#!/usr/bin/env python3
"""Count the privacy flags on requests to Google's ad server, per portal.

    python3 count_ad_flags.py                 # reads requests-noquery.csv.gz, the published request table
    python3 count_ad_flags.py --source DIR    # or a folder of your own measure.cjs captures

A "request to Google's ad server" is a request, finished within the first 60
seconds of a load, whose host is under doubleclick.net and whose path is
/gampad/ads (the Google Publisher Tag ad request). Two query parameters are
counted: npa=1 (ask for non-personalised ads) and tfcd=1 (tag for
child-directed treatment). Writes ad_flags.csv and ad_flag_requests.csv (one
row per counted request) and prints the table. Standard library only.
"""
import argparse, collections, csv, os, urllib.parse
from studylib import HERE, WINDOW_MS, hostname, open_loads, registrable

AD_SERVER_DOMAIN = "doubleclick.net"
AD_SERVER_PATH = "/gampad/ads"


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--source", default="table",
                    help="table (requests-noquery.csv.gz, the default) | path to another .csv.gz table or to a folder of measure.cjs captures")
    ap.add_argument("--out-dir", default=HERE)
    a = ap.parse_args()
    source, loads = open_loads(a.source)
    P = collections.defaultdict(lambda: collections.defaultdict(set))
    N = collections.defaultdict(collections.Counter)
    detail = []
    for load in loads:
        portal, page, key = load["portal"], load["page"], (load["page"], load["load"])
        P[portal]["pages"].add(page)
        P[portal]["loads"].add(key)
        all_npa = True
        for t_ms, url, _ in load["requests"]:
            sp = urllib.parse.urlsplit(url)
            if registrable(sp.hostname or "") != AD_SERVER_DOMAIN or sp.path != AD_SERVER_PATH:
                continue
            if t_ms > WINDOW_MS:
                N[portal]["after_window"] += 1
                continue
            q = urllib.parse.parse_qs(sp.query, keep_blank_values=True)
            npa, tfcd = "1" in q.get("npa", []), "1" in q.get("tfcd", [])
            N[portal]["requests"] += 1
            N[portal]["npa"] += npa
            N[portal]["tfcd"] += tfcd
            all_npa = all_npa and npa
            P[portal]["ad_loads"].add(key)
            P[portal]["ad_pages"].add(page)
            if npa:
                P[portal]["npa_loads"].add(key)
            if tfcd:
                P[portal]["tfcd_loads"].add(key)
                P[portal]["tfcd_pages"].add(page)
            detail.append({"portal": portal, "page": page, "load": load["load"], "seconds": f"{t_ms / 1000:.3f}",
                           "host": sp.hostname, "npa": ",".join(q.get("npa", [])), "tfcd": ",".join(q.get("tfcd", []))})
        if key in P[portal]["ad_loads"] and all_npa:
            P[portal]["all_npa_loads"].add(key)
    rows = []
    for portal in sorted(P):
        p, n = P[portal], N[portal]
        rows.append({
            "portal": portal, "pages": len(p["pages"]), "loads": len(p["loads"]),
            "ad_server_requests": n["requests"],
            "loads_with_ad_server_requests": len(p["ad_loads"]),
            "pages_with_ad_server_requests": len(p["ad_pages"]),
            "requests_npa1": n["npa"],
            "loads_where_every_ad_request_had_npa1": len(p["all_npa_loads"]),
            "requests_tfcd1": n["tfcd"], "loads_tfcd1": len(p["tfcd_loads"]), "pages_tfcd1": len(p["tfcd_pages"]),
            "pages_tfcd1_names": ";".join(sorted(p["tfcd_pages"])),
            "ad_server_requests_after_60s_not_counted": n["after_window"],
        })
    os.makedirs(a.out_dir, exist_ok=True)
    for name, data in (("ad_flags.csv", rows), ("ad_flag_requests.csv", sorted(
            detail, key=lambda d: (d["portal"], d["page"], d["load"], float(d["seconds"]), d["host"])))):
        with open(os.path.join(a.out_dir, name), "w", newline="", encoding="utf-8") as fh:
            w = csv.DictWriter(fh, fieldnames=list(data[0].keys()), lineterminator="\n")
            w.writeheader()
            w.writerows(data)
    print(f"source: {source}   window: first {WINDOW_MS // 1000} s   ad server: *.{AD_SERVER_DOMAIN}{AD_SERVER_PATH}")
    print(f"{'portal':9} {'pages':>5} {'loads':>5} {'ad req':>6} {'loads w/ad':>10} {'npa=1':>6} "
          f"{'loads all npa':>13} {'tfcd=1 req':>10} {'loads':>5} {'pages':>5}")
    for r in rows:
        print(f"{r['portal']:9} {r['pages']:>5} {r['loads']:>5} {r['ad_server_requests']:>6} "
              f"{r['loads_with_ad_server_requests']:>10} {r['requests_npa1']:>6} "
              f"{r['loads_where_every_ad_request_had_npa1']:>13} {r['requests_tfcd1']:>10} "
              f"{r['loads_tfcd1']:>5} {r['pages_tfcd1']:>5}  {r['pages_tfcd1_names']}")


if __name__ == "__main__":
    main()