#!/usr/bin/env python3
"""Compare the study's advertising/tracking label with EasyList + EasyPrivacy.

    python3 fetch_lists.py        # once
    python3 crosscheck.py         # reads requests-noquery.csv.gz, the published request table
    python3 crosscheck.py --source DIR        # or a folder of your own measure.cjs captures

Writes crosscheck.csv (one row per third-party registrable domain seen in the
first 60 seconds of any load) and summary.json, and prints the summary.
Standard library only. Definitions are in studylib.py and README.md.
"""
import argparse, collections, csv, json, os, re, sys
from studylib import HERE, WINDOW_MS, Entities, find, hostname, open_loads, registrable

RULE = re.compile(r"^\|\|([a-z0-9.-]+\.[a-z0-9-]+)\^(?:\$(?:third-party|3p))?$")


def domain_rules(path):
    """Hosts of pure domain-block rules: ||host^, no path, only $third-party or $3p allowed. @@ never matches."""
    hosts = set()
    for line in open(path, encoding="utf-8", errors="ignore"):
        m = RULE.match(line.strip().lower())
        if m:
            hosts.add(m.group(1))
    return hosts


def matching_rule(host, rules):
    """The rule host that blocks this hostname (itself or a parent domain), or None."""
    parts = host.split(".")
    for i in range(len(parts) - 1):
        cand = ".".join(parts[i:])
        if cand in rules:
            return cand
    return None


def pct(n, d):
    return round(100 * n / d, 1) if d else None


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--source", default="table",
                    help="table (requests-noquery.csv.gz, the default) | path to another .csv.gz table or to a folder of measure.cjs captures")
    ap.add_argument("--easylist", default=os.path.join(HERE, "lists", "easylist.txt"))
    ap.add_argument("--easyprivacy", default=os.path.join(HERE, "lists", "easyprivacy.txt"))
    ap.add_argument("--out-dir", default=HERE)
    a = ap.parse_args()
    for p in (a.easylist, a.easyprivacy):
        if not os.path.exists(p):
            sys.exit(f"missing {p}: run  python3 fetch_lists.py  first")
    E = Entities()
    supplement = {r["domain"]: r for r in csv.DictReader(open(find("supplement.csv"), encoding="utf-8"))}
    el, ep = domain_rules(a.easylist), domain_rules(a.easyprivacy)

    source, loads = open_loads(a.source)
    hosts = collections.defaultdict(set)       # third-party registrable domain -> hostnames requested
    pages = collections.defaultdict(set)       # ... -> pages it was third party on
    n_loads = n_requests = 0
    for load in loads:
        n_loads += 1
        page_owner = E.owner(registrable(hostname(load["page_url"])))
        for t_ms, url, _ in load["requests"]:
            if t_ms > WINDOW_MS:
                continue
            n_requests += 1
            host = hostname(url)
            reg = registrable(host)
            if E.owner(reg) != page_owner:     # third party
                hosts[reg].add(host)
                pages[reg].add(load["page"])

    rows = []
    for reg in sorted(hosts):
        hit_el = sorted({m for h in hosts[reg] for m in [matching_rule(h, el)] if m})
        hit_ep = sorted({m for h in hosts[reg] for m in [matching_rule(h, ep)] if m})
        which = "both" if hit_el and hit_ep else "EasyList" if hit_el else "EasyPrivacy" if hit_ep else ""
        e = E.map.get(reg) or {}
        rows.append({
            "domain": reg,
            "our_classification": E.label(reg),
            "list_verdict": "flagged" if which else "not flagged",
            "which_list": which,
            "owner": e.get("owner", ""),
            "categories": ";".join(e.get("categories", [])),
            "classification_source": "supplement.csv" if reg in supplement else
                                     ("third-party-web" if reg in E.map else ""),
            "pages": len(pages[reg]),
            "hostnames_seen": len(hosts[reg]),
            "matching_rules": ";".join(sorted(set(hit_el) | set(hit_ep))),
        })
    os.makedirs(a.out_dir, exist_ok=True)
    with open(os.path.join(a.out_dir, "crosscheck.csv"), "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()), lineterminator="\n")
        w.writeheader()
        w.writerows(rows)

    n = lambda f: sum(1 for r in rows if f(r))
    ours = lambda r: r["our_classification"] == "ad/tracking"
    flag = lambda r: r["list_verdict"] == "flagged"
    judged = lambda r: r["our_classification"] != "unknown"      # we have a category for it
    both = n(lambda r: ours(r) and flag(r))
    ours_only = n(lambda r: ours(r) and not flag(r))
    lists_only = n(lambda r: judged(r) and not ours(r) and flag(r))
    neither = n(lambda r: judged(r) and not ours(r) and not flag(r))
    not_network = [r["domain"] for r in rows if "." not in r["domain"]]
    pinned = os.path.join(os.path.dirname(a.easylist), "PINNED.json")
    summary = {
        "window_seconds": WINDOW_MS // 1000,
        "loads": n_loads,
        "requests_in_window": n_requests,
        "lists": {"easylist_domain_rules": len(el), "easyprivacy_domain_rules": len(ep),
                  "union_domain_rules": len(el | ep),
                  "source_commit": json.load(open(pinned)).get("commit") if os.path.exists(pinned) else None},
        "third_party_domains": len(rows),
        "third_party_domains_note": f"includes {len(not_network)} entries that are not internet domains "
                                    f"(pages probing for browser extensions, about:blank); the study's scripts count them too",
        "domains_we_classify_as_advertising_or_tracking": n(ours),
        "domains_the_lists_flag": n(flag),
        "domains_the_lists_flag_by_list": {"EasyList only": n(lambda r: r["which_list"] == "EasyList"),
                                           "EasyPrivacy only": n(lambda r: r["which_list"] == "EasyPrivacy"),
                                           "both": n(lambda r: r["which_list"] == "both")},
        "domains_we_have_a_category_for": n(judged),
        "domains_with_no_category": n(lambda r: not judged(r)),
        "domains_with_no_category_that_the_lists_flag": n(lambda r: not judged(r) and flag(r)),
        "table_of_domains_we_have_a_category_for": {
            "we_say_ad_tracking__lists_flag": both, "we_say_ad_tracking__lists_do_not": ours_only,
            "we_say_other__lists_flag": lists_only, "we_say_other__lists_do_not": neither},
        "agreement_overlap_of_flagged": {
            "definition": "Of the domains we have a category for that either we or the lists flag as "
                          "advertising/tracking, the share that both flag.",
            "numerator": both, "denominator": both + ours_only + lists_only,
            "percent": pct(both, both + ours_only + lists_only)},
        "agreement_same_verdict": {
            "definition": "Of all the domains we have a category for, the share where we and the lists give "
                          "the same verdict: both flag it, or neither does.",
            "numerator": both + neither, "denominator": both + ours_only + lists_only + neither,
            "percent": pct(both + neither, both + ours_only + lists_only + neither)},
    }
    json.dump(summary, open(os.path.join(a.out_dir, "summary.json"), "w"), indent=1)
    print(f"source: {source}")
    print(json.dumps(summary, indent=1))


if __name__ == "__main__":
    main()