"""Headline facts and the per-domain file, from the raw loads.

    python3 facts.py raw/ entities.json results/

Standard library only. Everything is limited to the first 60 seconds of each
load, the same window as the tables. Writes results/facts.json and
results/domains.csv (one row per load per registrable domain).
"""
import collections, csv, glob, json, os, statistics, sys, urllib.parse
from analyze import registrable, AD_TRACKING

WINDOW_MS = 60_000


def main(raw_dir, entities_path, out_dir):
    E = json.load(open(entities_path))
    owner = lambda g: (E.get(g) or {}).get("owner") or g
    is_ad = lambda g: bool({c.lower() for c in (E.get(g) or {}).get("categories", [])} & AD_TRACKING)
    ad_pages = collections.defaultdict(set)        # ad/tracking owner -> game labels
    per_load_requests = collections.defaultdict(list)
    rows = []
    for f in sorted(glob.glob(os.path.join(raw_dir, "*.json"))):
        d = json.load(open(f))
        po = owner(registrable(urllib.parse.urlsplit(d["url"]).hostname))
        reqs = [r for r in d["requests"] if r["t_ms"] <= WINDOW_MS and not r["url"].startswith(("data:", "blob:"))]
        per_load_requests[d["label"]].append(len(reqs))
        agg = collections.defaultdict(lambda: [0, 0])
        for r in reqs:
            g = registrable(urllib.parse.urlsplit(r["url"]).hostname)
            agg[g][0] += 1
            agg[g][1] += r.get("body_bytes") or 0
            if owner(g) != po and is_ad(g):
                ad_pages[owner(g)].add(d["label"])
        cookies = d.get("cookie_checkpoints", {}).get("60") or d["cookies"]
        ck = collections.Counter(registrable(c["domain"].lstrip(".")) for c in cookies)
        for g, (n, b) in sorted(agg.items(), key=lambda x: -x[1][0]):
            e = E.get(g) or {}
            rows.append({"label": d["label"], "run": d["run"], "domain": g, "owner": e.get("owner", ""),
                         "categories": ";".join(e.get("categories", [])),
                         "party": "first" if owner(g) == po else "third",
                         "ad_tracking": int(owner(g) != po and is_ad(g)),
                         "requests": n, "bytes": b, "cookies_set_by_60s": ck.get(g, 0)})
    os.makedirs(out_dir, exist_ok=True)
    with open(os.path.join(out_dir, "domains.csv"), "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
        w.writeheader()
        w.writerows(rows)
    labels = sorted(per_load_requests)
    portal_games = [l for l in labels if not l.startswith("manxlab-")]
    ranked = sorted(ad_pages, key=lambda o: (-len(ad_pages[o]), o))
    busiest = max(portal_games, key=lambda l: statistics.median(per_load_requests[l]))
    facts = {
        "window_seconds": WINDOW_MS // 1000,
        "loads_total": sum(len(v) for v in per_load_requests.values()),
        "games_total": len(labels), "portal_games": len(portal_games),
        "games_with_two_loads": [l for l in labels if len(per_load_requests[l]) < 3],
        "distinct_ad_tracking_companies_portal_games": len({o for o, ls in ad_pages.items()
                                                            if any(not l.startswith("manxlab-") for l in ls)}),
        "busiest_game": {"label": busiest, "median_requests": statistics.median(per_load_requests[busiest]),
                         "loads": sorted(per_load_requests[busiest])},
        "top_companies": [[o, len(ad_pages[o]), dict(collections.Counter(l.split("-")[0] for l in ad_pages[o]))]
                          for o in ranked[:15]],
    }
    json.dump(facts, open(os.path.join(out_dir, "facts.json"), "w"), indent=1)
    print(json.dumps({k: v for k, v in facts.items() if k != "top_companies"}, indent=1))
    for t in facts["top_companies"][:9]:
        print(t)


if __name__ == "__main__":
    main(*sys.argv[1:])