"""Turn measure.cjs runs into the published tables.

    python3 aggregate.py raw/ entities.json out_dir/

Standard library only; uses analyze.py for domain logic. Writes:
  runs.csv      one row per game per run, with checkpoint columns (5/10/30/60 s)
  games.csv     one row per game: the median of its runs
  portals.csv   one row per portal: the median of its games' medians, plus ranges
A checkpoint counts only requests that FINISHED by that second, and cookies
present at that second. "Game seen" means a <canvas> existed in some frame at
the end of the wait, with no click; it is a flag, not a filter.
"""
import csv, json, os, statistics, sys, urllib.parse
from analyze import registrable, AD_TRACKING

CHECKPOINTS = [5, 10, 30, 60]


def run_metrics(game, entities):
    page_reg = registrable(urllib.parse.urlsplit(game["url"]).hostname)
    owner = lambda reg: (entities.get(reg) or {}).get("owner") or reg
    page_owner = owner(page_reg)
    is_ad = lambda reg: bool({c.lower() for c in (entities.get(reg) or {}).get("categories", [])} & AD_TRACKING)
    reqs = [r for r in game["requests"] if not r["url"].startswith(("data:", "blob:"))]
    row = {"label": game["label"], "portal": game["label"].split("-")[0], "run": game["run"],
           "url": game["url"], "measured_at": game["measured_at"], "country": game.get("country"),
           "chrome": game.get("chrome"), "status": game["status"], "error": game.get("error") or "",
           "game_seen": any((f.get("canvases") or 0) > 0 for f in game.get("frames", []))}
    for s in CHECKPOINTS:
        if s > game["wait_s"]:
            continue
        upto = [r for r in reqs if r["t_ms"] <= s * 1000]
        regs = [registrable(urllib.parse.urlsplit(r["url"]).hostname) for r in upto]
        third = [(r, g) for r, g in zip(upto, regs) if owner(g) != page_owner]
        cookies = game.get("cookie_checkpoints", {}).get(str(s))
        if cookies is None and s == game["wait_s"]:
            cookies = game["cookies"]
        creg = {registrable(c["domain"].lstrip(".")) for c in (cookies or [])}
        row.update({
            f"requests_{s}s": len(upto),
            f"megabytes_{s}s": round(sum(r.get("body_bytes") or 0 for r in upto) / 1e6, 2),
            f"third_party_domains_{s}s": len({g for _, g in third}),
            f"ad_tracking_domains_{s}s": len({g for _, g in third if is_ad(g)}),
            f"ad_tracking_companies_{s}s": len({owner(g) for _, g in third if is_ad(g)}),
            f"third_party_companies_{s}s": len({owner(g) for _, g in third}),
            f"ad_tracking_requests_{s}s": sum(1 for _, g in third if is_ad(g)),
            f"cookies_{s}s": len(cookies or []),
            f"third_party_cookie_domains_{s}s": len({c for c in creg if owner(c) != page_owner}),
        })
    return row


def median(xs):
    xs = [x for x in xs if isinstance(x, (int, float))]
    return round(statistics.median(xs), 2) if xs else ""


def write(path, rows):
    with open(path, "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
        w.writeheader()
        w.writerows(rows)


def main(raw_dir, entities_path, out_dir):
    entities = json.load(open(entities_path))
    os.makedirs(out_dir, exist_ok=True)
    runs = [run_metrics(json.load(open(os.path.join(raw_dir, n))), entities)
            for n in sorted(os.listdir(raw_dir)) if n.endswith(".json")]
    write(os.path.join(out_dir, "runs.csv"), runs)
    metrics = [k for k in runs[0] if k.endswith("s") and "_" in k and k.split("_")[-1][:-1].isdigit()]
    games = []
    for label in sorted({r["label"] for r in runs}):
        rs = [r for r in runs if r["label"] == label]
        g = {"label": label, "portal": rs[0]["portal"], "url": rs[0]["url"], "runs": len(rs),
             "game_seen_runs": sum(r["game_seen"] for r in rs),
             "errors": sum(1 for r in rs if r["error"])}
        g.update({m: median([r.get(m) for r in rs]) for m in metrics})
        games.append(g)
    write(os.path.join(out_dir, "games.csv"), games)
    portals = []
    for p in sorted({g["portal"] for g in games}):
        gs = [g for g in games if g["portal"] == p]
        row = {"portal": p, "games": len(gs), "games_seen": sum(1 for g in gs if g["game_seen_runs"] >= 2)}
        for m in metrics:
            vals = [g[m] for g in gs if g[m] != ""]
            row[m] = median(vals)
            row[m + "_min"] = min(vals) if vals else ""
            row[m + "_max"] = max(vals) if vals else ""
        portals.append(row)
    write(os.path.join(out_dir, "portals.csv"), portals)
    s = max(c for c in CHECKPOINTS if f"requests_{c}s" in portals[0])
    key = f"ad_tracking_domains_{s}s"
    for r in portals:
        print(f"{r['portal']:9} games {r['games']:>2} seen {r['games_seen']:>2} | {s}s: "
              f"{r[f'requests_{s}s']} req, {r[f'megabytes_{s}s']} MB, {r[f'third_party_domains_{s}s']} 3p dom, "
              f"{r[key]} ad/track dom ({r[key + '_min']}-{r[key + '_max']}), "
              f"{r[f'third_party_cookie_domains_{s}s']} 3p cookie dom")


if __name__ == "__main__":
    main(*sys.argv[1:])