#!/usr/bin/env python3
"""Rebuild EasyList and EasyPrivacy as they stood when the study used them.

    python3 fetch_lists.py            # pinned build, checked against the study's recorded rule counts
    python3 fetch_lists.py --commit SHA   # any other commit of easylist/easylist (no check)

The study recorded list version 202609281501 (28 Sep 2026, 15:01 UTC). The
compiled files of that minute are no longer served by easylist.to, so this
script rebuilds them from the authors' source repository: it downloads the
GitHub archive of easylist/easylist at one commit and expands the two build
templates in it, easylist.template and easyprivacy.template, by replacing each
"%include easylist:PATH%" line with the contents of PATH. That is the same
assembly the authors' build performs. Result: lists/easylist.txt and
lists/easyprivacy.txt.

The lists are (c) The EasyList authors (https://easylist.to/), dual licensed
GPLv3+ / CC BY-SA 3.0+. This kit does not contain or redistribute them; do not
republish the lists/ folder it creates. Standard library only.
"""
import argparse, hashlib, io, json, os, re, sys, tarfile, urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "lists")

# Commit made at 2026-09-28T15:01:31Z. See README.md, "How the lists are pinned".
PINNED = "4300ac0edfef0a381cc8ca0b12c8fa965715d810"
# What the study's notes recorded for version 202609281501: pure domain rules
# in EasyList, in EasyPrivacy, and in their union.
EXPECTED = {"easylist": 45175, "easyprivacy": 46937, "union": 92082}
# sha256 of the sorted rule hosts, one per line, as rebuilt on 2026-10-07.
EXPECTED_SHA = {"easylist": "501ded3e0acdaedeacb0e51f40eb0e33da99aa1b4bc4b48a354e2327e0d503f6", "easyprivacy": "b11be8bd1ebdca30ad75fc60ae33b68fe96b3044086fa944ab8ff79ba69bb58e"}

RULE = re.compile(r"^\|\|([a-z0-9.-]+\.[a-z0-9-]+)\^(?:\$(?:third-party|3p))?$")
INCLUDE = re.compile(r"^%include easylist:(.+)%\s*$")


def domain_rules(lines):
    """Hosts of pure domain-block rules: ||host^ with no path, optionally $third-party or $3p."""
    hosts = set()
    for line in lines:
        m = RULE.match(line.strip().lower())
        if m:
            hosts.add(m.group(1))
    return hosts


def digest(hosts):
    return hashlib.sha256("\n".join(sorted(hosts)).encode()).hexdigest()


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--commit", default=PINNED)
    a = ap.parse_args()
    url = f"https://codeload.github.com/easylist/easylist/tar.gz/{a.commit}"
    print("downloading", url)
    req = urllib.request.Request(url, headers={"User-Agent": "game-weight-study-reproduce"})
    blob = urllib.request.urlopen(req, timeout=180).read()
    files = {}
    with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:   # read in memory, nothing is extracted
        for m in tar:
            if m.isfile():
                rel = m.name.split("/", 1)[1] if "/" in m.name else m.name
                if rel.endswith(".template") or rel == "template_header.txt" or \
                        rel.startswith(("easylist/", "easylist_adult/", "easyprivacy/")):
                    files[rel] = tar.extractfile(m).read().decode("utf-8", errors="ignore")
    os.makedirs(OUT, exist_ok=True)
    info = {"repository": "https://github.com/easylist/easylist", "commit": a.commit, "archive": url,
            "attribution": "EasyList and EasyPrivacy by The EasyList authors (https://easylist.to/), "
                           "GPLv3+ or CC BY-SA 3.0+", "lists": {}}
    rules = {}
    for name in ("easylist", "easyprivacy"):
        lines, included = [], []
        for line in files[name + ".template"].splitlines():
            m = INCLUDE.match(line)
            if m:
                included.append(m.group(1))
                lines += files[m.group(1)].splitlines()
            else:
                lines.append(line)
        with open(os.path.join(OUT, name + ".txt"), "w", encoding="utf-8") as fh:
            fh.write("\n".join(lines) + "\n")
        rules[name] = domain_rules(lines)
        info["lists"][name] = {"template": name + ".template", "included_paths": included,
                               "lines": len(lines), "domain_rules": len(rules[name]),
                               "domain_rules_sha256": digest(rules[name])}
        print(f"{name}: {len(included)} source files, {len(lines)} lines, {len(rules[name])} domain rules")
    info["union_domain_rules"] = len(rules["easylist"] | rules["easyprivacy"])
    print("union:", info["union_domain_rules"], "domain rules")
    json.dump(info, open(os.path.join(OUT, "PINNED.json"), "w"), indent=1)
    if a.commit == PINNED:
        got = {"easylist": len(rules["easylist"]), "easyprivacy": len(rules["easyprivacy"]),
               "union": info["union_domain_rules"]}
        if got != EXPECTED:
            sys.exit(f"MISMATCH: expected {EXPECTED}, got {got}. Do not use these lists for the reproduction.")
        print("OK: rule counts equal those the study recorded for version 202609281501 "
              "(45,175 / 46,937 / 92,082).")
        for name in rules:
            if digest(rules[name]) != EXPECTED_SHA[name]:
                sys.exit(f"MISMATCH: {name} domain rules differ from the kit's reference build.")
        print("OK: domain-rule sets are byte-identical to the kit's reference build.")
    else:
        print("note: not the pinned commit; counts are not checked.")
    print("wrote", OUT)


if __name__ == "__main__":
    main()