#!/usr/bin/env python3
"""How requests-noquery.csv.gz was made from the raw captures. A record, not a step.

YOU CANNOT RUN THIS WITH THE PUBLISHED FILES, AND YOU DO NOT NEED TO. It reads
the study's raw captures (one JSON file per load, 100 MB+), which are not
published because their full request addresses identify the connection the
measurement was run from. It is published so that the way the table was derived
and redacted is on the record. The published table was made with:

    python3 build_requests_table.py --raw RAW_DIR --strict --out requests-noquery.csv.gz --report url_scan_report_noquery.json

--strict drops every query string, keeping only npa and tfcd on requests to
Google's ad server, and replaces every path segment that contains "=" by
"REDACTED" (some ad-measurement addresses carry their parameters in the path,
as /name=value;name=value). Without --strict the script writes full URLs with only the
sensitive parts redacted (requests.csv.gz, 16.6 MB, also not published).
If you make your own captures with measure.cjs you can run it on those.

One row per request (data: and blob: URLs left out, as in the study):

    page, portal, load, page_url, seconds, url, resource_type

"seconds" is the time since navigation at which the request finished.

REDACTION. Ad servers echo the visitor's IP address and IP-derived location
back into URLs. A URL part is treated as sensitive when, after percent-decoding
it twice, it contains
  (1) an IPv4 address literal (except x.0.0.0, which is how browser versions
      are written),
  (2) a location or IP parameter (lat, lon, zip, postal code, city, dma,
      *geo_*, *ip) with a value, or
  (3) any term in private_terms.txt: the measuring connection's address (also
      as base64, hex or an integer, and as base64 of its four raw bytes) and its city, state, postcode, coordinates,
      media market, time zone and carrier. That file is not published.
A sensitive query string or fragment is replaced whole by "?REDACTED". A
sensitive path segment is replaced by "REDACTED". Scheme, host and the rest of
the path are never changed, so every count in this kit is unaffected; the build
ends by proving that (it re-reads the file it wrote and re-scans it).
Standard library only.
"""
import argparse, base64, collections, csv, getpass, gzip, io, json, os, re, socket, urllib.parse
from studylib import HERE, TABLE_COLUMNS, loads_from_raw, registrable

SPLIT = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*://[^/?#]*)([^?#]*)(.*)$", re.S)
IPV4 = re.compile(r"(?<![\d.])(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?![\d.])")
GEO_KEY = re.compile(r"(?:^|[?&;#])(?:\w*(?:lat|latitude|lon|lng|long|longitude)|\w*zip\w*|\w*postal\w*|\w*city|"
                     r"\w*dma|\w*geo_\w+|\w*ip)=[^&;]", re.I)
GEO_KEY_OK = re.compile(r"^(?:[?&;#])?(?:ip=[01]|dma=0)$", re.I)       # flags, not values


def twice(s):
    return urllib.parse.unquote(urllib.parse.unquote(s))


class Scanner:
    def __init__(self, private_path):
        self.terms, self.literals, self.binary = [], [], []
        if private_path and os.path.exists(private_path):
            for line in open(private_path, encoding="utf-8"):
                line = line.strip()
                if not line or line.startswith("#"):
                    continue
                if line.lower().startswith("ip:"):
                    self._add_ip(line.split(":", 1)[1].strip())
                else:
                    self.terms.append(re.compile(line, re.I))
        self.private_loaded = bool(self.terms or self.literals)

    def _add_ip(self, ip):
        o = ip.split(".")
        self.terms.append(re.compile(r"(?<!\d)" + r"[^0-9a-z]{1,3}".join(map(re.escape, o)) + r"(?!\d)", re.I))
        raw = ip.encode()
        for off in range(3):          # base64 of the address at each byte alignment
            enc = base64.b64encode(b"\0" * off + raw + b"\0\0").decode()
            first, last = -(-off * 8 // 6), (off + len(raw)) * 8 // 6
            self.literals.append(enc[first:last])
        self.literals.append(raw.hex())
        n = sum(int(x) << (8 * (3 - i)) for i, x in enumerate(o))
        self.literals += [str(n), f"{n:08x}"]
        four = bytes(int(x) for x in o)
        for off in range(3):          # base64 of the address's four raw bytes, as binary tokens carry it
            enc = base64.b64encode(b"\0" * off + four + b"\0\0").decode()
            self.binary.append(enc[-(-off * 8 // 6):(off + 4) * 8 // 6])

    def reasons(self, part):
        """Names of the rules a URL part trips. Never returns the matched text."""
        out = set()
        d = twice(part)
        for m in IPV4.finditer(d):
            if not m.group(0).endswith(".0.0.0"):
                out.add("ip-address literal")
        for m in GEO_KEY.finditer(d):
            if not GEO_KEY_OK.match(m.group(0)):
                out.add("location or ip parameter")
        if any(t.search(d) for t in self.terms):
            out.add("private term (measuring connection's address or location)")
        norm = part.replace("-", "+").replace("_", "/")
        low = d.lower()
        dnorm = d.replace("-", "+").replace("_", "/")
        if any(l in norm or l in part or l in d for l in self.literals[:3]) or \
                any(l in low for l in self.literals[3:]) or \
                any(l in norm or l in dnorm for l in self.binary):
            out.add("private term, encoded (base64 / hex / integer)")
        return out


OTHER = {
    "local file path": re.compile(r"file:/|/Users/|/home/[a-z]|[A-Za-z]:\\|/private/|/var/folders/"),
    "private or loopback address": re.compile(r"(?<![\w.])(?:localhost|127\.0\.0\.1|10\.\d+\.\d+\.\d+|192\.168\.\d+\.\d+|"
                                              r"172\.(?:1[6-9]|2\d|3[01])\.\d+\.\d+)(?![\w.])"),
    "email address": re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)*\.(?!webp|png|jpe?g|gif|svg|avif|js|css|mp4|json)[a-z]{2,}\b", re.I),
}


def redact(url, sc, tally):
    m = SPLIT.match(url)
    if not m:
        return url
    head, path, rest = m.groups()
    host = urllib.parse.urlsplit(url).hostname or ""
    if rest:
        why = sc.reasons(rest)
        if why:
            rest = "?REDACTED"
            for w in why:
                tally[("query string", w)] += 1
            tally["hosts"][host] += 1
    if path:
        segs = path.split("/")
        hit = False
        for i, s in enumerate(segs):
            why = sc.reasons("?" + s) - {"location or ip parameter"} if s else ()
            # a path has no parameters; test it as free text, and catch /ip/VALUE/ pairs through the literal rules
            if why:
                segs[i] = "REDACTED"
                hit = True
                for w in why:
                    tally[("path segment", w)] += 1
        if hit:
            path = "/".join(segs)
            tally["hosts"][host] += 1
    if sc.reasons("?" + head.split("://", 1)[1]) - {"location or ip parameter"} and not IPV4.fullmatch(host):
        tally[("HOST (not redacted)", "see report")] += 1
    return head + path + rest


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--raw", default=None, help="folder of raw captures from measure.cjs (the study's own are not published)")
    ap.add_argument("--out", default=os.path.join(HERE, "requests.csv.gz"))
    ap.add_argument("--private-terms", default=os.path.join(HERE, "private_terms.txt"))
    ap.add_argument("--strict", action="store_true", help="drop every query string; keep only npa/tfcd on ad-server requests")
    ap.add_argument("--report", default=None)
    a = ap.parse_args()
    if not a.raw or not os.path.isdir(a.raw):
        raise SystemExit("This script needs the raw captures, which are not published, so it cannot be run from the\n"
                         "published files. It documents how requests-noquery.csv.gz was made. To reproduce the\n"
                         "article's figures run fetch_lists.py, crosscheck.py and count_ad_flags.py; see README.md.\n"
                         "(With your own measure.cjs captures:  python3 build_requests_table.py --raw DIR)")
    sc = Scanner(a.private_terms)
    if not sc.private_loaded:
        print("WARNING: no private_terms.txt; only the generic rules (1) and (2) are applied.")
    tally = collections.Counter()
    tally["hosts"] = collections.Counter()
    other = collections.Counter()
    local = [x.lower() for x in {getpass.getuser(), socket.gethostname().split(".")[0]} if len(x) >= 4]
    n = changed = 0
    buf = io.StringIO()
    w = csv.writer(buf, lineterminator="\n")
    w.writerow(TABLE_COLUMNS)
    for load in loads_from_raw(a.raw):
        for t_ms, url, rtype in sorted(load["requests"], key=lambda r: r[0]):
            n += 1
            d = twice(url)
            for name, rx in OTHER.items():
                if rx.search(d):
                    other[name] += 1
            if any(x in d.lower() for x in local):
                other["local user or machine name"] += 1
            new = redact(url, sc, tally)
            if a.strict:
                m = SPLIT.match(new)
                if m and m.group(3):
                    sp = urllib.parse.urlsplit(url)
                    keep = ""
                    if registrable(sp.hostname or "") == "doubleclick.net" and sp.path == "/gampad/ads":
                        q = urllib.parse.parse_qsl(sp.query, keep_blank_values=True)
                        keep = "?" + urllib.parse.urlencode([(k, v) for k, v in q if k in ("npa", "tfcd")])
                    new = m.group(1) + m.group(2) + keep
                m = SPLIT.match(new)
                if m and "=" in m.group(2):            # parameters carried in the path
                    segs = m.group(2).split("/")
                    for i, s in enumerate(segs):
                        if "=" in s:
                            segs[i] = "REDACTED"
                            tally[("path segment", "parameter in the path (strict mode)")] += 1
                    new = m.group(1) + "/".join(segs) + m.group(3)
            changed += new != url
            w.writerow([load["page"], load["portal"], load["load"], load["page_url"], f"{t_ms / 1000:.3f}", new, rtype])
    data = buf.getvalue().encode("utf-8")
    with open(a.out, "wb") as fh:
        with gzip.GzipFile(filename="", mode="wb", fileobj=fh, compresslevel=9, mtime=0) as gz:
            gz.write(data)

    # Independent re-scan of what was actually written.
    left = collections.Counter()
    rows = 0
    with gzip.open(a.out, "rt", encoding="utf-8", newline="") as fh:
        for row in csv.DictReader(fh):
            rows += 1
            m = SPLIT.match(row["url"])
            if not m:
                continue
            for part in ("?" + m.group(2), m.group(3)):
                for why in sc.reasons(part) - ({"location or ip parameter"} if part.startswith("?/") else set()):
                    left[why] += 1
            d = twice(row["url"]).lower()
            if any(x in d for x in local):
                left["local user or machine name"] += 1
    hosts = tally.pop("hosts")
    report = {
        "output": os.path.basename(a.out), "mode": "strict" if a.strict else "redacted",
        "rows": rows, "bytes_compressed": os.path.getsize(a.out), "bytes_uncompressed": len(data),
        "urls_changed": changed,
        "private_terms_applied": sc.private_loaded,
        "redactions": {f"{where}: {why}": c for (where, why), c in sorted(tally.items())},
        "redacted_urls_by_host_top20": dict(hosts.most_common(20)),
        "redacted_urls_distinct_hosts": len(hosts),
        "other_checks_on_raw_urls": {k: other.get(k, 0) for k in list(OTHER) + ["local user or machine name"]},
        "left_after_redaction": dict(left),
    }
    json.dump(report, open(a.report or os.path.join(HERE, "url_scan_report.json"), "w"), indent=1)
    print(json.dumps(report, indent=1))
    if left:
        raise SystemExit("FAILED: sensitive patterns remain in the output; do not publish it.")
    print(f"OK: nothing matching the redaction rules remains. {a.out}: {os.path.getsize(a.out) / 1e6:.1f} MB")


if __name__ == "__main__":
    main()