DB-cleanup/merge-output/wayback_pass.py

"""
Second retrieval pass through the Wayback Machine for every linked source the live
fetch could not read (BLOCKED, DEAD, THIN, ERROR).

A snapshot is an archived copy taken by a third party on a recorded date, not the live
page, so every block staged here is labelled with the snapshot timestamp and the archive
URL, and the original live verdict is kept in the note. Extraction is the same
deterministic HTML pass as the live path, so the text is hashed and re-verifiable.

Resumable: a store block that already carries `archive_url` or `archive_tried` is not
retried. Nothing here touches KEEP or PARTIAL blocks. Run build_cases.py --all and
verify_all.py afterwards.
"""

import argparse
import hashlib
import json
import re
import time
from datetime import datetime, timezone
from pathlib import Path

from fetch_sources import STAGING, Extract, fetch_wayback

BASE = Path(__file__).resolve().parent
STORE = BASE / "sourced-articles.json"
LOG = BASE / "wayback-progress.json"
RETRY = ("BLOCKED", "DEAD", "THIN", "ERROR")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("cases", nargs="*", type=int, help="limit to these case ids")
    ap.add_argument("--delay", type=float, default=1.0)
    args = ap.parse_args()

    store = json.loads(STORE.read_text(encoding="utf-8"))
    progress = json.loads(LOG.read_text(encoding="utf-8")) if LOG.exists() else {}
    todo = [(cid, b) for cid, blocks in store.items() for b in blocks
            if b.get("verdict") in RETRY and not b.get("archive_url") and not b.get("archive_tried")
            and b.get("url") and (not args.cases or int(cid) in args.cases)]
    print(f"{len(todo)} failed source slots to retry through the Wayback Machine", flush=True)

    tally = {}
    for n, (cid, b) in enumerate(todo, 1):
        url = b["url"]
        stamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
        b["archive_tried"] = stamp
        try:
            ts, snap, html = fetch_wayback(url)
        except Exception as ex:
            outcome = f"ERROR {type(ex).__name__}"
            b["archive_note"] = f"Wayback lookup failed: {type(ex).__name__}: {str(ex)[:100]}"
        else:
            if not ts:
                outcome = "NO SNAPSHOT"
                b["archive_note"] = "no Wayback Machine snapshot exists for this URL"
            elif html.lstrip().startswith("%PDF"):
                outcome = "PDF"
                b["archive_note"] = f"Wayback snapshot {ts} is a PDF; not extracted"
                b["archive_url"] = snap
            else:
                x = Extract()
                x.feed(html)
                body = x.text()
                if len(body) < 400:
                    outcome = "THIN"
                    b["archive_note"] = (f"Wayback snapshot {ts} extracted to only {len(body)} chars "
                                         "(page needs a browser to render)")
                    b["archive_url"] = snap
                else:
                    outcome = "FETCHED"
                    cdir = STAGING / f"{int(cid):03d}"
                    cdir.mkdir(parents=True, exist_ok=True)
                    idx_path = cdir / "index.json"
                    index = json.loads(idx_path.read_text(encoding="utf-8")) if idx_path.exists() else []
                    entry = next((e for e in index if e.get("field") == b["field"]), None)
                    num = entry["n"] if entry else len(index) + 1
                    host = re.sub(r"^https?://(www\.)?([^/]+).*", r"\2", url)
                    fname = f"{num:02d}_{re.sub(r'[^a-z0-9]+', '-', host.lower())}-wayback.txt"
                    (cdir / fname).write_text(body, encoding="utf-8")
                    # Store what the file reads back as, not the in-memory body: a page
                    # with carriage returns inside its text would otherwise never match
                    # the staged file on re-verification.
                    body = (cdir / fname).read_text(encoding="utf-8")
                    live_verdict = b["verdict"]
                    method = f"Wayback Machine snapshot {ts}, deterministic extraction"
                    new_entry = {"case": int(cid), "n": num, "field": b["field"], "url": url,
                                 "host": host, "fetched_at": stamp, "verdict": "FETCHED",
                                 "method": "wayback", "snapshot": ts, "archive_url": snap,
                                 "chars": len(body),
                                 "sha256": hashlib.sha256(body.encode()).hexdigest(),
                                 "live_verdict": live_verdict}
                    if entry:
                        index[index.index(entry)] = new_entry
                    else:
                        index.append(new_entry)
                    idx_path.write_text(json.dumps(index, ensure_ascii=False, indent=2),
                                        encoding="utf-8")
                    b.clear()
                    b.update({
                        "field": new_entry["field"], "verdict": "UNTRIMMED", "url": url,
                        "source": host, "language": "unknown",
                        "text": body, "chars": len(body),
                        "sha256": new_entry["sha256"], "staged_file": fname,
                        "paragraphs_kept": f"ALL {len(body.split(chr(10) * 2))} paragraphs, untrimmed",
                        "method": method, "fetched_at": stamp, "http": None,
                        "archive_url": snap, "snapshot": ts,
                        "note": f"live URL was {live_verdict} on the first pass; text comes from "
                                f"an archived copy dated {ts[:4]}-{ts[4:6]}-{ts[6:8]}",
                        "cut_note": "NOT TRIMMED. Full page extract; trim on read-through.",
                    })
        tally[outcome] = tally.get(outcome, 0) + 1
        progress[f"{cid}:{b['field']}"] = outcome
        STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
        LOG.write_text(json.dumps(progress, ensure_ascii=False, indent=2), encoding="utf-8")
        print(f"[{n}/{len(todo)}] case {int(cid):>3} {b['field']:<7} {outcome}", flush=True)
        time.sleep(args.delay)

    print("\ndone " + "  ".join(f"{k}={v}" for k, v in sorted(tally.items())), flush=True)


if __name__ == "__main__":
    main()