DB-cleanup/merge-output/stage_all.py

"""
Stage every remaining case's sources, untrimmed.

The point is to race link rot. Retrieval is permanent; trimming and fact-checking can
happen any time afterwards against saved text. So this fetches broadly, files the full
extracted page, and marks every block UNTRIMMED so nobody mistakes it for curated text.

Resumable and non-destructive:
  - cases already present in sourced-articles.json are skipped entirely, so the curated
    entries for 456-462 are never overwritten
  - sources already staged are carried over rather than re-fetched, because re-fetching
    a live page shifts paragraph indices under any approved cut

Politeness: a delay between requests. We are reading other people's servers.
"""

import argparse
import hashlib
import json
import time
from pathlib import Path

from fetch_sources import STAGING, stage_case

BASE = Path(__file__).resolve().parent
JSONF = BASE.parent / "attacks-export-Gart-website.json"
STORE = BASE / "sourced-articles.json"
LOG = BASE / "staging-progress.json"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--limit", type=int, default=0, help="stop after N cases (0 = all)")
    ap.add_argument("--delay", type=float, default=1.5, help="seconds between cases")
    ap.add_argument("--include-sheet-cases", action="store_true",
                    help="also stage cases that already hold a spreadsheet article. The "
                         "sheet paste is often just the lede; this fetches the linked URLs "
                         "so the full text is on file before link rot takes it.")
    args = ap.parse_args()

    records = json.loads(JSONF.read_text(encoding="utf-8"))
    store = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {}
    progress = json.loads(LOG.read_text(encoding="utf-8")) if LOG.exists() else {}

    # Cases that already carry article text from the spreadsheet are not worth fetching
    # for: they have evidence on file, and the point of this pass is the ones with none.
    manifest = BASE / "case-files-manifest.json"
    have_articles = set()
    if manifest.exists() and not args.include_sheet_cases:
        have_articles = {e["case"] for e in
                         json.loads(manifest.read_text(encoding="utf-8"))["entries"]
                         if "reported_K&R sheet" in e.get("origin", "")}

    todo = [r for r in sorted(records, key=lambda r: -r["id"])
            if str(r["id"]) not in store and r["id"] not in have_articles]
    print(f"skipping {len(have_articles)} cases that already hold spreadsheet articles",
          flush=True)
    if args.limit:
        todo = todo[:args.limit]
    print(f"{len(todo)} cases to stage (skipping {len(store)} already in the store)\n", flush=True)

    for n, rec in enumerate(todo, 1):
        cid = rec["id"]
        try:
            entries = stage_case(rec, new_only=True)
        except Exception as ex:                       # one bad case must not stop the run
            progress[str(cid)] = {"error": f"{type(ex).__name__}: {str(ex)[:150]}"}
            print(f"[{n}/{len(todo)}] case {cid}  ERROR {type(ex).__name__}", flush=True)
            continue

        blocks, tally = [], {}
        for e in entries:
            tally[e["verdict"]] = tally.get(e["verdict"], 0) + 1
            if e["verdict"] != "FETCHED":
                blocks.append({k: e.get(k) for k in ("field", "verdict", "url", "source", "note")})
                continue
            path = STAGING / f"{cid:03d}" / f"{e['n']:02d}_" \
                   f"{e['host'].lower().replace('.', '-')}.txt"
            if not path.exists():
                cands = list((STAGING / f"{cid:03d}").glob(f"{e['n']:02d}_*.txt"))
                if not cands:
                    continue
                path = cands[0]
            text = path.read_text(encoding="utf-8")
            blocks.append({
                "field": e["field"], "verdict": "UNTRIMMED", "url": e["url"],
                "source": e["host"], "language": "unknown",
                "text": text, "chars": len(text),
                "sha256": hashlib.sha256(text.encode()).hexdigest(),
                "staged_file": path.name,
                "paragraphs_kept": f"ALL {len(text.split(chr(10) * 2))} paragraphs, untrimmed",
                "method": "raw HTTP retrieval, deterministic extraction",
                "fetched_at": e.get("fetched_at"), "http": e.get("http"),
                "cut_note": "NOT TRIMMED. Full page extract; trim on read-through.",
            })

        store[str(cid)] = blocks
        progress[str(cid)] = tally
        STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
        LOG.write_text(json.dumps(progress, ensure_ascii=False, indent=2), encoding="utf-8")
        print(f"[{n}/{len(todo)}] case {cid:>4}  " +
              "  ".join(f"{k}={v}" for k, v in sorted(tally.items())), flush=True)
        time.sleep(args.delay)

    print("\ndone", flush=True)


if __name__ == "__main__":
    main()