K&R database cleanup: handoff bundle

Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf

merge-output/stage_all.py

@@ -0,0 +1,105 @@
+"""
+Stage every remaining case's sources, untrimmed.
+
+The point is to race link rot. Retrieval is permanent; trimming and fact-checking can
+happen any time afterwards against saved text. So this fetches broadly, files the full
+extracted page, and marks every block UNTRIMMED so nobody mistakes it for curated text.
+
+Resumable and non-destructive:
+  - cases already present in sourced-articles.json are skipped entirely, so the curated
+    entries for 456-462 are never overwritten
+  - sources already staged are carried over rather than re-fetched, because re-fetching
+    a live page shifts paragraph indices under any approved cut
+
+Politeness: a delay between requests. We are reading other people's servers.
+"""
+
+import argparse
+import hashlib
+import json
+import time
+from pathlib import Path
+
+from fetch_sources import STAGING, stage_case
+
+BASE = Path(__file__).resolve().parent
+JSONF = BASE.parent / "attacks-export-Gart-website.json"
+STORE = BASE / "sourced-articles.json"
+LOG = BASE / "staging-progress.json"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--limit", type=int, default=0, help="stop after N cases (0 = all)")
+    ap.add_argument("--delay", type=float, default=1.5, help="seconds between cases")
+    args = ap.parse_args()
+
+    records = json.loads(JSONF.read_text(encoding="utf-8"))
+    store = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {}
+    progress = json.loads(LOG.read_text(encoding="utf-8")) if LOG.exists() else {}
+
+    # Cases that already carry article text from the spreadsheet are not worth fetching
+    # for: they have evidence on file, and the point of this pass is the ones with none.
+    manifest = BASE / "case-files-manifest.json"
+    have_articles = set()
+    if manifest.exists():
+        have_articles = {e["case"] for e in
+                         json.loads(manifest.read_text(encoding="utf-8"))["entries"]
+                         if "reported_K&R sheet" in e.get("origin", "")}
+
+    todo = [r for r in sorted(records, key=lambda r: -r["id"])
+            if str(r["id"]) not in store and r["id"] not in have_articles]
+    print(f"skipping {len(have_articles)} cases that already hold spreadsheet articles",
+          flush=True)
+    if args.limit:
+        todo = todo[:args.limit]
+    print(f"{len(todo)} cases to stage (skipping {len(store)} already in the store)\n", flush=True)
+
+    for n, rec in enumerate(todo, 1):
+        cid = rec["id"]
+        try:
+            entries = stage_case(rec, new_only=True)
+        except Exception as ex:                       # one bad case must not stop the run
+            progress[str(cid)] = {"error": f"{type(ex).__name__}: {str(ex)[:150]}"}
+            print(f"[{n}/{len(todo)}] case {cid}  ERROR {type(ex).__name__}", flush=True)
+            continue
+
+        blocks, tally = [], {}
+        for e in entries:
+            tally[e["verdict"]] = tally.get(e["verdict"], 0) + 1
+            if e["verdict"] != "FETCHED":
+                blocks.append({k: e.get(k) for k in ("field", "verdict", "url", "source", "note")})
+                continue
+            path = STAGING / f"{cid:03d}" / f"{e['n']:02d}_" \
+                   f"{e['host'].lower().replace('.', '-')}.txt"
+            if not path.exists():
+                cands = list((STAGING / f"{cid:03d}").glob(f"{e['n']:02d}_*.txt"))
+                if not cands:
+                    continue
+                path = cands[0]
+            text = path.read_text(encoding="utf-8")
+            blocks.append({
+                "field": e["field"], "verdict": "UNTRIMMED", "url": e["url"],
+                "source": e["host"], "language": "unknown",
+                "text": text, "chars": len(text),
+                "sha256": hashlib.sha256(text.encode()).hexdigest(),
+                "staged_file": path.name,
+                "paragraphs_kept": f"ALL {len(text.split(chr(10) * 2))} paragraphs, untrimmed",
+                "method": "raw HTTP retrieval, deterministic extraction",
+                "fetched_at": e.get("fetched_at"), "http": e.get("http"),
+                "cut_note": "NOT TRIMMED. Full page extract; trim on read-through.",
+            })
+
+        store[str(cid)] = blocks
+        progress[str(cid)] = tally
+        STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
+        LOG.write_text(json.dumps(progress, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"[{n}/{len(todo)}] case {cid:>4}  " +
+              "  ".join(f"{k}={v}" for k, v in sorted(tally.items())), flush=True)
+        time.sleep(args.delay)
+
+    print("\ndone", flush=True)
+
+
+if __name__ == "__main__":
+    main()