K&R database cleanup: handoff bundle

Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf

merge-output/reconcile_sources.py

@@ -0,0 +1,73 @@
+"""
+Source-coverage reconciliation. One pass over the whole dataset.
+
+For every DB case, decide whether it still holds a usable article:
+  - a fetched block with a text-bearing verdict (KEEP / PARTIAL / UNTRIMMED), or
+  - a phase-1 spreadsheet article (from the build manifest).
+A case with neither is a genuine gap: every fetched block is non-text
+(THIN / BLOCKED / DEAD / NOT AN ARTICLE / SYNDICATION / OFF-CASE) or it has no
+sources at all.
+
+Then reconcile against sofia-worklist.md in both directions:
+  - every gap must be named on the worklist (report the ones that are missing),
+  - anything named on the worklist that now HAS a usable source can come off.
+
+Mechanical only; prints a report and exits 0. Nothing is modified.
+"""
+
+import io
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
+BASE = Path(__file__).resolve().parent
+TEXT_VERDICTS = {"KEEP", "PARTIAL", "UNTRIMMED"}
+
+
+def main():
+    recs = json.loads((BASE.parent / "attacks-export-Gart-website.json").read_text(encoding="utf-8"))
+    ids = {r["id"] for r in recs}
+    victim = {r["id"]: r.get("victim") for r in recs}
+    store = json.loads((BASE / "sourced-articles.json").read_text(encoding="utf-8"))
+    manifest = json.loads((BASE / "case-files-manifest.json").read_text(encoding="utf-8"))
+
+    # fetched coverage: any text-bearing block
+    fetched_text = {int(cid) for cid, blocks in store.items()
+                    if any(b.get("verdict") in TEXT_VERDICTS for b in blocks)}
+    # sheet coverage: any phase-1 article filed by the build
+    sheet = {m["case"] for m in manifest["entries"] if "reported_K&R sheet" in m.get("origin", "")}
+    covered = fetched_text | sheet
+    gaps = sorted(ids - covered)
+
+    # what each non-covered case does still carry, for the report
+    def why(cid):
+        blocks = store.get(str(cid), [])
+        if not blocks:
+            return "no fetched sources and no spreadsheet article"
+        return "only non-text blocks: " + ", ".join(
+            f"{b.get('field')}={b.get('verdict')}" for b in blocks)
+
+    # worklist reconciliation
+    wl_path = BASE / "sofia-worklist.md"
+    wl = wl_path.read_text(encoding="utf-8") if wl_path.exists() else ""
+    wl_ids = {int(n) for n in re.findall(r"\b(?:case\s+)?(\d{2,3})\b", wl)} & ids
+
+    print(f"{len(ids)} DB cases | {len(covered)} covered "
+          f"({len(fetched_text)} fetched-text, {len(sheet)} sheet) | {len(gaps)} gaps\n")
+
+    print("GENUINE GAPS (no usable source anywhere):")
+    for cid in gaps:
+        on = "on worklist" if cid in wl_ids else ">>> NOT ON WORKLIST — ADD"
+        print(f"  {cid:>3} {str(victim[cid])[:34]:34} {on}")
+        print(f"        {why(cid)}")
+
+    stale = sorted(c for c in wl_ids if c in covered)
+    print(f"\nWORKLIST ENTRIES THAT NOW HAVE A SOURCE (candidates to remove): "
+          f"{stale if stale else 'none'}")
+    print("  (a worklist mention can be context rather than a gap claim — confirm before removing)")
+
+
+if __name__ == "__main__":
+    main()