K&R database cleanup: handoff bundle

Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf

merge-output/trim.py

@@ -0,0 +1,110 @@
+"""
+Apply approved trim decisions: convert UNTRIMMED full-page extracts into trimmed KEEP
+blocks in the article store.
+
+Reads trim-decisions.json — a map of case id -> list of per-article cuts. Each cut names
+the field, an optional headline paragraph, a body range, and optional interior exclusions
+(teaser headlines that sit inside the body). Every kept piece is verified as a verbatim
+substring of the staged file before it is written, so nothing is retyped and nothing can
+drift silently.
+
+Only touches UNTRIMMED blocks. Curated cases (456-462) are already KEEP and are ignored.
+Run build_cases.py + verify_all.py afterwards.
+"""
+
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+BASE = Path(__file__).resolve().parent
+STORE = BASE / "sourced-articles.json"
+STAGING = BASE / "staging"
+DECISIONS = BASE / "trim-decisions.json"
+
+
+def sha(t):
+    return hashlib.sha256(t.encode("utf-8")).hexdigest()
+
+
+def slice_paras(paras, start, end, title, exclude):
+    """Return (text, pieces) where pieces are the contiguous runs kept, for verification."""
+    keep = [i for i in range(start, end + 1) if i not in set(exclude)]
+    runs, run = [], []
+    for i in keep:
+        if run and i == run[-1] + 1:
+            run.append(i)
+        else:
+            if run:
+                runs.append(run)
+            run = [i]
+    if run:
+        runs.append(run)
+    pieces = []
+    if title is not None:
+        pieces.append(paras[title])
+    pieces += ["\n\n".join(paras[r[0]:r[-1] + 1]) for r in runs]
+    return "\n\n".join(pieces), pieces
+
+
+def main():
+    only = {int(a) for a in sys.argv[1:]} or None
+    store = json.loads(STORE.read_text(encoding="utf-8"))
+    decisions = json.loads(DECISIONS.read_text(encoding="utf-8"))
+
+    applied, fails = 0, []
+    for cid_s, cuts in decisions.items():
+        if only and int(cid_s) not in only:
+            continue
+        blocks = store.get(cid_s, [])
+        by_field = {b.get("field"): b for b in blocks}
+        for c in cuts:
+            b = by_field.get(c["field"])
+            if not b:
+                fails.append(f"case {cid_s} {c['field']}: no store block")
+                continue
+            if b.get("verdict") != "UNTRIMMED":
+                fails.append(f"case {cid_s} {c['field']}: verdict is {b.get('verdict')}, "
+                             f"not UNTRIMMED — refusing to re-trim")
+                continue
+            path = STAGING / f"{int(cid_s):03d}" / b["staged_file"]
+            full = path.read_text(encoding="utf-8")
+            paras = full.split("\n\n")
+            text, pieces = slice_paras(paras, c["start"], c["end"],
+                                       c.get("title"), c.get("exclude", []))
+            for p in pieces:
+                if p not in full:
+                    fails.append(f"case {cid_s} {c['field']}: a kept run is not a substring "
+                                 f"of the staged file (bad index?)")
+                    break
+            else:
+                title = c.get("title")
+                b.update(
+                    verdict="KEEP",
+                    text=text,
+                    chars=len(text),
+                    sha256=sha(text),
+                    paragraphs_kept=(f"headline [{title}] + " if title is not None else "")
+                                    + f"{c['start']}-{c['end']}"
+                                    + (f" excl {c['exclude']}" if c.get("exclude") else "")
+                                    + f" of {len(paras)}",
+                    cut_note=c.get("cut_note", "trimmed on read-through"),
+                    trimmed_by="Sofia review",
+                )
+                b.pop("carried_over", None)
+                applied += 1
+
+    if fails:
+        print(f"FAIL — {len(fails)} problem(s), nothing written:")
+        for f in fails:
+            print(f"  - {f}")
+        sys.exit(1)
+
+    STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"trimmed {applied} article(s)")
+    remaining = sum(1 for c in store.values() for b in c if b.get("verdict") == "UNTRIMMED")
+    print(f"UNTRIMMED blocks remaining in store: {remaining}")
+
+
+if __name__ == "__main__":
+    main()