SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
111 lines
4.1 KiB
Python
111 lines
4.1 KiB
Python
"""
|
|
Apply approved trim decisions: convert UNTRIMMED full-page extracts into trimmed KEEP
|
|
blocks in the article store.
|
|
|
|
Reads trim-decisions.json — a map of case id -> list of per-article cuts. Each cut names
|
|
the field, an optional headline paragraph, a body range, and optional interior exclusions
|
|
(teaser headlines that sit inside the body). Every kept piece is verified as a verbatim
|
|
substring of the staged file before it is written, so nothing is retyped and nothing can
|
|
drift silently.
|
|
|
|
Only touches UNTRIMMED blocks. Curated cases (456-462) are already KEEP and are ignored.
|
|
Run build_cases.py + verify_all.py afterwards.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
STORE = BASE / "sourced-articles.json"
|
|
STAGING = BASE / "staging"
|
|
DECISIONS = BASE / "trim-decisions.json"
|
|
|
|
|
|
def sha(t):
|
|
return hashlib.sha256(t.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def slice_paras(paras, start, end, title, exclude):
|
|
"""Return (text, pieces) where pieces are the contiguous runs kept, for verification."""
|
|
keep = [i for i in range(start, end + 1) if i not in set(exclude)]
|
|
runs, run = [], []
|
|
for i in keep:
|
|
if run and i == run[-1] + 1:
|
|
run.append(i)
|
|
else:
|
|
if run:
|
|
runs.append(run)
|
|
run = [i]
|
|
if run:
|
|
runs.append(run)
|
|
pieces = []
|
|
if title is not None:
|
|
pieces.append(paras[title])
|
|
pieces += ["\n\n".join(paras[r[0]:r[-1] + 1]) for r in runs]
|
|
return "\n\n".join(pieces), pieces
|
|
|
|
|
|
def main():
|
|
only = {int(a) for a in sys.argv[1:]} or None
|
|
store = json.loads(STORE.read_text(encoding="utf-8"))
|
|
decisions = json.loads(DECISIONS.read_text(encoding="utf-8"))
|
|
|
|
applied, fails = 0, []
|
|
for cid_s, cuts in decisions.items():
|
|
if only and int(cid_s) not in only:
|
|
continue
|
|
blocks = store.get(cid_s, [])
|
|
by_field = {b.get("field"): b for b in blocks}
|
|
for c in cuts:
|
|
b = by_field.get(c["field"])
|
|
if not b:
|
|
fails.append(f"case {cid_s} {c['field']}: no store block")
|
|
continue
|
|
if b.get("verdict") != "UNTRIMMED":
|
|
fails.append(f"case {cid_s} {c['field']}: verdict is {b.get('verdict')}, "
|
|
f"not UNTRIMMED — refusing to re-trim")
|
|
continue
|
|
path = STAGING / f"{int(cid_s):03d}" / b["staged_file"]
|
|
full = path.read_text(encoding="utf-8")
|
|
paras = full.split("\n\n")
|
|
text, pieces = slice_paras(paras, c["start"], c["end"],
|
|
c.get("title"), c.get("exclude", []))
|
|
for p in pieces:
|
|
if p not in full:
|
|
fails.append(f"case {cid_s} {c['field']}: a kept run is not a substring "
|
|
f"of the staged file (bad index?)")
|
|
break
|
|
else:
|
|
title = c.get("title")
|
|
b.update(
|
|
verdict="KEEP",
|
|
text=text,
|
|
chars=len(text),
|
|
sha256=sha(text),
|
|
paragraphs_kept=(f"headline [{title}] + " if title is not None else "")
|
|
+ f"{c['start']}-{c['end']}"
|
|
+ (f" excl {c['exclude']}" if c.get("exclude") else "")
|
|
+ f" of {len(paras)}",
|
|
cut_note=c.get("cut_note", "trimmed on read-through"),
|
|
trimmed_by="Sofia review",
|
|
)
|
|
b.pop("carried_over", None)
|
|
applied += 1
|
|
|
|
if fails:
|
|
print(f"FAIL — {len(fails)} problem(s), nothing written:")
|
|
for f in fails:
|
|
print(f" - {f}")
|
|
sys.exit(1)
|
|
|
|
STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"trimmed {applied} article(s)")
|
|
remaining = sum(1 for c in store.values() for b in c if b.get("verdict") == "UNTRIMMED")
|
|
print(f"UNTRIMMED blocks remaining in store: {remaining}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|