fix(trim): audit trail and review list rebuilt from the store; fetch list regenerated
- auto_trim.py: trim-decisions-auto.json and auto-trim-review.md are now derived from the whole store on every run (paragraphs_kept, cut_note, staged files), so a run limited to a few cases no longer overwrites the decisions for the rest. Restores the full trail: 378 decisions across 287 cases, 36 review entries (7 off-case suspects, 12 too small, 20 weak anchors). - make_fetch_list.py: regenerates the manual fetch list on sofia-worklist.md from the store between markers; replaces the hand-written July list. 65 sources remain unreadable by script, none on a case without other text. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
1387037367
commit
1309687723
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Regenerate Sofia's manual fetch list from the article store.
|
||||
|
||||
Lists every linked source the pipeline could not read, per case, ordered by how much
|
||||
else the case has: cases with NO usable text first. Verdicts: DEAD = 404/gone (no
|
||||
archive copy either), BLOCKED = the site refused an automated request (open it in a
|
||||
browser), THIN = the page needs a browser to render, ERROR = connection failure,
|
||||
PDF = an archived copy exists but is a PDF. Slots that the Wayback pass already
|
||||
recovered are not listed. Rewrites the top of sofia-worklist.md between the markers.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
||||
STORE = BASE / "sourced-articles.json"
|
||||
MANIFEST = BASE / "case-files-manifest.json"
|
||||
WORKLIST = BASE / "sofia-worklist.md"
|
||||
FAILED = ("BLOCKED", "THIN", "DEAD", "ERROR")
|
||||
|
||||
|
||||
def main():
|
||||
recs = {r["id"]: r for r in json.loads(JSONF.read_text(encoding="utf-8"))}
|
||||
store = json.loads(STORE.read_text(encoding="utf-8"))
|
||||
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"]
|
||||
if "sheet" in e.get("origin", "")}
|
||||
groups = {}
|
||||
for cid, r in recs.items():
|
||||
blocks = store.get(str(cid), [])
|
||||
have = sum(1 for b in blocks if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")) + (1 if cid in sheet else 0)
|
||||
for b in blocks:
|
||||
if b.get("verdict") not in FAILED or not b.get("url"):
|
||||
continue
|
||||
verdict = b["verdict"]
|
||||
note = b.get("archive_note") or ""
|
||||
if "PDF" in note:
|
||||
verdict = "PDF"
|
||||
groups.setdefault(have, []).append((cid, verdict, r, b["url"], b.get("archive_url")))
|
||||
lines = ["<!-- fetch-list:start (generated by make_fetch_list.py; do not edit by hand) -->",
|
||||
"## Manual fetch list (regenerated from the store)", "",
|
||||
"Every linked source the pipeline could not read, after the live fetch and the Wayback pass.",
|
||||
"Open each in your browser, copy the article text, and paste it back with the case number.",
|
||||
"Cases at the top have NO usable text at all; further down the missing source is a nice-to-have.",
|
||||
"DEAD = gone and not archived. BLOCKED = site refused the script, opens in a browser. THIN = needs",
|
||||
"a browser to render. PDF = an archived PDF exists at the archive link.", ""]
|
||||
total = 0
|
||||
for have in sorted(groups):
|
||||
title = "NO USABLE TEXT — priority" if have == 0 else f"has {have} other source(s) on file"
|
||||
lines += [f"### {title}", ""]
|
||||
for cid, verdict, r, url, arch in sorted(groups[have], key=lambda x: -x[0]):
|
||||
lines.append(f"- **case {cid}** `{verdict}` · {r.get('date')} · {str(r.get('victim'))[:60]} · {r.get('country')}")
|
||||
lines.append(f" <{url}>" + (f" archive: <{arch}>" if arch else ""))
|
||||
total += 1
|
||||
lines.append("")
|
||||
lines.append(f"{total} sources listed.")
|
||||
lines.append("<!-- fetch-list:end -->")
|
||||
block = "\n".join(lines)
|
||||
|
||||
wl = WORKLIST.read_text(encoding="utf-8")
|
||||
if "<!-- fetch-list:start" in wl:
|
||||
wl = re.sub(r"<!-- fetch-list:start.*?<!-- fetch-list:end -->", block, wl, flags=re.S)
|
||||
else:
|
||||
# replace the original hand-written list (from the intro up to the first '## Special')
|
||||
cut = wl.index("## Special:")
|
||||
wl = wl[:wl.index("## NO OTHER SOURCE")] + block + "\n\n" + wl[cut:]
|
||||
WORKLIST.write_text(wl, encoding="utf-8")
|
||||
print(f"{total} failed sources across {sum(len(v) for v in groups.values())} slots; "
|
||||
f"priority cases: {sorted({x[0] for x in groups.get(0, [])})}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user