StellarCrowandClaude Fable 5.1
1309687723
- auto_trim.py: trim-decisions-auto.json and auto-trim-review.md are now derived from the whole store on every run (paragraphs_kept, cut_note, staged files), so a run limited to a few cases no longer overwrites the decisions for the rest. Restores the full trail: 378 decisions across 287 cases, 36 review entries (7 off-case suspects, 12 too small, 20 weak anchors). - make_fetch_list.py: regenerates the manual fetch list on sofia-worklist.md from the store between markers; replaces the hand-written July list. 65 sources remain unreadable by script, none on a case without other text. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
75 lines
3.7 KiB
Python
75 lines
3.7 KiB
Python
"""
|
|
Regenerate Sofia's manual fetch list from the article store.
|
|
|
|
Lists every linked source the pipeline could not read, per case, ordered by how much
|
|
else the case has: cases with NO usable text first. Verdicts: DEAD = 404/gone (no
|
|
archive copy either), BLOCKED = the site refused an automated request (open it in a
|
|
browser), THIN = the page needs a browser to render, ERROR = connection failure,
|
|
PDF = an archived copy exists but is a PDF. Slots that the Wayback pass already
|
|
recovered are not listed. Rewrites the top of sofia-worklist.md between the markers.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
|
STORE = BASE / "sourced-articles.json"
|
|
MANIFEST = BASE / "case-files-manifest.json"
|
|
WORKLIST = BASE / "sofia-worklist.md"
|
|
FAILED = ("BLOCKED", "THIN", "DEAD", "ERROR")
|
|
|
|
|
|
def main():
|
|
recs = {r["id"]: r for r in json.loads(JSONF.read_text(encoding="utf-8"))}
|
|
store = json.loads(STORE.read_text(encoding="utf-8"))
|
|
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"]
|
|
if "sheet" in e.get("origin", "")}
|
|
groups = {}
|
|
for cid, r in recs.items():
|
|
blocks = store.get(str(cid), [])
|
|
have = sum(1 for b in blocks if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")) + (1 if cid in sheet else 0)
|
|
for b in blocks:
|
|
if b.get("verdict") not in FAILED or not b.get("url"):
|
|
continue
|
|
verdict = b["verdict"]
|
|
note = b.get("archive_note") or ""
|
|
if "PDF" in note:
|
|
verdict = "PDF"
|
|
groups.setdefault(have, []).append((cid, verdict, r, b["url"], b.get("archive_url")))
|
|
lines = ["<!-- fetch-list:start (generated by make_fetch_list.py; do not edit by hand) -->",
|
|
"## Manual fetch list (regenerated from the store)", "",
|
|
"Every linked source the pipeline could not read, after the live fetch and the Wayback pass.",
|
|
"Open each in your browser, copy the article text, and paste it back with the case number.",
|
|
"Cases at the top have NO usable text at all; further down the missing source is a nice-to-have.",
|
|
"DEAD = gone and not archived. BLOCKED = site refused the script, opens in a browser. THIN = needs",
|
|
"a browser to render. PDF = an archived PDF exists at the archive link.", ""]
|
|
total = 0
|
|
for have in sorted(groups):
|
|
title = "NO USABLE TEXT — priority" if have == 0 else f"has {have} other source(s) on file"
|
|
lines += [f"### {title}", ""]
|
|
for cid, verdict, r, url, arch in sorted(groups[have], key=lambda x: -x[0]):
|
|
lines.append(f"- **case {cid}** `{verdict}` · {r.get('date')} · {str(r.get('victim'))[:60]} · {r.get('country')}")
|
|
lines.append(f" <{url}>" + (f" archive: <{arch}>" if arch else ""))
|
|
total += 1
|
|
lines.append("")
|
|
lines.append(f"{total} sources listed.")
|
|
lines.append("<!-- fetch-list:end -->")
|
|
block = "\n".join(lines)
|
|
|
|
wl = WORKLIST.read_text(encoding="utf-8")
|
|
if "<!-- fetch-list:start" in wl:
|
|
wl = re.sub(r"<!-- fetch-list:start.*?<!-- fetch-list:end -->", block, wl, flags=re.S)
|
|
else:
|
|
# replace the original hand-written list (from the intro up to the first '## Special')
|
|
cut = wl.index("## Special:")
|
|
wl = wl[:wl.index("## NO OTHER SOURCE")] + block + "\n\n" + wl[cut:]
|
|
WORKLIST.write_text(wl, encoding="utf-8")
|
|
print(f"{total} failed sources across {sum(len(v) for v in groups.values())} slots; "
|
|
f"priority cases: {sorted({x[0] for x in groups.get(0, [])})}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|