SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
106 lines
4.5 KiB
Python
106 lines
4.5 KiB
Python
"""
|
|
Stage every remaining case's sources, untrimmed.
|
|
|
|
The point is to race link rot. Retrieval is permanent; trimming and fact-checking can
|
|
happen any time afterwards against saved text. So this fetches broadly, files the full
|
|
extracted page, and marks every block UNTRIMMED so nobody mistakes it for curated text.
|
|
|
|
Resumable and non-destructive:
|
|
- cases already present in sourced-articles.json are skipped entirely, so the curated
|
|
entries for 456-462 are never overwritten
|
|
- sources already staged are carried over rather than re-fetched, because re-fetching
|
|
a live page shifts paragraph indices under any approved cut
|
|
|
|
Politeness: a delay between requests. We are reading other people's servers.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from fetch_sources import STAGING, stage_case
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
|
STORE = BASE / "sourced-articles.json"
|
|
LOG = BASE / "staging-progress.json"
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--limit", type=int, default=0, help="stop after N cases (0 = all)")
|
|
ap.add_argument("--delay", type=float, default=1.5, help="seconds between cases")
|
|
args = ap.parse_args()
|
|
|
|
records = json.loads(JSONF.read_text(encoding="utf-8"))
|
|
store = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {}
|
|
progress = json.loads(LOG.read_text(encoding="utf-8")) if LOG.exists() else {}
|
|
|
|
# Cases that already carry article text from the spreadsheet are not worth fetching
|
|
# for: they have evidence on file, and the point of this pass is the ones with none.
|
|
manifest = BASE / "case-files-manifest.json"
|
|
have_articles = set()
|
|
if manifest.exists():
|
|
have_articles = {e["case"] for e in
|
|
json.loads(manifest.read_text(encoding="utf-8"))["entries"]
|
|
if "reported_K&R sheet" in e.get("origin", "")}
|
|
|
|
todo = [r for r in sorted(records, key=lambda r: -r["id"])
|
|
if str(r["id"]) not in store and r["id"] not in have_articles]
|
|
print(f"skipping {len(have_articles)} cases that already hold spreadsheet articles",
|
|
flush=True)
|
|
if args.limit:
|
|
todo = todo[:args.limit]
|
|
print(f"{len(todo)} cases to stage (skipping {len(store)} already in the store)\n", flush=True)
|
|
|
|
for n, rec in enumerate(todo, 1):
|
|
cid = rec["id"]
|
|
try:
|
|
entries = stage_case(rec, new_only=True)
|
|
except Exception as ex: # one bad case must not stop the run
|
|
progress[str(cid)] = {"error": f"{type(ex).__name__}: {str(ex)[:150]}"}
|
|
print(f"[{n}/{len(todo)}] case {cid} ERROR {type(ex).__name__}", flush=True)
|
|
continue
|
|
|
|
blocks, tally = [], {}
|
|
for e in entries:
|
|
tally[e["verdict"]] = tally.get(e["verdict"], 0) + 1
|
|
if e["verdict"] != "FETCHED":
|
|
blocks.append({k: e.get(k) for k in ("field", "verdict", "url", "source", "note")})
|
|
continue
|
|
path = STAGING / f"{cid:03d}" / f"{e['n']:02d}_" \
|
|
f"{e['host'].lower().replace('.', '-')}.txt"
|
|
if not path.exists():
|
|
cands = list((STAGING / f"{cid:03d}").glob(f"{e['n']:02d}_*.txt"))
|
|
if not cands:
|
|
continue
|
|
path = cands[0]
|
|
text = path.read_text(encoding="utf-8")
|
|
blocks.append({
|
|
"field": e["field"], "verdict": "UNTRIMMED", "url": e["url"],
|
|
"source": e["host"], "language": "unknown",
|
|
"text": text, "chars": len(text),
|
|
"sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"staged_file": path.name,
|
|
"paragraphs_kept": f"ALL {len(text.split(chr(10) * 2))} paragraphs, untrimmed",
|
|
"method": "raw HTTP retrieval, deterministic extraction",
|
|
"fetched_at": e.get("fetched_at"), "http": e.get("http"),
|
|
"cut_note": "NOT TRIMMED. Full page extract; trim on read-through.",
|
|
})
|
|
|
|
store[str(cid)] = blocks
|
|
progress[str(cid)] = tally
|
|
STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
LOG.write_text(json.dumps(progress, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"[{n}/{len(todo)}] case {cid:>4} " +
|
|
" ".join(f"{k}={v}" for k, v in sorted(tally.items())), flush=True)
|
|
time.sleep(args.delay)
|
|
|
|
print("\ndone", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|