SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
285 lines
16 KiB
Python
285 lines
16 KiB
Python
"""
|
|
Turn staged source text into the phase-2 article store.
|
|
|
|
The store, sourced-articles.json, is read by build_cases.py when it renders case
|
|
files, so rebuilds stay idempotent: nothing is ever appended to a .txt by hand.
|
|
|
|
Article bodies are SLICED from the staged files by paragraph index. No text is
|
|
retyped, so "verbatim" holds by construction, and each slice carries the sha256 of
|
|
both the staged file it came from and the slice itself.
|
|
|
|
Reviewed decisions live in DECISIONS. Everything here was signed off in session.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
STAGING = BASE / "staging"
|
|
STORE = BASE / "sourced-articles.json"
|
|
TRANSLATIONS = BASE / "translations.json"
|
|
|
|
# start/end are inclusive paragraph indices into the staged file, chosen by reading
|
|
# the file and finding where the article stops and site furniture begins.
|
|
DECISIONS = {
|
|
460: [
|
|
{"field": "url", "file": "01_thebalitimes-com.txt", "verdict": "KEEP",
|
|
"source": "The Bali Times", "lang": "en", "start": 0, "end": 13,
|
|
"cut_note": "dropped a site disclaimer and 5 unrelated recommended-article teasers"},
|
|
{"field": "url_2", "file": "02_beincrypto-com.txt", "verdict": "KEEP",
|
|
"source": "BeInCrypto", "lang": "en", "start": 0, "end": 12,
|
|
"note": "original wire story; url_3 and url_5 are syndications of this",
|
|
"cut_note": "dropped a self-promo line and 10 unrelated teasers"},
|
|
{"field": "url_3", "verdict": "SYNDICATION", "source": "CoinDesk", "of": "url_2"},
|
|
{"field": "url_5", "verdict": "SYNDICATION", "source": "Yahoo News", "of": "url_2",
|
|
"note": "carries one extra paragraph on the wider wrench-attack trend"},
|
|
{"field": "url_4", "file": "04_balidiscovery-com.txt", "verdict": "KEEP",
|
|
"source": "Bali Discovery", "lang": "en", "start": 0, "end": 15,
|
|
"cut_note": "nothing removed; the transcription contains no site furniture",
|
|
"method": "WebFetch transcription (site returns 403 to raw retrieval)",
|
|
"fetched_at": "2026-07-30 (WebFetch)",
|
|
"note": "The only source naming the victim and an amount, and it attributes BOTH to "
|
|
"an Instagram interview rather than to police. States explicitly that police "
|
|
"'have yet to share the amount of money lost'. Does not name the villa."},
|
|
],
|
|
462: [
|
|
{"field": "url", "file": "01_livecoins-com-br.txt", "verdict": "KEEP",
|
|
"source": "Livecoins", "lang": "pt", "start": 4, "end": 23, "translate": True,
|
|
"note": "headline says US$ 750 mil while the body says R$ 750 mil; the body matches "
|
|
"the record",
|
|
"cut_note": "dropped 4 leading site-branding lines, a WhatsApp promo and 16 teasers"},
|
|
{"field": "url_2", "verdict": "NOT AN ARTICLE", "source": "YouTube",
|
|
"note": "deliberately linked. Titled around an empresario; Brazilian outlets describe "
|
|
"this victim as both influencer and businessman. Not a mismatch."},
|
|
{"field": "extra_1", "file": "03_acidadeon-com.txt", "verdict": "KEEP",
|
|
"source": "ACidade ON Ribeirao Preto", "lang": "pt", "start": 0, "end": 17,
|
|
"translate": True,
|
|
"note": "NOT linked on the record. Found while resolving the amount. Reports "
|
|
"R$800,000 where Livecoins reports R$750,000. Record keeps R$750,000; the "
|
|
"conflict is preserved here.",
|
|
"cut_note": "dropped a tip-line promo, 3 unrelated local teasers, a tag list and a "
|
|
"cookie notice"},
|
|
],
|
|
461: [
|
|
{"field": "url", "verdict": "NOT AN ARTICLE", "source": "NYPD Tips (X)",
|
|
"note": "police wanted-bulletin post; recorded as a source, not transcribed"},
|
|
{"field": "url_2", "file": "02_hoodline-com.txt", "verdict": "KEEP",
|
|
"source": "Hoodline", "lang": "en", "title": 0, "start": 6, "end": 14,
|
|
"cut_note": "dropped city-nav lists at the head and 16 unrelated local teasers"},
|
|
],
|
|
459: [
|
|
{"field": "url", "file": "01_geo-tv.txt", "verdict": "KEEP", "source": "Geo News",
|
|
"lang": "en", "title": 0, "start": 5, "end": 13,
|
|
"cut_note": "dropped 12 unrelated national-news teasers"},
|
|
{"field": "url_2", "file": "02_abbtakk-tv.txt", "verdict": "KEEP", "source": "Abb Takk News",
|
|
"lang": "en", "title": 0, "start": 11, "end": 17,
|
|
"cut_note": "dropped 10 leading and 2 trailing unrelated headlines"},
|
|
{"field": "url_3", "file": "03_en-dailypakistan-com-pk.txt", "verdict": "KEEP",
|
|
"source": "Daily Pakistan", "lang": "en", "title": 0, "start": 23, "end": 30,
|
|
"cut_note": "dropped 22 blank or nav paragraphs and 33 unrelated teasers"},
|
|
{"field": "url_4", "file": "04_propakistani-pk.txt", "verdict": "KEEP",
|
|
"source": "ProPakistani", "lang": "en", "title": 0, "start": 1, "end": 16,
|
|
"cut_note": "dropped an inline related-story teaser and the site footer"},
|
|
],
|
|
458: [
|
|
{"field": "url", "file": "01_journalducoin-com.txt", "verdict": "KEEP",
|
|
"source": "Journal du Coin", "lang": "fr", "title": 0, "start": 5, "end": 22,
|
|
"note": "covers a series of incidents on the night of 13-14 June, including case 457",
|
|
"cut_note": "dropped site nav at the head, a Bitpanda advert, an author bio and a "
|
|
"cookie notice"},
|
|
{"field": "url_2", "file": "02_laprovence-com.txt", "verdict": "PARTIAL",
|
|
"source": "La Provence", "lang": "fr", "title": 0, "start": 3, "end": 10,
|
|
"note": "PAYWALLED. The page states 'Il vous reste 75% a lire'. Only the opening "
|
|
"quarter is retrievable; the rest sits behind a subscription.",
|
|
"cut_note": "dropped the paywall prompt and a subscribe block"},
|
|
{"field": "url_3", "file": "03_cryptopolitan-com.txt", "verdict": "KEEP",
|
|
"source": "Cryptopolitan (Spanish edition)", "lang": "es", "title": 0,
|
|
"start": 8, "end": 44,
|
|
"note": "covers the same series as Journal du Coin, including case 457",
|
|
"cut_note": "dropped 7 leading unrelated headlines and a repeated table of contents"},
|
|
],
|
|
457: [
|
|
{"field": "url", "file": "01_journalducoin-com.txt", "verdict": "KEEP",
|
|
"source": "Journal du Coin", "lang": "fr", "title": 0, "start": 5, "end": 22,
|
|
"note": "same article as case 458. Covers the series on the night of 13-14 June; this "
|
|
"record is the Marseille 13e attempted entry at around 3am",
|
|
"cut_note": "dropped site nav at the head, a Bitpanda advert, an author bio and a "
|
|
"cookie notice"},
|
|
{"field": "url_2", "file": "02_laprovence-com.txt", "verdict": "PARTIAL",
|
|
"source": "La Provence", "lang": "fr", "title": 0, "start": 3, "end": 10,
|
|
"note": "PAYWALLED, 75% unavailable. The retrievable portion opens on this incident: "
|
|
"'vers 3h du matin, ce soir du samedi 13 juin 2026, dans le 13e arrondissement'",
|
|
"cut_note": "dropped the paywall prompt and a subscribe block"},
|
|
{"field": "url_3", "file": "03_cryptopolitan-com.txt", "verdict": "KEEP",
|
|
"source": "Cryptopolitan (Spanish edition)", "lang": "es", "title": 0,
|
|
"start": 8, "end": 44,
|
|
"note": "same article as case 458",
|
|
"cut_note": "dropped 7 leading unrelated headlines and a repeated table of contents"},
|
|
],
|
|
456: [
|
|
{"field": "url", "verdict": "NOT AN ARTICLE", "source": "YouTube",
|
|
"note": "video; recorded as a source, not transcribed"},
|
|
{"field": "url_2", "file": "02_acessa-com.txt", "verdict": "KEEP",
|
|
"source": "Acessa (UOL/Folhapress)", "lang": "pt", "title": 0, "start": 2, "end": 11,
|
|
"cut_note": "dropped a section label and a newsletter promo"},
|
|
{"field": "url_3", "file": "03_serranoticiario-com-br.txt", "verdict": "KEEP",
|
|
"source": "Serra Noticiario", "lang": "pt", "title": 0, "start": 23, "end": 44,
|
|
"starts_with": "Criminosos exigiram transfer",
|
|
"ends_with": "Força Tática do 6º Batalhão.",
|
|
"cut_note": "dropped 22 leading unrelated teasers and a tip-line prompt. Range was "
|
|
"22-42 until the page was re-fetched and shifted by two paragraphs; "
|
|
"anchors now guard against that."},
|
|
{"field": "extra_1", "file": "06_tribunaonline-com-br.txt", "verdict": "KEEP",
|
|
"source": "A Tribuna (earlier same-day piece)", "lang": "pt",
|
|
"title": 215, "start": 223, "end": 228,
|
|
"starts_with": "O homem sequestrado em plena luz do dia",
|
|
"note": "Added after research. The only source reporting any assault during the "
|
|
"abduction: 'A vitima teria tentado resistir ao sequestro, mas foi agredida'. "
|
|
"Attributed to unnamed residents and grammatically conditional. Also carries "
|
|
"'Ate o momento, ninguem foi detido', which dates it before the arrest.",
|
|
"cut_note": "dropped a login prompt, subhead, datelines and 16 unrelated teasers"},
|
|
{"field": "extra_2", "file": "07_agazeta-com-br.txt", "verdict": "KEEP",
|
|
"source": "A Gazeta", "lang": "pt", "title": 0, "start": 4, "end": 6,
|
|
"starts_with": "Um homem de 25 anos foi detido em Jardim Carapina",
|
|
"note": "Added after research. Timestamped 11 June 18:19, which is what places the "
|
|
"arrest AFTER Sim Noticias published 'ninguem foi preso'. Confirms the "
|
|
"seizure inventory including three magazines.",
|
|
"cut_note": "this page carries several unrelated stories; kept only the three "
|
|
"paragraphs of this one. The auto-proposer picked a different article "
|
|
"entirely, about a woman arrested for threatening her daughter-in-law."},
|
|
{"field": "extra_3", "file": "08_simnoticias-com-br.txt", "verdict": "KEEP",
|
|
"source": "Sim Noticias (arrest follow-up)", "lang": "pt",
|
|
"title": 0, "start": 105, "end": 122,
|
|
"starts_with": "Um jovem de 25 anos foi preso nesta quinta-fe",
|
|
"ends_with": "car e localizar os demais suspeitos envolvidos no caso.",
|
|
"note": "Added after research. Published 12 June 08:30, the day after. Confirms the "
|
|
"arrest and the three-magazine seizure.",
|
|
"cut_note": "dropped 104 leading nav paragraphs and 15 trailing teasers"},
|
|
{"field": "url_4", "file": "04_simnoticias-com-br.txt", "verdict": "KEEP",
|
|
"source": "Sim Noticias", "lang": "pt", "title": 0, "start": 108, "end": 118,
|
|
"exclude": [114, 115],
|
|
"cut_note": "dropped 107 blank or nav paragraphs, 14 trailing teasers, and 2 teaser "
|
|
"headlines sitting INSIDE the article body (a lost-phone story and "
|
|
"'Gari que movimentou R$ 4 milhoes', an unrelated operation). The bulk "
|
|
"import appears to have folded that second teaser into this case's "
|
|
"summary as a drug-trafficking network claim."},
|
|
{"field": "url_5", "file": "05_tribunaonline-com-br.txt", "verdict": "KEEP",
|
|
"source": "A Tribuna", "lang": "pt", "start": 225, "end": 237,
|
|
"note": "reports the arrest and weapons seizure during the search rather than the "
|
|
"abduction itself. A paywall marker appears on the page but it gates the "
|
|
"comments, not the article.",
|
|
"cut_note": "dropped 224 leading nav paragraphs and a subscription pitch"},
|
|
],
|
|
}
|
|
|
|
|
|
def sha(t):
|
|
return hashlib.sha256(t.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def main():
|
|
store = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {}
|
|
translations = json.loads(TRANSLATIONS.read_text(encoding="utf-8")) if TRANSLATIONS.exists() else {}
|
|
|
|
for cid, decisions in DECISIONS.items():
|
|
index = {e["field"]: e for e in
|
|
json.loads((STAGING / f"{cid:03d}" / "index.json").read_text(encoding="utf-8"))}
|
|
out = []
|
|
for d in decisions:
|
|
src = index.get(d["field"], {})
|
|
entry = {k: d[k] for k in ("field", "verdict", "source", "note", "of") if k in d}
|
|
entry["url"] = src.get("url")
|
|
|
|
if d["verdict"] not in ("KEEP", "PARTIAL"):
|
|
out.append(entry)
|
|
continue
|
|
|
|
path = STAGING / f"{cid:03d}" / d["file"]
|
|
full = path.read_text(encoding="utf-8")
|
|
paras = full.split("\n\n")
|
|
|
|
# Optional headline paragraph, then one contiguous body range. Each piece is
|
|
# checked against the staged file separately, since prepending the title makes
|
|
# the joined result non-contiguous by design.
|
|
# Teaser headlines for unrelated stories sometimes sit INSIDE the body, so a
|
|
# single contiguous range is not enough. Excluded indices split the range into
|
|
# runs; each run is verified against the staged file separately.
|
|
keep = [i for i in range(d["start"], d["end"] + 1)
|
|
if i not in set(d.get("exclude", []))]
|
|
runs, run = [], []
|
|
for i in keep:
|
|
if run and i == run[-1] + 1:
|
|
run.append(i)
|
|
else:
|
|
if run:
|
|
runs.append(run)
|
|
run = [i]
|
|
if run:
|
|
runs.append(run)
|
|
|
|
pieces = []
|
|
if d.get("title") is not None:
|
|
pieces.append(paras[d["title"]])
|
|
pieces += ["\n\n".join(paras[r[0]:r[-1] + 1]) for r in runs]
|
|
for piece in pieces:
|
|
if piece not in full:
|
|
raise SystemExit(f"ABORT: slice for case {cid} {d['field']} is not a "
|
|
f"substring of the staged file")
|
|
body = "\n\n".join(pieces)
|
|
|
|
# Staged files are re-fetched from live pages, so paragraph indices drift when
|
|
# a site adds or drops a teaser. A shifted range still slices a valid substring
|
|
# and would pass the check above, so anchor the ends to their expected text.
|
|
first, last = paras[keep[0]], paras[keep[-1]]
|
|
if d.get("starts_with") and not first.startswith(d["starts_with"]):
|
|
raise SystemExit(
|
|
f"ABORT: case {cid} {d['field']} starts on the wrong paragraph.\n"
|
|
f" expected to start: {d['starts_with']!r}\n"
|
|
f" actually starts : {first[:80]!r}\n"
|
|
f" The staged file has changed. Re-derive the cut with propose_cuts.py.")
|
|
if d.get("ends_with") and not last.rstrip().endswith(d["ends_with"]):
|
|
raise SystemExit(
|
|
f"ABORT: case {cid} {d['field']} ends on the wrong paragraph.\n"
|
|
f" expected to end: {d['ends_with']!r}\n"
|
|
f" actually ends : {last[-80:]!r}\n"
|
|
f" The staged file has changed. Re-derive the cut with propose_cuts.py.")
|
|
|
|
key = f"{cid}:{d['field']}"
|
|
entry.update({
|
|
"language": d["lang"],
|
|
"text": body,
|
|
"chars": len(body),
|
|
"sha256": sha(body),
|
|
"staged_file": d["file"],
|
|
"staged_sha256": sha(full),
|
|
"paragraphs_kept": (f"headline [{d['title']}] + " if d.get("title") is not None
|
|
else "") + f"{d['start']}-{d['end']} of {len(paras)}",
|
|
"cut_note": d.get("cut_note"),
|
|
"method": d.get("method", "raw HTTP retrieval, deterministic extraction"),
|
|
"fetched_at": d.get("fetched_at", src.get("fetched_at")),
|
|
"http": None if d.get("method") else src.get("http"),
|
|
})
|
|
if d.get("translate"):
|
|
entry["translation"] = translations.get(key)
|
|
entry["translation_status"] = "present" if translations.get(key) else "PENDING"
|
|
out.append(entry)
|
|
|
|
store[str(cid)] = out
|
|
|
|
STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
for cid, entries in store.items():
|
|
print(f"\ncase {cid}")
|
|
for e in entries:
|
|
extra = ""
|
|
if e["verdict"] in ("KEEP", "PARTIAL"):
|
|
extra = (f"{e['chars']:,} chars, paras {e['paragraphs_kept']}, "
|
|
f"{e['language']}"
|
|
+ (f", translation {e['translation_status']}" if "translation_status" in e else ""))
|
|
print(f" [{e['verdict']:<15}] {e.get('field',''):<9} {str(e.get('source'))[:26]:28} {extra}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|