""" Turn staged source text into the phase-2 article store. The store, sourced-articles.json, is read by build_cases.py when it renders case files, so rebuilds stay idempotent: nothing is ever appended to a .txt by hand. Article bodies are SLICED from the staged files by paragraph index. No text is retyped, so "verbatim" holds by construction, and each slice carries the sha256 of both the staged file it came from and the slice itself. Reviewed decisions live in DECISIONS. Everything here was signed off in session. """ import hashlib import json from pathlib import Path BASE = Path(__file__).resolve().parent STAGING = BASE / "staging" STORE = BASE / "sourced-articles.json" TRANSLATIONS = BASE / "translations.json" # start/end are inclusive paragraph indices into the staged file, chosen by reading # the file and finding where the article stops and site furniture begins. DECISIONS = { 460: [ {"field": "url", "file": "01_thebalitimes-com.txt", "verdict": "KEEP", "source": "The Bali Times", "lang": "en", "start": 0, "end": 13, "cut_note": "dropped a site disclaimer and 5 unrelated recommended-article teasers"}, {"field": "url_2", "file": "02_beincrypto-com.txt", "verdict": "KEEP", "source": "BeInCrypto", "lang": "en", "start": 0, "end": 12, "note": "original wire story; url_3 and url_5 are syndications of this", "cut_note": "dropped a self-promo line and 10 unrelated teasers"}, {"field": "url_3", "verdict": "SYNDICATION", "source": "CoinDesk", "of": "url_2"}, {"field": "url_5", "verdict": "SYNDICATION", "source": "Yahoo News", "of": "url_2", "note": "carries one extra paragraph on the wider wrench-attack trend"}, {"field": "url_4", "file": "04_balidiscovery-com.txt", "verdict": "KEEP", "source": "Bali Discovery", "lang": "en", "start": 0, "end": 15, "cut_note": "nothing removed; the transcription contains no site furniture", "method": "WebFetch transcription (site returns 403 to raw retrieval)", "fetched_at": "2026-07-30 (WebFetch)", "note": "The only source naming the victim and an amount, and it attributes BOTH to " "an Instagram interview rather than to police. States explicitly that police " "'have yet to share the amount of money lost'. Does not name the villa."}, ], 462: [ {"field": "url", "file": "01_livecoins-com-br.txt", "verdict": "KEEP", "source": "Livecoins", "lang": "pt", "start": 4, "end": 23, "translate": True, "note": "headline says US$ 750 mil while the body says R$ 750 mil; the body matches " "the record", "cut_note": "dropped 4 leading site-branding lines, a WhatsApp promo and 16 teasers"}, {"field": "url_2", "verdict": "NOT AN ARTICLE", "source": "YouTube", "note": "deliberately linked. Titled around an empresario; Brazilian outlets describe " "this victim as both influencer and businessman. Not a mismatch."}, {"field": "extra_1", "file": "03_acidadeon-com.txt", "verdict": "KEEP", "source": "ACidade ON Ribeirao Preto", "lang": "pt", "start": 0, "end": 17, "translate": True, "note": "NOT linked on the record. Found while resolving the amount. Reports " "R$800,000 where Livecoins reports R$750,000. Record keeps R$750,000; the " "conflict is preserved here.", "cut_note": "dropped a tip-line promo, 3 unrelated local teasers, a tag list and a " "cookie notice"}, ], 461: [ {"field": "url", "verdict": "NOT AN ARTICLE", "source": "NYPD Tips (X)", "note": "police wanted-bulletin post; recorded as a source, not transcribed"}, {"field": "url_2", "file": "02_hoodline-com.txt", "verdict": "KEEP", "source": "Hoodline", "lang": "en", "title": 0, "start": 6, "end": 14, "cut_note": "dropped city-nav lists at the head and 16 unrelated local teasers"}, ], 459: [ {"field": "url", "file": "01_geo-tv.txt", "verdict": "KEEP", "source": "Geo News", "lang": "en", "title": 0, "start": 5, "end": 13, "cut_note": "dropped 12 unrelated national-news teasers"}, {"field": "url_2", "file": "02_abbtakk-tv.txt", "verdict": "KEEP", "source": "Abb Takk News", "lang": "en", "title": 0, "start": 11, "end": 17, "cut_note": "dropped 10 leading and 2 trailing unrelated headlines"}, {"field": "url_3", "file": "03_en-dailypakistan-com-pk.txt", "verdict": "KEEP", "source": "Daily Pakistan", "lang": "en", "title": 0, "start": 23, "end": 30, "cut_note": "dropped 22 blank or nav paragraphs and 33 unrelated teasers"}, {"field": "url_4", "file": "04_propakistani-pk.txt", "verdict": "KEEP", "source": "ProPakistani", "lang": "en", "title": 0, "start": 1, "end": 16, "cut_note": "dropped an inline related-story teaser and the site footer"}, ], 458: [ {"field": "url", "file": "01_journalducoin-com.txt", "verdict": "KEEP", "source": "Journal du Coin", "lang": "fr", "title": 0, "start": 5, "end": 22, "note": "covers a series of incidents on the night of 13-14 June, including case 457", "cut_note": "dropped site nav at the head, a Bitpanda advert, an author bio and a " "cookie notice"}, {"field": "url_2", "file": "02_laprovence-com.txt", "verdict": "PARTIAL", "source": "La Provence", "lang": "fr", "title": 0, "start": 3, "end": 10, "note": "PAYWALLED. The page states 'Il vous reste 75% a lire'. Only the opening " "quarter is retrievable; the rest sits behind a subscription.", "cut_note": "dropped the paywall prompt and a subscribe block"}, {"field": "url_3", "file": "03_cryptopolitan-com.txt", "verdict": "KEEP", "source": "Cryptopolitan (Spanish edition)", "lang": "es", "title": 0, "start": 8, "end": 44, "note": "covers the same series as Journal du Coin, including case 457", "cut_note": "dropped 7 leading unrelated headlines and a repeated table of contents"}, ], 457: [ {"field": "url", "file": "01_journalducoin-com.txt", "verdict": "KEEP", "source": "Journal du Coin", "lang": "fr", "title": 0, "start": 5, "end": 22, "note": "same article as case 458. Covers the series on the night of 13-14 June; this " "record is the Marseille 13e attempted entry at around 3am", "cut_note": "dropped site nav at the head, a Bitpanda advert, an author bio and a " "cookie notice"}, {"field": "url_2", "file": "02_laprovence-com.txt", "verdict": "PARTIAL", "source": "La Provence", "lang": "fr", "title": 0, "start": 3, "end": 10, "note": "PAYWALLED, 75% unavailable. The retrievable portion opens on this incident: " "'vers 3h du matin, ce soir du samedi 13 juin 2026, dans le 13e arrondissement'", "cut_note": "dropped the paywall prompt and a subscribe block"}, {"field": "url_3", "file": "03_cryptopolitan-com.txt", "verdict": "KEEP", "source": "Cryptopolitan (Spanish edition)", "lang": "es", "title": 0, "start": 8, "end": 44, "note": "same article as case 458", "cut_note": "dropped 7 leading unrelated headlines and a repeated table of contents"}, ], 456: [ {"field": "url", "verdict": "NOT AN ARTICLE", "source": "YouTube", "note": "video; recorded as a source, not transcribed"}, {"field": "url_2", "file": "02_acessa-com.txt", "verdict": "KEEP", "source": "Acessa (UOL/Folhapress)", "lang": "pt", "title": 0, "start": 2, "end": 11, "cut_note": "dropped a section label and a newsletter promo"}, {"field": "url_3", "file": "03_serranoticiario-com-br.txt", "verdict": "KEEP", "source": "Serra Noticiario", "lang": "pt", "title": 0, "start": 23, "end": 44, "starts_with": "Criminosos exigiram transfer", "ends_with": "Força Tática do 6º Batalhão.", "cut_note": "dropped 22 leading unrelated teasers and a tip-line prompt. Range was " "22-42 until the page was re-fetched and shifted by two paragraphs; " "anchors now guard against that."}, {"field": "extra_1", "file": "06_tribunaonline-com-br.txt", "verdict": "KEEP", "source": "A Tribuna (earlier same-day piece)", "lang": "pt", "title": 215, "start": 223, "end": 228, "starts_with": "O homem sequestrado em plena luz do dia", "note": "Added after research. The only source reporting any assault during the " "abduction: 'A vitima teria tentado resistir ao sequestro, mas foi agredida'. " "Attributed to unnamed residents and grammatically conditional. Also carries " "'Ate o momento, ninguem foi detido', which dates it before the arrest.", "cut_note": "dropped a login prompt, subhead, datelines and 16 unrelated teasers"}, {"field": "extra_2", "file": "07_agazeta-com-br.txt", "verdict": "KEEP", "source": "A Gazeta", "lang": "pt", "title": 0, "start": 4, "end": 6, "starts_with": "Um homem de 25 anos foi detido em Jardim Carapina", "note": "Added after research. Timestamped 11 June 18:19, which is what places the " "arrest AFTER Sim Noticias published 'ninguem foi preso'. Confirms the " "seizure inventory including three magazines.", "cut_note": "this page carries several unrelated stories; kept only the three " "paragraphs of this one. The auto-proposer picked a different article " "entirely, about a woman arrested for threatening her daughter-in-law."}, {"field": "extra_3", "file": "08_simnoticias-com-br.txt", "verdict": "KEEP", "source": "Sim Noticias (arrest follow-up)", "lang": "pt", "title": 0, "start": 105, "end": 122, "starts_with": "Um jovem de 25 anos foi preso nesta quinta-fe", "ends_with": "car e localizar os demais suspeitos envolvidos no caso.", "note": "Added after research. Published 12 June 08:30, the day after. Confirms the " "arrest and the three-magazine seizure.", "cut_note": "dropped 104 leading nav paragraphs and 15 trailing teasers"}, {"field": "url_4", "file": "04_simnoticias-com-br.txt", "verdict": "KEEP", "source": "Sim Noticias", "lang": "pt", "title": 0, "start": 108, "end": 118, "exclude": [114, 115], "cut_note": "dropped 107 blank or nav paragraphs, 14 trailing teasers, and 2 teaser " "headlines sitting INSIDE the article body (a lost-phone story and " "'Gari que movimentou R$ 4 milhoes', an unrelated operation). The bulk " "import appears to have folded that second teaser into this case's " "summary as a drug-trafficking network claim."}, {"field": "url_5", "file": "05_tribunaonline-com-br.txt", "verdict": "KEEP", "source": "A Tribuna", "lang": "pt", "start": 225, "end": 237, "note": "reports the arrest and weapons seizure during the search rather than the " "abduction itself. A paywall marker appears on the page but it gates the " "comments, not the article.", "cut_note": "dropped 224 leading nav paragraphs and a subscription pitch"}, ], } def sha(t): return hashlib.sha256(t.encode("utf-8")).hexdigest() def main(): store = json.loads(STORE.read_text(encoding="utf-8")) if STORE.exists() else {} translations = json.loads(TRANSLATIONS.read_text(encoding="utf-8")) if TRANSLATIONS.exists() else {} for cid, decisions in DECISIONS.items(): index = {e["field"]: e for e in json.loads((STAGING / f"{cid:03d}" / "index.json").read_text(encoding="utf-8"))} out = [] for d in decisions: src = index.get(d["field"], {}) entry = {k: d[k] for k in ("field", "verdict", "source", "note", "of") if k in d} entry["url"] = src.get("url") if d["verdict"] not in ("KEEP", "PARTIAL"): out.append(entry) continue path = STAGING / f"{cid:03d}" / d["file"] full = path.read_text(encoding="utf-8") paras = full.split("\n\n") # Optional headline paragraph, then one contiguous body range. Each piece is # checked against the staged file separately, since prepending the title makes # the joined result non-contiguous by design. # Teaser headlines for unrelated stories sometimes sit INSIDE the body, so a # single contiguous range is not enough. Excluded indices split the range into # runs; each run is verified against the staged file separately. keep = [i for i in range(d["start"], d["end"] + 1) if i not in set(d.get("exclude", []))] runs, run = [], [] for i in keep: if run and i == run[-1] + 1: run.append(i) else: if run: runs.append(run) run = [i] if run: runs.append(run) pieces = [] if d.get("title") is not None: pieces.append(paras[d["title"]]) pieces += ["\n\n".join(paras[r[0]:r[-1] + 1]) for r in runs] for piece in pieces: if piece not in full: raise SystemExit(f"ABORT: slice for case {cid} {d['field']} is not a " f"substring of the staged file") body = "\n\n".join(pieces) # Staged files are re-fetched from live pages, so paragraph indices drift when # a site adds or drops a teaser. A shifted range still slices a valid substring # and would pass the check above, so anchor the ends to their expected text. first, last = paras[keep[0]], paras[keep[-1]] if d.get("starts_with") and not first.startswith(d["starts_with"]): raise SystemExit( f"ABORT: case {cid} {d['field']} starts on the wrong paragraph.\n" f" expected to start: {d['starts_with']!r}\n" f" actually starts : {first[:80]!r}\n" f" The staged file has changed. Re-derive the cut with propose_cuts.py.") if d.get("ends_with") and not last.rstrip().endswith(d["ends_with"]): raise SystemExit( f"ABORT: case {cid} {d['field']} ends on the wrong paragraph.\n" f" expected to end: {d['ends_with']!r}\n" f" actually ends : {last[-80:]!r}\n" f" The staged file has changed. Re-derive the cut with propose_cuts.py.") key = f"{cid}:{d['field']}" entry.update({ "language": d["lang"], "text": body, "chars": len(body), "sha256": sha(body), "staged_file": d["file"], "staged_sha256": sha(full), "paragraphs_kept": (f"headline [{d['title']}] + " if d.get("title") is not None else "") + f"{d['start']}-{d['end']} of {len(paras)}", "cut_note": d.get("cut_note"), "method": d.get("method", "raw HTTP retrieval, deterministic extraction"), "fetched_at": d.get("fetched_at", src.get("fetched_at")), "http": None if d.get("method") else src.get("http"), }) if d.get("translate"): entry["translation"] = translations.get(key) entry["translation_status"] = "present" if translations.get(key) else "PENDING" out.append(entry) store[str(cid)] = out STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8") for cid, entries in store.items(): print(f"\ncase {cid}") for e in entries: extra = "" if e["verdict"] in ("KEEP", "PARTIAL"): extra = (f"{e['chars']:,} chars, paras {e['paragraphs_kept']}, " f"{e['language']}" + (f", translation {e['translation_status']}" if "translation_status" in e else "")) print(f" [{e['verdict']:<15}] {e.get('field',''):<9} {str(e.get('source'))[:26]:28} {extra}") if __name__ == "__main__": main()