K&R database cleanup: handoff bundle
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
Propose where a case's article starts and stops in each staged file, using the case
|
||||
record itself as the anchor.
|
||||
|
||||
The earlier version took the longest contiguous run of body-like paragraphs. That fails
|
||||
on two shapes seen in real pages:
|
||||
- a page carrying several stories, where the longest run is a DIFFERENT article
|
||||
(A Gazeta on case 456 opened on an unrelated arrest for threatening a relative)
|
||||
- an article whose key paragraph sits after a short sentence, so the run stops early
|
||||
(A Tribuna on case 456 truncated three paragraphs before the only line it was added for)
|
||||
|
||||
So: score every paragraph by how many case-specific terms it contains, drawn from the
|
||||
database record. Seed on the best-scoring paragraph and grow outward through body-like
|
||||
text, stopping at teaser runs. A page whose paragraphs never mention the case at all is
|
||||
flagged OFF-CASE, which is the deterministic half of the Phuket trap.
|
||||
|
||||
Emits exact anchors so apply_sources.py can guard against paragraph drift without
|
||||
anyone retyping them.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
STAGING = BASE / "staging"
|
||||
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
||||
|
||||
PAYWALL = re.compile(
|
||||
r"(il vous reste|% à lire|reste \d+% |assinante|already a subscriber|"
|
||||
r"subscribe to (read|continue)|log ?in to (read|continue)|para continuar lendo|"
|
||||
r"conteúdo exclusivo|abonnez-vous|déjà abonné)", re.I)
|
||||
|
||||
JUNK = re.compile(
|
||||
r"(cookie|adblock|bloqueador de an|javascript n|termos de uso|terms of use|"
|
||||
r"all rights reserved|todos os direitos|aviso legal|disclaimer|©|newsletter|"
|
||||
r"receba as principais|participe da nossa|join the groups|login page|please log in|"
|
||||
r"follow us|leia também|à lire aussi|is a writer|escritora y editora|"
|
||||
r"contact us|about us|sabe de alguma coisa)", re.I)
|
||||
|
||||
STOP = {"the", "and", "for", "was", "were", "with", "his", "her", "man", "men", "woman",
|
||||
"unidentified", "undisclosed", "unknown", "multiple", "victim", "repeat", "family",
|
||||
"crypto", "cryptocurrency", "bitcoin", "old", "year", "y/o", "de", "da", "do", "la",
|
||||
"le", "el", "en", "un", "una", "and"}
|
||||
|
||||
MONTHS = {
|
||||
1: ["january", "janeiro", "janvier", "enero"], 2: ["february", "fevereiro", "février", "febrero"],
|
||||
3: ["march", "março", "mars", "marzo"], 4: ["april", "abril", "avril"],
|
||||
5: ["may", "maio", "mai", "mayo"], 6: ["june", "junho", "juin", "junio"],
|
||||
7: ["july", "julho", "juillet", "julio"], 8: ["august", "agosto", "août"],
|
||||
9: ["september", "setembro", "septembre", "septiembre"], 10: ["october", "outubro", "octobre", "octubre"],
|
||||
11: ["november", "novembro", "novembre", "noviembre"], 12: ["december", "dezembro", "décembre", "diciembre"],
|
||||
}
|
||||
|
||||
|
||||
def fold(s):
|
||||
s = unicodedata.normalize("NFKD", str(s or ""))
|
||||
return "".join(c for c in s if not unicodedata.combining(c)).lower()
|
||||
|
||||
|
||||
def case_terms(rec):
|
||||
"""Distinctive strings that should appear in an article about THIS case."""
|
||||
terms = set()
|
||||
for field in ("victim", "location", "country"):
|
||||
for tok in re.split(r"[^A-Za-z0-9]+", fold(rec.get(field))):
|
||||
if len(tok) > 3 and tok not in STOP:
|
||||
terms.add(tok)
|
||||
if rec.get("month") in MONTHS:
|
||||
terms.update(MONTHS[rec["month"]])
|
||||
m = re.search(r"\b(\d{1,2})\b", str(rec.get("date") or ""))
|
||||
if m:
|
||||
terms.add(m.group(1))
|
||||
for field in ("money_wanted", "description"):
|
||||
for num in re.findall(r"\d[\d.,]{2,}", str(rec.get(field) or "")):
|
||||
terms.add(fold(num))
|
||||
# Distinctive nouns from the description carry most of the signal: a car model, a
|
||||
# street, a company. Without them a bare "11" matches half of any news page.
|
||||
for field in ("description", "notes"):
|
||||
for tok in re.findall(r"\b[A-Z][A-Za-z0-9]{4,}\b", str(rec.get(field) or "")):
|
||||
t = fold(tok)
|
||||
if t not in STOP:
|
||||
terms.add(t)
|
||||
return {t for t in terms if t}
|
||||
|
||||
|
||||
def score(para, terms):
|
||||
f = fold(para)
|
||||
return sum(1 for t in terms if t in f)
|
||||
|
||||
|
||||
def bodyish(p):
|
||||
p = p.strip()
|
||||
return len(p) > 110 and re.search(r"[.!?”»\"]$", p) and not JUNK.search(p)
|
||||
|
||||
|
||||
def propose(paras, terms, bridge=2):
|
||||
"""Runs decide the boundaries; case terms decide which run.
|
||||
|
||||
Boundaries come from contiguous body text, tolerating up to `bridge` short lines so
|
||||
a one-sentence paragraph cannot truncate an article. Selection then picks the run
|
||||
that actually talks about this case, which is what a multi-story page needs.
|
||||
"""
|
||||
scores = [score(p, terms) for p in paras]
|
||||
total_hits = sum(scores)
|
||||
|
||||
runs, cur, gap = [], [], 0
|
||||
for i, p in enumerate(paras):
|
||||
if bodyish(p):
|
||||
cur.append(i)
|
||||
gap = 0
|
||||
elif cur and len(p.strip()) <= 200 and not JUNK.search(p) and gap < bridge:
|
||||
cur.append(i)
|
||||
gap += 1
|
||||
else:
|
||||
if cur:
|
||||
runs.append(cur)
|
||||
cur, gap = [], 0
|
||||
if cur:
|
||||
runs.append(cur)
|
||||
if not runs:
|
||||
return None, None, total_hits
|
||||
|
||||
def run_score(r):
|
||||
# Distinct terms first: a long unrelated run accumulates raw hits just by being
|
||||
# long, which is how A Gazeta's story about a woman threatening her relative beat
|
||||
# the actual kidnapping. Density breaks ties; length only decides after that.
|
||||
text = fold(" ".join(paras[i] for i in r))
|
||||
distinct = sum(1 for t in terms if t in text)
|
||||
chars = sum(len(paras[i]) for i in r) or 1
|
||||
return (distinct, sum(scores[i] for i in r) / chars, chars)
|
||||
|
||||
best = max(runs, key=run_score)
|
||||
start, end = best[0], best[-1]
|
||||
# Trim bridged non-body lines off either end.
|
||||
while start < end and not bodyish(paras[start]):
|
||||
start += 1
|
||||
while end > start and not bodyish(paras[end]):
|
||||
end -= 1
|
||||
return start, end, total_hits
|
||||
|
||||
|
||||
def main():
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
||||
records = {r["id"]: r for r in json.loads(JSONF.read_text(encoding="utf-8"))}
|
||||
out = {}
|
||||
for case in sys.argv[1:]:
|
||||
cid = int(case)
|
||||
rec = records[cid]
|
||||
terms = case_terms(rec)
|
||||
print(f"\n### case {cid} terms: {' '.join(sorted(terms))[:110]}")
|
||||
for f in sorted((STAGING / f"{cid:03d}").glob("*.txt")):
|
||||
paras = f.read_text(encoding="utf-8").split("\n\n")
|
||||
start, end, hits = propose(paras, terms)
|
||||
flag = ""
|
||||
if hits == 0:
|
||||
flag = " !! OFF-CASE: no case term appears anywhere on this page"
|
||||
elif hits < 3:
|
||||
flag = f" !! WEAK: only {hits} case-term hits on the whole page"
|
||||
pay = [p.strip()[:60] for p in paras if PAYWALL.search(p)]
|
||||
title = paras[0].strip() if paras else ""
|
||||
use_title = start not in (0, None) and 20 < len(title) < 200 and not JUNK.search(title)
|
||||
key = f"{cid}:{f.name}"
|
||||
out[key] = {"start": start, "end": end, "title": 0 if use_title else None,
|
||||
"paras": len(paras), "case_term_hits": hits,
|
||||
"starts_with": paras[start][:45] if start is not None else None,
|
||||
"ends_with": paras[end][-55:] if end is not None else None,
|
||||
"paywall": pay}
|
||||
kept = len("\n\n".join(paras[start:end + 1])) if start is not None else 0
|
||||
print(f" {f.name:34} {len(paras):>4} paras -> "
|
||||
f"{'[0] + ' if use_title else ''}{start}..{end} {kept:>6,} chars hits={hits}{flag}")
|
||||
if pay:
|
||||
print(f" PAYWALL: {pay[0]}")
|
||||
if start is not None:
|
||||
print(f" starts: {paras[start][:88]}")
|
||||
print(f" ends : {paras[end][-88:]}")
|
||||
(STAGING / "proposed-cuts.json").write_text(json.dumps(out, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
print(f"\nwrote {len(out)} proposals with anchors -> staging/proposed-cuts.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user