SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
91 lines
3.7 KiB
Python
91 lines
3.7 KiB
Python
"""
|
|
Triage proposed cuts before a human sets final boundaries.
|
|
|
|
Two checks, both grounded in the case's own record terms:
|
|
END — does the paragraph after the cut still read like this case's article (under-cut),
|
|
or does the last kept paragraph carry no case signal (possible teaser over-cut)?
|
|
INNER — is there an ad or teaser wedged INSIDE the kept range (the end-only check misses these)?
|
|
|
|
Prints only what needs a human's eyes, with the surrounding lines. Run propose_cuts.py first.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
BASE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(BASE))
|
|
from propose_cuts import case_terms, score # noqa: E402
|
|
|
|
STORE = json.loads((BASE / "sourced-articles.json").read_text(encoding="utf-8"))
|
|
CUTS = json.loads((BASE / "staging" / "proposed-cuts.json").read_text(encoding="utf-8"))
|
|
RECS = {r["id"]: r for r in
|
|
json.loads((BASE.parent / "attacks-export-Gart-website.json").read_text(encoding="utf-8"))}
|
|
|
|
# Ad / teaser markers. A hit only matters on a paragraph that carries NO case term, so
|
|
# these substrings flagging inside real prose are dropped by the score==0 guard.
|
|
AD = re.compile(r"(bitpanda|etoro|\bkraken\b|\bledger\b|binance|bitvavo|bybit|coinbase\b|"
|
|
r"cashback|\bbonus\b|publicité|newsletter|whatsapp|telegram|s'inscrire|"
|
|
r"inscrivez|abonn|👉|🔔|🔒|🗞|📰|disclosure|disclaimer|sponsored|affiliate|"
|
|
r"loading more|read the latest|delivered free|was this writing|"
|
|
r"next article|previous article|recevez|découvrez)", re.I)
|
|
|
|
|
|
def clip(s, n=90):
|
|
return " ".join(str(s).split())[:n]
|
|
|
|
|
|
def field_of(cid):
|
|
return {b["staged_file"]: b["field"] for b in STORE[str(cid)] if b.get("staged_file")}
|
|
|
|
|
|
def main():
|
|
batch = {int(a) for a in sys.argv[1:]}
|
|
total = flagged = 0
|
|
for key in sorted(CUTS, key=lambda k: (-int(k.split(":")[0]), k)):
|
|
cid = int(key.split(":")[0])
|
|
if cid not in batch:
|
|
continue
|
|
total += 1
|
|
c = CUTS[key]
|
|
fn = key.split(":")[1]
|
|
field = field_of(cid).get(fn, "?")
|
|
terms = case_terms(RECS[cid])
|
|
paras = (BASE / "staging" / f"{cid:03d}" / fn).read_text(encoding="utf-8").split("\n\n")
|
|
s, e = c["start"], c["end"]
|
|
if s is None or e is None:
|
|
print(f"{cid} {field:6} {fn.split('_',1)[1].replace('.txt',''):20} "
|
|
f"NO BODY RUN — off-case or all-furniture page <<< INSPECT")
|
|
flagged += 1
|
|
continue
|
|
probs = []
|
|
if score(paras[e], terms) == 0 and len(paras[e]) > 60:
|
|
probs.append("LATE?")
|
|
nxt = paras[e + 1] if e + 1 < len(paras) else ""
|
|
if nxt and score(nxt, terms) >= 2:
|
|
probs.append("EARLY?")
|
|
if c["case_term_hits"] < 4:
|
|
probs.append("WEAK")
|
|
inner = [i for i in range(s, e + 1)
|
|
if score(paras[i], terms) == 0 and AD.search(paras[i])]
|
|
line = (f"{cid} {field:6} {fn.split('_',1)[1].replace('.txt',''):20} "
|
|
f"{'t+' if c.get('title') is not None else ''}{s}..{e:<4} h={c['case_term_hits']}")
|
|
if probs or inner:
|
|
flagged += 1
|
|
print(line, " <<<", "|".join(probs) + (f" INNER@{inner}" if inner else ""))
|
|
print(f" last[{e}]: {clip(paras[e])}")
|
|
if "EARLY?" in probs:
|
|
print(f" next[{e+1}]: {clip(nxt)}")
|
|
for i in inner:
|
|
print(f" inner[{i}]: {clip(paras[i])}")
|
|
else:
|
|
print(line)
|
|
print(f"\n{total} articles | {flagged} need a human look")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|