SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
42 lines
1.7 KiB
Python
42 lines
1.7 KiB
Python
import sys, io, json, re, os
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
BASE = os.path.dirname(os.path.abspath(__file__))
|
|
SRC = os.path.dirname(BASE)
|
|
recs = json.loads(open(os.path.join(SRC, "attacks-export-Gart-website.json"), encoding="utf-8").read())
|
|
sourced = json.loads(open(os.path.join(BASE, "sourced-articles.json"), encoding="utf-8").read())
|
|
|
|
CATS = ["Victims", "Attackers", "Attack Method", "Violence Used", "Crypto Demanded", "Status"]
|
|
|
|
def has_source(cid):
|
|
# sheet article present? check case file for "FETCHED ARTICLE" or "SPREADSHEET"/"ARTICLE" body
|
|
blocks = sourced.get(str(cid), [])
|
|
text_verdicts = {"KEEP", "PARTIAL", "UNTRIMMED"}
|
|
fetched = any(b.get("verdict") in text_verdicts for b in blocks) if isinstance(blocks, list) else False
|
|
return fetched
|
|
|
|
rows = []
|
|
for r in recs:
|
|
s = r.get("summary") or ""
|
|
n = len(s)
|
|
ncat = sum(1 for c in CATS if c.lower() in s.lower())
|
|
rows.append((r["id"], n, ncat, r.get("victim","")[:28]))
|
|
|
|
rows.sort()
|
|
print(f"{'id':>4} {'chars':>6} {'cats':>4} victim")
|
|
print("-"*60)
|
|
for cid, n, ncat, v in rows:
|
|
flag = ""
|
|
if n < 400: flag = " <== SHORT"
|
|
if ncat < 4: flag += " [unstructured]"
|
|
print(f"{cid:>4} {n:>6} {ncat:>4} {v}{flag}")
|
|
|
|
shorts = [cid for cid,n,ncat,v in rows if n < 400 or ncat < 4]
|
|
print("\nTOTAL cases:", len(rows))
|
|
print("SHORT (<400 chars) or unstructured (<4 cats):", len(shorts))
|
|
import statistics
|
|
lens = [n for _,n,_,_ in rows]
|
|
print("median chars:", statistics.median(lens), "| mean:", int(statistics.mean(lens)))
|
|
# latest 15 as reference
|
|
latest = sorted(rows, key=lambda x: x[0])[-15:]
|
|
print("latest 15 char range:", min(n for _,n,_,_ in latest), "-", max(n for _,n,_,_ in latest))
|