K&R database cleanup: handoff bundle

Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf

merge-output/survey_summaries.py

@@ -0,0 +1,41 @@
+import sys, io, json, re, os
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
+BASE = os.path.dirname(os.path.abspath(__file__))
+SRC = os.path.dirname(BASE)
+recs = json.loads(open(os.path.join(SRC, "attacks-export-Gart-website.json"), encoding="utf-8").read())
+sourced = json.loads(open(os.path.join(BASE, "sourced-articles.json"), encoding="utf-8").read())
+
+CATS = ["Victims", "Attackers", "Attack Method", "Violence Used", "Crypto Demanded", "Status"]
+
+def has_source(cid):
+    # sheet article present? check case file for "FETCHED ARTICLE" or "SPREADSHEET"/"ARTICLE" body
+    blocks = sourced.get(str(cid), [])
+    text_verdicts = {"KEEP", "PARTIAL", "UNTRIMMED"}
+    fetched = any(b.get("verdict") in text_verdicts for b in blocks) if isinstance(blocks, list) else False
+    return fetched
+
+rows = []
+for r in recs:
+    s = r.get("summary") or ""
+    n = len(s)
+    ncat = sum(1 for c in CATS if c.lower() in s.lower())
+    rows.append((r["id"], n, ncat, r.get("victim","")[:28]))
+
+rows.sort()
+print(f"{'id':>4} {'chars':>6} {'cats':>4}  victim")
+print("-"*60)
+for cid, n, ncat, v in rows:
+    flag = ""
+    if n < 400: flag = "  <== SHORT"
+    if ncat < 4: flag += "  [unstructured]"
+    print(f"{cid:>4} {n:>6} {ncat:>4}  {v}{flag}")
+
+shorts = [cid for cid,n,ncat,v in rows if n < 400 or ncat < 4]
+print("\nTOTAL cases:", len(rows))
+print("SHORT (<400 chars) or unstructured (<4 cats):", len(shorts))
+import statistics
+lens = [n for _,n,_,_ in rows]
+print("median chars:", statistics.median(lens), "| mean:", int(statistics.mean(lens)))
+# latest 15 as reference
+latest = sorted(rows, key=lambda x: x[0])[-15:]
+print("latest 15 char range:", min(n for _,n,_,_ in latest), "-", max(n for _,n,_,_ in latest))