import sys, io, json, re, os
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
BASE = os.path.dirname(os.path.abspath(__file__))
SRC = os.path.dirname(BASE)
recs = json.loads(open(os.path.join(SRC, "attacks-export-Gart-website.json"), encoding="utf-8").read())
sourced = json.loads(open(os.path.join(BASE, "sourced-articles.json"), encoding="utf-8").read())

CATS = ["Victims", "Attackers", "Attack Method", "Violence Used", "Crypto Demanded", "Status"]

def has_source(cid):
    # sheet article present? check case file for "FETCHED ARTICLE" or "SPREADSHEET"/"ARTICLE" body
    blocks = sourced.get(str(cid), [])
    text_verdicts = {"KEEP", "PARTIAL", "UNTRIMMED"}
    fetched = any(b.get("verdict") in text_verdicts for b in blocks) if isinstance(blocks, list) else False
    return fetched

rows = []
for r in recs:
    s = r.get("summary") or ""
    n = len(s)
    ncat = sum(1 for c in CATS if c.lower() in s.lower())
    rows.append((r["id"], n, ncat, r.get("victim","")[:28]))

rows.sort()
print(f"{'id':>4} {'chars':>6} {'cats':>4}  victim")
print("-"*60)
for cid, n, ncat, v in rows:
    flag = ""
    if n < 400: flag = "  <== SHORT"
    if ncat < 4: flag += "  [unstructured]"
    print(f"{cid:>4} {n:>6} {ncat:>4}  {v}{flag}")

shorts = [cid for cid,n,ncat,v in rows if n < 400 or ncat < 4]
print("\nTOTAL cases:", len(rows))
print("SHORT (<400 chars) or unstructured (<4 cats):", len(shorts))
import statistics
lens = [n for _,n,_,_ in rows]
print("median chars:", statistics.median(lens), "| mean:", int(statistics.mean(lens)))
# latest 15 as reference
latest = sorted(rows, key=lambda x: x[0])[-15:]
print("latest 15 char range:", min(n for _,n,_,_ in latest), "-", max(n for _,n,_,_ in latest))