#!/usr/bin/env python3 """Consolidate the verification batches into batches/VERIFICATION-REPORT.md and batches/contradictions-ALL.md. Re-run after every apply_batch.py.""" import json, glob, re, collections, datetime BASE='batches/' cases=[] for f in sorted(glob.glob(BASE+'batch-[0-9]*.json')): n=int(re.search(r'batch-(\d+)',f).group(1)) for c in json.load(open(f,encoding='utf-8')): c['_batch']=n; cases.append(c) byid={c['id']:c for c in cases} seqs=json.load(open('scenario-sequences.json',encoding='utf-8')) db=json.load(open('../attacks-export-Gart-website.json',encoding='utf-8')) recs=db if isinstance(db,list) else db.get('data',db) dbids={int(r['id']) for r in recs} name={int(r['id']):(r.get('victim') or r.get('name') or '') for r in recs} ov=json.load(open('summary-overrides.json',encoding='utf-8')) fp=json.load(open(BASE+'flag-proposals.json',encoding='utf-8')) today=datetime.date.today().isoformat() THEMES=[ ('Repeat victimisation / several incidents on one record', r'repeat|same household|consecutive days|second event|previous occasions|campaign|similar incident|one record|single record|carried on the same'), ('Insider, intimate or acquaintance perpetrator', r'insider|intimate|family member|own driver|own bodyguard|acquaintance|friend|relative'), ('Empty premises, unattended machines, infrastructure', r'empty premises|unattended|burglar|ATM|machine|infrastructure|mining|premises were empty|nobody home|no one home'), ('Extortion at a distance or threat at the door without entry', r'at a distance|without entry|at the door|no physical contact|Telegram|blackmail|threat only|extortion by'), ('Lured to and held at the attackers\' venue (captivity without transport)', r"attackers' (venue|premises|apartment|rented)|captivity without transport|held at the attackers|attackers' own"), ('Delivery, disguise or uniform ruse (mislabelled as Authority Misuse or Malicious Invitation)', r'delivery|disguise|uniform|DoorDash|courier|parcel|pretext'), ('Preparation: leaks, surveillance, trackers, data breaches', r'leak|surveillance|tracker|reconnaissance|preparation|data breach|breach|OSINT|knew about'), ('Drugging as the approach', r'drugg|drug as|sedat|spiked'), ('Unexplained death or undetermined circumstances', r'undetermined|unexplained|death with|cause of death|disappear'), ('Security measures deciding the outcome', r'security measures|guard|alarm|safe room|CCTV|panic'), ('Scope: fraud, non-violent, or not crypto-linked', r'fraud|non-violent|borderline|out of scope|scope|not crypto'), ('Multiple victims on one record', r'eleven victims|several victims|victims robbed in one|two intended victims|group of'), ] def theme(g): for t,rx in THEMES: if re.search(rx,g,re.I): return t return 'Other' gaps=collections.defaultdict(list) for c in cases: for g in c.get('gaps') or []: gaps[theme(g)].append((c['id'],g)) # contradictions FIELDMAP={'detention flag':'detention','kidnappings':'detention','held_duration':'detention','detention duration':'detention','held duration':'detention','duration':'detention','deaths':'life_taken','theft flag':'theft','crypto stolen':'theft','property taken':'theft','arrest':'arrests','charges':'arrests','sentence total':'arrests','attacker name':'attackers','attribution':'attackers','victim age':'victim','victim role':'victim','victim employer':'victim','age':'victim','role':'victim','children':'victim','time':'date','coin':'coin_type','violence_torture':'violence','injuries':'violence','drugs':'drugs_alcohol','summary (site)':'site summary','venue':'location','status':'status','certainty':'status','sequence of events':'description','attack method':'description','circumstances':'description','who raised the alarm':'description','reports':'description','source url':'notes','flags':'flags'} def norm(f): f=f.strip().lower() base=re.split(r'\s*[/(]\s*',f)[0].strip() return FIELDMAP.get(f,FIELDMAP.get(base,base)) ctr_field=collections.Counter(); ctr_case=collections.Counter(); ctr_total=0 for c in cases: for ct in c.get('contradictions') or []: ctr_total+=1; ctr_field[norm(ct.get('field','?'))]+=1; ctr_case[c['id']]+=1 # flags fl=collections.Counter((p['flag'],str(p['record']),str(p['proposed'])) for p in fp) det_up=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='1') det_down=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='0' and str(p['record'])=='1') life=[p for p in fp if p['flag']=='life_taken'] # sequences lab=collections.Counter(); multi=[]; empty=[]; unsup=[]; dodged=[] for k,v in seqs.items(): sq=v.get('sequence') or [] for l in sq: lab[l]+=1 if len(sq)>1: multi.append(int(k)) if not sq: empty.append(int(k)) if v.get('v1_label_supported') is False: unsup.append(int(k)) if 'Dodged Attack' in sq: dodged.append(int(k)) record_only=sorted(c['id'] for c in cases if c.get('record_only')) SCOPE_RX=r'belongs in this database|place in a crypto database|belongs in a K&R database|belongs in a physical-attack database|borderline for a physical|not crypto-related|candidate for removal|no person was held or attacked|nothing was stolen or demanded|lawful|doxing' scope=sorted({c['id'] for c in cases if re.search(SCOPE_RX,(c.get('notes_for_sofia') or '')+' '+(c.get('rationale') or '')+' '.join(c.get('gaps') or []),re.I)}) nobody=sorted(i for i in empty if i not in record_only) MERGE_RX=r'same incident|duplicate of|split across|split by hand|splitting this record|is a merge of|same operation is split|may be separate records|rest on the same paragraph|separate records with no link' merge=sorted({c['id'] for c in cases if re.search(MERGE_RX,(c.get('notes_for_sofia') or '')+' '.join(c.get('gaps') or []),re.I)}) out=[] w=out.append w(f"# Verification report — all batches ({today})\n") w("Every summary, scenario sequence, flag proposal and contradiction below was written from the case's own dossier (sources/_*.txt) by a writer agent and checked line by line by a read-only checker. The DB export is untouched: everything here is a proposal for Sofia. Per-batch detail sits in contradictions-NN.md; the merged list is contradictions-ALL.md.\n") w("## Coverage\n") w(f"| Item | Count |\n|---|---|") w(f"| Cases in the DB export | {len(dbids)} |") w(f"| Cases with a verified summary (summary-overrides.json) | {len(ov)} |") w(f"| Cases with a scenario sequence (scenario-sequences.json) | {len(seqs)} |") w(f"| Cases verified in batches 01–{max(c['_batch'] for c in cases):02d} | {len(cases)} |") w(f"| Cases still without a verified summary | {len(dbids-{int(k) for k in ov})} {sorted(dbids-{int(k) for k in ov}) or ''} |") w(f"| Record-only cases (no usable article on file) | {len(record_only)} {record_only} |") w("") w("## Contradictions between record and articles\n") w(f"{ctr_total} contradictions across {len(ctr_case)} cases. None was resolved; each is quoted with the article line in contradictions-ALL.md.\n") w("| Field | Contradictions |\n|---|---|") for k,n in ctr_field.most_common(): w(f"| {k} | {n} |") w("") w("Cases with the most contradictions (each needs a full re-entry):\n") for i,n in ctr_case.most_common(20): w(f"- case {i} ({name.get(i,'')}): {n}") w("") w("## Flag proposals\n") w(f"{len(fp)} proposals on {len({p['id'] for p in fp})} cases (batches/flag-proposals.json). 'unconfirmed' means no source on file supports the recorded value; 'threat only' means a weapon or violence was threatened, not used.\n") w("| Flag | Record | Proposed | Count |\n|---|---|---|---|") for (f_,r,p),n in sorted(fl.items(),key=lambda x:(x[0][0],-x[1])): w(f"| {f_} | {r} | {p} | {n} |") w("") w(f"Detention (DB field kidnappings) 0/null → 1 under the owner's rule ({len(det_up)}): {det_up}\n") w(f"Detention 1 → 0 ({len(det_down)}): {det_down}\n") w("Fatality flag (life_taken) proposals:\n") for p in life: w(f"- case {p['id']}: {p['record']} → {p['proposed']} — {p['evidence']}") w("") w("## Scenario sequences\n") w(f"| Item | Count |\n|---|---|\n| Cases with more than one label in sequence | {len(multi)} |\n| Cases where the v1 label is not supported by the file | {len(unsup)} |\n| Foiled attacks (Dodged Attack) | {len(dodged)} |\n| Cases no label fits (empty sequence) | {len(empty)} |\n") w("Label frequency across all sequences:\n") w("| Label | Uses |\n|---|---|") for l,n in lab.most_common(): w(f"| {l} | {n} |") w("") w("Cases no label fits, with the writer's reason:\n") for i in sorted(empty): r=(seqs[str(i)].get('rationale') or '').strip().replace('\n',' ') w(f"- case {i} ({name.get(i,'')}): {r[:260]}") w("") w("## Label gaps (what the eleven scenarios cannot express)\n") w("Gap notes grouped by theme. Counts are notes, not cases; a case can raise several.\n") w("| Theme | Notes | Cases |\n|---|---|---|") for t,_ in THEMES+[('Other','')]: items=gaps.get(t,[]) if not items: continue ids=sorted({i for i,_ in items}) w(f"| {t} | {len(items)} | {', '.join(map(str,ids))} |") w("") w("Proposed additions to the vocabulary, in order of how often the gap appears: (1) a `repeat_victimisation` / campaign link between records; (2) an `insider` initiation attribute (relative, partner, driver, guard, friend); (3) a label for attacks on unattended premises and machines (burglary, ATM ram-raid, mining rigs); (4) remote extortion with threat but no contact; (5) 'held at attackers' venue' as a Kidnapping sub-type; (6) a `ruse` attribute (delivery, uniform, disguise) separate from Authority Misuse and Malicious Invitation; (7) a preparation phase field (leak, surveillance, tracker).\n") for t,_ in THEMES+[('Other','')]: items=gaps.get(t,[]) if not items: continue w(f"### {t}\n") for i,g in sorted(items): w(f"- case {i}: {g}") w("") w("## Cases to decide on scope, merge or split\n") w(f"No scenario label fits (mostly empty premises, unattended machines and burglaries with nobody present) — decide whether a physical-attack database keeps them: {nobody}\n") w(f"Writer questioned scope for another reason (not crypto-related, lawful entry, doxing only, argument during a sale, non-violent device theft): {scope}\n") w(f"Possible merges or splits (same incident across records, or several incidents on one record): {merge}\n") w(f"Record-only (restructured from the DB record; nothing to verify against): {record_only}\n") open(BASE+'VERIFICATION-REPORT.md','w',encoding='utf-8').write('\n'.join(out)) # merged contradictions parts=[] for f in sorted(glob.glob(BASE+'contradictions-[0-9]*.md')): parts.append(open(f,encoding='utf-8').read()) open(BASE+'contradictions-ALL.md','w',encoding='utf-8').write(f"# All contradictions, flag proposals and notes for Sofia — merged from every batch ({today})\n\n"+'\n\n'.join(parts)) print('report:',len(out),'lines; cases',len(cases),'contradictions',ctr_total,'flag proposals',len(fp),'gap notes',sum(len(v) for v in gaps.values()))