This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/make_verification_report.py
T

155 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Consolidate the verification batches into batches/VERIFICATION-REPORT.md and
batches/contradictions-ALL.md. Re-run after every apply_batch.py."""
import json, glob, re, collections, datetime
BASE='batches/'
cases=[]
for f in sorted(glob.glob(BASE+'batch-[0-9]*.json')):
n=int(re.search(r'batch-(\d+)',f).group(1))
for c in json.load(open(f,encoding='utf-8')):
c['_batch']=n; cases.append(c)
byid={c['id']:c for c in cases}
seqs=json.load(open('scenario-sequences.json',encoding='utf-8'))
db=json.load(open('../attacks-export-Gart-website.json',encoding='utf-8'))
recs=db if isinstance(db,list) else db.get('data',db)
dbids={int(r['id']) for r in recs}
name={int(r['id']):(r.get('victim') or r.get('name') or '') for r in recs}
ov=json.load(open('summary-overrides.json',encoding='utf-8'))
fp=json.load(open(BASE+'flag-proposals.json',encoding='utf-8'))
today=datetime.date.today().isoformat()
THEMES=[
('Repeat victimisation / several incidents on one record', r'repeat|same household|consecutive days|second event|previous occasions|campaign|similar incident|one record|single record|carried on the same'),
('Insider, intimate or acquaintance perpetrator', r'insider|intimate|family member|own driver|own bodyguard|acquaintance|friend|relative'),
('Empty premises, unattended machines, infrastructure', r'empty premises|unattended|burglar|ATM|machine|infrastructure|mining|premises were empty|nobody home|no one home'),
('Extortion at a distance or threat at the door without entry', r'at a distance|without entry|at the door|no physical contact|Telegram|blackmail|threat only|extortion by'),
('Lured to and held at the attackers\' venue (captivity without transport)', r"attackers' (venue|premises|apartment|rented)|captivity without transport|held at the attackers|attackers' own"),
('Delivery, disguise or uniform ruse (mislabelled as Authority Misuse or Malicious Invitation)', r'delivery|disguise|uniform|DoorDash|courier|parcel|pretext'),
('Preparation: leaks, surveillance, trackers, data breaches', r'leak|surveillance|tracker|reconnaissance|preparation|data breach|breach|OSINT|knew about'),
('Drugging as the approach', r'drugg|drug as|sedat|spiked'),
('Unexplained death or undetermined circumstances', r'undetermined|unexplained|death with|cause of death|disappear'),
('Security measures deciding the outcome', r'security measures|guard|alarm|safe room|CCTV|panic'),
('Scope: fraud, non-violent, or not crypto-linked', r'fraud|non-violent|borderline|out of scope|scope|not crypto'),
('Multiple victims, proxy victims or one victim field', r'eleven victims|several victims|victims robbed in one|two intended victims|group of|one victim field|single victim field|proxy victim|held in his place|household used as lever|two victims held|three people were attacked'),
('Amount and coin fields cannot split crypto from cash, goods or fiat', r'money_wanted|coin_type|amount field|cash side|fiat|not crypto|not cryptocurrency|cannot show that only|silver|jewellery|watch'),
('Control moves between places; one venue field', r'one venue|single venue|two different labels|control shifted|two places|moves the victim|second address|neither a premises nor|stationary'),
('Outcome after the transfer (killing, escape, rescue) is not an initiation label', r'killed after|after the wallets|escape by|own escape|rescued|interrupted|broken by police|self-defence|the harm that did occur'),
('Crypto link or complainant account is itself unverified', r'unauthenticated|crypto connection|one complainant|doubting the complainant|wanted himself|edge of the crypto|only source is the victim'),
('Initiation or entry method not reported', r'not described|not reported|how the two got inside|how he was lured|no source gives a venue|is not in the file|appears in no article|cannot say whether'),
('Remote operator, phone drained after the event, or company infrastructure', r'remote-operator|coached them by telephone|drained with the stolen phone|discovered after the event|servers|company\'s'),
]
def theme(g):
for t,rx in THEMES:
if re.search(rx,g,re.I): return t
return 'Other'
gaps=collections.defaultdict(list)
for c in cases:
for g in c.get('gaps') or []:
gaps[theme(g)].append((c['id'],g))
# contradictions
FIELDMAP={'detention flag':'detention','kidnappings':'detention','held_duration':'detention','detention duration':'detention','held duration':'detention','duration':'detention','deaths':'life_taken','theft flag':'theft','crypto stolen':'theft','property taken':'theft','arrest':'arrests','charges':'arrests','sentence total':'arrests','attacker name':'attackers','attribution':'attackers','victim age':'victim','victim role':'victim','victim employer':'victim','age':'victim','role':'victim','children':'victim','time':'date','coin':'coin_type','violence_torture':'violence','injuries':'violence','drugs':'drugs_alcohol','summary (site)':'site summary','venue':'location','status':'status','certainty':'status','sequence of events':'description','attack method':'description','circumstances':'description','who raised the alarm':'description','reports':'description','source url':'notes','flags':'flags'}
def norm(f):
f=f.strip().lower()
base=re.split(r'\s*[/(]\s*',f)[0].strip()
return FIELDMAP.get(f,FIELDMAP.get(base,base))
ctr_field=collections.Counter(); ctr_case=collections.Counter(); ctr_total=0
for c in cases:
for ct in c.get('contradictions') or []:
ctr_total+=1; ctr_field[norm(ct.get('field','?'))]+=1; ctr_case[c['id']]+=1
# flags
fl=collections.Counter((p['flag'],str(p['record']),str(p['proposed'])) for p in fp)
det_up=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='1')
det_down=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='0' and str(p['record'])=='1')
life=[p for p in fp if p['flag']=='life_taken']
# sequences
lab=collections.Counter(); multi=[]; empty=[]; unsup=[]; dodged=[]
for k,v in seqs.items():
sq=v.get('sequence') or []
for l in sq: lab[l]+=1
if len(sq)>1: multi.append(int(k))
if not sq: empty.append(int(k))
if v.get('v1_label_supported') is False: unsup.append(int(k))
if 'Dodged Attack' in sq: dodged.append(int(k))
record_only=sorted(c['id'] for c in cases if c.get('record_only'))
SCOPE_RX=r'belongs in this database|place in a crypto database|belongs in a K&R database|belongs in a physical-attack database|borderline for a physical|not crypto-related|candidate for removal|no person was held or attacked|nothing was stolen or demanded|lawful|doxing'
scope=sorted({c['id'] for c in cases if re.search(SCOPE_RX,(c.get('notes_for_sofia') or '')+' '+(c.get('rationale') or '')+' '.join(c.get('gaps') or []),re.I)})
nobody=sorted(i for i in empty if i not in record_only)
MERGE_RX=r'same incident|duplicate of|split across|split by hand|splitting this record|is a merge of|same operation is split|may be separate records|rest on the same paragraph|separate records with no link'
merge=sorted({c['id'] for c in cases if re.search(MERGE_RX,(c.get('notes_for_sofia') or '')+' '.join(c.get('gaps') or []),re.I)})
out=[]
w=out.append
w(f"# Verification report — all batches ({today})\n")
w("Every summary, scenario sequence, flag proposal and contradiction below was written from the case's own dossier (sources/<id>_*.txt) by a writer agent and checked line by line by a read-only checker. The DB export is untouched: everything here is a proposal for Sofia. Per-batch detail sits in contradictions-NN.md; the merged list is contradictions-ALL.md.\n")
w("## Coverage\n")
w(f"| Item | Count |\n|---|---|")
w(f"| Cases in the DB export | {len(dbids)} |")
w(f"| Cases with a verified summary (summary-overrides.json) | {len(ov)} |")
w(f"| Cases with a scenario sequence (scenario-sequences.json) | {len(seqs)} |")
w(f"| Cases verified in batches 01–{max(c['_batch'] for c in cases):02d} | {len(cases)} |")
w(f"| Cases still without a verified summary | {len(dbids-{int(k) for k in ov})} {sorted(dbids-{int(k) for k in ov}) or ''} |")
w(f"| Record-only cases (no usable article on file) | {len(record_only)} {record_only} |")
w("")
w("## Contradictions between record and articles\n")
w(f"{ctr_total} contradictions across {len(ctr_case)} cases. None was resolved; each is quoted with the article line in contradictions-ALL.md.\n")
w("| Field | Contradictions |\n|---|---|")
for k,n in ctr_field.most_common(): w(f"| {k} | {n} |")
w("")
w("Cases with the most contradictions (each needs a full re-entry):\n")
for i,n in ctr_case.most_common(20): w(f"- case {i} ({name.get(i,'')}): {n}")
w("")
w("## Flag proposals\n")
w(f"{len(fp)} proposals on {len({p['id'] for p in fp})} cases (batches/flag-proposals.json). 'unconfirmed' means no source on file supports the recorded value; 'threat only' means a weapon or violence was threatened, not used.\n")
w("| Flag | Record | Proposed | Count |\n|---|---|---|---|")
for (f_,r,p),n in sorted(fl.items(),key=lambda x:(x[0][0],-x[1])): w(f"| {f_} | {r} | {p} | {n} |")
w("")
w(f"Detention (DB field kidnappings) 0/null → 1 under the owner's rule ({len(det_up)}): {det_up}\n")
w(f"Detention 1 → 0 ({len(det_down)}): {det_down}\n")
w("Fatality flag (life_taken) proposals:\n")
for p in life: w(f"- case {p['id']}: {p['record']} → {p['proposed']} — {p['evidence']}")
w("")
w("## Scenario sequences\n")
w(f"| Item | Count |\n|---|---|\n| Cases with more than one label in sequence | {len(multi)} |\n| Cases where the v1 label is not supported by the file | {len(unsup)} |\n| Foiled attacks (Dodged Attack) | {len(dodged)} |\n| Cases no label fits (empty sequence) | {len(empty)} |\n")
w("Label frequency across all sequences:\n")
w("| Label | Uses |\n|---|---|")
for l,n in lab.most_common(): w(f"| {l} | {n} |")
w("")
w("Cases no label fits, with the writer's reason:\n")
for i in sorted(empty):
r=(seqs[str(i)].get('rationale') or '').strip().replace('\n',' ')
w(f"- case {i} ({name.get(i,'')}): {r[:260]}")
w("")
w("## Label gaps (what the eleven scenarios cannot express)\n")
w("Gap notes grouped by theme. Counts are notes, not cases; a case can raise several.\n")
w("| Theme | Notes | Cases |\n|---|---|---|")
for t,_ in THEMES+[('Other','')]:
items=gaps.get(t,[])
if not items: continue
ids=sorted({i for i,_ in items})
w(f"| {t} | {len(items)} | {', '.join(map(str,ids))} |")
w("")
w("Proposed additions to the vocabulary, in order of how often the gap appears: (1) a `repeat_victimisation` / campaign link between records; (2) an `insider` initiation attribute (relative, partner, driver, guard, friend); (3) a label for attacks on unattended premises and machines (burglary, ATM ram-raid, mining rigs); (4) remote extortion with threat but no contact; (5) 'held at attackers' venue' as a Kidnapping sub-type; (6) a `ruse` attribute (delivery, uniform, disguise) separate from Authority Misuse and Malicious Invitation; (7) a preparation phase field (leak, surveillance, tracker).\n")
for t,_ in THEMES+[('Other','')]:
items=gaps.get(t,[])
if not items: continue
w(f"### {t}\n")
for i,g in sorted(items): w(f"- case {i}: {g}")
w("")
w("## Cases to decide on scope, merge or split\n")
w(f"No scenario label fits (mostly empty premises, unattended machines and burglaries with nobody present) — decide whether a physical-attack database keeps them: {nobody}\n")
w(f"Writer questioned scope for another reason (not crypto-related, lawful entry, doxing only, argument during a sale, non-violent device theft): {scope}\n")
w(f"Possible merges or splits (same incident across records, or several incidents on one record): {merge}\n")
w(f"Record-only (restructured from the DB record; nothing to verify against): {record_only}\n")
open(BASE+'VERIFICATION-REPORT.md','w',encoding='utf-8').write('\n'.join(out))
# merged contradictions
parts=[]
for f in sorted(glob.glob(BASE+'contradictions-[0-9]*.md')):
parts.append(open(f,encoding='utf-8').read())
open(BASE+'contradictions-ALL.md','w',encoding='utf-8').write(f"# All contradictions, flag proposals and notes for Sofia — merged from every batch ({today})\n\n"+'\n\n'.join(parts))
print('report:',len(out),'lines; cases',len(cases),'contradictions',ctr_total,'flag proposals',len(fp),'gap notes',sum(len(v) for v in gaps.values()))