Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
158 lines
12 KiB
Python
158 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Consolidate the verification batches into batches/VERIFICATION-REPORT.md and
|
||
batches/contradictions-ALL.md. Re-run after every apply_batch.py."""
|
||
import json, glob, re, collections, datetime
|
||
BASE='batches/'
|
||
cases=[]
|
||
for f in sorted(glob.glob(BASE+'batch-[0-9]*.json')):
|
||
m=re.search(r'batch-(\d+)\.json$',f)
|
||
if not m: continue # part files (batch-35a.json, batch-35b.json) repeat their parent batch
|
||
n=int(m.group(1))
|
||
for c in json.load(open(f,encoding='utf-8')):
|
||
c['_batch']=n; cases.append(c)
|
||
byid={c['id']:c for c in cases}
|
||
cases=list(byid.values()) # a case rewritten in a later batch counts once, in its latest version
|
||
seqs=json.load(open('scenario-sequences.json',encoding='utf-8'))
|
||
db=json.load(open('../attacks-export-Gart-website.json',encoding='utf-8'))
|
||
recs=db if isinstance(db,list) else db.get('data',db)
|
||
dbids={int(r['id']) for r in recs}
|
||
name={int(r['id']):(r.get('victim') or r.get('name') or '') for r in recs}
|
||
ov=json.load(open('summary-overrides.json',encoding='utf-8'))
|
||
fp=json.load(open(BASE+'flag-proposals.json',encoding='utf-8'))
|
||
today=datetime.date.today().isoformat()
|
||
|
||
THEMES=[
|
||
('Repeat victimisation / several incidents on one record', r'repeat|same household|consecutive days|second event|previous occasions|campaign|similar incident|one record|single record|carried on the same'),
|
||
('Insider, intimate or acquaintance perpetrator', r'insider|intimate|family member|own driver|own bodyguard|acquaintance|friend|relative'),
|
||
('Empty premises, unattended machines, infrastructure', r'empty premises|unattended|burglar|ATM|machine|infrastructure|mining|premises were empty|nobody home|no one home'),
|
||
('Extortion at a distance or threat at the door without entry', r'at a distance|without entry|at the door|no physical contact|Telegram|blackmail|threat only|extortion by'),
|
||
('Lured to and held at the attackers\' venue (captivity without transport)', r"attackers' (venue|premises|apartment|rented)|captivity without transport|held at the attackers|attackers' own"),
|
||
('Delivery, disguise or uniform ruse (mislabelled as Authority Misuse or Malicious Invitation)', r'delivery|disguise|uniform|DoorDash|courier|parcel|pretext'),
|
||
('Preparation: leaks, surveillance, trackers, data breaches', r'leak|surveillance|tracker|reconnaissance|preparation|data breach|breach|OSINT|knew about'),
|
||
('Drugging as the approach', r'drugg|drug as|sedat|spiked'),
|
||
('Unexplained death or undetermined circumstances', r'undetermined|unexplained|death with|cause of death|disappear'),
|
||
('Security measures deciding the outcome', r'security measures|guard|alarm|safe room|CCTV|panic'),
|
||
('Scope: fraud, non-violent, or not crypto-linked', r'fraud|non-violent|borderline|out of scope|scope|not crypto'),
|
||
('Multiple victims, proxy victims or one victim field', r'eleven victims|several victims|victims robbed in one|two intended victims|group of|one victim field|single victim field|proxy victim|held in his place|household used as lever|two victims held|three people were attacked'),
|
||
('Amount and coin fields cannot split crypto from cash, goods or fiat', r'money_wanted|coin_type|amount field|cash side|fiat|not crypto|not cryptocurrency|cannot show that only|silver|jewellery|watch'),
|
||
('Control moves between places; one venue field', r'one venue|single venue|two different labels|control shifted|two places|moves the victim|second address|neither a premises nor|stationary'),
|
||
('Outcome after the transfer (killing, escape, rescue) is not an initiation label', r'killed after|after the wallets|escape by|own escape|rescued|interrupted|broken by police|self-defence|the harm that did occur'),
|
||
('Crypto link or complainant account is itself unverified', r'unauthenticated|crypto connection|one complainant|doubting the complainant|wanted himself|edge of the crypto|only source is the victim'),
|
||
('Initiation or entry method not reported', r'not described|not reported|how the two got inside|how he was lured|no source gives a venue|is not in the file|appears in no article|cannot say whether'),
|
||
('Remote operator, phone drained after the event, or company infrastructure', r'remote-operator|coached them by telephone|drained with the stolen phone|discovered after the event|servers|company\'s'),
|
||
]
|
||
def theme(g):
|
||
for t,rx in THEMES:
|
||
if re.search(rx,g,re.I): return t
|
||
return 'Other'
|
||
gaps=collections.defaultdict(list)
|
||
for c in cases:
|
||
for g in c.get('gaps') or []:
|
||
gaps[theme(g)].append((c['id'],g))
|
||
|
||
# contradictions
|
||
FIELDMAP={'detention flag':'detention','kidnappings':'detention','held_duration':'detention','detention duration':'detention','held duration':'detention','duration':'detention','deaths':'life_taken','theft flag':'theft','crypto stolen':'theft','property taken':'theft','arrest':'arrests','charges':'arrests','sentence total':'arrests','attacker name':'attackers','attribution':'attackers','victim age':'victim','victim role':'victim','victim employer':'victim','age':'victim','role':'victim','children':'victim','time':'date','coin':'coin_type','violence_torture':'violence','injuries':'violence','drugs':'drugs_alcohol','summary (site)':'site summary','venue':'location','status':'status','certainty':'status','sequence of events':'description','attack method':'description','circumstances':'description','who raised the alarm':'description','reports':'description','source url':'notes','flags':'flags'}
|
||
def norm(f):
|
||
f=f.strip().lower()
|
||
base=re.split(r'\s*[/(]\s*',f)[0].strip()
|
||
return FIELDMAP.get(f,FIELDMAP.get(base,base))
|
||
ctr_field=collections.Counter(); ctr_case=collections.Counter(); ctr_total=0
|
||
for c in cases:
|
||
for ct in c.get('contradictions') or []:
|
||
ctr_total+=1; ctr_field[norm(ct.get('field','?'))]+=1; ctr_case[c['id']]+=1
|
||
|
||
# flags
|
||
fl=collections.Counter((p['flag'],str(p['record']),str(p['proposed'])) for p in fp)
|
||
det_up=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='1')
|
||
det_down=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='0' and str(p['record'])=='1')
|
||
life=[p for p in fp if p['flag']=='life_taken']
|
||
|
||
# sequences
|
||
lab=collections.Counter(); multi=[]; empty=[]; unsup=[]; dodged=[]
|
||
for k,v in seqs.items():
|
||
sq=v.get('sequence') or []
|
||
for l in sq: lab[l]+=1
|
||
if len(sq)>1: multi.append(int(k))
|
||
if not sq: empty.append(int(k))
|
||
if v.get('v1_label_supported') is False: unsup.append(int(k))
|
||
if 'Dodged Attack' in sq: dodged.append(int(k))
|
||
|
||
record_only=sorted(c['id'] for c in cases if c.get('record_only'))
|
||
SCOPE_RX=r'belongs in this database|place in a crypto database|belongs in a K&R database|belongs in a physical-attack database|borderline for a physical|not crypto-related|candidate for removal|no person was held or attacked|nothing was stolen or demanded|lawful|doxing'
|
||
scope=sorted({c['id'] for c in cases if re.search(SCOPE_RX,(c.get('notes_for_sofia') or '')+' '+(c.get('rationale') or '')+' '.join(c.get('gaps') or []),re.I)})
|
||
nobody=sorted(i for i in empty if i not in record_only)
|
||
MERGE_RX=r'same incident|duplicate of|split across|split by hand|splitting this record|is a merge of|same operation is split|may be separate records|rest on the same paragraph|separate records with no link'
|
||
merge=sorted({c['id'] for c in cases if re.search(MERGE_RX,(c.get('notes_for_sofia') or '')+' '.join(c.get('gaps') or []),re.I)})
|
||
|
||
out=[]
|
||
w=out.append
|
||
w(f"# Verification report — all batches ({today})\n")
|
||
w("Every summary, scenario sequence, flag proposal and contradiction below was written from the case's own dossier (sources/<id>_*.txt) by a writer agent and checked line by line by a read-only checker. The DB export is untouched: everything here is a proposal for Sofia. Per-batch detail sits in contradictions-NN.md; the merged list is contradictions-ALL.md.\n")
|
||
w("## Coverage\n")
|
||
w(f"| Item | Count |\n|---|---|")
|
||
w(f"| Cases in the DB export | {len(dbids)} |")
|
||
w(f"| Cases with a verified summary (summary-overrides.json) | {len(ov)} |")
|
||
w(f"| Cases with a scenario sequence (scenario-sequences.json) | {len(seqs)} |")
|
||
w(f"| Cases verified in batches 01–{max(c['_batch'] for c in cases):02d} | {len(cases)} |")
|
||
w(f"| Cases still without a verified summary | {len(dbids-{int(k) for k in ov})} {sorted(dbids-{int(k) for k in ov}) or ''} |")
|
||
w(f"| Record-only cases (no usable article on file) | {len(record_only)} {record_only} |")
|
||
w("")
|
||
w("## Contradictions between record and articles\n")
|
||
w(f"{ctr_total} contradictions across {len(ctr_case)} cases. None was resolved; each is quoted with the article line in contradictions-ALL.md.\n")
|
||
w("| Field | Contradictions |\n|---|---|")
|
||
for k,n in ctr_field.most_common(): w(f"| {k} | {n} |")
|
||
w("")
|
||
w("Cases with the most contradictions (each needs a full re-entry):\n")
|
||
for i,n in ctr_case.most_common(20): w(f"- case {i} ({name.get(i,'')}): {n}")
|
||
w("")
|
||
w("## Flag proposals\n")
|
||
w(f"{len(fp)} proposals on {len({p['id'] for p in fp})} cases (batches/flag-proposals.json). 'unconfirmed' means no source on file supports the recorded value; 'threat only' means a weapon or violence was threatened, not used.\n")
|
||
w("| Flag | Record | Proposed | Count |\n|---|---|---|---|")
|
||
for (f_,r,p),n in sorted(fl.items(),key=lambda x:(x[0][0],-x[1])): w(f"| {f_} | {r} | {p} | {n} |")
|
||
w("")
|
||
w(f"Detention (DB field kidnappings) 0/null → 1 under the owner's rule ({len(det_up)}): {det_up}\n")
|
||
w(f"Detention 1 → 0 ({len(det_down)}): {det_down}\n")
|
||
w("Fatality flag (life_taken) proposals:\n")
|
||
for p in life: w(f"- case {p['id']}: {p['record']} → {p['proposed']} — {p['evidence']}")
|
||
w("")
|
||
w("## Scenario sequences\n")
|
||
w(f"| Item | Count |\n|---|---|\n| Cases with more than one label in sequence | {len(multi)} |\n| Cases where the v1 label is not supported by the file | {len(unsup)} |\n| Foiled attacks (Dodged Attack) | {len(dodged)} |\n| Cases no label fits (empty sequence) | {len(empty)} |\n")
|
||
w("Label frequency across all sequences:\n")
|
||
w("| Label | Uses |\n|---|---|")
|
||
for l,n in lab.most_common(): w(f"| {l} | {n} |")
|
||
w("")
|
||
w("Cases no label fits, with the writer's reason:\n")
|
||
for i in sorted(empty):
|
||
r=(seqs[str(i)].get('rationale') or '').strip().replace('\n',' ')
|
||
w(f"- case {i} ({name.get(i,'')}): {r[:260]}")
|
||
w("")
|
||
w("## Label gaps (what the eleven scenarios cannot express)\n")
|
||
w("Gap notes grouped by theme. Counts are notes, not cases; a case can raise several.\n")
|
||
w("| Theme | Notes | Cases |\n|---|---|---|")
|
||
for t,_ in THEMES+[('Other','')]:
|
||
items=gaps.get(t,[])
|
||
if not items: continue
|
||
ids=sorted({i for i,_ in items})
|
||
w(f"| {t} | {len(items)} | {', '.join(map(str,ids))} |")
|
||
w("")
|
||
w("Proposed additions to the vocabulary, in order of how often the gap appears: (1) a `repeat_victimisation` / campaign link between records; (2) an `insider` initiation attribute (relative, partner, driver, guard, friend); (3) a label for attacks on unattended premises and machines (burglary, ATM ram-raid, mining rigs); (4) remote extortion with threat but no contact; (5) 'held at attackers' venue' as a Kidnapping sub-type; (6) a `ruse` attribute (delivery, uniform, disguise) separate from Authority Misuse and Malicious Invitation; (7) a preparation phase field (leak, surveillance, tracker).\n")
|
||
for t,_ in THEMES+[('Other','')]:
|
||
items=gaps.get(t,[])
|
||
if not items: continue
|
||
w(f"### {t}\n")
|
||
for i,g in sorted(items): w(f"- case {i}: {g}")
|
||
w("")
|
||
w("## Cases to decide on scope, merge or split\n")
|
||
w(f"No scenario label fits (mostly empty premises, unattended machines and burglaries with nobody present) — decide whether a physical-attack database keeps them: {nobody}\n")
|
||
w(f"Writer questioned scope for another reason (not crypto-related, lawful entry, doxing only, argument during a sale, non-violent device theft): {scope}\n")
|
||
w(f"Possible merges or splits (same incident across records, or several incidents on one record): {merge}\n")
|
||
w(f"Record-only (restructured from the DB record; nothing to verify against): {record_only}\n")
|
||
open(BASE+'VERIFICATION-REPORT.md','w',encoding='utf-8').write('\n'.join(out))
|
||
|
||
# merged contradictions
|
||
parts=[]
|
||
for f in sorted(glob.glob(BASE+'contradictions-[0-9]*.md')):
|
||
parts.append(open(f,encoding='utf-8').read())
|
||
open(BASE+'contradictions-ALL.md','w',encoding='utf-8').write(f"# All contradictions, flag proposals and notes for Sofia — merged from every batch ({today})\n\n"+'\n\n'.join(parts))
|
||
print('report:',len(out),'lines; cases',len(cases),'contradictions',ctr_total,'flag proposals',len(fp),'gap notes',sum(len(v) for v in gaps.values()))
|