docs(verify): consolidated verification report and README update

make_verification_report.py builds batches/VERIFICATION-REPORT.md (coverage,
contradictions by field, flag proposals, label frequency, cases no label fits,
label gaps by theme, scope/merge/split decisions) and batches/contradictions-ALL.md
from the applied batch files. README progress and quick-start rewritten for the
finished verification; the old enrichment method is kept for reference.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
This commit is contained in:
StellarCrow
2026-09-30 12:10:51 +02:00
co-authored by Claude Fable 5.1
parent 9470e6bb0c
commit c9d6a0f2fd
4 changed files with 3954 additions and 28 deletions
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""Consolidate the verification batches into batches/VERIFICATION-REPORT.md and
batches/contradictions-ALL.md. Re-run after every apply_batch.py."""
import json, glob, re, collections, datetime
BASE='batches/'
cases=[]
for f in sorted(glob.glob(BASE+'batch-[0-9]*.json')):
n=int(re.search(r'batch-(\d+)',f).group(1))
for c in json.load(open(f,encoding='utf-8')):
c['_batch']=n; cases.append(c)
byid={c['id']:c for c in cases}
seqs=json.load(open('scenario-sequences.json',encoding='utf-8'))
db=json.load(open('../attacks-export-Gart-website.json',encoding='utf-8'))
recs=db if isinstance(db,list) else db.get('data',db)
dbids={int(r['id']) for r in recs}
name={int(r['id']):(r.get('victim') or r.get('name') or '') for r in recs}
ov=json.load(open('summary-overrides.json',encoding='utf-8'))
fp=json.load(open(BASE+'flag-proposals.json',encoding='utf-8'))
today=datetime.date.today().isoformat()
THEMES=[
('Repeat victimisation / several incidents on one record', r'repeat|same household|consecutive days|second event|previous occasions|campaign|similar incident|one record|single record|carried on the same'),
('Insider, intimate or acquaintance perpetrator', r'insider|intimate|family member|own driver|own bodyguard|acquaintance|friend|relative'),
('Empty premises, unattended machines, infrastructure', r'empty premises|unattended|burglar|ATM|machine|infrastructure|mining|premises were empty|nobody home|no one home'),
('Extortion at a distance or threat at the door without entry', r'at a distance|without entry|at the door|no physical contact|Telegram|blackmail|threat only|extortion by'),
('Lured to and held at the attackers\' venue (captivity without transport)', r"attackers' (venue|premises|apartment|rented)|captivity without transport|held at the attackers|attackers' own"),
('Delivery, disguise or uniform ruse (mislabelled as Authority Misuse or Malicious Invitation)', r'delivery|disguise|uniform|DoorDash|courier|parcel|pretext'),
('Preparation: leaks, surveillance, trackers, data breaches', r'leak|surveillance|tracker|reconnaissance|preparation|data breach|breach|OSINT|knew about'),
('Drugging as the approach', r'drugg|drug as|sedat|spiked'),
('Unexplained death or undetermined circumstances', r'undetermined|unexplained|death with|cause of death|disappear'),
('Security measures deciding the outcome', r'security measures|guard|alarm|safe room|CCTV|panic'),
('Scope: fraud, non-violent, or not crypto-linked', r'fraud|non-violent|borderline|out of scope|scope|not crypto'),
('Multiple victims on one record', r'eleven victims|several victims|victims robbed in one|two intended victims|group of'),
]
def theme(g):
for t,rx in THEMES:
if re.search(rx,g,re.I): return t
return 'Other'
gaps=collections.defaultdict(list)
for c in cases:
for g in c.get('gaps') or []:
gaps[theme(g)].append((c['id'],g))
# contradictions
FIELDMAP={'detention flag':'detention','kidnappings':'detention','held_duration':'detention','detention duration':'detention','held duration':'detention','duration':'detention','deaths':'life_taken','theft flag':'theft','crypto stolen':'theft','property taken':'theft','arrest':'arrests','charges':'arrests','sentence total':'arrests','attacker name':'attackers','attribution':'attackers','victim age':'victim','victim role':'victim','victim employer':'victim','age':'victim','role':'victim','children':'victim','time':'date','coin':'coin_type','violence_torture':'violence','injuries':'violence','drugs':'drugs_alcohol','summary (site)':'site summary','venue':'location','status':'status','certainty':'status','sequence of events':'description','attack method':'description','circumstances':'description','who raised the alarm':'description','reports':'description','source url':'notes','flags':'flags'}
def norm(f):
f=f.strip().lower()
base=re.split(r'\s*[/(]\s*',f)[0].strip()
return FIELDMAP.get(f,FIELDMAP.get(base,base))
ctr_field=collections.Counter(); ctr_case=collections.Counter(); ctr_total=0
for c in cases:
for ct in c.get('contradictions') or []:
ctr_total+=1; ctr_field[norm(ct.get('field','?'))]+=1; ctr_case[c['id']]+=1
# flags
fl=collections.Counter((p['flag'],str(p['record']),str(p['proposed'])) for p in fp)
det_up=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='1')
det_down=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='0' and str(p['record'])=='1')
life=[p for p in fp if p['flag']=='life_taken']
# sequences
lab=collections.Counter(); multi=[]; empty=[]; unsup=[]; dodged=[]
for k,v in seqs.items():
sq=v.get('sequence') or []
for l in sq: lab[l]+=1
if len(sq)>1: multi.append(int(k))
if not sq: empty.append(int(k))
if v.get('v1_label_supported') is False: unsup.append(int(k))
if 'Dodged Attack' in sq: dodged.append(int(k))
record_only=sorted(c['id'] for c in cases if c.get('record_only'))
SCOPE_RX=r'belongs in this database|place in a crypto database|belongs in a K&R database|belongs in a physical-attack database|borderline for a physical|not crypto-related|candidate for removal|no person was held or attacked|nothing was stolen or demanded|lawful|doxing'
scope=sorted({c['id'] for c in cases if re.search(SCOPE_RX,(c.get('notes_for_sofia') or '')+' '+(c.get('rationale') or '')+' '.join(c.get('gaps') or []),re.I)})
nobody=sorted(i for i in empty if i not in record_only)
MERGE_RX=r'same incident|duplicate of|split across|split by hand|splitting this record|is a merge of|same operation is split|may be separate records|rest on the same paragraph|separate records with no link'
merge=sorted({c['id'] for c in cases if re.search(MERGE_RX,(c.get('notes_for_sofia') or '')+' '.join(c.get('gaps') or []),re.I)})
out=[]
w=out.append
w(f"# Verification report — all batches ({today})\n")
w("Every summary, scenario sequence, flag proposal and contradiction below was written from the case's own dossier (sources/<id>_*.txt) by a writer agent and checked line by line by a read-only checker. The DB export is untouched: everything here is a proposal for Sofia. Per-batch detail sits in contradictions-NN.md; the merged list is contradictions-ALL.md.\n")
w("## Coverage\n")
w(f"| Item | Count |\n|---|---|")
w(f"| Cases in the DB export | {len(dbids)} |")
w(f"| Cases with a verified summary (summary-overrides.json) | {len(ov)} |")
w(f"| Cases with a scenario sequence (scenario-sequences.json) | {len(seqs)} |")
w(f"| Cases verified in batches 01–{max(c['_batch'] for c in cases):02d} | {len(cases)} |")
w(f"| Cases still without a verified summary | {len(dbids-{int(k) for k in ov})} {sorted(dbids-{int(k) for k in ov}) or ''} |")
w(f"| Record-only cases (no usable article on file) | {len(record_only)} {record_only} |")
w("")
w("## Contradictions between record and articles\n")
w(f"{ctr_total} contradictions across {len(ctr_case)} cases. None was resolved; each is quoted with the article line in contradictions-ALL.md.\n")
w("| Field | Contradictions |\n|---|---|")
for k,n in ctr_field.most_common(): w(f"| {k} | {n} |")
w("")
w("Cases with the most contradictions (each needs a full re-entry):\n")
for i,n in ctr_case.most_common(20): w(f"- case {i} ({name.get(i,'')}): {n}")
w("")
w("## Flag proposals\n")
w(f"{len(fp)} proposals on {len({p['id'] for p in fp})} cases (batches/flag-proposals.json). 'unconfirmed' means no source on file supports the recorded value; 'threat only' means a weapon or violence was threatened, not used.\n")
w("| Flag | Record | Proposed | Count |\n|---|---|---|---|")
for (f_,r,p),n in sorted(fl.items(),key=lambda x:(x[0][0],-x[1])): w(f"| {f_} | {r} | {p} | {n} |")
w("")
w(f"Detention (DB field kidnappings) 0/null → 1 under the owner's rule ({len(det_up)}): {det_up}\n")
w(f"Detention 1 → 0 ({len(det_down)}): {det_down}\n")
w("Fatality flag (life_taken) proposals:\n")
for p in life: w(f"- case {p['id']}: {p['record']} → {p['proposed']} — {p['evidence']}")
w("")
w("## Scenario sequences\n")
w(f"| Item | Count |\n|---|---|\n| Cases with more than one label in sequence | {len(multi)} |\n| Cases where the v1 label is not supported by the file | {len(unsup)} |\n| Foiled attacks (Dodged Attack) | {len(dodged)} |\n| Cases no label fits (empty sequence) | {len(empty)} |\n")
w("Label frequency across all sequences:\n")
w("| Label | Uses |\n|---|---|")
for l,n in lab.most_common(): w(f"| {l} | {n} |")
w("")
w("Cases no label fits, with the writer's reason:\n")
for i in sorted(empty):
r=(seqs[str(i)].get('rationale') or '').strip().replace('\n',' ')
w(f"- case {i} ({name.get(i,'')}): {r[:260]}")
w("")
w("## Label gaps (what the eleven scenarios cannot express)\n")
w("Gap notes grouped by theme. Counts are notes, not cases; a case can raise several.\n")
w("| Theme | Notes | Cases |\n|---|---|---|")
for t,_ in THEMES+[('Other','')]:
items=gaps.get(t,[])
if not items: continue
ids=sorted({i for i,_ in items})
w(f"| {t} | {len(items)} | {', '.join(map(str,ids))} |")
w("")
w("Proposed additions to the vocabulary, in order of how often the gap appears: (1) a `repeat_victimisation` / campaign link between records; (2) an `insider` initiation attribute (relative, partner, driver, guard, friend); (3) a label for attacks on unattended premises and machines (burglary, ATM ram-raid, mining rigs); (4) remote extortion with threat but no contact; (5) 'held at attackers' venue' as a Kidnapping sub-type; (6) a `ruse` attribute (delivery, uniform, disguise) separate from Authority Misuse and Malicious Invitation; (7) a preparation phase field (leak, surveillance, tracker).\n")
for t,_ in THEMES+[('Other','')]:
items=gaps.get(t,[])
if not items: continue
w(f"### {t}\n")
for i,g in sorted(items): w(f"- case {i}: {g}")
w("")
w("## Cases to decide on scope, merge or split\n")
w(f"No scenario label fits (mostly empty premises, unattended machines and burglaries with nobody present) — decide whether a physical-attack database keeps them: {nobody}\n")
w(f"Writer questioned scope for another reason (not crypto-related, lawful entry, doxing only, argument during a sale, non-violent device theft): {scope}\n")
w(f"Possible merges or splits (same incident across records, or several incidents on one record): {merge}\n")
w(f"Record-only (restructured from the DB record; nothing to verify against): {record_only}\n")
open(BASE+'VERIFICATION-REPORT.md','w',encoding='utf-8').write('\n'.join(out))
# merged contradictions
parts=[]
for f in sorted(glob.glob(BASE+'contradictions-[0-9]*.md')):
parts.append(open(f,encoding='utf-8').read())
open(BASE+'contradictions-ALL.md','w',encoding='utf-8').write(f"# All contradictions, flag proposals and notes for Sofia — merged from every batch ({today})\n\n"+'\n\n'.join(parts))
print('report:',len(out),'lines; cases',len(cases),'contradictions',ctr_total,'flag proposals',len(fp),'gap notes',sum(len(v) for v in gaps.values()))