Archived
docs(verify): consolidated verification report and README update
make_verification_report.py builds batches/VERIFICATION-REPORT.md (coverage, contradictions by field, flag proposals, label frequency, cases no label fits, label gaps by theme, scope/merge/split decisions) and batches/contradictions-ALL.md from the applied batch files. README progress and quick-start rewritten for the finished verification; the old enrichment method is kept for reference. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
9470e6bb0c
commit
c9d6a0f2fd
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Consolidate the verification batches into batches/VERIFICATION-REPORT.md and
|
||||
batches/contradictions-ALL.md. Re-run after every apply_batch.py."""
|
||||
import json, glob, re, collections, datetime
|
||||
BASE='batches/'
|
||||
cases=[]
|
||||
for f in sorted(glob.glob(BASE+'batch-[0-9]*.json')):
|
||||
n=int(re.search(r'batch-(\d+)',f).group(1))
|
||||
for c in json.load(open(f,encoding='utf-8')):
|
||||
c['_batch']=n; cases.append(c)
|
||||
byid={c['id']:c for c in cases}
|
||||
seqs=json.load(open('scenario-sequences.json',encoding='utf-8'))
|
||||
db=json.load(open('../attacks-export-Gart-website.json',encoding='utf-8'))
|
||||
recs=db if isinstance(db,list) else db.get('data',db)
|
||||
dbids={int(r['id']) for r in recs}
|
||||
name={int(r['id']):(r.get('victim') or r.get('name') or '') for r in recs}
|
||||
ov=json.load(open('summary-overrides.json',encoding='utf-8'))
|
||||
fp=json.load(open(BASE+'flag-proposals.json',encoding='utf-8'))
|
||||
today=datetime.date.today().isoformat()
|
||||
|
||||
THEMES=[
|
||||
('Repeat victimisation / several incidents on one record', r'repeat|same household|consecutive days|second event|previous occasions|campaign|similar incident|one record|single record|carried on the same'),
|
||||
('Insider, intimate or acquaintance perpetrator', r'insider|intimate|family member|own driver|own bodyguard|acquaintance|friend|relative'),
|
||||
('Empty premises, unattended machines, infrastructure', r'empty premises|unattended|burglar|ATM|machine|infrastructure|mining|premises were empty|nobody home|no one home'),
|
||||
('Extortion at a distance or threat at the door without entry', r'at a distance|without entry|at the door|no physical contact|Telegram|blackmail|threat only|extortion by'),
|
||||
('Lured to and held at the attackers\' venue (captivity without transport)', r"attackers' (venue|premises|apartment|rented)|captivity without transport|held at the attackers|attackers' own"),
|
||||
('Delivery, disguise or uniform ruse (mislabelled as Authority Misuse or Malicious Invitation)', r'delivery|disguise|uniform|DoorDash|courier|parcel|pretext'),
|
||||
('Preparation: leaks, surveillance, trackers, data breaches', r'leak|surveillance|tracker|reconnaissance|preparation|data breach|breach|OSINT|knew about'),
|
||||
('Drugging as the approach', r'drugg|drug as|sedat|spiked'),
|
||||
('Unexplained death or undetermined circumstances', r'undetermined|unexplained|death with|cause of death|disappear'),
|
||||
('Security measures deciding the outcome', r'security measures|guard|alarm|safe room|CCTV|panic'),
|
||||
('Scope: fraud, non-violent, or not crypto-linked', r'fraud|non-violent|borderline|out of scope|scope|not crypto'),
|
||||
('Multiple victims on one record', r'eleven victims|several victims|victims robbed in one|two intended victims|group of'),
|
||||
]
|
||||
def theme(g):
|
||||
for t,rx in THEMES:
|
||||
if re.search(rx,g,re.I): return t
|
||||
return 'Other'
|
||||
gaps=collections.defaultdict(list)
|
||||
for c in cases:
|
||||
for g in c.get('gaps') or []:
|
||||
gaps[theme(g)].append((c['id'],g))
|
||||
|
||||
# contradictions
|
||||
FIELDMAP={'detention flag':'detention','kidnappings':'detention','held_duration':'detention','detention duration':'detention','held duration':'detention','duration':'detention','deaths':'life_taken','theft flag':'theft','crypto stolen':'theft','property taken':'theft','arrest':'arrests','charges':'arrests','sentence total':'arrests','attacker name':'attackers','attribution':'attackers','victim age':'victim','victim role':'victim','victim employer':'victim','age':'victim','role':'victim','children':'victim','time':'date','coin':'coin_type','violence_torture':'violence','injuries':'violence','drugs':'drugs_alcohol','summary (site)':'site summary','venue':'location','status':'status','certainty':'status','sequence of events':'description','attack method':'description','circumstances':'description','who raised the alarm':'description','reports':'description','source url':'notes','flags':'flags'}
|
||||
def norm(f):
|
||||
f=f.strip().lower()
|
||||
base=re.split(r'\s*[/(]\s*',f)[0].strip()
|
||||
return FIELDMAP.get(f,FIELDMAP.get(base,base))
|
||||
ctr_field=collections.Counter(); ctr_case=collections.Counter(); ctr_total=0
|
||||
for c in cases:
|
||||
for ct in c.get('contradictions') or []:
|
||||
ctr_total+=1; ctr_field[norm(ct.get('field','?'))]+=1; ctr_case[c['id']]+=1
|
||||
|
||||
# flags
|
||||
fl=collections.Counter((p['flag'],str(p['record']),str(p['proposed'])) for p in fp)
|
||||
det_up=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='1')
|
||||
det_down=sorted(p['id'] for p in fp if p['flag']=='kidnappings' and str(p['proposed'])=='0' and str(p['record'])=='1')
|
||||
life=[p for p in fp if p['flag']=='life_taken']
|
||||
|
||||
# sequences
|
||||
lab=collections.Counter(); multi=[]; empty=[]; unsup=[]; dodged=[]
|
||||
for k,v in seqs.items():
|
||||
sq=v.get('sequence') or []
|
||||
for l in sq: lab[l]+=1
|
||||
if len(sq)>1: multi.append(int(k))
|
||||
if not sq: empty.append(int(k))
|
||||
if v.get('v1_label_supported') is False: unsup.append(int(k))
|
||||
if 'Dodged Attack' in sq: dodged.append(int(k))
|
||||
|
||||
record_only=sorted(c['id'] for c in cases if c.get('record_only'))
|
||||
SCOPE_RX=r'belongs in this database|place in a crypto database|belongs in a K&R database|belongs in a physical-attack database|borderline for a physical|not crypto-related|candidate for removal|no person was held or attacked|nothing was stolen or demanded|lawful|doxing'
|
||||
scope=sorted({c['id'] for c in cases if re.search(SCOPE_RX,(c.get('notes_for_sofia') or '')+' '+(c.get('rationale') or '')+' '.join(c.get('gaps') or []),re.I)})
|
||||
nobody=sorted(i for i in empty if i not in record_only)
|
||||
MERGE_RX=r'same incident|duplicate of|split across|split by hand|splitting this record|is a merge of|same operation is split|may be separate records|rest on the same paragraph|separate records with no link'
|
||||
merge=sorted({c['id'] for c in cases if re.search(MERGE_RX,(c.get('notes_for_sofia') or '')+' '.join(c.get('gaps') or []),re.I)})
|
||||
|
||||
out=[]
|
||||
w=out.append
|
||||
w(f"# Verification report — all batches ({today})\n")
|
||||
w("Every summary, scenario sequence, flag proposal and contradiction below was written from the case's own dossier (sources/<id>_*.txt) by a writer agent and checked line by line by a read-only checker. The DB export is untouched: everything here is a proposal for Sofia. Per-batch detail sits in contradictions-NN.md; the merged list is contradictions-ALL.md.\n")
|
||||
w("## Coverage\n")
|
||||
w(f"| Item | Count |\n|---|---|")
|
||||
w(f"| Cases in the DB export | {len(dbids)} |")
|
||||
w(f"| Cases with a verified summary (summary-overrides.json) | {len(ov)} |")
|
||||
w(f"| Cases with a scenario sequence (scenario-sequences.json) | {len(seqs)} |")
|
||||
w(f"| Cases verified in batches 01–{max(c['_batch'] for c in cases):02d} | {len(cases)} |")
|
||||
w(f"| Cases still without a verified summary | {len(dbids-{int(k) for k in ov})} {sorted(dbids-{int(k) for k in ov}) or ''} |")
|
||||
w(f"| Record-only cases (no usable article on file) | {len(record_only)} {record_only} |")
|
||||
w("")
|
||||
w("## Contradictions between record and articles\n")
|
||||
w(f"{ctr_total} contradictions across {len(ctr_case)} cases. None was resolved; each is quoted with the article line in contradictions-ALL.md.\n")
|
||||
w("| Field | Contradictions |\n|---|---|")
|
||||
for k,n in ctr_field.most_common(): w(f"| {k} | {n} |")
|
||||
w("")
|
||||
w("Cases with the most contradictions (each needs a full re-entry):\n")
|
||||
for i,n in ctr_case.most_common(20): w(f"- case {i} ({name.get(i,'')}): {n}")
|
||||
w("")
|
||||
w("## Flag proposals\n")
|
||||
w(f"{len(fp)} proposals on {len({p['id'] for p in fp})} cases (batches/flag-proposals.json). 'unconfirmed' means no source on file supports the recorded value; 'threat only' means a weapon or violence was threatened, not used.\n")
|
||||
w("| Flag | Record | Proposed | Count |\n|---|---|---|---|")
|
||||
for (f_,r,p),n in sorted(fl.items(),key=lambda x:(x[0][0],-x[1])): w(f"| {f_} | {r} | {p} | {n} |")
|
||||
w("")
|
||||
w(f"Detention (DB field kidnappings) 0/null → 1 under the owner's rule ({len(det_up)}): {det_up}\n")
|
||||
w(f"Detention 1 → 0 ({len(det_down)}): {det_down}\n")
|
||||
w("Fatality flag (life_taken) proposals:\n")
|
||||
for p in life: w(f"- case {p['id']}: {p['record']} → {p['proposed']} — {p['evidence']}")
|
||||
w("")
|
||||
w("## Scenario sequences\n")
|
||||
w(f"| Item | Count |\n|---|---|\n| Cases with more than one label in sequence | {len(multi)} |\n| Cases where the v1 label is not supported by the file | {len(unsup)} |\n| Foiled attacks (Dodged Attack) | {len(dodged)} |\n| Cases no label fits (empty sequence) | {len(empty)} |\n")
|
||||
w("Label frequency across all sequences:\n")
|
||||
w("| Label | Uses |\n|---|---|")
|
||||
for l,n in lab.most_common(): w(f"| {l} | {n} |")
|
||||
w("")
|
||||
w("Cases no label fits, with the writer's reason:\n")
|
||||
for i in sorted(empty):
|
||||
r=(seqs[str(i)].get('rationale') or '').strip().replace('\n',' ')
|
||||
w(f"- case {i} ({name.get(i,'')}): {r[:260]}")
|
||||
w("")
|
||||
w("## Label gaps (what the eleven scenarios cannot express)\n")
|
||||
w("Gap notes grouped by theme. Counts are notes, not cases; a case can raise several.\n")
|
||||
w("| Theme | Notes | Cases |\n|---|---|---|")
|
||||
for t,_ in THEMES+[('Other','')]:
|
||||
items=gaps.get(t,[])
|
||||
if not items: continue
|
||||
ids=sorted({i for i,_ in items})
|
||||
w(f"| {t} | {len(items)} | {', '.join(map(str,ids))} |")
|
||||
w("")
|
||||
w("Proposed additions to the vocabulary, in order of how often the gap appears: (1) a `repeat_victimisation` / campaign link between records; (2) an `insider` initiation attribute (relative, partner, driver, guard, friend); (3) a label for attacks on unattended premises and machines (burglary, ATM ram-raid, mining rigs); (4) remote extortion with threat but no contact; (5) 'held at attackers' venue' as a Kidnapping sub-type; (6) a `ruse` attribute (delivery, uniform, disguise) separate from Authority Misuse and Malicious Invitation; (7) a preparation phase field (leak, surveillance, tracker).\n")
|
||||
for t,_ in THEMES+[('Other','')]:
|
||||
items=gaps.get(t,[])
|
||||
if not items: continue
|
||||
w(f"### {t}\n")
|
||||
for i,g in sorted(items): w(f"- case {i}: {g}")
|
||||
w("")
|
||||
w("## Cases to decide on scope, merge or split\n")
|
||||
w(f"No scenario label fits (mostly empty premises, unattended machines and burglaries with nobody present) — decide whether a physical-attack database keeps them: {nobody}\n")
|
||||
w(f"Writer questioned scope for another reason (not crypto-related, lawful entry, doxing only, argument during a sale, non-violent device theft): {scope}\n")
|
||||
w(f"Possible merges or splits (same incident across records, or several incidents on one record): {merge}\n")
|
||||
w(f"Record-only (restructured from the DB record; nothing to verify against): {record_only}\n")
|
||||
open(BASE+'VERIFICATION-REPORT.md','w',encoding='utf-8').write('\n'.join(out))
|
||||
|
||||
# merged contradictions
|
||||
parts=[]
|
||||
for f in sorted(glob.glob(BASE+'contradictions-[0-9]*.md')):
|
||||
parts.append(open(f,encoding='utf-8').read())
|
||||
open(BASE+'contradictions-ALL.md','w',encoding='utf-8').write(f"# All contradictions, flag proposals and notes for Sofia — merged from every batch ({today})\n\n"+'\n\n'.join(parts))
|
||||
print('report:',len(out),'lines; cases',len(cases),'contradictions',ctr_total,'flag proposals',len(fp),'gap notes',sum(len(v) for v in gaps.values()))
|
||||
Reference in New Issue
Block a user