This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/export_xlsx.py
T
StellarCrowandClaude Fable 5.1 2dafc871e0 feat(scenarios): scenario sequences assigned by reading for the 70 verified cases
scenario-sequences.json: for each of the 70 fact-checked summaries, the ordered
set of the eleven labels the summary supports, with a rationale, the pattern
that was attempted where the attack was foiled, and the coverage gaps met.
35 cases carry two scenarios, 6 three, 1 none (an empty-premises burglary).
Three site labels are not supported by their own summary (222, 231, 349).
Export: assigned scenarios show green in the S: columns with the sequence and
notes on the row; engine proposals stay blue for the unassigned cases.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
2026-09-18 23:58:45 +02:00

307 lines
16 KiB
Python

"""
Export the database as an .xlsx with every current field plus the fields of the proposed
rework, colour-coded so the gaps are visible at a glance.
Sheets
attacks one row per case: current columns, source coverage, summary status, the
six flags, and the new fact columns (venue_of_control, held_duration,
outcome, arrests, amount_demanded_usd, amount_taken_usd)
vocabulary the controlled value lists for every typed column
legend what the colours mean
Colours
red required value missing (null flag, no source text, empty field)
orange new column, not yet filled
yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor)
blue filled by a rule from existing data; a proposal, not a verified value
none present and verified
"""
import json
import re
import sys
from datetime import date
from pathlib import Path
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
BASE = Path(__file__).resolve().parent
JSONF = BASE.parent / "attacks-export-Gart-website.json"
STORE = BASE / "sourced-articles.json"
OVERRIDES = BASE / "summary-overrides.json"
MANIFEST = BASE / "case-files-manifest.json"
DETENTION = BASE / "detention-flag-corrections.json"
SEQUENCES = BASE / "scenario-sequences.json"
SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"
RED = PatternFill("solid", fgColor="F4B6B6")
ORANGE = PatternFill("solid", fgColor="F9D5A7")
YELLOW = PatternFill("solid", fgColor="FFF2A8")
BLUE = PatternFill("solid", fgColor="BDD7EE")
GREEN = PatternFill("solid", fgColor="C6E0B4")
HEAD = PatternFill("solid", fgColor="D9D9D9")
FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"]
CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol",
"weapons": "weapons", "theft": "theft", "life_taken": "fatality"}
VOCAB = {
"venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space",
"vehicle in motion", "public space", "none"],
"held_duration": ["none", "minutes", "hours", "days"],
"outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"],
"arrests": ["none", "some", "all", "convicted"],
"scenario (11 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space",
"Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse",
"Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting"],
"phase_role": ["initiation", "execution", "captivity", "outcome"],
"violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"],
"detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away",
"locked in", "fake arrest"],
"violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock",
"waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet",
"harm to pet", "pet killed"],
"weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive",
"armed police response, induced"],
"drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"],
"theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"],
"fatality subtypes": ["shot", "beaten", "other or unknown"],
"target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"],
}
def money_usd(text):
"""Best-effort USD number from money_wanted; None when not clearly USD."""
if not text:
return None
t = text.replace(",", "")
m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I)
if not m:
return None
v = float(m.group(1))
unit = (m.group(2) or "").lower()
if unit in ("million", "m"):
v *= 1e6
elif unit in ("k", "thousand"):
v *= 1e3
return v
def main():
recs = json.loads(JSONF.read_text(encoding="utf-8"))
store = json.loads(STORE.read_text(encoding="utf-8"))
overrides = json.loads(OVERRIDES.read_text(encoding="utf-8"))
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")}
det = {}
if DETENTION.exists():
for c in json.loads(DETENTION.read_text(encoding="utf-8")):
det[int(c["target"].split()[-1])] = c
seqs = json.loads(SEQUENCES.read_text(encoding="utf-8")) if SEQUENCES.exists() else {}
sys.path.insert(0, str(SITE))
try:
from scenario_signature import build_signature
except Exception:
build_signature = None
wb = Workbook()
# ---------------- attacks ----------------
ws = wb.active
ws.title = "attacks"
base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario",
"description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"]
cov_cols = ["dossier_file", "sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars",
"detention_decision"]
new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"]
SCEN = VOCAB["scenario (11 labels)"]
scen_cols = ["S: " + n for n in SCEN] + ["scenarios_in_order", "scenario_notes"]
cols = base_cols + scen_cols + FLAGS + cov_cols + new_cols + ["summary_db", "summary_revised"]
LABEL = {"kidnappings": "detention"} # the DB column keeps its old name; the export shows the glossary name
ws.append([LABEL.get(c, c) for c in cols])
for c in ws[1]:
c.font = Font(bold=True)
c.fill = HEAD
for r in recs:
cid = r["id"]
blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")]
n_src = len(blocks) + (1 if cid in sheet else 0)
chars = sum(b.get("chars") or 0 for b in blocks)
untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED")
if str(cid) in overrides:
sstat = "verified"
elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4:
sstat = "bulk, short"
else:
sstat = "bulk, unverified"
tags = build_signature(r, r.get("scenario")) if build_signature else [
{"scenario": r.get("scenario"), "status": "confirmed"}]
tagmap = {t["scenario"]: t["status"] for t in tags}
scen_vals = [(1 if tagmap.get(n) == "confirmed" else ("p" if tagmap.get(n) == "proposed" else None)) for n in SCEN]
seq_text = " > ".join([t["scenario"] for t in tags if t["status"] == "confirmed"]
+ [t["scenario"] + " (p)" for t in tags if t["status"] == "proposed"])
dossier = next((f.name for f in (BASE / "sources").glob(f"{cid:03d}_*.txt")), None)
d = det.get(cid)
ddec = (f"{d.get('current')} -> {d.get('proposed')} [{d.get('confidence')}] {d.get('status', 'open')}" if d else None)
row = [r.get(k) for k in base_cols] + scen_vals + [seq_text] + [r.get(k) for k in FLAGS] + \
[dossier, n_src, chars, untr, sstat, len(r.get("summary") or ""), ddec]
# rule-derived proposals for the new columns
venue = held = outcome = None
if r.get("scenario") == "Home Invasion":
venue = "victim's private space"
elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"):
venue = "public space"
if r.get("kidnappings") == 0:
held = "none"
if r.get("scenario") == "Dodged Attack":
outcome = "foiled"
elif (r.get("life_taken") or 0) > 0:
outcome = "killed"
demanded = money_usd(r.get("money_wanted"))
row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000],
(overrides.get(str(cid)) or "")[:32000] or None]
ws.append(row)
rown = ws.max_row
# colours
for j, k in enumerate(cols, 1):
cell = ws.cell(row=rown, column=j)
v = cell.value
if k.startswith("S: "):
if v == "p":
cell.fill = BLUE
elif v == "1?":
cell.fill = YELLOW
elif v == 1 and sq:
cell.fill = GREEN
elif v is None and r.get("scenario") not in SCEN:
cell.fill = RED
continue
if k == "scenarios_in_order":
cell.fill = GREEN if sq else ORANGE
continue
if k == "scenario_notes":
if sq and (sq["gaps"] or not sq["v1_label_supported"]):
cell.fill = YELLOW
continue
if k in FLAGS and v is None:
cell.fill = RED
elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \
and det[cid].get("proposed") != v:
cell.fill = YELLOW
cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})"
elif k in ("victim", "date", "country", "scenario", "description") and not v:
cell.fill = RED
elif k == "url" and not v:
cell.fill = RED
elif k == "sources_with_text" and v == 0:
cell.fill = RED
elif k == "untrimmed_blocks" and v:
cell.fill = YELLOW
elif k == "summary_status" and v != "verified":
cell.fill = YELLOW
elif k == "summary_revised" and not v:
cell.fill = YELLOW
elif k == "detention_decision" and v and ("proposed" in v or "open" in v):
cell.fill = YELLOW
elif k == "dossier_file" and not v:
cell.fill = RED
elif k in ("notes", "reports", "money_wanted", "coin_type") and not v:
cell.fill = YELLOW
elif k in new_cols:
cell.fill = BLUE if v not in (None, "") else ORANGE
ws.freeze_panes = "B2"
for j, k in enumerate(cols, 1):
ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary_db", "summary_revised", "description", "scenarios_in_order", "scenario_notes") else (
28 if k in ("victim", "location", "money_wanted", "url") else (8 if k.startswith("S: ") else 14))
ws.auto_filter.ref = ws.dimensions
# ---------------- sources ----------------
ws6 = wb.create_sheet("sources")
ws6.append(["case_id", "field", "on_db_record", "source", "verdict", "chars", "trimmed_by", "method", "fetched_at",
"archive_url", "url", "note"])
for c in ws6[1]:
c.font = Font(bold=True)
c.fill = HEAD
for r in recs:
cid = r["id"]
if cid in sheet:
ws6.append([cid, "spreadsheet", "sheet paste", "reported_K&R sheet", "SHEET", None, "Sofia trim pass 2026-08",
"copied from the spreadsheet cell", None, None, None, None])
for b in store.get(str(cid), []):
v = b.get("verdict")
ws6.append([cid, b.get("field"), "no" if str(b.get("field", "")).startswith("extra") else "yes", b.get("source"), v,
b.get("chars"), b.get("trimmed_by"), b.get("method"), b.get("fetched_at"), b.get("archive_url"),
b.get("url"), b.get("note") or b.get("archive_note")])
rown = ws6.max_row
if v == "UNTRIMMED":
ws6.cell(row=rown, column=5).fill = YELLOW
elif v in ("BLOCKED", "THIN", "DEAD", "ERROR"):
ws6.cell(row=rown, column=5).fill = RED
elif v == "OFF-CASE":
ws6.cell(row=rown, column=5).fill = ORANGE
if str(b.get("trimmed_by", "")).startswith("auto"):
ws6.cell(row=rown, column=7).fill = BLUE
ws6.freeze_panes = "A2"
ws6.auto_filter.ref = ws6.dimensions
for j, w in enumerate([8, 11, 10, 24, 12, 8, 26, 34, 20, 40, 60, 60], 1):
ws6.column_dimensions[get_column_letter(j)].width = w
# ---------------- detention review ----------------
ws7 = wb.create_sheet("detention_review")
ws7.append(["case_id", "current", "proposed", "confidence", "status", "basis / evidence", "note"])
for c in ws7[1]:
c.font = Font(bold=True)
c.fill = HEAD
for cid, d in sorted(det.items()):
ws7.append([cid, d.get("current"), d.get("proposed"), d.get("confidence"), d.get("status", "open"),
d.get("basis"), d.get("note")])
st = str(d.get("status", "open"))
ws7.cell(row=ws7.max_row, column=5).fill = (YELLOW if st.startswith(("proposed", "open")) else
ORANGE if st.startswith("WITHDRAWN") else BLUE)
ws7.freeze_panes = "A2"
ws7.auto_filter.ref = ws7.dimensions
for j, w in enumerate([8, 8, 9, 11, 40, 90, 60], 1):
ws7.column_dimensions[get_column_letter(j)].width = w
# ---------------- vocabulary ----------------
ws4 = wb.create_sheet("vocabulary")
ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"])
for c in ws4[1]:
c.font = Font(bold=True)
c.fill = HEAD
for k, vals in VOCAB.items():
ws4.append([k, " | ".join(vals)])
ws4.column_dimensions["A"].width = 28
ws4.column_dimensions["B"].width = 140
# ---------------- legend ----------------
ws5 = wb.create_sheet("legend")
rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"),
("orange", ORANGE, "new column or new row field, not yet filled"),
("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, "
"detention flag with an accepted or proposed change (shown as old -> new)"),
("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, "
"USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"),
("green", GREEN, "scenario assigned by reading the verified summary (scenario-sequences.json); 1? in yellow = the site's label is not supported by the summary"),
("none", None, "present and verified")]
ws5.append(["note", "column 'detention' on the attacks sheet is the DB field 'kidnappings' (the detention violence category, see DEFINITIONS.md)"])
ws5.append(["colour", "meaning"])
for name, fill, meaning in rows:
ws5.append([name, meaning])
if fill:
ws5.cell(row=ws5.max_row, column=1).fill = fill
ws5.append([])
ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"])
ws5.column_dimensions["B"].width = 150
out = BASE / "exports"
out.mkdir(exist_ok=True)
path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx"
wb.save(path)
print(f"wrote {path} : attacks {ws.max_row - 1} rows, sources {ws6.max_row - 1}, detention_review {ws7.max_row - 1}")
if __name__ == "__main__":
main()