This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/export_xlsx.py
T
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

326 lines
16 KiB
Python

"""
Export the database as an .xlsx with every current field plus the fields of the proposed
rework, colour-coded so the gaps are visible at a glance.
Sheets
attacks one row per case: current columns, source coverage, summary status, the
six flags, and the new fact columns (venue_of_control, held_duration,
outcome, arrests, amount_demanded_usd, amount_taken_usd)
vocabulary the controlled value lists for every typed column
legend what the colours mean
Colours
red required value missing (null flag, no source text, empty field)
orange new column, not yet filled
yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor)
blue filled by a rule from existing data; a proposal, not a verified value
none present and verified
"""
import json
import re
import sys
from datetime import date
from pathlib import Path
from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
BASE = Path(__file__).resolve().parent
JSONF = BASE.parent / "attacks-export-Gart-website.json"
STORE = BASE / "sourced-articles.json"
OVERRIDES = BASE / "summary-overrides.json"
MANIFEST = BASE / "case-files-manifest.json"
DETENTION = BASE / "detention-flag-corrections.json"
SEQUENCES = BASE / "scenario-sequences.json"
SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"
RED = PatternFill("solid", fgColor="F4B6B6")
ORANGE = PatternFill("solid", fgColor="F9D5A7")
YELLOW = PatternFill("solid", fgColor="FFF2A8")
BLUE = PatternFill("solid", fgColor="BDD7EE")
GREEN = PatternFill("solid", fgColor="C6E0B4")
HEAD = PatternFill("solid", fgColor="D9D9D9")
FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"]
CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol",
"weapons": "weapons", "theft": "theft", "life_taken": "fatality"}
VOCAB = {
"venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space",
"vehicle in motion", "public space", "none"],
"held_duration": ["none", "minutes", "hours", "days"],
"outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"],
"arrests": ["none", "some", "all", "convicted"],
"scenario (12 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space",
"Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse",
"Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting", "Burglary"],
"phase_role": ["initiation", "execution", "captivity", "outcome"],
"violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"],
"detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away",
"locked in", "fake arrest"],
"violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock",
"waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet",
"harm to pet", "pet killed"],
"weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive",
"armed police response, induced"],
"drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"],
"theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"],
"fatality subtypes": ["shot", "beaten", "other or unknown"],
"target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"],
}
def money_usd(text):
"""Best-effort USD number from money_wanted; None when not clearly USD."""
if not text:
return None
t = text.replace(",", "")
m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I)
if not m:
return None
v = float(m.group(1))
unit = (m.group(2) or "").lower()
if unit in ("million", "m"):
v *= 1e6
elif unit in ("k", "thousand"):
v *= 1e3
return v
def main():
recs = json.loads(JSONF.read_text(encoding="utf-8"))
store = json.loads(STORE.read_text(encoding="utf-8"))
overrides = json.loads(OVERRIDES.read_text(encoding="utf-8"))
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")}
det = {}
if DETENTION.exists():
for c in json.loads(DETENTION.read_text(encoding="utf-8")):
det[int(c["target"].split()[-1])] = c
seqs = json.loads(SEQUENCES.read_text(encoding="utf-8")) if SEQUENCES.exists() else {}
sys.path.insert(0, str(SITE))
try:
from scenario_signature import build_signature
except Exception:
build_signature = None
wb = Workbook()
# ---------------- attacks ----------------
ws = wb.active
ws.title = "attacks"
base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario",
"description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"]
cov_cols = ["dossier_file", "sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars",
"detention_decision"]
new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"]
SCEN = VOCAB["scenario (12 labels)"]
scen_cols = ["S: " + n for n in SCEN] + ["scenarios_in_order", "scenario_notes"]
cols = base_cols + scen_cols + FLAGS + cov_cols + new_cols + ["summary_db", "summary_revised"]
LABEL = {"kidnappings": "detention"} # the DB column keeps its old name; the export shows the glossary name
ws.append([LABEL.get(c, c) for c in cols])
for c in ws[1]:
c.font = Font(bold=True)
c.fill = HEAD
for r in recs:
cid = r["id"]
blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")]
n_src = len(blocks) + (1 if cid in sheet else 0)
chars = sum(b.get("chars") or 0 for b in blocks)
untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED")
if str(cid) in overrides:
sstat = "verified"
elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4:
sstat = "bulk, short"
else:
sstat = "bulk, unverified"
tags = build_signature(r, r.get("scenario")) if build_signature else [
{"scenario": r.get("scenario"), "status": "confirmed"}]
tagmap = {t["scenario"]: t["status"] for t in tags}
sq = seqs.get(str(cid))
if sq:
assigned = set(sq["sequence"])
scen_vals = []
for n in SCEN:
if n in assigned:
scen_vals.append(1)
elif n == r.get("scenario") and not sq["v1_label_supported"]:
scen_vals.append("1?")
else:
scen_vals.append(None)
seq_text = " > ".join(sq["sequence"]) or "(none of the twelve labels fits)"
notes = sq["rationale"]
if sq["attempted"]:
notes += " Attempted: " + ", ".join(sq["attempted"]) + "."
if sq["gaps"]:
notes += " GAP: " + "; ".join(sq["gaps"]) + "."
else:
scen_vals = [(1 if tagmap.get(n) == "confirmed" else ("p" if tagmap.get(n) == "proposed" else None)) for n in SCEN]
seq_text = " > ".join([t["scenario"] for t in tags if t["status"] == "confirmed"]
+ [t["scenario"] + " (p)" for t in tags if t["status"] == "proposed"])
notes = None
dossier = next((f.name for f in (BASE / "sources").glob(f"{cid:03d}_*.txt")), None)
d = det.get(cid)
ddec = (f"{d.get('current')} -> {d.get('proposed')} [{d.get('confidence')}] {d.get('status', 'open')}" if d else None)
row = [r.get(k) for k in base_cols] + scen_vals + [seq_text, notes] + [r.get(k) for k in FLAGS] + \
[dossier, n_src, chars, untr, sstat, len(r.get("summary") or ""), ddec]
# rule-derived proposals for the new columns
venue = held = outcome = None
if r.get("scenario") == "Home Invasion":
venue = "victim's private space"
elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"):
venue = "public space"
if r.get("kidnappings") == 0:
held = "none"
if r.get("scenario") == "Dodged Attack":
outcome = "foiled"
elif (r.get("life_taken") or 0) > 0:
outcome = "killed"
demanded = money_usd(r.get("money_wanted"))
row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000],
(overrides.get(str(cid)) or "")[:32000] or None]
ws.append(row)
rown = ws.max_row
# colours
for j, k in enumerate(cols, 1):
cell = ws.cell(row=rown, column=j)
v = cell.value
if k.startswith("S: "):
if v == "p":
cell.fill = BLUE
elif v == "1?":
cell.fill = YELLOW
elif v == 1 and sq:
cell.fill = GREEN
elif v is None and r.get("scenario") not in SCEN:
cell.fill = RED
continue
if k == "scenarios_in_order":
cell.fill = GREEN if sq else ORANGE
continue
if k == "scenario_notes":
if sq and (sq["gaps"] or not sq["v1_label_supported"]):
cell.fill = YELLOW
continue
if k in FLAGS and v is None:
cell.fill = RED
elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \
and det[cid].get("proposed") != v:
cell.fill = YELLOW
cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})"
elif k in ("victim", "date", "country", "scenario", "description") and not v:
cell.fill = RED
elif k == "url" and not v:
cell.fill = RED
elif k == "sources_with_text" and v == 0:
cell.fill = RED
elif k == "untrimmed_blocks" and v:
cell.fill = YELLOW
elif k == "summary_status" and v != "verified":
cell.fill = YELLOW
elif k == "summary_revised" and not v:
cell.fill = YELLOW
elif k == "detention_decision" and v and ("proposed" in v or "open" in v):
cell.fill = YELLOW
elif k == "dossier_file" and not v:
cell.fill = RED
elif k in ("notes", "reports", "money_wanted", "coin_type") and not v:
cell.fill = YELLOW
elif k in new_cols:
cell.fill = BLUE if v not in (None, "") else ORANGE
ws.freeze_panes = "B2"
for j, k in enumerate(cols, 1):
ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary_db", "summary_revised", "description", "scenarios_in_order", "scenario_notes") else (
28 if k in ("victim", "location", "money_wanted", "url") else (8 if k.startswith("S: ") else 14))
ws.auto_filter.ref = ws.dimensions
# ---------------- sources ----------------
ws6 = wb.create_sheet("sources")
ws6.append(["case_id", "field", "on_db_record", "source", "verdict", "chars", "trimmed_by", "method", "fetched_at",
"archive_url", "url", "note"])
for c in ws6[1]:
c.font = Font(bold=True)
c.fill = HEAD
for r in recs:
cid = r["id"]
if cid in sheet:
ws6.append([cid, "spreadsheet", "sheet paste", "reported_K&R sheet", "SHEET", None, "Sofia trim pass 2026-08",
"copied from the spreadsheet cell", None, None, None, None])
for b in store.get(str(cid), []):
v = b.get("verdict")
ws6.append([cid, b.get("field"), "no" if str(b.get("field", "")).startswith("extra") else "yes", b.get("source"), v,
b.get("chars"), b.get("trimmed_by"), b.get("method"), b.get("fetched_at"), b.get("archive_url"),
b.get("url"), b.get("note") or b.get("archive_note")])
rown = ws6.max_row
if v == "UNTRIMMED":
ws6.cell(row=rown, column=5).fill = YELLOW
elif v in ("BLOCKED", "THIN", "DEAD", "ERROR"):
ws6.cell(row=rown, column=5).fill = RED
elif v == "OFF-CASE":
ws6.cell(row=rown, column=5).fill = ORANGE
if str(b.get("trimmed_by", "")).startswith("auto"):
ws6.cell(row=rown, column=7).fill = BLUE
ws6.freeze_panes = "A2"
ws6.auto_filter.ref = ws6.dimensions
for j, w in enumerate([8, 11, 10, 24, 12, 8, 26, 34, 20, 40, 60, 60], 1):
ws6.column_dimensions[get_column_letter(j)].width = w
# ---------------- detention review ----------------
ws7 = wb.create_sheet("detention_review")
ws7.append(["case_id", "current", "proposed", "confidence", "status", "basis / evidence", "note"])
for c in ws7[1]:
c.font = Font(bold=True)
c.fill = HEAD
for cid, d in sorted(det.items()):
ws7.append([cid, d.get("current"), d.get("proposed"), d.get("confidence"), d.get("status", "open"),
d.get("basis"), d.get("note")])
st = str(d.get("status", "open"))
ws7.cell(row=ws7.max_row, column=5).fill = (YELLOW if st.startswith(("proposed", "open")) else
ORANGE if st.startswith("WITHDRAWN") else BLUE)
ws7.freeze_panes = "A2"
ws7.auto_filter.ref = ws7.dimensions
for j, w in enumerate([8, 8, 9, 11, 40, 90, 60], 1):
ws7.column_dimensions[get_column_letter(j)].width = w
# ---------------- vocabulary ----------------
ws4 = wb.create_sheet("vocabulary")
ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"])
for c in ws4[1]:
c.font = Font(bold=True)
c.fill = HEAD
for k, vals in VOCAB.items():
ws4.append([k, " | ".join(vals)])
ws4.column_dimensions["A"].width = 28
ws4.column_dimensions["B"].width = 140
# ---------------- legend ----------------
ws5 = wb.create_sheet("legend")
rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"),
("orange", ORANGE, "new column or new row field, not yet filled"),
("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, "
"detention flag with an accepted or proposed change (shown as old -> new)"),
("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, "
"USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"),
("green", GREEN, "scenario assigned by reading the verified summary (scenario-sequences.json); 1? in yellow = the site's label is not supported by the summary"),
("none", None, "present and verified")]
ws5.append(["note", "column 'detention' on the attacks sheet is the DB field 'kidnappings' (the detention violence category, see DEFINITIONS.md)"])
ws5.append(["colour", "meaning"])
for name, fill, meaning in rows:
ws5.append([name, meaning])
if fill:
ws5.cell(row=ws5.max_row, column=1).fill = fill
ws5.append([])
ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"])
ws5.column_dimensions["B"].width = 150
out = BASE / "exports"
out.mkdir(exist_ok=True)
path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx"
wb.save(path)
print(f"wrote {path} : attacks {ws.max_row - 1} rows, sources {ws6.max_row - 1}, detention_review {ws7.max_row - 1}")
if __name__ == "__main__":
main()