""" Export the database as an .xlsx with every current field plus the fields of the proposed rework, colour-coded so the gaps are visible at a glance. Sheets attacks one row per case: current columns, source coverage, summary status, the six flags, and the new fact columns (venue_of_control, held_duration, outcome, arrests, amount_demanded_usd, amount_taken_usd) case_scenarios one row per scenario tag: the confirmed v1 label (seq 1) and the tags the site's signature engine proposes (status proposed) case_violence one row per (case, category) seeded from the six flags; subtype, target and evidence are empty until a person fills them from the dossier vocabulary the controlled value lists for every typed column legend what the colours mean Colours red required value missing (null flag, no source text, empty field) orange new column, not yet filled yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor) blue filled by a rule from existing data; a proposal, not a verified value none present and verified """ import json import re import sys from datetime import date from pathlib import Path from openpyxl import Workbook from openpyxl.styles import Alignment, Font, PatternFill from openpyxl.utils import get_column_letter BASE = Path(__file__).resolve().parent JSONF = BASE.parent / "attacks-export-Gart-website.json" STORE = BASE / "sourced-articles.json" OVERRIDES = BASE / "summary-overrides.json" MANIFEST = BASE / "case-files-manifest.json" DETENTION = BASE / "detention-flag-corrections.json" SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3" RED = PatternFill("solid", fgColor="F4B6B6") ORANGE = PatternFill("solid", fgColor="F9D5A7") YELLOW = PatternFill("solid", fgColor="FFF2A8") BLUE = PatternFill("solid", fgColor="BDD7EE") HEAD = PatternFill("solid", fgColor="D9D9D9") FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"] CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol", "weapons": "weapons", "theft": "theft", "life_taken": "fatality"} VOCAB = { "venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space", "vehicle in motion", "public space", "none"], "held_duration": ["none", "minutes", "hours", "days"], "outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"], "arrests": ["none", "some", "all", "convicted"], "scenario (11 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space", "Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse", "Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting"], "phase_role": ["initiation", "execution", "captivity", "outcome"], "violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"], "detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away", "locked in", "fake arrest"], "violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock", "waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet", "harm to pet", "pet killed"], "weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive", "armed police response, induced"], "drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"], "theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"], "fatality subtypes": ["shot", "beaten", "other or unknown"], "target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"], } def money_usd(text): """Best-effort USD number from money_wanted; None when not clearly USD.""" if not text: return None t = text.replace(",", "") m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I) if not m: return None v = float(m.group(1)) unit = (m.group(2) or "").lower() if unit in ("million", "m"): v *= 1e6 elif unit in ("k", "thousand"): v *= 1e3 return v def main(): recs = json.loads(JSONF.read_text(encoding="utf-8")) store = json.loads(STORE.read_text(encoding="utf-8")) overrides = json.loads(OVERRIDES.read_text(encoding="utf-8")) sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")} det = {} if DETENTION.exists(): for c in json.loads(DETENTION.read_text(encoding="utf-8")): det[int(c["target"].split()[-1])] = c sys.path.insert(0, str(SITE)) try: from scenario_signature import build_signature except Exception: build_signature = None wb = Workbook() # ---------------- attacks ---------------- ws = wb.active ws.title = "attacks" base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario", "description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"] cov_cols = ["sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars"] new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"] cols = base_cols + FLAGS + cov_cols + new_cols + ["summary"] ws.append(cols) for c in ws[1]: c.font = Font(bold=True) c.fill = HEAD for r in recs: cid = r["id"] blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")] n_src = len(blocks) + (1 if cid in sheet else 0) chars = sum(b.get("chars") or 0 for b in blocks) untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED") if str(cid) in overrides: sstat = "verified" elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4: sstat = "bulk, short" else: sstat = "bulk, unverified" row = [r.get(k) for k in base_cols] + [r.get(k) for k in FLAGS] + \ [n_src, chars, untr, sstat, len(r.get("summary") or "")] # rule-derived proposals for the new columns venue = held = outcome = None if r.get("scenario") == "Home Invasion": venue = "victim's private space" elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"): venue = "public space" if r.get("kidnappings") == 0: held = "none" if r.get("scenario") == "Dodged Attack": outcome = "foiled" elif (r.get("life_taken") or 0) > 0: outcome = "killed" demanded = money_usd(r.get("money_wanted")) row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000]] ws.append(row) rown = ws.max_row # colours for j, k in enumerate(cols, 1): cell = ws.cell(row=rown, column=j) v = cell.value if k in FLAGS and v is None: cell.fill = RED elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \ and det[cid].get("proposed") != v: cell.fill = YELLOW cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})" elif k in ("victim", "date", "country", "scenario", "description") and not v: cell.fill = RED elif k == "url" and not v: cell.fill = RED elif k == "sources_with_text" and v == 0: cell.fill = RED elif k == "untrimmed_blocks" and v: cell.fill = YELLOW elif k == "summary_status" and v != "verified": cell.fill = YELLOW elif k in ("notes", "reports", "money_wanted", "coin_type") and not v: cell.fill = YELLOW elif k in new_cols: cell.fill = BLUE if v not in (None, "") else ORANGE ws.freeze_panes = "B2" for j, k in enumerate(cols, 1): ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary", "description") else ( 28 if k in ("victim", "location", "money_wanted", "url") else 14) ws.auto_filter.ref = ws.dimensions # ---------------- case_scenarios ---------------- ws2 = wb.create_sheet("case_scenarios") ws2.append(["case_id", "seq", "scenario", "phase_role", "status", "source", "rationale"]) for c in ws2[1]: c.font = Font(bold=True) c.fill = HEAD role = {"Costly Fame": "initiation", "Malicious Invitation - SocialEng": "initiation", "P2P Trade Gone Wrong": "initiation", "Authority Misuse": "initiation", "Swatting": "initiation", "Home Invasion": "execution", "Armed Robbery in Public Space": "execution", "Assault - Mugging": "execution", "Kidnapping": "captivity", "Express Kidnapping": "captivity", "Dodged Attack": "outcome"} for r in recs: tags = build_signature(r, r.get("scenario")) if build_signature else [ {"scenario": r.get("scenario"), "status": "confirmed", "source": "original", "rationale": "Original v1 scenario classification."}] seq = 1 for t in tags: ws2.append([r["id"], seq if t["status"] == "confirmed" else None, t["scenario"], role.get(t["scenario"]), t["status"], t["source"], t["rationale"]]) if t["status"] == "proposed": for c in ws2[ws2.max_row]: c.fill = BLUE else: ws2.cell(row=ws2.max_row, column=2).fill = ORANGE # order of phases still to set seq += 1 ws2.freeze_panes = "A2" ws2.auto_filter.ref = ws2.dimensions for j, w in enumerate([10, 6, 34, 12, 11, 10, 60], 1): ws2.column_dimensions[get_column_letter(j)].width = w # ---------------- case_violence ---------------- ws3 = wb.create_sheet("case_violence") ws3.append(["case_id", "category", "subtype", "target", "evidence", "source"]) for c in ws3[1]: c.font = Font(bold=True) c.fill = HEAD for r in recs: for f in FLAGS: v = r.get(f) if v is None: ws3.append([r["id"], CATEGORY_OF[f], None, None, None, "flag is null"]) for c in ws3[ws3.max_row]: c.fill = RED elif int(v) > 0: n = int(v) if f == "life_taken" else 1 for _ in range(n): ws3.append([r["id"], CATEGORY_OF[f], None, None, None, f"seeded from flag {f}={v}"]) for c in ws3[ws3.max_row][2:5]: c.fill = ORANGE ws3.freeze_panes = "A2" ws3.auto_filter.ref = ws3.dimensions for j, w in enumerate([10, 18, 32, 16, 60, 24], 1): ws3.column_dimensions[get_column_letter(j)].width = w # ---------------- vocabulary ---------------- ws4 = wb.create_sheet("vocabulary") ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"]) for c in ws4[1]: c.font = Font(bold=True) c.fill = HEAD for k, vals in VOCAB.items(): ws4.append([k, " | ".join(vals)]) ws4.column_dimensions["A"].width = 28 ws4.column_dimensions["B"].width = 140 # ---------------- legend ---------------- ws5 = wb.create_sheet("legend") rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"), ("orange", ORANGE, "new column or new row field, not yet filled"), ("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, " "detention flag with an accepted or proposed change (shown as old -> new)"), ("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, " "USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"), ("none", None, "present and verified")] ws5.append(["colour", "meaning"]) for name, fill, meaning in rows: ws5.append([name, meaning]) if fill: ws5.cell(row=ws5.max_row, column=1).fill = fill ws5.append([]) ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"]) ws5.column_dimensions["B"].width = 150 out = BASE / "exports" out.mkdir(exist_ok=True) path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx" wb.save(path) print(f"wrote {path} : attacks {ws.max_row - 1} rows, case_scenarios {ws2.max_row - 1}, case_violence {ws3.max_row - 1}") if __name__ == "__main__": main()