diff --git a/merge-output/export_xlsx.py b/merge-output/export_xlsx.py new file mode 100644 index 0000000..d24b9d2 --- /dev/null +++ b/merge-output/export_xlsx.py @@ -0,0 +1,272 @@ +""" +Export the database as an .xlsx with every current field plus the fields of the proposed +rework, colour-coded so the gaps are visible at a glance. + +Sheets + attacks one row per case: current columns, source coverage, summary status, the + six flags, and the new fact columns (venue_of_control, held_duration, + outcome, arrests, amount_demanded_usd, amount_taken_usd) + case_scenarios one row per scenario tag: the confirmed v1 label (seq 1) and the tags the + site's signature engine proposes (status proposed) + case_violence one row per (case, category) seeded from the six flags; subtype, target + and evidence are empty until a person fills them from the dossier + vocabulary the controlled value lists for every typed column + legend what the colours mean + +Colours + red required value missing (null flag, no source text, empty field) + orange new column, not yet filled + yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor) + blue filled by a rule from existing data; a proposal, not a verified value + none present and verified +""" + +import json +import re +import sys +from datetime import date +from pathlib import Path + +from openpyxl import Workbook +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter + +BASE = Path(__file__).resolve().parent +JSONF = BASE.parent / "attacks-export-Gart-website.json" +STORE = BASE / "sourced-articles.json" +OVERRIDES = BASE / "summary-overrides.json" +MANIFEST = BASE / "case-files-manifest.json" +DETENTION = BASE / "detention-flag-corrections.json" +SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3" + +RED = PatternFill("solid", fgColor="F4B6B6") +ORANGE = PatternFill("solid", fgColor="F9D5A7") +YELLOW = PatternFill("solid", fgColor="FFF2A8") +BLUE = PatternFill("solid", fgColor="BDD7EE") +HEAD = PatternFill("solid", fgColor="D9D9D9") + +FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"] +CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol", + "weapons": "weapons", "theft": "theft", "life_taken": "fatality"} + +VOCAB = { + "venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space", + "vehicle in motion", "public space", "none"], + "held_duration": ["none", "minutes", "hours", "days"], + "outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"], + "arrests": ["none", "some", "all", "convicted"], + "scenario (11 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space", + "Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse", + "Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting"], + "phase_role": ["initiation", "execution", "captivity", "outcome"], + "violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"], + "detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away", + "locked in", "fake arrest"], + "violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock", + "waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet", + "harm to pet", "pet killed"], + "weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive", + "armed police response, induced"], + "drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"], + "theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"], + "fatality subtypes": ["shot", "beaten", "other or unknown"], + "target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"], +} + + +def money_usd(text): + """Best-effort USD number from money_wanted; None when not clearly USD.""" + if not text: + return None + t = text.replace(",", "") + m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I) + if not m: + return None + v = float(m.group(1)) + unit = (m.group(2) or "").lower() + if unit in ("million", "m"): + v *= 1e6 + elif unit in ("k", "thousand"): + v *= 1e3 + return v + + +def main(): + recs = json.loads(JSONF.read_text(encoding="utf-8")) + store = json.loads(STORE.read_text(encoding="utf-8")) + overrides = json.loads(OVERRIDES.read_text(encoding="utf-8")) + sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")} + det = {} + if DETENTION.exists(): + for c in json.loads(DETENTION.read_text(encoding="utf-8")): + det[int(c["target"].split()[-1])] = c + sys.path.insert(0, str(SITE)) + try: + from scenario_signature import build_signature + except Exception: + build_signature = None + + wb = Workbook() + + # ---------------- attacks ---------------- + ws = wb.active + ws.title = "attacks" + base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario", + "description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"] + cov_cols = ["sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars"] + new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"] + cols = base_cols + FLAGS + cov_cols + new_cols + ["summary"] + ws.append(cols) + for c in ws[1]: + c.font = Font(bold=True) + c.fill = HEAD + for r in recs: + cid = r["id"] + blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")] + n_src = len(blocks) + (1 if cid in sheet else 0) + chars = sum(b.get("chars") or 0 for b in blocks) + untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED") + if str(cid) in overrides: + sstat = "verified" + elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4: + sstat = "bulk, short" + else: + sstat = "bulk, unverified" + row = [r.get(k) for k in base_cols] + [r.get(k) for k in FLAGS] + \ + [n_src, chars, untr, sstat, len(r.get("summary") or "")] + # rule-derived proposals for the new columns + venue = held = outcome = None + if r.get("scenario") == "Home Invasion": + venue = "victim's private space" + elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"): + venue = "public space" + if r.get("kidnappings") == 0: + held = "none" + if r.get("scenario") == "Dodged Attack": + outcome = "foiled" + elif (r.get("life_taken") or 0) > 0: + outcome = "killed" + demanded = money_usd(r.get("money_wanted")) + row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000]] + ws.append(row) + rown = ws.max_row + # colours + for j, k in enumerate(cols, 1): + cell = ws.cell(row=rown, column=j) + v = cell.value + if k in FLAGS and v is None: + cell.fill = RED + elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \ + and det[cid].get("proposed") != v: + cell.fill = YELLOW + cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})" + elif k in ("victim", "date", "country", "scenario", "description") and not v: + cell.fill = RED + elif k == "url" and not v: + cell.fill = RED + elif k == "sources_with_text" and v == 0: + cell.fill = RED + elif k == "untrimmed_blocks" and v: + cell.fill = YELLOW + elif k == "summary_status" and v != "verified": + cell.fill = YELLOW + elif k in ("notes", "reports", "money_wanted", "coin_type") and not v: + cell.fill = YELLOW + elif k in new_cols: + cell.fill = BLUE if v not in (None, "") else ORANGE + ws.freeze_panes = "B2" + for j, k in enumerate(cols, 1): + ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary", "description") else ( + 28 if k in ("victim", "location", "money_wanted", "url") else 14) + ws.auto_filter.ref = ws.dimensions + + # ---------------- case_scenarios ---------------- + ws2 = wb.create_sheet("case_scenarios") + ws2.append(["case_id", "seq", "scenario", "phase_role", "status", "source", "rationale"]) + for c in ws2[1]: + c.font = Font(bold=True) + c.fill = HEAD + role = {"Costly Fame": "initiation", "Malicious Invitation - SocialEng": "initiation", "P2P Trade Gone Wrong": "initiation", + "Authority Misuse": "initiation", "Swatting": "initiation", "Home Invasion": "execution", + "Armed Robbery in Public Space": "execution", "Assault - Mugging": "execution", "Kidnapping": "captivity", + "Express Kidnapping": "captivity", "Dodged Attack": "outcome"} + for r in recs: + tags = build_signature(r, r.get("scenario")) if build_signature else [ + {"scenario": r.get("scenario"), "status": "confirmed", "source": "original", "rationale": "Original v1 scenario classification."}] + seq = 1 + for t in tags: + ws2.append([r["id"], seq if t["status"] == "confirmed" else None, t["scenario"], role.get(t["scenario"]), + t["status"], t["source"], t["rationale"]]) + if t["status"] == "proposed": + for c in ws2[ws2.max_row]: + c.fill = BLUE + else: + ws2.cell(row=ws2.max_row, column=2).fill = ORANGE # order of phases still to set + seq += 1 + ws2.freeze_panes = "A2" + ws2.auto_filter.ref = ws2.dimensions + for j, w in enumerate([10, 6, 34, 12, 11, 10, 60], 1): + ws2.column_dimensions[get_column_letter(j)].width = w + + # ---------------- case_violence ---------------- + ws3 = wb.create_sheet("case_violence") + ws3.append(["case_id", "category", "subtype", "target", "evidence", "source"]) + for c in ws3[1]: + c.font = Font(bold=True) + c.fill = HEAD + for r in recs: + for f in FLAGS: + v = r.get(f) + if v is None: + ws3.append([r["id"], CATEGORY_OF[f], None, None, None, "flag is null"]) + for c in ws3[ws3.max_row]: + c.fill = RED + elif int(v) > 0: + n = int(v) if f == "life_taken" else 1 + for _ in range(n): + ws3.append([r["id"], CATEGORY_OF[f], None, None, None, f"seeded from flag {f}={v}"]) + for c in ws3[ws3.max_row][2:5]: + c.fill = ORANGE + ws3.freeze_panes = "A2" + ws3.auto_filter.ref = ws3.dimensions + for j, w in enumerate([10, 18, 32, 16, 60, 24], 1): + ws3.column_dimensions[get_column_letter(j)].width = w + + # ---------------- vocabulary ---------------- + ws4 = wb.create_sheet("vocabulary") + ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"]) + for c in ws4[1]: + c.font = Font(bold=True) + c.fill = HEAD + for k, vals in VOCAB.items(): + ws4.append([k, " | ".join(vals)]) + ws4.column_dimensions["A"].width = 28 + ws4.column_dimensions["B"].width = 140 + + # ---------------- legend ---------------- + ws5 = wb.create_sheet("legend") + rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"), + ("orange", ORANGE, "new column or new row field, not yet filled"), + ("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, " + "detention flag with an accepted or proposed change (shown as old -> new)"), + ("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, " + "USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"), + ("none", None, "present and verified")] + ws5.append(["colour", "meaning"]) + for name, fill, meaning in rows: + ws5.append([name, meaning]) + if fill: + ws5.cell(row=ws5.max_row, column=1).fill = fill + ws5.append([]) + ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"]) + ws5.column_dimensions["B"].width = 150 + + out = BASE / "exports" + out.mkdir(exist_ok=True) + path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx" + wb.save(path) + print(f"wrote {path} : attacks {ws.max_row - 1} rows, case_scenarios {ws2.max_row - 1}, case_violence {ws3.max_row - 1}") + + +if __name__ == "__main__": + main() diff --git a/merge-output/exports/gart-kr-database-2026-09-15.xlsx b/merge-output/exports/gart-kr-database-2026-09-15.xlsx new file mode 100644 index 0000000..382caa6 Binary files /dev/null and b/merge-output/exports/gart-kr-database-2026-09-15.xlsx differ