StellarCrowandClaude Fable 5.1
10fddba77d
Eleven S: columns on the attacks sheet (1 = confirmed v1 label, p = proposed by the site's keyword engine) plus scenarios_in_order; the separate case_scenarios sheet stays. Old dated export removed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
292 lines
15 KiB
Python
292 lines
15 KiB
Python
"""
|
|
Export the database as an .xlsx with every current field plus the fields of the proposed
|
|
rework, colour-coded so the gaps are visible at a glance.
|
|
|
|
Sheets
|
|
attacks one row per case: current columns, source coverage, summary status, the
|
|
six flags, and the new fact columns (venue_of_control, held_duration,
|
|
outcome, arrests, amount_demanded_usd, amount_taken_usd)
|
|
case_scenarios one row per scenario tag: the confirmed v1 label (seq 1) and the tags the
|
|
site's signature engine proposes (status proposed)
|
|
case_violence one row per (case, category) seeded from the six flags; subtype, target
|
|
and evidence are empty until a person fills them from the dossier
|
|
vocabulary the controlled value lists for every typed column
|
|
legend what the colours mean
|
|
|
|
Colours
|
|
red required value missing (null flag, no source text, empty field)
|
|
orange new column, not yet filled
|
|
yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor)
|
|
blue filled by a rule from existing data; a proposal, not a verified value
|
|
none present and verified
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from openpyxl import Workbook
|
|
from openpyxl.styles import Alignment, Font, PatternFill
|
|
from openpyxl.utils import get_column_letter
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
|
STORE = BASE / "sourced-articles.json"
|
|
OVERRIDES = BASE / "summary-overrides.json"
|
|
MANIFEST = BASE / "case-files-manifest.json"
|
|
DETENTION = BASE / "detention-flag-corrections.json"
|
|
SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"
|
|
|
|
RED = PatternFill("solid", fgColor="F4B6B6")
|
|
ORANGE = PatternFill("solid", fgColor="F9D5A7")
|
|
YELLOW = PatternFill("solid", fgColor="FFF2A8")
|
|
BLUE = PatternFill("solid", fgColor="BDD7EE")
|
|
HEAD = PatternFill("solid", fgColor="D9D9D9")
|
|
|
|
FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"]
|
|
CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol",
|
|
"weapons": "weapons", "theft": "theft", "life_taken": "fatality"}
|
|
|
|
VOCAB = {
|
|
"venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space",
|
|
"vehicle in motion", "public space", "none"],
|
|
"held_duration": ["none", "minutes", "hours", "days"],
|
|
"outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"],
|
|
"arrests": ["none", "some", "all", "convicted"],
|
|
"scenario (11 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space",
|
|
"Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse",
|
|
"Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting"],
|
|
"phase_role": ["initiation", "execution", "captivity", "outcome"],
|
|
"violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"],
|
|
"detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away",
|
|
"locked in", "fake arrest"],
|
|
"violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock",
|
|
"waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet",
|
|
"harm to pet", "pet killed"],
|
|
"weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive",
|
|
"armed police response, induced"],
|
|
"drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"],
|
|
"theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"],
|
|
"fatality subtypes": ["shot", "beaten", "other or unknown"],
|
|
"target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"],
|
|
}
|
|
|
|
|
|
def money_usd(text):
|
|
"""Best-effort USD number from money_wanted; None when not clearly USD."""
|
|
if not text:
|
|
return None
|
|
t = text.replace(",", "")
|
|
m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I)
|
|
if not m:
|
|
return None
|
|
v = float(m.group(1))
|
|
unit = (m.group(2) or "").lower()
|
|
if unit in ("million", "m"):
|
|
v *= 1e6
|
|
elif unit in ("k", "thousand"):
|
|
v *= 1e3
|
|
return v
|
|
|
|
|
|
def main():
|
|
recs = json.loads(JSONF.read_text(encoding="utf-8"))
|
|
store = json.loads(STORE.read_text(encoding="utf-8"))
|
|
overrides = json.loads(OVERRIDES.read_text(encoding="utf-8"))
|
|
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")}
|
|
det = {}
|
|
if DETENTION.exists():
|
|
for c in json.loads(DETENTION.read_text(encoding="utf-8")):
|
|
det[int(c["target"].split()[-1])] = c
|
|
sys.path.insert(0, str(SITE))
|
|
try:
|
|
from scenario_signature import build_signature
|
|
except Exception:
|
|
build_signature = None
|
|
|
|
wb = Workbook()
|
|
|
|
# ---------------- attacks ----------------
|
|
ws = wb.active
|
|
ws.title = "attacks"
|
|
base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario",
|
|
"description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"]
|
|
cov_cols = ["sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars"]
|
|
new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"]
|
|
SCEN = VOCAB["scenario (11 labels)"]
|
|
scen_cols = ["S: " + n for n in SCEN] + ["scenarios_in_order"]
|
|
cols = base_cols + scen_cols + FLAGS + cov_cols + new_cols + ["summary"]
|
|
LABEL = {"kidnappings": "detention"} # the DB column keeps its old name; the export shows the glossary name
|
|
ws.append([LABEL.get(c, c) for c in cols])
|
|
for c in ws[1]:
|
|
c.font = Font(bold=True)
|
|
c.fill = HEAD
|
|
for r in recs:
|
|
cid = r["id"]
|
|
blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")]
|
|
n_src = len(blocks) + (1 if cid in sheet else 0)
|
|
chars = sum(b.get("chars") or 0 for b in blocks)
|
|
untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED")
|
|
if str(cid) in overrides:
|
|
sstat = "verified"
|
|
elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4:
|
|
sstat = "bulk, short"
|
|
else:
|
|
sstat = "bulk, unverified"
|
|
tags = build_signature(r, r.get("scenario")) if build_signature else [
|
|
{"scenario": r.get("scenario"), "status": "confirmed"}]
|
|
tagmap = {t["scenario"]: t["status"] for t in tags}
|
|
scen_vals = [(1 if tagmap.get(n) == "confirmed" else ("p" if tagmap.get(n) == "proposed" else None)) for n in SCEN]
|
|
seq_text = " > ".join([t["scenario"] for t in tags if t["status"] == "confirmed"]
|
|
+ [t["scenario"] + " (p)" for t in tags if t["status"] == "proposed"])
|
|
row = [r.get(k) for k in base_cols] + scen_vals + [seq_text] + [r.get(k) for k in FLAGS] + \
|
|
[n_src, chars, untr, sstat, len(r.get("summary") or "")]
|
|
# rule-derived proposals for the new columns
|
|
venue = held = outcome = None
|
|
if r.get("scenario") == "Home Invasion":
|
|
venue = "victim's private space"
|
|
elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"):
|
|
venue = "public space"
|
|
if r.get("kidnappings") == 0:
|
|
held = "none"
|
|
if r.get("scenario") == "Dodged Attack":
|
|
outcome = "foiled"
|
|
elif (r.get("life_taken") or 0) > 0:
|
|
outcome = "killed"
|
|
demanded = money_usd(r.get("money_wanted"))
|
|
row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000]]
|
|
ws.append(row)
|
|
rown = ws.max_row
|
|
# colours
|
|
for j, k in enumerate(cols, 1):
|
|
cell = ws.cell(row=rown, column=j)
|
|
v = cell.value
|
|
if k.startswith("S: "):
|
|
if v == "p":
|
|
cell.fill = BLUE
|
|
elif v is None and r.get("scenario") not in SCEN:
|
|
cell.fill = RED
|
|
continue
|
|
if k == "scenarios_in_order":
|
|
cell.fill = ORANGE if "(p)" in (v or "") or len(tags) == 1 else None
|
|
continue
|
|
if k in FLAGS and v is None:
|
|
cell.fill = RED
|
|
elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \
|
|
and det[cid].get("proposed") != v:
|
|
cell.fill = YELLOW
|
|
cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})"
|
|
elif k in ("victim", "date", "country", "scenario", "description") and not v:
|
|
cell.fill = RED
|
|
elif k == "url" and not v:
|
|
cell.fill = RED
|
|
elif k == "sources_with_text" and v == 0:
|
|
cell.fill = RED
|
|
elif k == "untrimmed_blocks" and v:
|
|
cell.fill = YELLOW
|
|
elif k == "summary_status" and v != "verified":
|
|
cell.fill = YELLOW
|
|
elif k in ("notes", "reports", "money_wanted", "coin_type") and not v:
|
|
cell.fill = YELLOW
|
|
elif k in new_cols:
|
|
cell.fill = BLUE if v not in (None, "") else ORANGE
|
|
ws.freeze_panes = "B2"
|
|
for j, k in enumerate(cols, 1):
|
|
ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary", "description", "scenarios_in_order") else (
|
|
28 if k in ("victim", "location", "money_wanted", "url") else (8 if k.startswith("S: ") else 14))
|
|
ws.auto_filter.ref = ws.dimensions
|
|
|
|
# ---------------- case_scenarios ----------------
|
|
ws2 = wb.create_sheet("case_scenarios")
|
|
ws2.append(["case_id", "seq", "scenario", "phase_role", "status", "source", "rationale"])
|
|
for c in ws2[1]:
|
|
c.font = Font(bold=True)
|
|
c.fill = HEAD
|
|
role = {"Costly Fame": "initiation", "Malicious Invitation - SocialEng": "initiation", "P2P Trade Gone Wrong": "initiation",
|
|
"Authority Misuse": "initiation", "Swatting": "initiation", "Home Invasion": "execution",
|
|
"Armed Robbery in Public Space": "execution", "Assault - Mugging": "execution", "Kidnapping": "captivity",
|
|
"Express Kidnapping": "captivity", "Dodged Attack": "outcome"}
|
|
for r in recs:
|
|
tags = build_signature(r, r.get("scenario")) if build_signature else [
|
|
{"scenario": r.get("scenario"), "status": "confirmed", "source": "original", "rationale": "Original v1 scenario classification."}]
|
|
seq = 1
|
|
for t in tags:
|
|
ws2.append([r["id"], seq if t["status"] == "confirmed" else None, t["scenario"], role.get(t["scenario"]),
|
|
t["status"], t["source"], t["rationale"]])
|
|
if t["status"] == "proposed":
|
|
for c in ws2[ws2.max_row]:
|
|
c.fill = BLUE
|
|
else:
|
|
ws2.cell(row=ws2.max_row, column=2).fill = ORANGE # order of phases still to set
|
|
seq += 1
|
|
ws2.freeze_panes = "A2"
|
|
ws2.auto_filter.ref = ws2.dimensions
|
|
for j, w in enumerate([10, 6, 34, 12, 11, 10, 60], 1):
|
|
ws2.column_dimensions[get_column_letter(j)].width = w
|
|
|
|
# ---------------- case_violence ----------------
|
|
ws3 = wb.create_sheet("case_violence")
|
|
ws3.append(["case_id", "category", "subtype", "target", "evidence", "source"]) # category 'detention' = DB flag kidnappings
|
|
for c in ws3[1]:
|
|
c.font = Font(bold=True)
|
|
c.fill = HEAD
|
|
for r in recs:
|
|
for f in FLAGS:
|
|
v = r.get(f)
|
|
if v is None:
|
|
ws3.append([r["id"], CATEGORY_OF[f], None, None, None, "flag is null"])
|
|
for c in ws3[ws3.max_row]:
|
|
c.fill = RED
|
|
elif int(v) > 0:
|
|
n = int(v) if f == "life_taken" else 1
|
|
for _ in range(n):
|
|
ws3.append([r["id"], CATEGORY_OF[f], None, None, None, f"seeded from flag {LABEL.get(f, f)}={v}"])
|
|
for c in ws3[ws3.max_row][2:5]:
|
|
c.fill = ORANGE
|
|
ws3.freeze_panes = "A2"
|
|
ws3.auto_filter.ref = ws3.dimensions
|
|
for j, w in enumerate([10, 18, 32, 16, 60, 24], 1):
|
|
ws3.column_dimensions[get_column_letter(j)].width = w
|
|
|
|
# ---------------- vocabulary ----------------
|
|
ws4 = wb.create_sheet("vocabulary")
|
|
ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"])
|
|
for c in ws4[1]:
|
|
c.font = Font(bold=True)
|
|
c.fill = HEAD
|
|
for k, vals in VOCAB.items():
|
|
ws4.append([k, " | ".join(vals)])
|
|
ws4.column_dimensions["A"].width = 28
|
|
ws4.column_dimensions["B"].width = 140
|
|
|
|
# ---------------- legend ----------------
|
|
ws5 = wb.create_sheet("legend")
|
|
rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"),
|
|
("orange", ORANGE, "new column or new row field, not yet filled"),
|
|
("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, "
|
|
"detention flag with an accepted or proposed change (shown as old -> new)"),
|
|
("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, "
|
|
"USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"),
|
|
("none", None, "present and verified")]
|
|
ws5.append(["note", "column 'detention' on the attacks sheet is the DB field 'kidnappings' (the detention violence category, see DEFINITIONS.md)"])
|
|
ws5.append(["colour", "meaning"])
|
|
for name, fill, meaning in rows:
|
|
ws5.append([name, meaning])
|
|
if fill:
|
|
ws5.cell(row=ws5.max_row, column=1).fill = fill
|
|
ws5.append([])
|
|
ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"])
|
|
ws5.column_dimensions["B"].width = 150
|
|
|
|
out = BASE / "exports"
|
|
out.mkdir(exist_ok=True)
|
|
path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx"
|
|
wb.save(path)
|
|
print(f"wrote {path} : attacks {ws.max_row - 1} rows, case_scenarios {ws2.max_row - 1}, case_violence {ws3.max_row - 1}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|