feat(export): xlsx export with the proposed rework fields, colour-coded gaps
export_xlsx.py writes merge-output/exports/gart-kr-database-<date>.xlsx: attacks (current columns, coverage, summary status, six flags, new fact columns), case_scenarios (confirmed label plus the site engine's proposals), case_violence (rows seeded from the flags, subtype/target/evidence to fill), vocabulary, legend. Red = missing, orange = new field to fill, yellow = unverified, blue = rule-derived. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
This commit is contained in:
StellarCrow
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
58641b6984
commit
e5043eed41
@@ -0,0 +1,272 @@
|
||||
"""
|
||||
Export the database as an .xlsx with every current field plus the fields of the proposed
|
||||
rework, colour-coded so the gaps are visible at a glance.
|
||||
|
||||
Sheets
|
||||
attacks one row per case: current columns, source coverage, summary status, the
|
||||
six flags, and the new fact columns (venue_of_control, held_duration,
|
||||
outcome, arrests, amount_demanded_usd, amount_taken_usd)
|
||||
case_scenarios one row per scenario tag: the confirmed v1 label (seq 1) and the tags the
|
||||
site's signature engine proposes (status proposed)
|
||||
case_violence one row per (case, category) seeded from the six flags; subtype, target
|
||||
and evidence are empty until a person fills them from the dossier
|
||||
vocabulary the controlled value lists for every typed column
|
||||
legend what the colours mean
|
||||
|
||||
Colours
|
||||
red required value missing (null flag, no source text, empty field)
|
||||
orange new column, not yet filled
|
||||
yellow present but unverified or incomplete (bulk summary, untrimmed block, weak anchor)
|
||||
blue filled by a rule from existing data; a proposal, not a verified value
|
||||
none present and verified
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Alignment, Font, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
||||
STORE = BASE / "sourced-articles.json"
|
||||
OVERRIDES = BASE / "summary-overrides.json"
|
||||
MANIFEST = BASE / "case-files-manifest.json"
|
||||
DETENTION = BASE / "detention-flag-corrections.json"
|
||||
SITE = Path.home() / "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"
|
||||
|
||||
RED = PatternFill("solid", fgColor="F4B6B6")
|
||||
ORANGE = PatternFill("solid", fgColor="F9D5A7")
|
||||
YELLOW = PatternFill("solid", fgColor="FFF2A8")
|
||||
BLUE = PatternFill("solid", fgColor="BDD7EE")
|
||||
HEAD = PatternFill("solid", fgColor="D9D9D9")
|
||||
|
||||
FLAGS = ["kidnappings", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"]
|
||||
CATEGORY_OF = {"kidnappings": "detention", "violence_torture": "violence_torture", "drugs_alcohol": "drugs_alcohol",
|
||||
"weapons": "weapons", "theft": "theft", "life_taken": "fatality"}
|
||||
|
||||
VOCAB = {
|
||||
"venue_of_control": ["victim's private space", "victim's workplace", "attacker's or third-party space",
|
||||
"vehicle in motion", "public space", "none"],
|
||||
"held_duration": ["none", "minutes", "hours", "days"],
|
||||
"outcome": ["released", "escaped", "rescued", "killed", "foiled", "unharmed"],
|
||||
"arrests": ["none", "some", "all", "convicted"],
|
||||
"scenario (11 labels)": ["Kidnapping", "Home Invasion", "Armed Robbery in Public Space",
|
||||
"Malicious Invitation - SocialEng", "P2P Trade Gone Wrong", "Authority Misuse",
|
||||
"Dodged Attack", "Costly Fame", "Assault - Mugging", "Express Kidnapping", "Swatting"],
|
||||
"phase_role": ["initiation", "execution", "captivity", "outcome"],
|
||||
"violence category": ["detention", "violence_torture", "weapons", "drugs_alcohol", "theft", "fatality"],
|
||||
"detention subtypes": ["restrained (tied, taped, handcuffed)", "held in place under threat", "taken away",
|
||||
"locked in", "fake arrest"],
|
||||
"violence_torture subtypes": ["beating", "strangling", "burning", "cutting or mutilation", "electric shock",
|
||||
"waterboarding", "sexual assault", "threat of mutilation", "threat to family or pet",
|
||||
"harm to pet", "pet killed"],
|
||||
"weapons subtypes": ["handgun", "rifle", "knife or blade", "blunt object", "taser", "replica", "explosive",
|
||||
"armed police response, induced"],
|
||||
"drugs_alcohol subtypes": ["spiked drink", "forced consumption", "sedative injection"],
|
||||
"theft subtypes": ["forced transfer", "seed or device seized", "cash", "valuables", "nothing obtained"],
|
||||
"fatality subtypes": ["shot", "beaten", "other or unknown"],
|
||||
"target": ["principal", "family member", "unborn child", "staff or guard", "bystander", "attacker", "pet"],
|
||||
}
|
||||
|
||||
|
||||
def money_usd(text):
|
||||
"""Best-effort USD number from money_wanted; None when not clearly USD."""
|
||||
if not text:
|
||||
return None
|
||||
t = text.replace(",", "")
|
||||
m = re.search(r"(?:\$|USD\s?|US\$)\s?(\d+(?:\.\d+)?)\s*(million|m\b|k\b|thousand)?", t, re.I)
|
||||
if not m:
|
||||
return None
|
||||
v = float(m.group(1))
|
||||
unit = (m.group(2) or "").lower()
|
||||
if unit in ("million", "m"):
|
||||
v *= 1e6
|
||||
elif unit in ("k", "thousand"):
|
||||
v *= 1e3
|
||||
return v
|
||||
|
||||
|
||||
def main():
|
||||
recs = json.loads(JSONF.read_text(encoding="utf-8"))
|
||||
store = json.loads(STORE.read_text(encoding="utf-8"))
|
||||
overrides = json.loads(OVERRIDES.read_text(encoding="utf-8"))
|
||||
sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"] if "sheet" in e.get("origin", "")}
|
||||
det = {}
|
||||
if DETENTION.exists():
|
||||
for c in json.loads(DETENTION.read_text(encoding="utf-8")):
|
||||
det[int(c["target"].split()[-1])] = c
|
||||
sys.path.insert(0, str(SITE))
|
||||
try:
|
||||
from scenario_signature import build_signature
|
||||
except Exception:
|
||||
build_signature = None
|
||||
|
||||
wb = Workbook()
|
||||
|
||||
# ---------------- attacks ----------------
|
||||
ws = wb.active
|
||||
ws.title = "attacks"
|
||||
base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario",
|
||||
"description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"]
|
||||
cov_cols = ["sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars"]
|
||||
new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"]
|
||||
cols = base_cols + FLAGS + cov_cols + new_cols + ["summary"]
|
||||
ws.append(cols)
|
||||
for c in ws[1]:
|
||||
c.font = Font(bold=True)
|
||||
c.fill = HEAD
|
||||
for r in recs:
|
||||
cid = r["id"]
|
||||
blocks = [b for b in store.get(str(cid), []) if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")]
|
||||
n_src = len(blocks) + (1 if cid in sheet else 0)
|
||||
chars = sum(b.get("chars") or 0 for b in blocks)
|
||||
untr = sum(1 for b in blocks if b["verdict"] == "UNTRIMMED")
|
||||
if str(cid) in overrides:
|
||||
sstat = "verified"
|
||||
elif len(r.get("summary") or "") < 400 or len(re.findall(r"\*\*[^*]+:\*\*", r.get("summary") or "")) < 4:
|
||||
sstat = "bulk, short"
|
||||
else:
|
||||
sstat = "bulk, unverified"
|
||||
row = [r.get(k) for k in base_cols] + [r.get(k) for k in FLAGS] + \
|
||||
[n_src, chars, untr, sstat, len(r.get("summary") or "")]
|
||||
# rule-derived proposals for the new columns
|
||||
venue = held = outcome = None
|
||||
if r.get("scenario") == "Home Invasion":
|
||||
venue = "victim's private space"
|
||||
elif r.get("scenario") in ("Armed Robbery in Public Space", "Assault - Mugging"):
|
||||
venue = "public space"
|
||||
if r.get("kidnappings") == 0:
|
||||
held = "none"
|
||||
if r.get("scenario") == "Dodged Attack":
|
||||
outcome = "foiled"
|
||||
elif (r.get("life_taken") or 0) > 0:
|
||||
outcome = "killed"
|
||||
demanded = money_usd(r.get("money_wanted"))
|
||||
row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000]]
|
||||
ws.append(row)
|
||||
rown = ws.max_row
|
||||
# colours
|
||||
for j, k in enumerate(cols, 1):
|
||||
cell = ws.cell(row=rown, column=j)
|
||||
v = cell.value
|
||||
if k in FLAGS and v is None:
|
||||
cell.fill = RED
|
||||
elif k == "kidnappings" and cid in det and str(det[cid].get("status", "")).startswith(("ACCEPTED", "proposed")) \
|
||||
and det[cid].get("proposed") != v:
|
||||
cell.fill = YELLOW
|
||||
cell.value = f"{v} -> {det[cid]['proposed']} ({det[cid]['confidence']})"
|
||||
elif k in ("victim", "date", "country", "scenario", "description") and not v:
|
||||
cell.fill = RED
|
||||
elif k == "url" and not v:
|
||||
cell.fill = RED
|
||||
elif k == "sources_with_text" and v == 0:
|
||||
cell.fill = RED
|
||||
elif k == "untrimmed_blocks" and v:
|
||||
cell.fill = YELLOW
|
||||
elif k == "summary_status" and v != "verified":
|
||||
cell.fill = YELLOW
|
||||
elif k in ("notes", "reports", "money_wanted", "coin_type") and not v:
|
||||
cell.fill = YELLOW
|
||||
elif k in new_cols:
|
||||
cell.fill = BLUE if v not in (None, "") else ORANGE
|
||||
ws.freeze_panes = "B2"
|
||||
for j, k in enumerate(cols, 1):
|
||||
ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary", "description") else (
|
||||
28 if k in ("victim", "location", "money_wanted", "url") else 14)
|
||||
ws.auto_filter.ref = ws.dimensions
|
||||
|
||||
# ---------------- case_scenarios ----------------
|
||||
ws2 = wb.create_sheet("case_scenarios")
|
||||
ws2.append(["case_id", "seq", "scenario", "phase_role", "status", "source", "rationale"])
|
||||
for c in ws2[1]:
|
||||
c.font = Font(bold=True)
|
||||
c.fill = HEAD
|
||||
role = {"Costly Fame": "initiation", "Malicious Invitation - SocialEng": "initiation", "P2P Trade Gone Wrong": "initiation",
|
||||
"Authority Misuse": "initiation", "Swatting": "initiation", "Home Invasion": "execution",
|
||||
"Armed Robbery in Public Space": "execution", "Assault - Mugging": "execution", "Kidnapping": "captivity",
|
||||
"Express Kidnapping": "captivity", "Dodged Attack": "outcome"}
|
||||
for r in recs:
|
||||
tags = build_signature(r, r.get("scenario")) if build_signature else [
|
||||
{"scenario": r.get("scenario"), "status": "confirmed", "source": "original", "rationale": "Original v1 scenario classification."}]
|
||||
seq = 1
|
||||
for t in tags:
|
||||
ws2.append([r["id"], seq if t["status"] == "confirmed" else None, t["scenario"], role.get(t["scenario"]),
|
||||
t["status"], t["source"], t["rationale"]])
|
||||
if t["status"] == "proposed":
|
||||
for c in ws2[ws2.max_row]:
|
||||
c.fill = BLUE
|
||||
else:
|
||||
ws2.cell(row=ws2.max_row, column=2).fill = ORANGE # order of phases still to set
|
||||
seq += 1
|
||||
ws2.freeze_panes = "A2"
|
||||
ws2.auto_filter.ref = ws2.dimensions
|
||||
for j, w in enumerate([10, 6, 34, 12, 11, 10, 60], 1):
|
||||
ws2.column_dimensions[get_column_letter(j)].width = w
|
||||
|
||||
# ---------------- case_violence ----------------
|
||||
ws3 = wb.create_sheet("case_violence")
|
||||
ws3.append(["case_id", "category", "subtype", "target", "evidence", "source"])
|
||||
for c in ws3[1]:
|
||||
c.font = Font(bold=True)
|
||||
c.fill = HEAD
|
||||
for r in recs:
|
||||
for f in FLAGS:
|
||||
v = r.get(f)
|
||||
if v is None:
|
||||
ws3.append([r["id"], CATEGORY_OF[f], None, None, None, "flag is null"])
|
||||
for c in ws3[ws3.max_row]:
|
||||
c.fill = RED
|
||||
elif int(v) > 0:
|
||||
n = int(v) if f == "life_taken" else 1
|
||||
for _ in range(n):
|
||||
ws3.append([r["id"], CATEGORY_OF[f], None, None, None, f"seeded from flag {f}={v}"])
|
||||
for c in ws3[ws3.max_row][2:5]:
|
||||
c.fill = ORANGE
|
||||
ws3.freeze_panes = "A2"
|
||||
ws3.auto_filter.ref = ws3.dimensions
|
||||
for j, w in enumerate([10, 18, 32, 16, 60, 24], 1):
|
||||
ws3.column_dimensions[get_column_letter(j)].width = w
|
||||
|
||||
# ---------------- vocabulary ----------------
|
||||
ws4 = wb.create_sheet("vocabulary")
|
||||
ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"])
|
||||
for c in ws4[1]:
|
||||
c.font = Font(bold=True)
|
||||
c.fill = HEAD
|
||||
for k, vals in VOCAB.items():
|
||||
ws4.append([k, " | ".join(vals)])
|
||||
ws4.column_dimensions["A"].width = 28
|
||||
ws4.column_dimensions["B"].width = 140
|
||||
|
||||
# ---------------- legend ----------------
|
||||
ws5 = wb.create_sheet("legend")
|
||||
rows = [("red", RED, "required value missing: null flag, empty victim/date/country/scenario/description/url, no source text"),
|
||||
("orange", ORANGE, "new column or new row field, not yet filled"),
|
||||
("yellow", YELLOW, "present but unverified or incomplete: bulk summary, untrimmed source block, empty notes/reports/money, "
|
||||
"detention flag with an accepted or proposed change (shown as old -> new)"),
|
||||
("blue", BLUE, "filled by a rule from existing data (venue from scenario, held from the flag, outcome from Dodged/life_taken, "
|
||||
"USD parsed from money_wanted, scenario tags proposed by the site's keyword engine): a proposal, not verified"),
|
||||
("none", None, "present and verified")]
|
||||
ws5.append(["colour", "meaning"])
|
||||
for name, fill, meaning in rows:
|
||||
ws5.append([name, meaning])
|
||||
if fill:
|
||||
ws5.cell(row=ws5.max_row, column=1).fill = fill
|
||||
ws5.append([])
|
||||
ws5.append(["generated", date.today().isoformat(), f"{len(recs)} cases"])
|
||||
ws5.column_dimensions["B"].width = 150
|
||||
|
||||
out = BASE / "exports"
|
||||
out.mkdir(exist_ok=True)
|
||||
path = out / f"gart-kr-database-{date.today().isoformat()}.xlsx"
|
||||
wb.save(path)
|
||||
print(f"wrote {path} : attacks {ws.max_row - 1} rows, case_scenarios {ws2.max_row - 1}, case_violence {ws3.max_row - 1}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Reference in New Issue
Block a user