diff --git a/merge-output/export_xlsx.py b/merge-output/export_xlsx.py index fa66bd4..5230907 100644 --- a/merge-output/export_xlsx.py +++ b/merge-output/export_xlsx.py @@ -113,11 +113,12 @@ def main(): ws.title = "attacks" base_cols = ["id", "date", "year", "month", "quarter", "victim", "location", "country", "scenario", "description", "notes", "money_wanted", "coin_type", "reports", "url", "url_2", "url_3", "url_4", "url_5"] - cov_cols = ["sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars"] + cov_cols = ["dossier_file", "sources_with_text", "article_chars", "untrimmed_blocks", "summary_status", "summary_chars", + "detention_decision"] new_cols = ["venue_of_control", "held_duration", "outcome", "arrests", "amount_demanded_usd", "amount_taken_usd"] SCEN = VOCAB["scenario (11 labels)"] scen_cols = ["S: " + n for n in SCEN] + ["scenarios_in_order"] - cols = base_cols + scen_cols + FLAGS + cov_cols + new_cols + ["summary"] + cols = base_cols + scen_cols + FLAGS + cov_cols + new_cols + ["summary_db", "summary_revised"] LABEL = {"kidnappings": "detention"} # the DB column keeps its old name; the export shows the glossary name ws.append([LABEL.get(c, c) for c in cols]) for c in ws[1]: @@ -141,8 +142,11 @@ def main(): scen_vals = [(1 if tagmap.get(n) == "confirmed" else ("p" if tagmap.get(n) == "proposed" else None)) for n in SCEN] seq_text = " > ".join([t["scenario"] for t in tags if t["status"] == "confirmed"] + [t["scenario"] + " (p)" for t in tags if t["status"] == "proposed"]) + dossier = next((f.name for f in (BASE / "sources").glob(f"{cid:03d}_*.txt")), None) + d = det.get(cid) + ddec = (f"{d.get('current')} -> {d.get('proposed')} [{d.get('confidence')}] {d.get('status', 'open')}" if d else None) row = [r.get(k) for k in base_cols] + scen_vals + [seq_text] + [r.get(k) for k in FLAGS] + \ - [n_src, chars, untr, sstat, len(r.get("summary") or "")] + [dossier, n_src, chars, untr, sstat, len(r.get("summary") or ""), ddec] # rule-derived proposals for the new columns venue = held = outcome = None if r.get("scenario") == "Home Invasion": @@ -156,7 +160,8 @@ def main(): elif (r.get("life_taken") or 0) > 0: outcome = "killed" demanded = money_usd(r.get("money_wanted")) - row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000]] + row += [venue, held, outcome, None, demanded, None, (r.get("summary") or "")[:32000], + (overrides.get(str(cid)) or "")[:32000] or None] ws.append(row) rown = ws.max_row # colours @@ -188,13 +193,19 @@ def main(): cell.fill = YELLOW elif k == "summary_status" and v != "verified": cell.fill = YELLOW + elif k == "summary_revised" and not v: + cell.fill = YELLOW + elif k == "detention_decision" and v and ("proposed" in v or "open" in v): + cell.fill = YELLOW + elif k == "dossier_file" and not v: + cell.fill = RED elif k in ("notes", "reports", "money_wanted", "coin_type") and not v: cell.fill = YELLOW elif k in new_cols: cell.fill = BLUE if v not in (None, "") else ORANGE ws.freeze_panes = "B2" for j, k in enumerate(cols, 1): - ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary", "description", "scenarios_in_order") else ( + ws.column_dimensions[get_column_letter(j)].width = 60 if k in ("summary_db", "summary_revised", "description", "scenarios_in_order") else ( 28 if k in ("victim", "location", "money_wanted", "url") else (8 if k.startswith("S: ") else 14)) ws.auto_filter.ref = ws.dimensions @@ -250,6 +261,54 @@ def main(): for j, w in enumerate([10, 18, 32, 16, 60, 24], 1): ws3.column_dimensions[get_column_letter(j)].width = w + # ---------------- sources ---------------- + ws6 = wb.create_sheet("sources") + ws6.append(["case_id", "field", "on_db_record", "source", "verdict", "chars", "trimmed_by", "method", "fetched_at", + "archive_url", "url", "note"]) + for c in ws6[1]: + c.font = Font(bold=True) + c.fill = HEAD + for r in recs: + cid = r["id"] + if cid in sheet: + ws6.append([cid, "spreadsheet", "sheet paste", "reported_K&R sheet", "SHEET", None, "Sofia trim pass 2026-08", + "copied from the spreadsheet cell", None, None, None, None]) + for b in store.get(str(cid), []): + v = b.get("verdict") + ws6.append([cid, b.get("field"), "no" if str(b.get("field", "")).startswith("extra") else "yes", b.get("source"), v, + b.get("chars"), b.get("trimmed_by"), b.get("method"), b.get("fetched_at"), b.get("archive_url"), + b.get("url"), b.get("note") or b.get("archive_note")]) + rown = ws6.max_row + if v == "UNTRIMMED": + ws6.cell(row=rown, column=5).fill = YELLOW + elif v in ("BLOCKED", "THIN", "DEAD", "ERROR"): + ws6.cell(row=rown, column=5).fill = RED + elif v == "OFF-CASE": + ws6.cell(row=rown, column=5).fill = ORANGE + if str(b.get("trimmed_by", "")).startswith("auto"): + ws6.cell(row=rown, column=7).fill = BLUE + ws6.freeze_panes = "A2" + ws6.auto_filter.ref = ws6.dimensions + for j, w in enumerate([8, 11, 10, 24, 12, 8, 26, 34, 20, 40, 60, 60], 1): + ws6.column_dimensions[get_column_letter(j)].width = w + + # ---------------- detention review ---------------- + ws7 = wb.create_sheet("detention_review") + ws7.append(["case_id", "current", "proposed", "confidence", "status", "basis / evidence", "note"]) + for c in ws7[1]: + c.font = Font(bold=True) + c.fill = HEAD + for cid, d in sorted(det.items()): + ws7.append([cid, d.get("current"), d.get("proposed"), d.get("confidence"), d.get("status", "open"), + d.get("basis"), d.get("note")]) + st = str(d.get("status", "open")) + ws7.cell(row=ws7.max_row, column=5).fill = (YELLOW if st.startswith(("proposed", "open")) else + ORANGE if st.startswith("WITHDRAWN") else BLUE) + ws7.freeze_panes = "A2" + ws7.auto_filter.ref = ws7.dimensions + for j, w in enumerate([8, 8, 9, 11, 40, 90, 60], 1): + ws7.column_dimensions[get_column_letter(j)].width = w + # ---------------- vocabulary ---------------- ws4 = wb.create_sheet("vocabulary") ws4.append(["column", "allowed values (draft 2026-09, to be cut down)"]) diff --git a/merge-output/exports/.~lock.gart-kr-database-2026-09-15.xlsx# b/merge-output/exports/.~lock.gart-kr-database-2026-09-15.xlsx# deleted file mode 100644 index 40332dc..0000000 --- a/merge-output/exports/.~lock.gart-kr-database-2026-09-15.xlsx# +++ /dev/null @@ -1 +0,0 @@ -,disi,pop-os,17.09.2026 17:23,file:///home/disi/.config/libreoffice/4; \ No newline at end of file diff --git a/merge-output/exports/gart-kr-database-2026-09-17.xlsx b/merge-output/exports/gart-kr-database-2026-09-17.xlsx deleted file mode 100644 index 648a8ff..0000000 Binary files a/merge-output/exports/gart-kr-database-2026-09-17.xlsx and /dev/null differ diff --git a/merge-output/exports/gart-kr-database-2026-09-18.xlsx b/merge-output/exports/gart-kr-database-2026-09-18.xlsx new file mode 100644 index 0000000..9bfab54 Binary files /dev/null and b/merge-output/exports/gart-kr-database-2026-09-18.xlsx differ