This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/make_handover_worklist.ts
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

132 lines
8.2 KiB
TypeScript

// Build the per-case worklist for the handover to Sofia:
// WORKLIST-BY-CASE.md every ruling line, flag proposal and sequence, grouped by DB id
// worklist-tracker.csv one row per case, to tick off
// Usage (from merge-output/): bun make_handover_worklist.ts ../attacks-live-api-YYYY-MM-DD.json
import { readFileSync, readdirSync, writeFileSync } from "fs";
const livePath = process.argv[2];
if (!livePath) throw new Error("usage: bun make_handover_worklist.ts <live api snapshot.json>");
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const live: any[] = J(livePath);
const liveById = new Map<number, any>(live.map((r) => [Number(r.id), r]));
const exportIds = new Set<number>(J("../attacks-export-Gart-website.json").map((r: any) => Number(r.id)));
const seqs: Record<string, any> = J("scenario-sequences.json");
const flagProposals: any[] = J("batches/flag-proposals.json");
// latest batch version of every case (part files such as batch-35a.json are skipped)
const batchOf = new Map<number, any>();
for (const f of readdirSync("batches").filter((f) => /^batch-\d+\.json$/.test(f)).sort())
for (const c of J("batches/" + f)) batchOf.set(Number(c.id), { ...c, _batch: f.slice(6, 8) });
const lastBatch = [...batchOf.values()].reduce((m, c) => (c._batch > m ? c._batch : m), "00");
const untriaged = new Set<number>([...batchOf.values()].filter((c) => c._batch === lastBatch).map((c) => Number(c.id)));
// ruling lines from corrections-approved.md, grouped by DB id
const rulings = new Map<number, { section: string; text: string }[]>();
const add = (id: number, section: string, text: string) => {
if (!rulings.has(id)) rulings.set(id, []);
rulings.get(id)!.push({ section, text: text.trim() });
};
const ids = (s: string) => (s.match(/\d+/g) ?? []).map(Number);
let section = "";
let sectionId: number | null = null;
let open: { ids: number[]; section: string; text: string } | null = null;
const flush = () => {
if (open) for (const id of open.ids) add(id, open.section, open.text);
open = null;
};
for (const line of readFileSync("corrections-approved.md", "utf8").split("\n")) {
if (line.startsWith("## ")) {
flush();
section = line.slice(3).trim();
const m = section.match(/^DB id (\d+)\b/);
sectionId = m ? Number(m[1]) : null;
if (sectionId !== null) open = { ids: [sectionId], section, text: "" };
continue;
}
if (line.trim() === "---") { flush(); sectionId = null; continue; }
if (sectionId !== null && open) { open.text += line + "\n"; continue; }
const bullet = line.match(/^- \*\*DB ids? ([^*]+)\*\*/);
if (bullet) { flush(); open = { ids: ids(bullet[1].replace(/\(.*$/, "")), section, text: line.slice(2) }; continue; }
const set = line.match(/Set `kidnappings` to (\d) on ([^*]+)\*\*/);
if (set) {
flush();
for (const id of ids(set[2].replace(/\([^)]*\)/g, ""))) add(id, section, `kidnappings → ${set[1]} (detention flag list)`);
continue;
}
if (line.startsWith("- ") || line.trim() === "") { flush(); continue; }
if (open) open.text += " " + line.trim();
}
flush();
const BURGLARY = new Set([3, 6, 10, 54, 57, 62, 66, 71, 101, 105, 112, 158, 198, 21, 349, 99, 383, 449, 450]); // owner ruling 2026-10-06
const UNLABELLED = new Set([24, 195]); // kept by the owner, label still to be set
const STRUCTURAL: Record<number, string> = {
386: "delete after moving its sources to 295", 295: "receives the sources of 386",
446: "merge into 445, then delete", 445: "receives 446; date 10 to 11 May 2026",
440: "merge into 439, then delete", 441: "merge into 439, then delete", 439: "receives 440 and 441",
49: "split: new record for 21 August 2018", 38: "split into Irving and Mesquite", 5: "rewrite to the 27 May 2015 robbery",
430: "link to 138, 145, 156 (after rebuild)", 138: "link to 430 (after rebuild)", 145: "link to 430 (after rebuild)",
156: "link to 430 (after rebuild)", 106: "link to 124 (after rebuild)", 124: "link to 106 (after rebuild)",
45: "remove from the database (owner, 6 October)",
476: "replace the wrong AseanNow URL", 103: "replace the outlet credit", 458: "new Gardanne record pending La Provence text",
};
const allIds = [...new Set([...exportIds, ...liveById.keys()])].sort((a, b) => a - b);
const one = (s: any) => String(s ?? "").replace(/\s+/g, " ").trim();
const csv = (v: any) => `"${one(v).replace(/"/g, '""')}"`;
const md: string[] = [
"# Worklist by case",
"",
`Generated from corrections-approved.md, batches/flag-proposals.json, scenario-sequences.json and the live API pull ${livePath.replace(/^.*\//, "")}.`,
"Ruling lines are copied verbatim, oldest first. Where two lines on one case disagree, the later one wins.",
"",
];
const rows = [["id", "victim", "country", "date", "live_scenario", "final_sequence", "sequence_differs", "ruling_lines", "flag_proposals", "flags_already_live", "contradictions", "structural_operation", "label_to_be_set", "awaits_triage", "no_contradiction_pass", "done", "notes"].join(",")];
const stat = { withRulings: 0, rulingLines: 0, seqDiffers: 0, flagsLive: 0, flagsBinary: 0, noPass: 0, nothing: 0 };
for (const id of allIds) {
const r = liveById.get(id) ?? {};
const rl = rulings.get(id) ?? [];
const fp = flagProposals.filter((p) => Number(p.id) === id);
const sq = seqs[String(id)];
const sequence: string[] = sq?.sequence ?? [];
const attempted: string[] = sq?.attempted ?? [];
const seqDiffers = sequence.length > 0 && !(sequence.length === 1 && sequence[0] === r.scenario);
const nContra = (batchOf.get(id)?.contradictions ?? []).length;
const binary = fp.filter((p) => p.proposed === 0 || p.proposed === 1);
const liveDone = binary.filter((p) => r[p.flag] === p.proposed);
const noPass = exportIds.has(id) && !batchOf.has(id);
if (rl.length) stat.withRulings++;
stat.rulingLines += rl.length;
if (seqDiffers) stat.seqDiffers++;
stat.flagsBinary += binary.length;
stat.flagsLive += liveDone.length;
if (noPass) stat.noPass++;
if (!rl.length && !fp.length && !seqDiffers && !STRUCTURAL[id]) stat.nothing++;
md.push(`## case ${id} | ${one(r.victim) || "(not in live pull)"} | ${one(r.country)} | ${one(r.date)}`);
if (!exportIds.has(id)) md.push("- NEW in the live database since the 30 September export: no dossier, verified summary or sequence yet.");
if (STRUCTURAL[id]) md.push(`- STRUCTURAL: ${STRUCTURAL[id]}`);
if (BURGLARY.has(id)) md.push("- NEW LABEL: Burglary (owner, 6 October). Set it once the database offers the value.");
if (UNLABELLED.has(id)) md.push("- KEPT by the owner (6 October); label still to be set. Leave `scenario` as it is.");
if (untriaged.has(id)) md.push(`- AWAITS TRIAGE: rewritten in batch ${lastBatch} from new sources; its ${nContra} contradictions have no ruling yet.`);
if (noPass) md.push("- No contradiction pass: summary verified by the older method, record fields not compared with the articles.");
md.push(`- Scenario in live: ${one(r.scenario) || "(none)"} | verified sequence: ${sequence.join(" → ") || "(none)"}${attempted.length ? ` | attempted: ${attempted.join(", ")}` : ""}${seqDiffers ? " | DIFFERS" : ""}`);
if (rl.length) {
md.push("- Rulings:");
for (const x of rl) md.push(` - [${x.section}] ${one(x.text)}`);
} else md.push("- Rulings: none on file.");
if (fp.length) {
md.push("- Flag proposals (evidence; a ruling above overrides):");
for (const p of fp) md.push(` - \`${p.flag}\` record ${p.record} | proposed ${p.proposed} | live ${r[p.flag] ?? "null"}${r[p.flag] === p.proposed ? " (already in live)" : ""} | ${one(p.evidence)}`);
}
md.push(`- Contradictions on file: ${nContra}${nContra ? " (see contradictions-current.md)" : ""}`, "");
rows.push([id, csv(r.victim), csv(r.country), csv(r.date), csv(r.scenario), csv(sequence.join(" > ")), seqDiffers ? "yes" : "no", rl.length, fp.length, liveDone.length, nContra, csv(STRUCTURAL[id] ?? ""), UNLABELLED.has(id) ? "yes" : "no", untriaged.has(id) ? "yes" : "no", noPass ? "yes" : "no", "", ""].join(","));
}
writeFileSync("WORKLIST-BY-CASE.md", md.join("\n"));
writeFileSync("worklist-tracker.csv", rows.join("\n") + "\n");
const unknown = [...rulings.keys()].filter((id) => !allIds.includes(id));
console.log(JSON.stringify({ cases: allIds.length, lastBatch, awaitingTriage: untriaged.size, ...stat, flagProposals: flagProposals.length, rulingIdsNotInDb: unknown }));