Archived
chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
// Build the per-case worklist for the handover to Sofia:
|
||||
// WORKLIST-BY-CASE.md every ruling line, flag proposal and sequence, grouped by DB id
|
||||
// worklist-tracker.csv one row per case, to tick off
|
||||
// Usage (from merge-output/): bun make_handover_worklist.ts ../attacks-live-api-YYYY-MM-DD.json
|
||||
import { readFileSync, readdirSync, writeFileSync } from "fs";
|
||||
|
||||
const livePath = process.argv[2];
|
||||
if (!livePath) throw new Error("usage: bun make_handover_worklist.ts <live api snapshot.json>");
|
||||
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
||||
const live: any[] = J(livePath);
|
||||
const liveById = new Map<number, any>(live.map((r) => [Number(r.id), r]));
|
||||
const exportIds = new Set<number>(J("../attacks-export-Gart-website.json").map((r: any) => Number(r.id)));
|
||||
const seqs: Record<string, any> = J("scenario-sequences.json");
|
||||
const flagProposals: any[] = J("batches/flag-proposals.json");
|
||||
|
||||
// latest batch version of every case (part files such as batch-35a.json are skipped)
|
||||
const batchOf = new Map<number, any>();
|
||||
for (const f of readdirSync("batches").filter((f) => /^batch-\d+\.json$/.test(f)).sort())
|
||||
for (const c of J("batches/" + f)) batchOf.set(Number(c.id), { ...c, _batch: f.slice(6, 8) });
|
||||
const lastBatch = [...batchOf.values()].reduce((m, c) => (c._batch > m ? c._batch : m), "00");
|
||||
const untriaged = new Set<number>([...batchOf.values()].filter((c) => c._batch === lastBatch).map((c) => Number(c.id)));
|
||||
|
||||
// ruling lines from corrections-approved.md, grouped by DB id
|
||||
const rulings = new Map<number, { section: string; text: string }[]>();
|
||||
const add = (id: number, section: string, text: string) => {
|
||||
if (!rulings.has(id)) rulings.set(id, []);
|
||||
rulings.get(id)!.push({ section, text: text.trim() });
|
||||
};
|
||||
const ids = (s: string) => (s.match(/\d+/g) ?? []).map(Number);
|
||||
let section = "";
|
||||
let sectionId: number | null = null;
|
||||
let open: { ids: number[]; section: string; text: string } | null = null;
|
||||
const flush = () => {
|
||||
if (open) for (const id of open.ids) add(id, open.section, open.text);
|
||||
open = null;
|
||||
};
|
||||
for (const line of readFileSync("corrections-approved.md", "utf8").split("\n")) {
|
||||
if (line.startsWith("## ")) {
|
||||
flush();
|
||||
section = line.slice(3).trim();
|
||||
const m = section.match(/^DB id (\d+)\b/);
|
||||
sectionId = m ? Number(m[1]) : null;
|
||||
if (sectionId !== null) open = { ids: [sectionId], section, text: "" };
|
||||
continue;
|
||||
}
|
||||
if (line.trim() === "---") { flush(); sectionId = null; continue; }
|
||||
if (sectionId !== null && open) { open.text += line + "\n"; continue; }
|
||||
const bullet = line.match(/^- \*\*DB ids? ([^*]+)\*\*/);
|
||||
if (bullet) { flush(); open = { ids: ids(bullet[1].replace(/\(.*$/, "")), section, text: line.slice(2) }; continue; }
|
||||
const set = line.match(/Set `kidnappings` to (\d) on ([^*]+)\*\*/);
|
||||
if (set) {
|
||||
flush();
|
||||
for (const id of ids(set[2].replace(/\([^)]*\)/g, ""))) add(id, section, `kidnappings → ${set[1]} (detention flag list)`);
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("- ") || line.trim() === "") { flush(); continue; }
|
||||
if (open) open.text += " " + line.trim();
|
||||
}
|
||||
flush();
|
||||
|
||||
const BURGLARY = new Set([3, 6, 10, 54, 57, 62, 66, 71, 101, 105, 112, 158, 198, 21, 349, 99, 383, 449, 450]); // owner ruling 2026-10-06
|
||||
const UNLABELLED = new Set([24, 195]); // kept by the owner, label still to be set
|
||||
const STRUCTURAL: Record<number, string> = {
|
||||
386: "delete after moving its sources to 295", 295: "receives the sources of 386",
|
||||
446: "merge into 445, then delete", 445: "receives 446; date 10 to 11 May 2026",
|
||||
440: "merge into 439, then delete", 441: "merge into 439, then delete", 439: "receives 440 and 441",
|
||||
49: "split: new record for 21 August 2018", 38: "split into Irving and Mesquite", 5: "rewrite to the 27 May 2015 robbery",
|
||||
430: "link to 138, 145, 156 (after rebuild)", 138: "link to 430 (after rebuild)", 145: "link to 430 (after rebuild)",
|
||||
156: "link to 430 (after rebuild)", 106: "link to 124 (after rebuild)", 124: "link to 106 (after rebuild)",
|
||||
45: "remove from the database (owner, 6 October)",
|
||||
476: "replace the wrong AseanNow URL", 103: "replace the outlet credit", 458: "new Gardanne record pending La Provence text",
|
||||
};
|
||||
|
||||
const allIds = [...new Set([...exportIds, ...liveById.keys()])].sort((a, b) => a - b);
|
||||
const one = (s: any) => String(s ?? "").replace(/\s+/g, " ").trim();
|
||||
const csv = (v: any) => `"${one(v).replace(/"/g, '""')}"`;
|
||||
const md: string[] = [
|
||||
"# Worklist by case",
|
||||
"",
|
||||
`Generated from corrections-approved.md, batches/flag-proposals.json, scenario-sequences.json and the live API pull ${livePath.replace(/^.*\//, "")}.`,
|
||||
"Ruling lines are copied verbatim, oldest first. Where two lines on one case disagree, the later one wins.",
|
||||
"",
|
||||
];
|
||||
const rows = [["id", "victim", "country", "date", "live_scenario", "final_sequence", "sequence_differs", "ruling_lines", "flag_proposals", "flags_already_live", "contradictions", "structural_operation", "label_to_be_set", "awaits_triage", "no_contradiction_pass", "done", "notes"].join(",")];
|
||||
const stat = { withRulings: 0, rulingLines: 0, seqDiffers: 0, flagsLive: 0, flagsBinary: 0, noPass: 0, nothing: 0 };
|
||||
|
||||
for (const id of allIds) {
|
||||
const r = liveById.get(id) ?? {};
|
||||
const rl = rulings.get(id) ?? [];
|
||||
const fp = flagProposals.filter((p) => Number(p.id) === id);
|
||||
const sq = seqs[String(id)];
|
||||
const sequence: string[] = sq?.sequence ?? [];
|
||||
const attempted: string[] = sq?.attempted ?? [];
|
||||
const seqDiffers = sequence.length > 0 && !(sequence.length === 1 && sequence[0] === r.scenario);
|
||||
const nContra = (batchOf.get(id)?.contradictions ?? []).length;
|
||||
const binary = fp.filter((p) => p.proposed === 0 || p.proposed === 1);
|
||||
const liveDone = binary.filter((p) => r[p.flag] === p.proposed);
|
||||
const noPass = exportIds.has(id) && !batchOf.has(id);
|
||||
if (rl.length) stat.withRulings++;
|
||||
stat.rulingLines += rl.length;
|
||||
if (seqDiffers) stat.seqDiffers++;
|
||||
stat.flagsBinary += binary.length;
|
||||
stat.flagsLive += liveDone.length;
|
||||
if (noPass) stat.noPass++;
|
||||
if (!rl.length && !fp.length && !seqDiffers && !STRUCTURAL[id]) stat.nothing++;
|
||||
|
||||
md.push(`## case ${id} | ${one(r.victim) || "(not in live pull)"} | ${one(r.country)} | ${one(r.date)}`);
|
||||
if (!exportIds.has(id)) md.push("- NEW in the live database since the 30 September export: no dossier, verified summary or sequence yet.");
|
||||
if (STRUCTURAL[id]) md.push(`- STRUCTURAL: ${STRUCTURAL[id]}`);
|
||||
if (BURGLARY.has(id)) md.push("- NEW LABEL: Burglary (owner, 6 October). Set it once the database offers the value.");
|
||||
if (UNLABELLED.has(id)) md.push("- KEPT by the owner (6 October); label still to be set. Leave `scenario` as it is.");
|
||||
if (untriaged.has(id)) md.push(`- AWAITS TRIAGE: rewritten in batch ${lastBatch} from new sources; its ${nContra} contradictions have no ruling yet.`);
|
||||
if (noPass) md.push("- No contradiction pass: summary verified by the older method, record fields not compared with the articles.");
|
||||
md.push(`- Scenario in live: ${one(r.scenario) || "(none)"} | verified sequence: ${sequence.join(" → ") || "(none)"}${attempted.length ? ` | attempted: ${attempted.join(", ")}` : ""}${seqDiffers ? " | DIFFERS" : ""}`);
|
||||
if (rl.length) {
|
||||
md.push("- Rulings:");
|
||||
for (const x of rl) md.push(` - [${x.section}] ${one(x.text)}`);
|
||||
} else md.push("- Rulings: none on file.");
|
||||
if (fp.length) {
|
||||
md.push("- Flag proposals (evidence; a ruling above overrides):");
|
||||
for (const p of fp) md.push(` - \`${p.flag}\` record ${p.record} | proposed ${p.proposed} | live ${r[p.flag] ?? "null"}${r[p.flag] === p.proposed ? " (already in live)" : ""} | ${one(p.evidence)}`);
|
||||
}
|
||||
md.push(`- Contradictions on file: ${nContra}${nContra ? " (see contradictions-current.md)" : ""}`, "");
|
||||
|
||||
rows.push([id, csv(r.victim), csv(r.country), csv(r.date), csv(r.scenario), csv(sequence.join(" > ")), seqDiffers ? "yes" : "no", rl.length, fp.length, liveDone.length, nContra, csv(STRUCTURAL[id] ?? ""), UNLABELLED.has(id) ? "yes" : "no", untriaged.has(id) ? "yes" : "no", noPass ? "yes" : "no", "", ""].join(","));
|
||||
}
|
||||
|
||||
writeFileSync("WORKLIST-BY-CASE.md", md.join("\n"));
|
||||
writeFileSync("worklist-tracker.csv", rows.join("\n") + "\n");
|
||||
const unknown = [...rulings.keys()].filter((id) => !allIds.includes(id));
|
||||
console.log(JSON.stringify({ cases: allIds.length, lastBatch, awaitingTriage: untriaged.size, ...stat, flagProposals: flagProposals.length, rulingIdsNotInDb: unknown }));
|
||||
Reference in New Issue
Block a user