// Assemble the release data for the four-step database from the cleanup's side files: // exports/gart-kr-new-structure-.csv record fields, flags, facts, summary, sources (names withheld) // steps/merged.json the two-reader four-step pass // fields/merged.json, fields/not-applied.json written rulings turned into field values // structural-ops.json owner-approved deletions and merges // Writes into /release: data/cases.json, data/labels.json, data/admin-points.json, // export/gart-kr-database-.csv, export/admin-decisions-.csv and .md. // Rebuilds everything from the inputs on every run and writes to none of them. // Usage (from merge-output/): bun make_release_data.ts exports/gart-kr-new-structure-YYYY-MM-DD.csv import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs"; import { homedir } from "os"; import { join } from "path"; import { pointKey } from "./triage_pass"; import { csvCell, withhold } from "./withheld"; const basePath = process.argv[2]; if (!basePath) throw new Error("usage: bun make_release_data.ts "); const SITE = process.env.GART_STATS_SITE ?? join(homedir(), "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"); const OUT = join(SITE, "release"); const dataDate = /(\d{4}-\d{2}-\d{2})/.exec(basePath)?.[1] ?? "undated"; const buildDate = new Date().toLocaleDateString("sv"); // local date, ISO form const J = (p: string) => JSON.parse(readFileSync(p, "utf8")); const parseCsv = (text: string): string[][] => { const rows: string[][] = []; let row: string[] = [], field = "", quoted = false; for (let i = 0; i < text.length; i++) { const c = text[i]; if (quoted) { if (c === '"' && text[i + 1] === '"') (field += '"'), i++; else if (c === '"') quoted = false; else field += c; } else if (c === '"') quoted = true; else if (c === ",") row.push(field), (field = ""); else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = ""); else field += c; } if (quoted) throw new Error("unterminated quoted field"); if (field || row.length) row.push(field), rows.push(row); return rows; }; const table = (path: string) => { const [head, ...body] = parseCsv(readFileSync(path, "utf8").replace(/^/, "")); for (const r of body) if (r.length !== head.length) throw new Error(`${path}: case ${r[0]} has ${r.length} fields, expected ${head.length}`); return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record); }; const LABELS = J("step-labels.json"); const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"] as const; const records = table(basePath); const read = new Map(J("steps/merged.json").map((c: any) => [c.id, c])); const fieldChanges = new Map(existsSync("fields/merged.json") ? J("fields/merged.json").map((c: any) => [c.id, c.changes]) : []); const fieldNotApplied: any[] = existsSync("fields/not-applied.json") ? J("fields/not-applied.json") : []; // the field pass must be merged and current: a stale or partial merge would drop corrections without a trace const fieldBatches = existsSync("fields") ? readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)) : []; if (fieldBatches.length) { if (!existsSync("fields/merged.json")) throw new Error("fields/merged.json is missing: run `bun fields_pass.ts merge` first"); const mergedAt = statSync("fields/merged.json").mtimeMs; for (const f of readdirSync("fields").filter((f) => /^(out|check)-\d+\.json$/.test(f))) if (statSync(join("fields", f)).mtimeMs > mergedAt) throw new Error(`fields/${f} is newer than fields/merged.json: run \`bun fields_pass.ts merge\` again`); } const fieldPassIds = new Set(fieldBatches.flatMap((f) => J(join("fields", f)).map((c: any) => c.id))); const ops = J("structural-ops.json"); const withheldIds = new Set(Object.keys(J("names-withheld.json")).filter((k) => /^\d+$/.test(k)).map(Number)); // ---- dates ------------------------------------------------------------- const MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"]; const pad = (n: number) => String(n).padStart(2, "0"); // "March 8, 2026" | "February, 2015" | "January ?, 2020" | "May 10 to 11, 2026" | "2018" const parseDate = (text: string): { sort: string | null; precision: string } => { const twoMonths = /^(?:late|early|mid)[- ]([A-Za-z]+) or (?:late|early|mid)[- ][A-Za-z]+,? (\d{4})$/i.exec(text.trim()); if (twoMonths && MONTHS.includes(twoMonths[1].toLowerCase())) return { sort: `${twoMonths[2]}-${pad(MONTHS.indexOf(twoMonths[1].toLowerCase()) + 1)}-01`, precision: "year" }; const t = text.trim().replace(/^(?:late|early|mid)[- ]/i, ""); let m = /^([A-Za-z]+) (\d{1,2})(?: ?(?:to|-) ?(\d{1,2}))?, (\d{4})$/.exec(t); if (m && MONTHS.includes(m[1].toLowerCase())) { const month = MONTHS.indexOf(m[1].toLowerCase()) + 1, day = Number(m[2]), last = m[3] ? Number(m[3]) : day; const daysInMonth = new Date(Number(m[4]), month, 0).getDate(); // a day the calendar does not have, or a span that runs backwards, is not a date if (day < 1 || last > daysInMonth || last < day) return { sort: null, precision: "unknown" }; return { sort: `${m[4]}-${pad(month)}-${pad(day)}`, precision: m[3] ? "range" : "day" }; } m = /^([A-Za-z]+)(?: \?)?,? (\d{4})$/.exec(t); if (m && MONTHS.includes(m[1].toLowerCase())) return { sort: `${m[2]}-${pad(MONTHS.indexOf(m[1].toLowerCase()) + 1)}-01`, precision: "month" }; m = /^(\d{4})$/.exec(t); if (m) return { sort: `${m[1]}-01-01`, precision: "year" }; return { sort: null, precision: "unknown" }; }; // ---- cases ------------------------------------------------------------- const FLAG = { "1": "yes", "0": "no", unconfirmed: "unconfirmed", "threat only": "threat only", "": "" } as Record; const flag = (id: string, name: string, v: string) => { if (!(v in FLAG)) throw new Error(`case ${id}: flag ${name} has value "${v}"`); return FLAG[v]; }; const admin: Record[] = []; const point = (id: number | string, kind: string, detail: string, extra: Partial> = {}) => admin.push({ case_id: String(id), kind, field: "", detail, first_value: "", second_value: "", quote: "", evidence_file: "", ...extra }); const cases = new Map(); for (const r of records) { const id = Number(r.id); const p = read.get(id); if (!p) throw new Error(`case ${id}: no four-step reading in steps/merged.json`); const sources = Array.from({ length: 8 }, (_, i) => r[`source_${i + 1}`]).filter(Boolean); const c: any = { id, date: r.date, victim: r.victim, location: r.location, country: r.country, description: r.description, summary: r.summary, notes: r.notes, reports: r.reports, money_wanted: r.money_wanted, coin_type: r.coin_type, flags: { detention: flag(r.id, "detention", r.detention), violence_torture: flag(r.id, "violence_torture", r.violence_torture), drugs_alcohol: flag(r.id, "drugs_alcohol", r.drugs_alcohol), weapons: flag(r.id, "weapons", r.weapons), theft: flag(r.id, "theft", r.theft), }, life_taken: /^\d+$/.test(r.life_taken) ? Number(r.life_taken) : null, life_taken_unconfirmed: r.life_taken === "unconfirmed", venue_of_control: r.venue_of_control, held_duration: r.held_duration, arrests: r.arrests, steps: Object.fromEntries(STEPS.map((s) => [s, p[s]])), demanded: p.demanded, taken: p.taken, sources, linked_cases: r.linked_cases ? r.linked_cases.split(" | ").map(Number) : [], verification: r.verification, second_reader: p.second_reader, name_withheld: withheldIds.has(id), v1_scenario: r.v1_scenario, old_labels: [r.scenario_1, r.scenario_2, r.scenario_3].filter(Boolean), former_ids: [] as number[], dossier_file: r.dossier_file, internal: { evidence: p.evidence, open_points: p.open_points ?? 0, field_changes: [] as any[] }, }; // text the readers wrote also goes through the withheld-name list for (const k of Object.keys(c.internal.evidence)) c.internal.evidence[k] = withhold(id, "evidence", String(c.internal.evidence[k])); for (const m of [c.demanded, c.taken]) if (m?.as_stated) m.as_stated = withhold(id, "as_stated", m.as_stated); cases.set(id, c); } // 1. field rulings, confirmed by the second reader let changed = 0; const stale: any[] = []; for (const [id, changes] of fieldChanges) { const c = cases.get(id); if (!c) continue; for (const ch of changes) { if (!(ch.field in c)) throw new Error(`case ${id}: field ${ch.field} is not a record field`); // the change was read against a record value; if the record has moved since, a person decides if (String(ch.old ?? "").replace(/\s+/g, " ").trim() !== String(c[ch.field] ?? "").replace(/\s+/g, " ").trim()) { stale.push({ id, field: ch.field, proposed: ch.new, quote: ch.quote, record: c[ch.field] }); continue; } const value = withhold(id, ch.field, ch.new); c.internal.field_changes.push({ field: ch.field, old: c[ch.field], new: value, quote: ch.quote }); c[ch.field] = value; changed++; } } // 2. merges, then 3. deletions (owner-approved list only) const before = cases.size; const removed: number[] = []; for (const m of ops.merge) { const into = cases.get(m.into); if (!into) throw new Error(`merge target ${m.into} missing`); for (const from of m.from) { const src = cases.get(from); if (!src) throw new Error(`merge source ${from} missing`); if (m.move_sources) for (const u of src.sources) if (!into.sources.includes(u)) into.sources.push(u); for (const l of src.linked_cases) if (!into.linked_cases.includes(l)) into.linked_cases.push(l); into.former_ids.push(from); cases.delete(from); removed.push(from); point(from, "structural: applied in this build only", `Merged into case ${m.into} here (${m.reason}). The live database still holds it as a separate record.`, { evidence_file: "merge-output/structural-ops.json" }); } if (m.set) for (const [k, v] of Object.entries(m.set)) (into.internal.field_changes.push({ field: k, old: into[k], new: v, quote: `structural-ops.json: ${m.reason}` }), (into[k] = v)); if (m.append_note) into.notes = [into.notes, m.append_note].filter(Boolean).join(" "); } for (const d of ops.delete) { if (!cases.delete(d.id)) throw new Error(`delete target ${d.id} missing`); removed.push(d.id); point(d.id, "structural: applied in this build only", `Deleted here (${d.reason}). The live database still holds it.`, { evidence_file: "merge-output/structural-ops.json" }); } if (cases.size !== before - removed.length) throw new Error("case count does not reconcile after merges and deletions"); // links must not point at a removed record const alias = new Map(ops.merge.flatMap((m: any) => m.from.map((f: number) => [f, m.into]))); for (const c of cases.values()) c.linked_cases = [...new Set(c.linked_cases.map((x: number) => alias.get(x) ?? x))].filter((x) => x !== c.id && cases.has(x as number)); // Ruled splits and rewrites carried out from the dossiers (splits/out-.json, owner rulings of 2026-10-01 // executed on 2026-10-08). The record that keeps the id is overwritten; a new record gets the id 10000 + the // parent's id, which the live database cannot hand out, and the two are linked. const SPLIT_FLAGS = ["detention", "violence_torture", "drugs_alcohol", "weapons", "theft"]; const splitIds = new Set(); const applySplitRecord = (c: any, r: any) => { // an earlier change to a field that the split now rewrites is no longer the field's last word c.internal.field_changes = c.internal.field_changes.filter((ch: any) => typeof r[ch.field] !== "string"); for (const f of ["date", "victim", "location", "country", "description", "summary", "notes", "money_wanted", "coin_type"]) { if (typeof r[f] !== "string" || r[f] === c[f]) continue; if (f !== "summary") c.internal.field_changes.push({ field: f, old: c[f], new: r[f], quote: "split or rewrite ordered by the owner, written from the dossier on 2026-10-08" }); c[f] = r[f]; } for (const f of SPLIT_FLAGS) c.flags[f] = r[f]; c.life_taken_unconfirmed = r.life_taken === "unconfirmed"; c.life_taken = r.life_taken === "unconfirmed" ? null : Number(r.life_taken); for (const st of STEPS) c.steps[st] = r[st]; if (Array.isArray(r.sources) && r.sources.length) c.sources = r.sources; c.internal.evidence = Object.fromEntries((r.support ?? []).map((x: any, i: number) => [`split_${i + 1}`, `${x.claim}: ${x.quote}`])); splitIds.add(c.id); }; const newRecords = (existsSync("splits") ? readdirSync("splits").filter((f) => /^out-\d+\.json$/.test(f)) : []).length; for (const f of existsSync("splits") ? readdirSync("splits").filter((f) => /^out-\d+\.json$/.test(f)).sort() : []) { const o = J(join("splits", f)); const parent = cases.get(o.id); if (!parent) throw new Error(`splits/${f}: case ${o.id} is not in the data`); const fresh = JSON.parse(JSON.stringify(parent)); for (const r of o.records) { if (r.keeps_id) { applySplitRecord(parent, r); parent.verification = `rewritten from the dossier on 2026-10-08 to carry out the owner's ruling (${o.records.length > 1 ? "split" : "rewrite"}); one reader, the owner has not reviewed the text`; continue; } const c = { ...fresh, id: 10000 + o.id, former_ids: [], linked_cases: [o.id], old_labels: [], v1_scenario: "", internal: { evidence: {}, open_points: 0, field_changes: [] }, demanded: { amount: null, currency: null, as_stated: "", usd_stated: null }, taken: { amount: null, currency: null, as_stated: "", usd_stated: null }, venue_of_control: "", held_duration: "" }; applySplitRecord(c, r); c.verification = `written from the dossier of record ${o.id} on 2026-10-08, when the owner's ruling split that record; one reader, the owner has not reviewed the text`; cases.set(c.id, c); if (!parent.linked_cases.includes(c.id)) parent.linked_cases.push(c.id); } } // owner rulings on a record field of one case (step-rulings.json): set before names are withheld and dates are read const RECORD_FIELDS = ["date", "victim", "location", "country", "description", "summary", "notes"]; const ruledField = new Set(); // "id:field" the owner has ruled on; open items on it are closed for (const r of (existsSync("step-rulings.json") ? J("step-rulings.json").rulings : []).filter((r: any) => RECORD_FIELDS.includes(r.field))) { const c = cases.get(r.id); if (!c) continue; c.internal.field_changes.push({ field: r.field, old: c[r.field], new: r.value, quote: r.ruling }); c[r.field] = r.value; ruledField.add(`${r.id}:${r.field}`); } // night-shift triage of 2026-10-08: open items settled by precedent. Applied values are defaults the owner can overturn. const triage: { applied: any[]; closed: any[]; parked: any[] } = existsSync("triage/decisions.json") ? J("triage/decisions.json") : { applied: [], closed: [], parked: [] }; if (existsSync("triage/decisions.json")) { const at = statSync("triage/decisions.json").mtimeMs; for (const f of readdirSync("triage").filter((f) => /^(out|check)-\d+\.json$/.test(f))) if (statSync(join("triage", f)).mtimeMs > at) throw new Error(`triage/${f} is newer than triage/decisions.json: run \`bun triage_pass.ts merge\` again`); } const TRIAGE_RECORD = [...RECORD_FIELDS.filter((f) => f !== "summary"), "money_wanted", "coin_type"]; const nightDefaults: any[] = []; for (const t of triage.applied.filter((t) => TRIAGE_RECORD.includes(t.field))) { const c = cases.get(t.id); if (!c || splitIds.has(t.id) || ruledField.has(`${t.id}:${t.field}`)) continue; const value = t.op === "append" ? [c[t.field], t.value].filter(Boolean).join(" ") : t.value; c.internal.field_changes.push({ field: t.field, old: c[t.field], new: value, quote: `night-shift default (precedent ${t.precedent}): ${t.quote}` }); c[t.field] = value; nightDefaults.push(t); } // cases that arrived after the verification, compared with their own sources on 2026-10-08 (newcases/): // a value is taken when the first reader supports it with a passage of the fetched text and the second reader confirms it const newCaseValues = new Map>(); const normText = (t: unknown) => String(t ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim(); for (const f of existsSync("newcases") ? readdirSync("newcases").filter((f) => /^out-\d+\.json$/.test(f)) : []) { const o = J(join("newcases", f)); const checkPath = join("newcases", f.replace("out-", "check-")); const c = cases.get(o.id); if (!c || !existsSync(checkPath) || statSync(checkPath).mtimeMs <= statSync(join("newcases", f)).mtimeMs) continue; const verdicts: Record = J(checkPath).verdicts ?? {}; const text = normText(existsSync(`newcases/sources-${o.id}.txt`) ? readFileSync(`newcases/sources-${o.id}.txt`, "utf8") : ""); const taken: Record = {}; for (const [field, value] of Object.entries(o.values ?? {})) { if (verdicts[field] !== "confirm") continue; const quote = normText(o.support?.[field]); // a changed value needs its passage in the fetched text; the summary is checked as a whole by the second reader if ((o.changed ?? []).includes(field) && field !== "summary" && field !== "description" && (quote.length < 8 || !text.includes(quote))) continue; taken[field] = value; } newCaseValues.set(o.id, taken); const reasons: Record = J(checkPath).reasons ?? {}; for (const field of Object.keys(o.values ?? {}).filter((k) => !(k in taken))) point(o.id, "new case: value not confirmed", `Compared with the fetched sources on 2026-10-08. The first reader proposed a new ${field}; the second reader did not confirm it: ${reasons[field] ?? "no reason given"} The record's value stays.`, { field, second_value: typeof o.values[field] === "string" ? o.values[field] : JSON.stringify(o.values[field]), evidence_file: `merge-output/newcases/out-${o.id}.json` }); if ((o.not_fetched ?? []).length) point(o.id, "new case: source not read", `These sources of the record could not be fetched, so details that rest on them are unchecked: ${o.not_fetched.join("; ")}`, { evidence_file: `merge-output/newcases/sources-${o.id}.txt` }); for (const field of RECORD_FIELDS.filter((x) => x !== "notes")) { if (!(field in taken) || ruledField.has(`${o.id}:${field}`) || taken[field] === c[field]) continue; c.internal.field_changes.push({ field, old: c[field], new: taken[field], quote: `compared with ${(o.fetched ?? []).length} fetched sources, 2026-10-08 (night-shift default)` }); c[field] = taken[field]; } c.verification = `fields, flags and steps compared with ${(o.fetched ?? []).length} fetched sources on 2026-10-08 by two readers (night shift; the owner has not reviewed it)`; if (o.notes) c.notes = [c.notes, `Verification 2026-10-08: ${o.notes}`].filter(Boolean).join(" "); } // wording fixes to verified summaries that rulings ordered (rewrites/summary-fixes.json): exact passages replaced let summaryFixes = 0; for (const f of existsSync("rewrites/summary-fixes.json") ? J("rewrites/summary-fixes.json") : []) { const c = cases.get(f.id); if (!c || ruledField.has(`${f.id}:summary`)) continue; for (const e of f.edits) { if (c.summary.split(e.old).length !== 2) throw new Error(`rewrites/summary-fixes.json: case ${f.id}: the passage to replace is not found exactly once in the summary`); c.summary = c.summary.replace(e.old, () => e.new); c.internal.field_changes.push({ field: "summary", old: e.old, new: e.new, quote: `ruling wording fix: ${f.support ?? f.note ?? ""}` }); summaryFixes++; } } // names withheld at a victim's request: applied to the final text of every field, whatever the inputs carried const TEXT_FIELDS = ["victim", "location", "description", "summary", "notes", "reports", "money_wanted", "coin_type"]; for (const c of cases.values()) { if (!c.name_withheld) continue; for (const f of TEXT_FIELDS) c[f] = withhold(c.id, f, String(c[f] ?? "")); for (const ch of c.internal.field_changes) ch.new = withhold(c.id, ch.field, String(ch.new ?? "")); } // dates after all changes for (const c of cases.values()) { const d = parseDate(c.date); c.date_sort = d.sort; c.date_precision = d.precision; c.year = d.sort ? Number(d.sort.slice(0, 4)) : null; c.month = d.sort && d.precision !== "year" ? Number(d.sort.slice(5, 7)) : null; c.quarter = c.year && c.month ? `${c.year}-Q${Math.ceil(c.month / 3)}` : null; if (!d.sort) point(c.id, "date not readable", `The date "${c.date}" could not be turned into a sortable date.`, { field: "date" }); } // the owner's recheck of 2026-10-07 (who was attacked, outcome "fought off"): second-reader-confirmed changes if (existsSync("recheck/merged.json")) { const mergedAt = statSync("recheck/merged.json").mtimeMs; for (const f of readdirSync("recheck").filter((f) => /^(out|check)-\d+\.json$/.test(f))) if (statSync(join("recheck", f)).mtimeMs > mergedAt) throw new Error(`recheck/${f} is newer than recheck/merged.json: run \`bun recheck_pass.ts merge\` again`); } const recheck: any[] = existsSync("recheck/merged.json") ? J("recheck/merged.json") : []; const recheckNotApplied: any[] = existsSync("recheck/not-applied.json") ? J("recheck/not-applied.json") : []; let rechecked = 0; for (const r of recheck) { const c = cases.get(r.id); if (!c) continue; for (const ch of r.changes) { // read against a value that has since moved: leave it to a person if (JSON.stringify([c.steps[ch.field]].flat().sort()) !== JSON.stringify([ch.old].flat().sort())) { recheckNotApplied.push({ id: r.id, kind: "recheck: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; } c.steps[ch.field] = ch.new; rechecked++; } } // the full label check of 2026-10-07 (every label tested against LABEL-RULES.md): confirmed changes if (existsSync("labels/merged.json")) { const mergedAt = statSync("labels/merged.json").mtimeMs; for (const f of readdirSync("labels").filter((f) => /^(out|check)-\d+\.json$/.test(f) || f === "tiebreak-out.json")) if (statSync(join("labels", f)).mtimeMs > mergedAt) throw new Error(`labels/${f} is newer than labels/merged.json: run \`bun labels_pass.ts merge\` again`); } const labelCheck: any[] = existsSync("labels/merged.json") ? J("labels/merged.json") : []; const labelNotApplied: any[] = existsSync("labels/not-applied.json") ? J("labels/not-applied.json") : []; let labelsChanged = 0; for (const r of labelCheck) { const c = cases.get(r.id); if (!c || splitIds.has(r.id)) continue; for (const ch of r.changes) { if (JSON.stringify([c.steps[ch.field]].flat().sort()) !== JSON.stringify([ch.old].flat().sort())) { labelNotApplied.push({ id: r.id, kind: "label check: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; } c.steps[ch.field] = ch.new; c.internal.evidence[`label_check_${ch.field}`] = ch.quote; labelsChanged++; } } // Cases whose seven step fields went through the full label check (two readers, a third on disagreement). // That check supersedes older open questions on the same fields: the step pass, the recheck, Dodged Attack against foiled. const labelChecked = new Set(); if (existsSync("labels/merged.json")) for (const f of readdirSync("labels").filter((f) => /^in-\d+\.json$/.test(f))) { const out = join("labels", f.replace("in-", "out-")), check = join("labels", f.replace("in-", "check-")); if (!existsSync(out) || !existsSync(check) || statSync(check).mtimeMs <= statSync(out).mtimeMs) continue; // not second-read const read = new Set(J(out).map((o: any) => o.id)); for (const c of J(join("labels", f))) if (read.has(c.id)) labelChecked.add(c.id); } const olderRecheck = recheckNotApplied.splice(0).filter((n) => !labelChecked.has(n.id)); recheckNotApplied.push(...olderRecheck, ...labelNotApplied); // the flag check of 2026-10-07 (violence flags and deaths tested against FLAG-RULES.md): confirmed changes if (existsSync("flags/merged.json")) { const mergedAt = statSync("flags/merged.json").mtimeMs; for (const f of readdirSync("flags").filter((f) => /^(out|check)-\d+\.json$/.test(f))) if (statSync(join("flags", f)).mtimeMs > mergedAt) throw new Error(`flags/${f} is newer than flags/merged.json: run \`bun flags_pass.ts merge\` again`); } const flagCheck: any[] = existsSync("flags/merged.json") ? J("flags/merged.json") : []; const flagNotApplied: any[] = existsSync("flags/not-applied.json") ? J("flags/not-applied.json") : []; // unlocked flag values two readers tested and left alone, or changed together const flagChecked = new Set(existsSync("flags/checked.json") ? J("flags/checked.json") : []); let flagsChanged = 0; const deaths = (c: any) => (c.life_taken_unconfirmed ? "unconfirmed" : c.life_taken == null ? "" : String(c.life_taken)); for (const r of flagCheck) { const c = cases.get(r.id); if (!c || splitIds.has(r.id)) continue; for (const ch of r.changes) { const now = ch.field === "life_taken" ? deaths(c) : c.flags[ch.field]; if (now !== ch.old) { flagNotApplied.push({ id: r.id, kind: "flag check: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; } if (ch.field === "life_taken") { c.life_taken_unconfirmed = ch.new === "unconfirmed"; c.life_taken = ch.new === "unconfirmed" ? null : Number(ch.new); } else c.flags[ch.field] = ch.new; c.internal.evidence[`flag_check_${ch.field}`] = ch.quote; flagsChanged++; } } for (const t of triage.applied.filter((t) => !TRIAGE_RECORD.includes(t.field))) { const c = cases.get(t.id); if (!c) continue; if (t.field === "life_taken") { c.life_taken_unconfirmed = t.value === "unconfirmed"; c.life_taken = t.value === "unconfirmed" ? null : Number(t.value); } else if (t.field in c.flags) c.flags[t.field] = t.value; else throw new Error(`triage/decisions.json: case ${t.id} has no field ${t.field}`); c.internal.evidence[`night_shift_${t.field}`] = t.quote; nightDefaults.push(t); } for (const [id, taken] of newCaseValues) { const c = cases.get(id); for (const [field, value] of Object.entries(taken)) { if (field === "life_taken") { c.life_taken_unconfirmed = value === "unconfirmed"; c.life_taken = value === "unconfirmed" ? null : Number(value); } else if (field in c.flags) { if (["yes", "no", "unconfirmed", "threat only"].includes(value)) c.flags[field] = value; } else if (field in c.steps) c.steps[field] = value; // the vocabulary check below rejects an unknown label } } // owner rulings on a step value of one case const stepRulings: any[] = existsSync("step-rulings.json") ? J("step-rulings.json").rulings : []; const ruledStep = new Set(); for (const r of stepRulings) { const c = cases.get(r.id); if (!c) continue; if (RECORD_FIELDS.includes(r.field)) continue; // set earlier, before the dates were read if (r.field === "life_taken") { c.life_taken_unconfirmed = r.value === "unconfirmed"; c.life_taken = r.value === "unconfirmed" ? null : Number(r.value); } else if (r.field in c.flags) { if (!["yes", "no", "unconfirmed", "threat only"].includes(r.value)) throw new Error(`step-rulings.json: case ${r.id}: "${r.value}" is not a flag value`); c.flags[r.field] = r.value; } else if (r.field in c.steps) c.steps[r.field] = r.value; else throw new Error(`step-rulings.json: case ${r.id} has no field ${r.field}`); ruledStep.add(`${r.id}:${r.field}`); } // label check: nothing outside the vocabulary, nothing guessed for (const c of cases.values()) for (const s of STEPS) { const values: string[] = LABELS[s].several ? c.steps[s] : [c.steps[s]]; if (!values.length) throw new Error(`case ${c.id}: step ${s} is empty`); for (const v of values) if (!Object.hasOwn(LABELS[s].values, v)) throw new Error(`case ${c.id}: "${v}" is not a ${s} label`); } // ---- admin points -------------------------------------------------------- const live = (id: number) => cases.has(id); for (const c of cases.values()) { const dodged = c.old_labels.includes("Dodged Attack"); const foiled = c.steps.outcome === "foiled"; if (ruledStep.has(`${c.id}:outcome`) || labelChecked.has(c.id)) { // the owner has ruled on this outcome, or the label check tested it against the owner's rule for foiled } else if (dodged && !foiled) point(c.id, "foiled or not: owner decision", `Old label Dodged Attack, but the readers found someone was held, hurt or robbed: outcome "${c.steps.outcome}". Decide whether the case counts as foiled.`, { field: "outcome", first_value: c.steps.outcome, second_value: "foiled", quote: c.internal.evidence.outcome, evidence_file: `merge-output/sources/${c.dossier_file}` }); else if (!dodged && foiled) point(c.id, "foiled or not: owner decision", "Read as foiled, but the case has no Dodged Attack label in the verified sequence.", { field: "outcome", first_value: "foiled", quote: c.internal.evidence.outcome, evidence_file: `merge-output/sources/${c.dossier_file}` }); if (c.steps.attack.includes("no label fits")) point(c.id, "no attack label fits", "None of the attack labels describes this case. Decide a label or whether the case stays in the database.", { field: "attack", quote: c.internal.evidence.attack, evidence_file: `merge-output/sources/${c.dossier_file}` }); if (c.verification === "not verified") point(c.id, "not verified", "Arrived after the verification; summary, labels and fields are unchecked. The four steps were read from the site summary only.", { evidence_file: "attacks-live-api-" + dataDate + ".json" }); if (c.verification.startsWith("record only")) point(c.id, "no article on file", "No usable article is on file; every value rests on the record alone.", { evidence_file: "merge-output/sofia-worklist.md" }); if (c.verification.startsWith("summary and labels checked")) point(c.id, "record fields never compared with articles", "Summary and labels were verified by the older method; date, victim, location, amounts and flags were never compared with the articles.", { evidence_file: `merge-output/sources/${c.dossier_file}` }); } for (const p of table("steps/open-points.csv").filter((p) => p.applied === "no" && live(Number(p.id)) && !(labelChecked.has(Number(p.id)) && STEPS.includes(p.field)) && !ruledStep.has(`${p.id}:${p.field}`))) point(p.id, "two readers disagree", `Step value where the second reader prefers another reading (${p.kind}). The first reader's value is in the database.`, { field: p.field, first_value: withhold(p.id, "x", p.first_reader), second_value: withhold(p.id, "x", p.second_reader), quote: withhold(p.id, "x", p.quote), evidence_file: `merge-output/steps/open-points.csv` }); for (const n of fieldNotApplied.filter((n) => live(n.id) && !ruledField.has(`${n.id}:${n.field}`))) point(n.id, `field ruling: ${n.kind}`, n.detail ?? "", { field: n.field ?? "", second_value: n.proposed ?? "", quote: n.quote ?? "", evidence_file: "merge-output/corrections-approved.md" }); for (const n of recheckNotApplied.filter((n) => live(n.id) && !ruledStep.has(`${n.id}:${n.field}`))) point(n.id, n.kind, n.detail ?? "A change proposed in the recheck that was not applied.", { field: n.field ?? "", first_value: JSON.stringify(cases.get(n.id)?.steps[n.field] ?? ""), second_value: typeof n.proposed === "string" ? n.proposed : JSON.stringify(n.proposed ?? ""), quote: n.quote ?? "", evidence_file: "merge-output/recheck/not-applied.json" }); for (const n of flagNotApplied.filter((n) => live(n.id) && !ruledStep.has(`${n.id}:${n.field}`))) point(n.id, n.kind, n.detail ?? "A flag change proposed in the flag check that was not applied.", { field: n.field ?? "", first_value: n.field === "life_taken" ? deaths(cases.get(n.id)) : cases.get(n.id)?.flags[n.field] ?? "", second_value: String(n.proposed ?? ""), quote: n.quote ?? "", evidence_file: "merge-output/flags/not-applied.json" }); for (const f of J("exports/flag-values-from-proposals.json").filter((f: any) => live(f.id) && !flagChecked.has(`${f.id}:${f.flag}`))) point(f.id, "flag value from an unruled proposal", `The ${f.flag} flag follows our proposal "${f.value}"; no ruling confirms it.`, { field: f.flag, second_value: f.value, quote: f.evidence, evidence_file: "merge-output/batches/flag-proposals.json" }); for (const st of stale) if (live(st.id) && !ruledField.has(`${st.id}:${st.field}`)) point(st.id, "field ruling: record changed since the ruling was read", `The record's ${st.field} no longer holds the value the change was read against, so the change was not applied.`, { field: st.field, first_value: st.record, second_value: st.proposed, quote: st.quote, evidence_file: "merge-output/fields/merged.json" }); for (const id of fieldPassIds) if (!fieldChanges.has(id) && live(id) && !fieldNotApplied.some((n) => n.id === id && n.kind === "field pass not run")) point(id, "field ruling: field pass not run", "This case has ruling lines and no result from the field pass.", { evidence_file: "merge-output/WORKLIST-BY-CASE.md" }); for (const n of ops.not_applied) point(n.id, `structural: ${n.kind} not done`, n.detail, { evidence_file: "HANDOVER-SOFI.md" }); const fixed: [number, string, string, string][] = [ [290, "possible duplicate", "Reads as the same Dubai incident as case 296 (Al Nahda, three women, Dh300,000).", "merge-output/steps/out-10.json"], [296, "possible duplicate", "Reads as the same Dubai incident as case 290.", "merge-output/steps/out-10.json"], [476, "possible duplicate", "Sheet row 252 (Phuket, August 2025), listed as missing from the database, looks like this case.", "merge-output/cases-missing-from-database.json"], [476, "source fix", "The AseanNow URL on the record points at the French case 350. Correct thread: https://aseannow.com/topic/1371363-russian-fugitive-caught-hiding-in-car-at-koh-samui-ferry-port/", "merge-output/sofia-worklist.md"], [103, "source fix", 'Replace the "El Pais, Iustel" credit with Madrid Actual, Gacetin Madrid, Vozpopuli (2024) and El Pais (2022).', "merge-output/sofia-worklist.md"], [24, "on hold: owner", "The owner keeps the case; it has no label yet.", "merge-output/disputed-scenarios.md"], [195, "on hold: owner", "The owner keeps the case; it has no label yet.", "merge-output/disputed-scenarios.md"], [481, "on hold: owner", "life_taken stays 5 until the owner confirms that the unborn child counts.", "HANDOVER-SOFI.md"], [458, "on hold: owner", "The Gardanne attack of the same night gets a record only once someone has read the La Provence article.", "HANDOVER-SOFI.md"], [184, "live site still names the victim", "The victim asked not to be named. This build withholds the name; the live record still shows it in the victim field and the summary.", "merge-output/names-withheld.json"], ]; for (const [id, kind, detail, file] of fixed) if (live(id)) point(id, kind, detail, { evidence_file: file }); for (const id of [16, 103, 113, 114, 223, 373, 390, 399, 408, 412, 419, 427, 430, 434, 436, 444, 448, 465, 476, 485]) if (live(id)) point(id, "new contradictions without rulings", "Rewritten from new sources in batch 35; its fresh record-versus-article contradictions have no rulings yet.", { evidence_file: "merge-output/batches/contradictions-35.md" }); point("", "vocabulary not yet approved", 'Labels and fields added during the pass that the owner has not approved in a list: targeting "visible wealth", the field "who was attacked", and the attack labels as applied to approach-only cases.', { evidence_file: "merge-output/step-labels.json" }); point("", "sources not read", "65 links could not be read by the pipeline and wait for a manual fetch.", { evidence_file: "merge-output/sofia-worklist.md" }); point("", "old dashboard not migrated", "The v2 dashboard (index.html, backend_server.py) still reads the old schema and is not part of this release.", { evidence_file: "stats-gart-io-v3/index.html" }); admin.sort((a, b) => (Number(a.case_id) || 1e9) - (Number(b.case_id) || 1e9) || a.kind.localeCompare(b.kind)); // ---- write --------------------------------------------------------------- for (const d of ["data", "export"]) mkdirSync(join(OUT, d), { recursive: true }); const list = [...cases.values()].sort((a, b) => a.id - b.id); const meta = { data_date: dataDate, build_date: buildDate, cases: list.length, live_cases_at_data_date: records.length, removed_ids: removed.sort((a, b) => a - b), field_changes_applied: changed }; writeFileSync(join(OUT, "data/cases.json"), JSON.stringify({ meta, cases: list }, null, 1)); const groupOf = (step: string, name: string) => (step === "approach" && name.includes(": ") ? name.split(": ")[0] : ""); const labelRows = STEPS.flatMap((step) => Object.entries(LABELS[step].values as Record).map(([name, definition], i) => ({ step, name, group: groupOf(step, name), definition, display_order: i + 1 }))); writeFileSync(join(OUT, "data/labels.json"), JSON.stringify(labelRows, null, 1)); { const OLD_DATABASE_ONLY = ["structural: applied in this build only", "live site still names the victim", "old dashboard not migrated"]; const oldOnly = (a: Record) => OLD_DATABASE_ONLY.includes(a.kind) || a.detail.startsWith("merging and deleting records is not a field change"); const settled = new Set([...triage.applied, ...triage.closed].map((t) => t.key)); const parkedBy = new Map(triage.parked.map((t) => [t.key, t])); const byPrecedent = new Set((existsSync("closed-by-precedent.json") ? J("closed-by-precedent.json").closed : []).map((c: any) => [c.case_id, c.kind, c.field ?? ""].join("|"))); const kept = admin.filter((a) => !settled.has(pointKey(a)) && !oldOnly(a) && !byPrecedent.has([a.case_id, a.kind, a.field ?? ""].join("|"))); for (const a of kept) { const p = parkedBy.get(pointKey(a)); if (p) a.detail = [a.detail, `Night shift: ${p.reason}`, p.options && `Options: ${p.options}`, p.default && `Recommended: ${p.default}`].filter(Boolean).join(" | "); } console.log(`summary wording fixes: ${summaryFixes}`); console.log(`triage: ${admin.length - kept.length} open items settled (${nightDefaults.length} values set as night-shift defaults), ${kept.length} open`); admin.length = 0; admin.push(...kept); } writeFileSync(join(OUT, "data/admin-points.json"), JSON.stringify(admin, null, 1)); writeFileSync(join(OUT, "export", `night-shift-defaults-${buildDate}.json`), JSON.stringify(nightDefaults, null, 1)); // a cell that a spreadsheet would run as a formula gets a leading apostrophe const safe = (v: unknown) => { const s = v == null ? "" : String(v); return /^[=+\-@]/.test(s) ? "'" + s : s; }; const csv = (rows: unknown[][]) => "" + rows.map((r) => r.map((v) => csvCell(safe(v))).join(",")).join("\r\n") + "\r\n"; const amount = (m: any, k: string) => (m?.[k] == null ? "" : m[k]); const maxSources = Math.max(...list.map((c) => c.sources.length)); const head = [ "id", "date", "date_sort", "date_precision", "victim", "location", "country", "1_targeting", "1_targeting_accuracy", "1_who_was_attacked", "2_approach", "3_attack", "4_outcome", "4_obtained", "detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken", "demanded_amount", "demanded_currency", "demanded_usd_stated", "demanded_as_stated", "taken_amount", "taken_currency", "taken_usd_stated", "taken_as_stated", "money_wanted", "coin_type", "venue_of_control", "held_duration", "arrests", "description", "summary", "notes", "reports", "verification", "name_withheld", "linked_cases", "former_ids", "old_labels", "fields_changed_by_ruling", "admin_points", ...Array.from({ length: maxSources }, (_, i) => `source_${i + 1}`), ]; const adminCount = new Map(); for (const a of admin) adminCount.set(a.case_id, (adminCount.get(a.case_id) ?? 0) + 1); const rows = list.map((c) => [ c.id, c.date, c.date_sort ?? "", c.date_precision, c.victim, c.location, c.country, c.steps.targeting.join(" | "), c.steps.targeting_accuracy, c.steps.who_was_attacked.join(" | "), c.steps.approach.join(" | "), c.steps.attack.join(" | "), c.steps.outcome, c.steps.obtained.join(" | "), c.flags.detention, c.flags.violence_torture, c.flags.drugs_alcohol, c.flags.weapons, c.flags.theft, c.life_taken_unconfirmed ? "unconfirmed" : (c.life_taken ?? ""), amount(c.demanded, "amount"), amount(c.demanded, "currency"), amount(c.demanded, "usd_stated"), amount(c.demanded, "as_stated"), amount(c.taken, "amount"), amount(c.taken, "currency"), amount(c.taken, "usd_stated"), amount(c.taken, "as_stated"), c.money_wanted, c.coin_type, c.venue_of_control, c.held_duration, c.arrests, c.description, c.summary, c.notes, c.reports, c.verification, c.name_withheld ? "yes" : "", c.linked_cases.join(" | "), c.former_ids.join(" | "), c.old_labels.join(" > "), c.internal.field_changes.map((f: any) => f.field).join(" | "), adminCount.get(String(c.id)) ?? "", ...Array.from({ length: maxSources }, (_, i) => c.sources[i] ?? ""), ]); writeFileSync(join(OUT, `export/gart-kr-database-${buildDate}.csv`), csv([head, ...rows])); // every open item carries the case it belongs to: who, where, when, the four steps and what happened const section = (summary: string, name: string) => new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*([\\s\\S]*?)(?=\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary)?.[1].replace(/\s+/g, " ").trim() ?? ""; const shorten = (text: string, limit: number) => { if (text.length <= limit) return text; const head = text.slice(0, limit), stop = Math.max(head.lastIndexOf(". "), head.lastIndexOf("; ")); return (stop > limit * 0.5 ? head.slice(0, stop + 1) : head.trimEnd() + " ...").trim(); }; const removedRecord = new Map(records.filter((r) => removed.includes(Number(r.id))).map((r) => [r.id, r])); const context = (id: string) => { const c = cases.get(Number(id)); if (c) return { victim: c.victim, date: c.date, location: c.location, steps: `targeting: ${c.steps.targeting.join(", ")} | approach: ${c.steps.approach.join(", ")} | attack: ${c.steps.attack.join(", ")} | outcome: ${c.steps.outcome}`, what_happened: shorten(section(c.summary, "Attack Method") || c.description, 420), how_it_ended: shorten(section(c.summary, "Status"), 260) }; const r = removedRecord.get(id); return { victim: r?.victim ?? "", date: r?.date ?? "", location: r?.location ?? "", steps: "", what_happened: r ? shorten(section(r.summary, "Attack Method") || r.description, 420) : "", how_it_ended: r ? shorten(section(r.summary, "Status"), 260) : "" }; }; const CONTEXT = ["victim", "date", "location", "steps", "what_happened", "how_it_ended"] as const; for (const a of admin) Object.assign(a, a.case_id ? context(a.case_id) : Object.fromEntries(CONTEXT.map((k) => [k, ""]))); const adminHead = ["case_id", "victim", "date", "location", "kind", "field", "detail", "first_value", "second_value", "quote", "steps", "what_happened", "how_it_ended", "evidence_file"]; writeFileSync(join(OUT, `export/admin-decisions-${buildDate}.csv`), csv([adminHead, ...admin.map((a) => adminHead.map((h) => a[h]))])); // the same list case by case: the case first, then what is open on it const byCase = new Map[]>(); for (const a of admin) byCase.set(a.case_id, [...(byCase.get(a.case_id) ?? []), a]); const line = (a: Record) => `- **${a.kind}**${a.field ? ` (${a.field})` : ""}: ${a.detail}${a.first_value ? ` In the database: ${a.first_value}.` : ""}${a.second_value ? ` Other reading or proposal: ${a.second_value}.` : ""}${a.quote ? ` Text: "${a.quote.replace(/"/g, "'")}"` : ""}`; const caseMd = [ `# Open items, case by case (${buildDate})`, "", `${admin.length} open items on ${[...byCase.keys()].filter(Boolean).length} cases. An open item is one thing in one case that is unconfirmed, disputed or waiting for a decision. Each case below shows who, where and when, the four steps as they stand in the database, what happened, and then its open items.`, "", ...[...byCase.entries()].filter(([id]) => id).flatMap(([id, items]) => { const k = items[0]; return [ `## Case ${id}: ${k.victim || "(no victim field)"}`, "", `${[k.location, k.date].filter(Boolean).join(", ")}`, "", ...(k.steps ? [`**Steps:** ${k.steps}`, ""] : []), ...(k.what_happened ? [`**What happened:** ${k.what_happened}`, ""] : []), ...(k.how_it_ended ? [`**How it ended:** ${k.how_it_ended}`, ""] : []), `**Open (${items.length}):**`, "", ...items.map(line), "", ]; }), "## General", "", ...(byCase.get("") ?? []).map(line), ].join("\n"); writeFileSync(join(OUT, `export/admin-decisions-${buildDate}-by-case.md`), caseMd + "\n"); const byKind = new Map[]>(); for (const a of admin) byKind.set(a.kind, [...(byKind.get(a.kind) ?? []), a]); const kinds = [...byKind.entries()].sort((a, b) => a[0].localeCompare(b[0])); const md = [ `# Admin decisions and open discrepancies (${buildDate})`, "", `Built from the live pull of ${dataDate}. ${admin.length} points on ${new Set(admin.map((a) => a.case_id).filter(Boolean)).size} cases. The same rows, sortable, are in \`admin-decisions-${buildDate}.csv\`.`, "", "| Kind | Points |", "|---|---|", ...kinds.map(([k, v]) => `| ${k} | ${v.length} |`), "", ...kinds.flatMap(([k, v]) => [ `## ${k} (${v.length})`, "", ...(v.length > 40 && new Set(v.map((a) => a.detail)).size === 1 ? [v[0].detail, "", `Cases: ${v.map((a) => a.case_id).join(", ")}`, ""] : v.map((a) => `- **${a.case_id ? `Case ${a.case_id}, ${a.victim} (${[a.location, a.date].filter(Boolean).join(", ")})` : "General"}**${a.field ? ` (${a.field})` : ""}: ${a.detail}${a.first_value ? ` In the database: ${a.first_value}.` : ""}${a.second_value ? ` Proposed: ${a.second_value}.` : ""}${a.quote ? ` Text: "${a.quote.replace(/"/g, "'")}"` : ""}`).concat("")), ]), ].join("\n"); writeFileSync(join(OUT, `export/admin-decisions-${buildDate}.md`), md + "\n"); console.log(`${OUT}/data/cases.json: ${list.length} cases (${records.length} in the live pull, removed ${removed.join(", ")})`); console.log(`recheck changes applied: ${rechecked}; label-check changes applied: ${labelsChanged}; flag-check changes applied: ${flagsChanged}`); console.log(`field changes applied: ${changed}; admin points: ${admin.length} in ${kinds.length} kinds`); for (const [k, v] of kinds) console.log(` ${String(v.length).padStart(4)} ${k}`);