// Export the database in the proposed new structure, one CSV line per case: // record fields, verified scenario sequence, the six flags, the fact columns, the verified // summary and every source link. // Flags: a ruling on the case wins, then the flag proposal, then the database value. // Record fields (date, victim, location, description, amounts) are the database's own: // the rulings on them in corrections-approved.md are prose and are not applied here. // Usage (from merge-output/): bun make_new_structure_export.ts ../attacks-live-api-YYYY-MM-DD.json import { readFileSync, writeFileSync } from "fs"; import { csvCell, withhold } from "./withheld"; const livePath = process.argv[2]; if (!livePath) throw new Error("usage: bun make_new_structure_export.ts "); const J = (p: string) => JSON.parse(readFileSync(p, "utf8")); const live: any[] = J(livePath); const exportById = new Map(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r])); const seqs: Record = J("scenario-sequences.json"); const summaries: Record = J("summary-overrides.json"); const manifest: any[] = J("case-files-manifest.json").entries; const proposals = new Map(J("batches/flag-proposals.json").map((p: any) => [`${p.id}:${p.flag}`, p])); // DB column -> name in the new structure const FLAGS: [string, string][] = [ ["kidnappings", "detention"], ["violence_torture", "violence_torture"], ["drugs_alcohol", "drugs_alcohol"], ["weapons", "weapons"], ["theft", "theft"], ["life_taken", "life_taken"], ]; // proposal wording -> value in the new structure const flagValue = (v: unknown): string => { if (v == null || v === "null" || v === "") return ""; const s = String(v).trim(); if (/^unconfirmed|^unverified/.test(s)) return "unconfirmed"; if (s === "threat only") return "threat only"; const m = /^(\d+)/.exec(s); if (!m) throw new Error(`unknown flag value: ${s}`); return m[1]; }; // flag values stated in single-case ruling lines (WORKLIST-BY-CASE.md copies them oldest first, // so the last statement on a flag is the one in force); group rulings are left to the proposals const ruled = new Map(); const FLAG_IN_RULING = /`?\b(kidnappings|detention|violence_torture|drugs_alcohol|weapons|theft|life_taken)\b`?\s*(?:stays?\s+)?(?:(?:null|\d+|unconfirmed|unverified)\s*)?(?:→\s*)?(\d+|unconfirmed|unverified|threat only)?/g; for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) { const id = Number(/^\d+/.exec(block)![0]); for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l))) for (const m of line.matchAll(FLAG_IN_RULING)) { // "weapons 1 → unconfirmed" and "weapons → 1" carry the value last; "weapons 1" carries it first const value = m[2] ?? /(\d+|unconfirmed|unverified)\s*$/.exec(m[0].trim())?.[1]; if (value) ruled.set(`${id}:${m[1] === "detention" ? "kidnappings" : m[1]}`, value); } } // structural operations approved in HANDOVER-SOFI.md, not yet done in the database const PENDING: Record = { 5: "rewrite as the 27 May 2015 Crown Heights robbery", 38: "split: Irving 22 March 2018 and Mesquite 28 March 2018", 45: "delete (owner, 6 October 2026)", 49: "split: add a record for the 21 August 2018 attempt in Middelburg", 386: "merge into 295, then delete", 439: "absorbs 440 and 441", 440: "merge into 439", 441: "merge into 439", 445: "absorbs 446; date 10 to 11 May 2026", 446: "merge into 445", }; const LINKS: number[][] = [ [430, 138, 145, 156], [106, 124], [354, 355, 307, 162, 169], ]; const fetched = new Map(); const dossier = new Map(); for (const e of manifest) { const id = Number(e.case); dossier.set(id, e.file); const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? ""); if (m) fetched.set(id, [...(fetched.get(id) ?? []), m[1]]); } const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:"); const oneLine = (v: unknown) => (v == null ? "" : String(v)).replace(/\s*[\r\n]+\s*/g, " ").trim(); const cell = csvCell; const MAX_LABELS = 3; const MAX_SOURCES = 8; const header = [ "id", "date", "year", "month", "quarter", "victim", "location", "country", ...Array.from({ length: MAX_LABELS }, (_, i) => `scenario_${i + 1}`), "attempted", "v1_scenario", ...FLAGS.map(([, name]) => name), "venue_of_control", "held_duration", "outcome", "arrests", "money_wanted", "coin_type", "description", "summary", "notes", "reports", "verification", "linked_cases", "pending_action", ...Array.from({ length: MAX_SOURCES }, (_, i) => `source_${i + 1}`), "dossier_file", ]; const rows: unknown[][] = []; const origin = { ruling: 0, proposal: 0, record: 0 }; // flag values taken from a proposal that no ruling has confirmed; listed for the admin by make_release_data.ts const proposalFlags: { id: number; flag: string; value: string; evidence: string }[] = []; // where each flag value comes from, for the flag check (flags_pass.ts): a ruled value is not re-judged const flagSources: Record = {}; for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) { const id = Number(r.id); const old = exportById.get(id) ?? {}; // notes, reports, money_wanted, coin_type are hidden from the public API const seq = seqs[String(id)]; const labels: string[] = seq ? seq.sequence : [r.scenario].filter(Boolean); if (labels.length > MAX_LABELS) throw new Error(`case ${id}: ${labels.length} labels`); const flags = FLAGS.map(([col]) => { const key = `${id}:${col}`; const from = ruled.has(key) ? "ruling" : proposals.has(key) ? "proposal" : "record"; origin[from]++; flagSources[`${id}:${col === "kidnappings" ? "detention" : col}`] = from; if (from === "proposal") proposalFlags.push({ id, flag: col === "kidnappings" ? "detention" : col, value: String(proposals.get(key).proposed), evidence: proposals.get(key).evidence }); return flagValue(from === "ruling" ? ruled.get(key) : from === "proposal" ? proposals.get(key).proposed : r[col]); }); const sources: string[] = []; for (const u of [r.url, r.url_2, r.url_3, r.url_4, r.url_5, ...(fetched.get(id) ?? [])]) if (u && u.trim() && !sources.some((s) => norm(s) === norm(u))) sources.push(u.trim()); if (sources.length > MAX_SOURCES) throw new Error(`case ${id}: ${sources.length} sources`); const noArticle = seq && !dossier.has(id); // no article text in the manifest const verification = !seq ? "not verified" : noArticle ? "record only: no usable article on file" : seq.facts ? "summary, labels and record fields checked against articles" : "summary and labels checked; record fields not compared"; const linked = LINKS.filter((g) => g.includes(id)).flat().filter((x) => x !== id); rows.push([ id, r.date, r.year, r.month, r.quarter, r.victim, r.location, r.country, ...Array.from({ length: MAX_LABELS }, (_, i) => labels[i] ?? ""), (seq?.attempted ?? []).join(" | "), r.scenario, ...flags, seq?.facts?.venue_of_control, seq?.facts?.held_duration, seq?.facts?.outcome, seq?.facts?.arrests, old.money_wanted, old.coin_type, r.description, summaries[String(id)] ?? r.summary, old.notes, old.reports, verification, linked.join(" | "), PENDING[id] ?? "", ...Array.from({ length: MAX_SOURCES }, (_, i) => sources[i] ?? ""), dossier.get(id) ?? "", ]); } const out = `exports/gart-kr-new-structure-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`; const named = rows.map((row) => row.map((v, i) => withhold(row[0] as number, header[i], oneLine(v)))); writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n"); console.log(`${out}: ${rows.length} cases, ${header.length} columns`); writeFileSync("exports/flag-values-from-proposals.json", JSON.stringify(proposalFlags, null, 1)); writeFileSync("exports/flag-sources.json", JSON.stringify(flagSources, null, 1)); console.log(`flag values from a ruling ${origin.ruling}, from a proposal ${origin.proposal}, from the record ${origin.record}`);