// Writes the stats2 import files (one JSON per table, for sqlite-web's Import tab) for the new entries that // need no operator decision, and test-imports them into a scratch copy of the database. // bun make_import.ts [ ...] // Output: import/stats2-import-lopp-/0-labels.json (only when a label is missing), 1-cases.json, // 2-case_labels.json, 3-case_sources.json, 4-case_people.json, keys.json (Lopp key -> case id). import { Database } from "bun:sqlite"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; import { dateColumns } from "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2/dates.ts"; import { loadCase } from "./lib.ts"; const HERE = import.meta.dir; const TODAY = new Date().toLocaleDateString("sv"); const [firstArg, scratch, ...keys] = process.argv.slice(2); const first = Number(firstArg); if (!Number.isInteger(first) || !scratch || !keys.length) throw new Error("usage: bun make_import.ts ..."); if (new Set(keys).size !== keys.length) throw new Error("a key is given twice"); const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"]; // decided by the owner on 2026-10-17 and not yet in the live labels table const PENDING_LABELS = [{ step: "obtained", name: "Unknown", group_name: "", definition: "the sources do not say what, if anything, the attackers got", display_order: 7 }]; const cases = keys.map((key) => { const o = loadCase(key); if (o.status !== "new" || !o.values) throw new Error(`${key}: status ${o.status}, not a new case with values`); return { key, o, v: o.values, sort: dateColumns(o.values.date).date_sort ?? "9999" }; }); cases.sort((a, b) => a.sort.localeCompare(b.sort) || a.key.localeCompare(b.key)); const lifeTaken = (x: unknown) => { const s = String(x ?? ""); if (s === "") return { life_taken: null, life_taken_unconfirmed: 0 }; if (s === "unconfirmed") return { life_taken: null, life_taken_unconfirmed: 1 }; if (!/^\d+$/.test(s)) throw new Error(`life_taken "${s}"`); return { life_taken: Number(s), life_taken_unconfirmed: 0 }; }; const num = (x: unknown) => (x == null || x === "" ? null : Number(x)); const text = (x: unknown) => (x == null ? "" : String(x)); const rowsCases: any[] = [], rowsLabels: any[] = [], rowsSources: any[] = [], rowsPeople: any[] = [], map: Record = {}; cases.forEach(({ key, o, v }, i) => { const id = first + i; map[key] = id; const nSources = (v.sources ?? []).length; rowsCases.push({ id, date: text(v.date), victim: text(v.victim), location: text(v.location), country: text(v.country), description: text(v.description), summary: text(v.summary), case_details: text(v.case_details), notes: [text(o.notes), `From Jameson Lopp's list (PR 234), entry ${key}; dossier lopp-pr234/sources/.`].filter(Boolean).join("\n\n"), verification: `fields, flags and steps written from ${nSources} fetched source${nSources === 1 ? "" : "s"} on 2026-10-16 by two readers; flags and obtained re-read on 2026-10-17 under the owner's definitions; no open points or second-reader rejects; imported on the owner's instruction of ${TODAY}`, detention: text(v.detention), violence_torture: text(v.violence_torture), drugs_alcohol: text(v.drugs_alcohol), weapons: text(v.weapons), theft: text(v.theft), ...lifeTaken(v.life_taken), venue_of_control: text(v.venue_of_control), held_duration: text(v.held_duration), arrests: text(v.arrests), demanded_amount: num(v.demanded_amount), demanded_currency: v.demanded_currency ?? null, demanded_usd_stated: num(v.demanded_usd_stated), demanded_as_stated: text(v.demanded_as_stated), taken_amount: num(v.taken_amount), taken_currency: v.taken_currency ?? null, taken_usd_stated: num(v.taken_usd_stated), taken_as_stated: text(v.taken_as_stated), name_withheld: 0, }); for (const step of STEPS) [].concat(v[step] ?? []).forEach((label: string, j: number) => rowsLabels.push({ case_id: id, step, label, position: j + 1 })); (v.sources ?? []).forEach((url: string, j: number) => rowsSources.push({ case_id: id, position: j + 1, url })); for (const r of v.people ?? []) rowsPeople.push({ case_id: id, role: r.role, people: r.people, at_least: r.at_least, detail: text(r.detail) }); }); // test-import into the scratch copy, triggers on, the way sqlite-web would add the rows const db = new Database(scratch); db.exec("PRAGMA foreign_keys = ON"); const known = new Set((db.query("SELECT step || '|' || name k FROM labels").all() as any[]).map((r) => r.k)); const labelsNeeded = PENDING_LABELS.filter((l) => !known.has(`${l.step}|${l.name}`) && rowsLabels.some((r) => r.step === l.step && r.label === l.name)); const insert = (table: string, rows: any[]) => { for (const r of rows) { const cols = Object.keys(r); db.query(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`).run(...cols.map((c) => r[c])); } }; const last = Number((db.query("SELECT value FROM meta WHERE key = 'last_case_id'").get() as any)?.value ?? 0); if (last + 1 !== first) throw new Error(`the scratch copy's last_case_id is ${last}; the first id must be ${last + 1}`); db.transaction(() => { insert("labels", labelsNeeded); insert("cases", rowsCases); insert("case_labels", rowsLabels); insert("case_sources", rowsSources); insert("case_people", rowsPeople); for (const r of rowsCases) if (!db.query("SELECT 1 FROM cases WHERE id = ? AND victim = ?").get(r.id, r.victim)) throw new Error(`case ${r.id} did not keep its id`); const missing = db.query(`SELECT id FROM cases_missing_labels WHERE id >= ${first}`).all(); const mismatch = db.query(`SELECT * FROM cases_people_mismatch WHERE case_id >= ${first}`).all(); if (missing.length || mismatch.length) throw new Error(`test import failed: cases missing labels ${JSON.stringify(missing)}, people mismatches ${JSON.stringify(mismatch)}`); })(); // the checks run inside the transaction, so a failed test import leaves the scratch copy as it was const out = join(HERE, "import", `stats2-import-lopp-${TODAY}`); rmSync(out, { recursive: true, force: true }); // a rerun must not leave files of an earlier run beside the new ones mkdirSync(out, { recursive: true }); const w = (f: string, x: unknown) => writeFileSync(join(out, f), JSON.stringify(x, null, 1) + "\n"); if (labelsNeeded.length) w("0-labels.json", labelsNeeded); w("1-cases.json", rowsCases); w("2-case_labels.json", rowsLabels); w("3-case_sources.json", rowsSources); w("4-case_people.json", rowsPeople); w("keys.json", map); console.log(`${rowsCases.length} cases, ids ${first} to ${first + rowsCases.length - 1}; ${rowsLabels.length} labels, ${rowsSources.length} sources, ${rowsPeople.length} people rows${labelsNeeded.length ? `; ${labelsNeeded.length} new label` : ""}`); console.log("test import: ok; no case missing labels; no people mismatch"); console.log(`wrote ${out}`);