chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+95
View File
@@ -0,0 +1,95 @@
// Writes the stats2 import files (one JSON per table, for sqlite-web's Import tab) for the new entries that
// need no operator decision, and test-imports them into a scratch copy of the database.
// bun make_import.ts <first id> <scratch copy of gart_kr.db> <key> [<key> ...]
// Output: import/stats2-import-lopp-<date>/0-labels.json (only when a label is missing), 1-cases.json,
// 2-case_labels.json, 3-case_sources.json, 4-case_people.json, keys.json (Lopp key -> case id).
import { Database } from "bun:sqlite";
import { mkdirSync, rmSync, writeFileSync } from "fs";
import { join } from "path";
import { dateColumns } from "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2/dates.ts";
import { loadCase } from "./lib.ts";
const HERE = import.meta.dir;
const TODAY = new Date().toLocaleDateString("sv");
const [firstArg, scratch, ...keys] = process.argv.slice(2);
const first = Number(firstArg);
if (!Number.isInteger(first) || !scratch || !keys.length) throw new Error("usage: bun make_import.ts <first id> <scratch gart_kr.db> <key> ...");
if (new Set(keys).size !== keys.length) throw new Error("a key is given twice");
const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"];
// decided by the owner on 2026-10-17 and not yet in the live labels table
const PENDING_LABELS = [{ step: "obtained", name: "Unknown", group_name: "", definition: "the sources do not say what, if anything, the attackers got", display_order: 7 }];
const cases = keys.map((key) => {
const o = loadCase(key);
if (o.status !== "new" || !o.values) throw new Error(`${key}: status ${o.status}, not a new case with values`);
return { key, o, v: o.values, sort: dateColumns(o.values.date).date_sort ?? "9999" };
});
cases.sort((a, b) => a.sort.localeCompare(b.sort) || a.key.localeCompare(b.key));
const lifeTaken = (x: unknown) => {
const s = String(x ?? "");
if (s === "") return { life_taken: null, life_taken_unconfirmed: 0 };
if (s === "unconfirmed") return { life_taken: null, life_taken_unconfirmed: 1 };
if (!/^\d+$/.test(s)) throw new Error(`life_taken "${s}"`);
return { life_taken: Number(s), life_taken_unconfirmed: 0 };
};
const num = (x: unknown) => (x == null || x === "" ? null : Number(x));
const text = (x: unknown) => (x == null ? "" : String(x));
const rowsCases: any[] = [], rowsLabels: any[] = [], rowsSources: any[] = [], rowsPeople: any[] = [], map: Record<string, number> = {};
cases.forEach(({ key, o, v }, i) => {
const id = first + i;
map[key] = id;
const nSources = (v.sources ?? []).length;
rowsCases.push({
id, date: text(v.date), victim: text(v.victim), location: text(v.location), country: text(v.country),
description: text(v.description), summary: text(v.summary), case_details: text(v.case_details),
notes: [text(o.notes), `From Jameson Lopp's list (PR 234), entry ${key}; dossier lopp-pr234/sources/.`].filter(Boolean).join("\n\n"),
verification: `fields, flags and steps written from ${nSources} fetched source${nSources === 1 ? "" : "s"} on 2026-10-16 by two readers; flags and obtained re-read on 2026-10-17 under the owner's definitions; no open points or second-reader rejects; imported on the owner's instruction of ${TODAY}`,
detention: text(v.detention), violence_torture: text(v.violence_torture), drugs_alcohol: text(v.drugs_alcohol), weapons: text(v.weapons), theft: text(v.theft),
...lifeTaken(v.life_taken),
venue_of_control: text(v.venue_of_control), held_duration: text(v.held_duration), arrests: text(v.arrests),
demanded_amount: num(v.demanded_amount), demanded_currency: v.demanded_currency ?? null, demanded_usd_stated: num(v.demanded_usd_stated), demanded_as_stated: text(v.demanded_as_stated),
taken_amount: num(v.taken_amount), taken_currency: v.taken_currency ?? null, taken_usd_stated: num(v.taken_usd_stated), taken_as_stated: text(v.taken_as_stated),
name_withheld: 0,
});
for (const step of STEPS) [].concat(v[step] ?? []).forEach((label: string, j: number) => rowsLabels.push({ case_id: id, step, label, position: j + 1 }));
(v.sources ?? []).forEach((url: string, j: number) => rowsSources.push({ case_id: id, position: j + 1, url }));
for (const r of v.people ?? []) rowsPeople.push({ case_id: id, role: r.role, people: r.people, at_least: r.at_least, detail: text(r.detail) });
});
// test-import into the scratch copy, triggers on, the way sqlite-web would add the rows
const db = new Database(scratch);
db.exec("PRAGMA foreign_keys = ON");
const known = new Set((db.query("SELECT step || '|' || name k FROM labels").all() as any[]).map((r) => r.k));
const labelsNeeded = PENDING_LABELS.filter((l) => !known.has(`${l.step}|${l.name}`) && rowsLabels.some((r) => r.step === l.step && r.label === l.name));
const insert = (table: string, rows: any[]) => {
for (const r of rows) {
const cols = Object.keys(r);
db.query(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`).run(...cols.map((c) => r[c]));
}
};
const last = Number((db.query("SELECT value FROM meta WHERE key = 'last_case_id'").get() as any)?.value ?? 0);
if (last + 1 !== first) throw new Error(`the scratch copy's last_case_id is ${last}; the first id must be ${last + 1}`);
db.transaction(() => {
insert("labels", labelsNeeded);
insert("cases", rowsCases);
insert("case_labels", rowsLabels);
insert("case_sources", rowsSources);
insert("case_people", rowsPeople);
for (const r of rowsCases) if (!db.query("SELECT 1 FROM cases WHERE id = ? AND victim = ?").get(r.id, r.victim)) throw new Error(`case ${r.id} did not keep its id`);
const missing = db.query(`SELECT id FROM cases_missing_labels WHERE id >= ${first}`).all();
const mismatch = db.query(`SELECT * FROM cases_people_mismatch WHERE case_id >= ${first}`).all();
if (missing.length || mismatch.length) throw new Error(`test import failed: cases missing labels ${JSON.stringify(missing)}, people mismatches ${JSON.stringify(mismatch)}`);
})(); // the checks run inside the transaction, so a failed test import leaves the scratch copy as it was
const out = join(HERE, "import", `stats2-import-lopp-${TODAY}`);
rmSync(out, { recursive: true, force: true }); // a rerun must not leave files of an earlier run beside the new ones
mkdirSync(out, { recursive: true });
const w = (f: string, x: unknown) => writeFileSync(join(out, f), JSON.stringify(x, null, 1) + "\n");
if (labelsNeeded.length) w("0-labels.json", labelsNeeded);
w("1-cases.json", rowsCases); w("2-case_labels.json", rowsLabels); w("3-case_sources.json", rowsSources); w("4-case_people.json", rowsPeople);
w("keys.json", map);
console.log(`${rowsCases.length} cases, ids ${first} to ${first + rowsCases.length - 1}; ${rowsLabels.length} labels, ${rowsSources.length} sources, ${rowsPeople.length} people rows${labelsNeeded.length ? `; ${labelsNeeded.length} new label` : ""}`);
console.log("test import: ok; no case missing labels; no people mismatch");
console.log(`wrote ${out}`);