chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+159
View File
@@ -0,0 +1,159 @@
// Export the database in the proposed new structure, one CSV line per case:
// record fields, verified scenario sequence, the six flags, the fact columns, the verified
// summary and every source link.
// Flags: a ruling on the case wins, then the flag proposal, then the database value.
// Record fields (date, victim, location, description, amounts) are the database's own:
// the rulings on them in corrections-approved.md are prose and are not applied here.
// Usage (from merge-output/): bun make_new_structure_export.ts ../attacks-live-api-YYYY-MM-DD.json
import { readFileSync, writeFileSync } from "fs";
import { csvCell, withhold } from "./withheld";
const livePath = process.argv[2];
if (!livePath) throw new Error("usage: bun make_new_structure_export.ts <live api snapshot.json>");
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const live: any[] = J(livePath);
const exportById = new Map<number, any>(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r]));
const seqs: Record<string, any> = J("scenario-sequences.json");
const summaries: Record<string, string> = J("summary-overrides.json");
const manifest: any[] = J("case-files-manifest.json").entries;
const proposals = new Map<string, any>(J("batches/flag-proposals.json").map((p: any) => [`${p.id}:${p.flag}`, p]));
// DB column -> name in the new structure
const FLAGS: [string, string][] = [
["kidnappings", "detention"],
["violence_torture", "violence_torture"],
["drugs_alcohol", "drugs_alcohol"],
["weapons", "weapons"],
["theft", "theft"],
["life_taken", "life_taken"],
];
// proposal wording -> value in the new structure
const flagValue = (v: unknown): string => {
if (v == null || v === "null" || v === "") return "";
const s = String(v).trim();
if (/^unconfirmed|^unverified/.test(s)) return "unconfirmed";
if (s === "threat only") return "threat only";
const m = /^(\d+)/.exec(s);
if (!m) throw new Error(`unknown flag value: ${s}`);
return m[1];
};
// flag values stated in single-case ruling lines (WORKLIST-BY-CASE.md copies them oldest first,
// so the last statement on a flag is the one in force); group rulings are left to the proposals
const ruled = new Map<string, string>();
const FLAG_IN_RULING =
/`?\b(kidnappings|detention|violence_torture|drugs_alcohol|weapons|theft|life_taken)\b`?\s*(?:stays?\s+)?(?:(?:null|\d+|unconfirmed|unverified)\s*)?(?:→\s*)?(\d+|unconfirmed|unverified|threat only)?/g;
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = Number(/^\d+/.exec(block)![0]);
for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l)))
for (const m of line.matchAll(FLAG_IN_RULING)) {
// "weapons 1 → unconfirmed" and "weapons → 1" carry the value last; "weapons 1" carries it first
const value = m[2] ?? /(\d+|unconfirmed|unverified)\s*$/.exec(m[0].trim())?.[1];
if (value) ruled.set(`${id}:${m[1] === "detention" ? "kidnappings" : m[1]}`, value);
}
}
// structural operations approved in HANDOVER-SOFI.md, not yet done in the database
const PENDING: Record<number, string> = {
5: "rewrite as the 27 May 2015 Crown Heights robbery",
38: "split: Irving 22 March 2018 and Mesquite 28 March 2018",
45: "delete (owner, 6 October 2026)",
49: "split: add a record for the 21 August 2018 attempt in Middelburg",
386: "merge into 295, then delete",
439: "absorbs 440 and 441",
440: "merge into 439",
441: "merge into 439",
445: "absorbs 446; date 10 to 11 May 2026",
446: "merge into 445",
};
const LINKS: number[][] = [
[430, 138, 145, 156],
[106, 124],
[354, 355, 307, 162, 169],
];
const fetched = new Map<number, string[]>();
const dossier = new Map<number, string>();
for (const e of manifest) {
const id = Number(e.case);
dossier.set(id, e.file);
const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? "");
if (m) fetched.set(id, [...(fetched.get(id) ?? []), m[1]]);
}
const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:");
const oneLine = (v: unknown) => (v == null ? "" : String(v)).replace(/\s*[\r\n]+\s*/g, " ").trim();
const cell = csvCell;
const MAX_LABELS = 3;
const MAX_SOURCES = 8;
const header = [
"id", "date", "year", "month", "quarter", "victim", "location", "country",
...Array.from({ length: MAX_LABELS }, (_, i) => `scenario_${i + 1}`),
"attempted", "v1_scenario",
...FLAGS.map(([, name]) => name),
"venue_of_control", "held_duration", "outcome", "arrests",
"money_wanted", "coin_type",
"description", "summary", "notes", "reports",
"verification", "linked_cases", "pending_action",
...Array.from({ length: MAX_SOURCES }, (_, i) => `source_${i + 1}`),
"dossier_file",
];
const rows: unknown[][] = [];
const origin = { ruling: 0, proposal: 0, record: 0 };
// flag values taken from a proposal that no ruling has confirmed; listed for the admin by make_release_data.ts
const proposalFlags: { id: number; flag: string; value: string; evidence: string }[] = [];
// where each flag value comes from, for the flag check (flags_pass.ts): a ruled value is not re-judged
const flagSources: Record<string, string> = {};
for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) {
const id = Number(r.id);
const old = exportById.get(id) ?? {}; // notes, reports, money_wanted, coin_type are hidden from the public API
const seq = seqs[String(id)];
const labels: string[] = seq ? seq.sequence : [r.scenario].filter(Boolean);
if (labels.length > MAX_LABELS) throw new Error(`case ${id}: ${labels.length} labels`);
const flags = FLAGS.map(([col]) => {
const key = `${id}:${col}`;
const from = ruled.has(key) ? "ruling" : proposals.has(key) ? "proposal" : "record";
origin[from]++;
flagSources[`${id}:${col === "kidnappings" ? "detention" : col}`] = from;
if (from === "proposal") proposalFlags.push({ id, flag: col === "kidnappings" ? "detention" : col, value: String(proposals.get(key).proposed), evidence: proposals.get(key).evidence });
return flagValue(from === "ruling" ? ruled.get(key) : from === "proposal" ? proposals.get(key).proposed : r[col]);
});
const sources: string[] = [];
for (const u of [r.url, r.url_2, r.url_3, r.url_4, r.url_5, ...(fetched.get(id) ?? [])])
if (u && u.trim() && !sources.some((s) => norm(s) === norm(u))) sources.push(u.trim());
if (sources.length > MAX_SOURCES) throw new Error(`case ${id}: ${sources.length} sources`);
const noArticle = seq && !dossier.has(id); // no article text in the manifest
const verification = !seq
? "not verified"
: noArticle
? "record only: no usable article on file"
: seq.facts
? "summary, labels and record fields checked against articles"
: "summary and labels checked; record fields not compared";
const linked = LINKS.filter((g) => g.includes(id)).flat().filter((x) => x !== id);
rows.push([
id, r.date, r.year, r.month, r.quarter, r.victim, r.location, r.country,
...Array.from({ length: MAX_LABELS }, (_, i) => labels[i] ?? ""),
(seq?.attempted ?? []).join(" | "), r.scenario,
...flags,
seq?.facts?.venue_of_control, seq?.facts?.held_duration, seq?.facts?.outcome, seq?.facts?.arrests,
old.money_wanted, old.coin_type,
r.description, summaries[String(id)] ?? r.summary, old.notes, old.reports,
verification, linked.join(" | "), PENDING[id] ?? "",
...Array.from({ length: MAX_SOURCES }, (_, i) => sources[i] ?? ""),
dossier.get(id) ?? "",
]);
}
const out = `exports/gart-kr-new-structure-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`;
const named = rows.map((row) => row.map((v, i) => withhold(row[0] as number, header[i], oneLine(v))));
writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n");
console.log(`${out}: ${rows.length} cases, ${header.length} columns`);
writeFileSync("exports/flag-values-from-proposals.json", JSON.stringify(proposalFlags, null, 1));
writeFileSync("exports/flag-sources.json", JSON.stringify(flagSources, null, 1));
console.log(`flag values from a ruling ${origin.ruling}, from a proposal ${origin.proposal}, from the record ${origin.record}`);