Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
160 lines
8.1 KiB
TypeScript
160 lines
8.1 KiB
TypeScript
// Export the database in the proposed new structure, one CSV line per case:
|
|
// record fields, verified scenario sequence, the six flags, the fact columns, the verified
|
|
// summary and every source link.
|
|
// Flags: a ruling on the case wins, then the flag proposal, then the database value.
|
|
// Record fields (date, victim, location, description, amounts) are the database's own:
|
|
// the rulings on them in corrections-approved.md are prose and are not applied here.
|
|
// Usage (from merge-output/): bun make_new_structure_export.ts ../attacks-live-api-YYYY-MM-DD.json
|
|
import { readFileSync, writeFileSync } from "fs";
|
|
import { csvCell, withhold } from "./withheld";
|
|
|
|
const livePath = process.argv[2];
|
|
if (!livePath) throw new Error("usage: bun make_new_structure_export.ts <live api snapshot.json>");
|
|
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
|
const live: any[] = J(livePath);
|
|
const exportById = new Map<number, any>(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r]));
|
|
const seqs: Record<string, any> = J("scenario-sequences.json");
|
|
const summaries: Record<string, string> = J("summary-overrides.json");
|
|
const manifest: any[] = J("case-files-manifest.json").entries;
|
|
const proposals = new Map<string, any>(J("batches/flag-proposals.json").map((p: any) => [`${p.id}:${p.flag}`, p]));
|
|
|
|
// DB column -> name in the new structure
|
|
const FLAGS: [string, string][] = [
|
|
["kidnappings", "detention"],
|
|
["violence_torture", "violence_torture"],
|
|
["drugs_alcohol", "drugs_alcohol"],
|
|
["weapons", "weapons"],
|
|
["theft", "theft"],
|
|
["life_taken", "life_taken"],
|
|
];
|
|
// proposal wording -> value in the new structure
|
|
const flagValue = (v: unknown): string => {
|
|
if (v == null || v === "null" || v === "") return "";
|
|
const s = String(v).trim();
|
|
if (/^unconfirmed|^unverified/.test(s)) return "unconfirmed";
|
|
if (s === "threat only") return "threat only";
|
|
const m = /^(\d+)/.exec(s);
|
|
if (!m) throw new Error(`unknown flag value: ${s}`);
|
|
return m[1];
|
|
};
|
|
|
|
// flag values stated in single-case ruling lines (WORKLIST-BY-CASE.md copies them oldest first,
|
|
// so the last statement on a flag is the one in force); group rulings are left to the proposals
|
|
const ruled = new Map<string, string>();
|
|
const FLAG_IN_RULING =
|
|
/`?\b(kidnappings|detention|violence_torture|drugs_alcohol|weapons|theft|life_taken)\b`?\s*(?:stays?\s+)?(?:(?:null|\d+|unconfirmed|unverified)\s*)?(?:→\s*)?(\d+|unconfirmed|unverified|threat only)?/g;
|
|
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
|
|
const id = Number(/^\d+/.exec(block)![0]);
|
|
for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l)))
|
|
for (const m of line.matchAll(FLAG_IN_RULING)) {
|
|
// "weapons 1 → unconfirmed" and "weapons → 1" carry the value last; "weapons 1" carries it first
|
|
const value = m[2] ?? /(\d+|unconfirmed|unverified)\s*$/.exec(m[0].trim())?.[1];
|
|
if (value) ruled.set(`${id}:${m[1] === "detention" ? "kidnappings" : m[1]}`, value);
|
|
}
|
|
}
|
|
|
|
// structural operations approved in HANDOVER-SOFI.md, not yet done in the database
|
|
const PENDING: Record<number, string> = {
|
|
5: "rewrite as the 27 May 2015 Crown Heights robbery",
|
|
38: "split: Irving 22 March 2018 and Mesquite 28 March 2018",
|
|
45: "delete (owner, 6 October 2026)",
|
|
49: "split: add a record for the 21 August 2018 attempt in Middelburg",
|
|
386: "merge into 295, then delete",
|
|
439: "absorbs 440 and 441",
|
|
440: "merge into 439",
|
|
441: "merge into 439",
|
|
445: "absorbs 446; date 10 to 11 May 2026",
|
|
446: "merge into 445",
|
|
};
|
|
const LINKS: number[][] = [
|
|
[430, 138, 145, 156],
|
|
[106, 124],
|
|
[354, 355, 307, 162, 169],
|
|
];
|
|
|
|
const fetched = new Map<number, string[]>();
|
|
const dossier = new Map<number, string>();
|
|
for (const e of manifest) {
|
|
const id = Number(e.case);
|
|
dossier.set(id, e.file);
|
|
const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? "");
|
|
if (m) fetched.set(id, [...(fetched.get(id) ?? []), m[1]]);
|
|
}
|
|
const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:");
|
|
const oneLine = (v: unknown) => (v == null ? "" : String(v)).replace(/\s*[\r\n]+\s*/g, " ").trim();
|
|
const cell = csvCell;
|
|
|
|
const MAX_LABELS = 3;
|
|
const MAX_SOURCES = 8;
|
|
const header = [
|
|
"id", "date", "year", "month", "quarter", "victim", "location", "country",
|
|
...Array.from({ length: MAX_LABELS }, (_, i) => `scenario_${i + 1}`),
|
|
"attempted", "v1_scenario",
|
|
...FLAGS.map(([, name]) => name),
|
|
"venue_of_control", "held_duration", "outcome", "arrests",
|
|
"money_wanted", "coin_type",
|
|
"description", "summary", "notes", "reports",
|
|
"verification", "linked_cases", "pending_action",
|
|
...Array.from({ length: MAX_SOURCES }, (_, i) => `source_${i + 1}`),
|
|
"dossier_file",
|
|
];
|
|
|
|
const rows: unknown[][] = [];
|
|
const origin = { ruling: 0, proposal: 0, record: 0 };
|
|
// flag values taken from a proposal that no ruling has confirmed; listed for the admin by make_release_data.ts
|
|
const proposalFlags: { id: number; flag: string; value: string; evidence: string }[] = [];
|
|
// where each flag value comes from, for the flag check (flags_pass.ts): a ruled value is not re-judged
|
|
const flagSources: Record<string, string> = {};
|
|
for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) {
|
|
const id = Number(r.id);
|
|
const old = exportById.get(id) ?? {}; // notes, reports, money_wanted, coin_type are hidden from the public API
|
|
const seq = seqs[String(id)];
|
|
const labels: string[] = seq ? seq.sequence : [r.scenario].filter(Boolean);
|
|
if (labels.length > MAX_LABELS) throw new Error(`case ${id}: ${labels.length} labels`);
|
|
|
|
const flags = FLAGS.map(([col]) => {
|
|
const key = `${id}:${col}`;
|
|
const from = ruled.has(key) ? "ruling" : proposals.has(key) ? "proposal" : "record";
|
|
origin[from]++;
|
|
flagSources[`${id}:${col === "kidnappings" ? "detention" : col}`] = from;
|
|
if (from === "proposal") proposalFlags.push({ id, flag: col === "kidnappings" ? "detention" : col, value: String(proposals.get(key).proposed), evidence: proposals.get(key).evidence });
|
|
return flagValue(from === "ruling" ? ruled.get(key) : from === "proposal" ? proposals.get(key).proposed : r[col]);
|
|
});
|
|
|
|
const sources: string[] = [];
|
|
for (const u of [r.url, r.url_2, r.url_3, r.url_4, r.url_5, ...(fetched.get(id) ?? [])])
|
|
if (u && u.trim() && !sources.some((s) => norm(s) === norm(u))) sources.push(u.trim());
|
|
if (sources.length > MAX_SOURCES) throw new Error(`case ${id}: ${sources.length} sources`);
|
|
|
|
const noArticle = seq && !dossier.has(id); // no article text in the manifest
|
|
const verification = !seq
|
|
? "not verified"
|
|
: noArticle
|
|
? "record only: no usable article on file"
|
|
: seq.facts
|
|
? "summary, labels and record fields checked against articles"
|
|
: "summary and labels checked; record fields not compared";
|
|
const linked = LINKS.filter((g) => g.includes(id)).flat().filter((x) => x !== id);
|
|
|
|
rows.push([
|
|
id, r.date, r.year, r.month, r.quarter, r.victim, r.location, r.country,
|
|
...Array.from({ length: MAX_LABELS }, (_, i) => labels[i] ?? ""),
|
|
(seq?.attempted ?? []).join(" | "), r.scenario,
|
|
...flags,
|
|
seq?.facts?.venue_of_control, seq?.facts?.held_duration, seq?.facts?.outcome, seq?.facts?.arrests,
|
|
old.money_wanted, old.coin_type,
|
|
r.description, summaries[String(id)] ?? r.summary, old.notes, old.reports,
|
|
verification, linked.join(" | "), PENDING[id] ?? "",
|
|
...Array.from({ length: MAX_SOURCES }, (_, i) => sources[i] ?? ""),
|
|
dossier.get(id) ?? "",
|
|
]);
|
|
}
|
|
|
|
const out = `exports/gart-kr-new-structure-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`;
|
|
const named = rows.map((row) => row.map((v, i) => withhold(row[0] as number, header[i], oneLine(v))));
|
|
writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n");
|
|
console.log(`${out}: ${rows.length} cases, ${header.length} columns`);
|
|
writeFileSync("exports/flag-values-from-proposals.json", JSON.stringify(proposalFlags, null, 1));
|
|
writeFileSync("exports/flag-sources.json", JSON.stringify(flagSources, null, 1));
|
|
console.log(`flag values from a ruling ${origin.ruling}, from a proposal ${origin.proposal}, from the record ${origin.record}`);
|