// Reading pass for the four-step structure. // bun steps_pass.ts prepare exports/gart-kr-four-steps-YYYY-MM-DD.csv -> steps/in-NN.json (25 cases each) // bun steps_pass.ts validate -> checks every steps/out-NN.json import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs"; const LABELS = JSON.parse(readFileSync("step-labels.json", "utf8")); const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"]; const BATCH = 25; const isLabel = (step: string, x: unknown) => typeof x === "string" && Object.hasOwn(LABELS[step].values, x); const EMPTY_VALUES = ["unknown", "nothing", "unconfirmed", "no label fits"]; // a list may not mix an empty value with real ones const mixesEmpty = (values: unknown[]) => values.length > 1 && values.some((x) => EMPTY_VALUES.includes(String(x))); const isAmount = (m: any) => m && typeof m === "object" && !Array.isArray(m) && ["amount", "usd_stated"].every((n) => m[n] == null || typeof m[n] === "number") && (m.currency == null || typeof m.currency === "string") && typeof m.as_stated === "string" && m.as_stated.length > 0; const NOT_SORTED = "not sorted yet"; const parseCsv = (text: string): string[][] => { const rows: string[][] = []; let row: string[] = [], field = "", quoted = false; for (let i = 0; i < text.length; i++) { const c = text[i]; if (quoted) { if (c === '"' && text[i + 1] === '"') (field += '"'), i++; else if (c === '"') quoted = false; else field += c; } else if (c === '"') quoted = true; else if (c === ",") row.push(field), (field = ""); else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = ""); else field += c; } if (quoted) throw new Error("unterminated quoted field"); if (field || row.length) row.push(field), rows.push(row); return rows; }; const [cmd, arg] = process.argv.slice(2); if (cmd === "prepare") { const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, "")); const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record); const list = (v: string) => v.split(" | ").filter((x) => x && x !== NOT_SORTED); const items = cases.map((c) => ({ id: Number(c.id), victim: c.victim, date: c.date, location: c.location, country: c.country, verified_labels_old_names: c.old_labels, attempted: c.attempted, current: { targeting: list(c["1_targeting"]), targeting_note_from_ruling: c["1_targeting_note"], approach: list(c["2_approach"]), attack: list(c["3_attack"]), outcome: c["4_outcome"] === NOT_SORTED ? "" : c["4_outcome"], attack_failed: c["4_attack_failed"] === "yes", }, flags: { detention: c.detention, violence_torture: c.violence_torture, drugs_alcohol: c.drugs_alcohol, weapons: c.weapons, theft: c.theft, life_taken: c.life_taken }, money_wanted_db: c.money_wanted_db, summary_is_verified: c.verification !== "not verified", summary: c.summary, dossier: c.dossier_file ? `sources/${c.dossier_file}` : "", })); // dossier_file is not in the four-step export; take it from the new-structure export beside it const [h2, ...b2] = parseCsv(readFileSync(arg.replace("four-steps", "new-structure"), "utf8").replace(/^/, "")); const dossier = new Map(b2.map((r) => [Number(r[h2.indexOf("id")]), r[h2.indexOf("dossier_file")]])); for (const it of items) it.dossier = dossier.get(it.id) ? `sources/${dossier.get(it.id)}` : ""; for (let i = 0; i * BATCH < items.length; i++) { const nn = String(i + 1).padStart(2, "0"); writeFileSync(`steps/in-${nn}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1)); } console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches`); } else if (cmd === "validate") { let problems = 0, done = 0; for (const f of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) { const nn = f.slice(3, 5); const input: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8")); if (!existsSync(`steps/out-${nn}.json`)) { console.log(`out-${nn}.json missing`); problems++; continue; } let out: any[]; try { out = JSON.parse(readFileSync(`steps/out-${nn}.json`, "utf8")); } catch (e) { console.log(`out-${nn}.json: not valid JSON`); problems++; continue; } const say = (id: unknown, msg: string) => (problems++, console.log(`out-${nn} case ${id}: ${msg}`)); const ids = new Set(out.map((o) => o.id)); if (ids.size !== out.length) say("-", "duplicate case ids in the file"); for (const c of input) if (!ids.has(c.id)) say(c.id, "missing"); for (const o of out) { const src = input.find((c) => c.id === o.id); if (!src) { say(o.id, "not in this batch"); continue; } done++; for (const step of STEPS) { const spec = LABELS[step]; const v = o[step]; const values: unknown[] = spec.several ? v : [v]; if (spec.several ? !Array.isArray(v) || !v.length : typeof v !== "string") { say(o.id, `${step}: wrong shape`); continue; } for (const x of values) if (!isLabel(step, x)) say(o.id, `${step}: unknown label "${x}"`); if (spec.several && mixesEmpty(values)) say(o.id, `${step}: "${values.join(" | ")}" mixes an empty value with others`); } if (typeof o.attack_failed !== "boolean") say(o.id, "attack_failed: not true/false"); for (const k of ["demanded", "taken"]) { const m = o[k]; if (!isAmount(m)) { say(o.id, `${k}: not a complete amount object`); continue; } // a bare "$" from a Canadian or Australian source: currency may stay null when as_stated says so if (m.amount != null && !m.currency && !m.as_stated) say(o.id, `${k}: amount without currency`); } for (const step of ["targeting", "approach", "attack", "outcome", "amounts"]) if (typeof o.evidence?.[step] !== "string" || !o.evidence[step]) say(o.id, `evidence.${step}: empty or not text`); // a verified attack label may not be dropped if (Array.isArray(o.attack)) for (const a of src.current.attack) if (a !== "no label fits" && !o.attack.includes(a) && !o.dropped_verified_label_reason) say(o.id, `attack: verified label "${a}" dropped without a reason`); } } console.log(`${done} cases checked, ${problems} problems`); if (problems) process.exit(1); } else if (cmd === "merge") { // first reader's values, with the second reader's "contradicted" findings applied; // "arguable" findings change nothing and are listed in steps/open-points.csv const FIELDS = [...STEPS, "attack_failed", "demanded", "taken"]; const merged: any[] = []; const points: string[][] = [["id", "field", "kind", "applied", "first_reader", "second_reader", "quote", "source"]]; const esc = (v: unknown) => { const t = (typeof v === "string" ? v : JSON.stringify(v ?? "")).replace(/\s*[\r\n]+\s*/g, " "); return '"' + t.replace(/"/g, '""') + '"'; // every field quoted, so a semicolon cannot split a cell }; const validValue = (field: string, v: any) => { if (field === "attack_failed") return typeof v === "boolean"; if (field === "demanded" || field === "taken") return isAmount(v); const spec = LABELS[field]; const values = spec.several ? v : [v]; return (spec.several ? Array.isArray(v) && v.length > 0 && !mixesEmpty(v) : typeof v === "string") && values.every((x: unknown) => isLabel(field, x)); }; let applied = 0, rejected = 0, unchecked = 0; const seen = new Set(); for (const inFile of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) { const nn = inFile.slice(3, 5); const f = `out-${nn}.json`; if (!existsSync(`steps/${f}`)) throw new Error(`${f} missing: every input batch needs a first read before merging`); const out: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8")); const hasCheck = existsSync(`steps/check-${nn}.json`); const findings: any[] = hasCheck ? JSON.parse(readFileSync(`steps/check-${nn}.json`, "utf8")) : []; if (!hasCheck) unchecked += out.length; const byId = new Map(out.map((o) => [o.id, { ...o, second_reader: hasCheck ? "checked" : "not checked", open_points: 0 }])); if (byId.size !== out.length) throw new Error(`out-${nn}.json: duplicate case ids; run validate`); for (const id of byId.keys()) if (seen.has(id)) throw new Error(`case ${id} appears in two batches`); else seen.add(id); for (const x of findings) { const c = byId.get(x.id); // a finding written against an older first read no longer applies const current = c && JSON.stringify(c[x.field]) === JSON.stringify(x.reader_value); const ok = c && FIELDS.includes(x.field) && validValue(x.field, x.proposed_value) && x.quote && (current || x.kind !== "contradicted"); const apply = ok && x.kind === "contradicted"; if (!ok) rejected++; if (apply) (c[x.field] = x.proposed_value), applied++; else if (c) c.open_points++; points.push([x.id, x.field, ok ? x.kind : `${x.kind} (malformed, ignored)`, apply ? "yes" : "no", x.reader_value, x.proposed_value, x.quote, x.source].map(esc)); } merged.push(...byId.values()); } // keep attack_failed and obtained consistent after corrections // attack_failed is the readers' "attackers got nothing and were stopped". The owner ruled on 2026-10-06 // that this is not the same as foiled, so it stays in merged.json and is left out of the export: // outcome "foiled" carries the old Dodged Attack, and obtained "nothing" carries the empty-handed cases. writeFileSync("steps/merged.json", JSON.stringify(merged, null, 1)); writeFileSync("steps/open-points.csv", "\uFEFF" + points.map((r) => r.join(",")).join("\r\n") + "\r\n"); console.log(`${merged.length} cases merged; ${applied} corrections applied, ${points.length - 1 - applied - rejected} open points, ${rejected} malformed findings ignored, ${unchecked} cases without a second read`); } else throw new Error("usage: bun steps_pass.ts prepare | validate | merge");