Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
165 lines
10 KiB
TypeScript
165 lines
10 KiB
TypeScript
// Reading pass for the four-step structure.
|
||
// bun steps_pass.ts prepare exports/gart-kr-four-steps-YYYY-MM-DD.csv -> steps/in-NN.json (25 cases each)
|
||
// bun steps_pass.ts validate -> checks every steps/out-NN.json
|
||
import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
|
||
|
||
const LABELS = JSON.parse(readFileSync("step-labels.json", "utf8"));
|
||
const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"];
|
||
const BATCH = 25;
|
||
const isLabel = (step: string, x: unknown) => typeof x === "string" && Object.hasOwn(LABELS[step].values, x);
|
||
const EMPTY_VALUES = ["unknown", "nothing", "unconfirmed", "no label fits"];
|
||
// a list may not mix an empty value with real ones
|
||
const mixesEmpty = (values: unknown[]) => values.length > 1 && values.some((x) => EMPTY_VALUES.includes(String(x)));
|
||
const isAmount = (m: any) =>
|
||
m && typeof m === "object" && !Array.isArray(m) &&
|
||
["amount", "usd_stated"].every((n) => m[n] == null || typeof m[n] === "number") &&
|
||
(m.currency == null || typeof m.currency === "string") &&
|
||
typeof m.as_stated === "string" && m.as_stated.length > 0;
|
||
const NOT_SORTED = "not sorted yet";
|
||
|
||
const parseCsv = (text: string): string[][] => {
|
||
const rows: string[][] = [];
|
||
let row: string[] = [], field = "", quoted = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const c = text[i];
|
||
if (quoted) {
|
||
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
|
||
else if (c === '"') quoted = false;
|
||
else field += c;
|
||
} else if (c === '"') quoted = true;
|
||
else if (c === ",") row.push(field), (field = "");
|
||
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
|
||
else field += c;
|
||
}
|
||
if (quoted) throw new Error("unterminated quoted field");
|
||
if (field || row.length) row.push(field), rows.push(row);
|
||
return rows;
|
||
};
|
||
|
||
const [cmd, arg] = process.argv.slice(2);
|
||
if (cmd === "prepare") {
|
||
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, ""));
|
||
const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
|
||
const list = (v: string) => v.split(" | ").filter((x) => x && x !== NOT_SORTED);
|
||
const items = cases.map((c) => ({
|
||
id: Number(c.id),
|
||
victim: c.victim,
|
||
date: c.date,
|
||
location: c.location,
|
||
country: c.country,
|
||
verified_labels_old_names: c.old_labels,
|
||
attempted: c.attempted,
|
||
current: {
|
||
targeting: list(c["1_targeting"]),
|
||
targeting_note_from_ruling: c["1_targeting_note"],
|
||
approach: list(c["2_approach"]),
|
||
attack: list(c["3_attack"]),
|
||
outcome: c["4_outcome"] === NOT_SORTED ? "" : c["4_outcome"],
|
||
attack_failed: c["4_attack_failed"] === "yes",
|
||
},
|
||
flags: { detention: c.detention, violence_torture: c.violence_torture, drugs_alcohol: c.drugs_alcohol, weapons: c.weapons, theft: c.theft, life_taken: c.life_taken },
|
||
money_wanted_db: c.money_wanted_db,
|
||
summary_is_verified: c.verification !== "not verified",
|
||
summary: c.summary,
|
||
dossier: c.dossier_file ? `sources/${c.dossier_file}` : "",
|
||
}));
|
||
// dossier_file is not in the four-step export; take it from the new-structure export beside it
|
||
const [h2, ...b2] = parseCsv(readFileSync(arg.replace("four-steps", "new-structure"), "utf8").replace(/^/, ""));
|
||
const dossier = new Map(b2.map((r) => [Number(r[h2.indexOf("id")]), r[h2.indexOf("dossier_file")]]));
|
||
for (const it of items) it.dossier = dossier.get(it.id) ? `sources/${dossier.get(it.id)}` : "";
|
||
for (let i = 0; i * BATCH < items.length; i++) {
|
||
const nn = String(i + 1).padStart(2, "0");
|
||
writeFileSync(`steps/in-${nn}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
|
||
}
|
||
console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches`);
|
||
} else if (cmd === "validate") {
|
||
let problems = 0, done = 0;
|
||
for (const f of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) {
|
||
const nn = f.slice(3, 5);
|
||
const input: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8"));
|
||
if (!existsSync(`steps/out-${nn}.json`)) { console.log(`out-${nn}.json missing`); problems++; continue; }
|
||
let out: any[];
|
||
try { out = JSON.parse(readFileSync(`steps/out-${nn}.json`, "utf8")); } catch (e) { console.log(`out-${nn}.json: not valid JSON`); problems++; continue; }
|
||
const say = (id: unknown, msg: string) => (problems++, console.log(`out-${nn} case ${id}: ${msg}`));
|
||
const ids = new Set(out.map((o) => o.id));
|
||
if (ids.size !== out.length) say("-", "duplicate case ids in the file");
|
||
for (const c of input) if (!ids.has(c.id)) say(c.id, "missing");
|
||
for (const o of out) {
|
||
const src = input.find((c) => c.id === o.id);
|
||
if (!src) { say(o.id, "not in this batch"); continue; }
|
||
done++;
|
||
for (const step of STEPS) {
|
||
const spec = LABELS[step];
|
||
const v = o[step];
|
||
const values: unknown[] = spec.several ? v : [v];
|
||
if (spec.several ? !Array.isArray(v) || !v.length : typeof v !== "string") { say(o.id, `${step}: wrong shape`); continue; }
|
||
for (const x of values) if (!isLabel(step, x)) say(o.id, `${step}: unknown label "${x}"`);
|
||
if (spec.several && mixesEmpty(values)) say(o.id, `${step}: "${values.join(" | ")}" mixes an empty value with others`);
|
||
}
|
||
if (typeof o.attack_failed !== "boolean") say(o.id, "attack_failed: not true/false");
|
||
for (const k of ["demanded", "taken"]) {
|
||
const m = o[k];
|
||
if (!isAmount(m)) { say(o.id, `${k}: not a complete amount object`); continue; }
|
||
// a bare "$" from a Canadian or Australian source: currency may stay null when as_stated says so
|
||
if (m.amount != null && !m.currency && !m.as_stated) say(o.id, `${k}: amount without currency`);
|
||
}
|
||
for (const step of ["targeting", "approach", "attack", "outcome", "amounts"]) if (typeof o.evidence?.[step] !== "string" || !o.evidence[step]) say(o.id, `evidence.${step}: empty or not text`);
|
||
// a verified attack label may not be dropped
|
||
if (Array.isArray(o.attack))
|
||
for (const a of src.current.attack) if (a !== "no label fits" && !o.attack.includes(a) && !o.dropped_verified_label_reason) say(o.id, `attack: verified label "${a}" dropped without a reason`);
|
||
}
|
||
}
|
||
console.log(`${done} cases checked, ${problems} problems`);
|
||
if (problems) process.exit(1);
|
||
} else if (cmd === "merge") {
|
||
// first reader's values, with the second reader's "contradicted" findings applied;
|
||
// "arguable" findings change nothing and are listed in steps/open-points.csv
|
||
const FIELDS = [...STEPS, "attack_failed", "demanded", "taken"];
|
||
const merged: any[] = [];
|
||
const points: string[][] = [["id", "field", "kind", "applied", "first_reader", "second_reader", "quote", "source"]];
|
||
const esc = (v: unknown) => {
|
||
const t = (typeof v === "string" ? v : JSON.stringify(v ?? "")).replace(/\s*[\r\n]+\s*/g, " ");
|
||
return '"' + t.replace(/"/g, '""') + '"'; // every field quoted, so a semicolon cannot split a cell
|
||
};
|
||
const validValue = (field: string, v: any) => {
|
||
if (field === "attack_failed") return typeof v === "boolean";
|
||
if (field === "demanded" || field === "taken") return isAmount(v);
|
||
const spec = LABELS[field];
|
||
const values = spec.several ? v : [v];
|
||
return (spec.several ? Array.isArray(v) && v.length > 0 && !mixesEmpty(v) : typeof v === "string") && values.every((x: unknown) => isLabel(field, x));
|
||
};
|
||
let applied = 0, rejected = 0, unchecked = 0;
|
||
const seen = new Set<number>();
|
||
for (const inFile of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) {
|
||
const nn = inFile.slice(3, 5);
|
||
const f = `out-${nn}.json`;
|
||
if (!existsSync(`steps/${f}`)) throw new Error(`${f} missing: every input batch needs a first read before merging`);
|
||
const out: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8"));
|
||
const hasCheck = existsSync(`steps/check-${nn}.json`);
|
||
const findings: any[] = hasCheck ? JSON.parse(readFileSync(`steps/check-${nn}.json`, "utf8")) : [];
|
||
if (!hasCheck) unchecked += out.length;
|
||
const byId = new Map(out.map((o) => [o.id, { ...o, second_reader: hasCheck ? "checked" : "not checked", open_points: 0 }]));
|
||
if (byId.size !== out.length) throw new Error(`out-${nn}.json: duplicate case ids; run validate`);
|
||
for (const id of byId.keys()) if (seen.has(id)) throw new Error(`case ${id} appears in two batches`); else seen.add(id);
|
||
for (const x of findings) {
|
||
const c = byId.get(x.id);
|
||
// a finding written against an older first read no longer applies
|
||
const current = c && JSON.stringify(c[x.field]) === JSON.stringify(x.reader_value);
|
||
const ok = c && FIELDS.includes(x.field) && validValue(x.field, x.proposed_value) && x.quote && (current || x.kind !== "contradicted");
|
||
const apply = ok && x.kind === "contradicted";
|
||
if (!ok) rejected++;
|
||
if (apply) (c[x.field] = x.proposed_value), applied++;
|
||
else if (c) c.open_points++;
|
||
points.push([x.id, x.field, ok ? x.kind : `${x.kind} (malformed, ignored)`, apply ? "yes" : "no", x.reader_value, x.proposed_value, x.quote, x.source].map(esc));
|
||
}
|
||
merged.push(...byId.values());
|
||
}
|
||
// keep attack_failed and obtained consistent after corrections
|
||
// attack_failed is the readers' "attackers got nothing and were stopped". The owner ruled on 2026-10-06
|
||
// that this is not the same as foiled, so it stays in merged.json and is left out of the export:
|
||
// outcome "foiled" carries the old Dodged Attack, and obtained "nothing" carries the empty-handed cases.
|
||
writeFileSync("steps/merged.json", JSON.stringify(merged, null, 1));
|
||
writeFileSync("steps/open-points.csv", "\uFEFF" + points.map((r) => r.join(",")).join("\r\n") + "\r\n");
|
||
console.log(`${merged.length} cases merged; ${applied} corrections applied, ${points.length - 1 - applied - rejected} open points, ${rejected} malformed findings ignored, ${unchecked} cases without a second read`);
|
||
} else throw new Error("usage: bun steps_pass.ts prepare <four-steps export.csv> | validate | merge");
|