This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/steps_pass.ts
T
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

165 lines
10 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Reading pass for the four-step structure.
// bun steps_pass.ts prepare exports/gart-kr-four-steps-YYYY-MM-DD.csv -> steps/in-NN.json (25 cases each)
// bun steps_pass.ts validate -> checks every steps/out-NN.json
import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
const LABELS = JSON.parse(readFileSync("step-labels.json", "utf8"));
const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"];
const BATCH = 25;
const isLabel = (step: string, x: unknown) => typeof x === "string" && Object.hasOwn(LABELS[step].values, x);
const EMPTY_VALUES = ["unknown", "nothing", "unconfirmed", "no label fits"];
// a list may not mix an empty value with real ones
const mixesEmpty = (values: unknown[]) => values.length > 1 && values.some((x) => EMPTY_VALUES.includes(String(x)));
const isAmount = (m: any) =>
m && typeof m === "object" && !Array.isArray(m) &&
["amount", "usd_stated"].every((n) => m[n] == null || typeof m[n] === "number") &&
(m.currency == null || typeof m.currency === "string") &&
typeof m.as_stated === "string" && m.as_stated.length > 0;
const NOT_SORTED = "not sorted yet";
const parseCsv = (text: string): string[][] => {
const rows: string[][] = [];
let row: string[] = [], field = "", quoted = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (quoted) {
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
else if (c === '"') quoted = false;
else field += c;
} else if (c === '"') quoted = true;
else if (c === ",") row.push(field), (field = "");
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
else field += c;
}
if (quoted) throw new Error("unterminated quoted field");
if (field || row.length) row.push(field), rows.push(row);
return rows;
};
const [cmd, arg] = process.argv.slice(2);
if (cmd === "prepare") {
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, ""));
const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
const list = (v: string) => v.split(" | ").filter((x) => x && x !== NOT_SORTED);
const items = cases.map((c) => ({
id: Number(c.id),
victim: c.victim,
date: c.date,
location: c.location,
country: c.country,
verified_labels_old_names: c.old_labels,
attempted: c.attempted,
current: {
targeting: list(c["1_targeting"]),
targeting_note_from_ruling: c["1_targeting_note"],
approach: list(c["2_approach"]),
attack: list(c["3_attack"]),
outcome: c["4_outcome"] === NOT_SORTED ? "" : c["4_outcome"],
attack_failed: c["4_attack_failed"] === "yes",
},
flags: { detention: c.detention, violence_torture: c.violence_torture, drugs_alcohol: c.drugs_alcohol, weapons: c.weapons, theft: c.theft, life_taken: c.life_taken },
money_wanted_db: c.money_wanted_db,
summary_is_verified: c.verification !== "not verified",
summary: c.summary,
dossier: c.dossier_file ? `sources/${c.dossier_file}` : "",
}));
// dossier_file is not in the four-step export; take it from the new-structure export beside it
const [h2, ...b2] = parseCsv(readFileSync(arg.replace("four-steps", "new-structure"), "utf8").replace(/^/, ""));
const dossier = new Map(b2.map((r) => [Number(r[h2.indexOf("id")]), r[h2.indexOf("dossier_file")]]));
for (const it of items) it.dossier = dossier.get(it.id) ? `sources/${dossier.get(it.id)}` : "";
for (let i = 0; i * BATCH < items.length; i++) {
const nn = String(i + 1).padStart(2, "0");
writeFileSync(`steps/in-${nn}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
}
console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches`);
} else if (cmd === "validate") {
let problems = 0, done = 0;
for (const f of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) {
const nn = f.slice(3, 5);
const input: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8"));
if (!existsSync(`steps/out-${nn}.json`)) { console.log(`out-${nn}.json missing`); problems++; continue; }
let out: any[];
try { out = JSON.parse(readFileSync(`steps/out-${nn}.json`, "utf8")); } catch (e) { console.log(`out-${nn}.json: not valid JSON`); problems++; continue; }
const say = (id: unknown, msg: string) => (problems++, console.log(`out-${nn} case ${id}: ${msg}`));
const ids = new Set(out.map((o) => o.id));
if (ids.size !== out.length) say("-", "duplicate case ids in the file");
for (const c of input) if (!ids.has(c.id)) say(c.id, "missing");
for (const o of out) {
const src = input.find((c) => c.id === o.id);
if (!src) { say(o.id, "not in this batch"); continue; }
done++;
for (const step of STEPS) {
const spec = LABELS[step];
const v = o[step];
const values: unknown[] = spec.several ? v : [v];
if (spec.several ? !Array.isArray(v) || !v.length : typeof v !== "string") { say(o.id, `${step}: wrong shape`); continue; }
for (const x of values) if (!isLabel(step, x)) say(o.id, `${step}: unknown label "${x}"`);
if (spec.several && mixesEmpty(values)) say(o.id, `${step}: "${values.join(" | ")}" mixes an empty value with others`);
}
if (typeof o.attack_failed !== "boolean") say(o.id, "attack_failed: not true/false");
for (const k of ["demanded", "taken"]) {
const m = o[k];
if (!isAmount(m)) { say(o.id, `${k}: not a complete amount object`); continue; }
// a bare "$" from a Canadian or Australian source: currency may stay null when as_stated says so
if (m.amount != null && !m.currency && !m.as_stated) say(o.id, `${k}: amount without currency`);
}
for (const step of ["targeting", "approach", "attack", "outcome", "amounts"]) if (typeof o.evidence?.[step] !== "string" || !o.evidence[step]) say(o.id, `evidence.${step}: empty or not text`);
// a verified attack label may not be dropped
if (Array.isArray(o.attack))
for (const a of src.current.attack) if (a !== "no label fits" && !o.attack.includes(a) && !o.dropped_verified_label_reason) say(o.id, `attack: verified label "${a}" dropped without a reason`);
}
}
console.log(`${done} cases checked, ${problems} problems`);
if (problems) process.exit(1);
} else if (cmd === "merge") {
// first reader's values, with the second reader's "contradicted" findings applied;
// "arguable" findings change nothing and are listed in steps/open-points.csv
const FIELDS = [...STEPS, "attack_failed", "demanded", "taken"];
const merged: any[] = [];
const points: string[][] = [["id", "field", "kind", "applied", "first_reader", "second_reader", "quote", "source"]];
const esc = (v: unknown) => {
const t = (typeof v === "string" ? v : JSON.stringify(v ?? "")).replace(/\s*[\r\n]+\s*/g, " ");
return '"' + t.replace(/"/g, '""') + '"'; // every field quoted, so a semicolon cannot split a cell
};
const validValue = (field: string, v: any) => {
if (field === "attack_failed") return typeof v === "boolean";
if (field === "demanded" || field === "taken") return isAmount(v);
const spec = LABELS[field];
const values = spec.several ? v : [v];
return (spec.several ? Array.isArray(v) && v.length > 0 && !mixesEmpty(v) : typeof v === "string") && values.every((x: unknown) => isLabel(field, x));
};
let applied = 0, rejected = 0, unchecked = 0;
const seen = new Set<number>();
for (const inFile of readdirSync("steps").filter((f) => /^in-\d+\.json$/.test(f)).sort()) {
const nn = inFile.slice(3, 5);
const f = `out-${nn}.json`;
if (!existsSync(`steps/${f}`)) throw new Error(`${f} missing: every input batch needs a first read before merging`);
const out: any[] = JSON.parse(readFileSync(`steps/${f}`, "utf8"));
const hasCheck = existsSync(`steps/check-${nn}.json`);
const findings: any[] = hasCheck ? JSON.parse(readFileSync(`steps/check-${nn}.json`, "utf8")) : [];
if (!hasCheck) unchecked += out.length;
const byId = new Map(out.map((o) => [o.id, { ...o, second_reader: hasCheck ? "checked" : "not checked", open_points: 0 }]));
if (byId.size !== out.length) throw new Error(`out-${nn}.json: duplicate case ids; run validate`);
for (const id of byId.keys()) if (seen.has(id)) throw new Error(`case ${id} appears in two batches`); else seen.add(id);
for (const x of findings) {
const c = byId.get(x.id);
// a finding written against an older first read no longer applies
const current = c && JSON.stringify(c[x.field]) === JSON.stringify(x.reader_value);
const ok = c && FIELDS.includes(x.field) && validValue(x.field, x.proposed_value) && x.quote && (current || x.kind !== "contradicted");
const apply = ok && x.kind === "contradicted";
if (!ok) rejected++;
if (apply) (c[x.field] = x.proposed_value), applied++;
else if (c) c.open_points++;
points.push([x.id, x.field, ok ? x.kind : `${x.kind} (malformed, ignored)`, apply ? "yes" : "no", x.reader_value, x.proposed_value, x.quote, x.source].map(esc));
}
merged.push(...byId.values());
}
// keep attack_failed and obtained consistent after corrections
// attack_failed is the readers' "attackers got nothing and were stopped". The owner ruled on 2026-10-06
// that this is not the same as foiled, so it stays in merged.json and is left out of the export:
// outcome "foiled" carries the old Dodged Attack, and obtained "nothing" carries the empty-handed cases.
writeFileSync("steps/merged.json", JSON.stringify(merged, null, 1));
writeFileSync("steps/open-points.csv", "\uFEFF" + points.map((r) => r.join(",")).join("\r\n") + "\r\n");
console.log(`${merged.length} cases merged; ${applied} corrections applied, ${points.length - 1 - applied - rejected} open points, ${rejected} malformed findings ignored, ${unchecked} cases without a second read`);
} else throw new Error("usage: bun steps_pass.ts prepare <four-steps export.csv> | validate | merge");