chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+126
View File
@@ -0,0 +1,126 @@
// Field pass: turn the written rulings into field values, with a second reader confirming each change.
// bun fields_pass.ts prepare exports/gart-kr-new-structure-YYYY-MM-DD.csv -> fields/in-NN.json
// bun fields_pass.ts validate checks fields/out-NN.json mechanically
// bun fields_pass.ts merge -> fields/merged.json (confirmed changes only) and fields/not-applied.json
// A change is applied only when: its field is allowed, its quote is found in the case's ruling text,
// its expected_old equals the record value, and the second reader confirmed it.
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
const FIELDS = ["date", "victim", "location", "country", "description", "notes", "money_wanted", "coin_type", "reports", "arrests"];
const ARRESTS = ["none", "some", "all", "convicted"];
const BATCH = 25;
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
const parseCsv = (text: string): string[][] => {
const rows: string[][] = [];
let row: string[] = [], field = "", quoted = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (quoted) {
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
else if (c === '"') quoted = false;
else field += c;
} else if (c === '"') quoted = true;
else if (c === ",") row.push(field), (field = "");
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
else field += c;
}
if (quoted) throw new Error("unterminated quoted field");
if (field || row.length) row.push(field), rows.push(row);
return rows;
};
const batches = () => readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5));
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const [cmd, arg] = process.argv.slice(2);
if (cmd === "prepare") {
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, ""));
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
const items: any[] = [];
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
const rec = record.get(id);
if (!rulings.length || !rec) continue;
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
}
for (let i = 0; i * BATCH < items.length; i++)
writeFileSync(`fields/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
console.log(`${items.length} cases with rulings in ${Math.ceil(items.length / BATCH)} batches`);
} else if (cmd === "prepare-new") {
// cases whose ruling lines changed since their batch was prepared (or that had none): a new batch
// with the full current rulings. In merge, a later batch replaces the case's earlier reading.
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^\uFEFF/, ""));
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
const seen = new Map<number, string>();
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) seen.set(c.id, JSON.stringify(c.rulings));
const items: any[] = [];
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
const rec = record.get(id);
if (!rulings.length || !rec || seen.get(Number(id)) === JSON.stringify(rulings)) continue;
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
}
const first = Math.max(0, ...batches().map(Number)) + 1;
for (let i = 0; i * BATCH < items.length; i++)
writeFileSync(`fields/in-${String(first + i).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
console.log(`${items.length} cases with new or changed rulings in batches ${first} to ${first + Math.ceil(items.length / BATCH) - 1}`);
} else if (cmd === "validate" || cmd === "merge") {
const merged: any[] = [];
const notApplied: any[] = [];
let problems = 0, applied = 0, missing = 0;
const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`));
for (const nn of batches()) {
const input: any[] = J(`fields/in-${nn}.json`);
if (!existsSync(`fields/out-${nn}.json`)) {
missing++;
console.log(`out-${nn}.json missing`);
for (const c of input) notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: `${c.rulings.length} ruling lines not applied: first reader's batch ${nn} is missing` });
continue;
}
const out: any[] = J(`fields/out-${nn}.json`);
const verdicts: any[] = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs ? J(`fields/check-${nn}.json`) : [];
// a second read older than the first read it should be judging does not count
const checked = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs;
const byId = new Map(out.map((o) => [o.id, o]));
if (byId.size !== out.length) say(nn, "-", "duplicate case ids");
for (const c of input) {
const o = byId.get(c.id);
if (!o) { say(nn, c.id, "missing"); notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: "case missing from the first reader's output" }); continue; }
const text = norm(c.rulings.join(" \n "));
const changes: any[] = [];
const seen = new Set<string>();
for (const ch of Array.isArray(o.changes) ? o.changes : []) {
const fail = (why: string) => { say(nn, c.id, `${ch.field}: ${why}`); notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the script", field: ch.field, detail: why, proposed: ch.new, quote: ch.quote }); };
if (!FIELDS.includes(ch.field)) { fail("field not allowed"); continue; }
if (seen.has(ch.field)) { fail("field changed twice in one case"); continue; }
if (typeof ch.new !== "string" || typeof ch.quote !== "string" || !ch.quote) { fail("new value or quote missing"); continue; }
if (!text.includes(norm(ch.quote))) { fail("quote not found in the ruling text"); continue; }
if (norm(ch.expected_old) !== norm(c.record[ch.field])) { fail("expected_old does not match the record"); continue; }
if (norm(ch.new) === norm(c.record[ch.field])) { fail("new value equals the record"); continue; }
if (ch.field === "arrests" && !ARRESTS.includes(ch.new)) { fail("arrests value not in the list"); continue; }
seen.add(ch.field);
const v = verdicts.find((x) => x.id === c.id && x.field === ch.field);
if (!checked) { notApplied.push({ batch: nn, id: c.id, kind: "change not second-read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; }
if (!v || v.verdict !== "confirm") { notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the second reader", field: ch.field, proposed: ch.new, quote: ch.quote, detail: v?.reason ?? "no verdict given" }); continue; }
changes.push({ field: ch.field, old: c.record[ch.field], new: ch.new.trim(), quote: ch.quote });
applied++;
}
for (const n of Array.isArray(o.not_applied) ? o.not_applied : []) notApplied.push({ batch: nn, id: c.id, kind: "ruling needs a person", detail: n.reason, quote: n.quote });
for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ batch: nn, id: c.id, kind: "change the first reader missed", field: v.field, proposed: v.proposed, detail: v.reason, quote: v.quote });
merged.push({ id: c.id, changes, batch: nn });
}
}
// a case read again in a later batch (its rulings changed) keeps only the later reading
const lastBatch = new Map<number, string>();
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) lastBatch.set(c.id, nn);
const current = (id: number, nn?: string) => !nn || lastBatch.get(id) === nn;
for (let i = merged.length - 1; i >= 0; i--) if (!current(merged[i].id, merged[i].batch)) { applied -= merged[i].changes.length; merged.splice(i, 1); }
for (let i = notApplied.length - 1; i >= 0; i--) if (!current(notApplied[i].id, notApplied[i].batch)) notApplied.splice(i, 1);
if (cmd === "merge") {
writeFileSync("fields/merged.json", JSON.stringify(merged, null, 1));
writeFileSync("fields/not-applied.json", JSON.stringify(notApplied, null, 1));
}
console.log(`${merged.length} cases read, ${applied} changes ${cmd === "merge" ? "applied" : "would apply"}, ${notApplied.length} items not applied, ${problems} mechanical rejections, ${missing} batches missing`);
} else throw new Error("usage: bun fields_pass.ts prepare <new-structure export.csv> | prepare-new <new-structure export.csv> | validate | merge");