Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
127 lines
9.1 KiB
TypeScript
127 lines
9.1 KiB
TypeScript
// Field pass: turn the written rulings into field values, with a second reader confirming each change.
|
||
// bun fields_pass.ts prepare exports/gart-kr-new-structure-YYYY-MM-DD.csv -> fields/in-NN.json
|
||
// bun fields_pass.ts validate checks fields/out-NN.json mechanically
|
||
// bun fields_pass.ts merge -> fields/merged.json (confirmed changes only) and fields/not-applied.json
|
||
// A change is applied only when: its field is allowed, its quote is found in the case's ruling text,
|
||
// its expected_old equals the record value, and the second reader confirmed it.
|
||
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
|
||
|
||
const FIELDS = ["date", "victim", "location", "country", "description", "notes", "money_wanted", "coin_type", "reports", "arrests"];
|
||
const ARRESTS = ["none", "some", "all", "convicted"];
|
||
const BATCH = 25;
|
||
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
|
||
|
||
const parseCsv = (text: string): string[][] => {
|
||
const rows: string[][] = [];
|
||
let row: string[] = [], field = "", quoted = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const c = text[i];
|
||
if (quoted) {
|
||
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
|
||
else if (c === '"') quoted = false;
|
||
else field += c;
|
||
} else if (c === '"') quoted = true;
|
||
else if (c === ",") row.push(field), (field = "");
|
||
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
|
||
else field += c;
|
||
}
|
||
if (quoted) throw new Error("unterminated quoted field");
|
||
if (field || row.length) row.push(field), rows.push(row);
|
||
return rows;
|
||
};
|
||
const batches = () => readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5));
|
||
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
||
|
||
const [cmd, arg] = process.argv.slice(2);
|
||
if (cmd === "prepare") {
|
||
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, ""));
|
||
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
|
||
const items: any[] = [];
|
||
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
|
||
const id = /^\d+/.exec(block)![0];
|
||
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
|
||
const rec = record.get(id);
|
||
if (!rulings.length || !rec) continue;
|
||
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
|
||
}
|
||
for (let i = 0; i * BATCH < items.length; i++)
|
||
writeFileSync(`fields/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
|
||
console.log(`${items.length} cases with rulings in ${Math.ceil(items.length / BATCH)} batches`);
|
||
} else if (cmd === "prepare-new") {
|
||
// cases whose ruling lines changed since their batch was prepared (or that had none): a new batch
|
||
// with the full current rulings. In merge, a later batch replaces the case's earlier reading.
|
||
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^\uFEFF/, ""));
|
||
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
|
||
const seen = new Map<number, string>();
|
||
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) seen.set(c.id, JSON.stringify(c.rulings));
|
||
const items: any[] = [];
|
||
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
|
||
const id = /^\d+/.exec(block)![0];
|
||
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
|
||
const rec = record.get(id);
|
||
if (!rulings.length || !rec || seen.get(Number(id)) === JSON.stringify(rulings)) continue;
|
||
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
|
||
}
|
||
const first = Math.max(0, ...batches().map(Number)) + 1;
|
||
for (let i = 0; i * BATCH < items.length; i++)
|
||
writeFileSync(`fields/in-${String(first + i).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
|
||
console.log(`${items.length} cases with new or changed rulings in batches ${first} to ${first + Math.ceil(items.length / BATCH) - 1}`);
|
||
} else if (cmd === "validate" || cmd === "merge") {
|
||
const merged: any[] = [];
|
||
const notApplied: any[] = [];
|
||
let problems = 0, applied = 0, missing = 0;
|
||
const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`));
|
||
for (const nn of batches()) {
|
||
const input: any[] = J(`fields/in-${nn}.json`);
|
||
if (!existsSync(`fields/out-${nn}.json`)) {
|
||
missing++;
|
||
console.log(`out-${nn}.json missing`);
|
||
for (const c of input) notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: `${c.rulings.length} ruling lines not applied: first reader's batch ${nn} is missing` });
|
||
continue;
|
||
}
|
||
const out: any[] = J(`fields/out-${nn}.json`);
|
||
const verdicts: any[] = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs ? J(`fields/check-${nn}.json`) : [];
|
||
// a second read older than the first read it should be judging does not count
|
||
const checked = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs;
|
||
const byId = new Map(out.map((o) => [o.id, o]));
|
||
if (byId.size !== out.length) say(nn, "-", "duplicate case ids");
|
||
for (const c of input) {
|
||
const o = byId.get(c.id);
|
||
if (!o) { say(nn, c.id, "missing"); notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: "case missing from the first reader's output" }); continue; }
|
||
const text = norm(c.rulings.join(" \n "));
|
||
const changes: any[] = [];
|
||
const seen = new Set<string>();
|
||
for (const ch of Array.isArray(o.changes) ? o.changes : []) {
|
||
const fail = (why: string) => { say(nn, c.id, `${ch.field}: ${why}`); notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the script", field: ch.field, detail: why, proposed: ch.new, quote: ch.quote }); };
|
||
if (!FIELDS.includes(ch.field)) { fail("field not allowed"); continue; }
|
||
if (seen.has(ch.field)) { fail("field changed twice in one case"); continue; }
|
||
if (typeof ch.new !== "string" || typeof ch.quote !== "string" || !ch.quote) { fail("new value or quote missing"); continue; }
|
||
if (!text.includes(norm(ch.quote))) { fail("quote not found in the ruling text"); continue; }
|
||
if (norm(ch.expected_old) !== norm(c.record[ch.field])) { fail("expected_old does not match the record"); continue; }
|
||
if (norm(ch.new) === norm(c.record[ch.field])) { fail("new value equals the record"); continue; }
|
||
if (ch.field === "arrests" && !ARRESTS.includes(ch.new)) { fail("arrests value not in the list"); continue; }
|
||
seen.add(ch.field);
|
||
const v = verdicts.find((x) => x.id === c.id && x.field === ch.field);
|
||
if (!checked) { notApplied.push({ batch: nn, id: c.id, kind: "change not second-read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; }
|
||
if (!v || v.verdict !== "confirm") { notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the second reader", field: ch.field, proposed: ch.new, quote: ch.quote, detail: v?.reason ?? "no verdict given" }); continue; }
|
||
changes.push({ field: ch.field, old: c.record[ch.field], new: ch.new.trim(), quote: ch.quote });
|
||
applied++;
|
||
}
|
||
for (const n of Array.isArray(o.not_applied) ? o.not_applied : []) notApplied.push({ batch: nn, id: c.id, kind: "ruling needs a person", detail: n.reason, quote: n.quote });
|
||
for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ batch: nn, id: c.id, kind: "change the first reader missed", field: v.field, proposed: v.proposed, detail: v.reason, quote: v.quote });
|
||
merged.push({ id: c.id, changes, batch: nn });
|
||
}
|
||
}
|
||
// a case read again in a later batch (its rulings changed) keeps only the later reading
|
||
const lastBatch = new Map<number, string>();
|
||
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) lastBatch.set(c.id, nn);
|
||
const current = (id: number, nn?: string) => !nn || lastBatch.get(id) === nn;
|
||
for (let i = merged.length - 1; i >= 0; i--) if (!current(merged[i].id, merged[i].batch)) { applied -= merged[i].changes.length; merged.splice(i, 1); }
|
||
for (let i = notApplied.length - 1; i >= 0; i--) if (!current(notApplied[i].id, notApplied[i].batch)) notApplied.splice(i, 1);
|
||
if (cmd === "merge") {
|
||
writeFileSync("fields/merged.json", JSON.stringify(merged, null, 1));
|
||
writeFileSync("fields/not-applied.json", JSON.stringify(notApplied, null, 1));
|
||
}
|
||
console.log(`${merged.length} cases read, ${applied} changes ${cmd === "merge" ? "applied" : "would apply"}, ${notApplied.length} items not applied, ${problems} mechanical rejections, ${missing} batches missing`);
|
||
} else throw new Error("usage: bun fields_pass.ts prepare <new-structure export.csv> | prepare-new <new-structure export.csv> | validate | merge");
|