This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/fields_pass.ts
T
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

127 lines
9.1 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Field pass: turn the written rulings into field values, with a second reader confirming each change.
// bun fields_pass.ts prepare exports/gart-kr-new-structure-YYYY-MM-DD.csv -> fields/in-NN.json
// bun fields_pass.ts validate checks fields/out-NN.json mechanically
// bun fields_pass.ts merge -> fields/merged.json (confirmed changes only) and fields/not-applied.json
// A change is applied only when: its field is allowed, its quote is found in the case's ruling text,
// its expected_old equals the record value, and the second reader confirmed it.
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
const FIELDS = ["date", "victim", "location", "country", "description", "notes", "money_wanted", "coin_type", "reports", "arrests"];
const ARRESTS = ["none", "some", "all", "convicted"];
const BATCH = 25;
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
const parseCsv = (text: string): string[][] => {
const rows: string[][] = [];
let row: string[] = [], field = "", quoted = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (quoted) {
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
else if (c === '"') quoted = false;
else field += c;
} else if (c === '"') quoted = true;
else if (c === ",") row.push(field), (field = "");
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
else field += c;
}
if (quoted) throw new Error("unterminated quoted field");
if (field || row.length) row.push(field), rows.push(row);
return rows;
};
const batches = () => readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5));
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const [cmd, arg] = process.argv.slice(2);
if (cmd === "prepare") {
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, ""));
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
const items: any[] = [];
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
const rec = record.get(id);
if (!rulings.length || !rec) continue;
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
}
for (let i = 0; i * BATCH < items.length; i++)
writeFileSync(`fields/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
console.log(`${items.length} cases with rulings in ${Math.ceil(items.length / BATCH)} batches`);
} else if (cmd === "prepare-new") {
// cases whose ruling lines changed since their batch was prepared (or that had none): a new batch
// with the full current rulings. In merge, a later batch replaces the case's earlier reading.
const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^\uFEFF/, ""));
const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))]));
const seen = new Map<number, string>();
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) seen.set(c.id, JSON.stringify(c.rulings));
const items: any[] = [];
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, ""));
const rec = record.get(id);
if (!rulings.length || !rec || seen.get(Number(id)) === JSON.stringify(rulings)) continue;
items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings });
}
const first = Math.max(0, ...batches().map(Number)) + 1;
for (let i = 0; i * BATCH < items.length; i++)
writeFileSync(`fields/in-${String(first + i).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
console.log(`${items.length} cases with new or changed rulings in batches ${first} to ${first + Math.ceil(items.length / BATCH) - 1}`);
} else if (cmd === "validate" || cmd === "merge") {
const merged: any[] = [];
const notApplied: any[] = [];
let problems = 0, applied = 0, missing = 0;
const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`));
for (const nn of batches()) {
const input: any[] = J(`fields/in-${nn}.json`);
if (!existsSync(`fields/out-${nn}.json`)) {
missing++;
console.log(`out-${nn}.json missing`);
for (const c of input) notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: `${c.rulings.length} ruling lines not applied: first reader's batch ${nn} is missing` });
continue;
}
const out: any[] = J(`fields/out-${nn}.json`);
const verdicts: any[] = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs ? J(`fields/check-${nn}.json`) : [];
// a second read older than the first read it should be judging does not count
const checked = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs;
const byId = new Map(out.map((o) => [o.id, o]));
if (byId.size !== out.length) say(nn, "-", "duplicate case ids");
for (const c of input) {
const o = byId.get(c.id);
if (!o) { say(nn, c.id, "missing"); notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: "case missing from the first reader's output" }); continue; }
const text = norm(c.rulings.join(" \n "));
const changes: any[] = [];
const seen = new Set<string>();
for (const ch of Array.isArray(o.changes) ? o.changes : []) {
const fail = (why: string) => { say(nn, c.id, `${ch.field}: ${why}`); notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the script", field: ch.field, detail: why, proposed: ch.new, quote: ch.quote }); };
if (!FIELDS.includes(ch.field)) { fail("field not allowed"); continue; }
if (seen.has(ch.field)) { fail("field changed twice in one case"); continue; }
if (typeof ch.new !== "string" || typeof ch.quote !== "string" || !ch.quote) { fail("new value or quote missing"); continue; }
if (!text.includes(norm(ch.quote))) { fail("quote not found in the ruling text"); continue; }
if (norm(ch.expected_old) !== norm(c.record[ch.field])) { fail("expected_old does not match the record"); continue; }
if (norm(ch.new) === norm(c.record[ch.field])) { fail("new value equals the record"); continue; }
if (ch.field === "arrests" && !ARRESTS.includes(ch.new)) { fail("arrests value not in the list"); continue; }
seen.add(ch.field);
const v = verdicts.find((x) => x.id === c.id && x.field === ch.field);
if (!checked) { notApplied.push({ batch: nn, id: c.id, kind: "change not second-read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; }
if (!v || v.verdict !== "confirm") { notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the second reader", field: ch.field, proposed: ch.new, quote: ch.quote, detail: v?.reason ?? "no verdict given" }); continue; }
changes.push({ field: ch.field, old: c.record[ch.field], new: ch.new.trim(), quote: ch.quote });
applied++;
}
for (const n of Array.isArray(o.not_applied) ? o.not_applied : []) notApplied.push({ batch: nn, id: c.id, kind: "ruling needs a person", detail: n.reason, quote: n.quote });
for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ batch: nn, id: c.id, kind: "change the first reader missed", field: v.field, proposed: v.proposed, detail: v.reason, quote: v.quote });
merged.push({ id: c.id, changes, batch: nn });
}
}
// a case read again in a later batch (its rulings changed) keeps only the later reading
const lastBatch = new Map<number, string>();
for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) lastBatch.set(c.id, nn);
const current = (id: number, nn?: string) => !nn || lastBatch.get(id) === nn;
for (let i = merged.length - 1; i >= 0; i--) if (!current(merged[i].id, merged[i].batch)) { applied -= merged[i].changes.length; merged.splice(i, 1); }
for (let i = notApplied.length - 1; i >= 0; i--) if (!current(notApplied[i].id, notApplied[i].batch)) notApplied.splice(i, 1);
if (cmd === "merge") {
writeFileSync("fields/merged.json", JSON.stringify(merged, null, 1));
writeFileSync("fields/not-applied.json", JSON.stringify(notApplied, null, 1));
}
console.log(`${merged.length} cases read, ${applied} changes ${cmd === "merge" ? "applied" : "would apply"}, ${notApplied.length} items not applied, ${problems} mechanical rejections, ${missing} batches missing`);
} else throw new Error("usage: bun fields_pass.ts prepare <new-structure export.csv> | prepare-new <new-structure export.csv> | validate | merge");