// Field pass: turn the written rulings into field values, with a second reader confirming each change. // bun fields_pass.ts prepare exports/gart-kr-new-structure-YYYY-MM-DD.csv -> fields/in-NN.json // bun fields_pass.ts validate checks fields/out-NN.json mechanically // bun fields_pass.ts merge -> fields/merged.json (confirmed changes only) and fields/not-applied.json // A change is applied only when: its field is allowed, its quote is found in the case's ruling text, // its expected_old equals the record value, and the second reader confirmed it. import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs"; const FIELDS = ["date", "victim", "location", "country", "description", "notes", "money_wanted", "coin_type", "reports", "arrests"]; const ARRESTS = ["none", "some", "all", "convicted"]; const BATCH = 25; const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim(); const parseCsv = (text: string): string[][] => { const rows: string[][] = []; let row: string[] = [], field = "", quoted = false; for (let i = 0; i < text.length; i++) { const c = text[i]; if (quoted) { if (c === '"' && text[i + 1] === '"') (field += '"'), i++; else if (c === '"') quoted = false; else field += c; } else if (c === '"') quoted = true; else if (c === ",") row.push(field), (field = ""); else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = ""); else field += c; } if (quoted) throw new Error("unterminated quoted field"); if (field || row.length) row.push(field), rows.push(row); return rows; }; const batches = () => readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5)); const J = (p: string) => JSON.parse(readFileSync(p, "utf8")); const [cmd, arg] = process.argv.slice(2); if (cmd === "prepare") { const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^/, "")); const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))])); const items: any[] = []; for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) { const id = /^\d+/.exec(block)![0]; const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, "")); const rec = record.get(id); if (!rulings.length || !rec) continue; items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings }); } for (let i = 0; i * BATCH < items.length; i++) writeFileSync(`fields/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1)); console.log(`${items.length} cases with rulings in ${Math.ceil(items.length / BATCH)} batches`); } else if (cmd === "prepare-new") { // cases whose ruling lines changed since their batch was prepared (or that had none): a new batch // with the full current rulings. In merge, a later batch replaces the case's earlier reading. const [head, ...body] = parseCsv(readFileSync(arg, "utf8").replace(/^\uFEFF/, "")); const record = new Map(body.map((r) => [r[0], Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""]))])); const seen = new Map(); for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) seen.set(c.id, JSON.stringify(c.rulings)); const items: any[] = []; for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) { const id = /^\d+/.exec(block)![0]; const rulings = block.split("\n").filter((l) => /^\s+- \[/.test(l)).map((l) => l.replace(/^\s+- /, "")); const rec = record.get(id); if (!rulings.length || !rec || seen.get(Number(id)) === JSON.stringify(rulings)) continue; items.push({ id: Number(id), record: Object.fromEntries(FIELDS.map((f) => [f, rec[f] ?? ""])), rulings }); } const first = Math.max(0, ...batches().map(Number)) + 1; for (let i = 0; i * BATCH < items.length; i++) writeFileSync(`fields/in-${String(first + i).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1)); console.log(`${items.length} cases with new or changed rulings in batches ${first} to ${first + Math.ceil(items.length / BATCH) - 1}`); } else if (cmd === "validate" || cmd === "merge") { const merged: any[] = []; const notApplied: any[] = []; let problems = 0, applied = 0, missing = 0; const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`)); for (const nn of batches()) { const input: any[] = J(`fields/in-${nn}.json`); if (!existsSync(`fields/out-${nn}.json`)) { missing++; console.log(`out-${nn}.json missing`); for (const c of input) notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: `${c.rulings.length} ruling lines not applied: first reader's batch ${nn} is missing` }); continue; } const out: any[] = J(`fields/out-${nn}.json`); const verdicts: any[] = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs ? J(`fields/check-${nn}.json`) : []; // a second read older than the first read it should be judging does not count const checked = existsSync(`fields/check-${nn}.json`) && statSync(`fields/check-${nn}.json`).mtimeMs >= statSync(`fields/out-${nn}.json`).mtimeMs; const byId = new Map(out.map((o) => [o.id, o])); if (byId.size !== out.length) say(nn, "-", "duplicate case ids"); for (const c of input) { const o = byId.get(c.id); if (!o) { say(nn, c.id, "missing"); notApplied.push({ batch: nn, id: c.id, kind: "field pass not run", detail: "case missing from the first reader's output" }); continue; } const text = norm(c.rulings.join(" \n ")); const changes: any[] = []; const seen = new Set(); for (const ch of Array.isArray(o.changes) ? o.changes : []) { const fail = (why: string) => { say(nn, c.id, `${ch.field}: ${why}`); notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the script", field: ch.field, detail: why, proposed: ch.new, quote: ch.quote }); }; if (!FIELDS.includes(ch.field)) { fail("field not allowed"); continue; } if (seen.has(ch.field)) { fail("field changed twice in one case"); continue; } if (typeof ch.new !== "string" || typeof ch.quote !== "string" || !ch.quote) { fail("new value or quote missing"); continue; } if (!text.includes(norm(ch.quote))) { fail("quote not found in the ruling text"); continue; } if (norm(ch.expected_old) !== norm(c.record[ch.field])) { fail("expected_old does not match the record"); continue; } if (norm(ch.new) === norm(c.record[ch.field])) { fail("new value equals the record"); continue; } if (ch.field === "arrests" && !ARRESTS.includes(ch.new)) { fail("arrests value not in the list"); continue; } seen.add(ch.field); const v = verdicts.find((x) => x.id === c.id && x.field === ch.field); if (!checked) { notApplied.push({ batch: nn, id: c.id, kind: "change not second-read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; } if (!v || v.verdict !== "confirm") { notApplied.push({ batch: nn, id: c.id, kind: "change rejected by the second reader", field: ch.field, proposed: ch.new, quote: ch.quote, detail: v?.reason ?? "no verdict given" }); continue; } changes.push({ field: ch.field, old: c.record[ch.field], new: ch.new.trim(), quote: ch.quote }); applied++; } for (const n of Array.isArray(o.not_applied) ? o.not_applied : []) notApplied.push({ batch: nn, id: c.id, kind: "ruling needs a person", detail: n.reason, quote: n.quote }); for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ batch: nn, id: c.id, kind: "change the first reader missed", field: v.field, proposed: v.proposed, detail: v.reason, quote: v.quote }); merged.push({ id: c.id, changes, batch: nn }); } } // a case read again in a later batch (its rulings changed) keeps only the later reading const lastBatch = new Map(); for (const nn of batches()) for (const c of J(`fields/in-${nn}.json`)) lastBatch.set(c.id, nn); const current = (id: number, nn?: string) => !nn || lastBatch.get(id) === nn; for (let i = merged.length - 1; i >= 0; i--) if (!current(merged[i].id, merged[i].batch)) { applied -= merged[i].changes.length; merged.splice(i, 1); } for (let i = notApplied.length - 1; i >= 0; i--) if (!current(notApplied[i].id, notApplied[i].batch)) notApplied.splice(i, 1); if (cmd === "merge") { writeFileSync("fields/merged.json", JSON.stringify(merged, null, 1)); writeFileSync("fields/not-applied.json", JSON.stringify(notApplied, null, 1)); } console.log(`${merged.length} cases read, ${applied} changes ${cmd === "merge" ? "applied" : "would apply"}, ${notApplied.length} items not applied, ${problems} mechanical rejections, ${missing} batches missing`); } else throw new Error("usage: bun fields_pass.ts prepare | prepare-new | validate | merge");