// Full label check, ordered by the owner on 2026-10-07: every label of every case is tested // against LABEL-RULES.md by a first reader and confirmed by a second reader. // bun labels_pass.ts prepare /release/data/cases.json -> labels/in-NN.json // bun labels_pass.ts validate // bun labels_pass.ts merge -> labels/merged.json (second-reader-confirmed changes) and labels/not-applied.json // bun labels_pass.ts tiebreak-prepare -> labels/tiebreak-in.json: the changes the two readers disagree on // A change is applied only when its quote is found in the case text and the second reader confirmed it. // Where the readers disagree (one proposed, the other rejected or never saw it), a third reader decides: // labels/tiebreak-out.json, [{id, field, verdict: "apply" | "keep", value, quote, reason}]. An "apply" counts // only for a value one of the two readers proposed, with a quote found in the case text. import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs"; const LABELS = JSON.parse(readFileSync("step-labels.json", "utf8")); const BATCH = 25; const FIELDS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"] as const; const J = (p: string) => JSON.parse(readFileSync(p, "utf8")); const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim(); const same = (a: unknown, b: unknown) => JSON.stringify(Array.isArray(a) ? [...a].sort() : a) === JSON.stringify(Array.isArray(b) ? [...b].sort() : b); const batches = () => readdirSync("labels").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5)); const [cmd, arg] = process.argv.slice(2); if (cmd === "prepare") { const cases: any[] = J(arg).cases; const rulings = existsSync("step-rulings.json") ? J("step-rulings.json").rulings : []; const items = cases.map((c) => ({ id: c.id, victim_field: c.victim, current: Object.fromEntries(FIELDS.map((f) => [f, c.steps[f]])), locked_by_owner: rulings.filter((r: any) => r.id === c.id).map((r: any) => r.field), flags: c.flags, summary: c.summary, dossier: c.dossier_file ? `sources/${c.dossier_file}` : "", })); for (let i = 0; i * BATCH < items.length; i++) writeFileSync(`labels/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1)); console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches`); } else if (cmd === "validate" || cmd === "merge" || cmd === "tiebreak-prepare") { const merged: any[] = []; let notApplied: any[] = []; const caseOf = new Map(); let problems = 0, applied = 0, missing = 0, read = 0; const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`)); for (const nn of batches()) { const input: any[] = J(`labels/in-${nn}.json`); if (!existsSync(`labels/out-${nn}.json`)) { missing++; console.log(`out-${nn}.json missing`); continue; } const out: any[] = J(`labels/out-${nn}.json`); const checked = existsSync(`labels/check-${nn}.json`) && statSync(`labels/check-${nn}.json`).mtimeMs > statSync(`labels/out-${nn}.json`).mtimeMs; const verdicts: any[] = checked ? J(`labels/check-${nn}.json`) : []; const byId = new Map(out.map((o) => [o.id, o])); for (const c of input) { const o = byId.get(c.id); if (!o) { say(nn, c.id, "missing"); notApplied.push({ id: c.id, kind: "label check: case not read", detail: "missing from the first reader's output" }); continue; } read++; const text = norm(c.summary) + " " + (c.dossier && existsSync(c.dossier) ? norm(readFileSync(c.dossier, "utf8")) : ""); caseOf.set(c.id, { input: c, text }); const changes: any[] = []; for (const field of FIELDS) { if (c.locked_by_owner.includes(field)) continue; // an owner ruling stands const value = o[field]; const spec = LABELS[field]; const values: unknown[] = spec.several ? value : [value]; if (spec.several ? !Array.isArray(value) || !value.length : typeof value !== "string") { say(nn, c.id, `${field}: wrong shape`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, detail: "value has the wrong shape" }); continue; } if (values.some((x) => typeof x !== "string" || !Object.hasOwn(spec.values, x))) { say(nn, c.id, `${field}: label not in the list`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, proposed: value, detail: "label not in the list" }); continue; } if (same(value, c.current[field])) continue; const quote = o.evidence?.[field]; const reject = (why: string) => { say(nn, c.id, `${field}: ${why}`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, proposed: value, detail: why, quote }); }; if (typeof quote !== "string" || norm(quote).length < 8) { reject("no quote for the change"); continue; } if (!text.includes(norm(quote))) { reject("quote not found in the summary or the dossier"); continue; } const v = verdicts.find((x) => x.id === c.id && x.field === field); if (!checked) { notApplied.push({ id: c.id, kind: "label check: change not second-read", field, proposed: value, quote }); continue; } if (!v || v.verdict !== "confirm") { notApplied.push({ id: c.id, kind: "label check: change rejected by the second reader", field, proposed: value, quote, detail: v?.reason ?? "no verdict given" }); continue; } changes.push({ field, old: c.current[field], new: value, quote }); applied++; } for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ id: c.id, kind: "label check: change the first reader missed", field: v.field, proposed: v.proposed, quote: v.quote, detail: v.reason }); if (changes.length) merged.push({ id: c.id, changes }); } } const DISPUTED = ["label check: change rejected by the second reader", "label check: change the first reader missed"]; if (cmd === "tiebreak-prepare") { const items = notApplied.filter((n) => DISPUTED.includes(n.kind) && !caseOf.get(n.id)!.input.locked_by_owner.includes(n.field)).map((n) => { const c = caseOf.get(n.id)!.input; return { id: n.id, field: n.field, current: c.current[n.field], proposed: n.proposed, proposed_by: n.kind.includes("missed") ? "second reader" : "first reader", proposers_quote: n.quote ?? "", other_readers_objection: n.kind.includes("missed") ? "" : n.detail, all_current_labels: c.current, flags: c.flags, summary: c.summary, dossier: c.dossier }; }); writeFileSync("labels/tiebreak-in.json", JSON.stringify(items, null, 1)); console.log(`${items.length} disputed changes written to labels/tiebreak-in.json`); process.exit(0); } let tiebroken = 0; if (existsSync("labels/tiebreak-out.json")) { const at = statSync("labels/tiebreak-out.json").mtimeMs; for (const f of readdirSync("labels")) if (/^(out|check)-\d+\.json$/.test(f) && statSync(`labels/${f}`).mtimeMs > at) throw new Error(`labels/${f} is newer than labels/tiebreak-out.json: the tie-break is stale`); // a reader may hand a one-value field as a one-item list const one = (field: string, v: unknown) => (!LABELS[field]?.several && Array.isArray(v) && v.length === 1 ? v[0] : v); for (const t of J("labels/tiebreak-out.json") as any[]) { t.value = one(t.field, t.value); const item = notApplied.find((n) => n.id === t.id && n.field === t.field && DISPUTED.includes(n.kind) && same(one(n.field, n.proposed), t.value)); if (!item) continue; if (t.verdict !== "apply") { item.detail = `${item.detail ?? ""} | third reader: keep the current value. ${t.reason ?? ""}`.trim(); item.kind = "label check: change turned down by two of three readers"; continue; } const k = caseOf.get(t.id)!; const spec = LABELS[t.field]; const values: unknown[] = spec.several ? t.value : [t.value]; if (k.input.locked_by_owner.includes(t.field) || !Array.isArray(values) || !values.length || values.some((x) => typeof x !== "string" || !Object.hasOwn(spec.values, x))) { problems++; continue; } if (typeof t.quote !== "string" || norm(t.quote).length < 8 || !k.text.includes(norm(t.quote))) { problems++; item.detail = `${item.detail ?? ""} | third reader agreed, but the quote was not found in the text`; continue; } let entry = merged.find((m) => m.id === t.id); if (!entry) merged.push((entry = { id: t.id, changes: [] })); const prior = entry.changes.find((ch: any) => ch.field === t.field); if (prior) { prior.new = t.value; prior.quote = t.quote; } else entry.changes.push({ field: t.field, old: k.input.current[t.field], new: t.value, quote: t.quote }); notApplied = notApplied.filter((n) => n !== item); tiebroken++; } } if (cmd === "merge") { if (missing) throw new Error("a first-reader batch is missing; nothing merged"); writeFileSync("labels/merged.json", JSON.stringify(merged, null, 1)); writeFileSync("labels/not-applied.json", JSON.stringify(notApplied, null, 1)); } console.log(`${read} cases read, ${applied} changes confirmed by the second reader and ${tiebroken} by the third reader, on ${merged.length} cases, ${notApplied.length} not applied, ${problems} mechanical rejections, ${missing} batches missing`); } else throw new Error("usage: bun labels_pass.ts prepare | validate | merge");