Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
120 lines
9.4 KiB
TypeScript
120 lines
9.4 KiB
TypeScript
// Full label check, ordered by the owner on 2026-10-07: every label of every case is tested
|
|
// against LABEL-RULES.md by a first reader and confirmed by a second reader.
|
|
// bun labels_pass.ts prepare <site>/release/data/cases.json -> labels/in-NN.json
|
|
// bun labels_pass.ts validate
|
|
// bun labels_pass.ts merge -> labels/merged.json (second-reader-confirmed changes) and labels/not-applied.json
|
|
// bun labels_pass.ts tiebreak-prepare -> labels/tiebreak-in.json: the changes the two readers disagree on
|
|
// A change is applied only when its quote is found in the case text and the second reader confirmed it.
|
|
// Where the readers disagree (one proposed, the other rejected or never saw it), a third reader decides:
|
|
// labels/tiebreak-out.json, [{id, field, verdict: "apply" | "keep", value, quote, reason}]. An "apply" counts
|
|
// only for a value one of the two readers proposed, with a quote found in the case text.
|
|
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
|
|
|
|
const LABELS = JSON.parse(readFileSync("step-labels.json", "utf8"));
|
|
const BATCH = 25;
|
|
const FIELDS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"] as const;
|
|
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
|
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
|
|
const same = (a: unknown, b: unknown) => JSON.stringify(Array.isArray(a) ? [...a].sort() : a) === JSON.stringify(Array.isArray(b) ? [...b].sort() : b);
|
|
const batches = () => readdirSync("labels").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5));
|
|
|
|
const [cmd, arg] = process.argv.slice(2);
|
|
if (cmd === "prepare") {
|
|
const cases: any[] = J(arg).cases;
|
|
const rulings = existsSync("step-rulings.json") ? J("step-rulings.json").rulings : [];
|
|
const items = cases.map((c) => ({
|
|
id: c.id,
|
|
victim_field: c.victim,
|
|
current: Object.fromEntries(FIELDS.map((f) => [f, c.steps[f]])),
|
|
locked_by_owner: rulings.filter((r: any) => r.id === c.id).map((r: any) => r.field),
|
|
flags: c.flags,
|
|
summary: c.summary,
|
|
dossier: c.dossier_file ? `sources/${c.dossier_file}` : "",
|
|
}));
|
|
for (let i = 0; i * BATCH < items.length; i++)
|
|
writeFileSync(`labels/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
|
|
console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches`);
|
|
} else if (cmd === "validate" || cmd === "merge" || cmd === "tiebreak-prepare") {
|
|
const merged: any[] = [];
|
|
let notApplied: any[] = [];
|
|
const caseOf = new Map<number, { input: any; text: string }>();
|
|
let problems = 0, applied = 0, missing = 0, read = 0;
|
|
const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`));
|
|
for (const nn of batches()) {
|
|
const input: any[] = J(`labels/in-${nn}.json`);
|
|
if (!existsSync(`labels/out-${nn}.json`)) { missing++; console.log(`out-${nn}.json missing`); continue; }
|
|
const out: any[] = J(`labels/out-${nn}.json`);
|
|
const checked = existsSync(`labels/check-${nn}.json`) && statSync(`labels/check-${nn}.json`).mtimeMs > statSync(`labels/out-${nn}.json`).mtimeMs;
|
|
const verdicts: any[] = checked ? J(`labels/check-${nn}.json`) : [];
|
|
const byId = new Map(out.map((o) => [o.id, o]));
|
|
for (const c of input) {
|
|
const o = byId.get(c.id);
|
|
if (!o) { say(nn, c.id, "missing"); notApplied.push({ id: c.id, kind: "label check: case not read", detail: "missing from the first reader's output" }); continue; }
|
|
read++;
|
|
const text = norm(c.summary) + " " + (c.dossier && existsSync(c.dossier) ? norm(readFileSync(c.dossier, "utf8")) : "");
|
|
caseOf.set(c.id, { input: c, text });
|
|
const changes: any[] = [];
|
|
for (const field of FIELDS) {
|
|
if (c.locked_by_owner.includes(field)) continue; // an owner ruling stands
|
|
const value = o[field];
|
|
const spec = LABELS[field];
|
|
const values: unknown[] = spec.several ? value : [value];
|
|
if (spec.several ? !Array.isArray(value) || !value.length : typeof value !== "string") { say(nn, c.id, `${field}: wrong shape`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, detail: "value has the wrong shape" }); continue; }
|
|
if (values.some((x) => typeof x !== "string" || !Object.hasOwn(spec.values, x))) { say(nn, c.id, `${field}: label not in the list`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, proposed: value, detail: "label not in the list" }); continue; }
|
|
if (same(value, c.current[field])) continue;
|
|
const quote = o.evidence?.[field];
|
|
const reject = (why: string) => { say(nn, c.id, `${field}: ${why}`); notApplied.push({ id: c.id, kind: "label check: change rejected by the script", field, proposed: value, detail: why, quote }); };
|
|
if (typeof quote !== "string" || norm(quote).length < 8) { reject("no quote for the change"); continue; }
|
|
if (!text.includes(norm(quote))) { reject("quote not found in the summary or the dossier"); continue; }
|
|
const v = verdicts.find((x) => x.id === c.id && x.field === field);
|
|
if (!checked) { notApplied.push({ id: c.id, kind: "label check: change not second-read", field, proposed: value, quote }); continue; }
|
|
if (!v || v.verdict !== "confirm") { notApplied.push({ id: c.id, kind: "label check: change rejected by the second reader", field, proposed: value, quote, detail: v?.reason ?? "no verdict given" }); continue; }
|
|
changes.push({ field, old: c.current[field], new: value, quote });
|
|
applied++;
|
|
}
|
|
for (const v of verdicts.filter((x) => x.id === c.id && x.verdict === "missed")) notApplied.push({ id: c.id, kind: "label check: change the first reader missed", field: v.field, proposed: v.proposed, quote: v.quote, detail: v.reason });
|
|
if (changes.length) merged.push({ id: c.id, changes });
|
|
}
|
|
}
|
|
const DISPUTED = ["label check: change rejected by the second reader", "label check: change the first reader missed"];
|
|
if (cmd === "tiebreak-prepare") {
|
|
const items = notApplied.filter((n) => DISPUTED.includes(n.kind) && !caseOf.get(n.id)!.input.locked_by_owner.includes(n.field)).map((n) => {
|
|
const c = caseOf.get(n.id)!.input;
|
|
return { id: n.id, field: n.field, current: c.current[n.field], proposed: n.proposed, proposed_by: n.kind.includes("missed") ? "second reader" : "first reader", proposers_quote: n.quote ?? "", other_readers_objection: n.kind.includes("missed") ? "" : n.detail, all_current_labels: c.current, flags: c.flags, summary: c.summary, dossier: c.dossier };
|
|
});
|
|
writeFileSync("labels/tiebreak-in.json", JSON.stringify(items, null, 1));
|
|
console.log(`${items.length} disputed changes written to labels/tiebreak-in.json`);
|
|
process.exit(0);
|
|
}
|
|
let tiebroken = 0;
|
|
if (existsSync("labels/tiebreak-out.json")) {
|
|
const at = statSync("labels/tiebreak-out.json").mtimeMs;
|
|
for (const f of readdirSync("labels")) if (/^(out|check)-\d+\.json$/.test(f) && statSync(`labels/${f}`).mtimeMs > at) throw new Error(`labels/${f} is newer than labels/tiebreak-out.json: the tie-break is stale`);
|
|
// a reader may hand a one-value field as a one-item list
|
|
const one = (field: string, v: unknown) => (!LABELS[field]?.several && Array.isArray(v) && v.length === 1 ? v[0] : v);
|
|
for (const t of J("labels/tiebreak-out.json") as any[]) {
|
|
t.value = one(t.field, t.value);
|
|
const item = notApplied.find((n) => n.id === t.id && n.field === t.field && DISPUTED.includes(n.kind) && same(one(n.field, n.proposed), t.value));
|
|
if (!item) continue;
|
|
if (t.verdict !== "apply") { item.detail = `${item.detail ?? ""} | third reader: keep the current value. ${t.reason ?? ""}`.trim(); item.kind = "label check: change turned down by two of three readers"; continue; }
|
|
const k = caseOf.get(t.id)!;
|
|
const spec = LABELS[t.field];
|
|
const values: unknown[] = spec.several ? t.value : [t.value];
|
|
if (k.input.locked_by_owner.includes(t.field) || !Array.isArray(values) || !values.length || values.some((x) => typeof x !== "string" || !Object.hasOwn(spec.values, x))) { problems++; continue; }
|
|
if (typeof t.quote !== "string" || norm(t.quote).length < 8 || !k.text.includes(norm(t.quote))) { problems++; item.detail = `${item.detail ?? ""} | third reader agreed, but the quote was not found in the text`; continue; }
|
|
let entry = merged.find((m) => m.id === t.id);
|
|
if (!entry) merged.push((entry = { id: t.id, changes: [] }));
|
|
const prior = entry.changes.find((ch: any) => ch.field === t.field);
|
|
if (prior) { prior.new = t.value; prior.quote = t.quote; } else entry.changes.push({ field: t.field, old: k.input.current[t.field], new: t.value, quote: t.quote });
|
|
notApplied = notApplied.filter((n) => n !== item);
|
|
tiebroken++;
|
|
}
|
|
}
|
|
if (cmd === "merge") {
|
|
if (missing) throw new Error("a first-reader batch is missing; nothing merged");
|
|
writeFileSync("labels/merged.json", JSON.stringify(merged, null, 1));
|
|
writeFileSync("labels/not-applied.json", JSON.stringify(notApplied, null, 1));
|
|
}
|
|
console.log(`${read} cases read, ${applied} changes confirmed by the second reader and ${tiebroken} by the third reader, on ${merged.length} cases, ${notApplied.length} not applied, ${problems} mechanical rejections, ${missing} batches missing`);
|
|
} else throw new Error("usage: bun labels_pass.ts prepare <cases.json> | validate | merge");
|