This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/flags_pass.ts
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

85 lines
6.3 KiB
TypeScript

// Flag check, ordered by the owner on 2026-10-07 ("every single label"): the five violence flags
// and the death count of every case are tested against FLAG-RULES.md by a first reader and
// confirmed by a second reader. A value set by a written ruling is locked and not re-judged.
// bun flags_pass.ts prepare <site>/release/data/cases.json -> flags/in-NN.json
// bun flags_pass.ts validate
// bun flags_pass.ts merge -> flags/merged.json, flags/not-applied.json, flags/checked.json
// A change is applied only when its quote is found in the case text and the second reader confirmed it.
// flags/checked.json lists the unlocked values both readers left as they are.
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
const BATCH = 25;
const FLAGS = ["detention", "violence_torture", "drugs_alcohol", "weapons", "theft"] as const;
const FIELDS = [...FLAGS, "life_taken"] as const;
const allowed = (field: string, v: unknown) =>
typeof v === "string" && (field === "life_taken" ? /^(\d+|unconfirmed)$/.test(v) : ["yes", "no", "unconfirmed"].includes(v) || (field === "violence_torture" && v === "threat only"));
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
const batches = () => readdirSync("flags").filter((f) => /^in-\d+\.json$/.test(f)).sort().map((f) => f.slice(3, 5));
const [cmd, arg] = process.argv.slice(2);
if (cmd === "prepare") {
const cases: any[] = J(arg).cases;
const sources: Record<string, string> = J("exports/flag-sources.json");
const items = cases.map((c) => ({
id: c.id,
current: { ...Object.fromEntries(FLAGS.map((f) => [f, c.flags[f]])), life_taken: c.life_taken_unconfirmed ? "unconfirmed" : c.life_taken == null ? "" : String(c.life_taken) },
locked_by_ruling: FIELDS.filter((f) => sources[`${c.id}:${f}`] === "ruling"),
obtained: c.steps.obtained,
outcome: c.steps.outcome,
summary: c.summary,
dossier: c.dossier_file ? `sources/${c.dossier_file}` : "",
}));
for (let i = 0; i * BATCH < items.length; i++)
writeFileSync(`flags/in-${String(i + 1).padStart(2, "0")}.json`, JSON.stringify(items.slice(i * BATCH, (i + 1) * BATCH), null, 1));
const open = items.reduce((n, c) => n + FIELDS.length - c.locked_by_ruling.length, 0);
console.log(`${items.length} cases in ${Math.ceil(items.length / BATCH)} batches; ${open} values to test, ${items.length * FIELDS.length - open} locked by a ruling`);
} else if (cmd === "validate" || cmd === "merge") {
const merged: any[] = [];
const notApplied: any[] = [];
const checkedOk: string[] = [];
let problems = 0, applied = 0, missing = 0, read = 0;
const say = (nn: string, id: unknown, msg: string) => (problems++, cmd === "validate" && console.log(`out-${nn} case ${id}: ${msg}`));
for (const nn of batches()) {
const input: any[] = J(`flags/in-${nn}.json`);
if (!existsSync(`flags/out-${nn}.json`)) { missing++; console.log(`out-${nn}.json missing`); continue; }
const out: any[] = J(`flags/out-${nn}.json`);
const checked = existsSync(`flags/check-${nn}.json`) && statSync(`flags/check-${nn}.json`).mtimeMs > statSync(`flags/out-${nn}.json`).mtimeMs;
const verdicts: any[] = checked ? J(`flags/check-${nn}.json`) : [];
const byId = new Map(out.map((o) => [o.id, o]));
for (const c of input) {
const o = byId.get(c.id);
if (!o) { say(nn, c.id, "missing"); notApplied.push({ id: c.id, kind: "flag check: case not read", detail: "missing from the first reader's output" }); continue; }
read++;
const text = norm(c.summary) + " " + (c.dossier && existsSync(c.dossier) ? norm(readFileSync(c.dossier, "utf8")) : "");
const changes: any[] = [];
for (const field of FIELDS) {
if (c.locked_by_ruling.includes(field)) continue; // a written ruling stands
const value = o[field];
const missed = verdicts.find((x) => x.id === c.id && x.field === field && x.verdict === "missed");
if (missed) notApplied.push({ id: c.id, kind: "flag check: change the first reader missed", field, proposed: missed.proposed, quote: missed.quote, detail: missed.reason });
if (!allowed(field, value)) { say(nn, c.id, `${field}: value not allowed`); notApplied.push({ id: c.id, kind: "flag check: change rejected by the script", field, proposed: value, detail: "value not allowed" }); continue; }
if (value === c.current[field]) { if (checked && !missed) checkedOk.push(`${c.id}:${field}`); continue; }
const quote = o.evidence?.[field];
const reject = (why: string) => { say(nn, c.id, `${field}: ${why}`); notApplied.push({ id: c.id, kind: "flag check: change rejected by the script", field, proposed: value, detail: why, quote }); };
if (typeof quote !== "string" || norm(quote).length < 8) { reject("no quote for the change"); continue; }
if (!text.includes(norm(quote))) { reject("quote not found in the summary or the dossier"); continue; }
const v = verdicts.find((x) => x.id === c.id && x.field === field && x.verdict !== "missed");
if (!checked) { notApplied.push({ id: c.id, kind: "flag check: change not second-read", field, proposed: value, quote }); continue; }
if (!v || v.verdict !== "confirm") { notApplied.push({ id: c.id, kind: "flag check: change rejected by the second reader", field, proposed: value, quote, detail: v?.reason ?? "no verdict given" }); continue; }
changes.push({ field, old: c.current[field], new: value, quote });
checkedOk.push(`${c.id}:${field}`);
applied++;
}
if (changes.length) merged.push({ id: c.id, changes });
}
}
if (cmd === "merge") {
if (missing) throw new Error("a first-reader batch is missing; nothing merged");
writeFileSync("flags/merged.json", JSON.stringify(merged, null, 1));
writeFileSync("flags/not-applied.json", JSON.stringify(notApplied, null, 1));
writeFileSync("flags/checked.json", JSON.stringify(checkedOk));
}
console.log(`${read} cases read, ${applied} changes ${cmd === "merge" ? "applied" : "would apply"} on ${merged.length} cases, ${checkedOk.length} values confirmed, ${notApplied.length} not applied, ${problems} mechanical rejections, ${missing} batches missing`);
} else throw new Error("usage: bun flags_pass.ts prepare <cases.json> | validate | merge");