// Build the label audit: one CSV row per case, comparing the labels a case carries today // with the verified sequence (scenario-sequences.json). // live label the single `scenario` on stats.gart.io (live API snapshot) // v3 auto the extra labels the v3 site's engine (scenario_signature.py) would add // Usage (from merge-output/): bun make_label_audit.ts ../attacks-live-api-YYYY-MM-DD.json import { readFileSync, writeFileSync } from "fs"; import { csvCell, withhold } from "./withheld"; import { homedir } from "os"; import { join } from "path"; const livePath = process.argv[2]; if (!livePath) throw new Error("usage: bun make_label_audit.ts "); const SITE = process.env.GART_STATS_SITE ?? join(homedir(), "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3"); const J = (p: string) => JSON.parse(readFileSync(p, "utf8")); const live: any[] = J(livePath); const exportById = new Map(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r])); const seqs: Record = J("scenario-sequences.json"); const summaries: Record = J("summary-overrides.json"); const manifest: any[] = J("case-files-manifest.json").entries; // the v3 engine's proposed labels, computed on the same records (export text, live label) const engineInput = live.map((r) => ({ ...(exportById.get(Number(r.id)) ?? {}), ...r })); const py = Bun.spawnSync( [ "python3", "-c", "import sys, json\n" + "sys.path.insert(0, sys.argv[1])\n" + "from scenario_signature import build_signature\n" + "cases = json.load(sys.stdin)\n" + "json.dump({str(c['id']): [t for t in build_signature(c, c.get('scenario')) if t['status'] == 'proposed'] for c in cases}, sys.stdout)\n", SITE, ], { stdin: Buffer.from(JSON.stringify(engineInput)) }, ); if (py.exitCode !== 0) throw new Error("scenario_signature.py failed: " + py.stderr.toString()); const auto: Record = JSON.parse(py.stdout.toString()); // fetched source URLs per case, and the dossier file const fetched = new Map>(); const dossier = new Map(); for (const e of manifest) { const id = Number(e.case); dossier.set(id, e.file); const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? ""); if (m) (fetched.get(id) ?? fetched.set(id, new Set()).get(id)!).add(m[1]); } const section = (summary: string, name: string) => { const m = new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*([\\s\\S]*?)(?=\\n\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary); return m ? m[1].replace(/\s+/g, " ").trim() : ""; }; // cut at the last sentence end before the limit const shorten = (text: string, limit: number) => { if (text.length <= limit) return text; const head = text.slice(0, limit); const stop = Math.max(head.lastIndexOf(". "), head.lastIndexOf("; ")); return (stop > limit * 0.5 ? head.slice(0, stop + 1) : head.trimEnd() + " ...").trim(); }; const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:"); const cell = csvCell; const header = [ "case_id", "title", "date", "location", "live_label", "live_label_verdict", "missing_from_live", "verified_labels_in_order", "attempted", "v3_auto_labels_correct", "v3_auto_labels_wrong", "what_happened", "support_for_labels", "url_1", "url_2", "url_3", "url_4", "url_5", "other_source_urls", "dossier_file", ]; const rows: string[][] = []; const tally = { CORRECT: 0, WRONG: 0, "NO LABEL FITS": 0, "NOT VERIFIED": 0, missing: 0, autoOk: 0, autoWrong: 0 }; for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) { const id = Number(r.id); const seq = seqs[String(id)]; const liveLabel: string = r.scenario ?? ""; const verified: string[] = seq?.sequence ?? []; const verdict = !seq ? "NOT VERIFIED" : verified.length === 0 ? "NO LABEL FITS" : verified.includes(liveLabel) ? "CORRECT" : "WRONG"; tally[verdict]++; const missing = seq ? verified.filter((l) => l !== liveLabel) : []; if (missing.length) tally.missing++; const proposals = auto[String(id)] ?? []; const show = (p: { scenario: string; rationale: string }) => `${p.scenario} [${p.rationale}]`; const autoOk = seq ? proposals.filter((p) => verified.includes(p.scenario)) : []; const autoWrong = seq ? proposals.filter((p) => !verified.includes(p.scenario)) : []; tally.autoOk += autoOk.length; tally.autoWrong += autoWrong.length; // the Burglary rationales were written before the label existed const support = (seq?.rationale ?? "") + (verified.includes("Burglary") && !/Burglary/.test(seq.rationale ?? "") ? " Burglary set by owner ruling 2026-10-06 (DEFINITIONS.md)." : ""); const summary = summaries[String(id)] ?? ""; const urls = [r.url, r.url_2, r.url_3, r.url_4, r.url_5].map((u) => (u ?? "").trim()); const onRecord = new Set(urls.filter(Boolean).map(norm)); const others = [...(fetched.get(id) ?? [])].filter((u) => !onRecord.has(norm(u))); rows.push([ String(id), (r.victim ?? "").trim(), (r.date ?? "").trim(), (r.location ?? "").trim(), liveLabel, verdict, missing.join(" | "), verified.join(" > "), (seq?.attempted ?? []).join(" | "), autoOk.map(show).join(" | "), autoWrong.map(show).join(" | "), // empty for an unverified case: nothing to judge the proposals against summary ? shorten(section(summary, "Attack Method"), 420) : (r.description ?? "").trim(), support, ...urls, others.join(" | "), dossier.get(id) ?? "", ]); } const out = `exports/label-audit-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`; const named = rows.map((row) => row.map((v, i) => withhold(row[0], header[i], v))); writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n"); console.log(`${out}: ${rows.length} cases`); console.log(`live label CORRECT ${tally.CORRECT}, WRONG ${tally.WRONG}, NO LABEL FITS ${tally["NO LABEL FITS"]}, NOT VERIFIED ${tally["NOT VERIFIED"]}`); console.log(`cases with verified labels missing from live: ${tally.missing}`); console.log(`v3 auto labels correct ${tally.autoOk}, wrong ${tally.autoWrong}`);