This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/make_label_audit.ts
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

130 lines
6.1 KiB
TypeScript

// Build the label audit: one CSV row per case, comparing the labels a case carries today
// with the verified sequence (scenario-sequences.json).
// live label the single `scenario` on stats.gart.io (live API snapshot)
// v3 auto the extra labels the v3 site's engine (scenario_signature.py) would add
// Usage (from merge-output/): bun make_label_audit.ts ../attacks-live-api-YYYY-MM-DD.json
import { readFileSync, writeFileSync } from "fs";
import { csvCell, withhold } from "./withheld";
import { homedir } from "os";
import { join } from "path";
const livePath = process.argv[2];
if (!livePath) throw new Error("usage: bun make_label_audit.ts <live api snapshot.json>");
const SITE =
process.env.GART_STATS_SITE ??
join(homedir(), "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3");
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
const live: any[] = J(livePath);
const exportById = new Map<number, any>(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r]));
const seqs: Record<string, any> = J("scenario-sequences.json");
const summaries: Record<string, string> = J("summary-overrides.json");
const manifest: any[] = J("case-files-manifest.json").entries;
// the v3 engine's proposed labels, computed on the same records (export text, live label)
const engineInput = live.map((r) => ({ ...(exportById.get(Number(r.id)) ?? {}), ...r }));
const py = Bun.spawnSync(
[
"python3",
"-c",
"import sys, json\n" +
"sys.path.insert(0, sys.argv[1])\n" +
"from scenario_signature import build_signature\n" +
"cases = json.load(sys.stdin)\n" +
"json.dump({str(c['id']): [t for t in build_signature(c, c.get('scenario')) if t['status'] == 'proposed'] for c in cases}, sys.stdout)\n",
SITE,
],
{ stdin: Buffer.from(JSON.stringify(engineInput)) },
);
if (py.exitCode !== 0) throw new Error("scenario_signature.py failed: " + py.stderr.toString());
const auto: Record<string, { scenario: string; rationale: string }[]> = JSON.parse(py.stdout.toString());
// fetched source URLs per case, and the dossier file
const fetched = new Map<number, Set<string>>();
const dossier = new Map<number, string>();
for (const e of manifest) {
const id = Number(e.case);
dossier.set(id, e.file);
const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? "");
if (m) (fetched.get(id) ?? fetched.set(id, new Set()).get(id)!).add(m[1]);
}
const section = (summary: string, name: string) => {
const m = new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*([\\s\\S]*?)(?=\\n\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary);
return m ? m[1].replace(/\s+/g, " ").trim() : "";
};
// cut at the last sentence end before the limit
const shorten = (text: string, limit: number) => {
if (text.length <= limit) return text;
const head = text.slice(0, limit);
const stop = Math.max(head.lastIndexOf(". "), head.lastIndexOf("; "));
return (stop > limit * 0.5 ? head.slice(0, stop + 1) : head.trimEnd() + " ...").trim();
};
const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:");
const cell = csvCell;
const header = [
"case_id", "title", "date", "location",
"live_label", "live_label_verdict", "missing_from_live",
"verified_labels_in_order", "attempted",
"v3_auto_labels_correct", "v3_auto_labels_wrong",
"what_happened", "support_for_labels",
"url_1", "url_2", "url_3", "url_4", "url_5", "other_source_urls", "dossier_file",
];
const rows: string[][] = [];
const tally = { CORRECT: 0, WRONG: 0, "NO LABEL FITS": 0, "NOT VERIFIED": 0, missing: 0, autoOk: 0, autoWrong: 0 };
for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) {
const id = Number(r.id);
const seq = seqs[String(id)];
const liveLabel: string = r.scenario ?? "";
const verified: string[] = seq?.sequence ?? [];
const verdict = !seq ? "NOT VERIFIED" : verified.length === 0 ? "NO LABEL FITS" : verified.includes(liveLabel) ? "CORRECT" : "WRONG";
tally[verdict]++;
const missing = seq ? verified.filter((l) => l !== liveLabel) : [];
if (missing.length) tally.missing++;
const proposals = auto[String(id)] ?? [];
const show = (p: { scenario: string; rationale: string }) => `${p.scenario} [${p.rationale}]`;
const autoOk = seq ? proposals.filter((p) => verified.includes(p.scenario)) : [];
const autoWrong = seq ? proposals.filter((p) => !verified.includes(p.scenario)) : [];
tally.autoOk += autoOk.length;
tally.autoWrong += autoWrong.length;
// the Burglary rationales were written before the label existed
const support =
(seq?.rationale ?? "") +
(verified.includes("Burglary") && !/Burglary/.test(seq.rationale ?? "")
? " Burglary set by owner ruling 2026-10-06 (DEFINITIONS.md)."
: "");
const summary = summaries[String(id)] ?? "";
const urls = [r.url, r.url_2, r.url_3, r.url_4, r.url_5].map((u) => (u ?? "").trim());
const onRecord = new Set(urls.filter(Boolean).map(norm));
const others = [...(fetched.get(id) ?? [])].filter((u) => !onRecord.has(norm(u)));
rows.push([
String(id),
(r.victim ?? "").trim(),
(r.date ?? "").trim(),
(r.location ?? "").trim(),
liveLabel,
verdict,
missing.join(" | "),
verified.join(" > "),
(seq?.attempted ?? []).join(" | "),
autoOk.map(show).join(" | "),
autoWrong.map(show).join(" | "), // empty for an unverified case: nothing to judge the proposals against
summary ? shorten(section(summary, "Attack Method"), 420) : (r.description ?? "").trim(),
support,
...urls,
others.join(" | "),
dossier.get(id) ?? "",
]);
}
const out = `exports/label-audit-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`;
const named = rows.map((row) => row.map((v, i) => withhold(row[0], header[i], v)));
writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n");
console.log(`${out}: ${rows.length} cases`);
console.log(`live label CORRECT ${tally.CORRECT}, WRONG ${tally.WRONG}, NO LABEL FITS ${tally["NO LABEL FITS"]}, NOT VERIFIED ${tally["NOT VERIFIED"]}`);
console.log(`cases with verified labels missing from live: ${tally.missing}`);
console.log(`v3 auto labels correct ${tally.autoOk}, wrong ${tally.autoWrong}`);