Archived
chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
// Build the label audit: one CSV row per case, comparing the labels a case carries today
|
||||
// with the verified sequence (scenario-sequences.json).
|
||||
// live label the single `scenario` on stats.gart.io (live API snapshot)
|
||||
// v3 auto the extra labels the v3 site's engine (scenario_signature.py) would add
|
||||
// Usage (from merge-output/): bun make_label_audit.ts ../attacks-live-api-YYYY-MM-DD.json
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { csvCell, withhold } from "./withheld";
|
||||
import { homedir } from "os";
|
||||
import { join } from "path";
|
||||
|
||||
const livePath = process.argv[2];
|
||||
if (!livePath) throw new Error("usage: bun make_label_audit.ts <live api snapshot.json>");
|
||||
const SITE =
|
||||
process.env.GART_STATS_SITE ??
|
||||
join(homedir(), "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3");
|
||||
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
||||
const live: any[] = J(livePath);
|
||||
const exportById = new Map<number, any>(J("../attacks-export-Gart-website.json").map((r: any) => [Number(r.id), r]));
|
||||
const seqs: Record<string, any> = J("scenario-sequences.json");
|
||||
const summaries: Record<string, string> = J("summary-overrides.json");
|
||||
const manifest: any[] = J("case-files-manifest.json").entries;
|
||||
|
||||
// the v3 engine's proposed labels, computed on the same records (export text, live label)
|
||||
const engineInput = live.map((r) => ({ ...(exportById.get(Number(r.id)) ?? {}), ...r }));
|
||||
const py = Bun.spawnSync(
|
||||
[
|
||||
"python3",
|
||||
"-c",
|
||||
"import sys, json\n" +
|
||||
"sys.path.insert(0, sys.argv[1])\n" +
|
||||
"from scenario_signature import build_signature\n" +
|
||||
"cases = json.load(sys.stdin)\n" +
|
||||
"json.dump({str(c['id']): [t for t in build_signature(c, c.get('scenario')) if t['status'] == 'proposed'] for c in cases}, sys.stdout)\n",
|
||||
SITE,
|
||||
],
|
||||
{ stdin: Buffer.from(JSON.stringify(engineInput)) },
|
||||
);
|
||||
if (py.exitCode !== 0) throw new Error("scenario_signature.py failed: " + py.stderr.toString());
|
||||
const auto: Record<string, { scenario: string; rationale: string }[]> = JSON.parse(py.stdout.toString());
|
||||
|
||||
// fetched source URLs per case, and the dossier file
|
||||
const fetched = new Map<number, Set<string>>();
|
||||
const dossier = new Map<number, string>();
|
||||
for (const e of manifest) {
|
||||
const id = Number(e.case);
|
||||
dossier.set(id, e.file);
|
||||
const m = /^fetched (https?:\/\/\S+)/.exec(e.origin ?? "");
|
||||
if (m) (fetched.get(id) ?? fetched.set(id, new Set()).get(id)!).add(m[1]);
|
||||
}
|
||||
|
||||
const section = (summary: string, name: string) => {
|
||||
const m = new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*([\\s\\S]*?)(?=\\n\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary);
|
||||
return m ? m[1].replace(/\s+/g, " ").trim() : "";
|
||||
};
|
||||
// cut at the last sentence end before the limit
|
||||
const shorten = (text: string, limit: number) => {
|
||||
if (text.length <= limit) return text;
|
||||
const head = text.slice(0, limit);
|
||||
const stop = Math.max(head.lastIndexOf(". "), head.lastIndexOf("; "));
|
||||
return (stop > limit * 0.5 ? head.slice(0, stop + 1) : head.trimEnd() + " ...").trim();
|
||||
};
|
||||
const norm = (u: string) => u.trim().replace(/\/+$/, "").replace(/^http:/, "https:");
|
||||
const cell = csvCell;
|
||||
|
||||
const header = [
|
||||
"case_id", "title", "date", "location",
|
||||
"live_label", "live_label_verdict", "missing_from_live",
|
||||
"verified_labels_in_order", "attempted",
|
||||
"v3_auto_labels_correct", "v3_auto_labels_wrong",
|
||||
"what_happened", "support_for_labels",
|
||||
"url_1", "url_2", "url_3", "url_4", "url_5", "other_source_urls", "dossier_file",
|
||||
];
|
||||
const rows: string[][] = [];
|
||||
const tally = { CORRECT: 0, WRONG: 0, "NO LABEL FITS": 0, "NOT VERIFIED": 0, missing: 0, autoOk: 0, autoWrong: 0 };
|
||||
|
||||
for (const r of [...live].sort((a, b) => Number(a.id) - Number(b.id))) {
|
||||
const id = Number(r.id);
|
||||
const seq = seqs[String(id)];
|
||||
const liveLabel: string = r.scenario ?? "";
|
||||
const verified: string[] = seq?.sequence ?? [];
|
||||
const verdict = !seq ? "NOT VERIFIED" : verified.length === 0 ? "NO LABEL FITS" : verified.includes(liveLabel) ? "CORRECT" : "WRONG";
|
||||
tally[verdict]++;
|
||||
const missing = seq ? verified.filter((l) => l !== liveLabel) : [];
|
||||
if (missing.length) tally.missing++;
|
||||
const proposals = auto[String(id)] ?? [];
|
||||
const show = (p: { scenario: string; rationale: string }) => `${p.scenario} [${p.rationale}]`;
|
||||
const autoOk = seq ? proposals.filter((p) => verified.includes(p.scenario)) : [];
|
||||
const autoWrong = seq ? proposals.filter((p) => !verified.includes(p.scenario)) : [];
|
||||
tally.autoOk += autoOk.length;
|
||||
tally.autoWrong += autoWrong.length;
|
||||
|
||||
// the Burglary rationales were written before the label existed
|
||||
const support =
|
||||
(seq?.rationale ?? "") +
|
||||
(verified.includes("Burglary") && !/Burglary/.test(seq.rationale ?? "")
|
||||
? " Burglary set by owner ruling 2026-10-06 (DEFINITIONS.md)."
|
||||
: "");
|
||||
const summary = summaries[String(id)] ?? "";
|
||||
const urls = [r.url, r.url_2, r.url_3, r.url_4, r.url_5].map((u) => (u ?? "").trim());
|
||||
const onRecord = new Set(urls.filter(Boolean).map(norm));
|
||||
const others = [...(fetched.get(id) ?? [])].filter((u) => !onRecord.has(norm(u)));
|
||||
|
||||
rows.push([
|
||||
String(id),
|
||||
(r.victim ?? "").trim(),
|
||||
(r.date ?? "").trim(),
|
||||
(r.location ?? "").trim(),
|
||||
liveLabel,
|
||||
verdict,
|
||||
missing.join(" | "),
|
||||
verified.join(" > "),
|
||||
(seq?.attempted ?? []).join(" | "),
|
||||
autoOk.map(show).join(" | "),
|
||||
autoWrong.map(show).join(" | "), // empty for an unverified case: nothing to judge the proposals against
|
||||
summary ? shorten(section(summary, "Attack Method"), 420) : (r.description ?? "").trim(),
|
||||
support,
|
||||
...urls,
|
||||
others.join(" | "),
|
||||
dossier.get(id) ?? "",
|
||||
]);
|
||||
}
|
||||
|
||||
const out = `exports/label-audit-${/(\d{4}-\d{2}-\d{2})/.exec(livePath)?.[1] ?? "undated"}.csv`;
|
||||
const named = rows.map((row) => row.map((v, i) => withhold(row[0], header[i], v)));
|
||||
writeFileSync(out, "\uFEFF" + [header, ...named].map((row) => row.map(cell).join(",")).join("\r\n") + "\r\n");
|
||||
console.log(`${out}: ${rows.length} cases`);
|
||||
console.log(`live label CORRECT ${tally.CORRECT}, WRONG ${tally.WRONG}, NO LABEL FITS ${tally["NO LABEL FITS"]}, NOT VERIFIED ${tally["NOT VERIFIED"]}`);
|
||||
console.log(`cases with verified labels missing from live: ${tally.missing}`);
|
||||
console.log(`v3 auto labels correct ${tally.autoOk}, wrong ${tally.autoWrong}`);
|
||||
Reference in New Issue
Block a user