chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+182
View File
@@ -0,0 +1,182 @@
// Sort every case into the four steps of an attack, one CSV line per case:
// 1 targeting how the attackers knew of the victim (rulings' "targeting vector"; Costly Fame)
// 2 approach how they got to the victim (P2P, Social Engineering, Authority Misuse + subtypes)
// 3 attack what they did (Kidnapping, Home Invasion, Armed Robbery, ...)
// 4 outcome how it ended, and whether anything was obtained
// Only existing data is used. "not sorted yet" marks a cell that needs the articles read;
// "unknown" means the sources were read and do not say.
// Input: the new-structure export (run make_new_structure_export.ts first).
// Usage (from merge-output/): bun make_four_step_export.ts exports/gart-kr-new-structure-YYYY-MM-DD.csv
import { existsSync, readFileSync, writeFileSync } from "fs";
import { csvCell, withhold } from "./withheld";
const basePath = process.argv[2];
if (!basePath) throw new Error("usage: bun make_four_step_export.ts <new-structure export.csv>");
const NOT_SORTED = "not sorted yet";
const parseCsv = (text: string): string[][] => {
const rows: string[][] = [];
let row: string[] = [], field = "", quoted = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (quoted) {
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
else if (c === '"') quoted = false;
else field += c;
} else if (c === '"') quoted = true;
else if (c === ",") row.push(field), (field = "");
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
else field += c;
}
if (quoted) throw new Error("unterminated quoted field in the input CSV");
if (field || row.length) row.push(field), rows.push(row);
return rows;
};
const [head, ...body] = parseCsv(readFileSync(basePath, "utf8").replace(/^/, ""));
for (const h of ["id", "scenario_1", "scenario_2", "scenario_3", "verification", "outcome", "theft", "summary"])
if (!head.includes(h)) throw new Error(`input CSV lacks column ${h}`);
for (const r of body) if (r.length !== head.length) throw new Error(`case ${r[0]}: ${r.length} fields, expected ${head.length}`);
const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
const subtypes = JSON.parse(readFileSync("step-subtypes.json", "utf8"));
// result of the reading pass (steps_pass.ts merge); when present it replaces the values derived below
const readList: any[] = existsSync("steps/merged.json") ? JSON.parse(readFileSync("steps/merged.json", "utf8")) : [];
const read = new Map<string, any>(readList.map((c) => [String(c.id), c]));
if (read.size !== readList.length) throw new Error("steps/merged.json holds a case id twice");
// "targeting vector = ..." in single-case ruling lines; the last one on a case is in force
const targetingNote = new Map<string, string>();
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l)))
for (const m of line.matchAll(/targeting vector\s*(?:(?:=|→|:)\s*|(?=unknown|not stated))([^;]+?)(?=;|\.\s|\.$|$)/g)) targetingNote.set(id, m[1].trim());
}
// the owner's proposed values (DEFINITIONS.md, targeting vector), matched in the ruling's wording
const TARGETING: [string, RegExp][] = [
["data leak", /data leak|breach|darkweb/i],
["public records", /public record/i],
["insider tip-off", /insider|tip-off/i],
["family or partner", /intimate partner|^family \/ partner|girlfriend|boyfriend/i], // the attacker's tie to the victim, not the victim's to a holder
["acquaintance", /acquaintance|friend/i],
["business counterparty", /business counterparty|investor|collaborator/i],
["public profile", /public profile|displayed wealth|professional profile|streamer/i],
["social media", /social media/i],
["online lure", /online lure|fake business contact/i],
["surveillance", /surveillance|following|tailed/i],
["opportunistic", /opportunistic/i],
];
const targeting = (c: Record<string, string>, labels: string[]) => {
const note = targetingNote.get(c.id);
const asserted = (note ?? "").replace(/\([^)]*\)?/g, " "); // the value, without the explanation in brackets
const values: string[] = [];
if (note && !/^(unknown|not stated)/i.test(note)) for (const [v, re] of TARGETING) if (re.test(asserted)) values.push(v);
if (labels.includes("Costly Fame") && !values.includes("public profile")) values.push("public profile");
if (values.length) return values;
if (note) return [/^(unknown|not stated)/i.test(note) ? "unknown" : "other (see note)"];
return [NOT_SORTED];
};
const ATTACK: Record<string, string> = {
Kidnapping: "Kidnapping",
"Express Kidnapping": "Express Kidnapping",
"Home Invasion": "Home Invasion",
"Armed Robbery in Public Space": "Armed Robbery",
"Assault - Mugging": "Mugging",
Burglary: "Burglary",
Swatting: "Swatting",
};
// where the attack label itself says how the attackers got to the victim
// (not Home Invasion: the way in may be force, a trusted associate or unknown)
const APPROACH_BY_DEFINITION: Record<string, string> = {
Burglary: "break-in, nobody present",
Mugging: "approached in public",
Swatting: "none (remote)",
};
const approach = (id: string, labels: string[], attack: string[]) => {
const values: string[] = [];
for (const l of labels) {
if (l === "P2P Trade Gone Wrong") values.push("P2P Trade");
// "same fact as ...": the lure is already carried by the case's P2P Trade or Authority Misuse value
const lure = subtypes.social_engineering[id] ?? NOT_SORTED;
if (l === "Malicious Invitation - SocialEng" && !lure.startsWith("same fact as")) values.push(`Social Engineering: ${lure}`);
if (l === "Authority Misuse") values.push(...String(subtypes.authority_misuse[id] ?? NOT_SORTED).split(" | ").map((s) => `Authority Misuse: ${s}`));
}
if (values.length) return values;
const byDefinition = attack.map((a) => APPROACH_BY_DEFINITION[a]).filter(Boolean);
return byDefinition.length ? byDefinition : [NOT_SORTED];
};
const OBTAINED: Record<string, string> = { "1": "yes", "0": "nothing", unconfirmed: "unconfirmed" };
const summarySection = (summary: string, name: string) =>
new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*(.*?)(?=\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary)?.[1].trim() ?? "";
const cell = csvCell;
const header = [
"id", "date", "victim", "location", "country",
"1_targeting", "1_targeting_accuracy", "1_who_was_attacked", "1_targeting_note",
"2_approach",
"3_attack",
"4_outcome", "4_obtained",
"demanded_amount", "demanded_currency", "demanded_usd_stated", "demanded_as_stated",
"taken_amount", "taken_currency", "taken_usd_stated", "taken_as_stated",
"4_demanded_and_obtained_note", "money_wanted_db", "coin_type_db",
"detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken",
"venue_of_control", "held_duration", "arrests",
"old_labels", "attempted", "verification", "second_reader", "open_points", "linked_cases", "pending_action",
"evidence_targeting", "evidence_approach", "evidence_attack", "evidence_outcome", "evidence_amounts",
"summary",
...head.filter((h) => /^source_\d+$/.test(h)),
];
const count: Record<string, Record<string, number>> = { "1_targeting": {}, "2_approach": {}, "3_attack": {}, "4_outcome": {} };
const tally = (step: string, values: string[]) => values.forEach((v) => (count[step][v] = (count[step][v] ?? 0) + 1));
const rows = cases.map((c) => {
const verified = c.verification !== "not verified";
const labels = [c.scenario_1, c.scenario_2, c.scenario_3].filter(Boolean);
const attack = labels.map((l) => ATTACK[l]).filter(Boolean);
const t = verified ? targeting(c, labels) : [NOT_SORTED];
const a = verified ? approach(c.id, labels, attack) : [NOT_SORTED];
// a verified case with approach labels only (a P2P meeting, a lure) has no label for what was done there
const k = attack.length ? attack : [verified && !labels.length ? "no label fits" : NOT_SORTED];
const failed = labels.includes("Dodged Attack");
let outcome = c.outcome || (failed ? "foiled" : NOT_SORTED);
const p = read.get(c.id);
const num = (v: unknown) => (v == null ? "" : String(v));
const passCols: Record<string, string> = p
? {
"1_targeting_accuracy": p.targeting_accuracy,
"1_who_was_attacked": p.who_was_attacked.join(" | "),
"4_obtained": p.obtained.join(" | "),
demanded_amount: num(p.demanded.amount), demanded_currency: p.demanded.currency ?? "", demanded_usd_stated: num(p.demanded.usd_stated), demanded_as_stated: p.demanded.as_stated ?? "",
taken_amount: num(p.taken.amount), taken_currency: p.taken.currency ?? "", taken_usd_stated: num(p.taken.usd_stated), taken_as_stated: p.taken.as_stated ?? "",
second_reader: p.second_reader, open_points: p.open_points ? String(p.open_points) : "",
evidence_targeting: p.evidence.targeting, evidence_approach: p.evidence.approach, evidence_attack: p.evidence.attack, evidence_outcome: p.evidence.outcome, evidence_amounts: p.evidence.amounts,
}
: {};
if (p) (t.length = 0), t.push(...p.targeting), (a.length = 0), a.push(...p.approach), (k.length = 0), k.push(...p.attack), (outcome = p.outcome);
tally("1_targeting", t), tally("2_approach", a), tally("3_attack", k), tally("4_outcome", [outcome]);
const row: Record<string, string> = {
...c,
"1_targeting": t.join(" | "),
"1_targeting_note": targetingNote.get(c.id) ?? "",
"2_approach": a.join(" | "),
"3_attack": k.join(" | "),
"4_outcome": outcome,
"4_obtained": OBTAINED[c.theft] ?? "",
"4_demanded_and_obtained_note": summarySection(c.summary, "Crypto Demanded/Stolen"),
money_wanted_db: c.money_wanted,
coin_type_db: c.coin_type,
old_labels: labels.join(" > "),
...passCols,
};
return header.map((h) => withhold(c.id, h, row[h] ?? ""));
});
const out = basePath.replace("new-structure", "four-steps");
if (out === basePath) throw new Error("input name must contain 'new-structure'");
writeFileSync(out, "\uFEFF" + [header, ...rows].map((r) => r.map(cell).join(",")).join("\r\n") + "\r\n");
console.log(`${out}: ${rows.length} cases, ${header.length} columns`);
for (const [step, values] of Object.entries(count)) {
console.log(`\n${step}`);
for (const [v, n] of Object.entries(values).sort((x, y) => y[1] - x[1])) console.log(` ${String(n).padStart(4)} ${v}`);
}