This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/make_four_step_export.ts
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

183 lines
10 KiB
TypeScript
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Sort every case into the four steps of an attack, one CSV line per case:
// 1 targeting how the attackers knew of the victim (rulings' "targeting vector"; Costly Fame)
// 2 approach how they got to the victim (P2P, Social Engineering, Authority Misuse + subtypes)
// 3 attack what they did (Kidnapping, Home Invasion, Armed Robbery, ...)
// 4 outcome how it ended, and whether anything was obtained
// Only existing data is used. "not sorted yet" marks a cell that needs the articles read;
// "unknown" means the sources were read and do not say.
// Input: the new-structure export (run make_new_structure_export.ts first).
// Usage (from merge-output/): bun make_four_step_export.ts exports/gart-kr-new-structure-YYYY-MM-DD.csv
import { existsSync, readFileSync, writeFileSync } from "fs";
import { csvCell, withhold } from "./withheld";
const basePath = process.argv[2];
if (!basePath) throw new Error("usage: bun make_four_step_export.ts <new-structure export.csv>");
const NOT_SORTED = "not sorted yet";
const parseCsv = (text: string): string[][] => {
const rows: string[][] = [];
let row: string[] = [], field = "", quoted = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (quoted) {
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
else if (c === '"') quoted = false;
else field += c;
} else if (c === '"') quoted = true;
else if (c === ",") row.push(field), (field = "");
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
else field += c;
}
if (quoted) throw new Error("unterminated quoted field in the input CSV");
if (field || row.length) row.push(field), rows.push(row);
return rows;
};
const [head, ...body] = parseCsv(readFileSync(basePath, "utf8").replace(/^/, ""));
for (const h of ["id", "scenario_1", "scenario_2", "scenario_3", "verification", "outcome", "theft", "summary"])
if (!head.includes(h)) throw new Error(`input CSV lacks column ${h}`);
for (const r of body) if (r.length !== head.length) throw new Error(`case ${r[0]}: ${r.length} fields, expected ${head.length}`);
const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
const subtypes = JSON.parse(readFileSync("step-subtypes.json", "utf8"));
// result of the reading pass (steps_pass.ts merge); when present it replaces the values derived below
const readList: any[] = existsSync("steps/merged.json") ? JSON.parse(readFileSync("steps/merged.json", "utf8")) : [];
const read = new Map<string, any>(readList.map((c) => [String(c.id), c]));
if (read.size !== readList.length) throw new Error("steps/merged.json holds a case id twice");
// "targeting vector = ..." in single-case ruling lines; the last one on a case is in force
const targetingNote = new Map<string, string>();
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
const id = /^\d+/.exec(block)![0];
for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l)))
for (const m of line.matchAll(/targeting vector\s*(?:(?:=|→|:)\s*|(?=unknown|not stated))([^;]+?)(?=;|\.\s|\.$|$)/g)) targetingNote.set(id, m[1].trim());
}
// the owner's proposed values (DEFINITIONS.md, targeting vector), matched in the ruling's wording
const TARGETING: [string, RegExp][] = [
["data leak", /data leak|breach|darkweb/i],
["public records", /public record/i],
["insider tip-off", /insider|tip-off/i],
["family or partner", /intimate partner|^family \/ partner|girlfriend|boyfriend/i], // the attacker's tie to the victim, not the victim's to a holder
["acquaintance", /acquaintance|friend/i],
["business counterparty", /business counterparty|investor|collaborator/i],
["public profile", /public profile|displayed wealth|professional profile|streamer/i],
["social media", /social media/i],
["online lure", /online lure|fake business contact/i],
["surveillance", /surveillance|following|tailed/i],
["opportunistic", /opportunistic/i],
];
const targeting = (c: Record<string, string>, labels: string[]) => {
const note = targetingNote.get(c.id);
const asserted = (note ?? "").replace(/\([^)]*\)?/g, " "); // the value, without the explanation in brackets
const values: string[] = [];
if (note && !/^(unknown|not stated)/i.test(note)) for (const [v, re] of TARGETING) if (re.test(asserted)) values.push(v);
if (labels.includes("Costly Fame") && !values.includes("public profile")) values.push("public profile");
if (values.length) return values;
if (note) return [/^(unknown|not stated)/i.test(note) ? "unknown" : "other (see note)"];
return [NOT_SORTED];
};
const ATTACK: Record<string, string> = {
Kidnapping: "Kidnapping",
"Express Kidnapping": "Express Kidnapping",
"Home Invasion": "Home Invasion",
"Armed Robbery in Public Space": "Armed Robbery",
"Assault - Mugging": "Mugging",
Burglary: "Burglary",
Swatting: "Swatting",
};
// where the attack label itself says how the attackers got to the victim
// (not Home Invasion: the way in may be force, a trusted associate or unknown)
const APPROACH_BY_DEFINITION: Record<string, string> = {
Burglary: "break-in, nobody present",
Mugging: "approached in public",
Swatting: "none (remote)",
};
const approach = (id: string, labels: string[], attack: string[]) => {
const values: string[] = [];
for (const l of labels) {
if (l === "P2P Trade Gone Wrong") values.push("P2P Trade");
// "same fact as ...": the lure is already carried by the case's P2P Trade or Authority Misuse value
const lure = subtypes.social_engineering[id] ?? NOT_SORTED;
if (l === "Malicious Invitation - SocialEng" && !lure.startsWith("same fact as")) values.push(`Social Engineering: ${lure}`);
if (l === "Authority Misuse") values.push(...String(subtypes.authority_misuse[id] ?? NOT_SORTED).split(" | ").map((s) => `Authority Misuse: ${s}`));
}
if (values.length) return values;
const byDefinition = attack.map((a) => APPROACH_BY_DEFINITION[a]).filter(Boolean);
return byDefinition.length ? byDefinition : [NOT_SORTED];
};
const OBTAINED: Record<string, string> = { "1": "yes", "0": "nothing", unconfirmed: "unconfirmed" };
const summarySection = (summary: string, name: string) =>
new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*(.*?)(?=\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary)?.[1].trim() ?? "";
const cell = csvCell;
const header = [
"id", "date", "victim", "location", "country",
"1_targeting", "1_targeting_accuracy", "1_who_was_attacked", "1_targeting_note",
"2_approach",
"3_attack",
"4_outcome", "4_obtained",
"demanded_amount", "demanded_currency", "demanded_usd_stated", "demanded_as_stated",
"taken_amount", "taken_currency", "taken_usd_stated", "taken_as_stated",
"4_demanded_and_obtained_note", "money_wanted_db", "coin_type_db",
"detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken",
"venue_of_control", "held_duration", "arrests",
"old_labels", "attempted", "verification", "second_reader", "open_points", "linked_cases", "pending_action",
"evidence_targeting", "evidence_approach", "evidence_attack", "evidence_outcome", "evidence_amounts",
"summary",
...head.filter((h) => /^source_\d+$/.test(h)),
];
const count: Record<string, Record<string, number>> = { "1_targeting": {}, "2_approach": {}, "3_attack": {}, "4_outcome": {} };
const tally = (step: string, values: string[]) => values.forEach((v) => (count[step][v] = (count[step][v] ?? 0) + 1));
const rows = cases.map((c) => {
const verified = c.verification !== "not verified";
const labels = [c.scenario_1, c.scenario_2, c.scenario_3].filter(Boolean);
const attack = labels.map((l) => ATTACK[l]).filter(Boolean);
const t = verified ? targeting(c, labels) : [NOT_SORTED];
const a = verified ? approach(c.id, labels, attack) : [NOT_SORTED];
// a verified case with approach labels only (a P2P meeting, a lure) has no label for what was done there
const k = attack.length ? attack : [verified && !labels.length ? "no label fits" : NOT_SORTED];
const failed = labels.includes("Dodged Attack");
let outcome = c.outcome || (failed ? "foiled" : NOT_SORTED);
const p = read.get(c.id);
const num = (v: unknown) => (v == null ? "" : String(v));
const passCols: Record<string, string> = p
? {
"1_targeting_accuracy": p.targeting_accuracy,
"1_who_was_attacked": p.who_was_attacked.join(" | "),
"4_obtained": p.obtained.join(" | "),
demanded_amount: num(p.demanded.amount), demanded_currency: p.demanded.currency ?? "", demanded_usd_stated: num(p.demanded.usd_stated), demanded_as_stated: p.demanded.as_stated ?? "",
taken_amount: num(p.taken.amount), taken_currency: p.taken.currency ?? "", taken_usd_stated: num(p.taken.usd_stated), taken_as_stated: p.taken.as_stated ?? "",
second_reader: p.second_reader, open_points: p.open_points ? String(p.open_points) : "",
evidence_targeting: p.evidence.targeting, evidence_approach: p.evidence.approach, evidence_attack: p.evidence.attack, evidence_outcome: p.evidence.outcome, evidence_amounts: p.evidence.amounts,
}
: {};
if (p) (t.length = 0), t.push(...p.targeting), (a.length = 0), a.push(...p.approach), (k.length = 0), k.push(...p.attack), (outcome = p.outcome);
tally("1_targeting", t), tally("2_approach", a), tally("3_attack", k), tally("4_outcome", [outcome]);
const row: Record<string, string> = {
...c,
"1_targeting": t.join(" | "),
"1_targeting_note": targetingNote.get(c.id) ?? "",
"2_approach": a.join(" | "),
"3_attack": k.join(" | "),
"4_outcome": outcome,
"4_obtained": OBTAINED[c.theft] ?? "",
"4_demanded_and_obtained_note": summarySection(c.summary, "Crypto Demanded/Stolen"),
money_wanted_db: c.money_wanted,
coin_type_db: c.coin_type,
old_labels: labels.join(" > "),
...passCols,
};
return header.map((h) => withhold(c.id, h, row[h] ?? ""));
});
const out = basePath.replace("new-structure", "four-steps");
if (out === basePath) throw new Error("input name must contain 'new-structure'");
writeFileSync(out, "\uFEFF" + [header, ...rows].map((r) => r.map(cell).join(",")).join("\r\n") + "\r\n");
console.log(`${out}: ${rows.length} cases, ${header.length} columns`);
for (const [step, values] of Object.entries(count)) {
console.log(`\n${step}`);
for (const [v, n] of Object.entries(values).sort((x, y) => y[1] - x[1])) console.log(` ${String(n).padStart(4)} ${v}`);
}