Archived
chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
// Sort every case into the four steps of an attack, one CSV line per case:
|
||||
// 1 targeting how the attackers knew of the victim (rulings' "targeting vector"; Costly Fame)
|
||||
// 2 approach how they got to the victim (P2P, Social Engineering, Authority Misuse + subtypes)
|
||||
// 3 attack what they did (Kidnapping, Home Invasion, Armed Robbery, ...)
|
||||
// 4 outcome how it ended, and whether anything was obtained
|
||||
// Only existing data is used. "not sorted yet" marks a cell that needs the articles read;
|
||||
// "unknown" means the sources were read and do not say.
|
||||
// Input: the new-structure export (run make_new_structure_export.ts first).
|
||||
// Usage (from merge-output/): bun make_four_step_export.ts exports/gart-kr-new-structure-YYYY-MM-DD.csv
|
||||
import { existsSync, readFileSync, writeFileSync } from "fs";
|
||||
import { csvCell, withhold } from "./withheld";
|
||||
|
||||
const basePath = process.argv[2];
|
||||
if (!basePath) throw new Error("usage: bun make_four_step_export.ts <new-structure export.csv>");
|
||||
const NOT_SORTED = "not sorted yet";
|
||||
|
||||
const parseCsv = (text: string): string[][] => {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [], field = "", quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (quoted) {
|
||||
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
|
||||
else if (c === '"') quoted = false;
|
||||
else field += c;
|
||||
} else if (c === '"') quoted = true;
|
||||
else if (c === ",") row.push(field), (field = "");
|
||||
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
|
||||
else field += c;
|
||||
}
|
||||
if (quoted) throw new Error("unterminated quoted field in the input CSV");
|
||||
if (field || row.length) row.push(field), rows.push(row);
|
||||
return rows;
|
||||
};
|
||||
const [head, ...body] = parseCsv(readFileSync(basePath, "utf8").replace(/^/, ""));
|
||||
for (const h of ["id", "scenario_1", "scenario_2", "scenario_3", "verification", "outcome", "theft", "summary"])
|
||||
if (!head.includes(h)) throw new Error(`input CSV lacks column ${h}`);
|
||||
for (const r of body) if (r.length !== head.length) throw new Error(`case ${r[0]}: ${r.length} fields, expected ${head.length}`);
|
||||
const cases = body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
|
||||
|
||||
const subtypes = JSON.parse(readFileSync("step-subtypes.json", "utf8"));
|
||||
// result of the reading pass (steps_pass.ts merge); when present it replaces the values derived below
|
||||
const readList: any[] = existsSync("steps/merged.json") ? JSON.parse(readFileSync("steps/merged.json", "utf8")) : [];
|
||||
const read = new Map<string, any>(readList.map((c) => [String(c.id), c]));
|
||||
if (read.size !== readList.length) throw new Error("steps/merged.json holds a case id twice");
|
||||
|
||||
// "targeting vector = ..." in single-case ruling lines; the last one on a case is in force
|
||||
const targetingNote = new Map<string, string>();
|
||||
for (const block of readFileSync("WORKLIST-BY-CASE.md", "utf8").split(/^## case /m).slice(1)) {
|
||||
const id = /^\d+/.exec(block)![0];
|
||||
for (const line of block.split("\n").filter((l) => /^\s+- \[/.test(l) && /\*\*DB id \d+(?: \([^)]*\))?\*\*/.test(l)))
|
||||
for (const m of line.matchAll(/targeting vector\s*(?:(?:=|→|:)\s*|(?=unknown|not stated))([^;]+?)(?=;|\.\s|\.$|$)/g)) targetingNote.set(id, m[1].trim());
|
||||
}
|
||||
// the owner's proposed values (DEFINITIONS.md, targeting vector), matched in the ruling's wording
|
||||
const TARGETING: [string, RegExp][] = [
|
||||
["data leak", /data leak|breach|darkweb/i],
|
||||
["public records", /public record/i],
|
||||
["insider tip-off", /insider|tip-off/i],
|
||||
["family or partner", /intimate partner|^family \/ partner|girlfriend|boyfriend/i], // the attacker's tie to the victim, not the victim's to a holder
|
||||
["acquaintance", /acquaintance|friend/i],
|
||||
["business counterparty", /business counterparty|investor|collaborator/i],
|
||||
["public profile", /public profile|displayed wealth|professional profile|streamer/i],
|
||||
["social media", /social media/i],
|
||||
["online lure", /online lure|fake business contact/i],
|
||||
["surveillance", /surveillance|following|tailed/i],
|
||||
["opportunistic", /opportunistic/i],
|
||||
];
|
||||
const targeting = (c: Record<string, string>, labels: string[]) => {
|
||||
const note = targetingNote.get(c.id);
|
||||
const asserted = (note ?? "").replace(/\([^)]*\)?/g, " "); // the value, without the explanation in brackets
|
||||
const values: string[] = [];
|
||||
if (note && !/^(unknown|not stated)/i.test(note)) for (const [v, re] of TARGETING) if (re.test(asserted)) values.push(v);
|
||||
if (labels.includes("Costly Fame") && !values.includes("public profile")) values.push("public profile");
|
||||
if (values.length) return values;
|
||||
if (note) return [/^(unknown|not stated)/i.test(note) ? "unknown" : "other (see note)"];
|
||||
return [NOT_SORTED];
|
||||
};
|
||||
|
||||
const ATTACK: Record<string, string> = {
|
||||
Kidnapping: "Kidnapping",
|
||||
"Express Kidnapping": "Express Kidnapping",
|
||||
"Home Invasion": "Home Invasion",
|
||||
"Armed Robbery in Public Space": "Armed Robbery",
|
||||
"Assault - Mugging": "Mugging",
|
||||
Burglary: "Burglary",
|
||||
Swatting: "Swatting",
|
||||
};
|
||||
// where the attack label itself says how the attackers got to the victim
|
||||
// (not Home Invasion: the way in may be force, a trusted associate or unknown)
|
||||
const APPROACH_BY_DEFINITION: Record<string, string> = {
|
||||
Burglary: "break-in, nobody present",
|
||||
Mugging: "approached in public",
|
||||
Swatting: "none (remote)",
|
||||
};
|
||||
const approach = (id: string, labels: string[], attack: string[]) => {
|
||||
const values: string[] = [];
|
||||
for (const l of labels) {
|
||||
if (l === "P2P Trade Gone Wrong") values.push("P2P Trade");
|
||||
// "same fact as ...": the lure is already carried by the case's P2P Trade or Authority Misuse value
|
||||
const lure = subtypes.social_engineering[id] ?? NOT_SORTED;
|
||||
if (l === "Malicious Invitation - SocialEng" && !lure.startsWith("same fact as")) values.push(`Social Engineering: ${lure}`);
|
||||
if (l === "Authority Misuse") values.push(...String(subtypes.authority_misuse[id] ?? NOT_SORTED).split(" | ").map((s) => `Authority Misuse: ${s}`));
|
||||
}
|
||||
if (values.length) return values;
|
||||
const byDefinition = attack.map((a) => APPROACH_BY_DEFINITION[a]).filter(Boolean);
|
||||
return byDefinition.length ? byDefinition : [NOT_SORTED];
|
||||
};
|
||||
|
||||
const OBTAINED: Record<string, string> = { "1": "yes", "0": "nothing", unconfirmed: "unconfirmed" };
|
||||
const summarySection = (summary: string, name: string) =>
|
||||
new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*(.*?)(?=\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary)?.[1].trim() ?? "";
|
||||
const cell = csvCell;
|
||||
|
||||
const header = [
|
||||
"id", "date", "victim", "location", "country",
|
||||
"1_targeting", "1_targeting_accuracy", "1_who_was_attacked", "1_targeting_note",
|
||||
"2_approach",
|
||||
"3_attack",
|
||||
"4_outcome", "4_obtained",
|
||||
"demanded_amount", "demanded_currency", "demanded_usd_stated", "demanded_as_stated",
|
||||
"taken_amount", "taken_currency", "taken_usd_stated", "taken_as_stated",
|
||||
"4_demanded_and_obtained_note", "money_wanted_db", "coin_type_db",
|
||||
"detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken",
|
||||
"venue_of_control", "held_duration", "arrests",
|
||||
"old_labels", "attempted", "verification", "second_reader", "open_points", "linked_cases", "pending_action",
|
||||
"evidence_targeting", "evidence_approach", "evidence_attack", "evidence_outcome", "evidence_amounts",
|
||||
"summary",
|
||||
...head.filter((h) => /^source_\d+$/.test(h)),
|
||||
];
|
||||
const count: Record<string, Record<string, number>> = { "1_targeting": {}, "2_approach": {}, "3_attack": {}, "4_outcome": {} };
|
||||
const tally = (step: string, values: string[]) => values.forEach((v) => (count[step][v] = (count[step][v] ?? 0) + 1));
|
||||
|
||||
const rows = cases.map((c) => {
|
||||
const verified = c.verification !== "not verified";
|
||||
const labels = [c.scenario_1, c.scenario_2, c.scenario_3].filter(Boolean);
|
||||
const attack = labels.map((l) => ATTACK[l]).filter(Boolean);
|
||||
const t = verified ? targeting(c, labels) : [NOT_SORTED];
|
||||
const a = verified ? approach(c.id, labels, attack) : [NOT_SORTED];
|
||||
// a verified case with approach labels only (a P2P meeting, a lure) has no label for what was done there
|
||||
const k = attack.length ? attack : [verified && !labels.length ? "no label fits" : NOT_SORTED];
|
||||
const failed = labels.includes("Dodged Attack");
|
||||
let outcome = c.outcome || (failed ? "foiled" : NOT_SORTED);
|
||||
const p = read.get(c.id);
|
||||
const num = (v: unknown) => (v == null ? "" : String(v));
|
||||
const passCols: Record<string, string> = p
|
||||
? {
|
||||
"1_targeting_accuracy": p.targeting_accuracy,
|
||||
"1_who_was_attacked": p.who_was_attacked.join(" | "),
|
||||
"4_obtained": p.obtained.join(" | "),
|
||||
demanded_amount: num(p.demanded.amount), demanded_currency: p.demanded.currency ?? "", demanded_usd_stated: num(p.demanded.usd_stated), demanded_as_stated: p.demanded.as_stated ?? "",
|
||||
taken_amount: num(p.taken.amount), taken_currency: p.taken.currency ?? "", taken_usd_stated: num(p.taken.usd_stated), taken_as_stated: p.taken.as_stated ?? "",
|
||||
second_reader: p.second_reader, open_points: p.open_points ? String(p.open_points) : "",
|
||||
evidence_targeting: p.evidence.targeting, evidence_approach: p.evidence.approach, evidence_attack: p.evidence.attack, evidence_outcome: p.evidence.outcome, evidence_amounts: p.evidence.amounts,
|
||||
}
|
||||
: {};
|
||||
if (p) (t.length = 0), t.push(...p.targeting), (a.length = 0), a.push(...p.approach), (k.length = 0), k.push(...p.attack), (outcome = p.outcome);
|
||||
tally("1_targeting", t), tally("2_approach", a), tally("3_attack", k), tally("4_outcome", [outcome]);
|
||||
const row: Record<string, string> = {
|
||||
...c,
|
||||
"1_targeting": t.join(" | "),
|
||||
"1_targeting_note": targetingNote.get(c.id) ?? "",
|
||||
"2_approach": a.join(" | "),
|
||||
"3_attack": k.join(" | "),
|
||||
"4_outcome": outcome,
|
||||
"4_obtained": OBTAINED[c.theft] ?? "",
|
||||
"4_demanded_and_obtained_note": summarySection(c.summary, "Crypto Demanded/Stolen"),
|
||||
money_wanted_db: c.money_wanted,
|
||||
coin_type_db: c.coin_type,
|
||||
old_labels: labels.join(" > "),
|
||||
...passCols,
|
||||
};
|
||||
return header.map((h) => withhold(c.id, h, row[h] ?? ""));
|
||||
});
|
||||
|
||||
const out = basePath.replace("new-structure", "four-steps");
|
||||
if (out === basePath) throw new Error("input name must contain 'new-structure'");
|
||||
writeFileSync(out, "\uFEFF" + [header, ...rows].map((r) => r.map(cell).join(",")).join("\r\n") + "\r\n");
|
||||
console.log(`${out}: ${rows.length} cases, ${header.length} columns`);
|
||||
for (const [step, values] of Object.entries(count)) {
|
||||
console.log(`\n${step}`);
|
||||
for (const [v, n] of Object.entries(values).sort((x, y) => y[1] - x[1])) console.log(` ${String(n).padStart(4)} ${v}`);
|
||||
}
|
||||
Reference in New Issue
Block a user