Archived
Commits the work done since 2026-10-01 that was still local: the four-step label and flag passes, the triage and recheck files, the v1 exports of 2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch (scripts, agent prompts, reader and checker files, the flag re-read of 2026-10-17, the review workbook and the import set of the 57 cases added to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore). The stats2 material in use was copied to gart/gart-databases (stats2/) on 2026-10-17; README-START-HERE.md now says so. This repo is closed.
633 lines
46 KiB
TypeScript
633 lines
46 KiB
TypeScript
// Assemble the release data for the four-step database from the cleanup's side files:
|
||
// exports/gart-kr-new-structure-<date>.csv record fields, flags, facts, summary, sources (names withheld)
|
||
// steps/merged.json the two-reader four-step pass
|
||
// fields/merged.json, fields/not-applied.json written rulings turned into field values
|
||
// structural-ops.json owner-approved deletions and merges
|
||
// Writes into <site>/release: data/cases.json, data/labels.json, data/admin-points.json,
|
||
// export/gart-kr-database-<date>.csv, export/admin-decisions-<date>.csv and .md.
|
||
// Rebuilds everything from the inputs on every run and writes to none of them.
|
||
// Usage (from merge-output/): bun make_release_data.ts exports/gart-kr-new-structure-YYYY-MM-DD.csv
|
||
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs";
|
||
import { homedir } from "os";
|
||
import { join } from "path";
|
||
import { pointKey } from "./triage_pass";
|
||
import { csvCell, withhold } from "./withheld";
|
||
|
||
const basePath = process.argv[2];
|
||
if (!basePath) throw new Error("usage: bun make_release_data.ts <new-structure export.csv>");
|
||
const SITE =
|
||
process.env.GART_STATS_SITE ??
|
||
join(homedir(), "Documents/Claude/GART/05-strategy-marketing-sales/Marketing/gart-io/stats-gart-io-v3");
|
||
const OUT = join(SITE, "release");
|
||
const dataDate = /(\d{4}-\d{2}-\d{2})/.exec(basePath)?.[1] ?? "undated";
|
||
const buildDate = new Date().toLocaleDateString("sv"); // local date, ISO form
|
||
const J = (p: string) => JSON.parse(readFileSync(p, "utf8"));
|
||
|
||
const parseCsv = (text: string): string[][] => {
|
||
const rows: string[][] = [];
|
||
let row: string[] = [], field = "", quoted = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const c = text[i];
|
||
if (quoted) {
|
||
if (c === '"' && text[i + 1] === '"') (field += '"'), i++;
|
||
else if (c === '"') quoted = false;
|
||
else field += c;
|
||
} else if (c === '"') quoted = true;
|
||
else if (c === ",") row.push(field), (field = "");
|
||
else if (c === "\n") row.push(field.replace(/\r$/, "")), rows.push(row), (row = []), (field = "");
|
||
else field += c;
|
||
}
|
||
if (quoted) throw new Error("unterminated quoted field");
|
||
if (field || row.length) row.push(field), rows.push(row);
|
||
return rows;
|
||
};
|
||
const table = (path: string) => {
|
||
const [head, ...body] = parseCsv(readFileSync(path, "utf8").replace(/^/, ""));
|
||
for (const r of body) if (r.length !== head.length) throw new Error(`${path}: case ${r[0]} has ${r.length} fields, expected ${head.length}`);
|
||
return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ""])) as Record<string, string>);
|
||
};
|
||
|
||
const LABELS = J("step-labels.json");
|
||
const STEPS = ["targeting", "targeting_accuracy", "who_was_attacked", "approach", "attack", "outcome", "obtained"] as const;
|
||
const records = table(basePath);
|
||
const read = new Map<number, any>(J("steps/merged.json").map((c: any) => [c.id, c]));
|
||
const fieldChanges = new Map<number, any[]>(existsSync("fields/merged.json") ? J("fields/merged.json").map((c: any) => [c.id, c.changes]) : []);
|
||
const fieldNotApplied: any[] = existsSync("fields/not-applied.json") ? J("fields/not-applied.json") : [];
|
||
// the field pass must be merged and current: a stale or partial merge would drop corrections without a trace
|
||
const fieldBatches = existsSync("fields") ? readdirSync("fields").filter((f) => /^in-\d+\.json$/.test(f)) : [];
|
||
if (fieldBatches.length) {
|
||
if (!existsSync("fields/merged.json")) throw new Error("fields/merged.json is missing: run `bun fields_pass.ts merge` first");
|
||
const mergedAt = statSync("fields/merged.json").mtimeMs;
|
||
for (const f of readdirSync("fields").filter((f) => /^(out|check)-\d+\.json$/.test(f)))
|
||
if (statSync(join("fields", f)).mtimeMs > mergedAt) throw new Error(`fields/${f} is newer than fields/merged.json: run \`bun fields_pass.ts merge\` again`);
|
||
}
|
||
const fieldPassIds = new Set<number>(fieldBatches.flatMap((f) => J(join("fields", f)).map((c: any) => c.id)));
|
||
const ops = J("structural-ops.json");
|
||
const withheldIds = new Set(Object.keys(J("names-withheld.json")).filter((k) => /^\d+$/.test(k)).map(Number));
|
||
|
||
// ---- dates -------------------------------------------------------------
|
||
const MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"];
|
||
const pad = (n: number) => String(n).padStart(2, "0");
|
||
// "March 8, 2026" | "February, 2015" | "January ?, 2020" | "May 10 to 11, 2026" | "2018"
|
||
const parseDate = (text: string): { sort: string | null; precision: string } => {
|
||
const twoMonths = /^(?:late|early|mid)[- ]([A-Za-z]+) or (?:late|early|mid)[- ][A-Za-z]+,? (\d{4})$/i.exec(text.trim());
|
||
if (twoMonths && MONTHS.includes(twoMonths[1].toLowerCase())) return { sort: `${twoMonths[2]}-${pad(MONTHS.indexOf(twoMonths[1].toLowerCase()) + 1)}-01`, precision: "year" };
|
||
const t = text.trim().replace(/^(?:late|early|mid)[- ]/i, "");
|
||
let m = /^([A-Za-z]+) (\d{1,2})(?: ?(?:to|-) ?(\d{1,2}))?, (\d{4})$/.exec(t);
|
||
if (m && MONTHS.includes(m[1].toLowerCase())) {
|
||
const month = MONTHS.indexOf(m[1].toLowerCase()) + 1, day = Number(m[2]), last = m[3] ? Number(m[3]) : day;
|
||
const daysInMonth = new Date(Number(m[4]), month, 0).getDate();
|
||
// a day the calendar does not have, or a span that runs backwards, is not a date
|
||
if (day < 1 || last > daysInMonth || last < day) return { sort: null, precision: "unknown" };
|
||
return { sort: `${m[4]}-${pad(month)}-${pad(day)}`, precision: m[3] ? "range" : "day" };
|
||
}
|
||
m = /^([A-Za-z]+)(?: \?)?,? (\d{4})$/.exec(t);
|
||
if (m && MONTHS.includes(m[1].toLowerCase())) return { sort: `${m[2]}-${pad(MONTHS.indexOf(m[1].toLowerCase()) + 1)}-01`, precision: "month" };
|
||
m = /^(\d{4})$/.exec(t);
|
||
if (m) return { sort: `${m[1]}-01-01`, precision: "year" };
|
||
return { sort: null, precision: "unknown" };
|
||
};
|
||
|
||
// ---- cases -------------------------------------------------------------
|
||
const FLAG = { "1": "yes", "0": "no", unconfirmed: "unconfirmed", "threat only": "threat only", "": "" } as Record<string, string>;
|
||
const flag = (id: string, name: string, v: string) => {
|
||
if (!(v in FLAG)) throw new Error(`case ${id}: flag ${name} has value "${v}"`);
|
||
return FLAG[v];
|
||
};
|
||
const admin: Record<string, string>[] = [];
|
||
const point = (id: number | string, kind: string, detail: string, extra: Partial<Record<"field" | "first_value" | "second_value" | "quote" | "evidence_file", string>> = {}) =>
|
||
admin.push({ case_id: String(id), kind, field: "", detail, first_value: "", second_value: "", quote: "", evidence_file: "", ...extra });
|
||
|
||
const cases = new Map<number, any>();
|
||
for (const r of records) {
|
||
const id = Number(r.id);
|
||
const p = read.get(id);
|
||
if (!p) throw new Error(`case ${id}: no four-step reading in steps/merged.json`);
|
||
const sources = Array.from({ length: 8 }, (_, i) => r[`source_${i + 1}`]).filter(Boolean);
|
||
const c: any = {
|
||
id,
|
||
date: r.date, victim: r.victim, location: r.location, country: r.country,
|
||
description: r.description, summary: r.summary, notes: r.notes, reports: r.reports,
|
||
money_wanted: r.money_wanted, coin_type: r.coin_type,
|
||
flags: {
|
||
detention: flag(r.id, "detention", r.detention),
|
||
violence_torture: flag(r.id, "violence_torture", r.violence_torture),
|
||
drugs_alcohol: flag(r.id, "drugs_alcohol", r.drugs_alcohol),
|
||
weapons: flag(r.id, "weapons", r.weapons),
|
||
theft: flag(r.id, "theft", r.theft),
|
||
},
|
||
life_taken: /^\d+$/.test(r.life_taken) ? Number(r.life_taken) : null,
|
||
life_taken_unconfirmed: r.life_taken === "unconfirmed",
|
||
venue_of_control: r.venue_of_control, held_duration: r.held_duration, arrests: r.arrests,
|
||
steps: Object.fromEntries(STEPS.map((s) => [s, p[s]])),
|
||
demanded: p.demanded, taken: p.taken,
|
||
sources,
|
||
linked_cases: r.linked_cases ? r.linked_cases.split(" | ").map(Number) : [],
|
||
verification: r.verification,
|
||
second_reader: p.second_reader,
|
||
name_withheld: withheldIds.has(id),
|
||
v1_scenario: r.v1_scenario,
|
||
old_labels: [r.scenario_1, r.scenario_2, r.scenario_3].filter(Boolean),
|
||
former_ids: [] as number[],
|
||
dossier_file: r.dossier_file,
|
||
internal: { evidence: p.evidence, open_points: p.open_points ?? 0, field_changes: [] as any[] },
|
||
};
|
||
// text the readers wrote also goes through the withheld-name list
|
||
for (const k of Object.keys(c.internal.evidence)) c.internal.evidence[k] = withhold(id, "evidence", String(c.internal.evidence[k]));
|
||
for (const m of [c.demanded, c.taken]) if (m?.as_stated) m.as_stated = withhold(id, "as_stated", m.as_stated);
|
||
cases.set(id, c);
|
||
}
|
||
|
||
// 1. field rulings, confirmed by the second reader
|
||
let changed = 0;
|
||
const stale: any[] = [];
|
||
for (const [id, changes] of fieldChanges) {
|
||
const c = cases.get(id);
|
||
if (!c) continue;
|
||
for (const ch of changes) {
|
||
if (!(ch.field in c)) throw new Error(`case ${id}: field ${ch.field} is not a record field`);
|
||
// the change was read against a record value; if the record has moved since, a person decides
|
||
if (String(ch.old ?? "").replace(/\s+/g, " ").trim() !== String(c[ch.field] ?? "").replace(/\s+/g, " ").trim()) {
|
||
stale.push({ id, field: ch.field, proposed: ch.new, quote: ch.quote, record: c[ch.field] });
|
||
continue;
|
||
}
|
||
const value = withhold(id, ch.field, ch.new);
|
||
c.internal.field_changes.push({ field: ch.field, old: c[ch.field], new: value, quote: ch.quote });
|
||
c[ch.field] = value;
|
||
changed++;
|
||
}
|
||
}
|
||
|
||
// 2. merges, then 3. deletions (owner-approved list only)
|
||
const before = cases.size;
|
||
const removed: number[] = [];
|
||
for (const m of ops.merge) {
|
||
const into = cases.get(m.into);
|
||
if (!into) throw new Error(`merge target ${m.into} missing`);
|
||
for (const from of m.from) {
|
||
const src = cases.get(from);
|
||
if (!src) throw new Error(`merge source ${from} missing`);
|
||
if (m.move_sources) for (const u of src.sources) if (!into.sources.includes(u)) into.sources.push(u);
|
||
for (const l of src.linked_cases) if (!into.linked_cases.includes(l)) into.linked_cases.push(l);
|
||
into.former_ids.push(from);
|
||
cases.delete(from);
|
||
removed.push(from);
|
||
point(from, "structural: applied in this build only", `Merged into case ${m.into} here (${m.reason}). The live database still holds it as a separate record.`, { evidence_file: "merge-output/structural-ops.json" });
|
||
}
|
||
if (m.set) for (const [k, v] of Object.entries(m.set)) (into.internal.field_changes.push({ field: k, old: into[k], new: v, quote: `structural-ops.json: ${m.reason}` }), (into[k] = v));
|
||
if (m.append_note) into.notes = [into.notes, m.append_note].filter(Boolean).join(" ");
|
||
}
|
||
for (const d of ops.delete) {
|
||
if (!cases.delete(d.id)) throw new Error(`delete target ${d.id} missing`);
|
||
removed.push(d.id);
|
||
point(d.id, "structural: applied in this build only", `Deleted here (${d.reason}). The live database still holds it.`, { evidence_file: "merge-output/structural-ops.json" });
|
||
}
|
||
if (cases.size !== before - removed.length) throw new Error("case count does not reconcile after merges and deletions");
|
||
// links must not point at a removed record
|
||
const alias = new Map<number, number>(ops.merge.flatMap((m: any) => m.from.map((f: number) => [f, m.into])));
|
||
for (const c of cases.values()) c.linked_cases = [...new Set(c.linked_cases.map((x: number) => alias.get(x) ?? x))].filter((x) => x !== c.id && cases.has(x as number));
|
||
|
||
// Ruled splits and rewrites carried out from the dossiers (splits/out-<id>.json, owner rulings of 2026-10-01
|
||
// executed on 2026-10-08). The record that keeps the id is overwritten; a new record gets the id 10000 + the
|
||
// parent's id, which the live database cannot hand out, and the two are linked.
|
||
const SPLIT_FLAGS = ["detention", "violence_torture", "drugs_alcohol", "weapons", "theft"];
|
||
const splitIds = new Set<number>();
|
||
const applySplitRecord = (c: any, r: any) => {
|
||
// an earlier change to a field that the split now rewrites is no longer the field's last word
|
||
c.internal.field_changes = c.internal.field_changes.filter((ch: any) => typeof r[ch.field] !== "string");
|
||
for (const f of ["date", "victim", "location", "country", "description", "summary", "notes", "money_wanted", "coin_type"]) {
|
||
if (typeof r[f] !== "string" || r[f] === c[f]) continue;
|
||
if (f !== "summary") c.internal.field_changes.push({ field: f, old: c[f], new: r[f], quote: "split or rewrite ordered by the owner, written from the dossier on 2026-10-08" });
|
||
c[f] = r[f];
|
||
}
|
||
for (const f of SPLIT_FLAGS) c.flags[f] = r[f];
|
||
c.life_taken_unconfirmed = r.life_taken === "unconfirmed";
|
||
c.life_taken = r.life_taken === "unconfirmed" ? null : Number(r.life_taken);
|
||
for (const st of STEPS) c.steps[st] = r[st];
|
||
if (Array.isArray(r.sources) && r.sources.length) c.sources = r.sources;
|
||
c.internal.evidence = Object.fromEntries((r.support ?? []).map((x: any, i: number) => [`split_${i + 1}`, `${x.claim}: ${x.quote}`]));
|
||
splitIds.add(c.id);
|
||
};
|
||
const newRecords = (existsSync("splits") ? readdirSync("splits").filter((f) => /^out-\d+\.json$/.test(f)) : []).length;
|
||
for (const f of existsSync("splits") ? readdirSync("splits").filter((f) => /^out-\d+\.json$/.test(f)).sort() : []) {
|
||
const o = J(join("splits", f));
|
||
const parent = cases.get(o.id);
|
||
if (!parent) throw new Error(`splits/${f}: case ${o.id} is not in the data`);
|
||
const fresh = JSON.parse(JSON.stringify(parent));
|
||
for (const r of o.records) {
|
||
if (r.keeps_id) { applySplitRecord(parent, r); parent.verification = `rewritten from the dossier on 2026-10-08 to carry out the owner's ruling (${o.records.length > 1 ? "split" : "rewrite"}); one reader, the owner has not reviewed the text`; continue; }
|
||
const c = { ...fresh, id: 10000 + o.id, former_ids: [], linked_cases: [o.id], old_labels: [], v1_scenario: "", internal: { evidence: {}, open_points: 0, field_changes: [] },
|
||
demanded: { amount: null, currency: null, as_stated: "", usd_stated: null }, taken: { amount: null, currency: null, as_stated: "", usd_stated: null }, venue_of_control: "", held_duration: "" };
|
||
applySplitRecord(c, r);
|
||
c.verification = `written from the dossier of record ${o.id} on 2026-10-08, when the owner's ruling split that record; one reader, the owner has not reviewed the text`;
|
||
cases.set(c.id, c);
|
||
if (!parent.linked_cases.includes(c.id)) parent.linked_cases.push(c.id);
|
||
}
|
||
}
|
||
|
||
// owner rulings on a record field of one case (step-rulings.json): set before names are withheld and dates are read
|
||
const RECORD_FIELDS = ["date", "victim", "location", "country", "description", "summary", "notes"];
|
||
const ruledField = new Set<string>(); // "id:field" the owner has ruled on; open items on it are closed
|
||
for (const r of (existsSync("step-rulings.json") ? J("step-rulings.json").rulings : []).filter((r: any) => RECORD_FIELDS.includes(r.field))) {
|
||
const c = cases.get(r.id);
|
||
if (!c) continue;
|
||
c.internal.field_changes.push({ field: r.field, old: c[r.field], new: r.value, quote: r.ruling });
|
||
c[r.field] = r.value;
|
||
ruledField.add(`${r.id}:${r.field}`);
|
||
}
|
||
|
||
// night-shift triage of 2026-10-08: open items settled by precedent. Applied values are defaults the owner can overturn.
|
||
const triage: { applied: any[]; closed: any[]; parked: any[] } = existsSync("triage/decisions.json") ? J("triage/decisions.json") : { applied: [], closed: [], parked: [] };
|
||
if (existsSync("triage/decisions.json")) {
|
||
const at = statSync("triage/decisions.json").mtimeMs;
|
||
for (const f of readdirSync("triage").filter((f) => /^(out|check)-\d+\.json$/.test(f)))
|
||
if (statSync(join("triage", f)).mtimeMs > at) throw new Error(`triage/${f} is newer than triage/decisions.json: run \`bun triage_pass.ts merge\` again`);
|
||
}
|
||
const TRIAGE_RECORD = [...RECORD_FIELDS.filter((f) => f !== "summary"), "money_wanted", "coin_type"];
|
||
const nightDefaults: any[] = [];
|
||
for (const t of triage.applied.filter((t) => TRIAGE_RECORD.includes(t.field))) {
|
||
const c = cases.get(t.id);
|
||
if (!c || splitIds.has(t.id) || ruledField.has(`${t.id}:${t.field}`)) continue;
|
||
const value = t.op === "append" ? [c[t.field], t.value].filter(Boolean).join(" ") : t.value;
|
||
c.internal.field_changes.push({ field: t.field, old: c[t.field], new: value, quote: `night-shift default (precedent ${t.precedent}): ${t.quote}` });
|
||
c[t.field] = value;
|
||
nightDefaults.push(t);
|
||
}
|
||
|
||
// cases that arrived after the verification, compared with their own sources on 2026-10-08 (newcases/):
|
||
// a value is taken when the first reader supports it with a passage of the fetched text and the second reader confirms it
|
||
const newCaseValues = new Map<number, Record<string, any>>();
|
||
const normText = (t: unknown) => String(t ?? "").replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, " ").trim();
|
||
for (const f of existsSync("newcases") ? readdirSync("newcases").filter((f) => /^out-\d+\.json$/.test(f)) : []) {
|
||
const o = J(join("newcases", f));
|
||
const checkPath = join("newcases", f.replace("out-", "check-"));
|
||
const c = cases.get(o.id);
|
||
if (!c || !existsSync(checkPath) || statSync(checkPath).mtimeMs <= statSync(join("newcases", f)).mtimeMs) continue;
|
||
const verdicts: Record<string, string> = J(checkPath).verdicts ?? {};
|
||
const text = normText(existsSync(`newcases/sources-${o.id}.txt`) ? readFileSync(`newcases/sources-${o.id}.txt`, "utf8") : "");
|
||
const taken: Record<string, any> = {};
|
||
for (const [field, value] of Object.entries(o.values ?? {})) {
|
||
if (verdicts[field] !== "confirm") continue;
|
||
const quote = normText(o.support?.[field]);
|
||
// a changed value needs its passage in the fetched text; the summary is checked as a whole by the second reader
|
||
if ((o.changed ?? []).includes(field) && field !== "summary" && field !== "description" && (quote.length < 8 || !text.includes(quote))) continue;
|
||
taken[field] = value;
|
||
}
|
||
newCaseValues.set(o.id, taken);
|
||
const reasons: Record<string, string> = J(checkPath).reasons ?? {};
|
||
for (const field of Object.keys(o.values ?? {}).filter((k) => !(k in taken)))
|
||
point(o.id, "new case: value not confirmed", `Compared with the fetched sources on 2026-10-08. The first reader proposed a new ${field}; the second reader did not confirm it: ${reasons[field] ?? "no reason given"} The record's value stays.`, { field, second_value: typeof o.values[field] === "string" ? o.values[field] : JSON.stringify(o.values[field]), evidence_file: `merge-output/newcases/out-${o.id}.json` });
|
||
if ((o.not_fetched ?? []).length) point(o.id, "new case: source not read", `These sources of the record could not be fetched, so details that rest on them are unchecked: ${o.not_fetched.join("; ")}`, { evidence_file: `merge-output/newcases/sources-${o.id}.txt` });
|
||
for (const field of RECORD_FIELDS.filter((x) => x !== "notes")) {
|
||
if (!(field in taken) || ruledField.has(`${o.id}:${field}`) || taken[field] === c[field]) continue;
|
||
c.internal.field_changes.push({ field, old: c[field], new: taken[field], quote: `compared with ${(o.fetched ?? []).length} fetched sources, 2026-10-08 (night-shift default)` });
|
||
c[field] = taken[field];
|
||
}
|
||
c.verification = `fields, flags and steps compared with ${(o.fetched ?? []).length} fetched sources on 2026-10-08 by two readers (night shift; the owner has not reviewed it)`;
|
||
if (o.notes) c.notes = [c.notes, `Verification 2026-10-08: ${o.notes}`].filter(Boolean).join(" ");
|
||
}
|
||
|
||
// wording fixes to verified summaries that rulings ordered (rewrites/summary-fixes.json): exact passages replaced
|
||
let summaryFixes = 0;
|
||
for (const f of existsSync("rewrites/summary-fixes.json") ? J("rewrites/summary-fixes.json") : []) {
|
||
const c = cases.get(f.id);
|
||
if (!c || ruledField.has(`${f.id}:summary`)) continue;
|
||
for (const e of f.edits) {
|
||
if (c.summary.split(e.old).length !== 2) throw new Error(`rewrites/summary-fixes.json: case ${f.id}: the passage to replace is not found exactly once in the summary`);
|
||
c.summary = c.summary.replace(e.old, () => e.new);
|
||
c.internal.field_changes.push({ field: "summary", old: e.old, new: e.new, quote: `ruling wording fix: ${f.support ?? f.note ?? ""}` });
|
||
summaryFixes++;
|
||
}
|
||
}
|
||
|
||
// names withheld at a victim's request: applied to the final text of every field, whatever the inputs carried
|
||
const TEXT_FIELDS = ["victim", "location", "description", "summary", "notes", "reports", "money_wanted", "coin_type"];
|
||
for (const c of cases.values()) {
|
||
if (!c.name_withheld) continue;
|
||
for (const f of TEXT_FIELDS) c[f] = withhold(c.id, f, String(c[f] ?? ""));
|
||
for (const ch of c.internal.field_changes) ch.new = withhold(c.id, ch.field, String(ch.new ?? ""));
|
||
}
|
||
|
||
// dates after all changes
|
||
for (const c of cases.values()) {
|
||
const d = parseDate(c.date);
|
||
c.date_sort = d.sort;
|
||
c.date_precision = d.precision;
|
||
c.year = d.sort ? Number(d.sort.slice(0, 4)) : null;
|
||
c.month = d.sort && d.precision !== "year" ? Number(d.sort.slice(5, 7)) : null;
|
||
c.quarter = c.year && c.month ? `${c.year}-Q${Math.ceil(c.month / 3)}` : null;
|
||
if (!d.sort) point(c.id, "date not readable", `The date "${c.date}" could not be turned into a sortable date.`, { field: "date" });
|
||
}
|
||
|
||
// the owner's recheck of 2026-10-07 (who was attacked, outcome "fought off"): second-reader-confirmed changes
|
||
if (existsSync("recheck/merged.json")) {
|
||
const mergedAt = statSync("recheck/merged.json").mtimeMs;
|
||
for (const f of readdirSync("recheck").filter((f) => /^(out|check)-\d+\.json$/.test(f)))
|
||
if (statSync(join("recheck", f)).mtimeMs > mergedAt) throw new Error(`recheck/${f} is newer than recheck/merged.json: run \`bun recheck_pass.ts merge\` again`);
|
||
}
|
||
const recheck: any[] = existsSync("recheck/merged.json") ? J("recheck/merged.json") : [];
|
||
const recheckNotApplied: any[] = existsSync("recheck/not-applied.json") ? J("recheck/not-applied.json") : [];
|
||
let rechecked = 0;
|
||
for (const r of recheck) {
|
||
const c = cases.get(r.id);
|
||
if (!c) continue;
|
||
for (const ch of r.changes) {
|
||
// read against a value that has since moved: leave it to a person
|
||
if (JSON.stringify([c.steps[ch.field]].flat().sort()) !== JSON.stringify([ch.old].flat().sort())) {
|
||
recheckNotApplied.push({ id: r.id, kind: "recheck: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote });
|
||
continue;
|
||
}
|
||
c.steps[ch.field] = ch.new;
|
||
rechecked++;
|
||
}
|
||
}
|
||
|
||
// the full label check of 2026-10-07 (every label tested against LABEL-RULES.md): confirmed changes
|
||
if (existsSync("labels/merged.json")) {
|
||
const mergedAt = statSync("labels/merged.json").mtimeMs;
|
||
for (const f of readdirSync("labels").filter((f) => /^(out|check)-\d+\.json$/.test(f) || f === "tiebreak-out.json"))
|
||
if (statSync(join("labels", f)).mtimeMs > mergedAt) throw new Error(`labels/${f} is newer than labels/merged.json: run \`bun labels_pass.ts merge\` again`);
|
||
}
|
||
const labelCheck: any[] = existsSync("labels/merged.json") ? J("labels/merged.json") : [];
|
||
const labelNotApplied: any[] = existsSync("labels/not-applied.json") ? J("labels/not-applied.json") : [];
|
||
let labelsChanged = 0;
|
||
for (const r of labelCheck) {
|
||
const c = cases.get(r.id);
|
||
if (!c || splitIds.has(r.id)) continue;
|
||
for (const ch of r.changes) {
|
||
if (JSON.stringify([c.steps[ch.field]].flat().sort()) !== JSON.stringify([ch.old].flat().sort())) {
|
||
labelNotApplied.push({ id: r.id, kind: "label check: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote });
|
||
continue;
|
||
}
|
||
c.steps[ch.field] = ch.new;
|
||
c.internal.evidence[`label_check_${ch.field}`] = ch.quote;
|
||
labelsChanged++;
|
||
}
|
||
}
|
||
// Cases whose seven step fields went through the full label check (two readers, a third on disagreement).
|
||
// That check supersedes older open questions on the same fields: the step pass, the recheck, Dodged Attack against foiled.
|
||
const labelChecked = new Set<number>();
|
||
if (existsSync("labels/merged.json"))
|
||
for (const f of readdirSync("labels").filter((f) => /^in-\d+\.json$/.test(f))) {
|
||
const out = join("labels", f.replace("in-", "out-")), check = join("labels", f.replace("in-", "check-"));
|
||
if (!existsSync(out) || !existsSync(check) || statSync(check).mtimeMs <= statSync(out).mtimeMs) continue; // not second-read
|
||
const read = new Set<number>(J(out).map((o: any) => o.id));
|
||
for (const c of J(join("labels", f))) if (read.has(c.id)) labelChecked.add(c.id);
|
||
}
|
||
const olderRecheck = recheckNotApplied.splice(0).filter((n) => !labelChecked.has(n.id));
|
||
recheckNotApplied.push(...olderRecheck, ...labelNotApplied);
|
||
|
||
// the flag check of 2026-10-07 (violence flags and deaths tested against FLAG-RULES.md): confirmed changes
|
||
if (existsSync("flags/merged.json")) {
|
||
const mergedAt = statSync("flags/merged.json").mtimeMs;
|
||
for (const f of readdirSync("flags").filter((f) => /^(out|check)-\d+\.json$/.test(f)))
|
||
if (statSync(join("flags", f)).mtimeMs > mergedAt) throw new Error(`flags/${f} is newer than flags/merged.json: run \`bun flags_pass.ts merge\` again`);
|
||
}
|
||
const flagCheck: any[] = existsSync("flags/merged.json") ? J("flags/merged.json") : [];
|
||
const flagNotApplied: any[] = existsSync("flags/not-applied.json") ? J("flags/not-applied.json") : [];
|
||
// unlocked flag values two readers tested and left alone, or changed together
|
||
const flagChecked = new Set<string>(existsSync("flags/checked.json") ? J("flags/checked.json") : []);
|
||
let flagsChanged = 0;
|
||
const deaths = (c: any) => (c.life_taken_unconfirmed ? "unconfirmed" : c.life_taken == null ? "" : String(c.life_taken));
|
||
for (const r of flagCheck) {
|
||
const c = cases.get(r.id);
|
||
if (!c || splitIds.has(r.id)) continue;
|
||
for (const ch of r.changes) {
|
||
const now = ch.field === "life_taken" ? deaths(c) : c.flags[ch.field];
|
||
if (now !== ch.old) { flagNotApplied.push({ id: r.id, kind: "flag check: value changed since it was read", field: ch.field, proposed: ch.new, quote: ch.quote }); continue; }
|
||
if (ch.field === "life_taken") { c.life_taken_unconfirmed = ch.new === "unconfirmed"; c.life_taken = ch.new === "unconfirmed" ? null : Number(ch.new); }
|
||
else c.flags[ch.field] = ch.new;
|
||
c.internal.evidence[`flag_check_${ch.field}`] = ch.quote;
|
||
flagsChanged++;
|
||
}
|
||
}
|
||
|
||
for (const t of triage.applied.filter((t) => !TRIAGE_RECORD.includes(t.field))) {
|
||
const c = cases.get(t.id);
|
||
if (!c) continue;
|
||
if (t.field === "life_taken") { c.life_taken_unconfirmed = t.value === "unconfirmed"; c.life_taken = t.value === "unconfirmed" ? null : Number(t.value); }
|
||
else if (t.field in c.flags) c.flags[t.field] = t.value;
|
||
else throw new Error(`triage/decisions.json: case ${t.id} has no field ${t.field}`);
|
||
c.internal.evidence[`night_shift_${t.field}`] = t.quote;
|
||
nightDefaults.push(t);
|
||
}
|
||
|
||
for (const [id, taken] of newCaseValues) {
|
||
const c = cases.get(id);
|
||
for (const [field, value] of Object.entries(taken)) {
|
||
if (field === "life_taken") { c.life_taken_unconfirmed = value === "unconfirmed"; c.life_taken = value === "unconfirmed" ? null : Number(value); }
|
||
else if (field in c.flags) { if (["yes", "no", "unconfirmed", "threat only"].includes(value)) c.flags[field] = value; }
|
||
else if (field in c.steps) c.steps[field] = value; // the vocabulary check below rejects an unknown label
|
||
}
|
||
}
|
||
|
||
// owner rulings on a step value of one case
|
||
const stepRulings: any[] = existsSync("step-rulings.json") ? J("step-rulings.json").rulings : [];
|
||
const ruledStep = new Set<string>();
|
||
for (const r of stepRulings) {
|
||
const c = cases.get(r.id);
|
||
if (!c) continue;
|
||
if (RECORD_FIELDS.includes(r.field)) continue; // set earlier, before the dates were read
|
||
if (r.field === "life_taken") {
|
||
c.life_taken_unconfirmed = r.value === "unconfirmed";
|
||
c.life_taken = r.value === "unconfirmed" ? null : Number(r.value);
|
||
} else if (r.field in c.flags) {
|
||
if (!["yes", "no", "unconfirmed", "threat only"].includes(r.value)) throw new Error(`step-rulings.json: case ${r.id}: "${r.value}" is not a flag value`);
|
||
c.flags[r.field] = r.value;
|
||
} else if (r.field in c.steps) c.steps[r.field] = r.value;
|
||
else throw new Error(`step-rulings.json: case ${r.id} has no field ${r.field}`);
|
||
ruledStep.add(`${r.id}:${r.field}`);
|
||
}
|
||
|
||
// label check: nothing outside the vocabulary, nothing guessed
|
||
for (const c of cases.values())
|
||
for (const s of STEPS) {
|
||
const values: string[] = LABELS[s].several ? c.steps[s] : [c.steps[s]];
|
||
if (!values.length) throw new Error(`case ${c.id}: step ${s} is empty`);
|
||
for (const v of values) if (!Object.hasOwn(LABELS[s].values, v)) throw new Error(`case ${c.id}: "${v}" is not a ${s} label`);
|
||
}
|
||
|
||
// ---- admin points --------------------------------------------------------
|
||
const live = (id: number) => cases.has(id);
|
||
for (const c of cases.values()) {
|
||
const dodged = c.old_labels.includes("Dodged Attack");
|
||
const foiled = c.steps.outcome === "foiled";
|
||
if (ruledStep.has(`${c.id}:outcome`) || labelChecked.has(c.id)) {
|
||
// the owner has ruled on this outcome, or the label check tested it against the owner's rule for foiled
|
||
} else if (dodged && !foiled) point(c.id, "foiled or not: owner decision", `Old label Dodged Attack, but the readers found someone was held, hurt or robbed: outcome "${c.steps.outcome}". Decide whether the case counts as foiled.`, { field: "outcome", first_value: c.steps.outcome, second_value: "foiled", quote: c.internal.evidence.outcome, evidence_file: `merge-output/sources/${c.dossier_file}` });
|
||
else if (!dodged && foiled) point(c.id, "foiled or not: owner decision", "Read as foiled, but the case has no Dodged Attack label in the verified sequence.", { field: "outcome", first_value: "foiled", quote: c.internal.evidence.outcome, evidence_file: `merge-output/sources/${c.dossier_file}` });
|
||
if (c.steps.attack.includes("no label fits")) point(c.id, "no attack label fits", "None of the attack labels describes this case. Decide a label or whether the case stays in the database.", { field: "attack", quote: c.internal.evidence.attack, evidence_file: `merge-output/sources/${c.dossier_file}` });
|
||
if (c.verification === "not verified") point(c.id, "not verified", "Arrived after the verification; summary, labels and fields are unchecked. The four steps were read from the site summary only.", { evidence_file: "attacks-live-api-" + dataDate + ".json" });
|
||
if (c.verification.startsWith("record only")) point(c.id, "no article on file", "No usable article is on file; every value rests on the record alone.", { evidence_file: "merge-output/sofia-worklist.md" });
|
||
if (c.verification.startsWith("summary and labels checked")) point(c.id, "record fields never compared with articles", "Summary and labels were verified by the older method; date, victim, location, amounts and flags were never compared with the articles.", { evidence_file: `merge-output/sources/${c.dossier_file}` });
|
||
}
|
||
for (const p of table("steps/open-points.csv").filter((p) => p.applied === "no" && live(Number(p.id)) && !(labelChecked.has(Number(p.id)) && STEPS.includes(p.field)) && !ruledStep.has(`${p.id}:${p.field}`)))
|
||
point(p.id, "two readers disagree", `Step value where the second reader prefers another reading (${p.kind}). The first reader's value is in the database.`, { field: p.field, first_value: withhold(p.id, "x", p.first_reader), second_value: withhold(p.id, "x", p.second_reader), quote: withhold(p.id, "x", p.quote), evidence_file: `merge-output/steps/open-points.csv` });
|
||
for (const n of fieldNotApplied.filter((n) => live(n.id) && !ruledField.has(`${n.id}:${n.field}`)))
|
||
point(n.id, `field ruling: ${n.kind}`, n.detail ?? "", { field: n.field ?? "", second_value: n.proposed ?? "", quote: n.quote ?? "", evidence_file: "merge-output/corrections-approved.md" });
|
||
for (const n of recheckNotApplied.filter((n) => live(n.id) && !ruledStep.has(`${n.id}:${n.field}`)))
|
||
point(n.id, n.kind, n.detail ?? "A change proposed in the recheck that was not applied.", { field: n.field ?? "", first_value: JSON.stringify(cases.get(n.id)?.steps[n.field] ?? ""), second_value: typeof n.proposed === "string" ? n.proposed : JSON.stringify(n.proposed ?? ""), quote: n.quote ?? "", evidence_file: "merge-output/recheck/not-applied.json" });
|
||
for (const n of flagNotApplied.filter((n) => live(n.id) && !ruledStep.has(`${n.id}:${n.field}`)))
|
||
point(n.id, n.kind, n.detail ?? "A flag change proposed in the flag check that was not applied.", { field: n.field ?? "", first_value: n.field === "life_taken" ? deaths(cases.get(n.id)) : cases.get(n.id)?.flags[n.field] ?? "", second_value: String(n.proposed ?? ""), quote: n.quote ?? "", evidence_file: "merge-output/flags/not-applied.json" });
|
||
for (const f of J("exports/flag-values-from-proposals.json").filter((f: any) => live(f.id) && !flagChecked.has(`${f.id}:${f.flag}`)))
|
||
point(f.id, "flag value from an unruled proposal", `The ${f.flag} flag follows our proposal "${f.value}"; no ruling confirms it.`, { field: f.flag, second_value: f.value, quote: f.evidence, evidence_file: "merge-output/batches/flag-proposals.json" });
|
||
for (const st of stale) if (live(st.id) && !ruledField.has(`${st.id}:${st.field}`)) point(st.id, "field ruling: record changed since the ruling was read", `The record's ${st.field} no longer holds the value the change was read against, so the change was not applied.`, { field: st.field, first_value: st.record, second_value: st.proposed, quote: st.quote, evidence_file: "merge-output/fields/merged.json" });
|
||
for (const id of fieldPassIds) if (!fieldChanges.has(id) && live(id) && !fieldNotApplied.some((n) => n.id === id && n.kind === "field pass not run")) point(id, "field ruling: field pass not run", "This case has ruling lines and no result from the field pass.", { evidence_file: "merge-output/WORKLIST-BY-CASE.md" });
|
||
for (const n of ops.not_applied) point(n.id, `structural: ${n.kind} not done`, n.detail, { evidence_file: "HANDOVER-SOFI.md" });
|
||
const fixed: [number, string, string, string][] = [
|
||
[290, "possible duplicate", "Reads as the same Dubai incident as case 296 (Al Nahda, three women, Dh300,000).", "merge-output/steps/out-10.json"],
|
||
[296, "possible duplicate", "Reads as the same Dubai incident as case 290.", "merge-output/steps/out-10.json"],
|
||
[476, "possible duplicate", "Sheet row 252 (Phuket, August 2025), listed as missing from the database, looks like this case.", "merge-output/cases-missing-from-database.json"],
|
||
[476, "source fix", "The AseanNow URL on the record points at the French case 350. Correct thread: https://aseannow.com/topic/1371363-russian-fugitive-caught-hiding-in-car-at-koh-samui-ferry-port/", "merge-output/sofia-worklist.md"],
|
||
[103, "source fix", 'Replace the "El Pais, Iustel" credit with Madrid Actual, Gacetin Madrid, Vozpopuli (2024) and El Pais (2022).', "merge-output/sofia-worklist.md"],
|
||
[24, "on hold: owner", "The owner keeps the case; it has no label yet.", "merge-output/disputed-scenarios.md"],
|
||
[195, "on hold: owner", "The owner keeps the case; it has no label yet.", "merge-output/disputed-scenarios.md"],
|
||
[481, "on hold: owner", "life_taken stays 5 until the owner confirms that the unborn child counts.", "HANDOVER-SOFI.md"],
|
||
[458, "on hold: owner", "The Gardanne attack of the same night gets a record only once someone has read the La Provence article.", "HANDOVER-SOFI.md"],
|
||
[184, "live site still names the victim", "The victim asked not to be named. This build withholds the name; the live record still shows it in the victim field and the summary.", "merge-output/names-withheld.json"],
|
||
];
|
||
for (const [id, kind, detail, file] of fixed) if (live(id)) point(id, kind, detail, { evidence_file: file });
|
||
for (const id of [16, 103, 113, 114, 223, 373, 390, 399, 408, 412, 419, 427, 430, 434, 436, 444, 448, 465, 476, 485])
|
||
if (live(id)) point(id, "new contradictions without rulings", "Rewritten from new sources in batch 35; its fresh record-versus-article contradictions have no rulings yet.", { evidence_file: "merge-output/batches/contradictions-35.md" });
|
||
point("", "vocabulary not yet approved", 'Labels and fields added during the pass that the owner has not approved in a list: targeting "visible wealth", the field "who was attacked", and the attack labels as applied to approach-only cases.', { evidence_file: "merge-output/step-labels.json" });
|
||
point("", "sources not read", "65 links could not be read by the pipeline and wait for a manual fetch.", { evidence_file: "merge-output/sofia-worklist.md" });
|
||
point("", "old dashboard not migrated", "The v2 dashboard (index.html, backend_server.py) still reads the old schema and is not part of this release.", { evidence_file: "stats-gart-io-v3/index.html" });
|
||
admin.sort((a, b) => (Number(a.case_id) || 1e9) - (Number(b.case_id) || 1e9) || a.kind.localeCompare(b.kind));
|
||
|
||
// ---- write ---------------------------------------------------------------
|
||
for (const d of ["data", "export"]) mkdirSync(join(OUT, d), { recursive: true });
|
||
const list = [...cases.values()].sort((a, b) => a.id - b.id);
|
||
const meta = { data_date: dataDate, build_date: buildDate, cases: list.length, live_cases_at_data_date: records.length, removed_ids: removed.sort((a, b) => a - b), field_changes_applied: changed };
|
||
writeFileSync(join(OUT, "data/cases.json"), JSON.stringify({ meta, cases: list }, null, 1));
|
||
const groupOf = (step: string, name: string) => (step === "approach" && name.includes(": ") ? name.split(": ")[0] : "");
|
||
const labelRows = STEPS.flatMap((step) => Object.entries(LABELS[step].values as Record<string, string>).map(([name, definition], i) => ({ step, name, group: groupOf(step, name), definition, display_order: i + 1 })));
|
||
writeFileSync(join(OUT, "data/labels.json"), JSON.stringify(labelRows, null, 1));
|
||
{
|
||
const OLD_DATABASE_ONLY = ["structural: applied in this build only", "live site still names the victim", "old dashboard not migrated"];
|
||
const oldOnly = (a: Record<string, string>) => OLD_DATABASE_ONLY.includes(a.kind) || a.detail.startsWith("merging and deleting records is not a field change");
|
||
const settled = new Set<string>([...triage.applied, ...triage.closed].map((t) => t.key));
|
||
const parkedBy = new Map<string, any>(triage.parked.map((t) => [t.key, t]));
|
||
const byPrecedent = new Set<string>((existsSync("closed-by-precedent.json") ? J("closed-by-precedent.json").closed : []).map((c: any) => [c.case_id, c.kind, c.field ?? ""].join("|")));
|
||
const kept = admin.filter((a) => !settled.has(pointKey(a)) && !oldOnly(a) && !byPrecedent.has([a.case_id, a.kind, a.field ?? ""].join("|")));
|
||
for (const a of kept) {
|
||
const p = parkedBy.get(pointKey(a));
|
||
if (p) a.detail = [a.detail, `Night shift: ${p.reason}`, p.options && `Options: ${p.options}`, p.default && `Recommended: ${p.default}`].filter(Boolean).join(" | ");
|
||
}
|
||
console.log(`summary wording fixes: ${summaryFixes}`);
|
||
console.log(`triage: ${admin.length - kept.length} open items settled (${nightDefaults.length} values set as night-shift defaults), ${kept.length} open`);
|
||
admin.length = 0;
|
||
admin.push(...kept);
|
||
}
|
||
writeFileSync(join(OUT, "data/admin-points.json"), JSON.stringify(admin, null, 1));
|
||
writeFileSync(join(OUT, "export", `night-shift-defaults-${buildDate}.json`), JSON.stringify(nightDefaults, null, 1));
|
||
|
||
// a cell that a spreadsheet would run as a formula gets a leading apostrophe
|
||
const safe = (v: unknown) => { const s = v == null ? "" : String(v); return /^[=+\-@]/.test(s) ? "'" + s : s; };
|
||
const csv = (rows: unknown[][]) => "" + rows.map((r) => r.map((v) => csvCell(safe(v))).join(",")).join("\r\n") + "\r\n";
|
||
const amount = (m: any, k: string) => (m?.[k] == null ? "" : m[k]);
|
||
const maxSources = Math.max(...list.map((c) => c.sources.length));
|
||
const head = [
|
||
"id", "date", "date_sort", "date_precision", "victim", "location", "country",
|
||
"1_targeting", "1_targeting_accuracy", "1_who_was_attacked", "2_approach", "3_attack", "4_outcome", "4_obtained",
|
||
"detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken",
|
||
"demanded_amount", "demanded_currency", "demanded_usd_stated", "demanded_as_stated",
|
||
"taken_amount", "taken_currency", "taken_usd_stated", "taken_as_stated",
|
||
"money_wanted", "coin_type", "venue_of_control", "held_duration", "arrests",
|
||
"description", "summary", "notes", "reports",
|
||
"verification", "name_withheld", "linked_cases", "former_ids", "old_labels", "fields_changed_by_ruling", "admin_points",
|
||
...Array.from({ length: maxSources }, (_, i) => `source_${i + 1}`),
|
||
];
|
||
const adminCount = new Map<string, number>();
|
||
for (const a of admin) adminCount.set(a.case_id, (adminCount.get(a.case_id) ?? 0) + 1);
|
||
const rows = list.map((c) => [
|
||
c.id, c.date, c.date_sort ?? "", c.date_precision, c.victim, c.location, c.country,
|
||
c.steps.targeting.join(" | "), c.steps.targeting_accuracy, c.steps.who_was_attacked.join(" | "), c.steps.approach.join(" | "), c.steps.attack.join(" | "), c.steps.outcome, c.steps.obtained.join(" | "),
|
||
c.flags.detention, c.flags.violence_torture, c.flags.drugs_alcohol, c.flags.weapons, c.flags.theft, c.life_taken_unconfirmed ? "unconfirmed" : (c.life_taken ?? ""),
|
||
amount(c.demanded, "amount"), amount(c.demanded, "currency"), amount(c.demanded, "usd_stated"), amount(c.demanded, "as_stated"),
|
||
amount(c.taken, "amount"), amount(c.taken, "currency"), amount(c.taken, "usd_stated"), amount(c.taken, "as_stated"),
|
||
c.money_wanted, c.coin_type, c.venue_of_control, c.held_duration, c.arrests,
|
||
c.description, c.summary, c.notes, c.reports,
|
||
c.verification, c.name_withheld ? "yes" : "", c.linked_cases.join(" | "), c.former_ids.join(" | "), c.old_labels.join(" > "),
|
||
c.internal.field_changes.map((f: any) => f.field).join(" | "), adminCount.get(String(c.id)) ?? "",
|
||
...Array.from({ length: maxSources }, (_, i) => c.sources[i] ?? ""),
|
||
]);
|
||
writeFileSync(join(OUT, `export/gart-kr-database-${buildDate}.csv`), csv([head, ...rows]));
|
||
// every open item carries the case it belongs to: who, where, when, the four steps and what happened
|
||
const section = (summary: string, name: string) => new RegExp(`\\*\\*${name}[^*]*:\\*\\*\\s*([\\s\\S]*?)(?=\\s*\\*\\*[^*]+:\\*\\*|$)`).exec(summary)?.[1].replace(/\s+/g, " ").trim() ?? "";
|
||
const shorten = (text: string, limit: number) => {
|
||
if (text.length <= limit) return text;
|
||
const head = text.slice(0, limit), stop = Math.max(head.lastIndexOf(". "), head.lastIndexOf("; "));
|
||
return (stop > limit * 0.5 ? head.slice(0, stop + 1) : head.trimEnd() + " ...").trim();
|
||
};
|
||
const removedRecord = new Map(records.filter((r) => removed.includes(Number(r.id))).map((r) => [r.id, r]));
|
||
const context = (id: string) => {
|
||
const c = cases.get(Number(id));
|
||
if (c) return { victim: c.victim, date: c.date, location: c.location,
|
||
steps: `targeting: ${c.steps.targeting.join(", ")} | approach: ${c.steps.approach.join(", ")} | attack: ${c.steps.attack.join(", ")} | outcome: ${c.steps.outcome}`,
|
||
what_happened: shorten(section(c.summary, "Attack Method") || c.description, 420), how_it_ended: shorten(section(c.summary, "Status"), 260) };
|
||
const r = removedRecord.get(id);
|
||
return { victim: r?.victim ?? "", date: r?.date ?? "", location: r?.location ?? "", steps: "", what_happened: r ? shorten(section(r.summary, "Attack Method") || r.description, 420) : "", how_it_ended: r ? shorten(section(r.summary, "Status"), 260) : "" };
|
||
};
|
||
const CONTEXT = ["victim", "date", "location", "steps", "what_happened", "how_it_ended"] as const;
|
||
for (const a of admin) Object.assign(a, a.case_id ? context(a.case_id) : Object.fromEntries(CONTEXT.map((k) => [k, ""])));
|
||
const adminHead = ["case_id", "victim", "date", "location", "kind", "field", "detail", "first_value", "second_value", "quote", "steps", "what_happened", "how_it_ended", "evidence_file"];
|
||
writeFileSync(join(OUT, `export/admin-decisions-${buildDate}.csv`), csv([adminHead, ...admin.map((a) => adminHead.map((h) => a[h]))]));
|
||
|
||
// the same list case by case: the case first, then what is open on it
|
||
const byCase = new Map<string, Record<string, string>[]>();
|
||
for (const a of admin) byCase.set(a.case_id, [...(byCase.get(a.case_id) ?? []), a]);
|
||
const line = (a: Record<string, string>) => `- **${a.kind}**${a.field ? ` (${a.field})` : ""}: ${a.detail}${a.first_value ? ` In the database: ${a.first_value}.` : ""}${a.second_value ? ` Other reading or proposal: ${a.second_value}.` : ""}${a.quote ? ` Text: "${a.quote.replace(/"/g, "'")}"` : ""}`;
|
||
const caseMd = [
|
||
`# Open items, case by case (${buildDate})`,
|
||
"",
|
||
`${admin.length} open items on ${[...byCase.keys()].filter(Boolean).length} cases. An open item is one thing in one case that is unconfirmed, disputed or waiting for a decision. Each case below shows who, where and when, the four steps as they stand in the database, what happened, and then its open items.`,
|
||
"",
|
||
...[...byCase.entries()].filter(([id]) => id).flatMap(([id, items]) => {
|
||
const k = items[0];
|
||
return [
|
||
`## Case ${id}: ${k.victim || "(no victim field)"}`,
|
||
"",
|
||
`${[k.location, k.date].filter(Boolean).join(", ")}`,
|
||
"",
|
||
...(k.steps ? [`**Steps:** ${k.steps}`, ""] : []),
|
||
...(k.what_happened ? [`**What happened:** ${k.what_happened}`, ""] : []),
|
||
...(k.how_it_ended ? [`**How it ended:** ${k.how_it_ended}`, ""] : []),
|
||
`**Open (${items.length}):**`,
|
||
"",
|
||
...items.map(line),
|
||
"",
|
||
];
|
||
}),
|
||
"## General",
|
||
"",
|
||
...(byCase.get("") ?? []).map(line),
|
||
].join("\n");
|
||
writeFileSync(join(OUT, `export/admin-decisions-${buildDate}-by-case.md`), caseMd + "\n");
|
||
|
||
const byKind = new Map<string, Record<string, string>[]>();
|
||
for (const a of admin) byKind.set(a.kind, [...(byKind.get(a.kind) ?? []), a]);
|
||
const kinds = [...byKind.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
||
const md = [
|
||
`# Admin decisions and open discrepancies (${buildDate})`,
|
||
"",
|
||
`Built from the live pull of ${dataDate}. ${admin.length} points on ${new Set(admin.map((a) => a.case_id).filter(Boolean)).size} cases. The same rows, sortable, are in \`admin-decisions-${buildDate}.csv\`.`,
|
||
"",
|
||
"| Kind | Points |",
|
||
"|---|---|",
|
||
...kinds.map(([k, v]) => `| ${k} | ${v.length} |`),
|
||
"",
|
||
...kinds.flatMap(([k, v]) => [
|
||
`## ${k} (${v.length})`,
|
||
"",
|
||
...(v.length > 40 && new Set(v.map((a) => a.detail)).size === 1
|
||
? [v[0].detail, "", `Cases: ${v.map((a) => a.case_id).join(", ")}`, ""]
|
||
: v.map((a) => `- **${a.case_id ? `Case ${a.case_id}, ${a.victim} (${[a.location, a.date].filter(Boolean).join(", ")})` : "General"}**${a.field ? ` (${a.field})` : ""}: ${a.detail}${a.first_value ? ` In the database: ${a.first_value}.` : ""}${a.second_value ? ` Proposed: ${a.second_value}.` : ""}${a.quote ? ` Text: "${a.quote.replace(/"/g, "'")}"` : ""}`).concat("")),
|
||
]),
|
||
].join("\n");
|
||
writeFileSync(join(OUT, `export/admin-decisions-${buildDate}.md`), md + "\n");
|
||
|
||
console.log(`${OUT}/data/cases.json: ${list.length} cases (${records.length} in the live pull, removed ${removed.join(", ")})`);
|
||
console.log(`recheck changes applied: ${rechecked}; label-check changes applied: ${labelsChanged}; flag-check changes applied: ${flagsChanged}`);
|
||
console.log(`field changes applied: ${changed}; admin points: ${admin.length} in ${kinds.length} kinds`);
|
||
for (const [k, v] of kinds) console.log(` ${String(v.length).padStart(4)} ${k}`);
|