chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+131
View File
@@ -0,0 +1,131 @@
// Mechanical checks of the readers' files work/out/<key>.json.
// bun validate.ts [batch-NN ...] (no argument: every out file)
// Prints one line per problem and writes work/validation.json. A problem is something a rule forbids,
// never a judgement call: those are the second reader's job.
import { Database } from "bun:sqlite";
import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
import { join } from "path";
import { loadCase, pageParas, parseKeep } from "./lib.ts";
const HERE = import.meta.dir;
const S2 = join(HERE, "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2");
const { dateColumns } = await import(join(S2, "dates.ts"));
const db = new Database(join(S2, "gart_kr.db"), { readonly: true });
const LABELS = new Map<string, Set<string>>();
for (const r of db.query("SELECT step, name FROM labels").all() as any[]) {
if (!LABELS.has(r.step)) LABELS.set(r.step, new Set());
LABELS.get(r.step)!.add(r.name);
}
// decided by the owner on 2026-10-17 and not yet in the database copy: accepted until the labels table has it
LABELS.get("obtained")?.add("Unknown");
const STEPS_LIST = ["targeting", "who_was_attacked", "approach", "attack", "obtained"];
const STEPS_ONE = ["targeting_accuracy", "outcome"];
// owner, 2026-10-16: empty = unknown (the information is not available), never written as no
const FLAG = new Set(["yes", "no", "unconfirmed", ""]);
const VENUE = new Set(["victim's private space", "victim's workplace", "attacker's or third-party space", "vehicle in motion", "public space", "none", ""]);
const HELD = new Set(["none", "minutes", "hours", "days", ""]);
const ARRESTS = new Set(["none", "some", "all", "convicted", ""]);
const STATUS = new Set(["new", "duplicate", "out of scope", "no source"]);
const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”«»„]/g, '"').replace(/\s+/g, " ").trim();
const args = process.argv.slice(2);
const keys = args.length
? args.flatMap((b) => JSON.parse(readFileSync(join(HERE, "batches", `${b.replace(/\.json$/, "")}.json`), "utf8")))
: readdirSync(join(HERE, "work/out")).filter((f) => f.endsWith(".json")).map((f) => f.replace(/\.json$/, ""));
const report: Record<string, string[]> = {};
for (const key of keys) {
const p: string[] = (report[key] = []);
const file = join(HERE, "work/out", `${key}.json`);
if (!existsSync(file)) { p.push("no out file"); continue; }
let o: any;
try { o = loadCase(key); } catch (e: any) { p.push(`not JSON: ${e.message}`); continue; }
if (o.key !== key) p.push(`key is ${o.key}`);
if (!STATUS.has(o.status)) p.push(`status "${o.status}"`);
// kept text of the case
const kept: string[] = [];
for (const pg of o.pages ?? []) {
if (!existsSync(join(HERE, pg.file))) { p.push(`page file missing: ${pg.file}`); continue; }
const paras = pageParas(pg.file);
const nums = parseKeep(String(pg.keep ?? ""));
if (nums === null) { p.push(`${pg.file}: keep "${pg.keep}" is not a paragraph list`); continue; }
// the headline counts as kept text only on a page whose article is kept
const title = nums && nums.length ? /^Title {6}(.*)$/m.exec(readFileSync(join(HERE, pg.file), "utf8"))?.[1] ?? "" : "";
if (title) kept.push(title);
for (const n of nums) {
if (!paras.has(n)) p.push(`${pg.file}: paragraph ${n} does not exist`);
else kept.push(paras.get(n)!);
}
if (!nums.length && !pg.reason) p.push(`${pg.file}: nothing kept and no reason`);
}
// compared without whitespace: PDF text layers break lines inside sentences
const keptText = norm(kept.join(" ")).replace(/\s+/g, "");
if (o.status === "duplicate") {
if (o.duplicate_of == null) p.push("duplicate without duplicate_of");
if (typeof o.duplicate_of === "number" && !db.query("SELECT 1 FROM cases WHERE id = ?").get(o.duplicate_of)) p.push(`duplicate_of ${o.duplicate_of} is no stats2 case`);
if (o.values && Object.keys(o.values).length && typeof o.duplicate_of === "number") p.push("values written for a duplicate of a stats2 case");
}
const v = o.values && Object.keys(o.values).length ? o.values : null;
if (!v) { if (o.status !== "duplicate") p.push("no values"); continue; }
if (String(v.date ?? "").trim() && !dateColumns(v.date).date_sort) p.push(`date "${v.date}" is not a stats2 date form`);
for (const f of ["victim", "description", "summary", "case_details"]) if (!String(v[f] ?? "").trim()) p.push(`${f} empty`);
for (const f of ["description", "summary", "victim", "location"]) if (/[—–]/.test(v[f] ?? "")) p.push(`${f} has an em or en dash`);
const sl = String(v.summary ?? "").length;
if (sl > 900) p.push(`summary is ${sl} characters (limit 900)`);
if (sl && sl < 600 && o.status === "new") p.push(`summary is ${sl} characters (floor 600; fine when the source is thin)`);
if (/[“”‘’]/.test(v.summary ?? "")) p.push("summary has curly quotes");
if (v.case_details && !/\*\*Victims?:\*\*[\s\S]*\*\*Attackers?:\*\*[\s\S]*\*\*Attack Method:\*\*[\s\S]*\*\*Violence Used:\*\*[\s\S]*\*\*Crypto Demanded\/Stolen:\*\*[\s\S]*\*\*Status:\*\*/.test(v.case_details)) p.push("case_details lacks the six bold labels in order");
for (const f of ["detention", "drugs_alcohol", "weapons", "theft"]) if (!FLAG.has(v[f])) p.push(`${f} "${v[f]}"`);
if (!FLAG.has(v.violence_torture) && v.violence_torture !== "threat only") p.push(`violence_torture "${v.violence_torture}"`);
if (!/^(\d+|unconfirmed|)$/.test(String(v.life_taken ?? ""))) p.push(`life_taken "${v.life_taken}"`);
if (!VENUE.has(v.venue_of_control ?? "")) p.push(`venue_of_control "${v.venue_of_control}"`);
if (!HELD.has(v.held_duration ?? "")) p.push(`held_duration "${v.held_duration}"`);
if (!ARRESTS.has(v.arrests ?? "")) p.push(`arrests "${v.arrests}"`);
for (const k of ["demanded", "taken"]) {
for (const f of ["amount", "usd_stated"]) if (v[`${k}_${f}`] != null && typeof v[`${k}_${f}`] !== "number") p.push(`${k}_${f} is not a number`);
if (v[`${k}_amount`] != null && !v[`${k}_currency`]) p.push(`${k}_amount without currency`);
if (!String(v[`${k}_as_stated`] ?? "").trim()) p.push(`${k}_as_stated empty`);
}
for (const s of STEPS_LIST) {
if (!Array.isArray(v[s]) || !v[s].length) { p.push(`${s} is not a non-empty list`); continue; }
for (const l of v[s]) if (!LABELS.get(s)?.has(l)) p.push(`${s} label "${l}" is not in labels`);
if (v[s].length > 1 && v[s].includes("Unknown")) p.push(`${s}: Unknown beside another label`);
if (s === "obtained" && v[s].length > 1 && ["Nothing", "Unconfirmed", "Unknown"].some((l) => v[s].includes(l))) p.push("obtained: Nothing, Unconfirmed or Unknown beside another label");
}
for (const s of STEPS_ONE) if (!LABELS.get(s)?.has(v[s])) p.push(`${s} "${v[s]}" is not in labels`);
if (v.theft === "yes" && (v.obtained ?? []).includes("Nothing")) p.push("theft yes but obtained Nothing");
if (v.theft === "no" && (v.obtained ?? []).some((l: string) => l !== "Nothing")) p.push("theft no but obtained names something");
if ((v.obtained ?? []).includes("Unknown") && (v.theft ?? "") !== "") p.push(`obtained Unknown but theft "${v.theft}"`);
if ((v.theft ?? "") === "" && !(v.obtained ?? []).includes("Unknown")) p.push("theft empty (unknown) but obtained is not Unknown");
// people agree with who_was_attacked
const roles = new Set((v.people ?? []).map((r: any) => r.role));
for (const r of v.people ?? []) {
if (!LABELS.get("who_was_attacked")?.has(r.role) || r.role === "Business premises or machine") p.push(`people role "${r.role}"`);
if (!Number.isInteger(r.people) || r.people < 1) p.push(`people count "${r.people}" for ${r.role}`);
if (![0, 1].includes(r.at_least)) p.push(`at_least "${r.at_least}" for ${r.role}`);
if (!(v.who_was_attacked ?? []).includes(r.role)) p.push(`people row ${r.role} without the label`);
}
for (const l of v.who_was_attacked ?? []) if (l !== "Business premises or machine" && !roles.has(l)) p.push(`who_was_attacked ${l} without a people row`);
if (!Array.isArray(v.sources) || !v.sources.length) p.push("no sources");
// support passages must be in the kept text (pages fetched by WebFetch count as kept text too)
if (o.status !== "no source") {
// an empty flag is unknown and has no passage; neither has obtained Unknown
const need = [...(String(v.date ?? "").trim() ? ["date"] : []), ...["detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"].filter((f) => String(v[f] ?? "") !== "")];
for (const s of [...STEPS_LIST, ...STEPS_ONE]) for (const l of [].concat(v[s] ?? [])) if (!(s === "obtained" && l === "Unknown")) need.push(`${s}: ${l}`);
for (const f of ["venue_of_control", "held_duration", "arrests"]) if (String(v[f] ?? "").trim()) need.push(f);
for (const f of ["demanded_amount", "taken_amount"]) if (v[f] != null) need.push(f);
for (const r of v.people ?? []) need.push(`people: ${r.role}`);
const has = (f: string) => o.support?.[f] || o.support?.[f.split(":")[0]] || o.support?.[f.replace(/_amount$/, "")] || o.support?.[f.replace(/_amount$/, "_as_stated")];
for (const f of need) if (!has(f)) p.push(`no support for ${f}`);
for (const [f, q] of Object.entries(o.support ?? {})) {
const quote = norm(q);
// 8 characters, or 3 in Chinese, Japanese or Korean script, where a few characters carry a phrase
if (quote.length < (/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(quote) ? 3 : 8)) p.push(`support ${f} too short`);
else if (!keptText.includes(quote.replace(/\s+/g, ""))) p.push(`support ${f} not found in the kept paragraphs: "${quote.slice(0, 80)}"`);
}
}
}
writeFileSync(join(HERE, "work/validation.json"), JSON.stringify(report, null, 1));
let n = 0;
for (const [k, ps] of Object.entries(report)) for (const x of ps) (n++, console.log(`${k} ${x}`));
console.log(`${keys.length} keys checked, ${n} problems, ${Object.values(report).filter((x) => !x.length).length} clean`);