// Mechanical checks of the readers' files work/out/.json. // bun validate.ts [batch-NN ...] (no argument: every out file) // Prints one line per problem and writes work/validation.json. A problem is something a rule forbids, // never a judgement call: those are the second reader's job. import { Database } from "bun:sqlite"; import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs"; import { join } from "path"; import { loadCase, pageParas, parseKeep } from "./lib.ts"; const HERE = import.meta.dir; const S2 = join(HERE, "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2"); const { dateColumns } = await import(join(S2, "dates.ts")); const db = new Database(join(S2, "gart_kr.db"), { readonly: true }); const LABELS = new Map>(); for (const r of db.query("SELECT step, name FROM labels").all() as any[]) { if (!LABELS.has(r.step)) LABELS.set(r.step, new Set()); LABELS.get(r.step)!.add(r.name); } // decided by the owner on 2026-10-17 and not yet in the database copy: accepted until the labels table has it LABELS.get("obtained")?.add("Unknown"); const STEPS_LIST = ["targeting", "who_was_attacked", "approach", "attack", "obtained"]; const STEPS_ONE = ["targeting_accuracy", "outcome"]; // owner, 2026-10-16: empty = unknown (the information is not available), never written as no const FLAG = new Set(["yes", "no", "unconfirmed", ""]); const VENUE = new Set(["victim's private space", "victim's workplace", "attacker's or third-party space", "vehicle in motion", "public space", "none", ""]); const HELD = new Set(["none", "minutes", "hours", "days", ""]); const ARRESTS = new Set(["none", "some", "all", "convicted", ""]); const STATUS = new Set(["new", "duplicate", "out of scope", "no source"]); const norm = (s: unknown) => String(s ?? "").replace(/[‘’]/g, "'").replace(/[“”«»„]/g, '"').replace(/\s+/g, " ").trim(); const args = process.argv.slice(2); const keys = args.length ? args.flatMap((b) => JSON.parse(readFileSync(join(HERE, "batches", `${b.replace(/\.json$/, "")}.json`), "utf8"))) : readdirSync(join(HERE, "work/out")).filter((f) => f.endsWith(".json")).map((f) => f.replace(/\.json$/, "")); const report: Record = {}; for (const key of keys) { const p: string[] = (report[key] = []); const file = join(HERE, "work/out", `${key}.json`); if (!existsSync(file)) { p.push("no out file"); continue; } let o: any; try { o = loadCase(key); } catch (e: any) { p.push(`not JSON: ${e.message}`); continue; } if (o.key !== key) p.push(`key is ${o.key}`); if (!STATUS.has(o.status)) p.push(`status "${o.status}"`); // kept text of the case const kept: string[] = []; for (const pg of o.pages ?? []) { if (!existsSync(join(HERE, pg.file))) { p.push(`page file missing: ${pg.file}`); continue; } const paras = pageParas(pg.file); const nums = parseKeep(String(pg.keep ?? "")); if (nums === null) { p.push(`${pg.file}: keep "${pg.keep}" is not a paragraph list`); continue; } // the headline counts as kept text only on a page whose article is kept const title = nums && nums.length ? /^Title {6}(.*)$/m.exec(readFileSync(join(HERE, pg.file), "utf8"))?.[1] ?? "" : ""; if (title) kept.push(title); for (const n of nums) { if (!paras.has(n)) p.push(`${pg.file}: paragraph ${n} does not exist`); else kept.push(paras.get(n)!); } if (!nums.length && !pg.reason) p.push(`${pg.file}: nothing kept and no reason`); } // compared without whitespace: PDF text layers break lines inside sentences const keptText = norm(kept.join(" ")).replace(/\s+/g, ""); if (o.status === "duplicate") { if (o.duplicate_of == null) p.push("duplicate without duplicate_of"); if (typeof o.duplicate_of === "number" && !db.query("SELECT 1 FROM cases WHERE id = ?").get(o.duplicate_of)) p.push(`duplicate_of ${o.duplicate_of} is no stats2 case`); if (o.values && Object.keys(o.values).length && typeof o.duplicate_of === "number") p.push("values written for a duplicate of a stats2 case"); } const v = o.values && Object.keys(o.values).length ? o.values : null; if (!v) { if (o.status !== "duplicate") p.push("no values"); continue; } if (String(v.date ?? "").trim() && !dateColumns(v.date).date_sort) p.push(`date "${v.date}" is not a stats2 date form`); for (const f of ["victim", "description", "summary", "case_details"]) if (!String(v[f] ?? "").trim()) p.push(`${f} empty`); for (const f of ["description", "summary", "victim", "location"]) if (/[—–]/.test(v[f] ?? "")) p.push(`${f} has an em or en dash`); const sl = String(v.summary ?? "").length; if (sl > 900) p.push(`summary is ${sl} characters (limit 900)`); if (sl && sl < 600 && o.status === "new") p.push(`summary is ${sl} characters (floor 600; fine when the source is thin)`); if (/[“”‘’]/.test(v.summary ?? "")) p.push("summary has curly quotes"); if (v.case_details && !/\*\*Victims?:\*\*[\s\S]*\*\*Attackers?:\*\*[\s\S]*\*\*Attack Method:\*\*[\s\S]*\*\*Violence Used:\*\*[\s\S]*\*\*Crypto Demanded\/Stolen:\*\*[\s\S]*\*\*Status:\*\*/.test(v.case_details)) p.push("case_details lacks the six bold labels in order"); for (const f of ["detention", "drugs_alcohol", "weapons", "theft"]) if (!FLAG.has(v[f])) p.push(`${f} "${v[f]}"`); if (!FLAG.has(v.violence_torture) && v.violence_torture !== "threat only") p.push(`violence_torture "${v.violence_torture}"`); if (!/^(\d+|unconfirmed|)$/.test(String(v.life_taken ?? ""))) p.push(`life_taken "${v.life_taken}"`); if (!VENUE.has(v.venue_of_control ?? "")) p.push(`venue_of_control "${v.venue_of_control}"`); if (!HELD.has(v.held_duration ?? "")) p.push(`held_duration "${v.held_duration}"`); if (!ARRESTS.has(v.arrests ?? "")) p.push(`arrests "${v.arrests}"`); for (const k of ["demanded", "taken"]) { for (const f of ["amount", "usd_stated"]) if (v[`${k}_${f}`] != null && typeof v[`${k}_${f}`] !== "number") p.push(`${k}_${f} is not a number`); if (v[`${k}_amount`] != null && !v[`${k}_currency`]) p.push(`${k}_amount without currency`); if (!String(v[`${k}_as_stated`] ?? "").trim()) p.push(`${k}_as_stated empty`); } for (const s of STEPS_LIST) { if (!Array.isArray(v[s]) || !v[s].length) { p.push(`${s} is not a non-empty list`); continue; } for (const l of v[s]) if (!LABELS.get(s)?.has(l)) p.push(`${s} label "${l}" is not in labels`); if (v[s].length > 1 && v[s].includes("Unknown")) p.push(`${s}: Unknown beside another label`); if (s === "obtained" && v[s].length > 1 && ["Nothing", "Unconfirmed", "Unknown"].some((l) => v[s].includes(l))) p.push("obtained: Nothing, Unconfirmed or Unknown beside another label"); } for (const s of STEPS_ONE) if (!LABELS.get(s)?.has(v[s])) p.push(`${s} "${v[s]}" is not in labels`); if (v.theft === "yes" && (v.obtained ?? []).includes("Nothing")) p.push("theft yes but obtained Nothing"); if (v.theft === "no" && (v.obtained ?? []).some((l: string) => l !== "Nothing")) p.push("theft no but obtained names something"); if ((v.obtained ?? []).includes("Unknown") && (v.theft ?? "") !== "") p.push(`obtained Unknown but theft "${v.theft}"`); if ((v.theft ?? "") === "" && !(v.obtained ?? []).includes("Unknown")) p.push("theft empty (unknown) but obtained is not Unknown"); // people agree with who_was_attacked const roles = new Set((v.people ?? []).map((r: any) => r.role)); for (const r of v.people ?? []) { if (!LABELS.get("who_was_attacked")?.has(r.role) || r.role === "Business premises or machine") p.push(`people role "${r.role}"`); if (!Number.isInteger(r.people) || r.people < 1) p.push(`people count "${r.people}" for ${r.role}`); if (![0, 1].includes(r.at_least)) p.push(`at_least "${r.at_least}" for ${r.role}`); if (!(v.who_was_attacked ?? []).includes(r.role)) p.push(`people row ${r.role} without the label`); } for (const l of v.who_was_attacked ?? []) if (l !== "Business premises or machine" && !roles.has(l)) p.push(`who_was_attacked ${l} without a people row`); if (!Array.isArray(v.sources) || !v.sources.length) p.push("no sources"); // support passages must be in the kept text (pages fetched by WebFetch count as kept text too) if (o.status !== "no source") { // an empty flag is unknown and has no passage; neither has obtained Unknown const need = [...(String(v.date ?? "").trim() ? ["date"] : []), ...["detention", "violence_torture", "drugs_alcohol", "weapons", "theft", "life_taken"].filter((f) => String(v[f] ?? "") !== "")]; for (const s of [...STEPS_LIST, ...STEPS_ONE]) for (const l of [].concat(v[s] ?? [])) if (!(s === "obtained" && l === "Unknown")) need.push(`${s}: ${l}`); for (const f of ["venue_of_control", "held_duration", "arrests"]) if (String(v[f] ?? "").trim()) need.push(f); for (const f of ["demanded_amount", "taken_amount"]) if (v[f] != null) need.push(f); for (const r of v.people ?? []) need.push(`people: ${r.role}`); const has = (f: string) => o.support?.[f] || o.support?.[f.split(":")[0]] || o.support?.[f.replace(/_amount$/, "")] || o.support?.[f.replace(/_amount$/, "_as_stated")]; for (const f of need) if (!has(f)) p.push(`no support for ${f}`); for (const [f, q] of Object.entries(o.support ?? {})) { const quote = norm(q); // 8 characters, or 3 in Chinese, Japanese or Korean script, where a few characters carry a phrase if (quote.length < (/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(quote) ? 3 : 8)) p.push(`support ${f} too short`); else if (!keptText.includes(quote.replace(/\s+/g, ""))) p.push(`support ${f} not found in the kept paragraphs: "${quote.slice(0, 80)}"`); } } } writeFileSync(join(HERE, "work/validation.json"), JSON.stringify(report, null, 1)); let n = 0; for (const [k, ps] of Object.entries(report)) for (const x of ps) (n++, console.log(`${k} ${x}`)); console.log(`${keys.length} keys checked, ${n} problems, ${Object.values(report).filter((x) => !x.length).length} clean`);