This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/lopp-pr234/prep.ts
T
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

84 lines
5.4 KiB
TypeScript

// Writes the readers' inputs from work/entries.json and work/fetched.json:
// work/articles/<key>-<n>.txt one fetched page, paragraphs numbered [1], [2], ... (text as extracted, unchanged)
// work/in/<key>.json the Lopp entry, the fetched pages, the stats2 cases it might duplicate
// batches/batch-NN.json the keys of each reader batch
import { Database } from "bun:sqlite";
import { mkdirSync, readFileSync, writeFileSync } from "fs";
import { join } from "path";
const HERE = import.meta.dir;
const S2 = join(HERE, "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2");
const { dateColumns } = await import(join(S2, "dates.ts"));
const BATCH = 10;
const entries: any[] = JSON.parse(readFileSync(join(HERE, "work/entries.json"), "utf8"));
const fetched: Record<string, any[]> = JSON.parse(readFileSync(join(HERE, "work/fetched.json"), "utf8"));
const db = new Database(join(S2, "gart_kr.db"), { readonly: true });
const cases = db.query("SELECT id, date, victim, location, country, description FROM cases").all() as any[];
const srcs = db.query("SELECT case_id, url FROM case_sources").all() as any[];
for (const d of ["articles", "in"]) mkdirSync(join(HERE, "work", d), { recursive: true });
// Lopp writes "September 24 2018" and "July 2021"; stats2 writes "September 24, 2018" and "July, 2021"
export const toS2Date = (raw: string) => raw.trim().replace(/^(\w+) (\d{1,2}) (\d{4})$/, "$1 $2, $3").replace(/^(\w+) (\d{4})$/, "$1, $2");
const country = (s: string) => {
const c = String(s ?? "").trim().toLowerCase();
return ({ usa: "united states", us: "united states", uk: "united kingdom", "england": "united kingdom" } as Record<string, string>)[c] ?? c;
};
const days = (a: string, b: string) => Math.abs(+new Date(a) - +new Date(b)) / 864e5;
const normUrl = (u: string) => u.trim().replace(/^https?:\/\/(www\.)?/, "").replace(/[?#].*$/, "").replace(/\/+$/, "").toLowerCase();
const urlCase = new Map(srcs.map((r) => [normUrl(r.url), r.case_id]));
for (const e of entries) {
e.s2_date = toS2Date(e.date);
e.dc = dateColumns(e.s2_date);
e.country_en = (e.location.split(",").pop() ?? "").trim();
}
// stats2 cases (and other new entries) in the same country within 120 days, or the same year when either date is a year only
function near(e: any) {
const lim = (a: any, b: any) => (a.date_precision === "year" || b.date_precision === "year" ? 366 : 120);
const out: any[] = [];
for (const c of cases) {
const cd = dateColumns(c.date);
if (!cd.date_sort || !e.dc.date_sort || country(c.country) !== country(e.country_en)) continue;
const dd = days(cd.date_sort, e.dc.date_sort);
if (dd <= lim(cd, e.dc)) out.push({ stats2_id: c.id, days_apart: Math.round(dd), date: c.date, victim: c.victim, location: c.location, description: c.description });
}
const urlHits = [...new Set(e.links.map((l: any) => urlCase.get(normUrl(l.url))).filter(Boolean))];
for (const id of urlHits) if (!out.some((o) => o.stats2_id === id)) {
const c = cases.find((x) => x.id === id);
out.push({ stats2_id: id, days_apart: null, date: c.date, victim: c.victim, location: c.location, description: c.description });
}
const sib = entries.filter((o) => o !== e && o.dc.date_sort && e.dc.date_sort && country(o.country_en) === country(e.country_en) && days(o.dc.date_sort, e.dc.date_sort) <= 30)
.map((o) => ({ key: o.key, date: o.date, victim: o.victim, location: o.location, lopp_label: o.lopp_label }));
out.sort((a, b) => (a.days_apart ?? -1) - (b.days_apart ?? -1));
return { urlHits, candidates: out.slice(0, 12), siblings: sib.slice(0, 6) };
}
for (const e of entries) {
const pages = (fetched[e.key] ?? []).map((g: any, i: number) => {
const file = `work/articles/${e.key}-${i + 1}.txt`;
const head = [`URL ${g.url}`, `Via ${g.via} (status ${g.status})`, `Retrieved ${g.retrieved}`, `Title ${g.title}`, `Published ${g.published || "(not given)"}`, `Byline ${g.byline || "(not given)"}`, `Length ${g.chars} chars, ${g.paras.length} paragraphs`, ""];
writeFileSync(join(HERE, file), head.join("\n") + "\n" + g.paras.map((p: string, j: number) => `[${j + 1}] ${p}`).join("\n\n") + "\n");
return { file, url: g.url, via: g.via, status: g.status, chars: g.chars, paragraphs: g.paras.length, title: g.title };
});
const n = near(e);
writeFileSync(join(HERE, "work/in", `${e.key}.json`), JSON.stringify({
key: e.key,
lopp: { date: e.date, date_as_stats2: e.s2_date, victim: e.victim, location: e.location, summary_line: e.lopp_label, links: e.links },
pages,
stats2_url_match: n.urlHits,
stats2_candidates: n.candidates,
new_entry_siblings: n.siblings,
}, null, 1));
}
const keys = entries.map((e) => e.key);
for (let i = 0; i < keys.length; i += BATCH) {
const nn = String(i / BATCH + 1).padStart(2, "0");
writeFileSync(join(HERE, "batches", `batch-${nn}.json`), JSON.stringify(keys.slice(i, i + BATCH)));
}
const big = entries.flatMap((e) => (fetched[e.key] ?? []).filter((g: any) => g.chars > 40000).map((g: any) => `${e.key} ${g.chars}`));
console.log(`entries ${entries.length}, batches ${Math.ceil(keys.length / BATCH)}, pages ${Object.values(fetched).flat().length}, pages over 40k chars: ${big.length} (${big.join(", ")})`);
console.log(`with stats2 URL match: ${entries.filter((e) => near(e).urlHits.length).length}; with any stats2 candidate: ${entries.filter((e) => near(e).candidates.length).length}`);