// Writes the readers' inputs from work/entries.json and work/fetched.json: // work/articles/-.txt one fetched page, paragraphs numbered [1], [2], ... (text as extracted, unchanged) // work/in/.json the Lopp entry, the fetched pages, the stats2 cases it might duplicate // batches/batch-NN.json the keys of each reader batch import { Database } from "bun:sqlite"; import { mkdirSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; const HERE = import.meta.dir; const S2 = join(HERE, "../../../../05-strategy-marketing-sales/Marketing/gart-io/databases/stats2"); const { dateColumns } = await import(join(S2, "dates.ts")); const BATCH = 10; const entries: any[] = JSON.parse(readFileSync(join(HERE, "work/entries.json"), "utf8")); const fetched: Record = JSON.parse(readFileSync(join(HERE, "work/fetched.json"), "utf8")); const db = new Database(join(S2, "gart_kr.db"), { readonly: true }); const cases = db.query("SELECT id, date, victim, location, country, description FROM cases").all() as any[]; const srcs = db.query("SELECT case_id, url FROM case_sources").all() as any[]; for (const d of ["articles", "in"]) mkdirSync(join(HERE, "work", d), { recursive: true }); // Lopp writes "September 24 2018" and "July 2021"; stats2 writes "September 24, 2018" and "July, 2021" export const toS2Date = (raw: string) => raw.trim().replace(/^(\w+) (\d{1,2}) (\d{4})$/, "$1 $2, $3").replace(/^(\w+) (\d{4})$/, "$1, $2"); const country = (s: string) => { const c = String(s ?? "").trim().toLowerCase(); return ({ usa: "united states", us: "united states", uk: "united kingdom", "england": "united kingdom" } as Record)[c] ?? c; }; const days = (a: string, b: string) => Math.abs(+new Date(a) - +new Date(b)) / 864e5; const normUrl = (u: string) => u.trim().replace(/^https?:\/\/(www\.)?/, "").replace(/[?#].*$/, "").replace(/\/+$/, "").toLowerCase(); const urlCase = new Map(srcs.map((r) => [normUrl(r.url), r.case_id])); for (const e of entries) { e.s2_date = toS2Date(e.date); e.dc = dateColumns(e.s2_date); e.country_en = (e.location.split(",").pop() ?? "").trim(); } // stats2 cases (and other new entries) in the same country within 120 days, or the same year when either date is a year only function near(e: any) { const lim = (a: any, b: any) => (a.date_precision === "year" || b.date_precision === "year" ? 366 : 120); const out: any[] = []; for (const c of cases) { const cd = dateColumns(c.date); if (!cd.date_sort || !e.dc.date_sort || country(c.country) !== country(e.country_en)) continue; const dd = days(cd.date_sort, e.dc.date_sort); if (dd <= lim(cd, e.dc)) out.push({ stats2_id: c.id, days_apart: Math.round(dd), date: c.date, victim: c.victim, location: c.location, description: c.description }); } const urlHits = [...new Set(e.links.map((l: any) => urlCase.get(normUrl(l.url))).filter(Boolean))]; for (const id of urlHits) if (!out.some((o) => o.stats2_id === id)) { const c = cases.find((x) => x.id === id); out.push({ stats2_id: id, days_apart: null, date: c.date, victim: c.victim, location: c.location, description: c.description }); } const sib = entries.filter((o) => o !== e && o.dc.date_sort && e.dc.date_sort && country(o.country_en) === country(e.country_en) && days(o.dc.date_sort, e.dc.date_sort) <= 30) .map((o) => ({ key: o.key, date: o.date, victim: o.victim, location: o.location, lopp_label: o.lopp_label })); out.sort((a, b) => (a.days_apart ?? -1) - (b.days_apart ?? -1)); return { urlHits, candidates: out.slice(0, 12), siblings: sib.slice(0, 6) }; } for (const e of entries) { const pages = (fetched[e.key] ?? []).map((g: any, i: number) => { const file = `work/articles/${e.key}-${i + 1}.txt`; const head = [`URL ${g.url}`, `Via ${g.via} (status ${g.status})`, `Retrieved ${g.retrieved}`, `Title ${g.title}`, `Published ${g.published || "(not given)"}`, `Byline ${g.byline || "(not given)"}`, `Length ${g.chars} chars, ${g.paras.length} paragraphs`, ""]; writeFileSync(join(HERE, file), head.join("\n") + "\n" + g.paras.map((p: string, j: number) => `[${j + 1}] ${p}`).join("\n\n") + "\n"); return { file, url: g.url, via: g.via, status: g.status, chars: g.chars, paragraphs: g.paras.length, title: g.title }; }); const n = near(e); writeFileSync(join(HERE, "work/in", `${e.key}.json`), JSON.stringify({ key: e.key, lopp: { date: e.date, date_as_stats2: e.s2_date, victim: e.victim, location: e.location, summary_line: e.lopp_label, links: e.links }, pages, stats2_url_match: n.urlHits, stats2_candidates: n.candidates, new_entry_siblings: n.siblings, }, null, 1)); } const keys = entries.map((e) => e.key); for (let i = 0; i < keys.length; i += BATCH) { const nn = String(i / BATCH + 1).padStart(2, "0"); writeFileSync(join(HERE, "batches", `batch-${nn}.json`), JSON.stringify(keys.slice(i, i + BATCH))); } const big = entries.flatMap((e) => (fetched[e.key] ?? []).filter((g: any) => g.chars > 40000).map((g: any) => `${e.key} ${g.chars}`)); console.log(`entries ${entries.length}, batches ${Math.ceil(keys.length / BATCH)}, pages ${Object.values(fetched).flat().length}, pages over 40k chars: ${big.length} (${big.join(", ")})`); console.log(`with stats2 URL match: ${entries.filter((e) => near(e).urlHits.length).length}; with any stats2 candidate: ${entries.filter((e) => near(e).candidates.length).length}`);