This repository has been archived on 2026-10-19. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
DB-cleanup/merge-output/lopp-pr234/fetch.ts
StellarCrow 41766c68a2 chore: final state before the move to gart/gart-databases
Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
2026-10-17 10:47:37 +02:00

159 lines
9.8 KiB
TypeScript

// Fetches every source link of the new Lopp entries (PR 234) and extracts the article paragraphs.
// bun fetch.ts fetch all keys not yet in work/fetched.json
// bun fetch.ts --retry fetch again the keys with no usable article
// Input work/entries.json (written on first run from work/fresh2.json), output work/fetched.json, raw HTML in raw/.
import { Readability } from "@mozilla/readability";
import { parseHTML } from "linkedom";
import { existsSync, readFileSync, writeFileSync } from "fs";
import { createHash } from "crypto";
import { join } from "path";
const HERE = import.meta.dir;
const W = (p: string) => join(HERE, "work", p);
const UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
const MIN_CHARS = 400;
type Link = { label: string; url: string };
type Entry = { key: string; date: string; victim: string; location: string; country: string; country_code: string; lat: number; lng: number; lopp_label: string; links: Link[]; dup: string; s2_url_match: number[]; candidates: any[] };
type Got = { url: string; via: string; status: number | string; title: string; byline: string; published: string; paras: string[]; chars: number; sha256: string; retrieved: string };
if (!existsSync(W("entries.json"))) {
const fresh = JSON.parse(readFileSync(W("fresh2.json"), "utf8"));
const entries: Entry[] = fresh.map((e: any, i: number) => ({
key: `L${String(i + 1).padStart(3, "0")}`,
date: e.date.raw, victim: e.victim, location: e.location,
country: e.country, country_code: e.country_code, lat: e.lat, lng: e.lng,
lopp_label: e.links.find((l: Link) => !l.label.startsWith("(original"))?.label ?? e.links[0]?.label ?? "",
links: e.links, dup: e._dup ?? "", s2_url_match: e._s2 ?? [], candidates: e._cands ?? [],
}));
writeFileSync(W("entries.json"), JSON.stringify(entries, null, 1));
}
const entries: Entry[] = JSON.parse(readFileSync(W("entries.json"), "utf8"));
const fetched: Record<string, Got[]> = existsSync(W("fetched.json")) ? JSON.parse(readFileSync(W("fetched.json"), "utf8")) : {};
const retry = process.argv.includes("--retry");
const extra = process.argv.includes("--extra"); // entries whose archive link is a different page from their original links
const usable = (g: Got[] | undefined) => (g ?? []).some((x) => x.chars >= MIN_CHARS && typeof x.status === "number" && x.status < 400);
const isArchive = (u: string) => /^https?:\/\/(web\.)?archive\.(org|is|ph|today|li|vn|md)\//.test(u);
const wayback = (u: string) => /^https?:\/\/web\.archive\.org\/web\/(\d+)[a-z_]*\/(.+)$/.exec(u);
function charsetOf(ct: string, head: string): string {
const m = /charset=["']?([\w-]+)/i.exec(ct) ?? /<meta[^>]+charset=["']?([\w-]+)/i.exec(head);
return (m?.[1] ?? "utf-8").toLowerCase();
}
async function get(url: string): Promise<{ status: number | string; html: string; pdf?: Uint8Array; xml?: boolean }> {
try {
const r = await fetch(url, { headers: { "User-Agent": UA, "Accept-Language": "en,fr;q=0.8,*;q=0.5", Accept: "text/html,*/*" }, redirect: "follow", signal: AbortSignal.timeout(30000) });
const buf = new Uint8Array(await r.arrayBuffer());
const head = new TextDecoder("latin1").decode(buf.slice(0, 4000));
const ct = r.headers.get("content-type") ?? "";
if (head.startsWith("%PDF") || /pdf/i.test(ct)) return { status: r.status, html: "", pdf: buf };
if (/^\s*<\?xml/.test(head) && /rechtspraak/.test(url)) return { status: r.status, html: new TextDecoder("utf-8").decode(buf), xml: true };
let cs = charsetOf(r.headers.get("content-type") ?? "", head);
if (cs === "gb2312") cs = "gbk";
let html: string;
try { html = new TextDecoder(cs).decode(buf); } catch { html = new TextDecoder("utf-8").decode(buf); }
return { status: r.status, html };
} catch (e: any) {
return { status: String(e?.name === "TimeoutError" ? "timeout" : e?.message ?? e).slice(0, 120), html: "" };
}
}
function extract(html: string, url: string) {
const { document } = parseHTML(html);
const meta = (sel: string) => document.querySelector(sel)?.getAttribute("content") ?? "";
const published = meta('meta[property="article:published_time"]') || meta('meta[name="pubdate"]') || meta('meta[name="publishdate"]') || meta('meta[itemprop="datePublished"]') || meta('meta[name="date"]') || (document.querySelector("time[datetime]")?.getAttribute("datetime") ?? "");
let art: any = null;
try { art = new Readability(document as any, { charThreshold: 200 }).parse(); } catch {}
const paras: string[] = [];
if (art?.content) {
const { document: d2 } = parseHTML(`<html><body>${art.content}</body></html>`);
for (const el of d2.querySelectorAll("h1,h2,h3,h4,p,li,blockquote,pre,td")) {
if (el.querySelector("p,li,blockquote")) continue; // keep the innermost block only
const t = (el.textContent ?? "").replace(/\s+/g, " ").trim();
if (t) paras.push(t);
}
if (!paras.length && art.textContent) paras.push(...art.textContent.split(/\n\s*\n/).map((s: string) => s.replace(/\s+/g, " ").trim()).filter(Boolean));
}
const title = (art?.title || document.querySelector("title")?.textContent || "").replace(/\s+/g, " ").trim();
return { title, byline: (art?.byline ?? "").replace(/\s+/g, " ").trim(), published, paras };
}
async function one(e: Entry): Promise<Got[]> {
const originals = e.links.filter((l) => !isArchive(l.url)).map((l) => l.url);
const archives = e.links.filter((l) => isArchive(l.url)).map((l) => l.url);
const out: Got[] = [];
const tried = new Set<string>();
const attempt = async (url: string, via: string, sourceUrl = url) => {
const ecli = /uitspraken\.rechtspraak\.nl\/(?:details|inziendocument)\?id=(ECLI:[^&]+)/i.exec(url);
if (ecli) url = `https://data.rechtspraak.nl/uitspraken/content?id=${ecli[1]}`;
if (tried.has(url)) return null;
tried.add(url);
const r = await get(url);
let x = { title: "", byline: "", published: "", paras: [] as string[] };
if (r.pdf) {
const f = join(HERE, "raw", `${e.key}-${out.length + 1}.pdf`);
writeFileSync(f, r.pdf);
const t = Bun.spawnSync(["pdftotext", "-enc", "UTF-8", f, "-"]).stdout.toString();
x.paras = t.split(/\n\s*\n/).map((s) => s.replace(/\s+/g, " ").trim()).filter((s) => s.length > 1);
x.title = x.paras[0]?.slice(0, 200) ?? "";
via += " (PDF, text layer by pdftotext)";
} else if (r.xml) {
const { document } = parseHTML(r.html.replace(/<\?xml[^>]*>/, ""));
x.paras = [...document.querySelectorAll("para, title")].map((el: any) => (el.textContent ?? "").replace(/\s+/g, " ").trim()).filter(Boolean);
x.title = document.querySelector("dcterms\\:identifier, identifier")?.textContent ?? "";
x.published = document.querySelector("dcterms\\:date, date")?.textContent ?? "";
via += " (court XML)";
} else if (r.html) x = extract(r.html, url);
const text = x.paras.join("\n\n");
const n = Object.values(fetched).flat().length + out.length;
if (r.html) writeFileSync(join(HERE, "raw", `${e.key}-${out.length + 1}.html`), r.html);
const g: Got = { url: sourceUrl, via, status: r.status, ...x, chars: text.length, sha256: createHash("sha256").update(text).digest("hex"), retrieved: new Date().toISOString().replace(/\.\d+Z$/, "Z") };
out.push(g);
return g;
};
// every original link, and its archive copy when the original gives no article
for (const u of originals) {
const g = await attempt(u, "raw HTTP retrieval");
if (g && g.chars >= MIN_CHARS && typeof g.status === "number" && g.status < 400) continue;
const wb = archives.map(wayback).find(Boolean);
const viaWb = wb && wb[2].replace(/^https?:\/\//, "") === u.replace(/^https?:\/\//, "") ? `https://web.archive.org/web/${wb[1]}id_/${wb[2]}` : `https://web.archive.org/web/2026id_/${u}`;
await attempt(viaWb, "web.archive.org copy", u);
}
// archive links whose page is not one of the original links (a different article), or with no original at all
const bare = (u: string) => u.replace(/^https?:\/\/(www\.)?/, "").replace(/\/+$/, "");
for (const a of archives) {
const inner = wayback(a)?.[2];
if (originals.length && inner && originals.some((o) => bare(o) === bare(inner))) continue;
if (originals.length && !inner) continue;
const wb = wayback(a);
if (wb) await attempt(`https://web.archive.org/web/${wb[1]}id_/${wb[2]}`, "web.archive.org copy", wb[2]);
else out.push({ url: a, via: "archive.is (not fetched: captcha)", status: "skipped", title: "", byline: "", published: "", paras: [], chars: 0, sha256: "", retrieved: "" });
}
return out;
}
const bareU = (u: string) => u.replace(/^https?:\/\/(www\.)?/, "").replace(/\/+$/, "");
const hasExtra = (e: Entry) => { const o = e.links.filter((l) => !isArchive(l.url)).map((l) => bareU(l.url)); return o.length > 0 && e.links.some((l) => { const w = wayback(l.url); return w && !o.includes(bareU(w[2])); }); };
const todo = entries.filter((e) => (extra ? hasExtra(e) : retry ? !usable(fetched[e.key]) : !fetched[e.key]));
console.log(`to fetch: ${todo.length} entries`);
const hostBusy = new Map<string, number>();
let done = 0;
const queue = [...todo];
async function worker() {
while (queue.length) {
const i = queue.findIndex((e) => { const h = new URL(e.links[0].url).host; return (hostBusy.get(h) ?? 0) < 2; });
const e = queue.splice(i === -1 ? 0 : i, 1)[0];
const h = new URL(e.links[0].url).host;
hostBusy.set(h, (hostBusy.get(h) ?? 0) + 1);
fetched[e.key] = await one(e);
hostBusy.set(h, (hostBusy.get(h) ?? 1) - 1);
if (++done % 20 === 0) { writeFileSync(W("fetched.json"), JSON.stringify(fetched)); console.log(`${done}/${todo.length}`); }
}
}
await Promise.all(Array.from({ length: 12 }, worker));
writeFileSync(W("fetched.json"), JSON.stringify(fetched));
const all = entries.map((e) => fetched[e.key]);
console.log(`entries with a usable article: ${all.filter(usable).length}/${entries.length}; none: ${all.filter((g) => !usable(g)).length}`);