// Fetches every source link of the new Lopp entries (PR 234) and extracts the article paragraphs. // bun fetch.ts fetch all keys not yet in work/fetched.json // bun fetch.ts --retry fetch again the keys with no usable article // Input work/entries.json (written on first run from work/fresh2.json), output work/fetched.json, raw HTML in raw/. import { Readability } from "@mozilla/readability"; import { parseHTML } from "linkedom"; import { existsSync, readFileSync, writeFileSync } from "fs"; import { createHash } from "crypto"; import { join } from "path"; const HERE = import.meta.dir; const W = (p: string) => join(HERE, "work", p); const UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"; const MIN_CHARS = 400; type Link = { label: string; url: string }; type Entry = { key: string; date: string; victim: string; location: string; country: string; country_code: string; lat: number; lng: number; lopp_label: string; links: Link[]; dup: string; s2_url_match: number[]; candidates: any[] }; type Got = { url: string; via: string; status: number | string; title: string; byline: string; published: string; paras: string[]; chars: number; sha256: string; retrieved: string }; if (!existsSync(W("entries.json"))) { const fresh = JSON.parse(readFileSync(W("fresh2.json"), "utf8")); const entries: Entry[] = fresh.map((e: any, i: number) => ({ key: `L${String(i + 1).padStart(3, "0")}`, date: e.date.raw, victim: e.victim, location: e.location, country: e.country, country_code: e.country_code, lat: e.lat, lng: e.lng, lopp_label: e.links.find((l: Link) => !l.label.startsWith("(original"))?.label ?? e.links[0]?.label ?? "", links: e.links, dup: e._dup ?? "", s2_url_match: e._s2 ?? [], candidates: e._cands ?? [], })); writeFileSync(W("entries.json"), JSON.stringify(entries, null, 1)); } const entries: Entry[] = JSON.parse(readFileSync(W("entries.json"), "utf8")); const fetched: Record = existsSync(W("fetched.json")) ? JSON.parse(readFileSync(W("fetched.json"), "utf8")) : {}; const retry = process.argv.includes("--retry"); const extra = process.argv.includes("--extra"); // entries whose archive link is a different page from their original links const usable = (g: Got[] | undefined) => (g ?? []).some((x) => x.chars >= MIN_CHARS && typeof x.status === "number" && x.status < 400); const isArchive = (u: string) => /^https?:\/\/(web\.)?archive\.(org|is|ph|today|li|vn|md)\//.test(u); const wayback = (u: string) => /^https?:\/\/web\.archive\.org\/web\/(\d+)[a-z_]*\/(.+)$/.exec(u); function charsetOf(ct: string, head: string): string { const m = /charset=["']?([\w-]+)/i.exec(ct) ?? /]+charset=["']?([\w-]+)/i.exec(head); return (m?.[1] ?? "utf-8").toLowerCase(); } async function get(url: string): Promise<{ status: number | string; html: string; pdf?: Uint8Array; xml?: boolean }> { try { const r = await fetch(url, { headers: { "User-Agent": UA, "Accept-Language": "en,fr;q=0.8,*;q=0.5", Accept: "text/html,*/*" }, redirect: "follow", signal: AbortSignal.timeout(30000) }); const buf = new Uint8Array(await r.arrayBuffer()); const head = new TextDecoder("latin1").decode(buf.slice(0, 4000)); const ct = r.headers.get("content-type") ?? ""; if (head.startsWith("%PDF") || /pdf/i.test(ct)) return { status: r.status, html: "", pdf: buf }; if (/^\s*<\?xml/.test(head) && /rechtspraak/.test(url)) return { status: r.status, html: new TextDecoder("utf-8").decode(buf), xml: true }; let cs = charsetOf(r.headers.get("content-type") ?? "", head); if (cs === "gb2312") cs = "gbk"; let html: string; try { html = new TextDecoder(cs).decode(buf); } catch { html = new TextDecoder("utf-8").decode(buf); } return { status: r.status, html }; } catch (e: any) { return { status: String(e?.name === "TimeoutError" ? "timeout" : e?.message ?? e).slice(0, 120), html: "" }; } } function extract(html: string, url: string) { const { document } = parseHTML(html); const meta = (sel: string) => document.querySelector(sel)?.getAttribute("content") ?? ""; const published = meta('meta[property="article:published_time"]') || meta('meta[name="pubdate"]') || meta('meta[name="publishdate"]') || meta('meta[itemprop="datePublished"]') || meta('meta[name="date"]') || (document.querySelector("time[datetime]")?.getAttribute("datetime") ?? ""); let art: any = null; try { art = new Readability(document as any, { charThreshold: 200 }).parse(); } catch {} const paras: string[] = []; if (art?.content) { const { document: d2 } = parseHTML(`${art.content}`); for (const el of d2.querySelectorAll("h1,h2,h3,h4,p,li,blockquote,pre,td")) { if (el.querySelector("p,li,blockquote")) continue; // keep the innermost block only const t = (el.textContent ?? "").replace(/\s+/g, " ").trim(); if (t) paras.push(t); } if (!paras.length && art.textContent) paras.push(...art.textContent.split(/\n\s*\n/).map((s: string) => s.replace(/\s+/g, " ").trim()).filter(Boolean)); } const title = (art?.title || document.querySelector("title")?.textContent || "").replace(/\s+/g, " ").trim(); return { title, byline: (art?.byline ?? "").replace(/\s+/g, " ").trim(), published, paras }; } async function one(e: Entry): Promise { const originals = e.links.filter((l) => !isArchive(l.url)).map((l) => l.url); const archives = e.links.filter((l) => isArchive(l.url)).map((l) => l.url); const out: Got[] = []; const tried = new Set(); const attempt = async (url: string, via: string, sourceUrl = url) => { const ecli = /uitspraken\.rechtspraak\.nl\/(?:details|inziendocument)\?id=(ECLI:[^&]+)/i.exec(url); if (ecli) url = `https://data.rechtspraak.nl/uitspraken/content?id=${ecli[1]}`; if (tried.has(url)) return null; tried.add(url); const r = await get(url); let x = { title: "", byline: "", published: "", paras: [] as string[] }; if (r.pdf) { const f = join(HERE, "raw", `${e.key}-${out.length + 1}.pdf`); writeFileSync(f, r.pdf); const t = Bun.spawnSync(["pdftotext", "-enc", "UTF-8", f, "-"]).stdout.toString(); x.paras = t.split(/\n\s*\n/).map((s) => s.replace(/\s+/g, " ").trim()).filter((s) => s.length > 1); x.title = x.paras[0]?.slice(0, 200) ?? ""; via += " (PDF, text layer by pdftotext)"; } else if (r.xml) { const { document } = parseHTML(r.html.replace(/<\?xml[^>]*>/, "")); x.paras = [...document.querySelectorAll("para, title")].map((el: any) => (el.textContent ?? "").replace(/\s+/g, " ").trim()).filter(Boolean); x.title = document.querySelector("dcterms\\:identifier, identifier")?.textContent ?? ""; x.published = document.querySelector("dcterms\\:date, date")?.textContent ?? ""; via += " (court XML)"; } else if (r.html) x = extract(r.html, url); const text = x.paras.join("\n\n"); const n = Object.values(fetched).flat().length + out.length; if (r.html) writeFileSync(join(HERE, "raw", `${e.key}-${out.length + 1}.html`), r.html); const g: Got = { url: sourceUrl, via, status: r.status, ...x, chars: text.length, sha256: createHash("sha256").update(text).digest("hex"), retrieved: new Date().toISOString().replace(/\.\d+Z$/, "Z") }; out.push(g); return g; }; // every original link, and its archive copy when the original gives no article for (const u of originals) { const g = await attempt(u, "raw HTTP retrieval"); if (g && g.chars >= MIN_CHARS && typeof g.status === "number" && g.status < 400) continue; const wb = archives.map(wayback).find(Boolean); const viaWb = wb && wb[2].replace(/^https?:\/\//, "") === u.replace(/^https?:\/\//, "") ? `https://web.archive.org/web/${wb[1]}id_/${wb[2]}` : `https://web.archive.org/web/2026id_/${u}`; await attempt(viaWb, "web.archive.org copy", u); } // archive links whose page is not one of the original links (a different article), or with no original at all const bare = (u: string) => u.replace(/^https?:\/\/(www\.)?/, "").replace(/\/+$/, ""); for (const a of archives) { const inner = wayback(a)?.[2]; if (originals.length && inner && originals.some((o) => bare(o) === bare(inner))) continue; if (originals.length && !inner) continue; const wb = wayback(a); if (wb) await attempt(`https://web.archive.org/web/${wb[1]}id_/${wb[2]}`, "web.archive.org copy", wb[2]); else out.push({ url: a, via: "archive.is (not fetched: captcha)", status: "skipped", title: "", byline: "", published: "", paras: [], chars: 0, sha256: "", retrieved: "" }); } return out; } const bareU = (u: string) => u.replace(/^https?:\/\/(www\.)?/, "").replace(/\/+$/, ""); const hasExtra = (e: Entry) => { const o = e.links.filter((l) => !isArchive(l.url)).map((l) => bareU(l.url)); return o.length > 0 && e.links.some((l) => { const w = wayback(l.url); return w && !o.includes(bareU(w[2])); }); }; const todo = entries.filter((e) => (extra ? hasExtra(e) : retry ? !usable(fetched[e.key]) : !fetched[e.key])); console.log(`to fetch: ${todo.length} entries`); const hostBusy = new Map(); let done = 0; const queue = [...todo]; async function worker() { while (queue.length) { const i = queue.findIndex((e) => { const h = new URL(e.links[0].url).host; return (hostBusy.get(h) ?? 0) < 2; }); const e = queue.splice(i === -1 ? 0 : i, 1)[0]; const h = new URL(e.links[0].url).host; hostBusy.set(h, (hostBusy.get(h) ?? 0) + 1); fetched[e.key] = await one(e); hostBusy.set(h, (hostBusy.get(h) ?? 1) - 1); if (++done % 20 === 0) { writeFileSync(W("fetched.json"), JSON.stringify(fetched)); console.log(`${done}/${todo.length}`); } } } await Promise.all(Array.from({ length: 12 }, worker)); writeFileSync(W("fetched.json"), JSON.stringify(fetched)); const all = entries.map((e) => fetched[e.key]); console.log(`entries with a usable article: ${all.filter(usable).length}/${entries.length}; none: ${all.filter((g) => !usable(g)).length}`);