// Writes blocked-sources.csv: every link of the entries whose fetch gave no usable article, one row per link. import { readFileSync, writeFileSync } from "fs"; const entries: any[] = JSON.parse(readFileSync("work/entries.json", "utf8")); const fetched: Record = JSON.parse(readFileSync("work/fetched.json", "utf8")); const q = (v: unknown) => `"${String(v ?? "").replace(/"/g, '""')}"`; const isArchive = (u: string) => /^https?:\/\/(web\.)?archive\.(org|is|ph|today|li|vn|md)\//.test(u); const blocked = entries.filter((e) => !(fetched[e.key] ?? []).some((g) => g.chars >= 400 && typeof g.status === "number" && g.status < 400)); const rows = [["lopp_key", "date", "victim", "location", "link_type", "url", "fetch_result", "stats2_case_id_same_url", "stats2_cases_to_check_for_duplicate", "lopp_summary_line"].join(",")]; for (const e of blocked) { const inn = JSON.parse(readFileSync(`work/in/${e.key}.json`, "utf8")); const near = inn.stats2_candidates.filter((c: any) => c.days_apart !== null && c.days_apart <= 30).map((c: any) => `${c.stats2_id} (${c.date}, ${c.location})`).join(" | "); for (const l of e.links) { const tries = (fetched[e.key] ?? []).filter((g) => g.url === l.url || (isArchive(l.url) && l.url.includes(g.url.replace(/^https?:\/\//, "")))); const res = isArchive(l.url) && /archive\.(is|ph|today)/.test(l.url) ? "not fetched: archive.is captcha" : tries.map((g) => `${g.via}: ${g.status}${g.chars ? `, ${g.chars} chars` : ""}`).join("; ") || "not fetched"; rows.push([e.key, e.date, e.victim, e.location, isArchive(l.url) ? "archive" : "original", l.url, res, inn.stats2_url_match.join(" | "), near, e.lopp_label].map(q).join(",")); } } writeFileSync("blocked-sources.csv", rows.join("\n") + "\n"); console.log(`${blocked.length} entries, ${rows.length - 1} links; with a same-URL stats2 case: ${blocked.filter((e) => JSON.parse(readFileSync(`work/in/${e.key}.json`, "utf8")).stats2_url_match.length).length}`);