chore: final state before the move to gart/gart-databases

Commits the work done since 2026-10-01 that was still local: the four-step
label and flag passes, the triage and recheck files, the v1 exports of
2026-10-06 and 2026-10-07, the split dossiers, and the Lopp PR 234 batch
(scripts, agent prompts, reader and checker files, the flag re-read of
2026-10-17, the review workbook and the import set of the 57 cases added
to stats2 as 506 to 562). Raw HTML and node_modules stay out (.gitignore).

The stats2 material in use was copied to gart/gart-databases (stats2/) on
2026-10-17; README-START-HERE.md now says so. This repo is closed.
This commit is contained in:
StellarCrow
2026-10-17 10:47:37 +02:00
parent 0f7a691151
commit 41766c68a2
3196 changed files with 374809 additions and 485 deletions
+19
View File
@@ -0,0 +1,19 @@
// Writes blocked-sources.csv: every link of the entries whose fetch gave no usable article, one row per link.
import { readFileSync, writeFileSync } from "fs";
const entries: any[] = JSON.parse(readFileSync("work/entries.json", "utf8"));
const fetched: Record<string, any[]> = JSON.parse(readFileSync("work/fetched.json", "utf8"));
const q = (v: unknown) => `"${String(v ?? "").replace(/"/g, '""')}"`;
const isArchive = (u: string) => /^https?:\/\/(web\.)?archive\.(org|is|ph|today|li|vn|md)\//.test(u);
const blocked = entries.filter((e) => !(fetched[e.key] ?? []).some((g) => g.chars >= 400 && typeof g.status === "number" && g.status < 400));
const rows = [["lopp_key", "date", "victim", "location", "link_type", "url", "fetch_result", "stats2_case_id_same_url", "stats2_cases_to_check_for_duplicate", "lopp_summary_line"].join(",")];
for (const e of blocked) {
const inn = JSON.parse(readFileSync(`work/in/${e.key}.json`, "utf8"));
const near = inn.stats2_candidates.filter((c: any) => c.days_apart !== null && c.days_apart <= 30).map((c: any) => `${c.stats2_id} (${c.date}, ${c.location})`).join(" | ");
for (const l of e.links) {
const tries = (fetched[e.key] ?? []).filter((g) => g.url === l.url || (isArchive(l.url) && l.url.includes(g.url.replace(/^https?:\/\//, ""))));
const res = isArchive(l.url) && /archive\.(is|ph|today)/.test(l.url) ? "not fetched: archive.is captcha" : tries.map((g) => `${g.via}: ${g.status}${g.chars ? `, ${g.chars} chars` : ""}`).join("; ") || "not fetched";
rows.push([e.key, e.date, e.victim, e.location, isArchive(l.url) ? "archive" : "original", l.url, res, inn.stats2_url_match.join(" | "), near, e.lopp_label].map(q).join(","));
}
}
writeFileSync("blocked-sources.csv", rows.join("\n") + "\n");
console.log(`${blocked.length} entries, ${rows.length - 1} links; with a same-URL stats2 case: ${blocked.filter((e) => JSON.parse(readFileSync(`work/in/${e.key}.json`, "utf8")).stats2_url_match.length).length}`);