K&R database cleanup: handoff bundle
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,211 @@
|
||||
"""
|
||||
Stage source text for one or more cases.
|
||||
|
||||
Raw HTTP first, because that path is deterministic: the bytes come off the wire and
|
||||
the extracted text is hashed, so it can be re-verified later. Sites that refuse a raw
|
||||
request are recorded as BLOCKED and left for the WebFetch path, which is a model
|
||||
transcription and must be labelled as such.
|
||||
|
||||
Nothing here writes to case files. Output goes to merge-output/staging/.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from datetime import datetime, timezone
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
||||
STAGING = BASE / "staging"
|
||||
|
||||
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
NOT_ARTICLE = re.compile(r"(youtube\.com|youtu\.be|x\.com|twitter\.com|instagram\.com|"
|
||||
r"facebook\.com|tiktok\.com|t\.me)", re.I)
|
||||
|
||||
# Furniture that survives tag-stripping and has to go by content, not markup.
|
||||
JUNK = re.compile(r"^(facebook|instagram|linkedin|rss|twitter|x|whatsapp|telegram|share|"
|
||||
r"compartilhar|publicidade|advertisement|newsletter|cookies?|assine|"
|
||||
r"leia (também|mais)|read more|related( articles)?|tags?|"
|
||||
r"siga o|follow us|subscribe)\b", re.I)
|
||||
|
||||
|
||||
class Extract(HTMLParser):
|
||||
"""Pull block-level text.
|
||||
|
||||
Deliberately simple. Suppression is tracked with a stack of the tag names that
|
||||
opened it, so a container that never closes cleanly cannot swallow the rest of the
|
||||
document. Furniture is removed by content afterwards, not by guessing at CSS class
|
||||
names, because class conventions differ on every site and a wrong guess here fails
|
||||
silently by returning an empty page.
|
||||
"""
|
||||
|
||||
DROP = {"script", "style", "noscript", "nav", "header", "footer", "aside",
|
||||
"form", "svg", "button", "figure", "iframe", "select"}
|
||||
BLOCK = {"p", "h1", "h2", "h3", "h4", "li", "blockquote", "div", "section", "br"}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.stack = []
|
||||
self.cur = []
|
||||
self.blocks = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in self.DROP:
|
||||
self.stack.append(tag)
|
||||
return
|
||||
if tag in self.BLOCK:
|
||||
self._flush()
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.stack and self.stack[-1] == tag:
|
||||
self.stack.pop()
|
||||
return
|
||||
if not self.stack and tag in self.BLOCK:
|
||||
self._flush()
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.stack:
|
||||
self.cur.append(data)
|
||||
|
||||
def _flush(self):
|
||||
if self.cur:
|
||||
t = re.sub(r"[ \t\xa0]+", " ", "".join(self.cur)).strip()
|
||||
if t:
|
||||
self.blocks.append(t)
|
||||
self.cur = []
|
||||
|
||||
def text(self):
|
||||
self._flush()
|
||||
out, seen = [], set()
|
||||
for t in self.blocks:
|
||||
if len(t) < 40 and not re.search(r"[.!?]$", t):
|
||||
continue # nav labels, buttons, breadcrumbs
|
||||
if JUNK.match(t):
|
||||
continue
|
||||
if t in seen:
|
||||
continue
|
||||
seen.add(t)
|
||||
out.append(t)
|
||||
return "\n\n".join(out)
|
||||
|
||||
|
||||
def fetch_raw(url, timeout=30):
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": UA,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en,pt;q=0.9,fr;q=0.8,id;q=0.7,es;q=0.6",
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
raw = r.read()
|
||||
enc = (r.headers.get("Content-Encoding") or "").lower()
|
||||
if enc == "gzip":
|
||||
raw = gzip.decompress(raw)
|
||||
elif enc == "deflate":
|
||||
raw = zlib.decompress(raw)
|
||||
charset = r.headers.get_content_charset() or "utf-8"
|
||||
return r.status, raw.decode(charset, errors="replace"), r.geturl()
|
||||
|
||||
|
||||
def stage_case(rec, extra_urls=(), new_only=False):
|
||||
cid = rec["id"]
|
||||
out = STAGING / f"{cid:03d}"
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
entries = []
|
||||
|
||||
# Re-fetching a source that is already staged is pure cost, and worse: live pages
|
||||
# add and drop teasers, so the paragraph indices in an approved cut silently shift.
|
||||
# With --new-only, anything already fetched is carried over untouched.
|
||||
prior = {}
|
||||
idx_path = out / "index.json"
|
||||
if new_only and idx_path.exists():
|
||||
prior = {e["field"]: e for e in json.loads(idx_path.read_text(encoding="utf-8"))}
|
||||
|
||||
urls = [(f, rec[f]) for f in ("url", "url_2", "url_3", "url_4", "url_5") if rec.get(f)]
|
||||
urls += [(f"extra_{i+1}", u) for i, u in enumerate(extra_urls)]
|
||||
|
||||
for n, (field, url) in enumerate(urls, 1):
|
||||
host = re.sub(r"^https?://(www\.)?([^/]+).*", r"\2", url)
|
||||
if field in prior and prior[field].get("verdict") in ("FETCHED", "NOT AN ARTICLE"):
|
||||
carried = dict(prior[field])
|
||||
carried["carried_over"] = True
|
||||
entries.append(carried)
|
||||
continue
|
||||
|
||||
e = {"case": cid, "n": n, "field": field, "url": url, "host": host,
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(timespec="seconds")}
|
||||
|
||||
if NOT_ARTICLE.search(url):
|
||||
e.update(verdict="NOT AN ARTICLE", method="none",
|
||||
note="video or social post; recorded as a source reference, not transcribed")
|
||||
entries.append(e)
|
||||
continue
|
||||
|
||||
try:
|
||||
status, html, final = fetch_raw(url)
|
||||
text = Extract()
|
||||
text.feed(html)
|
||||
body = text.text()
|
||||
if len(body) < 400:
|
||||
e.update(verdict="THIN", method="raw", http=status, chars=len(body),
|
||||
note="extractor recovered very little; needs the WebFetch path or a "
|
||||
"site-specific rule")
|
||||
else:
|
||||
e.update(verdict="FETCHED", method="raw", http=status, chars=len(body),
|
||||
sha256=hashlib.sha256(body.encode()).hexdigest(),
|
||||
final_url=final if final != url else None)
|
||||
(out / f"{n:02d}_{re.sub(r'[^a-z0-9]+', '-', host.lower())}.txt").write_text(
|
||||
body, encoding="utf-8")
|
||||
except urllib.error.HTTPError as ex:
|
||||
e.update(verdict="BLOCKED" if ex.code in (401, 403, 429) else "DEAD",
|
||||
method="raw", http=ex.code,
|
||||
note="raw request refused; try the WebFetch path, which is a model "
|
||||
"transcription and must be labelled as such")
|
||||
except Exception as ex:
|
||||
e.update(verdict="ERROR", method="raw", note=f"{type(ex).__name__}: {str(ex)[:120]}")
|
||||
entries.append(e)
|
||||
|
||||
(out / "index.json").write_text(json.dumps(entries, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
return entries
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("cases", nargs="+", type=int)
|
||||
ap.add_argument("--extra", action="append", default=[],
|
||||
help="CASEID=URL for a source not on the record")
|
||||
ap.add_argument("--new-only", action="store_true",
|
||||
help="skip sources already staged; use this whenever adding a source to "
|
||||
"a case that has approved cuts, or their paragraph indices will drift")
|
||||
args = ap.parse_args()
|
||||
|
||||
extra = {}
|
||||
for spec in args.extra:
|
||||
cid, _, url = spec.partition("=")
|
||||
extra.setdefault(int(cid), []).append(url)
|
||||
|
||||
records = {r["id"]: r for r in json.loads(JSONF.read_text(encoding="utf-8"))}
|
||||
STAGING.mkdir(exist_ok=True)
|
||||
|
||||
for cid in args.cases:
|
||||
entries = stage_case(records[cid], extra.get(cid, ()), new_only=args.new_only)
|
||||
print(f"\ncase {cid}")
|
||||
for e in entries:
|
||||
bits = "carried" if e.get("carried_over") else (f"HTTP {e.get('http')}" if e.get("http") else "")
|
||||
size = f"{e['chars']:,} chars" if e.get("chars") else ""
|
||||
print(f" [{e['verdict']:<15}] {e['field']:<8} {e['host']:<26} {bits:<9} {size}")
|
||||
if e.get("note"):
|
||||
print(f" {e['note']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user