K&R database cleanup: handoff bundle
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Pre-trim deterministic audit. Mechanical checks only; no judgment.
|
||||
|
||||
Everything here is re-runnable and its result is match/no-match, so it is not
|
||||
'marking my own homework' — anyone can run it and get the same answer. The judgment
|
||||
call (is an article under the right case) is left to an independent agent.
|
||||
|
||||
Exit non-zero if anything fails.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
CF = BASE / "sources"
|
||||
STORE = BASE / "sourced-articles.json"
|
||||
MANIFEST = BASE / "case-files-manifest.json"
|
||||
STAGING = BASE / "staging"
|
||||
JSONF = BASE.parent / "attacks-export-Gart-website.json"
|
||||
|
||||
|
||||
def sha(t):
|
||||
return hashlib.sha256(t.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
recs = json.loads(JSONF.read_text(encoding="utf-8"))
|
||||
ids = {r["id"] for r in recs}
|
||||
store = json.loads(STORE.read_text(encoding="utf-8"))
|
||||
fails, checks = [], 0
|
||||
|
||||
def check(cond, msg):
|
||||
nonlocal checks
|
||||
checks += 1
|
||||
if not cond:
|
||||
fails.append(msg)
|
||||
|
||||
# 1. one file per DB record, filename prefix matches id
|
||||
files = {f: int(f.name.split("_")[0]) for f in CF.glob("*.txt")}
|
||||
check(set(files.values()) == ids, "case-file id set does not equal DB id set")
|
||||
check(len(files) == len(ids), f"file count {len(files)} != record count {len(ids)}")
|
||||
file_by_id = {cid: f for f, cid in files.items()}
|
||||
|
||||
# 2. every stored article block is byte-present in its case file, and its hash re-derives
|
||||
WITH_TEXT = ("KEEP", "PARTIAL", "UNTRIMMED")
|
||||
n_blocks = 0
|
||||
for cid_s, blocks in store.items():
|
||||
cid = int(cid_s)
|
||||
if cid not in file_by_id:
|
||||
fails.append(f"case {cid} in store but has no case file")
|
||||
continue
|
||||
body = file_by_id[cid].read_text(encoding="utf-8")
|
||||
for e in blocks:
|
||||
if e.get("verdict") not in WITH_TEXT:
|
||||
continue
|
||||
n_blocks += 1
|
||||
txt = e.get("text")
|
||||
check(isinstance(txt, str) and txt, f"case {cid} {e.get('field')}: empty text")
|
||||
if not txt:
|
||||
continue
|
||||
check(txt in body, f"case {cid} {e.get('field')}: article NOT verbatim in case file")
|
||||
check(sha(txt) == e.get("sha256"), f"case {cid} {e.get('field')}: stored sha256 mismatch")
|
||||
# untrimmed blocks must carry the warning banner
|
||||
if e["verdict"] == "UNTRIMMED":
|
||||
check("RAW EXTRACT — NOT YET TRIMMED" in body,
|
||||
f"case {cid}: has an UNTRIMMED block but no banner in the file")
|
||||
# raw-retrieved blocks must still match their staged file on disk. The stored
|
||||
# text may be non-contiguous (a prepended headline plus a later body range), so
|
||||
# the right invariant is per-paragraph: every paragraph of the stored text must
|
||||
# appear verbatim as a paragraph in the staged file. That catches any altered or
|
||||
# fabricated paragraph regardless of gaps.
|
||||
sf = e.get("staged_file")
|
||||
if sf and "raw HTTP" in (e.get("method") or ""):
|
||||
p = STAGING / f"{cid:03d}" / sf
|
||||
if p.exists():
|
||||
staged_paras = set(p.read_text(encoding="utf-8").split("\n\n"))
|
||||
bad = [para for para in txt.split("\n\n") if para and para not in staged_paras]
|
||||
check(not bad,
|
||||
f"case {cid} {e.get('field')}: {len(bad)} paragraph(s) not found in "
|
||||
f"staged file (altered or drifted)")
|
||||
|
||||
# 3. no DB id receives article text under two different provenance sources
|
||||
# (spreadsheet double-assignment guard, re-checked from the manifest)
|
||||
if MANIFEST.exists():
|
||||
man = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
# One sheet article (row 164, the Manchester Evening News piece) documents five
|
||||
# Salford incidents and is deliberately filed into all five cases. So a case
|
||||
# legitimately holds its own row plus row 164. Flag only two DIFFERENT own-rows.
|
||||
SHARED_ROWS = {"164"}
|
||||
sheet_seen = defaultdict(set)
|
||||
for m in man["entries"]:
|
||||
if "reported_K&R sheet" in m.get("origin", ""):
|
||||
row = re.search(r"row (\d+)", m["origin"])
|
||||
if row:
|
||||
sheet_seen[m["case"]].add(row.group(1))
|
||||
for cid, rowset in sheet_seen.items():
|
||||
own = rowset - SHARED_ROWS
|
||||
if len(own) > 1:
|
||||
fails.append(f"case {cid}: sheet articles from two different own-rows {sorted(own)}")
|
||||
# every manifest article still verbatim in its file
|
||||
for m in man["entries"]:
|
||||
cid = m["case"]
|
||||
if cid in file_by_id:
|
||||
body = file_by_id[cid].read_text(encoding="utf-8")
|
||||
# manifest doesn't store text, only sha + chars; re-derive from the store
|
||||
pass # covered by check 2 for store-backed blocks
|
||||
|
||||
# 4. spreadsheet phase-1 spot check: sample sheet-origin articles re-verified
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(BASE.parent / "KR-reports_analysis_Sofi.xlsx",
|
||||
data_only=True, read_only=True)
|
||||
rows = list(wb["reported_K&R"].iter_rows(values_only=True))
|
||||
idx = {h: i for i, h in enumerate(rows[0]) if h is not None}
|
||||
col = {"Articles": "Articles", "Article 2": "Article 2", "Article 3": "Article 3"}
|
||||
if MANIFEST.exists():
|
||||
import random
|
||||
sheet_arts = [m for m in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"]
|
||||
if "reported_K&R sheet" in m.get("origin", "")]
|
||||
random.seed(1)
|
||||
for m in random.sample(sheet_arts, min(40, len(sheet_arts))):
|
||||
rn = int(re.search(r"row (\d+)", m["origin"]).group(1))
|
||||
cn = re.search(r'column "([^"]+)"', m["origin"]).group(1)
|
||||
cell = rows[rn - 1][idx[col[cn]]]
|
||||
check(isinstance(cell, str) and sha(cell) == m["sha256"],
|
||||
f"case {m['case']}: sheet article no longer matches cell (row {rn}, {cn})")
|
||||
|
||||
print(f"ran {checks} checks over {n_blocks} phase-2 blocks and {len(files)} case files\n")
|
||||
if fails:
|
||||
print(f"FAIL — {len(fails)} problem(s):")
|
||||
for f in fails:
|
||||
print(f" - {f}")
|
||||
sys.exit(1)
|
||||
print("PASS — every mechanical check clean:")
|
||||
print(" * one file per DB record, filenames match ids")
|
||||
print(" * every filed article byte-present in its case file")
|
||||
print(" * every stored sha256 re-derives from the text")
|
||||
print(" * raw-retrieved text still matches its staged file (no drift)")
|
||||
print(" * every UNTRIMMED block carries its warning banner")
|
||||
print(" * no case has spreadsheet articles from two different rows")
|
||||
print(" * sampled phase-1 articles still match their spreadsheet cells")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user