DB-cleanup/merge-output/verify_all.py

"""
Pre-trim deterministic audit. Mechanical checks only; no judgment.

Everything here is re-runnable and its result is match/no-match, so it is not
'marking my own homework' — anyone can run it and get the same answer. The judgment
call (is an article under the right case) is left to an independent agent.

Exit non-zero if anything fails.
"""

import hashlib
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

BASE = Path(__file__).resolve().parent
CF = BASE / "sources"
STORE = BASE / "sourced-articles.json"
MANIFEST = BASE / "case-files-manifest.json"
STAGING = BASE / "staging"
JSONF = BASE.parent / "attacks-export-Gart-website.json"


def sha(t):
    return hashlib.sha256(t.encode("utf-8")).hexdigest()


def main():
    recs = json.loads(JSONF.read_text(encoding="utf-8"))
    ids = {r["id"] for r in recs}
    store = json.loads(STORE.read_text(encoding="utf-8"))
    fails, checks = [], 0

    def check(cond, msg):
        nonlocal checks
        checks += 1
        if not cond:
            fails.append(msg)

    # 1. one file per DB record, filename prefix matches id
    files = {f: int(f.name.split("_")[0]) for f in CF.glob("*.txt")}
    check(set(files.values()) == ids, "case-file id set does not equal DB id set")
    check(len(files) == len(ids), f"file count {len(files)} != record count {len(ids)}")
    file_by_id = {cid: f for f, cid in files.items()}

    # 2. every stored article block is byte-present in its case file, and its hash re-derives
    WITH_TEXT = ("KEEP", "PARTIAL", "UNTRIMMED")
    n_blocks = 0
    for cid_s, blocks in store.items():
        cid = int(cid_s)
        if cid not in file_by_id:
            fails.append(f"case {cid} in store but has no case file")
            continue
        body = file_by_id[cid].read_text(encoding="utf-8")
        for e in blocks:
            if e.get("verdict") not in WITH_TEXT:
                continue
            n_blocks += 1
            txt = e.get("text")
            check(isinstance(txt, str) and txt, f"case {cid} {e.get('field')}: empty text")
            if not txt:
                continue
            check(txt in body, f"case {cid} {e.get('field')}: article NOT verbatim in case file")
            check(sha(txt) == e.get("sha256"), f"case {cid} {e.get('field')}: stored sha256 mismatch")
            # untrimmed blocks must carry the warning banner
            if e["verdict"] == "UNTRIMMED":
                check("RAW EXTRACT — NOT YET TRIMMED" in body,
                      f"case {cid}: has an UNTRIMMED block but no banner in the file")
            # raw-retrieved blocks must still match their staged file on disk. The stored
            # text may be non-contiguous (a prepended headline plus a later body range), so
            # the right invariant is per-paragraph: every paragraph of the stored text must
            # appear verbatim as a paragraph in the staged file. That catches any altered or
            # fabricated paragraph regardless of gaps.
            sf = e.get("staged_file")
            if e["verdict"] == "KEEP" and str(e.get("trimmed_by", "")).startswith("auto"):
                check("AUTO-TRIMMED — UNREVIEWED" in body,
                      f"case {cid}: has an auto-trimmed block but no banner in the file")
            if sf and ("raw HTTP" in (e.get("method") or "") or "Wayback" in (e.get("method") or "")):
                p = STAGING / f"{cid:03d}" / sf
                if p.exists():
                    staged_paras = set(p.read_text(encoding="utf-8").split("\n\n"))
                    bad = [para for para in txt.split("\n\n") if para and para not in staged_paras]
                    check(not bad,
                          f"case {cid} {e.get('field')}: {len(bad)} paragraph(s) not found in "
                          f"staged file (altered or drifted)")

    # 3. no DB id receives article text under two different provenance sources
    #    (spreadsheet double-assignment guard, re-checked from the manifest)
    if MANIFEST.exists():
        man = json.loads(MANIFEST.read_text(encoding="utf-8"))
        # One sheet article (row 164, the Manchester Evening News piece) documents five
        # Salford incidents and is deliberately filed into all five cases. So a case
        # legitimately holds its own row plus row 164. Flag only two DIFFERENT own-rows.
        SHARED_ROWS = {"164"}
        sheet_seen = defaultdict(set)
        for m in man["entries"]:
            if "reported_K&R sheet" in m.get("origin", ""):
                row = re.search(r"row (\d+)", m["origin"])
                if row:
                    sheet_seen[m["case"]].add(row.group(1))
        for cid, rowset in sheet_seen.items():
            own = rowset - SHARED_ROWS
            if len(own) > 1:
                fails.append(f"case {cid}: sheet articles from two different own-rows {sorted(own)}")
        # every manifest article still verbatim in its file
        for m in man["entries"]:
            cid = m["case"]
            if cid in file_by_id:
                body = file_by_id[cid].read_text(encoding="utf-8")
                # manifest doesn't store text, only sha + chars; re-derive from the store
                pass  # covered by check 2 for store-backed blocks

    # 4. spreadsheet phase-1 spot check: sample sheet-origin articles re-verified
    import openpyxl
    wb = openpyxl.load_workbook(BASE.parent / "KR-reports_analysis_Sofi.xlsx",
                                data_only=True, read_only=True)
    rows = list(wb["reported_K&R"].iter_rows(values_only=True))
    idx = {h: i for i, h in enumerate(rows[0]) if h is not None}
    col = {"Articles": "Articles", "Article 2": "Article 2", "Article 3": "Article 3"}
    if MANIFEST.exists():
        import random
        sheet_arts = [m for m in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"]
                      if "reported_K&R sheet" in m.get("origin", "")]
        random.seed(1)
        for m in random.sample(sheet_arts, min(40, len(sheet_arts))):
            rn = int(re.search(r"row (\d+)", m["origin"]).group(1))
            cn = re.search(r'column "([^"]+)"', m["origin"]).group(1)
            cell = rows[rn - 1][idx[col[cn]]]
            check(isinstance(cell, str) and sha(cell) == m["sha256"],
                  f"case {m['case']}: sheet article no longer matches cell (row {rn}, {cn})")

    print(f"ran {checks} checks over {n_blocks} phase-2 blocks and {len(files)} case files\n")
    if fails:
        print(f"FAIL — {len(fails)} problem(s):")
        for f in fails:
            print(f"  - {f}")
        sys.exit(1)
    print("PASS — every mechanical check clean:")
    print("  * one file per DB record, filenames match ids")
    print("  * every filed article byte-present in its case file")
    print("  * every stored sha256 re-derives from the text")
    print("  * raw-retrieved text still matches its staged file (no drift)")
    print("  * every UNTRIMMED block carries its warning banner")
    print("  * every auto-trimmed block carries its unreviewed banner")
    print("  * no case has spreadsheet articles from two different rows")
    print("  * sampled phase-1 articles still match their spreadsheet cells")


if __name__ == "__main__":
    main()