SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
74 lines
3.0 KiB
Python
74 lines
3.0 KiB
Python
"""
|
|
Source-coverage reconciliation. One pass over the whole dataset.
|
|
|
|
For every DB case, decide whether it still holds a usable article:
|
|
- a fetched block with a text-bearing verdict (KEEP / PARTIAL / UNTRIMMED), or
|
|
- a phase-1 spreadsheet article (from the build manifest).
|
|
A case with neither is a genuine gap: every fetched block is non-text
|
|
(THIN / BLOCKED / DEAD / NOT AN ARTICLE / SYNDICATION / OFF-CASE) or it has no
|
|
sources at all.
|
|
|
|
Then reconcile against sofia-worklist.md in both directions:
|
|
- every gap must be named on the worklist (report the ones that are missing),
|
|
- anything named on the worklist that now HAS a usable source can come off.
|
|
|
|
Mechanical only; prints a report and exits 0. Nothing is modified.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
BASE = Path(__file__).resolve().parent
|
|
TEXT_VERDICTS = {"KEEP", "PARTIAL", "UNTRIMMED"}
|
|
|
|
|
|
def main():
|
|
recs = json.loads((BASE.parent / "attacks-export-Gart-website.json").read_text(encoding="utf-8"))
|
|
ids = {r["id"] for r in recs}
|
|
victim = {r["id"]: r.get("victim") for r in recs}
|
|
store = json.loads((BASE / "sourced-articles.json").read_text(encoding="utf-8"))
|
|
manifest = json.loads((BASE / "case-files-manifest.json").read_text(encoding="utf-8"))
|
|
|
|
# fetched coverage: any text-bearing block
|
|
fetched_text = {int(cid) for cid, blocks in store.items()
|
|
if any(b.get("verdict") in TEXT_VERDICTS for b in blocks)}
|
|
# sheet coverage: any phase-1 article filed by the build
|
|
sheet = {m["case"] for m in manifest["entries"] if "reported_K&R sheet" in m.get("origin", "")}
|
|
covered = fetched_text | sheet
|
|
gaps = sorted(ids - covered)
|
|
|
|
# what each non-covered case does still carry, for the report
|
|
def why(cid):
|
|
blocks = store.get(str(cid), [])
|
|
if not blocks:
|
|
return "no fetched sources and no spreadsheet article"
|
|
return "only non-text blocks: " + ", ".join(
|
|
f"{b.get('field')}={b.get('verdict')}" for b in blocks)
|
|
|
|
# worklist reconciliation
|
|
wl_path = BASE / "sofia-worklist.md"
|
|
wl = wl_path.read_text(encoding="utf-8") if wl_path.exists() else ""
|
|
wl_ids = {int(n) for n in re.findall(r"\b(?:case\s+)?(\d{2,3})\b", wl)} & ids
|
|
|
|
print(f"{len(ids)} DB cases | {len(covered)} covered "
|
|
f"({len(fetched_text)} fetched-text, {len(sheet)} sheet) | {len(gaps)} gaps\n")
|
|
|
|
print("GENUINE GAPS (no usable source anywhere):")
|
|
for cid in gaps:
|
|
on = "on worklist" if cid in wl_ids else ">>> NOT ON WORKLIST — ADD"
|
|
print(f" {cid:>3} {str(victim[cid])[:34]:34} {on}")
|
|
print(f" {why(cid)}")
|
|
|
|
stale = sorted(c for c in wl_ids if c in covered)
|
|
print(f"\nWORKLIST ENTRIES THAT NOW HAVE A SOURCE (candidates to remove): "
|
|
f"{stale if stale else 'none'}")
|
|
print(" (a worklist mention can be context rather than a gap claim — confirm before removing)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|