"""
Regenerate Sofia's manual fetch list from the article store.

Lists every linked source the pipeline could not read, per case, ordered by how much
else the case has: cases with NO usable text first. Verdicts: DEAD = 404/gone (no
archive copy either), BLOCKED = the site refused an automated request (open it in a
browser), THIN = the page needs a browser to render, ERROR = connection failure,
PDF = an archived copy exists but is a PDF. Slots that the Wayback pass already
recovered are not listed. Rewrites the top of sofia-worklist.md between the markers.
"""

import json
import re
from pathlib import Path

BASE = Path(__file__).resolve().parent
JSONF = BASE.parent / "attacks-export-Gart-website.json"
STORE = BASE / "sourced-articles.json"
MANIFEST = BASE / "case-files-manifest.json"
WORKLIST = BASE / "sofia-worklist.md"
FAILED = ("BLOCKED", "THIN", "DEAD", "ERROR")


def main():
    recs = {r["id"]: r for r in json.loads(JSONF.read_text(encoding="utf-8"))}
    store = json.loads(STORE.read_text(encoding="utf-8"))
    sheet = {e["case"] for e in json.loads(MANIFEST.read_text(encoding="utf-8"))["entries"]
             if "sheet" in e.get("origin", "")}
    groups = {}
    for cid, r in recs.items():
        blocks = store.get(str(cid), [])
        have = sum(1 for b in blocks if b.get("verdict") in ("KEEP", "PARTIAL", "UNTRIMMED")) + (1 if cid in sheet else 0)
        for b in blocks:
            if b.get("verdict") not in FAILED or not b.get("url"):
                continue
            verdict = b["verdict"]
            note = b.get("archive_note") or ""
            if "PDF" in note:
                verdict = "PDF"
            groups.setdefault(have, []).append((cid, verdict, r, b["url"], b.get("archive_url")))
    lines = ["<!-- fetch-list:start (generated by make_fetch_list.py; do not edit by hand) -->",
             "## Manual fetch list (regenerated from the store)", "",
             "Every linked source the pipeline could not read, after the live fetch and the Wayback pass.",
             "Open each in your browser, copy the article text, and paste it back with the case number.",
             "Cases at the top have NO usable text at all; further down the missing source is a nice-to-have.",
             "DEAD = gone and not archived. BLOCKED = site refused the script, opens in a browser. THIN = needs",
             "a browser to render. PDF = an archived PDF exists at the archive link.", ""]
    total = 0
    for have in sorted(groups):
        title = "NO USABLE TEXT — priority" if have == 0 else f"has {have} other source(s) on file"
        lines += [f"### {title}", ""]
        for cid, verdict, r, url, arch in sorted(groups[have], key=lambda x: -x[0]):
            lines.append(f"- **case {cid}** `{verdict}` · {r.get('date')} · {str(r.get('victim'))[:60]} · {r.get('country')}")
            lines.append(f"  <{url}>" + (f"  archive: <{arch}>" if arch else ""))
            total += 1
        lines.append("")
    lines.append(f"{total} sources listed.")
    lines.append("<!-- fetch-list:end -->")
    block = "\n".join(lines)

    wl = WORKLIST.read_text(encoding="utf-8")
    if "<!-- fetch-list:start" in wl:
        wl = re.sub(r"<!-- fetch-list:start.*?<!-- fetch-list:end -->", block, wl, flags=re.S)
    else:
        # replace the original hand-written list (from the intro up to the first '## Special')
        cut = wl.index("## Special:")
        wl = wl[:wl.index("## NO OTHER SOURCE")] + block + "\n\n" + wl[cut:]
    WORKLIST.write_text(wl, encoding="utf-8")
    print(f"{total} failed sources across {sum(len(v) for v in groups.values())} slots; "
          f"priority cases: {sorted({x[0] for x in groups.get(0, [])})}")


if __name__ == "__main__":
    main()