"""
File staged extra_* sources (found by web search, NOT on the DB record) into the article
store as UNTRIMMED blocks, so build_cases.py renders them and auto_trim.py can cut them.

Reads staging/<case>/index.json; every extra_N entry that is not yet in the store becomes
a block. FETCHED entries carry the staged text, hashed; anything else (THIN, BLOCKED, DEAD,
NOT AN ARTICLE) is filed as a reference with its verdict. Each block says in its note that
the URL is not on the record and when it was found, so provenance stays visible.

Usage: python add_extra_sources.py 258 260 ...   then auto_trim.py <cases>, build, verify.
"""

import hashlib
import json
import sys
from datetime import date
from pathlib import Path

BASE = Path(__file__).resolve().parent
STAGING = BASE / "staging"
STORE = BASE / "sourced-articles.json"

NAMES = {"document.no": "Document.no", "alt.no": "Avisa Oslo (alt.no)", "nrk.no": "NRK", "ao.no": "Avisa Oslo",
         "mothership.sg": "Mothership", "khaosodenglish.com": "Khaosod English", "8newsnow.com": "8 News Now (KLAS)",
         "thedefiant.io": "The Defiant", "actu17.fr": "Actu17", "lagazette-yvelines.fr": "La Gazette en Yvelines",
         "mantes-actu.net": "Mantes Actu", "vrt.be": "VRT NWS", "nnieuws.be": "NNieuws", "indegazette.be": "In de Gazette",
         "justice.gov": "US Department of Justice", "decrypt.co": "Decrypt", "fortune.com": "Fortune",
         "ukrinform.ua": "Ukrinform", "hromadske.radio": "Hromadske Radio", "sud.ua": "Sud.ua", "atn.ua": "ATN"}


def main():
    cases = [int(a) for a in sys.argv[1:]]
    store = json.loads(STORE.read_text(encoding="utf-8"))
    today = date.today().isoformat()
    added = 0
    for cid in cases:
        idx_path = STAGING / f"{cid:03d}" / "index.json"
        if not idx_path.exists():
            print(f"case {cid}: no staging index"); continue
        blocks = store.setdefault(str(cid), [])
        have = {b.get("field") for b in blocks}
        for e in json.loads(idx_path.read_text(encoding="utf-8")):
            if not e["field"].startswith("extra_") or e["field"] in have:
                continue
            src = NAMES.get(e["host"], e["host"])
            note = f"NOT on the DB record. Found {today} by web search because the linked source(s) gave no usable text."
            if e["verdict"] != "FETCHED":
                blocks.append({"field": e["field"], "verdict": e["verdict"], "url": e["url"], "source": src,
                               "note": note + (" " + e["note"] if e.get("note") else "")})
                added += 1
                continue
            path = next(STAGING.joinpath(f"{cid:03d}").glob(f"{e['n']:02d}_*.txt"))
            text = path.read_text(encoding="utf-8")
            blocks.append({"field": e["field"], "verdict": "UNTRIMMED", "url": e["url"], "source": src,
                           "language": "unknown", "text": text, "chars": len(text),
                           "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
                           "staged_file": path.name,
                           "paragraphs_kept": f"ALL {len(text.split(chr(10) * 2))} paragraphs, untrimmed",
                           "method": "raw HTTP retrieval, deterministic extraction",
                           "fetched_at": e.get("fetched_at"), "http": e.get("http"), "note": note,
                           "cut_note": "NOT TRIMMED. Full page extract; trim on read-through."})
            added += 1
            print(f"case {cid}: + {e['field']} {src} {len(text):,} chars")
    STORE.write_text(json.dumps(store, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"added {added} block(s)")


if __name__ == "__main__":
    main()