"""
Pull the current case list from stats.gart.io and merge it into the DB export.

The live site's /api/attacks is the only public export path. It carries every record
but not the notes, reports, original_date and has_processed_date fields that the
original export had, so this merges instead of replacing:
  - records already in the export keep every field; fields the live copy changed
    are updated and listed (created_at is ignored)
  - new records are appended with the four missing fields null
The previous export is kept beside the new one, dated, so any diff can be re-run.
The raw API response is saved as attacks-live-api-<date>.json for provenance.

Run from merge-output/, then: stage_all.py, wayback_pass.py <new ids>,
auto_trim.py <new ids>, build_cases.py --all, verify_all.py.
"""

import copy
import json
import shutil
import sys
import urllib.request
from datetime import date
from pathlib import Path

BASE = Path(__file__).resolve().parent
EXPORT = BASE.parent / "attacks-export-Gart-website.json"
API = "https://stats.gart.io/api/attacks"
MISSING_IN_API = ("notes", "reports", "original_date", "has_processed_date")


def main():
    today = date.today().isoformat()
    raw = urllib.request.urlopen(urllib.request.Request(API, headers={"User-Agent": "Mozilla/5.0"}),
                                 timeout=120).read()
    (BASE.parent / f"attacks-live-api-{today}.json").write_bytes(raw)
    live = {r["id"]: r for r in json.loads(raw.decode("utf-8"))}

    old = json.loads(EXPORT.read_text(encoding="utf-8"))
    keys = list(old[0].keys())
    backup = BASE.parent / f"attacks-export-Gart-website-{today}-before.json"
    if "--dry-run" not in sys.argv:
        shutil.copy(EXPORT, backup)

    merged, changed, new = [], [], []
    for r in old:
        n = copy.deepcopy(r)
        l = live.get(r["id"])
        if l:
            for k in keys:
                if k in l and k != "created_at" and str(r.get(k) or "").strip() != str(l.get(k) or "").strip():
                    n[k] = l[k]
                    changed.append((r["id"], k))
        merged.append(n)
    for i in sorted(set(live) - {r["id"] for r in old}):
        n = {k: live[i].get(k) for k in keys}
        for k in MISSING_IN_API:
            n[k] = 0 if k == "has_processed_date" else None
        merged.append(n)
        new.append(i)

    print(f"live records: {len(live)} | export before: {len(old)} | after: {len(merged)}")
    print(f"changed fields on existing records: {changed}")
    print(f"new ids: {new}")
    if "--dry-run" in sys.argv:
        return
    EXPORT.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"export written; previous copy kept as {backup.name}")


if __name__ == "__main__":
    main()