StellarCrowandClaude Fable 5.1
1387037367
- pull_live_cases.py: merges https://stats.gart.io/api/attacks into the export. The feed lacks notes/reports/original_date/has_processed_date, so existing records keep their fields; changed values are taken from live (record 188: victim, violence_torture, weapons, summary) and new records are appended with those four fields null. Previous export kept as attacks-export-Gart-website-2026-09-09-before.json; raw feed saved as attacks-live-api-2026-09-09.json. Export now has 389 records. - 25 new dossiers built; 93 linked URL slots fetched (77 fetched, 8 blocked, 4 thin, 4 video/social); 76 blocks auto-trimmed. Wayback retry for the 12 failed slots hit archive.org rate limits (429) and is left for a later run. - Sheet row 252 (the Phuket robbery listed in cases-missing-from-database.json) approved as the source for new case 476, which is that event. - Stale dossier for renamed case 188 removed. - Detention flag check on the new cases: 479 proposed 0->1 (forced entry plus assault), 468 null->0 proposed, 483 and 486 confirmed 0; review file and corrections JSON updated. Worklist and README carry the procedure. verify_all.py PASSES (2776 checks, 389 case files). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
72 lines
2.7 KiB
Python
72 lines
2.7 KiB
Python
"""
|
|
Pull the current case list from stats.gart.io and merge it into the DB export.
|
|
|
|
The live site's /api/attacks is the only public export path. It carries every record
|
|
but not the notes, reports, original_date and has_processed_date fields that the
|
|
original export had, so this merges instead of replacing:
|
|
- records already in the export keep every field; fields the live copy changed
|
|
are updated and listed (created_at is ignored)
|
|
- new records are appended with the four missing fields null
|
|
The previous export is kept beside the new one, dated, so any diff can be re-run.
|
|
The raw API response is saved as attacks-live-api-<date>.json for provenance.
|
|
|
|
Run from merge-output/, then: stage_all.py, wayback_pass.py <new ids>,
|
|
auto_trim.py <new ids>, build_cases.py --all, verify_all.py.
|
|
"""
|
|
|
|
import copy
|
|
import json
|
|
import shutil
|
|
import sys
|
|
import urllib.request
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
EXPORT = BASE.parent / "attacks-export-Gart-website.json"
|
|
API = "https://stats.gart.io/api/attacks"
|
|
MISSING_IN_API = ("notes", "reports", "original_date", "has_processed_date")
|
|
|
|
|
|
def main():
|
|
today = date.today().isoformat()
|
|
raw = urllib.request.urlopen(urllib.request.Request(API, headers={"User-Agent": "Mozilla/5.0"}),
|
|
timeout=120).read()
|
|
(BASE.parent / f"attacks-live-api-{today}.json").write_bytes(raw)
|
|
live = {r["id"]: r for r in json.loads(raw.decode("utf-8"))}
|
|
|
|
old = json.loads(EXPORT.read_text(encoding="utf-8"))
|
|
keys = list(old[0].keys())
|
|
backup = BASE.parent / f"attacks-export-Gart-website-{today}-before.json"
|
|
if "--dry-run" not in sys.argv:
|
|
shutil.copy(EXPORT, backup)
|
|
|
|
merged, changed, new = [], [], []
|
|
for r in old:
|
|
n = copy.deepcopy(r)
|
|
l = live.get(r["id"])
|
|
if l:
|
|
for k in keys:
|
|
if k in l and k != "created_at" and str(r.get(k) or "").strip() != str(l.get(k) or "").strip():
|
|
n[k] = l[k]
|
|
changed.append((r["id"], k))
|
|
merged.append(n)
|
|
for i in sorted(set(live) - {r["id"] for r in old}):
|
|
n = {k: live[i].get(k) for k in keys}
|
|
for k in MISSING_IN_API:
|
|
n[k] = 0 if k == "has_processed_date" else None
|
|
merged.append(n)
|
|
new.append(i)
|
|
|
|
print(f"live records: {len(live)} | export before: {len(old)} | after: {len(merged)}")
|
|
print(f"changed fields on existing records: {changed}")
|
|
print(f"new ids: {new}")
|
|
if "--dry-run" in sys.argv:
|
|
return
|
|
EXPORT.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"export written; previous copy kept as {backup.name}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|