feat(cases): pull 25 new cases (463-487) from stats.gart.io; dossiers, sources, trim
- pull_live_cases.py: merges https://stats.gart.io/api/attacks into the export. The feed lacks notes/reports/original_date/has_processed_date, so existing records keep their fields; changed values are taken from live (record 188: victim, violence_torture, weapons, summary) and new records are appended with those four fields null. Previous export kept as attacks-export-Gart-website-2026-09-09-before.json; raw feed saved as attacks-live-api-2026-09-09.json. Export now has 389 records. - 25 new dossiers built; 93 linked URL slots fetched (77 fetched, 8 blocked, 4 thin, 4 video/social); 76 blocks auto-trimmed. Wayback retry for the 12 failed slots hit archive.org rate limits (429) and is left for a later run. - Sheet row 252 (the Phuket robbery listed in cases-missing-from-database.json) approved as the source for new case 476, which is that event. - Stale dossier for renamed case 188 removed. - Detention flag check on the new cases: 479 proposed 0->1 (forced entry plus assault), 468 null->0 proposed, 483 and 486 confirmed 0; review file and corrections JSON updated. Worklist and README carry the procedure. verify_all.py PASSES (2776 checks, 389 case files). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GZZENdTLzNGsbNy4DyF1yt
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
93cf7ae53a
commit
1387037367
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Pull the current case list from stats.gart.io and merge it into the DB export.
|
||||
|
||||
The live site's /api/attacks is the only public export path. It carries every record
|
||||
but not the notes, reports, original_date and has_processed_date fields that the
|
||||
original export had, so this merges instead of replacing:
|
||||
- records already in the export keep every field; fields the live copy changed
|
||||
are updated and listed (created_at is ignored)
|
||||
- new records are appended with the four missing fields null
|
||||
The previous export is kept beside the new one, dated, so any diff can be re-run.
|
||||
The raw API response is saved as attacks-live-api-<date>.json for provenance.
|
||||
|
||||
Run from merge-output/, then: stage_all.py, wayback_pass.py <new ids>,
|
||||
auto_trim.py <new ids>, build_cases.py --all, verify_all.py.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path(__file__).resolve().parent
|
||||
EXPORT = BASE.parent / "attacks-export-Gart-website.json"
|
||||
API = "https://stats.gart.io/api/attacks"
|
||||
MISSING_IN_API = ("notes", "reports", "original_date", "has_processed_date")
|
||||
|
||||
|
||||
def main():
|
||||
today = date.today().isoformat()
|
||||
raw = urllib.request.urlopen(urllib.request.Request(API, headers={"User-Agent": "Mozilla/5.0"}),
|
||||
timeout=120).read()
|
||||
(BASE.parent / f"attacks-live-api-{today}.json").write_bytes(raw)
|
||||
live = {r["id"]: r for r in json.loads(raw.decode("utf-8"))}
|
||||
|
||||
old = json.loads(EXPORT.read_text(encoding="utf-8"))
|
||||
keys = list(old[0].keys())
|
||||
backup = BASE.parent / f"attacks-export-Gart-website-{today}-before.json"
|
||||
if "--dry-run" not in sys.argv:
|
||||
shutil.copy(EXPORT, backup)
|
||||
|
||||
merged, changed, new = [], [], []
|
||||
for r in old:
|
||||
n = copy.deepcopy(r)
|
||||
l = live.get(r["id"])
|
||||
if l:
|
||||
for k in keys:
|
||||
if k in l and k != "created_at" and str(r.get(k) or "").strip() != str(l.get(k) or "").strip():
|
||||
n[k] = l[k]
|
||||
changed.append((r["id"], k))
|
||||
merged.append(n)
|
||||
for i in sorted(set(live) - {r["id"] for r in old}):
|
||||
n = {k: live[i].get(k) for k in keys}
|
||||
for k in MISSING_IN_API:
|
||||
n[k] = 0 if k == "has_processed_date" else None
|
||||
merged.append(n)
|
||||
new.append(i)
|
||||
|
||||
print(f"live records: {len(live)} | export before: {len(old)} | after: {len(merged)}")
|
||||
print(f"changed fields on existing records: {changed}")
|
||||
print(f"new ids: {new}")
|
||||
if "--dry-run" in sys.argv:
|
||||
return
|
||||
EXPORT.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"export written; previous copy kept as {backup.name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user