SofiaandClaude Opus 4.8
ba7529ea50
Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
92 lines
3.9 KiB
Python
92 lines
3.9 KiB
Python
"""
|
|
Propose furniture lines to drop from phase-1 spreadsheet articles.
|
|
|
|
The sheet articles were copied in verbatim and never trimmed. A subset carries site
|
|
furniture (interior ad markers, share buttons, newsletter/subscribe blocks, related-article
|
|
teasers, copyright footers). This proposes ONLY high-confidence furniture lines, from the
|
|
exact cell text, for human review. Nothing is invented: every proposed drop is a real line
|
|
from the cell, quoted in full.
|
|
|
|
Canonical split is cell.split("\\n") over ALL lines (blanks included) so drop indices line up
|
|
exactly with what the trimmer/build will use. Output -> staging/sheet-cuts.json.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import openpyxl
|
|
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
|
BASE = Path(__file__).resolve().parent
|
|
XLSX = BASE.parent / "KR-reports_analysis_Sofi.xlsx"
|
|
|
|
# Standalone-furniture patterns. A line is furniture only if the token is essentially the
|
|
# whole line (short line), or it is an unambiguous block marker. Body lines that merely
|
|
# mention "Telegram"/"WhatsApp" in a sentence are NOT dropped (length guard handles them).
|
|
STRONG = re.compile(r"^\s*("
|
|
r"advertisement|skip advertisement|sponsored|"
|
|
r"you may (also )?like|related articles?|more from|"
|
|
r"read more|read next|next article|previous article|"
|
|
r"follow us|share this|share on|sign up|subscribe|"
|
|
r"most read|trending|recommended|"
|
|
r"©|copyright|all rights reserved|terms of service|privacy policy"
|
|
r")\b", re.I)
|
|
SHARE = re.compile(r"^\s*(whatsapp|telegram|facebook|twitter|x\b|linkedin|share)"
|
|
r"[\s|/,·-]*(whatsapp|telegram|facebook|twitter|linkedin|email|share|copy)?\s*$", re.I)
|
|
NEWS = re.compile(r"(newsletter|sign[- ]up for|subscribe to our|get the latest|"
|
|
r"delivered (to your inbox|free)|enter your email)", re.I)
|
|
|
|
|
|
def is_furniture(line):
|
|
s = line.strip()
|
|
if not s:
|
|
return None
|
|
if STRONG.search(s) and len(s) < 90:
|
|
return "marker"
|
|
if SHARE.match(s) and len(s) < 60:
|
|
return "share"
|
|
if NEWS.search(s) and len(s) < 120:
|
|
return "newsletter"
|
|
return None
|
|
|
|
|
|
def main():
|
|
cases = [int(a) for a in sys.argv[1:]]
|
|
wb = openpyxl.load_workbook(XLSX, data_only=True, read_only=True)
|
|
rows = list(wb["reported_K&R"].iter_rows(values_only=True))
|
|
idx = {h: i for i, h in enumerate(rows[0]) if h is not None}
|
|
# map case -> (row, col) via the build manifest origins
|
|
man = json.loads((BASE / "case-files-manifest.json").read_text(encoding="utf-8"))
|
|
locs = {}
|
|
for m in man["entries"]:
|
|
o = m.get("origin", "")
|
|
if "reported_K&R sheet" in o:
|
|
rn = int(re.search(r"row (\d+)", o).group(1))
|
|
col = re.search(r'column "([^"]+)"', o).group(1)
|
|
locs.setdefault(m["case"], []).append((rn, col))
|
|
|
|
out = {}
|
|
for cid in cases:
|
|
for rn, col in locs.get(cid, []):
|
|
cell = str(rows[rn - 1][idx[col]])
|
|
lines = cell.split("\n")
|
|
drops = [(i, is_furniture(l), l) for i, l in enumerate(lines) if is_furniture(l)]
|
|
key = f"{cid}:{col}"
|
|
print(f"\n### {key} (row {rn}, {len(lines)} lines) — {len(drops)} proposed drops")
|
|
for i, kind, l in drops:
|
|
ctx = " ".join(lines[i].split())[:100]
|
|
out.setdefault(key, {"row": rn, "col": col, "drop": []})
|
|
out[key]["drop"].append(i)
|
|
print(f" drop [{i}] ({kind}): {ctx}")
|
|
(BASE / "staging" / "sheet-cuts.json").write_text(
|
|
json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"\nwrote {sum(len(v['drop']) for v in out.values())} proposed drops "
|
|
f"across {len(out)} articles -> staging/sheet-cuts.json")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|