K&R database cleanup: handoff bundle

Self-contained bundle to continue the case-summary enrichment pass (70/256 done). sources/ holds the 364 .txt dossiers; scripts use relative paths. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018FJRciSWZc9HftS2edbzBf
2026-09-02 19:49:46 -03:00
commit ba7529ea50
735 changed files with 90597 additions and 0 deletions
@@ -0,0 +1,91 @@
+"""
+Propose furniture lines to drop from phase-1 spreadsheet articles.
+
+The sheet articles were copied in verbatim and never trimmed. A subset carries site
+furniture (interior ad markers, share buttons, newsletter/subscribe blocks, related-article
+teasers, copyright footers). This proposes ONLY high-confidence furniture lines, from the
+exact cell text, for human review. Nothing is invented: every proposed drop is a real line
+from the cell, quoted in full.
+
+Canonical split is cell.split("\\n") over ALL lines (blanks included) so drop indices line up
+exactly with what the trimmer/build will use. Output -> staging/sheet-cuts.json.
+"""
+
+import io
+import json
+import re
+import sys
+from pathlib import Path
+
+import openpyxl
+
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
+BASE = Path(__file__).resolve().parent
+XLSX = BASE.parent / "KR-reports_analysis_Sofi.xlsx"
+
+# Standalone-furniture patterns. A line is furniture only if the token is essentially the
+# whole line (short line), or it is an unambiguous block marker. Body lines that merely
+# mention "Telegram"/"WhatsApp" in a sentence are NOT dropped (length guard handles them).
+STRONG = re.compile(r"^\s*("
+                    r"advertisement|skip advertisement|sponsored|"
+                    r"you may (also )?like|related articles?|more from|"
+                    r"read more|read next|next article|previous article|"
+                    r"follow us|share this|share on|sign up|subscribe|"
+                    r"most read|trending|recommended|"
+                    r"©|copyright|all rights reserved|terms of service|privacy policy"
+                    r")\b", re.I)
+SHARE = re.compile(r"^\s*(whatsapp|telegram|facebook|twitter|x\b|linkedin|share)"
+                   r"[\s|/,·-]*(whatsapp|telegram|facebook|twitter|linkedin|email|share|copy)?\s*$", re.I)
+NEWS = re.compile(r"(newsletter|sign[- ]up for|subscribe to our|get the latest|"
+                  r"delivered (to your inbox|free)|enter your email)", re.I)
+
+
+def is_furniture(line):
+    s = line.strip()
+    if not s:
+        return None
+    if STRONG.search(s) and len(s) < 90:
+        return "marker"
+    if SHARE.match(s) and len(s) < 60:
+        return "share"
+    if NEWS.search(s) and len(s) < 120:
+        return "newsletter"
+    return None
+
+
+def main():
+    cases = [int(a) for a in sys.argv[1:]]
+    wb = openpyxl.load_workbook(XLSX, data_only=True, read_only=True)
+    rows = list(wb["reported_K&R"].iter_rows(values_only=True))
+    idx = {h: i for i, h in enumerate(rows[0]) if h is not None}
+    # map case -> (row, col) via the build manifest origins
+    man = json.loads((BASE / "case-files-manifest.json").read_text(encoding="utf-8"))
+    locs = {}
+    for m in man["entries"]:
+        o = m.get("origin", "")
+        if "reported_K&R sheet" in o:
+            rn = int(re.search(r"row (\d+)", o).group(1))
+            col = re.search(r'column "([^"]+)"', o).group(1)
+            locs.setdefault(m["case"], []).append((rn, col))
+
+    out = {}
+    for cid in cases:
+        for rn, col in locs.get(cid, []):
+            cell = str(rows[rn - 1][idx[col]])
+            lines = cell.split("\n")
+            drops = [(i, is_furniture(l), l) for i, l in enumerate(lines) if is_furniture(l)]
+            key = f"{cid}:{col}"
+            print(f"\n### {key}  (row {rn}, {len(lines)} lines) — {len(drops)} proposed drops")
+            for i, kind, l in drops:
+                ctx = " ".join(lines[i].split())[:100]
+                out.setdefault(key, {"row": rn, "col": col, "drop": []})
+                out[key]["drop"].append(i)
+                print(f"  drop [{i}] ({kind}): {ctx}")
+    (BASE / "staging" / "sheet-cuts.json").write_text(
+        json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"\nwrote {sum(len(v['drop']) for v in out.values())} proposed drops "
+          f"across {len(out)} articles -> staging/sheet-cuts.json")
+
+
+if __name__ == "__main__":
+    main()