From 8cb4f8766fbd9c1db3e3c3a24ce5e6c62cec5021 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:30:59 +0200 Subject: [PATCH 1/9] Add do-no-harm extraction gate for web capture (issue #2 Phase 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Web URLs that yield no usable text (login walls, JS-only SPAs, dead newsletter links) were silently classified into junk artifacts: capture fell back to feeding the bare URL string / nav cruft to the LLM. Centralise fetch + a confidence gate in lib/webextract and make capture refuse a low-confidence extraction with an actionable message instead of polluting the vault. Verified against the issue #2 examples: refuses the newsletter pollution cases (LinkedIn too-short, AlphaSignal fetch-fail, Substack index JS-wall) while passing genuine articles. Digest-shape pages (ethical.institute) still pass — that is Phase 1's archetype-routing job, not an extraction failure. Co-Authored-By: Claude Opus 4.8 --- infra/smoke.sh | 2 + jobs/capture.py | 39 +++++++------- lib/webextract.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 lib/webextract.py diff --git a/infra/smoke.sh b/infra/smoke.sh index c28dc98..754fab4 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -61,6 +61,7 @@ check "lib.llm imports" uv run python -c "from lib import llm" check "lib.embeddings imports" uv run python -c "from lib import embeddings" check "lib.github imports" uv run python -c "from lib import github" check "lib.body imports" uv run python -c "from lib import body" +check "lib.webextract imports" uv run python -c "from lib import webextract" check "all jobs import" uv run python -c "from jobs import capture, bulk_capture, enrich, digest, search, backfill, reevaluate, seed, cli" echo "[lib smoke tests]" @@ -68,6 +69,7 @@ check "lib.vault smoke" uv run python -m lib.vault check_skip_ok "lib.embeddings smoke" uv run python -m lib.embeddings check_skip_ok "lib.llm smoke" uv run python -m lib.llm check "lib.github smoke (uses network)" uv run python -m lib.github +check_skip_ok "lib.webextract smoke (uses network)" uv run python -m lib.webextract echo "[CLI wiring]" check "raidar --help" uv run raidar --help diff --git a/jobs/capture.py b/jobs/capture.py index 845f070..8f84935 100644 --- a/jobs/capture.py +++ b/jobs/capture.py @@ -19,10 +19,9 @@ from datetime import date, datetime, timezone from typing import Any -import trafilatura import typer -from lib import config, vault +from lib import config, vault, webextract from lib.embeddings import Index from lib.body import render_artifact, render_artifact_summary_table, render_concept from lib.github import ( @@ -181,16 +180,6 @@ def _truncate(text: str, limit: int = _CONTENT_LIMIT) -> str: return text[:limit] + f"\n\n[...truncated at {limit} chars]" -def _fetch_web_text(url: str) -> str | None: - raw = trafilatura.fetch_url(url) - if not raw: - return None - extracted = trafilatura.extract(raw) - if not extracted or len(extracted.strip()) < 100: - return None - return extracted - - # --------------------------------------------------------------------------- # Concept context builder # --------------------------------------------------------------------------- @@ -486,10 +475,17 @@ def _capture_one( elif _is_url(input_str): source = "web" log.info("fetching web URL %s", input_str) - web_text = _fetch_web_text(input_str) + min_chars = int(cfg.thresholds.get("min_web_extract_chars", webextract.MIN_USABLE_CHARS)) + web_text, reason = webextract.fetch_main_text(input_str, min_chars=min_chars) if web_text is None: - log.warning("falling back to text mode for %s", input_str) - source = "text" + # Doctrine: a skipped capture beats a polluted vault. Never fall back + # to classifying the bare URL string — that fabricates a junk artifact + # from a login wall, JS-only page, or dead newsletter link. + raise CaptureSkipped( + f"could not extract usable content from {input_str} ({reason}); " + "refusing to capture. If this is a newsletter or JS/login-walled " + "page, paste the per-article link instead." + ) else: source = "text" @@ -758,8 +754,17 @@ def capture( parsed_repo = None snapshot = fetch_repo(*parsed_repo) if parsed_repo else None readme = fetch_readme(*parsed_repo) if parsed_repo else None - web_text = _fetch_web_text(input) if _is_url(input) and not parsed_repo else None - source = "github" if parsed_repo else ("web" if _is_url(input) and web_text else "text") + web_text = None + web_reason = "" + if _is_url(input) and not parsed_repo: + min_chars = int(cfg.thresholds.get("min_web_extract_chars", webextract.MIN_USABLE_CHARS)) + web_text, web_reason = webextract.fetch_main_text(input, min_chars=min_chars) + if web_text is None: + print( + f"NOTE: a real capture would be REFUSED — could not extract usable " + f"content ({web_reason}).\n" + ) + source = "github" if parsed_repo else ("web" if web_text else "text") try: context_md = cfg.context_path.read_text(encoding="utf-8") except FileNotFoundError: diff --git a/lib/webextract.py b/lib/webextract.py new file mode 100644 index 0000000..208cdc5 --- /dev/null +++ b/lib/webextract.py @@ -0,0 +1,126 @@ +"""Web extraction with a confidence gate. + +raidar captures arbitrary web URLs by running them through trafilatura. Some +pages return little or no usable text: auth walls (LinkedIn newsletters), +JS-only SPAs (Substack archive/index pages), or per-recipient newsletter email +links that 404/403. Feeding that thin output to the classifier fabricates junk +artifacts/concepts and silently pollutes the vault — which violates the project +doctrine: *a skipped capture beats a polluted knowledge base.* + +This module centralises the fetch + a quality gate so capture (and, later, +bulk-capture) can refuse a low-confidence extraction instead of degrading to +classifying nav cruft or the bare URL string. + +`assess()` is pure (no network) and is the unit the smoke test covers. +`fetch_main_text()` wraps trafilatura and applies the gate. +""" + +from __future__ import annotations + +import logging + +import trafilatura + +log = logging.getLogger(__name__) + +# A real article extracts to thousands of characters; the observed pollution +# cases were 281 chars (LinkedIn login-wall nav) and 1035 chars (Substack index +# title soup). A 500-char floor clears genuine short posts — the smallest real +# article in the issue #2 sample extracted to ~4.7k — while rejecting nav/teaser +# cruft. Callers may override via thresholds.min_web_extract_chars. +MIN_USABLE_CHARS = 500 + +# Phrases that betray a JS-only or login-walled page whose "extraction" is really +# boilerplate chrome. Only treated as fatal when the extraction is ALSO short: a +# long article that merely mentions JavaScript or "sign in" must not be rejected. +_WALL_SENTINELS = ( + "this site requires javascript", + "please enable javascript", + "turn on javascript", + "enable javascript to", + "javascript is required", + "you must be logged in", + "sign in to continue", + "log in to continue", +) +_WALL_MAX_CHARS = 3000 + + +def assess(text: str | None, *, min_chars: int = MIN_USABLE_CHARS) -> tuple[bool, str]: + """Judge whether extracted text is usable as artifact content. + + Returns ``(usable, reason)``. ``reason`` is a short tag with context: + ``"ok"``, ``"empty"``, ``"too-short (...)"`` or ``"js-or-login-wall (...)"``. + """ + if not text or not text.strip(): + return False, "empty" + stripped = text.strip() + n = len(stripped) + if n < min_chars: + return False, f"too-short ({n} chars < {min_chars})" + if n < _WALL_MAX_CHARS: + low = stripped.lower() + hit = next((s for s in _WALL_SENTINELS if s in low), None) + if hit is not None: + return False, f"js-or-login-wall (matched {hit!r}, only {n} chars)" + return True, "ok" + + +def fetch_main_text( + url: str, *, min_chars: int = MIN_USABLE_CHARS +) -> tuple[str | None, str]: + """Fetch ``url`` and return ``(clean_text, reason)``. + + ``clean_text`` is None when the page could not be fetched or the extraction + is not usable (see :func:`assess`); ``reason`` explains why so callers can + give the user an actionable message. Never raises on network errors. + """ + try: + raw = trafilatura.fetch_url(url) + except Exception as exc: # noqa: BLE001 — network/parse errors must not crash capture + log.warning("fetch failed for %s: %s", url, exc) + return None, f"fetch-failed ({exc})" + if not raw: + return None, "fetch-failed (no response / blocked)" + extracted = trafilatura.extract(raw) + usable, reason = assess(extracted, min_chars=min_chars) + return (extracted if usable else None), reason + + +def _smoke() -> int: + # --- offline: assess() on representative strings (the real failure cases) --- + assert assess("x" * 600)[0], "long clean text should be usable" + assert not assess("")[0], "empty -> unusable" + assert not assess(None)[0], "None -> unusable" + assert not assess(" ")[0], "whitespace-only -> unusable" + # C2: LinkedIn login-wall nav (281 chars in the wild). + assert not assess("Skip to main content\nArtificial Engineering\n" * 4)[0], ( + "short nav cruft -> unusable" + ) + # E1: Substack index — above the length floor but a JS wall. + js_wall = ("Latest Top Discussions " * 30) + " This site requires JavaScript to run correctly. Please turn on JavaScript" + assert not assess(js_wall)[0], "short JS-wall page -> unusable" + # A long article that merely mentions JS must still pass. + long_js = ("Real substantive article body. " * 300) + " (this site requires javascript for comments)" + assert assess(long_js)[0], "long article mentioning JS should remain usable" + print("assess() offline checks passed.") + + # --- online (best-effort): a known-good article should fetch + pass; treat + # any network failure as a skip so the offline smoke run stays green. --- + try: + text, reason = fetch_main_text("https://ghuntley.com/ralph/") + except Exception as exc: # noqa: BLE001 + print(f"live fetch skipped (unreachable: {exc})") + return 0 + if text is None: + print(f"live fetch skipped (could not fetch sample: {reason})") + return 0 + assert len(text) > 1000, f"expected a substantial article body, got {len(text)}" + print(f"live fetch ok ({len(text)} chars, reason={reason}).") + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(_smoke()) From fb2bb23780a6886b7eb7792e8357cbf867548173 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:38:26 +0200 Subject: [PATCH 2/9] Route newsletters/digests away from single-artifact capture (issue #2 Phase 1) capture no longer writes a junk artifact for multi-item pages: - pre-fetch URL hints for known newsletter landing/index/email links (LinkedIn, AlphaSignal, Substack archive) that point at the per-article URL; - a new LLM page_kind field (article|digest|index|other), biased toward 'article', that refuses web 'digest'/'index' pages after classification. Verified live: ethical.institute -> digest, ghuntley/ralph -> article, anthropic opus -> article, refactoring.fm/s/essays -> index. bulk-capture gains an awesome-list shape gate (repo count + owner spread + diversity, config-driven via thresholds.bulk_*) so it stops ingesting newsletter publisher footers and incidental partner links; --force overrides. Verified: the 6-link ethical.institute newsletter and the 1-link Gemma announcement are refused, a 40-repo diverse list passes, a single-org dump is refused. Co-Authored-By: Claude Opus 4.8 --- jobs/bulk_capture.py | 51 +++++++++++++++++++++++++++++++++++++++++- jobs/capture.py | 46 ++++++++++++++++++++++++++++++++------ lib/webextract.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 8 deletions(-) diff --git a/jobs/bulk_capture.py b/jobs/bulk_capture.py index 6750490..8058989 100644 --- a/jobs/bulk_capture.py +++ b/jobs/bulk_capture.py @@ -59,6 +59,37 @@ def _extract_github_slugs(text: str) -> list[str]: return result +def _link_list_shape(slugs: list[str]) -> tuple[int, int, float]: + """Return (n_repos, n_owners, owner_diversity) for a set of owner/name slugs.""" + n_repos = len(slugs) + owners = {s.split("/", 1)[0].lower() for s in slugs} + n_owners = len(owners) + diversity = (n_owners / n_repos) if n_repos else 0.0 + return n_repos, n_owners, diversity + + +def _looks_like_awesome_list( + slugs: list[str], *, min_repos: int, min_owners: int, min_diversity: float +) -> tuple[bool, str]: + """Decide whether a page's GitHub links look like a curated awesome-list. + + bulk-capture is correct for awesome-lists (links ARE the content) and harmful + for newsletters/announcements (the links are a publisher self-promo footer or + an incidental partner mention — verified on the issue #2 examples). The robust + separators are repo COUNT (newsletter footers carry a handful; lists carry + dozens) and owner spread (a single org's repo dump is not a curated list). + """ + n_repos, n_owners, diversity = _link_list_shape(slugs) + stats = f"{n_repos} repo link(s) from {n_owners} owner(s), diversity {diversity:.2f}" + if n_repos < min_repos: + return False, f"{stats} — too few links to be an awesome-list (need ≥{min_repos})" + if n_owners < min_owners: + return False, f"{stats} — too few distinct owners (need ≥{min_owners})" + if diversity < min_diversity: + return False, f"{stats} — one owner dominates (need diversity ≥{min_diversity})" + return True, stats + + def _fetch_source_text(url: str) -> str | None: """Return raw text for slug extraction. @@ -86,7 +117,7 @@ def bulk( url: Annotated[str, typer.Argument(help="GitHub awesome-list URL or any web page.")], dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be captured without writing."), limit: int | None = typer.Option(None, "--limit", help="Max repos to process (default: all)."), - force: bool = typer.Option(False, "--force", help="Bypass soft dedup on each item."), + force: bool = typer.Option(False, "--force", help="Bypass the awesome-list shape gate and soft dedup on each item."), ) -> None: """Bulk-capture repos extracted from an awesome-list or web page.""" cfg = config.load() @@ -103,6 +134,24 @@ def bulk( print("No GitHub repo links found in the page.") raise typer.Exit(code=0) + # Shape gate: only blanket-capture pages that look like a curated awesome-list. + # Newsletters/announcements carry a few publisher-owned or incidental links that + # would pollute the vault if ingested wholesale. --force overrides. + ok, stats = _looks_like_awesome_list( + slugs, + min_repos=int(cfg.thresholds.get("bulk_min_repos", 10)), + min_owners=int(cfg.thresholds.get("bulk_min_owners", 3)), + min_diversity=float(cfg.thresholds.get("bulk_min_owner_diversity", 0.3)), + ) + if not ok and not force: + print(f"Refusing to bulk-capture: {stats}.") + print( + "bulk-capture is for awesome-list-shaped pages. For a newsletter or " + "single article, capture items individually (`raidar capture `), " + "or pass --force to override." + ) + raise typer.Exit(code=0) + if limit is not None: slugs = slugs[:limit] diff --git a/jobs/capture.py b/jobs/capture.py index 8f84935..a8d7660 100644 --- a/jobs/capture.py +++ b/jobs/capture.py @@ -56,6 +56,7 @@ _EVALUATIONS = ["new", "promising", "recommended", "deprecated", "hype"] _RELATIONSHIPS = ["introduces", "implements", "extends", "applies", "discusses"] _CONCEPT_STATUSES = ["emerging", "watch", "invest"] +_PAGE_KINDS = ["article", "digest", "index", "other"] _SCHEMA: dict[str, Any] = { "type": "object", @@ -72,6 +73,7 @@ "concept_label", "is_new_concept", "relationship", + "page_kind", ], "properties": { "artifact_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9_-]*$"}, @@ -85,6 +87,7 @@ "concept_label": {"type": "string"}, "is_new_concept": {"type": "boolean"}, "relationship": {"type": "string", "enum": _RELATIONSHIPS}, + "page_kind": {"type": "string", "enum": _PAGE_KINDS}, "concept_what_it_is": {"type": ["string", "null"]}, "concept_why_it_matters": {"type": ["string", "null"]}, "review_needed": {"type": "boolean"}, @@ -106,6 +109,13 @@ "'hype'=thin community, self-reported benchmarks, not safe to depend on.\n" "what_it_is: 2-3 sentences describing what this thing is.\n" "evaluation_rationale: 2-3 sentences explaining the evaluation, citing concrete evidence.\n" + "page_kind: the SHAPE of the source. 'article'=a single self-contained post/paper/" + "announcement/repo about ONE thing (the DEFAULT — a post that merely cites several " + "links is still an article). 'digest'=a newsletter/roundup whose primary purpose is to " + "link out to MANY separate, unrelated items. 'index'=an archive or listing page (a table " + "of contents of posts), not content itself. 'other'=free-text note or anything else. " + "Bias toward 'article'; only say 'digest'/'index' when the page is clearly a multi-item " + "container, because those will NOT be captured as a single artifact.\n" "\n" "CONCEPT MAPPING:\n" "A concept is a CAPABILITY or APPROACH category that several artifacts could implement as " @@ -474,6 +484,11 @@ def _capture_one( readme = fetch_readme(owner, name) elif _is_url(input_str): source = "web" + # Refuse known newsletter landing/index/email URLs pre-fetch, with a hint + # pointing at the per-article link that *is* capturable. + hint = webextract.newsletter_url_hint(input_str) + if hint: + raise CaptureSkipped(f"refusing to capture {input_str}: {hint}") log.info("fetching web URL %s", input_str) min_chars = int(cfg.thresholds.get("min_web_extract_chars", webextract.MIN_USABLE_CHARS)) web_text, reason = webextract.fetch_main_text(input_str, min_chars=min_chars) @@ -553,6 +568,19 @@ def _capture_one( if parsed_repo is not None: parsed["github_repo"] = f"{parsed_repo[0]}/{parsed_repo[1]}" + # ----- 4b. archetype gate (web pages only) ---------------------------- + # A newsletter/roundup ('digest') is N independent items across N concepts; + # an archive listing ('index') is navigation. Mapping either to ONE artifact + + # ONE concept forces a doctrine-violating umbrella concept or drops most of the + # content. Refuse rather than pollute — the per-item links are the real artifacts. + if source == "web" and parsed.get("page_kind") in ("digest", "index"): + kind = parsed["page_kind"] + raise CaptureSkipped( + f"{input_str} looks like a {kind} (newsletter/archive), not a single " + "article — refusing to capture it as one artifact. Capture the individual " + "items instead; for a Substack, paste the per-article /p/ link." + ) + # ----- 5. ID collision handling --------------------------------------- artifact_id = _unique_artifact_id(parsed["artifact_id"]) parsed["artifact_id"] = artifact_id @@ -757,13 +785,17 @@ def capture( web_text = None web_reason = "" if _is_url(input) and not parsed_repo: - min_chars = int(cfg.thresholds.get("min_web_extract_chars", webextract.MIN_USABLE_CHARS)) - web_text, web_reason = webextract.fetch_main_text(input, min_chars=min_chars) - if web_text is None: - print( - f"NOTE: a real capture would be REFUSED — could not extract usable " - f"content ({web_reason}).\n" - ) + hint = webextract.newsletter_url_hint(input) + if hint: + print(f"NOTE: a real capture would be REFUSED pre-fetch — {hint}\n") + else: + min_chars = int(cfg.thresholds.get("min_web_extract_chars", webextract.MIN_USABLE_CHARS)) + web_text, web_reason = webextract.fetch_main_text(input, min_chars=min_chars) + if web_text is None: + print( + f"NOTE: a real capture would be REFUSED — could not extract usable " + f"content ({web_reason}).\n" + ) source = "github" if parsed_repo else ("web" if web_text else "text") try: context_md = cfg.context_path.read_text(encoding="utf-8") diff --git a/lib/webextract.py b/lib/webextract.py index 208cdc5..50a6dea 100644 --- a/lib/webextract.py +++ b/lib/webextract.py @@ -18,6 +18,7 @@ from __future__ import annotations import logging +import re import trafilatura @@ -66,6 +67,40 @@ def assess(text: str | None, *, min_chars: int = MIN_USABLE_CHARS) -> tuple[bool return True, "ok" +# Known newsletter-platform URL shapes that are landing/index/email pages, not a +# single article. Detected pre-fetch so we can refuse with an actionable hint +# (most also fail the extraction gate, but the generic "too-short" message is +# less useful than telling the user which per-article URL to paste). Substack +# custom domains (e.g. refactoring.fm) can't be spotted from the URL alone — the +# extraction gate catches their JS-gated index pages instead. +_NEWSLETTER_URL_HINTS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"linkedin\.com/newsletters/", re.I), + "LinkedIn newsletter landing pages are auth-walled. Paste the individual " + "article URL instead (linkedin.com/pulse/).", + ), + ( + re.compile(r"alphasignal\.ai/email/", re.I), + "AlphaSignal per-email links are not publicly fetchable. Use the Substack " + "mirror (alphasignalai.substack.com) and paste a per-article /p/ link.", + ), + ( + re.compile(r"substack\.com/(?:archive|s/)", re.I), + "This is a Substack archive/index, not a single article. Paste a " + "per-article /p/ link instead.", + ), +) + + +def newsletter_url_hint(url: str) -> str | None: + """Return an actionable hint if `url` is a known newsletter landing/index/email + page (not a single article), else None. Pure; used to refuse pre-fetch.""" + for pattern, msg in _NEWSLETTER_URL_HINTS: + if pattern.search(url): + return msg + return None + + def fetch_main_text( url: str, *, min_chars: int = MIN_USABLE_CHARS ) -> tuple[str | None, str]: @@ -105,6 +140,24 @@ def _smoke() -> int: assert assess(long_js)[0], "long article mentioning JS should remain usable" print("assess() offline checks passed.") + # --- offline: newsletter_url_hint() on the issue #2 platform examples --- + assert newsletter_url_hint( + "https://www.linkedin.com/newsletters/7324756381780140032/?displayConfirmation=true" + ), "LinkedIn newsletter landing should be hinted" + assert newsletter_url_hint("https://alphasignal.ai/email/305ad305be75cdf5"), ( + "AlphaSignal email link should be hinted" + ) + assert newsletter_url_hint("https://foo.substack.com/archive?sort=new"), ( + "Substack archive should be hinted" + ) + assert newsletter_url_hint("https://ghuntley.com/ralph/") is None, ( + "a real article URL must NOT be hinted" + ) + assert newsletter_url_hint("https://refactoring.fm/p/monday") is None, ( + "a Substack per-article /p/ URL must NOT be hinted" + ) + print("newsletter_url_hint() offline checks passed.") + # --- online (best-effort): a known-good article should fetch + pass; treat # any network failure as a skip so the offline smoke run stays green. --- try: From 88aa4ad4133ce35c60c926ab68c91b2eb9f750db Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:44:39 +0200 Subject: [PATCH 3/9] Add Substack newsletter expansion + provenance (issue #2 Phase 2a) Manual capture of a Substack landing/archive URL now discovers the per-article permalinks via the unauthenticated /api/v1/posts API (works for *.substack.com and custom domains like refactoring.fm) and either lists them (default, so the user/agent picks) or captures them all (--expand-all). A per-post /p/ URL and ordinary articles on arbitrary domains are never expanded. Captured items now carry provenance: source_url (the page captured) and via (the newsletter that surfaced it). Verified live against refactoring.fm in a sandbox vault: expansion lists /p/ permalinks, --expand-all captures with both provenance fields written. Co-Authored-By: Claude Opus 4.8 --- infra/smoke.sh | 2 + jobs/capture.py | 91 ++++++++++++++++++++++++- lib/substack.py | 172 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 lib/substack.py diff --git a/infra/smoke.sh b/infra/smoke.sh index 754fab4..7001eb8 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -62,6 +62,7 @@ check "lib.embeddings imports" uv run python -c "from lib import embeddings" check "lib.github imports" uv run python -c "from lib import github" check "lib.body imports" uv run python -c "from lib import body" check "lib.webextract imports" uv run python -c "from lib import webextract" +check "lib.substack imports" uv run python -c "from lib import substack" check "all jobs import" uv run python -c "from jobs import capture, bulk_capture, enrich, digest, search, backfill, reevaluate, seed, cli" echo "[lib smoke tests]" @@ -70,6 +71,7 @@ check_skip_ok "lib.embeddings smoke" uv run python -m lib.embeddings check_skip_ok "lib.llm smoke" uv run python -m lib.llm check "lib.github smoke (uses network)" uv run python -m lib.github check_skip_ok "lib.webextract smoke (uses network)" uv run python -m lib.webextract +check_skip_ok "lib.substack smoke (uses network)" uv run python -m lib.substack echo "[CLI wiring]" check "raidar --help" uv run raidar --help diff --git a/jobs/capture.py b/jobs/capture.py index a8d7660..73c8f24 100644 --- a/jobs/capture.py +++ b/jobs/capture.py @@ -460,8 +460,13 @@ def _capture_one( force: bool = False, today: str | None = None, backfill: bool = False, + via: str | None = None, ) -> str: - """Capture a single input. Returns the artifact_id.""" + """Capture a single input. Returns the artifact_id. + + `via` records provenance — the newsletter/source URL that surfaced this item — + stamped on the artifact frontmatter so recall can trace where it came from. + """ today = today or date.today().isoformat() # ----- 1. classify input shape ---------------------------------------- @@ -673,6 +678,11 @@ def _capture_one( } if parsed.get("github_repo"): artifact_fm["source_url"] = f"https://github.com/{parsed['github_repo']}" + elif source == "web": + artifact_fm["source_url"] = input_str + if via: + # Provenance: the newsletter/source that surfaced this item. + artifact_fm["via"] = via artifact = Artifact(id=artifact_id, frontmatter=artifact_fm, body=artifact_body) write_artifact(artifact) @@ -751,6 +761,63 @@ def _capture_one( return artifact_id +# --------------------------------------------------------------------------- +# Substack expansion (Mode A: manual capture of a newsletter URL) +# --------------------------------------------------------------------------- + + +def _expand_substack( + base: str, + cfg, + *, + today: str, + force: bool, + dry_run: bool, + expand_all: bool, + limit: int, + backfill: bool, +) -> None: + """A pasted Substack landing/archive URL is a CONTAINER of posts, not one + artifact. Discover the per-post permalinks and either list them (default, so + the user/agent picks) or capture them all (--expand-all). Each captured post + carries `via=base` provenance. + """ + from lib import substack + + posts = substack.list_posts(base, limit=limit) + if not posts: + print(f"No posts found via the Substack API at {base}.") + return + + print(f"Substack publication: {base} — {len(posts)} recent post(s).") + if dry_run or not expand_all: + for i, p in enumerate(posts, 1): + print(f" {i:2d}. {p.post_date or ' '} {p.title}") + print(f" {p.url}") + if dry_run: + print("\n(dry run) --expand-all would capture the posts above.") + else: + print( + "\nCapture one with `raidar capture `, or all of the above with " + "`raidar capture --expand-all `." + ) + return + + captured = skipped = failed = 0 + for p in posts: + try: + aid = _capture_one(p.url, cfg, force=force, today=today, backfill=backfill, via=base) + print(f" ✓ {aid} ← {p.url}") + captured += 1 + except CaptureSkipped as exc: + log.debug("expand skip %s: %s", p.url, exc) + skipped += 1 + except Exception as exc: # noqa: BLE001 + print(f" ERROR {p.url}: {exc}", file=sys.stderr) + failed += 1 + print(f"\nDone: {captured} captured, {skipped} skipped, {failed} failed.") + + # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- @@ -769,12 +836,34 @@ def capture( "--backfill/--no-backfill", help="Automatically backfill star history for captured GitHub repos.", ), + expand_all: bool = typer.Option( + False, + "--expand-all", + help="If the URL is a Substack newsletter, capture ALL discovered posts " + "(default: just list them so you can pick).", + ), + expand_limit: int = typer.Option( + 20, "--expand-limit", help="Max Substack posts to discover/expand.", + ), ) -> None: """Capture an artifact from a URL or free-text note.""" cfg = config.load() setup_logging(level=cfg.log_level, log_file=cfg.log_file) today = date.today().isoformat() + # Substack expansion (Mode A): a pasted newsletter landing/archive URL is a + # container of posts, not one artifact — discover per-post permalinks instead. + if _is_url(input) and parse_repo_url(input) is None: + from lib import substack + if substack.is_expandable_candidate(input): + base = substack.resolve_pub(input) + if base: + _expand_substack( + base, cfg, today=today, force=force, dry_run=dry_run, + expand_all=expand_all, limit=expand_limit, backfill=backfill, + ) + return + if dry_run: parsed_repo = parse_repo_url(input) if _is_url(input) or "/" in input else None if parsed_repo is not None and not _is_url(input): diff --git a/lib/substack.py b/lib/substack.py new file mode 100644 index 0000000..034f56c --- /dev/null +++ b/lib/substack.py @@ -0,0 +1,172 @@ +"""Substack discovery: turn a newsletter landing/archive URL into the per-article +permalinks raidar can actually capture. + +Substack publications — both the default ``*.substack.com`` and custom domains +like ``refactoring.fm`` — expose an UNAUTHENTICATED JSON API:: + + GET https://{pub}/api/v1/posts?limit=N&offset=M + +returning post objects with ``canonical_url`` / ``title`` / ``slug`` / +``post_date``. The landing, ``/archive`` and ``/s/
`` pages are JS-gated +and extract to title-soup, but the API and the per-post ``/p/`` pages are +fully server-rendered. So we discover per-post URLs via the API and hand each to +the normal capture pipeline — no headless browser, no auth. + +``is_post_url`` / ``is_expandable_candidate`` are pure (smoke-tested offline); +``resolve_pub`` / ``list_posts`` hit the network. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from urllib.parse import urlparse + +import httpx + +log = logging.getLogger(__name__) + +_API_PATH = "/api/v1/posts" +_TIMEOUT_S = 20.0 +_UA = "raidar/0.1 (+https://github.com/janikmu/raidar)" + + +@dataclass(frozen=True) +class Post: + title: str + url: str # canonical per-post permalink + slug: str + post_date: str # ISO date (YYYY-MM-DD) or "" + + +def _host_base(url: str) -> str | None: + p = urlparse(url) + if not p.scheme or not p.netloc: + return None + return f"{p.scheme}://{p.netloc}" + + +def is_post_url(url: str) -> bool: + """True if the URL already points at a single Substack post (``/p/``).""" + return "/p/" in urlparse(url).path + + +def is_expandable_candidate(url: str) -> bool: + """Cheap, network-free pre-check: does this URL look like a Substack + publication landing/archive/section page worth probing for expansion? + + Deliberately narrow so a normal article URL on an arbitrary domain (e.g. + ``ghuntley.com/ralph/``) is never probed. A per-post ``/p/`` URL is + NOT expandable — it is captured directly. + """ + p = urlparse(url) + if not p.netloc or is_post_url(url): + return False + path = p.path.rstrip("/") + return path in ("", "/archive") or path.startswith("/s/") + + +def _fetch_posts_json(base: str, *, limit: int, offset: int) -> list | None: + api = f"{base}{_API_PATH}?limit={limit}&offset={offset}" + try: + r = httpx.get(api, headers={"User-Agent": _UA}, timeout=_TIMEOUT_S, follow_redirects=True) + except Exception as exc: # noqa: BLE001 — network errors are expected, not fatal + log.debug("substack api fetch failed for %s: %s", base, exc) + return None + if r.status_code != 200: + return None + try: + data = r.json() + except Exception: # noqa: BLE001 + return None + return data if isinstance(data, list) else None + + +def _looks_like_posts(data: list) -> bool: + return bool(data) and isinstance(data[0], dict) and ( + "canonical_url" in data[0] or "slug" in data[0] + ) + + +def resolve_pub(url: str) -> str | None: + """Return the publication base (``https://host``) if `url` belongs to a + Substack publication, else None. + + Fast-path: any ``*.substack.com`` host. Otherwise probe the API once, which + confirms custom-domain publications (refactoring.fm, alphasignalai mirror, …). + """ + base = _host_base(url) + if base is None: + return None + host = urlparse(base).netloc.lower() + if host.endswith(".substack.com"): + return base + data = _fetch_posts_json(base, limit=1, offset=0) + if data is not None and _looks_like_posts(data): + return base + return None + + +def list_posts(base: str, *, limit: int = 20) -> list[Post]: + """Return up to `limit` recent posts for a publication base URL, newest first.""" + out: list[Post] = [] + offset = 0 + while len(out) < limit: + want = min(50, limit - len(out)) + data = _fetch_posts_json(base, limit=want, offset=offset) + if not data: + break + for d in data: + slug = d.get("slug") or "" + url = d.get("canonical_url") or (f"{base}/p/{slug}" if slug else "") + if not url: + continue + out.append( + Post( + title=(d.get("title") or "").strip() or "(untitled)", + url=url, + slug=slug, + post_date=(d.get("post_date") or "")[:10], + ) + ) + if len(data) < want: + break + offset += len(data) + return out[:limit] + + +def _smoke() -> int: + # --- offline: pure helpers --- + assert is_post_url("https://refactoring.fm/p/monday") + assert not is_post_url("https://refactoring.fm/s/essays") + assert is_expandable_candidate("https://refactoring.fm/s/essays") + assert is_expandable_candidate("https://foo.substack.com/") + assert is_expandable_candidate("https://foo.substack.com/archive") + assert not is_expandable_candidate("https://refactoring.fm/p/monday"), "per-post is not expandable" + assert not is_expandable_candidate("https://ghuntley.com/ralph/"), "arbitrary article is not a candidate" + assert _looks_like_posts([{"canonical_url": "x", "slug": "y"}]) + assert not _looks_like_posts([]) + print("substack offline checks passed.") + + # --- online (best-effort): discover refactoring.fm posts; skip on failure --- + try: + base = resolve_pub("https://refactoring.fm/s/essays") + except Exception as exc: # noqa: BLE001 + print(f"live discovery skipped (unreachable: {exc})") + return 0 + if not base: + print("live discovery skipped (could not resolve publication)") + return 0 + posts = list_posts(base, limit=5) + if not posts: + print("live discovery skipped (no posts returned)") + return 0 + assert all("/p/" in p.url for p in posts), "every discovered URL should be a /p/ permalink" + print(f"live discovery ok ({len(posts)} posts; newest: {posts[0].title!r} {posts[0].url})") + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(_smoke()) From da612efe6e82b50860f88f0f6dd5eb5439313c83 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:43:13 +0200 Subject: [PATCH 4/9] Add newsletter subscriptions + recurring poll (issue #2 Phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mode B: `raidar newsletter {subscribe,unsubscribe,list,poll}`. subscribe registers a Substack publication (custom domains included) and baselines its current posts so only NEW editions are captured later; --catch-up N also grabs the latest N now. poll is the recurring entry point — it captures every post not yet seen across subscriptions, stamping `via` provenance on each. Cadence- and form-agnostic by design: "new" is decided by canonical post URL (never URL arithmetic or an assumed schedule), so weekly/monthly/irregular newsletters are all handled by one fixed-clock poll. The registry is a small JSON list beside the active config (tool state, not a vault concept/artifact) with a capped seen-set. Only Substack is pollable today; other newsletter forms are refused at subscribe time rather than stored as something we can't honor. Verified in a sandbox vault: --catch-up captures the latest and baselines the window; poll reports 0 new when caught up and exactly the new edition when one appears. Co-Authored-By: Claude Opus 4.8 --- infra/smoke.sh | 6 +- jobs/cli.py | 2 + jobs/newsletter.py | 194 +++++++++++++++++++++++++++++++++++++++++++ lib/subscriptions.py | 144 ++++++++++++++++++++++++++++++++ 4 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 jobs/newsletter.py create mode 100644 lib/subscriptions.py diff --git a/infra/smoke.sh b/infra/smoke.sh index 7001eb8..40b65ae 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -63,7 +63,8 @@ check "lib.github imports" uv run python -c "from lib import github" check "lib.body imports" uv run python -c "from lib import body" check "lib.webextract imports" uv run python -c "from lib import webextract" check "lib.substack imports" uv run python -c "from lib import substack" -check "all jobs import" uv run python -c "from jobs import capture, bulk_capture, enrich, digest, search, backfill, reevaluate, seed, cli" +check "lib.subscriptions imports" uv run python -c "from lib import subscriptions" +check "all jobs import" uv run python -c "from jobs import capture, bulk_capture, enrich, digest, search, backfill, reevaluate, seed, newsletter, cli" echo "[lib smoke tests]" check "lib.vault smoke" uv run python -m lib.vault @@ -72,6 +73,7 @@ check_skip_ok "lib.llm smoke" uv run python -m lib.llm check "lib.github smoke (uses network)" uv run python -m lib.github check_skip_ok "lib.webextract smoke (uses network)" uv run python -m lib.webextract check_skip_ok "lib.substack smoke (uses network)" uv run python -m lib.substack +check "lib.subscriptions smoke" uv run python -m lib.subscriptions echo "[CLI wiring]" check "raidar --help" uv run raidar --help @@ -88,6 +90,8 @@ check "merge-concept --help" uv run python -m jobs.merge --help check "rename-concept --help" uv run python -m jobs.rename --help check "reindex --help" uv run python -m jobs.reindex --help check "install-launchd --help" uv run python -m jobs.launchd --help +check "newsletter --help" uv run python -m jobs.newsletter --help +check "newsletter poll --help" uv run python -m jobs.newsletter poll --help check "search list-concepts against empty vault" uv run python -m jobs.search list-concepts check "health against empty vault" uv run python -m jobs.health diff --git a/jobs/cli.py b/jobs/cli.py index e8cb414..6eaf804 100644 --- a/jobs/cli.py +++ b/jobs/cli.py @@ -59,6 +59,7 @@ from jobs.rename import rename_concept as _rename_cmd # noqa: E402 from jobs.reindex import reindex as _reindex_cmd # noqa: E402 from jobs.launchd import install_launchd as _install_launchd_cmd # noqa: E402 +from jobs.newsletter import app as _newsletter_app # noqa: E402 app.command("capture")(capture) app.command("bulk-capture")(bulk) @@ -73,6 +74,7 @@ app.command("reindex")(_reindex_cmd) app.command("install-launchd")(_install_launchd_cmd) app.add_typer(_search_app, name="search") +app.add_typer(_newsletter_app, name="newsletter") # --------------------------------------------------------------------------- # init diff --git a/jobs/newsletter.py b/jobs/newsletter.py new file mode 100644 index 0000000..e8a1ab8 --- /dev/null +++ b/jobs/newsletter.py @@ -0,0 +1,194 @@ +"""Newsletter subscriptions (Mode B: recurring auto-capture). + + raidar newsletter subscribe [--catch-up N] [--dry-run] + raidar newsletter unsubscribe + raidar newsletter list + raidar newsletter poll [--limit N] [--only URL] [--dry-run] + +`poll` is the recurring entry point (run it from launchd/cron). It captures every +post not yet seen for each subscription — so it is cadence-agnostic: a weekly, +monthly, or irregular newsletter is all handled by the same fixed-clock poll, and +"new" is decided by canonical post URL, never by URL arithmetic or a schedule. + +Only Substack publications can be subscribed today (custom domains included). Other +newsletter forms (LinkedIn, RSS, generic) are refused at subscribe time rather than +stored as something the poller cannot faithfully honor. +""" + +from __future__ import annotations + +import logging +import sys +from datetime import date + +import typer + +from lib import config, substack, subscriptions +from lib.logging_setup import setup as setup_logging +from jobs.capture import CaptureSkipped, _capture_one + +log = logging.getLogger(__name__) + +app = typer.Typer(add_completion=False, help=__doc__) + + +def _capture_posts(posts: list[substack.Post], via: str, cfg, today: str) -> int: + """Capture each post via the normal pipeline, stamping `via` provenance. + Returns the number successfully captured.""" + n = 0 + for p in posts: + try: + aid = _capture_one(p.url, cfg, today=today, backfill=False, via=via) + print(f" ✓ {aid} ← {p.url}") + n += 1 + except CaptureSkipped as exc: + log.debug("subscription capture skip %s: %s", p.url, exc) + except Exception as exc: # noqa: BLE001 + print(f" ERROR {p.url}: {exc}", file=sys.stderr) + return n + + +@app.command() +def subscribe( + url: str = typer.Argument(..., help="Newsletter URL (Substack landing/archive/post)."), + catch_up: int = typer.Option( + 0, "--catch-up", + help="Capture the latest N editions now. Default 0 = baseline only " + "(mark current posts seen and watch forward).", + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Show what would happen, write nothing."), +) -> None: + """Subscribe a newsletter for recurring auto-capture via `newsletter poll`.""" + cfg = config.load() + setup_logging(level=cfg.log_level, log_file=cfg.log_file) + today = date.today().isoformat() + + base = substack.resolve_pub(url) + if not base: + print( + "Not a recognized Substack publication. Only Substack newsletters can be " + "subscribed right now — LinkedIn, RSS and generic newsletters are future work.\n" + "(For a one-off, capture an individual article URL directly with `raidar capture`.)", + file=sys.stderr, + ) + raise typer.Exit(code=1) + + if subscriptions.find(base): + print(f"Already subscribed: {base}") + raise typer.Exit(code=0) + + posts = substack.list_posts(base, limit=max(catch_up, 20)) + sub = subscriptions.Subscription(url=base, kind="substack", added=today) + + if dry_run: + action = f"capture the latest {catch_up} now" if catch_up else "mark current posts as baseline" + print(f"(dry run) would subscribe to {base} ({len(posts)} recent posts) and {action}.") + return + + # Baseline EVERY currently-listed post as seen so future polls capture only + # NEW editions, never the backlog. --catch-up additionally captures the latest N now. + if catch_up > 0: + to_capture = posts[:catch_up] + print(f"Subscribing to {base}; capturing latest {len(to_capture)} edition(s):") + _capture_posts(to_capture, base, cfg, today) + subscriptions.mark_seen(sub, [p.url for p in posts]) + if catch_up > 0: + print( + f"Subscribed to {base}; captured latest {min(catch_up, len(posts))}, " + f"baselined {len(posts)} current post(s). Future editions: `raidar newsletter poll`." + ) + else: + print( + f"Subscribed to {base}. {len(posts)} current post(s) marked as baseline — " + "future editions are captured by `raidar newsletter poll`." + ) + subscriptions.add(sub) + + +@app.command() +def unsubscribe( + url: str = typer.Argument(..., help="Subscription URL (as shown by `newsletter list`)."), +) -> None: + """Remove a subscription (does not delete already-captured artifacts).""" + if subscriptions.remove(url): + print(f"Unsubscribed: {url}") + return + base = substack.resolve_pub(url) + if base and subscriptions.remove(base): + print(f"Unsubscribed: {base}") + return + print(f"Not subscribed: {url}", file=sys.stderr) + raise typer.Exit(code=1) + + +@app.command("list") +def list_subs() -> None: + """List current newsletter subscriptions.""" + subs = subscriptions.load() + if not subs: + print("No subscriptions. Add one with `raidar newsletter subscribe `.") + return + for s in subs: + print( + f"- {s.url} [{s.kind}] added={s.added} " + f"last_run={s.last_run or '-'} seen={len(s.seen)}" + ) + + +@app.command() +def poll( + limit: int = typer.Option(20, "--limit", help="Max recent posts to inspect per subscription."), + only: str = typer.Option(None, "--only", help="Poll just this subscription URL."), + dry_run: bool = typer.Option(False, "--dry-run", help="Show new posts, capture nothing."), +) -> None: + """Capture every not-yet-seen post across subscriptions (the recurring entry point).""" + cfg = config.load() + setup_logging(level=cfg.log_level, log_file=cfg.log_file) + today = date.today().isoformat() + + subs = subscriptions.load() + if only: + subs = [s for s in subs if s.url == only] + if not subs: + print("No subscriptions to poll." if not only else f"No subscription matching {only}.") + return + + total_new = 0 + for sub in subs: + if sub.kind != "substack": + print(f"- {sub.url}: kind '{sub.kind}' is not pollable yet — skipping.") + continue + try: + posts = substack.list_posts(sub.url, limit=limit) + except Exception as exc: # noqa: BLE001 + print(f"- {sub.url}: discovery failed ({exc}) — skipping.", file=sys.stderr) + continue + + seen = set(sub.seen) + new = [p for p in posts if p.url not in seen] + if not new: + print(f"- {sub.url}: no new posts.") + if not dry_run: + sub.last_run = today + subscriptions.update(sub) + continue + + if dry_run: + print(f"- {sub.url}: {len(new)} new post(s) (dry run):") + for p in new: + print(f" {p.post_date or ' '} {p.title}\n {p.url}") + continue + + print(f"- {sub.url}: {len(new)} new post(s):") + captured = _capture_posts(new, sub.url, cfg, today) + subscriptions.mark_seen(sub, [p.url for p in new]) + sub.last_run = today + subscriptions.update(sub) + total_new += captured + + if not dry_run: + print(f"\nDone: {total_new} new artifact(s) across {len(subs)} subscription(s).") + + +if __name__ == "__main__": + app() diff --git a/lib/subscriptions.py b/lib/subscriptions.py new file mode 100644 index 0000000..f748ffa --- /dev/null +++ b/lib/subscriptions.py @@ -0,0 +1,144 @@ +"""Newsletter subscription registry (Mode B: recurring auto-capture). + +A small JSON list stored next to the active config — deliberately NOT a vault +concept/artifact (it is tool state, like .env, not knowledge). Each entry records +a newsletter source and the per-post URLs already captured, so a recurring poll +captures only genuinely new editions. + +Design notes (newsletters vary a lot): +- "New" is decided by *have we seen this canonical post URL*, never by URL + arithmetic (slugs don't increment) or by an assumed schedule. +- Cadence is not modelled: the poll runs on a fixed clock and captures whatever + is unseen; a monthly or irregular newsletter just yields nothing most polls. +- ``kind`` tags the source so the poller can dispatch per form. Only ``substack`` + is supported today; other kinds are reserved for future work and must not be + added until they can be faithfully polled. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from lib import config + +log = logging.getLogger(__name__) + +# We only ever rediscover *recent* posts (the API returns newest-first, capped), +# so the seen-set never needs the full archive — cap its growth well above any +# realistic per-poll discovery window. +_SEEN_CAP = 500 + +KINDS = ("substack",) # forms we can faithfully poll today + + +def registry_path() -> Path: + """Location of the subscriptions registry (sibling of the active config).""" + return config.get_config_path().parent / "subscriptions.json" + + +@dataclass +class Subscription: + url: str # resolved publication base (the canonical source key) + kind: str # one of KINDS + added: str # ISO date + last_run: str | None = None + seen: list[str] = field(default_factory=list) # canonical post URLs already captured + + +def _read() -> list[Subscription]: + p = registry_path() + if not p.exists(): + return [] + try: + raw = json.loads(p.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + log.warning("could not parse %s (%s); treating as empty", p, exc) + return [] + out: list[Subscription] = [] + for d in raw.get("subscriptions", []): + try: + out.append(Subscription(**d)) + except TypeError as exc: # tolerate forward/backward schema drift + log.warning("skipping malformed subscription %r (%s)", d, exc) + return out + + +def _write(subs: list[Subscription]) -> None: + p = registry_path() + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text( + json.dumps({"subscriptions": [asdict(s) for s in subs]}, indent=2) + "\n", + encoding="utf-8", + ) + + +def load() -> list[Subscription]: + return _read() + + +def find(url: str) -> Subscription | None: + return next((s for s in _read() if s.url == url), None) + + +def add(sub: Subscription) -> None: + subs = _read() + if any(s.url == sub.url for s in subs): + raise ValueError(f"already subscribed: {sub.url}") + subs.append(sub) + _write(subs) + + +def remove(url: str) -> bool: + subs = _read() + kept = [s for s in subs if s.url != url] + if len(kept) == len(subs): + return False + _write(kept) + return True + + +def update(sub: Subscription) -> None: + """Persist changes to an existing subscription (matched by url).""" + subs = _read() + for i, s in enumerate(subs): + if s.url == sub.url: + subs[i] = sub + _write(subs) + return + raise ValueError(f"not subscribed: {sub.url}") + + +def mark_seen(sub: Subscription, urls: list[str]) -> None: + """Record `urls` as captured on `sub` (in-memory; caller persists via update).""" + existing = set(sub.seen) + for u in urls: + if u not in existing: + sub.seen.append(u) + existing.add(u) + if len(sub.seen) > _SEEN_CAP: + sub.seen = sub.seen[-_SEEN_CAP:] + + +def _smoke() -> int: + # Pure, offline: exercise the dataclass + seen-set semantics without touching + # the real registry file. + s = Subscription(url="https://x.example", kind="substack", added="2026-01-01") + mark_seen(s, ["https://x.example/p/a", "https://x.example/p/b", "https://x.example/p/a"]) + assert s.seen == ["https://x.example/p/a", "https://x.example/p/b"], "dedup on mark_seen" + mark_seen(s, ["https://x.example/p/b"]) + assert s.seen.count("https://x.example/p/b") == 1, "no duplicate re-add" + # round-trip through dict (the on-disk shape) + rt = Subscription(**asdict(s)) + assert rt == s + print("subscriptions offline checks passed.") + print(f"(registry would live at: {registry_path()})") + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(_smoke()) From 5cb52ab4dae6492223c4f6769e617d9fcd22b07e Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:47:13 +0200 Subject: [PATCH 5/9] Schedule weekly newsletter poll via launchd (issue #2 Phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a com.airadar.newsletters launchd agent that runs `raidar newsletter poll` Sundays at 19:00 — before enrich (20:00) and digest (21:00) — so new editions land in the vault ahead of the weekly passes. One fixed schedule suffices for all newsletter cadences since poll captures whatever is unseen. install_launchd.sh now maps each plist label to its subcommand (handling the two-token `newsletter poll`) and templates all three agents; launchd.py and the health launchd check pick up the new agent automatically, so `raidar health` nudges existing installs to reinstall and gain it. Verified: the generated plist validates and its ProgramArguments are three separate strings under bash. Co-Authored-By: Claude Opus 4.8 --- infra/install_launchd.sh | 34 ++++++++------ infra/launchd/com.airadar.newsletters.plist | 49 +++++++++++++++++++++ infra/smoke.sh | 1 + jobs/health.py | 5 ++- jobs/launchd.py | 11 +++-- 5 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 infra/launchd/com.airadar.newsletters.plist diff --git a/infra/install_launchd.sh b/infra/install_launchd.sh index c9c7e55..fb62d6e 100755 --- a/infra/install_launchd.sh +++ b/infra/install_launchd.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Install AI Radar launchd agents (enrich + digest). +# Install AI Radar launchd agents (newsletters + enrich + digest). # # Safeguard: checks if the OS is macOS (Darwin). # Detects if a global `raidar` binary is available in PATH. If so, templates @@ -27,7 +27,7 @@ TOOL_DIR="$(cd "$(dirname "$0")/.." && pwd)" UV_PATH="$(command -v uv || true)" RAIDAR_PATH="$(command -v raidar || true)" LA_DIR="$HOME/Library/LaunchAgents" -PLISTS=("com.airadar.enrich.plist" "com.airadar.digest.plist") +PLISTS=("com.airadar.enrich.plist" "com.airadar.digest.plist" "com.airadar.newsletters.plist") if [[ -z "${UV_PATH}" ]]; then echo "error: uv not found in PATH" >&2 @@ -78,19 +78,25 @@ install() { label="${plist%.plist}" echo "installing ${label}" - # Build execution arguments - if [[ "${label}" == "com.airadar.enrich" ]]; then - if [[ -n "${RAIDAR_PATH}" ]]; then - exec_args=" ${RAIDAR_PATH}\n enrich" - else - exec_args=" ${UV_PATH}\n run\n raidar\n enrich" - fi + # Map the plist label to its raidar subcommand(s). + case "${label}" in + com.airadar.enrich) sub="enrich" ;; + com.airadar.digest) sub="digest" ;; + com.airadar.newsletters) sub="newsletter poll" ;; + *) echo "error: unknown plist label ${label}" >&2; exit 1 ;; + esac + + # One per subcommand token (handles multi-token like "newsletter poll"). + sub_strings="" + for tok in ${sub}; do + sub_strings="${sub_strings}\n ${tok}" + done + + # Build execution arguments: prefer the global raidar, else uv-run fallback. + if [[ -n "${RAIDAR_PATH}" ]]; then + exec_args=" ${RAIDAR_PATH}${sub_strings}" else - if [[ -n "${RAIDAR_PATH}" ]]; then - exec_args=" ${RAIDAR_PATH}\n digest" - else - exec_args=" ${UV_PATH}\n run\n raidar\n digest" - fi + exec_args=" ${UV_PATH}\n run\n raidar${sub_strings}" fi # Bootout any existing instance first so updates take effect. diff --git a/infra/launchd/com.airadar.newsletters.plist b/infra/launchd/com.airadar.newsletters.plist new file mode 100644 index 0000000..824294f --- /dev/null +++ b/infra/launchd/com.airadar.newsletters.plist @@ -0,0 +1,49 @@ + + + + + + Label + com.airadar.newsletters + + ProgramArguments + + __PROGRAM_ARGUMENTS__ + + + WorkingDirectory + __WORKING_DIRECTORY__ + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + + + StartCalendarInterval + + Weekday + 0 + Hour + 19 + Minute + 0 + + + RunAtLoad + + + StandardOutPath + __LOG_DIR__/launchd.newsletters.out.log + StandardErrorPath + __LOG_DIR__/launchd.newsletters.err.log + + diff --git a/infra/smoke.sh b/infra/smoke.sh index 40b65ae..1695488 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -101,6 +101,7 @@ check "capture --dry-run on free text" \ echo "[infra]" check "install_launchd.sh syntax" bash -n infra/install_launchd.sh +check "newsletters.plist validates" plutil -lint infra/launchd/com.airadar.newsletters.plist check "enrich.plist validates" plutil -lint infra/launchd/com.airadar.enrich.plist check "digest.plist validates" plutil -lint infra/launchd/com.airadar.digest.plist diff --git a/jobs/health.py b/jobs/health.py index 33cc781..f5955ce 100644 --- a/jobs/health.py +++ b/jobs/health.py @@ -341,8 +341,9 @@ def check_orphan_signals(v: _VaultView, cfg: config_module.Config) -> list[Findi def check_launchd_agents() -> list[Finding]: - """macOS only: flag scheduled enrich/digest agents that were never installed - (or were lost — e.g. after a fresh machine setup that skipped that step).""" + """macOS only: flag scheduled agents (newsletter poll / enrich / digest) that + were never installed (or were lost — e.g. after a fresh machine setup that + skipped that step, or after an upgrade that added a new agent).""" import platform if platform.system() != "Darwin": return [] diff --git a/jobs/launchd.py b/jobs/launchd.py index 69db496..0e1866d 100644 --- a/jobs/launchd.py +++ b/jobs/launchd.py @@ -1,4 +1,5 @@ -"""Install/remove the macOS launchd agents that run enrich + digest on schedule. +"""Install/remove the macOS launchd agents that run newsletter poll + enrich + +digest on schedule. Thin wrapper around infra/install_launchd.sh — that script stays the source of truth (and what infra/smoke.sh lints), this just makes it reachable from the @@ -20,7 +21,11 @@ app = typer.Typer(add_completion=False, help=__doc__) -_PLISTS = ("com.airadar.enrich.plist", "com.airadar.digest.plist") +_PLISTS = ( + "com.airadar.newsletters.plist", + "com.airadar.enrich.plist", + "com.airadar.digest.plist", +) EXPECTED_LABELS = tuple(p[: -len(".plist")] for p in _PLISTS) @@ -63,7 +68,7 @@ def install_launchd( False, "--uninstall", help="Unload and remove the agents instead of installing them.", ), ) -> None: - """Install (or remove) the macOS launchd agents for enrich + digest.""" + """Install (or remove) the macOS launchd agents for newsletter poll + enrich + digest.""" if platform.system() != "Darwin": print("launchd is macOS-only — on Linux, configure cron or systemd instead.", file=sys.stderr) raise typer.Exit(code=1) From 1c4713fb1c82e7066d6cd0a955b0b64cbd3621a4 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:51:59 +0200 Subject: [PATCH 6/9] Follow in-body implementation repos for articles (issue #2 Phase 3) A post that introduces a concept often links its reference implementation (ghuntley.com/ralph -> repomirrorhq/repomirror). The classifier now returns linked_repos for articles; _capture_one grounds them against the article text (dropping hallucinated/off-page links, capped at 3) and, with --follow-repos, captures each as a sibling repo artifact with `via` provenance. Default is to SUGGEST, not auto-ingest, per the do-no-harm stance. Verified live: ralph -> linked_repos=[repomirrorhq/repomirror] (Gemma's incidental NVIDIA-NeMo link correctly returns []); --follow-repos captures the repo into the SAME concept (post introduces / repo implements) with signal tracking; default run only prints the suggestion. Co-Authored-By: Claude Opus 4.8 --- jobs/capture.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/jobs/capture.py b/jobs/capture.py index 73c8f24..a5570b9 100644 --- a/jobs/capture.py +++ b/jobs/capture.py @@ -88,6 +88,7 @@ "is_new_concept": {"type": "boolean"}, "relationship": {"type": "string", "enum": _RELATIONSHIPS}, "page_kind": {"type": "string", "enum": _PAGE_KINDS}, + "linked_repos": {"type": "array", "items": {"type": "string"}}, "concept_what_it_is": {"type": ["string", "null"]}, "concept_why_it_matters": {"type": ["string", "null"]}, "review_needed": {"type": "boolean"}, @@ -116,6 +117,11 @@ "of contents of posts), not content itself. 'other'=free-text note or anything else. " "Bias toward 'article'; only say 'digest'/'index' when the page is clearly a multi-item " "container, because those will NOT be captured as a single artifact.\n" + "linked_repos: list of GitHub 'owner/repo' slugs that THIS article presents as the primary " + "IMPLEMENTATION(S) of the concept it introduces — e.g. a post introducing a pattern that links " + "its reference implementation. Include a repo ONLY if the article is substantially about it or " + "presents it as THE implementation; EXCLUDE incidental mentions, partner/integration links, " + "'see also', benchmarks, and footer/promo repos. Usually empty; at most 2-3. Use [] when none.\n" "\n" "CONCEPT MAPPING:\n" "A concept is a CAPABILITY or APPROACH category that several artifacts could implement as " @@ -453,6 +459,33 @@ def __init__(self, reason: str, existing_id: str | None = None) -> None: self.existing_id = existing_id +def _grounded_repos(linked_repos: Any, text: str | None, *, limit: int = 3) -> list[str]: + """Filter LLM-proposed in-body repos to those that ACTUALLY appear in the + article text (guards against hallucinated or off-page links), normalised to + 'owner/repo' and capped. The presence check is the key safety gate.""" + low = (text or "").lower() + out: list[str] = [] + seen: set[str] = set() + for raw in (linked_repos or []): + if not isinstance(raw, str): + continue + parsed = parse_repo_url(raw) or parse_repo_url(f"https://github.com/{raw.strip()}") + if parsed is None: + continue + slug = f"{parsed[0]}/{parsed[1]}" + key = slug.lower() + if key in seen: + continue + if key not in low: + log.info("dropping ungrounded linked_repo %r (not present in article text)", slug) + continue + seen.add(key) + out.append(slug) + if len(out) >= limit: + break + return out + + def _capture_one( input_str: str, cfg, @@ -461,6 +494,7 @@ def _capture_one( today: str | None = None, backfill: bool = False, via: str | None = None, + follow_repos: bool = False, ) -> str: """Capture a single input. Returns the artifact_id. @@ -758,6 +792,32 @@ def _capture_one( except Exception as exc: log.warning("failed to automatically backfill %s: %s", artifact_id, exc) + # ----- 14. in-body implementation repos (articles only) --------------- + # A post that introduces a concept often links its reference implementation + # (e.g. ghuntley.com/ralph -> repomirrorhq/repomirror). Surface those repos — + # grounded against the article text — and, with --follow-repos, capture each + # as a sibling artifact (with provenance) so it gets signal tracking. We + # SUGGEST by default rather than auto-ingest, per the do-no-harm stance. + if source == "web" and parsed.get("page_kind") == "article": + repos = _grounded_repos(parsed.get("linked_repos"), web_text) + if repos and follow_repos: + print(f"Following {len(repos)} in-body implementation repo(s):") + for slug in repos: + gh_url = f"https://github.com/{slug}" + try: + _capture_one( + gh_url, cfg, force=force, today=today, + backfill=backfill, via=input_str, follow_repos=False, + ) + except CaptureSkipped as exc: + print(f" ↪ {slug}: {exc.existing_id or exc}") + except Exception as exc: # noqa: BLE001 + print(f" ↪ ERROR {slug}: {exc}", file=sys.stderr) + elif repos: + print("In-body implementation repo(s) detected (re-run with --follow-repos to capture):") + for slug in repos: + print(f" - https://github.com/{slug}") + return artifact_id @@ -845,6 +905,12 @@ def capture( expand_limit: int = typer.Option( 20, "--expand-limit", help="Max Substack posts to discover/expand.", ), + follow_repos: bool = typer.Option( + False, + "--follow-repos", + help="For a single article, also capture the in-body implementation repo(s) " + "it introduces (default: just suggest them).", + ), ) -> None: """Capture an artifact from a URL or free-text note.""" cfg = config.load() @@ -911,7 +977,7 @@ def capture( return try: - _capture_one(input, cfg, force=force, today=today, backfill=backfill) + _capture_one(input, cfg, force=force, today=today, backfill=backfill, follow_repos=follow_repos) except CaptureSkipped as exc: if exc.existing_id: print(f"Already tracked as {exc.existing_id}") From 2db54a5722a98c05dcb447223fd9985e8d17f886 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:53:48 +0200 Subject: [PATCH 7/9] Document newsletter capture + subscriptions (issue #2) README: four jobs (incl. weekly newsletter poll), new lib/job modules (webextract, substack, subscriptions, newsletter), third launchd agent. SKILL.md: capture refusal behavior, Substack expansion, --follow-repos, and the `raidar newsletter` subscription commands. Co-Authored-By: Claude Opus 4.8 --- README.md | 18 +++++++++++------- SKILL.md | 23 +++++++++++++++++++++-- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8ffd844..829fe9a 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ A concept is a *capability several artifacts could implement as alternatives* ## Architecture in one breath -- **Three jobs**: `capture` (on-demand), `enrich` (Sunday 20:00 via launchd), - `digest` (Sunday 21:00 via launchd). All jobs and query utilities are accessed via a unified global CLI executable. -- **Unified CLI**: `raidar` with subcommands (`capture`, `enrich`, `digest`, `seed`, `backfill`, `reevaluate`, `search`, plus vault-hygiene commands `health`, `merge-concept`, `reindex`). +- **Four jobs**: `capture` (on-demand), `newsletter poll` (Sunday 19:00 via launchd), + `enrich` (Sunday 20:00 via launchd), `digest` (Sunday 21:00 via launchd). All jobs and query utilities are accessed via a unified global CLI executable. +- **Unified CLI**: `raidar` with subcommands (`capture`, `newsletter`, `enrich`, `digest`, `seed`, `backfill`, `reevaluate`, `search`, plus vault-hygiene commands `health`, `merge-concept`, `reindex`). - **One LLM router** (`lib/llm.py`) routes per-task to a configured chain of OpenAI-wire-compatible providers (academic proxy → local LMStudio fallback). - **Local embeddings** via LMStudio (any OpenAI-compatible embedding model), flat JSON indexes, numpy cosine. @@ -102,7 +102,7 @@ Ensure automated background processing runs weekly: ```bash raidar install-launchd -launchctl list | grep airadar # both com.airadar.enrich and com.airadar.digest should appear +launchctl list | grep airadar # com.airadar.newsletters, com.airadar.enrich and com.airadar.digest should appear ``` (equivalently `./infra/install_launchd.sh`, which the command wraps — or pass `--launchd` to `raidar init` to do this in the same step). This setup: @@ -121,8 +121,9 @@ Open this directory as a Claude Cowork project with filesystem access and shell ``` ai-radar-tool/ (this repo - stateless utility) jobs/ - capture.py on-demand capture (URL or text -> artifact + concept) - bulk_capture.py bulk capture from awesome-lists and newsletter pages + capture.py on-demand capture (URL or text -> artifact + concept); Substack expand + in-body repo follow + bulk_capture.py bulk capture from awesome-lists (awesome-list shape gate) + newsletter.py newsletter subscriptions + recurring poll (Substack) enrich.py weekly signal refresh + LLM re-evaluation (two passes) digest.py weekly markdown digest backfill.py bulk star-history backfill for artifacts @@ -140,11 +141,14 @@ ai-radar-tool/ (this repo - stateless utility) embeddings.py Ollama embeddings + split numpy indexes github.py GitHub API client (httpx + tenacity) body.py canonical body renderer/parser for concepts and artifacts + webextract.py trafilatura fetch + extraction-quality gate (refuses junk pages) + substack.py Substack discovery (landing/archive -> per-post permalinks via API) + subscriptions.py newsletter subscription registry (JSON, beside config) config.py config.yaml loader / active config resolver secrets.py .env / env-var access logging_setup.py logging configured once per process infra/ - launchd/com.airadar.{enrich,digest}.plist templates (placeholders substituted on install) + launchd/com.airadar.{newsletters,enrich,digest}.plist templates (placeholders substituted on install) install_launchd.sh install / uninstall with OS safeguards & PATH detection smoke.sh offline acceptance test test_sandbox.sh sandboxed isolated integration test diff --git a/SKILL.md b/SKILL.md index 76ecee6..fa2356e 100644 --- a/SKILL.md +++ b/SKILL.md @@ -33,11 +33,30 @@ raidar init --vault "~/raidar-vault" ### Capture — add or update an artifact/concept ```bash raidar capture "" -raidar capture --force "" # bypass dedup warning -raidar capture --dry-run "" # preview, no writes +raidar capture --force "" # bypass dedup warning +raidar capture --dry-run "" # preview, no writes +raidar capture --follow-repos "
" # also capture the article's in-body implementation repo(s) +raidar capture --expand-all "" # capture ALL posts of a pasted Substack newsletter ``` Accepts a GitHub URL, any other web URL, or free-form text. The LLM automatically classifies the artifact, maps it to a concept (or creates a new one), and outputs the resulting IDs. +**Capture refuses what it cannot understand** (do-no-harm: a skipped capture beats a polluted vault): +- **Unfetchable / thin pages** (login walls, JS-only SPAs, dead newsletter email links) are refused with a hint, never classified from nav cruft or the bare URL. +- **Newsletters / digests / archive indexes** are refused as single artifacts — they are containers of many items, not one piece of evidence. The message tells you to capture the individual items (or, for Substack, see below). +- **A pasted Substack landing/archive URL auto-expands**: capture lists the per-article permalinks so you can capture the ones you want, or `--follow-repos`/`--expand-all` act on them. Each captured item records provenance (`source_url`, and `via` = the newsletter it came from). +- **Single articles that introduce a concept** may link a reference implementation repo (e.g. ghuntley.com/ralph → repomirror). Capture *suggests* it; `--follow-repos` captures it as a sibling under the same concept with signal tracking. + +### Newsletters — recurring subscriptions (Substack) +```bash +raidar newsletter subscribe "" # watch forward (baseline current posts) +raidar newsletter subscribe --catch-up 3 "" # also capture the latest 3 now +raidar newsletter list # show subscriptions +raidar newsletter poll # capture every not-yet-seen post (runs weekly via launchd) +raidar newsletter poll --dry-run # show what's new without capturing +raidar newsletter unsubscribe "" +``` +`poll` is the recurring entry point (scheduled Sundays 19:00 by `install-launchd`). It is cadence-agnostic — "new" is decided by canonical post URL, so weekly/monthly/irregular newsletters are all handled. Only Substack publications can be subscribed today; other newsletter forms are refused at subscribe time. For a one-off non-Substack article, capture its individual URL directly. + ### Search — query the vault ```bash raidar search keyword "" # frontmatter substring match (both layers) From 909ae1967d134f54054897a68c5a0a4729b5fd44 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:43:22 +0200 Subject: [PATCH 8/9] Show help for bare `raidar newsletter` (issue #2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set no_args_is_help=True on the newsletter Typer group so invoking it without a subcommand lists the available commands instead of erroring with "Missing command" — matching `raidar search` and the root CLI. Co-Authored-By: Claude Opus 4.8 --- jobs/newsletter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/newsletter.py b/jobs/newsletter.py index e8a1ab8..5241099 100644 --- a/jobs/newsletter.py +++ b/jobs/newsletter.py @@ -29,7 +29,7 @@ log = logging.getLogger(__name__) -app = typer.Typer(add_completion=False, help=__doc__) +app = typer.Typer(add_completion=False, help=__doc__, no_args_is_help=True) def _capture_posts(posts: list[substack.Post], via: str, cfg, today: str) -> int: From ab6c87f6bfd9518875fd8c72979c55ecf01783f0 Mon Sep 17 00:00:00 2001 From: Janik Muires <2056743+janikmu@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:14:17 +0200 Subject: [PATCH 9/9] Ingest HTML-email newsletter digests (issue #2 Mode C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many newsletters (AlphaSignal, Beehiiv/Mailchimp senders) arrive as HTML email, not a fetchable page — the web archive is auth/bot-walled, so the email itself is the reliable surface. `raidar newsletter ingest ` (or stdin) parses the MIME, LLM-triages the links to drop ads/sponsors and chrome BEFORE resolving anything (so sponsor trackers are never clicked), follows the click-tracking redirects to real destinations, then auto-captures the GitHub repos (with `via` provenance) and prints a review list of everything else (HF models, articles) with ready-to-run capture commands — never auto-writing artifact types raidar cannot yet model (do-no-harm). lib/email_digest.py handles MIME parsing, anchor/href extraction (tracker-dup collapse, & unescaping, chrome filtering) and redirect resolution. Verified against a real AlphaSignal issue (--dry-run): triage = 4 content / 1 ad / 1 chrome, with the Lambda sponsor link never resolved; spec-kit + LangBot route to GitHub capture; the Nemotron HF model and the Anthropic news article go to the review list with their resolved URLs. Co-Authored-By: Claude Opus 4.8 --- README.md | 3 +- SKILL.md | 9 ++ infra/smoke.sh | 3 + jobs/newsletter.py | 162 ++++++++++++++++++++++++++++- lib/email_digest.py | 248 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 423 insertions(+), 2 deletions(-) create mode 100644 lib/email_digest.py diff --git a/README.md b/README.md index 829fe9a..e5485d0 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ ai-radar-tool/ (this repo - stateless utility) jobs/ capture.py on-demand capture (URL or text -> artifact + concept); Substack expand + in-body repo follow bulk_capture.py bulk capture from awesome-lists (awesome-list shape gate) - newsletter.py newsletter subscriptions + recurring poll (Substack) + newsletter.py newsletter subscriptions + recurring poll (Substack); HTML-email digest ingest enrich.py weekly signal refresh + LLM re-evaluation (two passes) digest.py weekly markdown digest backfill.py bulk star-history backfill for artifacts @@ -144,6 +144,7 @@ ai-radar-tool/ (this repo - stateless utility) webextract.py trafilatura fetch + extraction-quality gate (refuses junk pages) substack.py Substack discovery (landing/archive -> per-post permalinks via API) subscriptions.py newsletter subscription registry (JSON, beside config) + email_digest.py HTML-email newsletter parsing (MIME -> text + links, tracker resolution) config.py config.yaml loader / active config resolver secrets.py .env / env-var access logging_setup.py logging configured once per process diff --git a/SKILL.md b/SKILL.md index fa2356e..5b46421 100644 --- a/SKILL.md +++ b/SKILL.md @@ -57,6 +57,15 @@ raidar newsletter unsubscribe "" ``` `poll` is the recurring entry point (scheduled Sundays 19:00 by `install-launchd`). It is cadence-agnostic — "new" is decided by canonical post URL, so weekly/monthly/irregular newsletters are all handled. Only Substack publications can be subscribed today; other newsletter forms are refused at subscribe time. For a one-off non-Substack article, capture its individual URL directly. +### Newsletters — HTML email digests (AlphaSignal, Beehiiv, Mailchimp, …) +Many newsletters arrive as HTML email, not a fetchable web page. Save the message as a `.eml` file and ingest it: +```bash +raidar newsletter ingest "" # harvest the GitHub repos it surfaces; list the rest +raidar newsletter ingest --dry-run "" # show the decomposition, capture nothing +cat message.eml | raidar newsletter ingest - # or pipe raw email on stdin +``` +Ingest parses the email, uses the LLM to drop ads/sponsors and navigation **before resolving any link** (so sponsor trackers are never clicked), follows the remaining click-tracking redirects to their real destinations, then: **auto-captures the GitHub repos** (with `via` = the newsletter as provenance) and prints a **review list** of everything else (HuggingFace models, articles) with a ready-to-run `raidar capture ""` line for each — because raidar has no artifact type for models/papers yet, those are never auto-written (do-no-harm). + ### Search — query the vault ```bash raidar search keyword "" # frontmatter substring match (both layers) diff --git a/infra/smoke.sh b/infra/smoke.sh index 1695488..010b7d2 100755 --- a/infra/smoke.sh +++ b/infra/smoke.sh @@ -64,6 +64,7 @@ check "lib.body imports" uv run python -c "from lib import body" check "lib.webextract imports" uv run python -c "from lib import webextract" check "lib.substack imports" uv run python -c "from lib import substack" check "lib.subscriptions imports" uv run python -c "from lib import subscriptions" +check "lib.email_digest imports" uv run python -c "from lib import email_digest" check "all jobs import" uv run python -c "from jobs import capture, bulk_capture, enrich, digest, search, backfill, reevaluate, seed, newsletter, cli" echo "[lib smoke tests]" @@ -74,6 +75,7 @@ check "lib.github smoke (uses network)" uv run python -m lib.github check_skip_ok "lib.webextract smoke (uses network)" uv run python -m lib.webextract check_skip_ok "lib.substack smoke (uses network)" uv run python -m lib.substack check "lib.subscriptions smoke" uv run python -m lib.subscriptions +check "lib.email_digest smoke" uv run python -m lib.email_digest echo "[CLI wiring]" check "raidar --help" uv run raidar --help @@ -92,6 +94,7 @@ check "reindex --help" uv run python -m jobs.reindex --help check "install-launchd --help" uv run python -m jobs.launchd --help check "newsletter --help" uv run python -m jobs.newsletter --help check "newsletter poll --help" uv run python -m jobs.newsletter poll --help +check "newsletter ingest --help" uv run python -m jobs.newsletter ingest --help check "search list-concepts against empty vault" uv run python -m jobs.search list-concepts check "health against empty vault" uv run python -m jobs.health diff --git a/jobs/newsletter.py b/jobs/newsletter.py index 5241099..81cca12 100644 --- a/jobs/newsletter.py +++ b/jobs/newsletter.py @@ -20,10 +20,16 @@ import logging import sys from datetime import date +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +import httpx import typer -from lib import config, substack, subscriptions +from lib import config, email_digest, substack, subscriptions +from lib.github import parse_repo_url +from lib.llm import Router from lib.logging_setup import setup as setup_logging from jobs.capture import CaptureSkipped, _capture_one @@ -190,5 +196,159 @@ def poll( print(f"\nDone: {total_new} new artifact(s) across {len(subs)} subscription(s).") +# --------------------------------------------------------------------------- +# ingest — decompose an HTML newsletter email (Mode C: email digests) +# --------------------------------------------------------------------------- + +_TRIAGE_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["index", "label"], + "properties": { + "index": {"type": "integer"}, + "label": {"type": "string", "enum": ["content", "ad", "chrome"]}, + }, + }, + } + }, +} + +_TRIAGE_SYSTEM = ( + "You triage the links in an AI/dev newsletter email. For EACH candidate link " + "(given by index + anchor text), assign one label:\n" + "- 'content' = a real editorial item worth tracking: a tool, GitHub repo, model, " + "paper, or genuine news story the newsletter is reporting on.\n" + "- 'ad' = a paid sponsor/advertisement. Signals: 'Presented by', 'In Partnership " + "with', 'Sponsored', promotional CTAs, or a product pitch unrelated to the news.\n" + "- 'chrome' = navigation/subscribe/login/archive/feedback/forward/'work with us' " + "or other boilerplate.\n" + "Use the anchor text and the email body for context. Return a label for EVERY index. " + "Be strict: when a link is a sponsor plug, label it 'ad', not 'content'." +) + + +def _kind_for(url: str) -> str: + host = urlparse(url).netloc.lower().lstrip("www.") + if "huggingface.co" in host: + return "model (HuggingFace)" + if "arxiv.org" in host: + return "paper (arXiv)" + if "github.com" in host: + return "repo (non-canonical path)" + return "article/other" + + +def _triage_links(cfg, subject: str, text: str, links: list) -> dict[int, str]: + """Ask the LLM to label each candidate link content/ad/chrome BEFORE we resolve + any redirect (so sponsor trackers are never clicked). Unlabeled indices default + to 'chrome' (conservative — we act only on explicit 'content').""" + enumerated = "\n".join(f"[{i}] {l.text[:140]}" for i, l in enumerate(links)) + prompt = ( + f"## Newsletter subject\n{subject}\n\n" + f"## Body (for context, truncated)\n{text[:4000]}\n\n" + f"## Candidate links\n{enumerated}\n\n" + "Label every index. Return ONLY JSON matching the schema." + ) + try: + comp = Router(cfg).generate( + task="classification", prompt=prompt, system=_TRIAGE_SYSTEM, + response_schema=_TRIAGE_SCHEMA, max_tokens=2048, + ) + parsed = comp.parsed or {} + except Exception as exc: # noqa: BLE001 + log.warning("link triage failed (%s); treating all links as chrome", exc) + return {} + out: dict[int, str] = {} + for it in parsed.get("items", []): + try: + out[int(it["index"])] = str(it["label"]) + except (KeyError, TypeError, ValueError): + continue + return out + + +@app.command() +def ingest( + path: str = typer.Argument(..., help="Path to a saved .eml file, or '-' to read raw email from stdin."), + dry_run: bool = typer.Option(False, "--dry-run", help="Show the decomposition; capture nothing."), +) -> None: + """Decompose an HTML newsletter email: auto-capture the GitHub repos it surfaces + (with provenance) and list the rest (models, articles) for you to capture by hand.""" + cfg = config.load() + setup_logging(level=cfg.log_level, log_file=cfg.log_file) + today = date.today().isoformat() + + raw = sys.stdin.read() if path == "-" else Path(path).expanduser().read_text(encoding="utf-8", errors="replace") + email = email_digest.parse_email(raw) + if not email.links: + print("No links found in the email (is it an HTML newsletter?).") + return + + via = email.provenance() + print(f"Newsletter: {email.subject or '(no subject)'}") + print(f" via: {via}") + print(f" {len(email.links)} candidate link(s) found; triaging…") + + labels = _triage_links(cfg, email.subject, email.text, email.links) + content = [(i, l) for i, l in enumerate(email.links) if labels.get(i, "chrome") == "content"] + n_ad = sum(1 for v in labels.values() if v == "ad") + n_chrome = len(email.links) - len(content) - n_ad + print(f" triage: {len(content)} content, {n_ad} ad(s), {n_chrome} chrome/skipped\n") + + if not content: + print("No content items to harvest.") + return + + harvested = 0 + skipped = 0 + review: list[tuple[str, str | None, str]] = [] + with httpx.Client( + follow_redirects=True, timeout=25.0, + headers={"User-Agent": email_digest._UA}, + ) as client: + for _i, link in content: + final = email_digest.resolve_final_url(link.href, client=client) + if not final: + review.append((link.text, None, "unresolved")) + continue + repo = parse_repo_url(final) + if repo is not None: + slug = f"{repo[0]}/{repo[1]}" + if dry_run: + print(f" would capture repo: {slug} ← {link.text[:70]}") + harvested += 1 + continue + try: + aid = _capture_one(final, cfg, today=today, backfill=False, via=via) + print(f" ✓ captured {aid} ({slug})") + harvested += 1 + except CaptureSkipped as exc: + print(f" · skip {slug}: {exc.existing_id or exc}") + skipped += 1 + except Exception as exc: # noqa: BLE001 + print(f" ERROR {slug}: {exc}", file=sys.stderr) + else: + review.append((link.text, final, _kind_for(final))) + + verb = "would harvest" if dry_run else "harvested" + print(f"\n{verb} {harvested} GitHub repo(s)" + (f", skipped {skipped} already-tracked" if skipped else "") + ".") + + if review: + print(f"\nReview list — {len(review)} item(s) raidar can't auto-capture yet " + "(models/articles); capture the ones you want individually:") + for text, url, kind in review: + if url: + print(f" - [{kind}] {text[:70]}\n raidar capture \"{url}\"") + else: + print(f" - [unresolved] {text[:70]}") + + if __name__ == "__main__": app() diff --git a/lib/email_digest.py b/lib/email_digest.py new file mode 100644 index 0000000..dd14d6a --- /dev/null +++ b/lib/email_digest.py @@ -0,0 +1,248 @@ +"""Parse an HTML newsletter email (.eml / raw MIME) into its text + outbound links. + +Many newsletters (AlphaSignal, Beehiiv/Mailchimp senders, …) are delivered as +HTML email rather than a fetchable web page — the web archive is often auth- or +bot-walled. The email itself is the reliable, complete surface, so raidar ingests +it directly (see `jobs.newsletter ingest`). + +Two wrinkles this module handles: +- Content links are usually **click-tracking redirects** (e.g. app.alphasignal.ai/ + c?…) with no destination in the markup — `resolve_final_url` follows them to the + real URL (github.com/…, huggingface.co/…, the original article). +- Ads/sponsors are styled like content — we surface the anchor text + context so a + caller (the LLM) can filter them out BEFORE any redirect is resolved (so sponsor + trackers are never clicked). + +`parse_email` / `extract_links` are pure (offline smoke-tested); `resolve_final_url` +hits the network. +""" + +from __future__ import annotations + +import html +import logging +from dataclasses import dataclass, field +from email import message_from_string +from email.policy import default as default_policy +from html.parser import HTMLParser +from urllib.parse import parse_qs, urlparse + +import httpx + +log = logging.getLogger(__name__) + +_UA = "Mozilla/5.0 (compatible; raidar/0.1; +https://github.com/janikmu/raidar)" +_TIMEOUT_S = 25.0 + +# Anchor hrefs that are never content: mail actions, unsubscribe/feedback/open +# beacons, and social. Matched as substrings against the raw href. +_CHROME_HREF_MARKERS = ( + "mailto:", + "/unsubscribe", + "/us?", + "/fb/", + "/o?", # open-tracking pixel + "utm_source=forward", + "twitter.com", + "x.com/", + "linkedin.com/", + "facebook.com", +) + + +@dataclass +class Link: + text: str # anchor text (reassembled across nested tags) + href: str # raw href as it appears in the email (usually a tracker) + + +@dataclass +class ParsedEmail: + subject: str + sender: str + list_id: str + archived_at: str + text: str # text/plain part if present, else stripped HTML text + links: list[Link] = field(default_factory=list) + + def provenance(self) -> str: + """Best stable identifier for the newsletter, for `via` provenance.""" + return self.archived_at or self.list_id or self.sender or "newsletter-email" + + +class _LinkExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._href: str | None = None + self._buf: list[str] = [] + self.links: list[Link] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + href = dict(attrs).get("href") + if href: + self._href = href + self._buf = [] + + def handle_data(self, data: str) -> None: + if self._href is not None: + self._buf.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._href is not None: + text = " ".join("".join(self._buf).split()) + self.links.append(Link(text=text, href=self._href)) + self._href = None + self._buf = [] + + +def extract_links(markup: str) -> list[Link]: + """Return (anchor_text, href) pairs from HTML, dropping obvious chrome and + collapsing tracker duplicates (same destination link id) to their most + descriptive anchor text.""" + p = _LinkExtractor() + p.feed(markup) + + best: dict[str, Link] = {} + for link in p.links: + href = html.unescape(link.href.strip()) # & -> & so trackers resolve + link = Link(text=link.text, href=href) + if not href or href.startswith("#"): + continue + low = href.lower() + if any(m in low for m in _CHROME_HREF_MARKERS): + continue + key = _dedup_key(href) + cur = best.get(key) + if cur is None or len(link.text) > len(cur.text): + best[key] = link + # Preserve first-seen order roughly by anchor length is fine; sort by nothing. + return [l for l in best.values() if l.text] + + +def _dedup_key(href: str) -> str: + """Collapse the same click-tracked destination (identified by a link-id query + param like `lid`) so an item's title link and its READ-MORE button merge.""" + q = parse_qs(urlparse(href).query) + for k in ("lid", "l", "link_id"): + if k in q and q[k]: + return f"lid:{q[k][0]}" + return href + + +def _strip_html_text(markup: str) -> str: + class _T(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.out: list[str] = [] + self._skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style"): + self._skip += 1 + + def handle_endtag(self, tag): + if tag in ("script", "style") and self._skip: + self._skip -= 1 + + def handle_data(self, data): + if not self._skip and data.strip(): + self.out.append(data.strip()) + + t = _T() + t.feed(markup) + return "\n".join(t.out) + + +def parse_email(raw: str) -> ParsedEmail: + """Parse a raw RFC-822 email (as text) into a ParsedEmail.""" + msg = message_from_string(raw, policy=default_policy) + + subject = str(msg.get("Subject", "")).strip() + sender = str(msg.get("From", "")).strip() + list_id = str(msg.get("List-Id", "")).strip() + archived_at = str(msg.get("Archived-At", "")).strip().strip("<>") + + text_plain = "" + text_html = "" + if msg.is_multipart(): + for part in msg.walk(): + ctype = part.get_content_type() + if ctype == "text/plain" and not text_plain: + text_plain = part.get_content() + elif ctype == "text/html" and not text_html: + text_html = part.get_content() + else: + if msg.get_content_type() == "text/html": + text_html = msg.get_content() + else: + text_plain = msg.get_content() + + links = extract_links(text_html) if text_html else [] + text = text_plain.strip() or _strip_html_text(text_html) + return ParsedEmail( + subject=subject, + sender=sender, + list_id=list_id, + archived_at=archived_at, + text=text, + links=links, + ) + + +def resolve_final_url(url: str, *, client: httpx.Client | None = None) -> str | None: + """Follow redirects for a tracker `url` and return the final destination, or + None if it can't be resolved. Never raises.""" + owns = client is None + c = client or httpx.Client( + follow_redirects=True, timeout=_TIMEOUT_S, headers={"User-Agent": _UA} + ) + try: + # Stream a GET so we get the final URL without downloading the whole body. + with c.stream("GET", url) as r: + return str(r.url) + except Exception as exc: # noqa: BLE001 + log.debug("could not resolve %s: %s", url, exc) + return None + finally: + if owns: + c.close() + + +def _smoke() -> int: + sample = ( + "From: AlphaSignal \n" + "Subject: Test issue\n" + "List-Id: AlphaSignal Newsletter \n" + "Archived-At: \n" + 'Content-Type: multipart/alternative; boundary="B"\n\n' + "--B\nContent-Type: text/plain\n\n" + "Top News\nGitHub releases spec-kit\nPresented by Lambda: MFU guide\n" + "--B\nContent-Type: text/html\n\n" + "" + 'Signup' + 'GitHub releases ' + 'spec-kit' + 'READ MORE' # dup of BBB + 'unsubscribe' + "\n--B--\n" + ) + p = parse_email(sample) + assert p.subject == "Test issue", p.subject + assert p.list_id.endswith("news.alphasignal.ai>") or "alphasignal" in p.list_id + assert p.archived_at == "https://alphasignal.ai/email/abc123", p.archived_at + assert p.provenance() == "https://alphasignal.ai/email/abc123" + texts = {l.text for l in p.links} + # mailto dropped; BBB deduped to its descriptive title (not "READ MORE"). + assert "GitHub releases spec-kit" in texts, texts + assert "unsubscribe" not in texts, texts + assert not any(l.text == "READ MORE" for l in p.links), "dup tracker should merge to title" + assert len(p.links) == 2, [l.text for l in p.links] # Signup + spec-kit + print("email_digest offline checks passed.") + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(_smoke())