From 211df0deaaab574ad8fae23eb5b29d7545b420dc Mon Sep 17 00:00:00 2001 From: Dave Morin Date: Fri, 8 May 2026 10:36:26 -0700 Subject: [PATCH 1/3] feat(web): auto-enrich Reddit URLs from web search via JSON API Web search backends (Brave, Exa, Serper) can return Reddit URLs as results. Claude Code's WebFetch blocks reddit.com, so the model can't retrieve full thread content. After web search, detect Reddit URLs and fetch body text + top comments via reddit.com/.json endpoint using the skill's own HTTP library. Fixes #324 --- skills/last30days/scripts/lib/grounding.py | 59 ++++++++++++++++++---- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.py index 3aa4a98..9d7835e 100644 --- a/skills/last30days/scripts/lib/grounding.py +++ b/skills/last30days/scripts/lib/grounding.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys import urllib.parse from datetime import datetime from urllib.parse import urlparse @@ -205,29 +206,69 @@ def web_search( backend = "parallel" else: return [], {} + items: list[dict] = [] + artifact: dict = {} if backend == "brave": key = config.get("BRAVE_API_KEY") if not key: raise RuntimeError("BRAVE_API_KEY is required when web_backend='brave'") - return brave_search(query, date_range, key) - if backend == "exa": + items, artifact = brave_search(query, date_range, key) + elif backend == "exa": key = config.get("EXA_API_KEY") if not key: raise RuntimeError("EXA_API_KEY is required when web_backend='exa'") - return exa_search(query, date_range, key) - if backend == "serper": + items, artifact = exa_search(query, date_range, key) + elif backend == "serper": key = config.get("SERPER_API_KEY") if not key: raise RuntimeError("SERPER_API_KEY is required when web_backend='serper'") - return serper_search(query, date_range, key) - if backend == "parallel": + items, artifact = serper_search(query, date_range, key) + elif backend == "parallel": key = config.get("PARALLEL_API_KEY") if not key: raise RuntimeError("PARALLEL_API_KEY is required when web_backend='parallel'") - return parallel_search(query, date_range, key) - if backend != "none": + items, artifact = parallel_search(query, date_range, key) + elif backend != "none": raise ValueError(f"Unsupported web backend: {backend!r}") - return [], {} + else: + return [], {} + if items: + items = _enrich_reddit_items(items) + return items, artifact + + +def _enrich_reddit_items(items: list[dict]) -> list[dict]: + """Enrich web search results that are Reddit URLs with thread body and comments. + + Claude Code's WebFetch blocks reddit.com, so the model can't retrieve + Reddit content from web search results. This fetches it via the public + JSON API (reddit.com/.../.json) which bypasses that restriction. + """ + from . import reddit_enrich + + for item in items: + url = item.get("url", "") + if "reddit.com" not in url or "/comments/" not in url: + continue + try: + thread_data = reddit_enrich.fetch_thread_data(url, timeout=8) + if not thread_data: + continue + parsed = reddit_enrich.parse_thread_data(thread_data) + selftext = parsed.get("selftext", "") + if selftext: + item["snippet"] = selftext[:2000] + comments = parsed.get("comments", []) + top = reddit_enrich.get_top_comments(comments) + if top: + item["top_comments"] = [ + {"score": c.get("score", 0), "excerpt": (c.get("body") or "")[:200]} + for c in top[:5] + ] + item["enriched_via"] = "reddit_json_api" + except Exception as exc: + sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n") + return items # --------------------------------------------------------------------------- From 791c0a57a0b025c0a63887e954e1c01020cebec3 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Sat, 16 May 2026 22:52:06 -0700 Subject: [PATCH 2/3] review: gate web Reddit enrichment behind EXCLUDE_SOURCES PR #366 routes Reddit URLs found in web-search results through the public Reddit JSON API to recover thread body + top comments (the Claude Code WebFetch tool blocks reddit.com directly). That bypass is sound and the fixed problem is real - but the always-on shape ignores user intent on source gating. A user who sets EXCLUDE_SOURCES=reddit to suppress Reddit results would still get Reddit content smuggled back in via web-search URLs that happen to point at reddit.com threads. This contradicts the suppression contract that EXCLUDE_SOURCES is supposed to provide (see lib/pipeline.available_sources where the same env var gates the top-level Reddit source). Add a _reddit_excluded(config) check in web_search() that mirrors the parsing pattern from lib/pipeline (comma-separated, case-insensitive, whitespace-tolerant). When reddit is in EXCLUDE_SOURCES, skip the enrichment pass entirely - the web results themselves still flow through, but they're not augmented with Reddit body/comments. Four new tests in test_grounding_v3.py cover: - EXCLUDE_SOURCES=reddit skips enrichment - case-insensitive parsing matches REDDIT/Reddit/whitespace-padded/csv - Other sources in EXCLUDE_SOURCES don't trigger the gate - Enrichment runs normally when reddit isn't excluded 19/19 grounding tests pass. --- skills/last30days/scripts/lib/grounding.py | 17 +++++++++- tests/test_grounding_v3.py | 37 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.py index 9d7835e..ad33ede 100644 --- a/skills/last30days/scripts/lib/grounding.py +++ b/skills/last30days/scripts/lib/grounding.py @@ -232,17 +232,32 @@ def web_search( raise ValueError(f"Unsupported web backend: {backend!r}") else: return [], {} - if items: + if items and not _reddit_excluded(config): items = _enrich_reddit_items(items) return items, artifact +def _reddit_excluded(config: dict) -> bool: + """Return True when EXCLUDE_SOURCES contains 'reddit'. + + Respects the same suppression knob the pipeline uses for source gating, + so a user who set EXCLUDE_SOURCES=reddit doesn't get Reddit content + smuggled back in via web-search URLs. + """ + raw = (config.get("EXCLUDE_SOURCES") or "").split(",") + return any(s.strip().lower() == "reddit" for s in raw) + + def _enrich_reddit_items(items: list[dict]) -> list[dict]: """Enrich web search results that are Reddit URLs with thread body and comments. Claude Code's WebFetch blocks reddit.com, so the model can't retrieve Reddit content from web search results. This fetches it via the public JSON API (reddit.com/.../.json) which bypasses that restriction. + + Callers should gate this with EXCLUDE_SOURCES=reddit handling (see + `_reddit_excluded`) so a user who explicitly excluded Reddit doesn't + get Reddit content via web-search URLs. """ from . import reddit_enrich diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.py index b9867d6..0a662f5 100644 --- a/tests/test_grounding_v3.py +++ b/tests/test_grounding_v3.py @@ -190,5 +190,42 @@ class WebSearchDispatchTests(unittest.TestCase): grounding.web_search("test", ("2026-02-25", "2026-03-27"), {}, backend="google") +class RedditEnrichmentGateTests(unittest.TestCase): + """EXCLUDE_SOURCES=reddit must suppress the web-search Reddit enrichment. + + Otherwise a user who explicitly excluded Reddit would still get Reddit + content smuggled back in via web-search URLs that happen to point at + reddit.com threads. + """ + + def test_reddit_excluded_via_exclude_sources_skips_enrichment(self): + config = {"BRAVE_API_KEY": "k", "EXCLUDE_SOURCES": "reddit"} + items = [{"url": "https://www.reddit.com/r/python/comments/abc/title/", "snippet": "original"}] + with patch("lib.grounding.brave_search", return_value=(items, {})), \ + patch("lib.grounding._enrich_reddit_items") as enrich_mock: + grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto") + enrich_mock.assert_not_called() + + def test_reddit_excluded_case_insensitive(self): + for value in ("REDDIT", "Reddit", " reddit ", "x,reddit,y"): + config = {"BRAVE_API_KEY": "k", "EXCLUDE_SOURCES": value} + self.assertTrue( + grounding._reddit_excluded(config), + msg=f"_reddit_excluded should be True for EXCLUDE_SOURCES={value!r}", + ) + + def test_reddit_not_excluded_when_other_sources_listed(self): + config = {"EXCLUDE_SOURCES": "tiktok,instagram"} + self.assertFalse(grounding._reddit_excluded(config)) + + def test_enrichment_runs_when_reddit_not_excluded(self): + config = {"BRAVE_API_KEY": "k"} + items = [{"url": "https://www.reddit.com/r/python/comments/abc/title/", "snippet": "original"}] + with patch("lib.grounding.brave_search", return_value=(items, {})), \ + patch("lib.grounding._enrich_reddit_items", return_value=items) as enrich_mock: + grounding.web_search("test", ("2026-02-25", "2026-03-27"), config, backend="auto") + enrich_mock.assert_called_once() + + if __name__ == "__main__": unittest.main() From e2d9d705f6ee35e5c3372c234442789110cbc62a Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Sat, 16 May 2026 23:37:00 -0700 Subject: [PATCH 3/3] review: fix selftext key path + break on RedditRateLimitError --- skills/last30days/scripts/lib/grounding.py | 8 +++- tests/test_grounding_v3.py | 46 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/skills/last30days/scripts/lib/grounding.py b/skills/last30days/scripts/lib/grounding.py index ad33ede..aa06289 100644 --- a/skills/last30days/scripts/lib/grounding.py +++ b/skills/last30days/scripts/lib/grounding.py @@ -260,6 +260,7 @@ def _enrich_reddit_items(items: list[dict]) -> list[dict]: get Reddit content via web-search URLs. """ from . import reddit_enrich + from .reddit_enrich import RedditRateLimitError for item in items: url = item.get("url", "") @@ -270,7 +271,8 @@ def _enrich_reddit_items(items: list[dict]) -> list[dict]: if not thread_data: continue parsed = reddit_enrich.parse_thread_data(thread_data) - selftext = parsed.get("selftext", "") + # selftext lives under parsed["submission"], not at the top level + selftext = (parsed.get("submission") or {}).get("selftext", "") if selftext: item["snippet"] = selftext[:2000] comments = parsed.get("comments", []) @@ -281,6 +283,10 @@ def _enrich_reddit_items(items: list[dict]) -> list[dict]: for c in top[:5] ] item["enriched_via"] = "reddit_json_api" + except RedditRateLimitError as exc: + # Stop iterating to avoid flooding more 429s + sys.stderr.write(f"[Web] Reddit rate-limited, halting enrichment: {exc}\n") + break except Exception as exc: sys.stderr.write(f"[Web] Reddit enrichment failed for {url}: {exc}\n") return items diff --git a/tests/test_grounding_v3.py b/tests/test_grounding_v3.py index 0a662f5..30d663b 100644 --- a/tests/test_grounding_v3.py +++ b/tests/test_grounding_v3.py @@ -227,5 +227,51 @@ class RedditEnrichmentGateTests(unittest.TestCase): enrich_mock.assert_called_once() +class RedditEnrichItemsTests(unittest.TestCase): + """Direct tests for `_enrich_reddit_items` covering the selftext key path + and the RedditRateLimitError early-exit behavior. + """ + + def test_selftext_under_submission_populates_snippet(self): + from lib import reddit_enrich + + item = { + "url": "https://www.reddit.com/r/python/comments/abc/title/", + "snippet": "original", + } + parsed = { + "submission": {"selftext": "thread body content"}, + "comments": [], + } + with patch.object(reddit_enrich, "fetch_thread_data", return_value={"raw": True}), \ + patch.object(reddit_enrich, "parse_thread_data", return_value=parsed): + result = grounding._enrich_reddit_items([item]) + self.assertEqual("thread body content", result[0]["snippet"]) + self.assertEqual("reddit_json_api", result[0]["enriched_via"]) + + def test_rate_limit_error_halts_iteration(self): + from lib import reddit_enrich + + item1 = {"url": "https://www.reddit.com/r/python/comments/aaa/x/"} + item2 = {"url": "https://www.reddit.com/r/python/comments/bbb/y/"} + + def fake_fetch(url, *args, **kwargs): + raise reddit_enrich.RedditRateLimitError(f"429 for {url}") + + captured_stderr: list[str] = [] + + with patch.object(reddit_enrich, "fetch_thread_data", side_effect=fake_fetch) as fetch_mock, \ + patch("lib.grounding.sys.stderr.write", side_effect=lambda s: captured_stderr.append(s)): + grounding._enrich_reddit_items([item1, item2]) + + # Only the first item should have triggered a fetch attempt + self.assertEqual(1, fetch_mock.call_count) + # A stderr message about the rate-limit halt should have been emitted + self.assertTrue( + any("rate-limited" in msg.lower() or "rate limited" in msg.lower() for msg in captured_stderr), + msg=f"Expected a rate-limit stderr message, got: {captured_stderr!r}", + ) + + if __name__ == "__main__": unittest.main()