From 4f6b86c4569946f20275e0875aee60776f4d0661 Mon Sep 17 00:00:00 2001 From: Tobi Date: Sat, 16 May 2026 00:54:18 +0200 Subject: [PATCH 1/3] feat: honor EXCLUDE_SOURCES env var in source count + pipeline filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a per-run denylist via the existing-but-unused EXCLUDE_SOURCES config key. Two coupled changes: 1. pipeline.available_sources() filters out any source listed in config["EXCLUDE_SOURCES"] (comma-separated, case-insensitive, whitespace-tolerant) before returning. 2. hooks/scripts/check-config.sh "Ready — N sources active" banner subtracts excluded sources from the ScrapeCreators +3 (Reddit comments + TikTok + Instagram) so the count matches what the pipeline actually runs. Use case: skip TikTok/Instagram on runs where you only want text-substantive sources, without unsetting SCRAPECREATORS_API_KEY (which would also kill Reddit comments). The existing INCLUDE_SOURCES allowlist covers Perplexity opt-in but doesn't cover this denylist case — tiktok and instagram are added unconditionally when SCRAPECREATORS_API_KEY is set, with no opt-out short of removing the key. Tests (tests/test_pipeline_v3.py::TestExcludeSources): - excludes tiktok+instagram when listed - no exclusion when env unset or empty string - case-insensitive + whitespace-tolerant parsing - works for any source (e.g. EXCLUDE_SOURCES=hackernews), not just SC-backed --- hooks/scripts/check-config.sh | 11 ++++- skills/last30days/scripts/lib/pipeline.py | 3 ++ tests/test_pipeline_v3.py | 52 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 39b4bbd..81da6d2 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -97,7 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then SOURCE_COUNT=$((SOURCE_COUNT + 1)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then - SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram + # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES + SC_ADD=3 + EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}" + if [[ ",$EXCLUDED," == *",tiktok,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + if [[ ",$EXCLUDED," == *",instagram,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index ebedd45..30bcbbf 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -128,6 +128,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non available.append("pinterest") if env.is_xquik_available(config): available.append("xquik") + exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()} + if exclude: + available = [s for s in available if s not in exclude] return available diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py index 08b6109..54dcc18 100644 --- a/tests/test_pipeline_v3.py +++ b/tests/test_pipeline_v3.py @@ -904,5 +904,57 @@ class TestZeroKeyPipelineRun(unittest.TestCase): self.assertEqual("fallback-local-score", candidate.explanation) +class TestExcludeSources(unittest.TestCase): + """EXCLUDE_SOURCES env var filters sources out of available_sources(). + + The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does + not cover this case — tiktok and instagram are added unconditionally + when SCRAPECREATORS_API_KEY is set, with no way to opt out short of + unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist. + """ + + def test_excludes_tiktok_and_instagram(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "tiktok,instagram", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + self.assertIn("reddit", sources) + self.assertIn("hackernews", sources) + + def test_no_exclusion_when_unset(self): + config = {"SCRAPECREATORS_API_KEY": "test-key"} + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_empty_exclude_sources_is_noop(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "", + } + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_whitespace_and_case_insensitive(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": " TikTok , INSTAGRAM ", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + + def test_excludes_non_scrapecreators_source(self): + """EXCLUDE_SOURCES applies to any source, not just SC-backed ones.""" + config = {"EXCLUDE_SOURCES": "hackernews"} + sources = pipeline.available_sources(config) + self.assertNotIn("hackernews", sources) + self.assertIn("reddit", sources) + + if __name__ == "__main__": unittest.main() From 095bcae915ad02e3bd27d874c890ab6d6e563ff3 Mon Sep 17 00:00:00 2001 From: Tobi Date: Sat, 16 May 2026 08:05:34 +0200 Subject: [PATCH 2/3] fix(check-config): normalize EXCLUDE_SOURCES (lowercase + whitespace) before matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bash banner accounting used raw substring matching while pipeline.py normalises EXCLUDE_SOURCES via .strip().lower(). With EXCLUDE_SOURCES=TikTok,Instagram (or with surrounding spaces), pipeline correctly excludes the sources but the banner did not deduct them — count showed 1-2 higher than what the pipeline actually runs. Normalisation now mirrors the Python side (lowercase, collapse whitespace around commas, strip outer whitespace). Reproducer (clean HOME with config EXCLUDE_SOURCES=TikTok,Instagram): before: /last30days: Ready — 7 sources active. after: /last30days: Ready — 5 sources active. Addresses Greptile review comment P1 on #399. --- hooks/scripts/check-config.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 81da6d2..85403cc 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -97,13 +97,17 @@ if [[ -n "$HAS_BSKY" ]]; then SOURCE_COUNT=$((SOURCE_COUNT + 1)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then - # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES + # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES. + # Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer + # whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing. SC_ADD=3 EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}" - if [[ ",$EXCLUDED," == *",tiktok,"* ]]; then + EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//') + if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then SC_ADD=$((SC_ADD - 1)) fi - if [[ ",$EXCLUDED," == *",instagram,"* ]]; then + if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then SC_ADD=$((SC_ADD - 1)) fi SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD)) From 306d8c2d73330aa2af307e196a1fb606a678e6a2 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Sat, 16 May 2026 19:30:27 -0700 Subject: [PATCH 3/3] fix(env): wire EXCLUDE_SOURCES through get_config + SKILL.md integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original PR added EXCLUDE_SOURCES filtering to pipeline.available_sources() and to the check-config.sh banner, but env.py::get_config() builds its config dict from a hardcoded keys list that didn't include EXCLUDE_SOURCES. The result: setting EXCLUDE_SOURCES in the environment silently no-op'd through the Python pipeline. Only the bash hook (which reads shell env directly) worked. The PR's unit tests didn't catch this because they construct config dicts directly, bypassing get_config(). Changes: - Add ('EXCLUDE_SOURCES', '') to env.py's keys list so the env var actually propagates into config. - Add an end-to-end regression test that goes through get_config() rather than constructing config dicts directly. - Document EXCLUDE_SOURCES in SKILL.md's source-list checklist so the model invoking the skill knows to subtract excluded sources before displaying the active-sources line. (Per AGENTS.md: engine flags without SKILL.md prose are incomplete — the agent invoking the skill won't know the flag exists.) --- skills/last30days/SKILL.md | 1 + skills/last30days/scripts/lib/env.py | 1 + tests/test_pipeline_v3.py | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index a7db05f..2b47dc0 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -327,6 +327,7 @@ Common patterns: - If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest - If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky - If OPENROUTER_API_KEY is set: add Perplexity +- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma): diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index e78f012..81f5641 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -265,6 +265,7 @@ def get_config() -> dict[str, Any]: ('FROM_BROWSER', None), ('SETUP_COMPLETE', None), ('INCLUDE_SOURCES', ''), + ('EXCLUDE_SOURCES', ''), ] for key, default in keys: diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py index 54dcc18..5ccddd1 100644 --- a/tests/test_pipeline_v3.py +++ b/tests/test_pipeline_v3.py @@ -956,5 +956,29 @@ class TestExcludeSources(unittest.TestCase): self.assertIn("reddit", sources) +class TestExcludeSourcesEndToEnd(unittest.TestCase): + """Wiring regression: EXCLUDE_SOURCES from the process environment must + reach available_sources() via env.get_config(). The unit tests above + construct config dicts directly; this one exercises the env-to-config + path so a missing entry in env.py's keys list is caught immediately.""" + + def test_exclude_sources_from_env_propagates_through_get_config(self): + import os + from unittest.mock import patch as _patch + from lib import env as env_mod + from importlib import reload + with _patch.dict(os.environ, { + "LAST30DAYS_CONFIG_DIR": "", + "EXCLUDE_SOURCES": "tiktok,instagram", + "SCRAPECREATORS_API_KEY": "fake", + }, clear=False): + reload(env_mod) + cfg = env_mod.get_config() + self.assertEqual(cfg.get("EXCLUDE_SOURCES"), "tiktok,instagram") + sources = pipeline.available_sources(cfg) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + + if __name__ == "__main__": unittest.main()