From 4f6b86c4569946f20275e0875aee60776f4d0661 Mon Sep 17 00:00:00 2001 From: Tobi Date: Sat, 16 May 2026 00:54:18 +0200 Subject: [PATCH] feat: honor EXCLUDE_SOURCES env var in source count + pipeline filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a per-run denylist via the existing-but-unused EXCLUDE_SOURCES config key. Two coupled changes: 1. pipeline.available_sources() filters out any source listed in config["EXCLUDE_SOURCES"] (comma-separated, case-insensitive, whitespace-tolerant) before returning. 2. hooks/scripts/check-config.sh "Ready — N sources active" banner subtracts excluded sources from the ScrapeCreators +3 (Reddit comments + TikTok + Instagram) so the count matches what the pipeline actually runs. Use case: skip TikTok/Instagram on runs where you only want text-substantive sources, without unsetting SCRAPECREATORS_API_KEY (which would also kill Reddit comments). The existing INCLUDE_SOURCES allowlist covers Perplexity opt-in but doesn't cover this denylist case — tiktok and instagram are added unconditionally when SCRAPECREATORS_API_KEY is set, with no opt-out short of removing the key. Tests (tests/test_pipeline_v3.py::TestExcludeSources): - excludes tiktok+instagram when listed - no exclusion when env unset or empty string - case-insensitive + whitespace-tolerant parsing - works for any source (e.g. EXCLUDE_SOURCES=hackernews), not just SC-backed --- hooks/scripts/check-config.sh | 11 ++++- skills/last30days/scripts/lib/pipeline.py | 3 ++ tests/test_pipeline_v3.py | 52 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 39b4bbd..81da6d2 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -97,7 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then SOURCE_COUNT=$((SOURCE_COUNT + 1)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then - SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram + # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES + SC_ADD=3 + EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}" + if [[ ",$EXCLUDED," == *",tiktok,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + if [[ ",$EXCLUDED," == *",instagram,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index ebedd45..30bcbbf 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -128,6 +128,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non available.append("pinterest") if env.is_xquik_available(config): available.append("xquik") + exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()} + if exclude: + available = [s for s in available if s not in exclude] return available diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py index 08b6109..54dcc18 100644 --- a/tests/test_pipeline_v3.py +++ b/tests/test_pipeline_v3.py @@ -904,5 +904,57 @@ class TestZeroKeyPipelineRun(unittest.TestCase): self.assertEqual("fallback-local-score", candidate.explanation) +class TestExcludeSources(unittest.TestCase): + """EXCLUDE_SOURCES env var filters sources out of available_sources(). + + The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does + not cover this case — tiktok and instagram are added unconditionally + when SCRAPECREATORS_API_KEY is set, with no way to opt out short of + unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist. + """ + + def test_excludes_tiktok_and_instagram(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "tiktok,instagram", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + self.assertIn("reddit", sources) + self.assertIn("hackernews", sources) + + def test_no_exclusion_when_unset(self): + config = {"SCRAPECREATORS_API_KEY": "test-key"} + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_empty_exclude_sources_is_noop(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "", + } + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_whitespace_and_case_insensitive(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": " TikTok , INSTAGRAM ", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + + def test_excludes_non_scrapecreators_source(self): + """EXCLUDE_SOURCES applies to any source, not just SC-backed ones.""" + config = {"EXCLUDE_SOURCES": "hackernews"} + sources = pipeline.available_sources(config) + self.assertNotIn("hackernews", sources) + self.assertIn("reddit", sources) + + if __name__ == "__main__": unittest.main()