feat: honor EXCLUDE_SOURCES env var in source count + pipeline filter

Adds a per-run denylist via the existing-but-unused EXCLUDE_SOURCES
config key. Two coupled changes:

1. pipeline.available_sources() filters out any source listed in
   config["EXCLUDE_SOURCES"] (comma-separated, case-insensitive,
   whitespace-tolerant) before returning.
2. hooks/scripts/check-config.sh "Ready — N sources active" banner
   subtracts excluded sources from the ScrapeCreators +3 (Reddit
   comments + TikTok + Instagram) so the count matches what the
   pipeline actually runs.

Use case: skip TikTok/Instagram on runs where you only want
text-substantive sources, without unsetting SCRAPECREATORS_API_KEY
(which would also kill Reddit comments). The existing INCLUDE_SOURCES
allowlist covers Perplexity opt-in but doesn't cover this denylist case
— tiktok and instagram are added unconditionally when
SCRAPECREATORS_API_KEY is set, with no opt-out short of removing the key.

Tests (tests/test_pipeline_v3.py::TestExcludeSources):
- excludes tiktok+instagram when listed
- no exclusion when env unset or empty string
- case-insensitive + whitespace-tolerant parsing
- works for any source (e.g. EXCLUDE_SOURCES=hackernews), not just SC-backed
This commit is contained in:
Tobi
2026-05-16 00:54:18 +02:00
parent 80a1a47eef
commit 4f6b86c456
3 changed files with 65 additions and 1 deletions
+10 -1
View File
@@ -97,7 +97,16 @@ if [[ -n "$HAS_BSKY" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 1)) SOURCE_COUNT=$((SOURCE_COUNT + 1))
fi fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then if [[ -n "$HAS_SCRAPECREATORS" ]]; then
SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES
SC_ADD=3
EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
if [[ ",$EXCLUDED," == *",tiktok,"* ]]; then
SC_ADD=$((SC_ADD - 1))
fi
if [[ ",$EXCLUDED," == *",instagram,"* ]]; then
SC_ADD=$((SC_ADD - 1))
fi
SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
fi fi
if [[ -n "$HAS_SCRAPECREATORS" ]]; then if [[ -n "$HAS_SCRAPECREATORS" ]]; then
@@ -128,6 +128,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
available.append("pinterest") available.append("pinterest")
if env.is_xquik_available(config): if env.is_xquik_available(config):
available.append("xquik") available.append("xquik")
exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
if exclude:
available = [s for s in available if s not in exclude]
return available return available
+52
View File
@@ -904,5 +904,57 @@ class TestZeroKeyPipelineRun(unittest.TestCase):
self.assertEqual("fallback-local-score", candidate.explanation) self.assertEqual("fallback-local-score", candidate.explanation)
class TestExcludeSources(unittest.TestCase):
"""EXCLUDE_SOURCES env var filters sources out of available_sources().
The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does
not cover this case — tiktok and instagram are added unconditionally
when SCRAPECREATORS_API_KEY is set, with no way to opt out short of
unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist.
"""
def test_excludes_tiktok_and_instagram(self):
config = {
"SCRAPECREATORS_API_KEY": "test-key",
"EXCLUDE_SOURCES": "tiktok,instagram",
}
sources = pipeline.available_sources(config)
self.assertNotIn("tiktok", sources)
self.assertNotIn("instagram", sources)
self.assertIn("reddit", sources)
self.assertIn("hackernews", sources)
def test_no_exclusion_when_unset(self):
config = {"SCRAPECREATORS_API_KEY": "test-key"}
sources = pipeline.available_sources(config)
self.assertIn("tiktok", sources)
self.assertIn("instagram", sources)
def test_empty_exclude_sources_is_noop(self):
config = {
"SCRAPECREATORS_API_KEY": "test-key",
"EXCLUDE_SOURCES": "",
}
sources = pipeline.available_sources(config)
self.assertIn("tiktok", sources)
self.assertIn("instagram", sources)
def test_whitespace_and_case_insensitive(self):
config = {
"SCRAPECREATORS_API_KEY": "test-key",
"EXCLUDE_SOURCES": " TikTok , INSTAGRAM ",
}
sources = pipeline.available_sources(config)
self.assertNotIn("tiktok", sources)
self.assertNotIn("instagram", sources)
def test_excludes_non_scrapecreators_source(self):
"""EXCLUDE_SOURCES applies to any source, not just SC-backed ones."""
config = {"EXCLUDE_SOURCES": "hackernews"}
sources = pipeline.available_sources(config)
self.assertNotIn("hackernews", sources)
self.assertIn("reddit", sources)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()