feat: honor EXCLUDE_SOURCES env var in source count + pipeline filter
Adds a per-run denylist via the existing-but-unused EXCLUDE_SOURCES config key. Two coupled changes: 1. pipeline.available_sources() filters out any source listed in config["EXCLUDE_SOURCES"] (comma-separated, case-insensitive, whitespace-tolerant) before returning. 2. hooks/scripts/check-config.sh "Ready — N sources active" banner subtracts excluded sources from the ScrapeCreators +3 (Reddit comments + TikTok + Instagram) so the count matches what the pipeline actually runs. Use case: skip TikTok/Instagram on runs where you only want text-substantive sources, without unsetting SCRAPECREATORS_API_KEY (which would also kill Reddit comments). The existing INCLUDE_SOURCES allowlist covers Perplexity opt-in but doesn't cover this denylist case — tiktok and instagram are added unconditionally when SCRAPECREATORS_API_KEY is set, with no opt-out short of removing the key. Tests (tests/test_pipeline_v3.py::TestExcludeSources): - excludes tiktok+instagram when listed - no exclusion when env unset or empty string - case-insensitive + whitespace-tolerant parsing - works for any source (e.g. EXCLUDE_SOURCES=hackernews), not just SC-backed
This commit is contained in:
@@ -904,5 +904,57 @@ class TestZeroKeyPipelineRun(unittest.TestCase):
|
||||
self.assertEqual("fallback-local-score", candidate.explanation)
|
||||
|
||||
|
||||
class TestExcludeSources(unittest.TestCase):
|
||||
"""EXCLUDE_SOURCES env var filters sources out of available_sources().
|
||||
|
||||
The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does
|
||||
not cover this case — tiktok and instagram are added unconditionally
|
||||
when SCRAPECREATORS_API_KEY is set, with no way to opt out short of
|
||||
unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist.
|
||||
"""
|
||||
|
||||
def test_excludes_tiktok_and_instagram(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": "tiktok,instagram",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("tiktok", sources)
|
||||
self.assertNotIn("instagram", sources)
|
||||
self.assertIn("reddit", sources)
|
||||
self.assertIn("hackernews", sources)
|
||||
|
||||
def test_no_exclusion_when_unset(self):
|
||||
config = {"SCRAPECREATORS_API_KEY": "test-key"}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertIn("tiktok", sources)
|
||||
self.assertIn("instagram", sources)
|
||||
|
||||
def test_empty_exclude_sources_is_noop(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": "",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertIn("tiktok", sources)
|
||||
self.assertIn("instagram", sources)
|
||||
|
||||
def test_whitespace_and_case_insensitive(self):
|
||||
config = {
|
||||
"SCRAPECREATORS_API_KEY": "test-key",
|
||||
"EXCLUDE_SOURCES": " TikTok , INSTAGRAM ",
|
||||
}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("tiktok", sources)
|
||||
self.assertNotIn("instagram", sources)
|
||||
|
||||
def test_excludes_non_scrapecreators_source(self):
|
||||
"""EXCLUDE_SOURCES applies to any source, not just SC-backed ones."""
|
||||
config = {"EXCLUDE_SOURCES": "hackernews"}
|
||||
sources = pipeline.available_sources(config)
|
||||
self.assertNotIn("hackernews", sources)
|
||||
self.assertIn("reddit", sources)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user