diff --git a/hooks/scripts/check-config.sh b/hooks/scripts/check-config.sh index 39b4bbd..85403cc 100755 --- a/hooks/scripts/check-config.sh +++ b/hooks/scripts/check-config.sh @@ -97,7 +97,20 @@ if [[ -n "$HAS_BSKY" ]]; then SOURCE_COUNT=$((SOURCE_COUNT + 1)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then - SOURCE_COUNT=$((SOURCE_COUNT + 3)) # Reddit comments + TikTok + Instagram + # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES. + # Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer + # whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing. + SC_ADD=3 + EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}" + EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//') + if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then + SC_ADD=$((SC_ADD - 1)) + fi + SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD)) fi if [[ -n "$HAS_SCRAPECREATORS" ]]; then diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 06a63ca..16f89b6 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -336,6 +336,7 @@ Common patterns: - If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest - If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky - If OPENROUTER_API_KEY is set: add Perplexity +- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma): diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index b8dee18..58751a4 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -328,6 +328,7 @@ def get_config() -> dict[str, Any]: ('FROM_BROWSER', None), ('SETUP_COMPLETE', None), ('INCLUDE_SOURCES', ''), + ('EXCLUDE_SOURCES', ''), ] for key, default in keys: diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py index ebedd45..30bcbbf 100644 --- a/skills/last30days/scripts/lib/pipeline.py +++ b/skills/last30days/scripts/lib/pipeline.py @@ -128,6 +128,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non available.append("pinterest") if env.is_xquik_available(config): available.append("xquik") + exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()} + if exclude: + available = [s for s in available if s not in exclude] return available diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py index 08b6109..5ccddd1 100644 --- a/tests/test_pipeline_v3.py +++ b/tests/test_pipeline_v3.py @@ -904,5 +904,81 @@ class TestZeroKeyPipelineRun(unittest.TestCase): self.assertEqual("fallback-local-score", candidate.explanation) +class TestExcludeSources(unittest.TestCase): + """EXCLUDE_SOURCES env var filters sources out of available_sources(). + + The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does + not cover this case — tiktok and instagram are added unconditionally + when SCRAPECREATORS_API_KEY is set, with no way to opt out short of + unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist. + """ + + def test_excludes_tiktok_and_instagram(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "tiktok,instagram", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + self.assertIn("reddit", sources) + self.assertIn("hackernews", sources) + + def test_no_exclusion_when_unset(self): + config = {"SCRAPECREATORS_API_KEY": "test-key"} + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_empty_exclude_sources_is_noop(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": "", + } + sources = pipeline.available_sources(config) + self.assertIn("tiktok", sources) + self.assertIn("instagram", sources) + + def test_whitespace_and_case_insensitive(self): + config = { + "SCRAPECREATORS_API_KEY": "test-key", + "EXCLUDE_SOURCES": " TikTok , INSTAGRAM ", + } + sources = pipeline.available_sources(config) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + + def test_excludes_non_scrapecreators_source(self): + """EXCLUDE_SOURCES applies to any source, not just SC-backed ones.""" + config = {"EXCLUDE_SOURCES": "hackernews"} + sources = pipeline.available_sources(config) + self.assertNotIn("hackernews", sources) + self.assertIn("reddit", sources) + + +class TestExcludeSourcesEndToEnd(unittest.TestCase): + """Wiring regression: EXCLUDE_SOURCES from the process environment must + reach available_sources() via env.get_config(). The unit tests above + construct config dicts directly; this one exercises the env-to-config + path so a missing entry in env.py's keys list is caught immediately.""" + + def test_exclude_sources_from_env_propagates_through_get_config(self): + import os + from unittest.mock import patch as _patch + from lib import env as env_mod + from importlib import reload + with _patch.dict(os.environ, { + "LAST30DAYS_CONFIG_DIR": "", + "EXCLUDE_SOURCES": "tiktok,instagram", + "SCRAPECREATORS_API_KEY": "fake", + }, clear=False): + reload(env_mod) + cfg = env_mod.get_config() + self.assertEqual(cfg.get("EXCLUDE_SOURCES"), "tiktok,instagram") + sources = pipeline.available_sources(cfg) + self.assertNotIn("tiktok", sources) + self.assertNotIn("instagram", sources) + + if __name__ == "__main__": unittest.main()