Merge pull request #399 from spiky02plateau/feat/exclude-sources-banner-and-pipeline

feat: honor EXCLUDE_SOURCES env var in source count + pipeline filter

hooks/scripts/check-config.sh

@@ -97,7 +97,20 @@ if [[ -n "$HAS_BSKY" ]]; then
   SOURCE_COUNT=$((SOURCE_COUNT + 1))
 fi
 if [[ -n "$HAS_SCRAPECREATORS" ]]; then
-  SOURCE_COUNT=$((SOURCE_COUNT + 3))  # Reddit comments + TikTok + Instagram
+  # Start with Reddit comments + TikTok + Instagram, subtract any in EXCLUDE_SOURCES.
+  # Normalise EXCLUDED (lowercase + collapse whitespace around commas + strip outer
+  # whitespace) so the matching mirrors pipeline.py's .strip().lower() parsing.
+  SC_ADD=3
+  EXCLUDED="${ENV_EXCLUDE_SOURCES:-${EXCLUDE_SOURCES:-}}"
+  EXCLUDED_NORM=$(printf '%s' "$EXCLUDED" | tr '[:upper:]' '[:lower:]' \
+    | sed -E 's/[[:space:]]*,[[:space:]]*/,/g; s/^[[:space:]]+//; s/[[:space:]]+$//')
+  if [[ ",$EXCLUDED_NORM," == *",tiktok,"* ]]; then
+    SC_ADD=$((SC_ADD - 1))
+  fi
+  if [[ ",$EXCLUDED_NORM," == *",instagram,"* ]]; then
+    SC_ADD=$((SC_ADD - 1))
+  fi
+  SOURCE_COUNT=$((SOURCE_COUNT + SC_ADD))
 fi
 
 if [[ -n "$HAS_SCRAPECREATORS" ]]; then

skills/last30days/SKILL.md

@@ -336,6 +336,7 @@ Common patterns:
 - If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest
 - If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky
 - If OPENROUTER_API_KEY is set: add Perplexity
+- If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying
 
 Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma):
 

skills/last30days/scripts/lib/env.py

@@ -328,6 +328,7 @@ def get_config() -> dict[str, Any]:
         ('FROM_BROWSER', None),
         ('SETUP_COMPLETE', None),
         ('INCLUDE_SOURCES', ''),
+        ('EXCLUDE_SOURCES', ''),
     ]
 
     for key, default in keys:

skills/last30days/scripts/lib/pipeline.py

@@ -128,6 +128,9 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
         available.append("pinterest")
     if env.is_xquik_available(config):
         available.append("xquik")
+    exclude = {s.strip().lower() for s in (config.get("EXCLUDE_SOURCES") or "").split(",") if s.strip()}
+    if exclude:
+        available = [s for s in available if s not in exclude]
     return available
 
 

tests/test_pipeline_v3.py

@@ -904,5 +904,81 @@ class TestZeroKeyPipelineRun(unittest.TestCase):
             self.assertEqual("fallback-local-score", candidate.explanation)
 
 
+class TestExcludeSources(unittest.TestCase):
+    """EXCLUDE_SOURCES env var filters sources out of available_sources().
+
+    The existing INCLUDE_SOURCES allowlist (used by Perplexity opt-in) does
+    not cover this case — tiktok and instagram are added unconditionally
+    when SCRAPECREATORS_API_KEY is set, with no way to opt out short of
+    unsetting the key. EXCLUDE_SOURCES gives runs a per-invocation denylist.
+    """
+
+    def test_excludes_tiktok_and_instagram(self):
+        config = {
+            "SCRAPECREATORS_API_KEY": "test-key",
+            "EXCLUDE_SOURCES": "tiktok,instagram",
+        }
+        sources = pipeline.available_sources(config)
+        self.assertNotIn("tiktok", sources)
+        self.assertNotIn("instagram", sources)
+        self.assertIn("reddit", sources)
+        self.assertIn("hackernews", sources)
+
+    def test_no_exclusion_when_unset(self):
+        config = {"SCRAPECREATORS_API_KEY": "test-key"}
+        sources = pipeline.available_sources(config)
+        self.assertIn("tiktok", sources)
+        self.assertIn("instagram", sources)
+
+    def test_empty_exclude_sources_is_noop(self):
+        config = {
+            "SCRAPECREATORS_API_KEY": "test-key",
+            "EXCLUDE_SOURCES": "",
+        }
+        sources = pipeline.available_sources(config)
+        self.assertIn("tiktok", sources)
+        self.assertIn("instagram", sources)
+
+    def test_whitespace_and_case_insensitive(self):
+        config = {
+            "SCRAPECREATORS_API_KEY": "test-key",
+            "EXCLUDE_SOURCES": " TikTok , INSTAGRAM ",
+        }
+        sources = pipeline.available_sources(config)
+        self.assertNotIn("tiktok", sources)
+        self.assertNotIn("instagram", sources)
+
+    def test_excludes_non_scrapecreators_source(self):
+        """EXCLUDE_SOURCES applies to any source, not just SC-backed ones."""
+        config = {"EXCLUDE_SOURCES": "hackernews"}
+        sources = pipeline.available_sources(config)
+        self.assertNotIn("hackernews", sources)
+        self.assertIn("reddit", sources)
+
+
+class TestExcludeSourcesEndToEnd(unittest.TestCase):
+    """Wiring regression: EXCLUDE_SOURCES from the process environment must
+    reach available_sources() via env.get_config(). The unit tests above
+    construct config dicts directly; this one exercises the env-to-config
+    path so a missing entry in env.py's keys list is caught immediately."""
+
+    def test_exclude_sources_from_env_propagates_through_get_config(self):
+        import os
+        from unittest.mock import patch as _patch
+        from lib import env as env_mod
+        from importlib import reload
+        with _patch.dict(os.environ, {
+            "LAST30DAYS_CONFIG_DIR": "",
+            "EXCLUDE_SOURCES": "tiktok,instagram",
+            "SCRAPECREATORS_API_KEY": "fake",
+        }, clear=False):
+            reload(env_mod)
+            cfg = env_mod.get_config()
+        self.assertEqual(cfg.get("EXCLUDE_SOURCES"), "tiktok,instagram")
+        sources = pipeline.available_sources(cfg)
+        self.assertNotIn("tiktok", sources)
+        self.assertNotIn("instagram", sources)
+
+
 if __name__ == "__main__":
     unittest.main()