From 61904b31e3ec16ce5bee77710196ecbd1129c5df Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Mon, 30 Mar 2026 06:22:37 -0700 Subject: [PATCH] feat: INCLUDE_SOURCES config + TikTok/Instagram opt-in in NUX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: INCLUDE_SOURCES config + TikTok/Instagram opt-in in NUX - INCLUDE_SOURCES=tiktok,instagram in .env forces sources on for all query types, bypassing the tier system - NUX shows opt-in modal after ScrapeCreators key is saved: "Also search TikTok and Instagram?" with honest call-usage warning - Tier system preserved as default — override only when INCLUDE_SOURCES set Co-Authored-By: Claude Opus 4.6 (1M context) * fix: neutral call-usage copy — works for free and paid tiers --------- Co-authored-by: Matt Van Horn Co-authored-by: Claude Opus 4.6 (1M context) --- SKILL.md | 12 ++++++- scripts/last30days.py | 12 +++++++ scripts/lib/env.py | 1 + tests/test_source_priority.py | 67 +++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/SKILL.md b/SKILL.md index baa0c5f..0531b5f 100644 --- a/SKILL.md +++ b/SKILL.md @@ -112,7 +112,17 @@ Options: - "I have a key — let me paste it" — accept the key, write to .env - "Skip for now — start researching" — proceed without ScrapeCreators -**After ScrapeCreators modal (or skip), show the first research topic modal:** +**After SC key is saved (not if skipped), show the TikTok/Instagram opt-in:** + +Your ScrapeCreators key also powers TikTok and Instagram search. Want those on for every research run? (Each additional source uses a ScrapeCreators call per search.) + +**Call AskUserQuestion:** +Question: "Also search TikTok and Instagram?" +Options: +- "Yes, search everything — TikTok + Instagram on every run" — append `INCLUDE_SOURCES=tiktok,instagram` to ~/.config/last30days/.env. Confirm: "All set — Reddit comments, TikTok, and Instagram will run on every search." +- "Just Reddit comments for now" — don't write the flag. Confirm: "Got it — Reddit comments are on. TikTok and Instagram will kick in automatically for relevant topics like product reviews and trends. Add INCLUDE_SOURCES=tiktok,instagram to your .env anytime to force them on." + +**After TikTok/Instagram opt-in (or SC skip), show the first research topic modal:** **Call AskUserQuestion:** Question: "What do you want to research first?" diff --git a/scripts/last30days.py b/scripts/last30days.py index 48ebf1e..92e0f7c 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -1735,6 +1735,18 @@ def main(): search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type) search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type) search_run_xiaohongshu = has_xiaohongshu + + # INCLUDE_SOURCES override: force specific sources on regardless of tier + _include_sources = {s.strip().lower() for s in config.get('INCLUDE_SOURCES', '').split(',') if s.strip()} + if _include_sources: + if 'tiktok' in _include_sources and has_tiktok: + if not search_run_tiktok: + sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing tiktok\n") + search_run_tiktok = True + if 'instagram' in _include_sources and has_instagram: + if not search_run_instagram: + sys.stderr.write("[Config] INCLUDE_SOURCES override: forcing instagram\n") + search_run_instagram = True if args.search: search_sources = parse_search_flag(args.search) has_reddit = "reddit" in search_sources diff --git a/scripts/lib/env.py b/scripts/lib/env.py index 6058833..c678278 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -374,6 +374,7 @@ def get_config() -> Dict[str, Any]: ('TRUTHSOCIAL_TOKEN', None), ('FROM_BROWSER', None), ('SETUP_COMPLETE', None), + ('INCLUDE_SOURCES', None), ] for key, default in keys: diff --git a/tests/test_source_priority.py b/tests/test_source_priority.py index 45006b0..1f75e66 100644 --- a/tests/test_source_priority.py +++ b/tests/test_source_priority.py @@ -668,3 +668,70 @@ class TestBirdDisableBrowserCookiesEnvVar: assert 'BIRD_DISABLE_BROWSER_COOKIES' not in os.environ finally: os.environ.pop('BIRD_DISABLE_BROWSER_COOKIES', None) + + +# --------------------------------------------------------------------------- +# Tests: INCLUDE_SOURCES config override +# --------------------------------------------------------------------------- + +class TestIncludeSourcesOverride: + """Test that INCLUDE_SOURCES forces sources on regardless of tier.""" + + def _simulate_source_decisions(self, config, query_type="breaking_news"): + """Simulate the source decision logic from last30days.py main flow. + + Returns (search_run_tiktok, search_run_instagram) after tier + override. + """ + from scripts.lib import query_type as qt + + has_tiktok = env.is_tiktok_available(config) + has_instagram = env.is_instagram_available(config) + + # Tier system decision + search_run_tiktok = has_tiktok and qt.is_source_enabled("tiktok", query_type) + search_run_instagram = has_instagram and qt.is_source_enabled("instagram", query_type) + + # INCLUDE_SOURCES override (mirrors last30days.py logic) + _include_sources = { + s.strip().lower() + for s in config.get('INCLUDE_SOURCES', '').split(',') + if s.strip() + } + if _include_sources: + if 'tiktok' in _include_sources and has_tiktok: + if not search_run_tiktok: + search_run_tiktok = True + if 'instagram' in _include_sources and has_instagram: + if not search_run_instagram: + search_run_instagram = True + + return search_run_tiktok, search_run_instagram + + def test_include_sources_forces_tiktok_and_instagram_on(self): + """INCLUDE_SOURCES=tiktok,instagram + SC key + GENERAL query -> both forced on.""" + config = _base_config( + SCRAPECREATORS_API_KEY="sc-key", + INCLUDE_SOURCES="tiktok,instagram", + ) + run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news") + assert run_tiktok is True + assert run_instagram is True + + def test_no_include_sources_tier_controls(self): + """No INCLUDE_SOURCES + SC key + GENERAL query -> tier system controls (both off).""" + config = _base_config( + SCRAPECREATORS_API_KEY="sc-key", + ) + run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news") + # breaking_news tier doesn't include tiktok or instagram + assert run_tiktok is False + assert run_instagram is False + + def test_include_sources_no_sc_key_still_off(self): + """INCLUDE_SOURCES=tiktok but no SC key -> TikTok still off (no backend).""" + config = _base_config( + INCLUDE_SOURCES="tiktok", + ) + run_tiktok, run_instagram = self._simulate_source_decisions(config, "breaking_news") + assert run_tiktok is False + assert run_instagram is False