From 9f08bb68b5dcab6f516360a3f5bbb1ada5861249 Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Sat, 16 May 2026 20:14:35 -0700 Subject: [PATCH] fix(sources): align SC source gating between code and docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related drifts surfaced while reviewing PR #399 (EXCLUDE_SOURCES) — docs claimed several SC-backed sources required INCLUDE_SOURCES opt-in that the code didn't actually enforce, and threads was inconsistently gated relative to its same-key siblings. This commit picks the "code as source of truth + EXCLUDE_SOURCES as suppression knob" model and aligns docs to match. It also promotes threads to the same auto-on tier as tiktok and instagram, since all three share the SC key and per-call cost shape — there was no real product reason for threads being opt-in while the other two weren't. The resulting source-gating model is three-tier and intentional: • **Auto-on if backing infra present** (suppress via EXCLUDE_SOURCES): reddit, HN, polymarket, X, youtube, github, bluesky, truthsocial, grounding, **tiktok, instagram, threads** • **INCLUDE_SOURCES persistent opt-in** (cost/billing reasons): perplexity (different paid API — OpenRouter), tiktok_comments / youtube_comments (N× extra SC calls per video) • **--search per-query opt-in** (relevance reasons): pinterest (visual pins, narrow utility), xiaohongshu (Chinese-market specific) Changes: - env.py: `is_threads_available()` drops the INCLUDE_SOURCES check, now mirrors tiktok/instagram (SC key → True). Docstring updated. - tests/test_env_v3.py: new `ThreadsAvailabilityTests` class locks in the new contract and includes a regression guard ("INCLUDE_SOURCES should not be needed"). - SKILL.md: lines 333-338 rewritten so the model's "Build ACTIVE_SOURCES_LIST" checklist reflects what the engine actually runs. Drops false INCLUDE_SOURCES requirement for tiktok/instagram/threads; corrects pinterest to mention --search; adds missing INCLUDE_SOURCES=perplexity requirement. - README: same alignment for the user-facing "Everything else in v3" section. Note on EXCLUDE_SOURCES references in the new docs: the suppression flag is wired up in PR #399. SKILL.md and README mention EXCLUDE_SOURCES as the opt-out path; that prose is forward-looking until #399 lands. The behavior changes in this PR (threads auto-on) are self-contained and don't require #399 to function — but for users who want to suppress the newly-auto-on threads source, #399 needs to land first. --- README.md | 6 ++++-- skills/last30days/SKILL.md | 8 +++----- skills/last30days/scripts/lib/env.py | 10 +++++----- tests/test_env_v3.py | 20 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index dfdfbf2..0731525 100644 --- a/README.md +++ b/README.md @@ -152,8 +152,10 @@ Say "eli5 on" after any research run. The synthesis rewrites in plain language. - **Free Reddit comments.** Public JSON gives you threads + top comments with upvote counts. No API key, no ScrapeCreators. Just works. - **YouTube transcripts that actually work.** Widened candidate pool 3x past music videos to reach talk/review content with captions. -- **Threads, Pinterest, YouTube + TikTok comments.** Opt-in sources via ScrapeCreators. Set `INCLUDE_SOURCES=tiktok,instagram` and add threads, pinterest, youtube_comments, tiktok_comments for more. `youtube_comments` and `tiktok_comments` surface top comments with vote counts the same way Reddit does. -- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` to unlock. +- **TikTok, Instagram, Threads.** All three activate automatically once `SCRAPECREATORS_API_KEY` is set — same key, same per-call cost. Suppress any of them with `EXCLUDE_SOURCES=tiktok,instagram,threads` (any comma-separated subset). +- **Pinterest.** Per-query opt-in (visual pins, narrow utility): the model passes `--search=pinterest` for the runs that need it. Requires `SCRAPECREATORS_API_KEY`. +- **YouTube + TikTok comments.** Persistent opt-in via `INCLUDE_SOURCES=youtube_comments,tiktok_comments` because each video pulls N extra ScrapeCreators calls on top of the base search. Surface top comments with vote counts the same way Reddit does. +- **Perplexity Sonar.** Grounded web search with citations via OpenRouter. Add `OPENROUTER_API_KEY` and `INCLUDE_SOURCES=perplexity` (it's a separate paid API — opt-in keeps you from being surprise-billed). - **Polymarket noise filtering.** Common-word disambiguation prevents "Apple" from matching "Will Apple release a car?" - **Resilient Reddit.** Timeout budgets and runtime fallback. One slow thread doesn't kill the whole run. - **Fun judge v2.** Humor scoring baked into the narrative. Reddit's cleverest one-liners mixed into the synthesis where they fit, not dumped in a separate section. diff --git a/skills/last30days/SKILL.md b/skills/last30days/SKILL.md index 16f89b6..a6c6b03 100644 --- a/skills/last30days/SKILL.md +++ b/skills/last30days/SKILL.md @@ -330,12 +330,10 @@ Common patterns: - If digg-pp-cli is installed (check `which digg-pp-cli`): add Digg - If AUTH_TOKEN/CT0 or XAI_API_KEY or FROM_BROWSER is set, or xurl CLI is installed and authenticated: add X - If yt-dlp is installed (check `which yt-dlp`): add YouTube -- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains tiktok: add TikTok -- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains instagram: add Instagram -- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains threads: add Threads -- If SCRAPECREATORS_API_KEY is set and INCLUDE_SOURCES contains pinterest: add Pinterest +- If SCRAPECREATORS_API_KEY is set: add TikTok, Instagram, Threads (suppress any of these via EXCLUDE_SOURCES) +- If SCRAPECREATORS_API_KEY is set and the user explicitly requested pinterest for this query (e.g. via `--search=pinterest`): add Pinterest - If BSKY_HANDLE and BSKY_APP_PASSWORD are set: add Bluesky -- If OPENROUTER_API_KEY is set: add Perplexity +- If OPENROUTER_API_KEY is set and INCLUDE_SOURCES contains perplexity: add Perplexity - If EXCLUDE_SOURCES is set (comma-separated, case-insensitive): drop any matching source from the list above before displaying Then display (use "and more" if 5+ sources, otherwise list all with Oxford comma): diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 58751a4..870b75a 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -584,12 +584,12 @@ def _parse_include_sources(config: dict[str, Any]) -> set[str]: def is_threads_available(config: dict[str, Any]) -> bool: """Check if Threads source is available. - Requires SCRAPECREATORS_API_KEY AND 'threads' in INCLUDE_SOURCES. - Threads is an opt-in source - it is not activated by default. + Returns True when SCRAPECREATORS_API_KEY is set. Threads runs alongside + TikTok and Instagram as part of the SC family — same key, same per-call + cost shape, so the same default-on rule applies. Suppress via + EXCLUDE_SOURCES=threads. """ - if not config.get('SCRAPECREATORS_API_KEY'): - return False - return 'threads' in _parse_include_sources(config) + return bool(config.get('SCRAPECREATORS_API_KEY')) def is_instagram_available(config: dict[str, Any]) -> bool: diff --git a/tests/test_env_v3.py b/tests/test_env_v3.py index a79ec69..b2a1f51 100644 --- a/tests/test_env_v3.py +++ b/tests/test_env_v3.py @@ -42,5 +42,25 @@ class EnvV3Tests(unittest.TestCase): self.assertIsNone(bird_x.is_bird_authenticated()) +class ThreadsAvailabilityTests(unittest.TestCase): + """Threads is in the SC default-on family: same key, same per-call cost + shape as TikTok / Instagram, so the same default-on rule applies. + Suppression goes through EXCLUDE_SOURCES, not gated opt-in.""" + + def test_threads_available_with_sc_key_only(self): + self.assertTrue(env.is_threads_available({"SCRAPECREATORS_API_KEY": "k"})) + + def test_threads_unavailable_without_sc_key(self): + self.assertFalse(env.is_threads_available({})) + self.assertFalse(env.is_threads_available({"INCLUDE_SOURCES": "threads"})) + + def test_threads_does_not_require_include_sources(self): + """Regression guard: INCLUDE_SOURCES should not be needed.""" + self.assertTrue(env.is_threads_available({ + "SCRAPECREATORS_API_KEY": "k", + "INCLUDE_SOURCES": "", + })) + + if __name__ == "__main__": unittest.main()