From 9ca84e495ee2f92769231f63b4f561069a2cbd9c Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 15:56:14 -0700 Subject: [PATCH 1/6] Fix stale test assertions and truthsocial pytest dependency - test_models: update xAI model expectations to grok-4-1-fast (matching current XAI_POLICY_MAP) - test_openai_reddit: update fallback order assertion to gpt-4.1 (matching current MODEL_FALLBACK_ORDER) - test_codex_auth: expect 'reddit' not 'web' when no API keys (Reddit is available via public JSON fallback) - test_truthsocial: convert from pytest-style classes to unittest.TestCase, fix import path to use sys.path.insert pattern (matching all other tests) --- tests/test_codex_auth.py | 3 +- tests/test_models.py | 8 +- tests/test_openai_reddit.py | 6 +- tests/test_truthsocial.py | 142 +++++++++++++++++++----------------- 4 files changed, 84 insertions(+), 75 deletions(-) diff --git a/tests/test_codex_auth.py b/tests/test_codex_auth.py index 595971e..c71a520 100644 --- a/tests/test_codex_auth.py +++ b/tests/test_codex_auth.py @@ -140,7 +140,8 @@ class TestGetAvailableSourcesWithAuth(unittest.TestCase): "XAI_API_KEY": None, } result = env.get_available_sources(config) - self.assertEqual(result, "web") + # Reddit is available via public JSON fallback even without OpenAI auth + self.assertEqual(result, "reddit") class TestParseCodexStream(unittest.TestCase): diff --git a/tests/test_models.py b/tests/test_models.py index 0baa42b..1b862ec 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -84,7 +84,7 @@ class TestSelectXAIModel(unittest.TestCase): "fake-key", policy="latest" ) - self.assertEqual(result, "grok-4-latest") + self.assertEqual(result, "grok-4-1-fast") def test_stable_policy(self): # Clear cache first to avoid interference @@ -94,7 +94,7 @@ class TestSelectXAIModel(unittest.TestCase): "fake-key", policy="stable" ) - self.assertEqual(result, "grok-4") + self.assertEqual(result, "grok-4-1-fast") def test_pinned_policy(self): result = models.select_xai_model( @@ -125,10 +125,10 @@ class TestGetModels(unittest.TestCase): "XAI_API_KEY": "xai-test", } mock_openai = [{"id": "gpt-5.2", "created": 1704067200}] - mock_xai = [{"id": "grok-4-latest", "created": 1704067200}] + mock_xai = [{"id": "grok-4-1-fast", "created": 1704067200}] result = models.get_models(config, mock_openai, mock_xai) self.assertEqual(result["openai"], "gpt-5.2") - self.assertEqual(result["xai"], "grok-4-latest") + self.assertEqual(result["xai"], "grok-4-1-fast") if __name__ == "__main__": diff --git a/tests/test_openai_reddit.py b/tests/test_openai_reddit.py index 2748007..607be97 100644 --- a/tests/test_openai_reddit.py +++ b/tests/test_openai_reddit.py @@ -68,9 +68,9 @@ class TestModelFallbackOrder(unittest.TestCase): """Fallback list should include gpt-4o.""" self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER) - def test_gpt4o_is_first(self): - """gpt-4o should be the first fallback option.""" - self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4o") + def test_gpt41_is_first(self): + """gpt-4.1 should be the first fallback option.""" + self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4.1") if __name__ == "__main__": diff --git a/tests/test_truthsocial.py b/tests/test_truthsocial.py index 2901e1e..e827eab 100644 --- a/tests/test_truthsocial.py +++ b/tests/test_truthsocial.py @@ -1,129 +1,133 @@ """Tests for Truth Social source module.""" -import pytest +import sys +import unittest +from pathlib import Path from unittest.mock import patch, MagicMock -from scripts.lib import truthsocial +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import truthsocial -class TestStripHtml: +class TestStripHtml(unittest.TestCase): """Test HTML tag stripping.""" def test_basic_paragraph(self): - assert truthsocial._strip_html("

Hello world

") == "Hello world" + self.assertEqual(truthsocial._strip_html("

Hello world

"), "Hello world") def test_br_tags(self): - assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" - assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" - assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" + self.assertEqual(truthsocial._strip_html("Line 1
Line 2"), "Line 1\nLine 2") + self.assertEqual(truthsocial._strip_html("Line 1
Line 2"), "Line 1\nLine 2") + self.assertEqual(truthsocial._strip_html("Line 1
Line 2"), "Line 1\nLine 2") def test_nested_tags(self): - assert truthsocial._strip_html("

Hello world

") == "Hello world" + self.assertEqual(truthsocial._strip_html("

Hello world

"), "Hello world") def test_empty_string(self): - assert truthsocial._strip_html("") == "" + self.assertEqual(truthsocial._strip_html(""), "") def test_no_tags(self): - assert truthsocial._strip_html("plain text") == "plain text" + self.assertEqual(truthsocial._strip_html("plain text"), "plain text") def test_entities_preserved(self): - assert truthsocial._strip_html("

& test

") == "& test" + self.assertEqual(truthsocial._strip_html("

& test

"), "& test") -class TestExtractCoreSubject: +class TestExtractCoreSubject(unittest.TestCase): """Test query preprocessing.""" def test_strips_question_prefix(self): - assert truthsocial._extract_core_subject("what are people saying about tariffs") == "tariffs" + self.assertEqual(truthsocial._extract_core_subject("what are people saying about tariffs"), "tariffs") def test_strips_noise_words(self): - assert truthsocial._extract_core_subject("latest trending crypto news") == "crypto" + self.assertEqual(truthsocial._extract_core_subject("latest trending crypto news"), "crypto") def test_preserves_core_topic(self): - assert truthsocial._extract_core_subject("tariffs") == "tariffs" + self.assertEqual(truthsocial._extract_core_subject("tariffs"), "tariffs") def test_strips_trailing_punctuation(self): - assert truthsocial._extract_core_subject("what is bitcoin?") == "bitcoin" + self.assertEqual(truthsocial._extract_core_subject("what is bitcoin?"), "bitcoin") -class TestParseDate: +class TestParseDate(unittest.TestCase): """Test date parsing from Mastodon status.""" def test_iso_date(self): - assert truthsocial._parse_date({"created_at": "2026-03-09T12:00:00.000Z"}) == "2026-03-09" + self.assertEqual(truthsocial._parse_date({"created_at": "2026-03-09T12:00:00.000Z"}), "2026-03-09") def test_missing_date(self): - assert truthsocial._parse_date({}) is None + self.assertIsNone(truthsocial._parse_date({})) def test_short_date(self): - assert truthsocial._parse_date({"created_at": "short"}) is None + self.assertIsNone(truthsocial._parse_date({"created_at": "short"})) def test_none_value(self): - assert truthsocial._parse_date({"created_at": None}) is None + self.assertIsNone(truthsocial._parse_date({"created_at": None})) -class TestDepthConfig: +class TestDepthConfig(unittest.TestCase): """Test depth configuration.""" def test_all_depths_exist(self): - assert "quick" in truthsocial.DEPTH_CONFIG - assert "default" in truthsocial.DEPTH_CONFIG - assert "deep" in truthsocial.DEPTH_CONFIG + self.assertIn("quick", truthsocial.DEPTH_CONFIG) + self.assertIn("default", truthsocial.DEPTH_CONFIG) + self.assertIn("deep", truthsocial.DEPTH_CONFIG) def test_depth_ordering(self): - assert truthsocial.DEPTH_CONFIG["quick"] < truthsocial.DEPTH_CONFIG["default"] - assert truthsocial.DEPTH_CONFIG["default"] < truthsocial.DEPTH_CONFIG["deep"] + self.assertLess(truthsocial.DEPTH_CONFIG["quick"], truthsocial.DEPTH_CONFIG["default"]) + self.assertLess(truthsocial.DEPTH_CONFIG["default"], truthsocial.DEPTH_CONFIG["deep"]) -class TestSearchTruthSocial: +class TestSearchTruthSocial(unittest.TestCase): """Test search function auth handling.""" def test_no_config_returns_error(self): result = truthsocial.search_truthsocial("test", "2026-02-09", "2026-03-09") - assert result["statuses"] == [] - assert "not configured" in result["error"] + self.assertEqual(result["statuses"], []) + self.assertIn("not configured", result["error"]) def test_empty_token_returns_error(self): result = truthsocial.search_truthsocial( "test", "2026-02-09", "2026-03-09", config={"TRUTHSOCIAL_TOKEN": ""}, ) - assert result["statuses"] == [] - assert "not configured" in result["error"] + self.assertEqual(result["statuses"], []) + self.assertIn("not configured", result["error"]) - @patch("scripts.lib.truthsocial.http.request") + @patch("lib.truthsocial.http.request") def test_401_returns_token_expired(self, mock_request): - from scripts.lib.http import HTTPError + from lib.http import HTTPError mock_request.side_effect = HTTPError("Unauthorized", status_code=401) result = truthsocial.search_truthsocial( "test", "2026-02-09", "2026-03-09", config={"TRUTHSOCIAL_TOKEN": "expired_token"}, ) - assert result["statuses"] == [] - assert "expired" in result["error"] + self.assertEqual(result["statuses"], []) + self.assertIn("expired", result["error"]) - @patch("scripts.lib.truthsocial.http.request") + @patch("lib.truthsocial.http.request") def test_403_returns_access_denied(self, mock_request): - from scripts.lib.http import HTTPError + from lib.http import HTTPError mock_request.side_effect = HTTPError("Forbidden", status_code=403) result = truthsocial.search_truthsocial( "test", "2026-02-09", "2026-03-09", config={"TRUTHSOCIAL_TOKEN": "blocked_token"}, ) - assert result["statuses"] == [] - assert "Cloudflare" in result["error"] + self.assertEqual(result["statuses"], []) + self.assertIn("Cloudflare", result["error"]) - @patch("scripts.lib.truthsocial.http.request") + @patch("lib.truthsocial.http.request") def test_429_returns_rate_limited(self, mock_request): - from scripts.lib.http import HTTPError + from lib.http import HTTPError mock_request.side_effect = HTTPError("Too Many Requests", status_code=429) result = truthsocial.search_truthsocial( "test", "2026-02-09", "2026-03-09", config={"TRUTHSOCIAL_TOKEN": "rate_limited_token"}, ) - assert result["statuses"] == [] - assert "rate limited" in result["error"] + self.assertEqual(result["statuses"], []) + self.assertIn("rate limited", result["error"]) - @patch("scripts.lib.truthsocial.http.request") + @patch("lib.truthsocial.http.request") def test_successful_search(self, mock_request): mock_request.return_value = { "statuses": [ @@ -142,13 +146,13 @@ class TestSearchTruthSocial: "tariffs", "2026-02-09", "2026-03-09", config={"TRUTHSOCIAL_TOKEN": "valid_token"}, ) - assert len(result["statuses"]) == 1 + self.assertEqual(len(result["statuses"]), 1) # Verify bearer token was passed call_args = mock_request.call_args - assert call_args[1]["headers"]["Authorization"] == "Bearer valid_token" + self.assertEqual(call_args[1]["headers"]["Authorization"], "Bearer valid_token") -class TestParseTruthSocialResponse: +class TestParseTruthSocialResponse(unittest.TestCase): """Test response parsing.""" def test_basic_post(self): @@ -166,21 +170,21 @@ class TestParseTruthSocialResponse: ] } items = truthsocial.parse_truthsocial_response(response) - assert len(items) == 1 + self.assertEqual(len(items), 1) item = items[0] - assert item["handle"] == "testuser" - assert item["display_name"] == "Test User" - assert item["text"] == "Hello from Truth Social" # HTML stripped - assert item["url"] == "https://truthsocial.com/@testuser/456" - assert item["date"] == "2026-03-09" - assert item["engagement"]["likes"] == 100 - assert item["engagement"]["reposts"] == 50 - assert item["engagement"]["replies"] == 25 - assert item["relevance"] > 0 + self.assertEqual(item["handle"], "testuser") + self.assertEqual(item["display_name"], "Test User") + self.assertEqual(item["text"], "Hello from Truth Social") + self.assertEqual(item["url"], "https://truthsocial.com/@testuser/456") + self.assertEqual(item["date"], "2026-03-09") + self.assertEqual(item["engagement"]["likes"], 100) + self.assertEqual(item["engagement"]["reposts"], 50) + self.assertEqual(item["engagement"]["replies"], 25) + self.assertGreater(item["relevance"], 0) def test_empty_response(self): items = truthsocial.parse_truthsocial_response({"statuses": []}) - assert items == [] + self.assertEqual(items, []) def test_missing_fields(self): response = { @@ -192,10 +196,10 @@ class TestParseTruthSocialResponse: ] } items = truthsocial.parse_truthsocial_response(response) - assert len(items) == 1 - assert items[0]["handle"] == "" - assert items[0]["text"] == "" - assert items[0]["engagement"]["likes"] == 0 + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["handle"], "") + self.assertEqual(items[0]["text"], "") + self.assertEqual(items[0]["engagement"]["likes"], 0) def test_relevance_ordering(self): response = { @@ -206,8 +210,8 @@ class TestParseTruthSocialResponse: ] } items = truthsocial.parse_truthsocial_response(response) - assert items[0]["relevance"] >= items[1]["relevance"] - assert items[1]["relevance"] >= items[2]["relevance"] + self.assertGreaterEqual(items[0]["relevance"], items[1]["relevance"]) + self.assertGreaterEqual(items[1]["relevance"], items[2]["relevance"]) def test_html_stripping_in_parse(self): response = { @@ -222,5 +226,9 @@ class TestParseTruthSocialResponse: ] } items = truthsocial.parse_truthsocial_response(response) - assert "<" not in items[0]["text"] - assert ">" not in items[0]["text"] + self.assertNotIn("<", items[0]["text"]) + self.assertNotIn(">", items[0]["text"]) + + +if __name__ == "__main__": + unittest.main() From 3e9e2f632b23fbd4e80d2f9c24fe5ece06a6d0ed Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 16:42:49 -0700 Subject: [PATCH 2/6] Update stale API endpoints and model chains - Instagram: migrate /v1/ to /v2/ ScrapeCreators endpoint (v1 deprecated Feb 2026) - OpenAI: switch fallback chain to [gpt-5-mini, gpt-4.1-mini, gpt-4.1] (8x cheaper, gpt-5-mini is the first mini model supporting web_search with filters.allowed_domains) - xAI: use explicit grok-4-1-fast-non-reasoning (bare name aliases to reasoning variant) - xAI: pass from_date/to_date natively to x_search tool config instead of prompt-only - Polymarket: correct rate limit comment (15K/10s, not 350/10s) --- scripts/lib/instagram.py | 4 ++-- scripts/lib/models.py | 4 ++-- scripts/lib/openai_reddit.py | 5 +++-- scripts/lib/polymarket.py | 2 +- scripts/lib/xai_x.py | 4 ++-- tests/test_models.py | 8 ++++---- tests/test_openai_reddit.py | 12 ++++++------ 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/lib/instagram.py b/scripts/lib/instagram.py index a826113..3861a93 100644 --- a/scripts/lib/instagram.py +++ b/scripts/lib/instagram.py @@ -217,7 +217,7 @@ def search_instagram( try: resp = _requests.get( - f"{SCRAPECREATORS_BASE}/v1/instagram/reels/search", + f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search", params={"query": core_topic}, headers=_sc_headers(token), timeout=30, @@ -228,7 +228,7 @@ def search_instagram( _log(f"ScrapeCreators error: {e}") return {"items": [], "error": f"{type(e).__name__}: {e}"} - # Items are in the 'reels' array (ScrapeCreators v1 response) + # Items are in the 'reels' array (ScrapeCreators v2 response) raw_items = data.get("reels") or data.get("items") or data.get("data") or [] # Limit to configured count diff --git a/scripts/lib/models.py b/scripts/lib/models.py index 13ed3bb..2d5ec7f 100644 --- a/scripts/lib/models.py +++ b/scripts/lib/models.py @@ -13,8 +13,8 @@ CODEX_FALLBACK_MODELS = ["gpt-5.1-codex-mini", "gpt-5.2"] # xAI API - Agent Tools API requires grok-4 family XAI_MODELS_URL = "https://api.x.ai/v1/models" XAI_ALIASES = { - "latest": "grok-4-1-fast", # Required for x_search tool - "stable": "grok-4-1-fast", + "latest": "grok-4-1-fast-non-reasoning", # Explicit: bare grok-4-1-fast aliases to reasoning variant + "stable": "grok-4-1-fast-non-reasoning", } diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index f98e8e4..d4e4a1e 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -8,8 +8,9 @@ from typing import Any, Dict, List, Optional from . import http, env # Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5) -# Note: gpt-4o-mini does NOT support web_search with filters param, so exclude it -MODEL_FALLBACK_ORDER = ["gpt-4.1", "gpt-4o"] +# gpt-5-mini: $0.25/1M input (8x cheaper than gpt-4.1), supports web_search with filters +# gpt-4o-mini does NOT support web_search with filters param, so exclude it +MODEL_FALLBACK_ORDER = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1"] def _log_error(msg: str): diff --git a/scripts/lib/polymarket.py b/scripts/lib/polymarket.py index ee9922a..7ce87e0 100644 --- a/scripts/lib/polymarket.py +++ b/scripts/lib/polymarket.py @@ -1,7 +1,7 @@ """Polymarket prediction market search via Gamma API (free, no auth required). Uses gamma-api.polymarket.com for event/market discovery. -No API key needed - public read-only API with generous rate limits (350 req/10s). +No API key needed - public read-only API with generous rate limits (15K req/10s). """ import json diff --git a/scripts/lib/xai_x.py b/scripts/lib/xai_x.py index 3642dac..b00e215 100644 --- a/scripts/lib/xai_x.py +++ b/scripts/lib/xai_x.py @@ -91,11 +91,11 @@ def search_x( # Adjust timeout based on depth (generous for API response time) timeout = 90 if depth == "quick" else 120 if depth == "default" else 180 - # Use Agent Tools API with x_search tool + # Use Agent Tools API with x_search tool (native date filtering) payload = { "model": model, "tools": [ - {"type": "x_search"} + {"type": "x_search", "from_date": from_date, "to_date": to_date} ], "input": [ { diff --git a/tests/test_models.py b/tests/test_models.py index 1b862ec..493a4d7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -84,7 +84,7 @@ class TestSelectXAIModel(unittest.TestCase): "fake-key", policy="latest" ) - self.assertEqual(result, "grok-4-1-fast") + self.assertEqual(result, "grok-4-1-fast-non-reasoning") def test_stable_policy(self): # Clear cache first to avoid interference @@ -94,7 +94,7 @@ class TestSelectXAIModel(unittest.TestCase): "fake-key", policy="stable" ) - self.assertEqual(result, "grok-4-1-fast") + self.assertEqual(result, "grok-4-1-fast-non-reasoning") def test_pinned_policy(self): result = models.select_xai_model( @@ -125,10 +125,10 @@ class TestGetModels(unittest.TestCase): "XAI_API_KEY": "xai-test", } mock_openai = [{"id": "gpt-5.2", "created": 1704067200}] - mock_xai = [{"id": "grok-4-1-fast", "created": 1704067200}] + mock_xai = [{"id": "grok-4-1-fast-non-reasoning", "created": 1704067200}] result = models.get_models(config, mock_openai, mock_xai) self.assertEqual(result["openai"], "gpt-5.2") - self.assertEqual(result["xai"], "grok-4-1-fast") + self.assertEqual(result["xai"], "grok-4-1-fast-non-reasoning") if __name__ == "__main__": diff --git a/tests/test_openai_reddit.py b/tests/test_openai_reddit.py index 607be97..550fa9c 100644 --- a/tests/test_openai_reddit.py +++ b/tests/test_openai_reddit.py @@ -64,13 +64,13 @@ class TestIsModelAccessError(unittest.TestCase): class TestModelFallbackOrder(unittest.TestCase): """Tests for MODEL_FALLBACK_ORDER constant.""" - def test_contains_gpt4o(self): - """Fallback list should include gpt-4o.""" - self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER) + def test_contains_gpt41(self): + """Fallback list should include gpt-4.1 as last resort.""" + self.assertIn("gpt-4.1", MODEL_FALLBACK_ORDER) - def test_gpt41_is_first(self): - """gpt-4.1 should be the first fallback option.""" - self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4.1") + def test_gpt5_mini_is_first(self): + """gpt-5-mini should be the first fallback option (cheapest with web_search support).""" + self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-5-mini") if __name__ == "__main__": From e568ef8af94c8d93c941bdd7fd50c25346df088c Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 18:02:33 -0700 Subject: [PATCH 3/6] Revert MODEL_FALLBACK_ORDER to upstream values Model optimization (mini-first fallback, is_search_capable_model) belongs in PR #67. This PR stays focused on endpoint/API fixes only. Also fixes pre-existing test bug where test asserted gpt-4o was first in MODEL_FALLBACK_ORDER when it was actually gpt-4.1. --- scripts/lib/openai_reddit.py | 5 ++--- tests/test_openai_reddit.py | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index d4e4a1e..f98e8e4 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -8,9 +8,8 @@ from typing import Any, Dict, List, Optional from . import http, env # Fallback models when the selected model isn't accessible (e.g., org not verified for GPT-5) -# gpt-5-mini: $0.25/1M input (8x cheaper than gpt-4.1), supports web_search with filters -# gpt-4o-mini does NOT support web_search with filters param, so exclude it -MODEL_FALLBACK_ORDER = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1"] +# Note: gpt-4o-mini does NOT support web_search with filters param, so exclude it +MODEL_FALLBACK_ORDER = ["gpt-4.1", "gpt-4o"] def _log_error(msg: str): diff --git a/tests/test_openai_reddit.py b/tests/test_openai_reddit.py index 550fa9c..607be97 100644 --- a/tests/test_openai_reddit.py +++ b/tests/test_openai_reddit.py @@ -64,13 +64,13 @@ class TestIsModelAccessError(unittest.TestCase): class TestModelFallbackOrder(unittest.TestCase): """Tests for MODEL_FALLBACK_ORDER constant.""" - def test_contains_gpt41(self): - """Fallback list should include gpt-4.1 as last resort.""" - self.assertIn("gpt-4.1", MODEL_FALLBACK_ORDER) + def test_contains_gpt4o(self): + """Fallback list should include gpt-4o.""" + self.assertIn("gpt-4o", MODEL_FALLBACK_ORDER) - def test_gpt5_mini_is_first(self): - """gpt-5-mini should be the first fallback option (cheapest with web_search support).""" - self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-5-mini") + def test_gpt41_is_first(self): + """gpt-4.1 should be the first fallback option.""" + self.assertEqual(MODEL_FALLBACK_ORDER[0], "gpt-4.1") if __name__ == "__main__": From dd9a3f14826b94f259d5ac56a401134d1f94d77d Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Thu, 12 Mar 2026 21:07:04 -0700 Subject: [PATCH 4/6] Disable browser cookie fallback for local X auth Prefer injected AUTH_TOKEN/CT0 for bundled Bird, disable browser-cookie probing in repo-invoked subprocesses, and keep repo-invoked yt-dlp from inheriting browser-cookie settings. Validation: uv run python -m unittest tests.test_bird_x tests.test_youtube_yt --- scripts/lib/bird_x.py | 14 +++++- scripts/lib/vendor/bird-search/lib/cookies.js | 20 +++++++- scripts/lib/youtube_yt.py | 4 ++ tests/test_bird_x.py | 30 ++++++++++++ tests/test_youtube_yt.py | 49 +++++++++++++++++++ 5 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 tests/test_youtube_yt.py diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py index 66dc13c..548b81c 100644 --- a/scripts/lib/bird_x.py +++ b/scripts/lib/bird_x.py @@ -36,10 +36,19 @@ def set_credentials(auth_token: Optional[str], ct0: Optional[str]): _credentials['CT0'] = ct0 +def _has_injected_credentials() -> bool: + """Return True when both X session cookies were injected from config.""" + return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0')) + + def _subprocess_env() -> Dict[str, str]: """Build env dict for Node subprocesses, merging injected credentials.""" env = os.environ.copy() env.update(_credentials) + # When repo config already provides cookies, disable browser-cookie fallback + # so vendored Bird never hits Safari/Chrome keychain during automation. + if _has_injected_credentials(): + env.setdefault("BIRD_DISABLE_BROWSER_COOKIES", "1") return env @@ -126,6 +135,9 @@ def is_bird_authenticated() -> Optional[str]: if not is_bird_installed(): return None + if _has_injected_credentials(): + return "env AUTH_TOKEN" + try: result = subprocess.run( ["node", str(_BIRD_SEARCH_MJS), "--whoami"], @@ -473,4 +485,4 @@ def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: items.append(item) - return items \ No newline at end of file + return items diff --git a/scripts/lib/vendor/bird-search/lib/cookies.js b/scripts/lib/vendor/bird-search/lib/cookies.js index 83ca799..49789a1 100644 --- a/scripts/lib/vendor/bird-search/lib/cookies.js +++ b/scripts/lib/vendor/bird-search/lib/cookies.js @@ -14,6 +14,13 @@ function normalizeValue(value) { const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : null; } +function envFlagEnabled(name) { + const value = normalizeValue(process.env[name]); + if (!value) { + return false; + } + return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); +} function cookieHeader(authToken, ct0) { return `auth_token=${authToken}; ct0=${ct0}`; } @@ -123,6 +130,8 @@ export async function extractCookiesFromFirefox(profile) { export async function resolveCredentials(options) { const warnings = []; const cookies = buildEmpty(); + const disableBrowserCookies = envFlagEnabled('BIRD_DISABLE_BROWSER_COOKIES') || + envFlagEnabled('LAST30DAYS_DISABLE_BROWSER_COOKIES'); const cookieTimeoutMs = typeof options.cookieTimeoutMs === 'number' && Number.isFinite(options.cookieTimeoutMs) && options.cookieTimeoutMs > 0 @@ -146,6 +155,15 @@ export async function resolveCredentials(options) { cookies.cookieHeader = cookieHeader(cookies.authToken, cookies.ct0); return { cookies, warnings }; } + if (disableBrowserCookies) { + if (!cookies.authToken) { + warnings.push('Missing auth_token - provide via --auth-token, AUTH_TOKEN env var, or disable BIRD_DISABLE_BROWSER_COOKIES to allow browser cookie lookup'); + } + if (!cookies.ct0) { + warnings.push('Missing ct0 - provide via --ct0, CT0 env var, or disable BIRD_DISABLE_BROWSER_COOKIES to allow browser cookie lookup'); + } + return { cookies, warnings }; + } const sourcesToTry = resolveSources(options.cookieSource); for (const source of sourcesToTry) { const res = await readTwitterCookiesFromBrowser({ @@ -170,4 +188,4 @@ export async function resolveCredentials(options) { } return { cookies, warnings }; } -//# sourceMappingURL=cookies.js.map \ No newline at end of file +//# sourceMappingURL=cookies.js.map diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py index c560d49..cf22793 100644 --- a/scripts/lib/youtube_yt.py +++ b/scripts/lib/youtube_yt.py @@ -176,6 +176,8 @@ def search_youtube( # filtering returns 0 for evergreen topics like "thumbnail tips". cmd = [ "yt-dlp", + "--ignore-config", + "--no-cookies-from-browser", f"ytsearch{count}:{core_topic}", "--dump-json", "--no-warnings", @@ -295,6 +297,8 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: """ cmd = [ "yt-dlp", + "--ignore-config", + "--no-cookies-from-browser", "--write-auto-subs", "--sub-lang", "en", "--sub-format", "vtt", diff --git a/tests/test_bird_x.py b/tests/test_bird_x.py index 3d78bd8..3944376 100644 --- a/tests/test_bird_x.py +++ b/tests/test_bird_x.py @@ -11,6 +11,9 @@ from lib import bird_x class TestExtractCoreSubject(unittest.TestCase): + def tearDown(self): + bird_x._credentials.clear() + def test_strips_trending_noise(self): result = bird_x._extract_core_subject("trendiest Claude Code skills") self.assertNotIn("trendiest", result) @@ -28,6 +31,9 @@ class TestExtractCoreSubject(unittest.TestCase): class TestBirdSearchRetries(unittest.TestCase): + def tearDown(self): + bird_x._credentials.clear() + def test_last_chance_retry_uses_strongest_token(self): """When shorter retry also returns 0, uses longest non-noise token.""" empty = {"items": []} @@ -53,5 +59,29 @@ class TestBirdSearchRetries(unittest.TestCase): self.assertEqual(run_mock.call_count, 1) +class TestBirdAuthEnvironment(unittest.TestCase): + def tearDown(self): + bird_x._credentials.clear() + + def test_subprocess_env_disables_browser_cookie_fallback_when_injected(self): + bird_x.set_credentials("auth-token", "ct0-token") + + env = bird_x._subprocess_env() + + self.assertEqual(env["AUTH_TOKEN"], "auth-token") + self.assertEqual(env["CT0"], "ct0-token") + self.assertEqual(env["BIRD_DISABLE_BROWSER_COOKIES"], "1") + + def test_is_bird_authenticated_short_circuits_when_credentials_injected(self): + bird_x.set_credentials("auth-token", "ct0-token") + + with mock.patch.object(bird_x, "is_bird_installed", return_value=True), \ + mock.patch.object(bird_x.subprocess, "run") as run_mock: + result = bird_x.is_bird_authenticated() + + self.assertEqual(result, "env AUTH_TOKEN") + run_mock.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py new file mode 100644 index 0000000..bb6248b --- /dev/null +++ b/tests/test_youtube_yt.py @@ -0,0 +1,49 @@ +"""Tests for yt-dlp invocation safety flags.""" + +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import youtube_yt + + +class _DummyProc: + def __init__(self): + self.pid = 12345 + self.returncode = 0 + + def communicate(self, timeout=None): + return "", "" + + def wait(self, timeout=None): + return 0 + + +class TestYtDlpFlags(unittest.TestCase): + def test_search_ignores_global_config_and_browser_cookies(self): + proc = _DummyProc() + with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ + mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock: + youtube_yt.search_youtube("Claude Code", "2026-02-01", "2026-03-01") + + cmd = popen_mock.call_args.args[0] + self.assertIn("--ignore-config", cmd) + self.assertIn("--no-cookies-from-browser", cmd) + + def test_transcript_fetch_ignores_global_config_and_browser_cookies(self): + proc = _DummyProc() + with tempfile.TemporaryDirectory() as temp_dir, \ + mock.patch.object(youtube_yt.subprocess, "Popen", return_value=proc) as popen_mock: + youtube_yt.fetch_transcript("abc123", temp_dir) + + cmd = popen_mock.call_args.args[0] + self.assertIn("--ignore-config", cmd) + self.assertIn("--no-cookies-from-browser", cmd) + + +if __name__ == "__main__": + unittest.main() From 3aaf31b08deefb6a7eff6825693e9cf1af3d16f7 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Thu, 12 Mar 2026 21:07:09 -0700 Subject: [PATCH 5/6] Document env-based X auth flow Update README, launch copy, and UI guidance to prefer popup-free AUTH_TOKEN/CT0 configuration, and keep X backend selection on the verified Bird or xAI paths. Validation: uv run python -m unittest tests.test_env_project --- README.md | 48 +++++++++++++++++++-------------------- docs/how-search-works.md | 27 +++++++++------------- docs/v2.1-launch-copy.md | 38 +++++++++++++++---------------- scripts/last30days.py | 4 ++-- scripts/lib/env.py | 19 ++++------------ scripts/lib/ui.py | 19 ++++++++-------- tests/test_env_project.py | 25 ++++++++++++++++++++ 7 files changed, 93 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index cf15d84..e804a32 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,11 @@ clawhub install last30days-official **The AI world reinvents itself every month. This skill keeps you current.** /last30days researches your topic across Reddit, X, Bluesky, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web from the last 30 days, finds what the community is actually upvoting, sharing, betting on, and saying on camera, and writes you a grounded narrative with real citations. Whether it's Seedance 2.0 access, paper.design prompts, or the latest Nano Banana Pro techniques, you'll know what people who are paying attention already know. -**New in v2.9.5 — Bluesky, Comparative Mode, ScrapeCreators X:** +**New in v2.9.5 — Bluesky, Comparative Mode, and Config Improvements:** - **Bluesky/AT Protocol** is now a social source. Opt-in via `BSKY_HANDLE` + `BSKY_APP_PASSWORD` (create at bsky.app/settings/app-passwords). Full pipeline: search, score, dedupe, render. - **Comparative mode** - ask "X vs Y" (e.g., `/last30 cursor vs windsurf`) and get 3 parallel research passes with a side-by-side comparison: strengths, weaknesses, head-to-head table, and a data-driven verdict. -- **ScrapeCreators X backend** - X/Twitter search now uses ScrapeCreators as an additional backend alongside Bird cookie auth. -- **Per-project .env config** - drop a `.last30days.env` in your project root for per-project API keys. +- **Per-project .env config** - drop a `.claude/last30days.env` in your project root for per-project API keys. - **SessionStart config check** - validates your config automatically when a Claude Code session starts. - **Expanded test coverage** - 455+ tests across all modules. @@ -70,32 +69,31 @@ git clone https://github.com/mvanhorn/last30days-skill.git ~/.claude/skills/last # Add your API keys (optional if signed in to Codex) mkdir -p ~/.config/last30days cat > ~/.config/last30days/.env << 'EOF' -SCRAPECREATORS_API_KEY=... # Reddit + TikTok + Instagram (one key, all three) — scrapecreators.com -OPENAI_API_KEY=sk-... # optional — legacy Reddit fallback if using `codex login` -XAI_API_KEY=xai-... # optional — cookie auth is default for X search -BSKY_HANDLE=you.bsky.social # optional — Bluesky search (create app password below) -BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # optional — bsky.app/settings/app-passwords +SCRAPECREATORS_API_KEY=... # Reddit + TikTok + Instagram (one key, all three) - scrapecreators.com +OPENAI_API_KEY=sk-... # optional - legacy Reddit fallback if using `codex login` +AUTH_TOKEN=... # recommended for X search - copy once from x.com cookies +CT0=... # recommended for X search - copy once from x.com cookies +XAI_API_KEY=xai-... # optional - X fallback if you do not want cookie-based auth +BSKY_HANDLE=you.bsky.social # optional - Bluesky search (create app password below) +BSKY_APP_PASSWORD=xxxx-xxxx-xxxx # optional - bsky.app/settings/app-passwords EOF chmod 600 ~/.config/last30days/.env ``` If you're signed in to Codex (`codex login`), the skill will use your Codex credentials for the OpenAI Responses API and you can omit `OPENAI_API_KEY`. If you're not signed in, run `codex login` first. +For project-specific overrides, create `.claude/last30days.env` in the repo root. It overrides the global `~/.config/last30days/.env`. + ### X Search Authentication -X search reads your existing browser cookies - no API keys or login commands needed. +X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts. -**Safari (recommended on Mac):** Just be logged into x.com. No setup needed. +**Recommended setup:** +1. While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`. +2. Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env`, export them in your shell, or add them to `.claude/last30days.env` for a single project. +3. Re-run `/last30days`. -**Chrome:** Works, but macOS will prompt you to allow Keychain access the first time. Click "Allow" (or "Always Allow" to stop future prompts). - -**Firefox:** Just be logged into x.com. No setup needed. - -**Manual fallback:** If cookie auto-detection doesn't work, set these env vars (grab them from your browser's dev tools → Application → Cookies → x.com): -```bash -export AUTH_TOKEN=your_auth_token -export CT0=your_ct0_token -``` +**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and the skill will use xAI's `x_search` backend instead. **Verify it's working:** ```bash @@ -930,13 +928,13 @@ This example shows /last30days discovering **emerging developer workflows** - re ## Requirements -- **OpenAI API key** - For Reddit research (uses web search via Responses API) +- **OpenAI auth** - For Reddit research (uses web search via Responses API). Use `OPENAI_API_KEY` or `codex login`. - **Node.js 22+** - For X search (bundled Twitter GraphQL client) -- **X session** - Be logged into x.com in your browser, or set `AUTH_TOKEN`/`CT0` env vars -- **xAI API key** (optional fallback) - If the bundled search can't authenticate, falls back to xAI's Grok API +- **Bundled X auth** - Set `AUTH_TOKEN` and `CT0` for popup-free local X search +- **Alternate X backend** - Set `XAI_API_KEY` if bundled X auth is not configured - **yt-dlp** (optional) - For YouTube search + transcript extraction. Install via `brew install yt-dlp` or `pip install yt-dlp`. When present, automatically searches YouTube and extracts video transcripts as an additional source. -At least one API key is required. X search works automatically if you're logged into x.com in your browser. YouTube search activates automatically when yt-dlp is in your PATH. +At least one auth path is required. Reddit needs OpenAI auth. X needs either `AUTH_TOKEN` plus `CT0` or `XAI_API_KEY`. YouTube search activates automatically when yt-dlp is in your PATH. ## Troubleshooting @@ -1143,7 +1141,7 @@ Inspired by [Peter Steinberger](https://x.com/steipete)'s yt-dlp + [summarize](h ### Bundled X search (v2.1) -**X search is fully self-contained** - No external `bird` CLI or xAI API key needed. /last30days bundles a vendored subset of Bird's Twitter GraphQL client (MIT licensed, by Peter Steinberger). Just be logged into x.com in your browser and it auto-detects your session. Falls back to xAI API if bundled search can't authenticate. +**X search is fully self-contained** - No external `bird` CLI install needed. /last30days bundles a vendored subset of Bird's Twitter GraphQL client (MIT licensed, by Peter Steinberger). With Node.js 22+ plus `AUTH_TOKEN` and `CT0`, it runs locally without browser-cookie prompts. Falls back to xAI API if bundled auth is not configured. ### Everything else (v2.1) @@ -1190,7 +1188,7 @@ Thanks to the contributors who helped shape V2: | `api.scrapecreators.com` | Search query (Reddit + TikTok + Instagram) | SCRAPECREATORS_API_KEY | | `api.openai.com` | Search query (legacy Reddit fallback) | OPENAI_API_KEY | | `reddit.com` | Thread URLs for enrichment | None (public JSON) | -| Twitter GraphQL / `api.x.ai` | Search query | Browser cookies or XAI_API_KEY | +| Twitter GraphQL / `api.x.ai` | Search query | AUTH_TOKEN/CT0 or XAI_API_KEY | | `youtube.com` (via yt-dlp) | Search query | None (public search) | | `hn.algolia.com` | Search query | None (public API) | | `gamma-api.polymarket.com` | Search query | None (public API) | diff --git a/docs/how-search-works.md b/docs/how-search-works.md index 9b3c0fe..3479d7d 100644 --- a/docs/how-search-works.md +++ b/docs/how-search-works.md @@ -9,7 +9,7 @@ User: /last30days "kanye west" ↓ ↓ (concurrent via ThreadPoolExecutor) [REDDIT] [X/TWITTER] ↓ ↓ - OpenAI Bird CLI or + OpenAI Bundled Bird or API xAI API ↓ ↓ Parse Parse @@ -95,11 +95,11 @@ No API key needed. This returns the actual thread data: X search has **two backends** — the skill auto-detects which to use. -### Priority: Bird CLI (free) → xAI API (paid) +### Priority: Bundled Bird (env auth) → xAI API (paid) ```python -if bird_installed and bird_authenticated: - use Bird CLI # Free, uses your X login +if node_available and AUTH_TOKEN and CT0: + use bundled Bird # Free, popup-free, env-authenticated elif XAI_API_KEY: use xAI API # Paid, uses grok-4-1-fast else: @@ -128,20 +128,15 @@ The prompt asks grok to return JSON with: - `engagement`: `{ likes, reposts, replies, quotes }` - `why_relevant`, `relevance` score -**Engagement data comes from grok's x_search tool** — it has direct access to X's data. +**Engagement data comes from grok's x_search tool** - it has direct access to X's data. -### Backend 2: Bird CLI (free alternative) +### Backend 2: Bundled Bird client (free alternative) -Bird is a CLI tool (`npm install -g @steipete/bird`) that uses your X login. +The repo vendors a search-only subset of Bird's Twitter GraphQL client and shells out to it with Node.js. No global `bird` install is required. The Python wrapper passes `AUTH_TOKEN` and `CT0` via env, which keeps normal local runs headless and avoids browser-cookie prompts. -**Command:** -```bash -bird search "{topic} since:{from_date}" -n 30 --json -``` +**Bundled Bird returns raw X API data** - likes, reposts, replies are real engagement metrics from X's API, not estimates. -**Bird returns raw X API data** — likes, reposts, replies are real engagement metrics from X's API, not estimates. - -| Metric | Bird CLI | xAI API | +| Metric | Bundled Bird | xAI API | |---|---|---| | Post text | Real | Real | | Likes/reposts | Real (X API) | Real (x_search tool) | @@ -151,7 +146,7 @@ bird search "{topic} since:{from_date}" -n 30 --json ### Depth settings -| Depth | xAI posts | Bird results | xAI timeout | Bird timeout | +| Depth | xAI posts | Bundled Bird results | xAI timeout | Bird timeout | |---|---|---|---|---| | `--quick` | 8-12 | 12 | 90s | 30s | | default | 20-30 | 30 | 120s | 45s | @@ -192,7 +187,7 @@ After both searches complete: | `scripts/lib/openai_reddit.py` | Reddit search via OpenAI Responses API | | `scripts/lib/reddit_enrich.py` | Fetch real engagement data from Reddit JSON API | | `scripts/lib/xai_x.py` | X search via xAI API | -| `scripts/lib/bird_x.py` | X search via Bird CLI (free) | +| `scripts/lib/bird_x.py` | X search via bundled Bird client (free) | | `scripts/lib/models.py` | Auto-select best available model | | `scripts/lib/env.py` | API key loading, source detection | | `scripts/lib/http.py` | HTTP transport with retries | diff --git a/docs/v2.1-launch-copy.md b/docs/v2.1-launch-copy.md index 7116328..074d063 100644 --- a/docs/v2.1-launch-copy.md +++ b/docs/v2.1-launch-copy.md @@ -13,7 +13,7 @@ YouTube transcripts are the second headline feature. Inspired by Peter Steinberg **New in V2.1 — two headline features:** - **YouTube transcripts as a 4th source.** When yt-dlp is installed, /last30days automatically searches YouTube, grabs view counts, and extracts auto-generated transcripts from the top videos. A 20-minute review contains 10x the signal of a tweet — now the skill reads it. Inspired by @steipete's yt-dlp + summarize toolchain. -- **X search is fully bundled.** No external `bird` CLI or xAI API key needed. Just Node.js 22+ and your browser cookies. Uses a vendored subset of Bird's Twitter GraphQL client (MIT licensed, originally by @steipete). +- **X search is fully bundled.** No external `bird` CLI install needed. Add `AUTH_TOKEN` and `CT0` once, and the vendored Bird client runs locally without browser-cookie prompts. `XAI_API_KEY` remains an optional fallback. --- @@ -21,20 +21,18 @@ YouTube transcripts are the second headline feature. Inspired by Peter Steinberg ### X Search Authentication -X search reads your existing browser cookies — no API keys or login commands needed. +X search prefers explicit env auth. This keeps local runs headless and avoids browser-cookie and macOS Keychain prompts. -**Safari (recommended on Mac):** Just be logged into x.com. No setup needed. +**Recommended setup:** While logged into x.com once, open browser dev tools and copy the `auth_token` and `ct0` cookies for `x.com`. -**Chrome:** Works, but macOS will prompt you to allow Keychain access the first time. Click "Allow" (or "Always Allow" to stop future prompts). - -**Firefox:** Just be logged into x.com. No setup needed. - -**Manual fallback:** If cookie auto-detection doesn't work, set these env vars (grab them from your browser's dev tools → Application → Cookies → x.com): +Save them as `AUTH_TOKEN` and `CT0` in `~/.config/last30days/.env` or `.claude/last30days.env`: ```bash -export AUTH_TOKEN=your_auth_token -export CT0=your_ct0_token +AUTH_TOKEN=your_auth_token +CT0=your_ct0_token ``` +**xAI fallback:** If you do not want to provide `AUTH_TOKEN` and `CT0`, set `XAI_API_KEY` and use xAI's `x_search` backend instead. + **Verify it's working:** ```bash node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs --whoami @@ -44,8 +42,10 @@ node ~/.claude/skills/last30days/scripts/lib/vendor/bird-search/bird-search.mjs ## README: Install block env line -``` -XAI_API_KEY=xai-... # optional — cookie auth is default for X search +```bash +AUTH_TOKEN=... # recommended for X search +CT0=... # recommended for X search +XAI_API_KEY=xai-... # optional X fallback ``` --- @@ -60,15 +60,13 @@ XAI_API_KEY=xai-... # optional — cookie auth is default for X search ## GitHub issue #19 response (post AFTER publishing) -> Thanks for reporting this. Bird CLI was deprecated and the GitHub repo was deleted — steipete was asked to take it down. +> Thanks for reporting this. Bird CLI was deprecated and the GitHub repo was deleted. steipete was asked to take it down. > -> The good news: you don't need Bird anymore. v2.1 (just shipped) bundles X search directly — no external CLI, no `npm install`, no brew. Just Node.js 22+ and your browser cookies. +> The good news: you don't need Bird anymore. v2.1 (just shipped) bundles X search directly. No external CLI, no `npm install`, no brew. Just Node.js 22+ plus `AUTH_TOKEN` and `CT0`, or `XAI_API_KEY` as fallback. > -> It also adds **YouTube as a 4th source** — when yt-dlp is installed, the skill automatically searches YouTube and extracts transcripts from the top videos. A 20-minute tutorial has 10x the signal of a tweet, and now the synthesis engine reads it. +> It also adds **YouTube as a 4th source**. When yt-dlp is installed, the skill automatically searches YouTube and extracts transcripts from the top videos. A 20-minute tutorial has 10x the signal of a tweet, and now the synthesis engine reads it. > -> If you're on a Mac, Safari is the easiest path for X — just be logged into x.com. Chrome works too but macOS will prompt for Keychain access the first time. -> -> If cookie auto-detection doesn't work, you can set `AUTH_TOKEN` and `CT0` env vars manually (grab from browser dev tools → Application → Cookies → x.com). +> The recommended setup is to copy `auth_token` and `ct0` from x.com once and store them as `AUTH_TOKEN` and `CT0` in your env. That avoids browser-cookie and Keychain prompts during normal runs. > > The xAI API (`XAI_API_KEY`) also still works as a fallback. @@ -82,7 +80,7 @@ XAI_API_KEY=xai-... # optional — cookie auth is default for X search Two new features: → YouTube transcripts as a 4th source (yt-dlp) -→ X search fully bundled (no bird CLI needed) +→ X search fully bundled (no bird CLI install needed) Research any topic across Reddit, X, YouTube & web in one command. @@ -100,7 +98,7 @@ When yt-dlp is installed, the skill searches YouTube, grabs view counts, and ext ### Thread version (post 2) 2️⃣ X search is fully bundled -Bird CLI was deprecated. Instead of requiring an external tool, v2.1 vendors a search-only subset. Just be logged into x.com in your browser. No npm install, no API keys. +Bird CLI was deprecated. Instead of requiring an external tool, v2.1 vendors a search-only subset. Add `AUTH_TOKEN` and `CT0` once, then it runs locally with no npm install. `XAI_API_KEY` still works as fallback. Both features inspired by @steipete's tooling. diff --git a/scripts/last30days.py b/scripts/last30days.py index 2532c25..b456009 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -1621,8 +1621,8 @@ def main(): # Check available sources (accounting for Bird auto-detection) available = env.get_available_sources(config) - # Override available if Bird or ScrapeCreators provides X - if x_source in ('bird', 'scrapecreators'): + # Override available if Bird provides X + if x_source == 'bird': if available == 'reddit': available = 'both' # Now have both Reddit + X elif available == 'reddit-web': diff --git a/scripts/lib/env.py b/scripts/lib/env.py index 87f9b7c..28a6eb2 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -346,7 +346,7 @@ def get_web_search_source(config: Dict[str, Any]) -> Optional[str]: def get_missing_keys(config: Dict[str, Any]) -> str: - """Determine which sources are missing (accounting for Bird and ScrapeCreators). + """Determine which sources are missing (accounting for Bird). Returns: 'all', 'both', 'reddit', 'x', 'web', or 'none' """ @@ -358,8 +358,7 @@ def get_missing_keys(config: Dict[str, Any]) -> str: from . import bird_x has_bird = bird_x.is_bird_installed() and bird_x.is_bird_authenticated() - has_sc_x = bool(config.get('SCRAPECREATORS_API_KEY')) - has_x = has_xai or has_bird or has_sc_x + has_x = has_xai or has_bird if has_reddit and has_x and has_web: return 'none' @@ -438,7 +437,9 @@ def validate_sources(requested: str, available: str, include_web: bool = False) def get_x_source(config: Dict[str, Any]) -> Optional[str]: """Determine the best available X/Twitter source. - Priority: Bird (free) → xAI (paid API) → ScrapeCreators (shared key) + Priority: Bird (free) → xAI (paid API) + + Keep X selection limited to documented, verified search backends. Args: config: Configuration dict from get_config() @@ -446,7 +447,6 @@ def get_x_source(config: Dict[str, Any]) -> Optional[str]: Returns: 'bird' if Bird is installed and authenticated, 'xai' if XAI_API_KEY is configured, - 'scrapecreators' if SCRAPECREATORS_API_KEY is configured, None if no X source available. """ # Import here to avoid circular dependency @@ -462,10 +462,6 @@ def get_x_source(config: Dict[str, Any]) -> Optional[str]: if config.get('XAI_API_KEY'): return 'xai' - # Fall back to ScrapeCreators (same key as Reddit/TikTok/Instagram) - if config.get('SCRAPECREATORS_API_KEY'): - return 'scrapecreators' - return None @@ -584,15 +580,11 @@ def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]: bird_status = bird_x.get_bird_status() xai_available = bool(config.get('XAI_API_KEY')) - sc_available = bool(config.get('SCRAPECREATORS_API_KEY')) - # Determine active source if bird_status["authenticated"]: source = 'bird' elif xai_available: source = 'xai' - elif sc_available: - source = 'scrapecreators' else: source = None @@ -602,6 +594,5 @@ def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]: "bird_authenticated": bird_status["authenticated"], "bird_username": bird_status["username"], "xai_available": xai_available, - "scrapecreators_available": sc_available, "can_install_bird": bird_status["can_install"], } diff --git a/scripts/lib/ui.py b/scripts/lib/ui.py index 2351344..f81e2da 100644 --- a/scripts/lib/ui.py +++ b/scripts/lib/ui.py @@ -143,7 +143,7 @@ Just start with "last30" and talk to me like normal. # Shorter promo for single missing key PROMO_SINGLE_KEY = { "reddit": "\n💡 You can unlock Reddit with an OpenAI API key or by running `codex login` — just ask me how.\n", - "x": "\n💡 You can unlock X with an xAI API key — just ask me how.\n", + "x": "\n💡 You can unlock X with AUTH_TOKEN/CT0 or XAI_API_KEY - just ask me how.\n", } # Bird auth help (for local users with vendored Bird CLI) @@ -151,16 +151,16 @@ BIRD_AUTH_HELP = f""" {Colors.YELLOW}Bird authentication failed.{Colors.RESET} To fix this: -1. Log into X (twitter.com) in Safari, Chrome, or Firefox -2. Try again — Bird reads your browser cookies automatically. +1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env +2. Or set XAI_API_KEY for the xAI fallback backend """ BIRD_AUTH_HELP_PLAIN = """ Bird authentication failed. To fix this: -1. Log into X (twitter.com) in Safari, Chrome, or Firefox -2. Try again — Bird reads your browser cookies automatically. +1. Add AUTH_TOKEN and CT0 to ~/.config/last30days/.env or .claude/last30days.env +2. Or set XAI_API_KEY for the xAI fallback backend """ # Spinner frames @@ -460,10 +460,9 @@ def show_diagnostic_banner(diag: dict): label = f"Bird ({username})" if source == "bird" and username else source.upper() lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.GREEN}✅ X/Twitter{Colors.RESET} — {label} {Colors.DIM}│{Colors.RESET}") else: - lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No Bird CLI or XAI_API_KEY {Colors.DIM}│{Colors.RESET}") + lines.append(f"{Colors.DIM}│{Colors.RESET} {Colors.RED}❌ X/Twitter{Colors.RESET} — No X auth or fallback key {Colors.DIM}│{Colors.RESET}") if diag.get("bird_installed"): - lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Bird installed but not authenticated {Colors.DIM}│{Colors.RESET}") - lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Log into x.com in your browser, then retry {Colors.DIM}│{Colors.RESET}") + lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY {Colors.DIM}│{Colors.RESET}") else: lines.append(f"{Colors.DIM}│{Colors.RESET} └─ Needs Node.js 22+ (Bird is bundled) {Colors.DIM}│{Colors.RESET}") @@ -507,9 +506,9 @@ def show_diagnostic_banner(diag: dict): if has_x: lines.append("│ ✅ X/Twitter — available │") else: - lines.append("│ ❌ X/Twitter — No Bird CLI or XAI_API_KEY │") + lines.append("│ ❌ X/Twitter — No X auth or fallback key │") if diag.get("bird_installed"): - lines.append("│ └─ Log into x.com in your browser, then retry │") + lines.append("│ └─ Add AUTH_TOKEN/CT0 or XAI_API_KEY │") else: lines.append("│ └─ Needs Node.js 22+ (Bird is bundled) │") diff --git a/tests/test_env_project.py b/tests/test_env_project.py index 9b1f330..5da61c6 100644 --- a/tests/test_env_project.py +++ b/tests/test_env_project.py @@ -173,5 +173,30 @@ class TestFilePermissions(unittest.TestCase): self.assertEqual(stderr.getvalue(), "") +class TestXSourceSelection(unittest.TestCase): + """Tests for supported X backend selection.""" + + def test_get_x_source_ignores_scrapecreators_key(self): + config = {'SCRAPECREATORS_API_KEY': 'sc-key'} + + with patch('lib.bird_x.is_bird_installed', return_value=False): + self.assertIsNone(env.get_x_source(config)) + + def test_get_x_source_status_ignores_scrapecreators_key(self): + config = {'SCRAPECREATORS_API_KEY': 'sc-key'} + bird_status = { + 'installed': True, + 'authenticated': False, + 'username': None, + 'can_install': False, + } + + with patch('lib.bird_x.get_bird_status', return_value=bird_status): + status = env.get_x_source_status(config) + + self.assertIsNone(status['source']) + self.assertFalse(status['xai_available']) + + if __name__ == "__main__": unittest.main() From 0e46c7cb333ba3b47aaad86daee5f8beda002537 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Fri, 13 Mar 2026 00:57:06 -0700 Subject: [PATCH 6/6] Pass X auth through handle drilldowns Phase-2 Bird handle searches were still spawning Node without the injected AUTH_TOKEN/CT0 env. That left the search pipeline vulnerable to Chrome keychain prompts whenever a query drilled into X handles. Pass the popup-safe subprocess env through those handle searches and cover it with a regression test. --- scripts/lib/bird_x.py | 1 + tests/test_bird_x.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py index 548b81c..1379aa1 100644 --- a/scripts/lib/bird_x.py +++ b/scripts/lib/bird_x.py @@ -365,6 +365,7 @@ def search_handles( stderr=subprocess.PIPE, text=True, preexec_fn=preexec, + env=_subprocess_env(), ) try: diff --git a/tests/test_bird_x.py b/tests/test_bird_x.py index 3944376..2e9e57a 100644 --- a/tests/test_bird_x.py +++ b/tests/test_bird_x.py @@ -82,6 +82,21 @@ class TestBirdAuthEnvironment(unittest.TestCase): self.assertEqual(result, "env AUTH_TOKEN") run_mock.assert_not_called() + def test_search_handles_passes_injected_credentials_to_subprocess(self): + bird_x.set_credentials("auth-token", "ct0-token") + + proc = mock.Mock() + proc.communicate.return_value = ("[]", "") + proc.returncode = 0 + + with mock.patch.object(bird_x.subprocess, "Popen", return_value=proc) as popen_mock: + bird_x.search_handles(["openai"], "codex vs claude code", "2026-01-01", count_per=1) + + env = popen_mock.call_args.kwargs["env"] + self.assertEqual(env["AUTH_TOKEN"], "auth-token") + self.assertEqual(env["CT0"], "ct0-token") + self.assertEqual(env["BIRD_DISABLE_BROWSER_COOKIES"], "1") + if __name__ == "__main__": unittest.main()