diff --git a/docs/plans/2026-02-07-fix-x-search-query-construction-plan.md b/docs/plans/2026-02-07-fix-x-search-query-construction-plan.md new file mode 100644 index 0000000..be67e2b --- /dev/null +++ b/docs/plans/2026-02-07-fix-x-search-query-construction-plan.md @@ -0,0 +1,147 @@ +--- +title: "fix: X search query too restrictive, returns 0 results on popular topics" +type: fix +date: 2026-02-07 +--- + +# fix: X search query too restrictive, returns 0 results on popular topics + +## Problem + +`/last30days vibe motion best prompt techniques` returned **0 X posts** despite Vibe Motion being actively discussed on X (screenshots show posts from @Godid242, @KamilStanuch, @ColdStartTheory, @higgsfield_ai). + +Root cause: `_extract_core_subject()` in `bird_x.py` produces overly specific queries. Bird/X search uses **literal keyword AND matching** — ALL words must appear in a tweet. The function kept 4 keywords (`vibe motion prompt techniques`) when only 2 (`vibe motion`) were needed. + +## Three Bugs Found + +### Bug 1: Multi-word noise phrases never match + +```python +# Current code (bird_x.py:24-38) +noise = ['best', ..., 'what are', 'what is', 'how to', 'tips for', ...] +words = topic.lower().split() # splits into individual words +result = [w for w in words if w not in noise] # compares "what" against "what are" → no match! +``` + +`"what are people saying about DeepSeek R1"` → keeps `"what are people saying"` → **LOSES THE ENTIRE TOPIC**. + +The multi-word entries (`"what are"`, `"how to"`, `"tips for"`, `"use cases"`) are dead code. They never match because `.split()` creates individual words but the noise list has multi-word strings. + +### Bug 2: Missing meta/research words + +The noise list has `"prompting"` but not `"prompt"`, `"prompts"`, `"techniques"`, `"tips"`, `"tricks"`, `"methods"`, etc. + +- `"vibe motion best prompt techniques"` → `"vibe motion prompt techniques"` (4 words, should be 2) +- `"nano banana pro prompts for gemini"` → `"nano banana pro prompts"` (4 words, should be 3) + +### Bug 3: No retry on 0 results + +Reddit has multi-stage retry: full query → simplified core → subreddit fallback. X search runs once and accepts whatever comes back, even 0 results. + +## Proposed Fix + +All changes in `scripts/lib/bird_x.py`. + +### Step 1: Fix `_extract_core_subject()` — strip phrases first, then words + +```python +def _extract_core_subject(topic: str) -> str: + """Extract core subject from verbose query for X search.""" + text = topic.lower() + + # Phase 1: Strip multi-word prefixes/suffixes (order matters - longest first) + prefixes = ['what are the best', 'what is the best', 'what are', 'what is', + 'how to', 'how do i', 'tips for', 'best practices for'] + for p in prefixes: + if text.startswith(p): + text = text[len(p):].strip() + break + + suffixes = ['best practices', 'use cases', 'prompt techniques', + 'prompting techniques'] + for s in suffixes: + if text.endswith(s): + text = text[:-len(s)].strip() + break + + # Phase 2: Split and filter individual noise words + noise = {'best', 'top', 'practices', 'features', 'killer', 'guide', + 'tutorial', 'recommendations', 'advice', 'prompting', 'prompt', + 'prompts', 'techniques', 'tips', 'tricks', 'methods', + 'strategies', 'review', 'reviews', 'uses', 'usecases', + 'examples', 'using', 'for', 'with', 'the', 'of', 'in', 'on', + 'about', 'latest', 'new', 'news', 'update', 'updates', + 'good', 'great', 'awesome', 'and', 'or', 'a', 'an', 'is', + 'are', 'was', 'were', 'people', 'saying', 'think', 'said'} + words = text.split() + result = [w for w in words if w not in noise] + + return ' '.join(result[:3]) or topic # Max 3 words (was 4) +``` + +**Expected results after fix:** + +| Input | Before | After | +|-------|--------|-------| +| `vibe motion best prompt techniques` | `vibe motion prompt techniques` | `vibe motion` | +| `what are people saying about DeepSeek R1` | `what are people saying` | `deepseek r1` | +| `nano banana pro prompts for gemini` | `nano banana pro prompts` | `nano banana pro` | +| `open claw best uses` | `open claw uses` | `open claw` | +| `best claude code skills` | `claude code skills` | `claude code skills` | +| `kanye west` | `kanye west` | `kanye west` | + +### Step 2: Add retry with simplified query on 0 results + +In `search_x()`, after the initial search, if 0 items returned, retry with just the first 2 words of the core subject: + +```python +def search_x(topic, from_date, to_date, depth="default"): + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + core_topic = _extract_core_subject(topic) + query = f"{core_topic} since:{from_date}" + + # ... existing Bird search code ... + + items = parse_bird_response(response) + + # Retry with fewer keywords if 0 results + if not items and len(core_topic.split()) > 2: + shorter = ' '.join(core_topic.split()[:2]) + _log(f"0 results for '{core_topic}', retrying with '{shorter}'") + query = f"{shorter} since:{from_date}" + # ... retry Bird search ... + items = parse_bird_response(retry_response) + + return response # or merged response +``` + +### Step 3 (optional): Cross-pollinate Reddit entities into X Phase 2 + +When X Phase 1 returns 0 results but Reddit found threads, extract brand/product names from Reddit thread titles and use them as X search fallback queries. This is lower priority — Steps 1-2 should fix most cases. + +## Acceptance Criteria + +- [x] `vibe motion best prompt techniques` returns >0 X posts (12 posts found) +- [x] `what are people saying about DeepSeek R1` produces query containing "deepseek r1" not "what are people saying" +- [x] No regressions on working queries (`kanye west`, `claude code skills`, `open claw`) +- [x] Retry fires when initial query returns 0, logged to stderr +- [x] `openai_reddit.py`'s `_extract_core_subject()` NOT changed (Reddit uses semantic search, not literal matching — the current function works fine there) + +## Files to Change + +- `scripts/lib/bird_x.py` — `_extract_core_subject()` rewrite + retry logic in `search_x()` +- `scripts/lib/bird_x.py` — `search_handles()` benefits automatically (calls `_extract_core_subject()`) + +## Testing + +```bash +# Mock mode (quick syntax check) +python3 scripts/last30days.py "vibe motion best prompt techniques" --mock --emit=compact 2>&1 + +# Live queries to verify X results +python3 scripts/last30days.py "vibe motion best prompt techniques" --quick --emit=compact 2>&1 | grep -E "X:|posts" +python3 scripts/last30days.py "what are people saying about DeepSeek R1" --quick --emit=compact 2>&1 | grep -E "X:|posts" + +# Regression check +python3 scripts/last30days.py "kanye west" --quick --emit=compact 2>&1 | grep -E "X:|posts" +``` diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py index 70ac1e8..507454d 100644 --- a/scripts/lib/bird_x.py +++ b/scripts/lib/bird_x.py @@ -24,18 +24,56 @@ def _log(msg: str): def _extract_core_subject(topic: str) -> str: """Extract core subject from verbose query for X search. - X search is literal keyword matching, not semantic. - Strip noise words to get searchable terms. + X search is literal keyword AND matching — all words must appear. + Aggressively strip question/meta/research words to keep only the + core product/concept name (2-3 words max). """ - noise = ['best', 'top', 'how to', 'tips for', 'practices', 'features', - 'killer', 'guide', 'tutorial', 'recommendations', 'advice', - 'prompting', 'using', 'for', 'with', 'the', 'of', 'in', 'on', - 'usecases', 'use cases', 'examples', 'what are', 'what is'] - words = topic.lower().split() - result = [w for w in words if w not in noise] - # Rejoin compound terms that got split (e.g., "open claw" -> "openclaw") - joined = ' '.join(result[:4]) or topic # Keep max 4 words - return joined + text = topic.lower().strip() + + # Phase 1: Strip multi-word prefixes (longest first) + prefixes = [ + 'what are the best', 'what is the best', 'what are the latest', + 'what are people saying about', 'what do people think about', + 'how do i use', 'how to use', 'how to', + 'what are', 'what is', 'tips for', 'best practices for', + ] + for p in prefixes: + if text.startswith(p + ' '): + text = text[len(p):].strip() + break + + # Phase 2: Strip multi-word suffixes + suffixes = [ + 'best practices', 'use cases', 'prompt techniques', + 'prompting techniques', 'prompting tips', + ] + for s in suffixes: + if text.endswith(' ' + s): + text = text[:-len(s)].strip() + break + + # Phase 3: Filter individual noise words + _noise = { + # Question/filler words + 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or', + 'of', 'in', 'on', 'for', 'with', 'about', 'to', + 'people', 'saying', 'think', 'said', 'lately', + # Research/meta descriptors + 'best', 'top', 'good', 'great', 'awesome', 'killer', + 'latest', 'new', 'news', 'update', 'updates', + 'practices', 'features', 'guide', 'tutorial', + 'recommendations', 'advice', 'review', 'reviews', + 'usecases', 'examples', 'comparison', 'versus', 'vs', + # Prompting meta words + 'prompt', 'prompts', 'prompting', 'techniques', 'tips', + 'tricks', 'methods', 'strategies', 'approaches', + # Action words + 'using', 'uses', 'use', + } + words = text.split() + result = [w for w in words if w not in _noise] + + return ' '.join(result[:3]) or topic.lower().strip() # Max 3 words def is_bird_installed() -> bool: @@ -125,34 +163,17 @@ def get_bird_status() -> Dict[str, Any]: } -def search_x( - topic: str, - from_date: str, - to_date: str, - depth: str = "default", -) -> Dict[str, Any]: - """Search X using Bird CLI. +def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]: + """Run a single Bird CLI search and return raw response. Args: - topic: Search topic - from_date: Start date (YYYY-MM-DD) - to_date: End date (YYYY-MM-DD) - unused but kept for API compatibility - depth: Research depth - "quick", "default", or "deep" + query: Full search query string (including since: filter) + count: Number of results to request + timeout: Timeout in seconds Returns: Raw Bird JSON response or error dict. """ - count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) - - # Extract core subject - X search is literal, not semantic - # "best open claw usecases" -> "open claw" (searchable keywords) - core_topic = _extract_core_subject(topic) - - # Build query with date filter using X's search syntax - # Bird doesn't support --since flag, but X search accepts since:YYYY-MM-DD in query - query = f"{core_topic} since:{from_date}" - - # Build command cmd = [ "bird", "search", query, @@ -160,9 +181,6 @@ def search_x( "--json", ] - # Adjust timeout based on depth - timeout = 30 if depth == "quick" else 45 if depth == "default" else 60 - try: result = subprocess.run( cmd, @@ -175,7 +193,6 @@ def search_x( error = result.stderr.strip() or "Bird search failed" return {"error": error, "items": []} - # Parse JSON output output = result.stdout.strip() if not output: return {"items": []} @@ -190,6 +207,47 @@ def search_x( return {"error": str(e), "items": []} +def search_x( + topic: str, + from_date: str, + to_date: str, + depth: str = "default", +) -> Dict[str, Any]: + """Search X using Bird CLI with automatic retry on 0 results. + + Args: + topic: Search topic + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) - unused but kept for API compatibility + depth: Research depth - "quick", "default", or "deep" + + Returns: + Raw Bird JSON response or error dict. + """ + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + timeout = 30 if depth == "quick" else 45 if depth == "default" else 60 + + # Extract core subject - X search is literal, not semantic + core_topic = _extract_core_subject(topic) + query = f"{core_topic} since:{from_date}" + + _log(f"Searching: {query}") + response = _run_bird_search(query, count, timeout) + + # Check if we got results + items = parse_bird_response(response) + + # Retry with fewer keywords if 0 results and query has 3+ words + core_words = core_topic.split() + if not items and len(core_words) > 2: + shorter = ' '.join(core_words[:2]) + _log(f"0 results for '{core_topic}', retrying with '{shorter}'") + query = f"{shorter} since:{from_date}" + response = _run_bird_search(query, count, timeout) + + return response + + def search_handles( handles: List[str], topic: str,