feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources

v3 rewrites the search engine from the ground up:

- Intelligent pre-research: resolves X handles, GitHub repos, subreddits,
  TikTok hashtags, and YouTube channels before searching
- GitHub person-mode: PR velocity, top repos by stars, release notes
- GitHub project-mode: live star counts, README, releases, top issues
- ELI5 mode: plain language synthesis, no jargon
- 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket,
  GitHub, Threads, Pinterest, Perplexity, Bluesky, Web
- Free Reddit comments via public JSON (no API key needed)
- Fun judge v2: humor scoring baked into narrative
- Cookie consent before browser scanning
- 10,000 free ScrapeCreators calls
- 1,012 tests

Thank you to the community contributors whose issues and PRs shaped v3:
@uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116),
@thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78),
@zl190 (#115), @hnshah (#84, #85, #86)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-04-08 10:52:23 -07:00
parent 61904b31e3
commit 0a9ff16dfc
397 changed files with 21427 additions and 53106 deletions
+37 -35
View File
@@ -1,7 +1,8 @@
"""Bird X search client - vendored Twitter GraphQL search for /last30days v2.1.
"""Bird X search client for the v3.0.0 last30days pipeline.
Uses a vendored subset of @steipete/bird v0.8.0 (MIT License) to search X
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 22+.
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js.
See scripts/lib/vendor/bird-search/package.json for authoritative version.
"""
import json
@@ -11,11 +12,21 @@ import shutil
import subprocess
import sys
from pathlib import Path
from . import http, log
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from .relevance import token_overlap_relevance as _compute_relevance
def _first_of(*values):
"""Return first value that is not None."""
for v in values:
if v is not None:
return v
return None
# Path to the vendored bird-search wrapper
_BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
@@ -43,21 +54,23 @@ def _has_injected_credentials() -> bool:
return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0'))
def _has_process_credentials() -> bool:
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
return bool(os.environ.get("AUTH_TOKEN") and os.environ.get("CT0"))
def _subprocess_env() -> Dict[str, str]:
"""Build env dict for Node subprocesses, merging injected credentials."""
env = os.environ.copy()
env.update(_credentials)
# When repo config already provides cookies, disable browser-cookie fallback
# so vendored Bird never hits Safari/Chrome keychain during automation.
if _has_injected_credentials():
env.setdefault("BIRD_DISABLE_BROWSER_COOKIES", "1")
# Hard-disable browser-cookie fallback so normal pipeline runs never hit
# Safari/Chrome Keychain prompts during source detection or search.
env["BIRD_DISABLE_BROWSER_COOKIES"] = "1"
return env
def _log(msg: str):
"""Log to stderr."""
sys.stderr.write(f"[Bird] {msg}\n")
sys.stderr.flush()
log.source_log("Bird", msg, tty_only=False)
def _extract_core_subject(topic: str) -> str:
@@ -75,7 +88,7 @@ def is_bird_installed() -> bool:
"""Check if vendored Bird search module is available.
Returns:
True if bird-search.mjs exists and Node.js 22+ is in PATH.
True if bird-search.mjs exists and Node.js is in PATH.
"""
if not _BIRD_SEARCH_MJS.exists():
return False
@@ -83,7 +96,7 @@ def is_bird_installed() -> bool:
def is_bird_authenticated() -> Optional[str]:
"""Check if X credentials are available (env vars or browser cookies).
"""Check if explicit X credentials are available.
Returns:
Auth source string if authenticated, None otherwise.
@@ -93,20 +106,9 @@ def is_bird_authenticated() -> Optional[str]:
if _has_injected_credentials():
return "env AUTH_TOKEN"
try:
result = subprocess.run(
["node", str(_BIRD_SEARCH_MJS), "--whoami"],
capture_output=True,
text=True,
timeout=15,
env=_subprocess_env(),
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout.strip().split('\n')[0]
return None
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
return None
if _has_process_credentials():
return "env AUTH_TOKEN"
return None
def check_npm_available() -> bool:
@@ -119,13 +121,13 @@ def check_npm_available() -> bool:
def install_bird() -> Tuple[bool, str]:
"""No-op - Bird search is vendored in v2.1, no installation needed.
"""No-op. Bird search is vendored in v3.0.0, no installation needed.
Returns:
Tuple of (success, message).
"""
if is_bird_installed():
return True, "Bird search is bundled with /last30days v2.1 - no installation needed."
return True, "Bird search is bundled with /last30days v3.0.0 - no installation needed."
if not shutil.which("node"):
return False, "Node.js 22+ is required for X search. Install Node.js first."
return False, f"Vendored bird-search.mjs not found at {_BIRD_SEARCH_MJS}"
@@ -144,7 +146,7 @@ def get_bird_status() -> Dict[str, Any]:
"installed": installed,
"authenticated": auth_source is not None,
"username": auth_source, # Now returns auth source (e.g., "Safari", "env AUTH_TOKEN")
"can_install": True, # Always vendored in v2.1
"can_install": True, # Always vendored in v3.0.0
}
@@ -200,7 +202,7 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
try:
from last30days import unregister_child_pid
unregister_child_pid(proc.pid)
except (ImportError, Exception):
except Exception:
pass
if proc.returncode != 0:
@@ -361,7 +363,7 @@ def search_handles(
except json.JSONDecodeError:
_log(f"Invalid JSON from handle search for @{handle}")
except Exception as e:
except (OSError, subprocess.SubprocessError) as e:
_log(f"Handle search error for @{handle}: {e}")
return all_items
@@ -428,10 +430,10 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
# Build engagement dict (Bird uses camelCase: likeCount, retweetCount, etc.)
engagement = {
"likes": tweet.get("likeCount") or tweet.get("like_count") or tweet.get("favorite_count"),
"reposts": tweet.get("retweetCount") or tweet.get("retweet_count"),
"replies": tweet.get("replyCount") or tweet.get("reply_count"),
"quotes": tweet.get("quoteCount") or tweet.get("quote_count"),
"likes": _first_of(tweet.get("likeCount"), tweet.get("like_count"), tweet.get("favorite_count")),
"reposts": _first_of(tweet.get("retweetCount"), tweet.get("retweet_count")),
"replies": _first_of(tweet.get("replyCount"), tweet.get("reply_count")),
"quotes": _first_of(tweet.get("quoteCount"), tweet.get("quote_count")),
}
# Convert to int where possible
for key in engagement:
@@ -448,7 +450,7 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
"url": url,
"author_handle": author_handle.lstrip("@"),
"date": date,
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
"engagement": engagement,
"why_relevant": "", # Bird doesn't provide relevance explanations
"relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
}