feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources
v3 rewrites the search engine from the ground up: - Intelligent pre-research: resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching - GitHub person-mode: PR velocity, top repos by stars, release notes - GitHub project-mode: live star counts, README, releases, top issues - ELI5 mode: plain language synthesis, no jargon - 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, Threads, Pinterest, Perplexity, Bluesky, Web - Free Reddit comments via public JSON (no API key needed) - Fun judge v2: humor scoring baked into narrative - Cookie consent before browser scanning - 10,000 free ScrapeCreators calls - 1,012 tests Thank you to the community contributors whose issues and PRs shaped v3: @uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116), @thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78), @zl190 (#115), @hnshah (#84, #85, #86) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+37
-35
@@ -1,7 +1,8 @@
|
||||
"""Bird X search client - vendored Twitter GraphQL search for /last30days v2.1.
|
||||
"""Bird X search client for the v3.0.0 last30days pipeline.
|
||||
|
||||
Uses a vendored subset of @steipete/bird v0.8.0 (MIT License) to search X
|
||||
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 22+.
|
||||
via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js.
|
||||
See scripts/lib/vendor/bird-search/package.json for authoritative version.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -11,11 +12,21 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import http, log
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
|
||||
def _first_of(*values):
|
||||
"""Return first value that is not None."""
|
||||
for v in values:
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
|
||||
# Path to the vendored bird-search wrapper
|
||||
_BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
|
||||
|
||||
@@ -43,21 +54,23 @@ def _has_injected_credentials() -> bool:
|
||||
return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0'))
|
||||
|
||||
|
||||
def _has_process_credentials() -> bool:
|
||||
"""Return True when AUTH_TOKEN/CT0 are present in process env."""
|
||||
return bool(os.environ.get("AUTH_TOKEN") and os.environ.get("CT0"))
|
||||
|
||||
|
||||
def _subprocess_env() -> Dict[str, str]:
|
||||
"""Build env dict for Node subprocesses, merging injected credentials."""
|
||||
env = os.environ.copy()
|
||||
env.update(_credentials)
|
||||
# When repo config already provides cookies, disable browser-cookie fallback
|
||||
# so vendored Bird never hits Safari/Chrome keychain during automation.
|
||||
if _has_injected_credentials():
|
||||
env.setdefault("BIRD_DISABLE_BROWSER_COOKIES", "1")
|
||||
# Hard-disable browser-cookie fallback so normal pipeline runs never hit
|
||||
# Safari/Chrome Keychain prompts during source detection or search.
|
||||
env["BIRD_DISABLE_BROWSER_COOKIES"] = "1"
|
||||
return env
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr."""
|
||||
sys.stderr.write(f"[Bird] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
log.source_log("Bird", msg, tty_only=False)
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
@@ -75,7 +88,7 @@ def is_bird_installed() -> bool:
|
||||
"""Check if vendored Bird search module is available.
|
||||
|
||||
Returns:
|
||||
True if bird-search.mjs exists and Node.js 22+ is in PATH.
|
||||
True if bird-search.mjs exists and Node.js is in PATH.
|
||||
"""
|
||||
if not _BIRD_SEARCH_MJS.exists():
|
||||
return False
|
||||
@@ -83,7 +96,7 @@ def is_bird_installed() -> bool:
|
||||
|
||||
|
||||
def is_bird_authenticated() -> Optional[str]:
|
||||
"""Check if X credentials are available (env vars or browser cookies).
|
||||
"""Check if explicit X credentials are available.
|
||||
|
||||
Returns:
|
||||
Auth source string if authenticated, None otherwise.
|
||||
@@ -93,20 +106,9 @@ def is_bird_authenticated() -> Optional[str]:
|
||||
|
||||
if _has_injected_credentials():
|
||||
return "env AUTH_TOKEN"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["node", str(_BIRD_SEARCH_MJS), "--whoami"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
env=_subprocess_env(),
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip().split('\n')[0]
|
||||
return None
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
|
||||
return None
|
||||
if _has_process_credentials():
|
||||
return "env AUTH_TOKEN"
|
||||
return None
|
||||
|
||||
|
||||
def check_npm_available() -> bool:
|
||||
@@ -119,13 +121,13 @@ def check_npm_available() -> bool:
|
||||
|
||||
|
||||
def install_bird() -> Tuple[bool, str]:
|
||||
"""No-op - Bird search is vendored in v2.1, no installation needed.
|
||||
"""No-op. Bird search is vendored in v3.0.0, no installation needed.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message).
|
||||
"""
|
||||
if is_bird_installed():
|
||||
return True, "Bird search is bundled with /last30days v2.1 - no installation needed."
|
||||
return True, "Bird search is bundled with /last30days v3.0.0 - no installation needed."
|
||||
if not shutil.which("node"):
|
||||
return False, "Node.js 22+ is required for X search. Install Node.js first."
|
||||
return False, f"Vendored bird-search.mjs not found at {_BIRD_SEARCH_MJS}"
|
||||
@@ -144,7 +146,7 @@ def get_bird_status() -> Dict[str, Any]:
|
||||
"installed": installed,
|
||||
"authenticated": auth_source is not None,
|
||||
"username": auth_source, # Now returns auth source (e.g., "Safari", "env AUTH_TOKEN")
|
||||
"can_install": True, # Always vendored in v2.1
|
||||
"can_install": True, # Always vendored in v3.0.0
|
||||
}
|
||||
|
||||
|
||||
@@ -200,7 +202,7 @@ def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
|
||||
try:
|
||||
from last30days import unregister_child_pid
|
||||
unregister_child_pid(proc.pid)
|
||||
except (ImportError, Exception):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if proc.returncode != 0:
|
||||
@@ -361,7 +363,7 @@ def search_handles(
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
except Exception as e:
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
|
||||
return all_items
|
||||
@@ -428,10 +430,10 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
|
||||
|
||||
# Build engagement dict (Bird uses camelCase: likeCount, retweetCount, etc.)
|
||||
engagement = {
|
||||
"likes": tweet.get("likeCount") or tweet.get("like_count") or tweet.get("favorite_count"),
|
||||
"reposts": tweet.get("retweetCount") or tweet.get("retweet_count"),
|
||||
"replies": tweet.get("replyCount") or tweet.get("reply_count"),
|
||||
"quotes": tweet.get("quoteCount") or tweet.get("quote_count"),
|
||||
"likes": _first_of(tweet.get("likeCount"), tweet.get("like_count"), tweet.get("favorite_count")),
|
||||
"reposts": _first_of(tweet.get("retweetCount"), tweet.get("retweet_count")),
|
||||
"replies": _first_of(tweet.get("replyCount"), tweet.get("reply_count")),
|
||||
"quotes": _first_of(tweet.get("quoteCount"), tweet.get("quote_count")),
|
||||
}
|
||||
# Convert to int where possible
|
||||
for key in engagement:
|
||||
@@ -448,7 +450,7 @@ def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[
|
||||
"url": url,
|
||||
"author_handle": author_handle.lstrip("@"),
|
||||
"date": date,
|
||||
"engagement": engagement if any(v is not None for v in engagement.values()) else None,
|
||||
"engagement": engagement,
|
||||
"why_relevant": "", # Bird doesn't provide relevance explanations
|
||||
"relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user