feat: add Xquik as X/Twitter search source
Add Xquik (xquik.com) as a new X/Twitter search source that uses a REST API with full engagement metrics (likes, retweets, replies, quotes, views, bookmarks). Uses stdlib urllib only -- no new dependencies. - scripts/lib/xquik.py: source module with search, parse, query expansion - tests/test_xquik.py: 32 unit tests covering all functions - env.py: XQUIK_API_KEY config and availability check - pipeline.py: source registration and retrieve dispatch - normalize.py: reuses _normalize_x (same item format as Bird) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Burak Bayır
@@ -0,0 +1,227 @@
|
||||
"""Xquik X search source for the v3.0.0 last30days pipeline.
|
||||
|
||||
Uses the Xquik REST API (https://xquik.com/api/v1) to search X/Twitter
|
||||
with full engagement metrics (likes, retweets, replies, quotes, views,
|
||||
bookmarks). Requires an API key from xquik.com.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from . import http, log
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
# Depth configurations: number of results to request per query
|
||||
DEPTH_CONFIG = {
|
||||
"quick": {"limit": 10, "queries": 1},
|
||||
"default": {"limit": 20, "queries": 2},
|
||||
"deep": {"limit": 40, "queries": 3},
|
||||
}
|
||||
|
||||
_BASE_URL = "https://xquik.com/api/v1"
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
log.source_log("Xquik", msg, tty_only=False)
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject for X search queries."""
|
||||
from .query import extract_core_subject
|
||||
return extract_core_subject(topic, max_words=5, strip_suffixes=True)
|
||||
|
||||
|
||||
def expand_xquik_queries(topic: str, depth: str) -> List[str]:
|
||||
"""Generate query variants based on depth.
|
||||
|
||||
Args:
|
||||
topic: Research topic
|
||||
depth: "quick", "default", or "deep"
|
||||
|
||||
Returns:
|
||||
List of query strings (1 for quick, 2 for default, 3 for deep).
|
||||
"""
|
||||
core = _extract_core_subject(topic)
|
||||
queries = [core]
|
||||
|
||||
# Add original topic if meaningfully different
|
||||
if topic.lower().strip() != core.lower().strip():
|
||||
queries.append(topic.strip())
|
||||
|
||||
# Add compound term variant for deep searches
|
||||
if len(queries) < 3:
|
||||
from .query import extract_compound_terms
|
||||
compounds = extract_compound_terms(topic)
|
||||
if compounds:
|
||||
or_parts = " OR ".join(f'"{t}"' for t in compounds[:3])
|
||||
queries.append(f"({or_parts})")
|
||||
|
||||
cap = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])["queries"]
|
||||
return queries[:cap]
|
||||
|
||||
|
||||
def search_xquik(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
token: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search X via Xquik REST API.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD)
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: Research depth - "quick", "default", or "deep"
|
||||
token: Xquik API key
|
||||
|
||||
Returns:
|
||||
Dict with "items" list and optional "error" string.
|
||||
"""
|
||||
if not token:
|
||||
return {"items": [], "error": "No XQUIK_API_KEY configured"}
|
||||
|
||||
cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
queries = expand_xquik_queries(topic, depth)
|
||||
all_items: List[Dict[str, Any]] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
for query_text in queries:
|
||||
try:
|
||||
url = f"{_BASE_URL}/x/tweets/search"
|
||||
# Build query with date filter
|
||||
q = f"{query_text} since:{from_date} until:{to_date}"
|
||||
params = f"q={_url_encode(q)}&queryType=Top&limit={cfg['limit']}"
|
||||
full_url = f"{url}?{params}"
|
||||
|
||||
_log(f"Searching: {query_text}")
|
||||
response = http.get(
|
||||
full_url,
|
||||
headers={"X-Api-Key": token},
|
||||
timeout=30,
|
||||
retries=2,
|
||||
)
|
||||
|
||||
tweets = response.get("tweets", [])
|
||||
if not isinstance(tweets, list):
|
||||
continue
|
||||
|
||||
for i, tweet in enumerate(tweets):
|
||||
if not isinstance(tweet, dict):
|
||||
continue
|
||||
tweet_id = str(tweet.get("id", ""))
|
||||
if tweet_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(tweet_id)
|
||||
|
||||
item = _parse_tweet(tweet, i + len(all_items), query_text)
|
||||
if item:
|
||||
all_items.append(item)
|
||||
|
||||
except http.HTTPError as exc:
|
||||
status = getattr(exc, "status_code", None)
|
||||
if status in (401, 403):
|
||||
return {"items": [], "error": f"Xquik auth failed ({status})"}
|
||||
_log(f"HTTP error for query '{query_text}': {exc}")
|
||||
except Exception as exc:
|
||||
_log(f"Error for query '{query_text}': {exc}")
|
||||
|
||||
return {"items": all_items}
|
||||
|
||||
|
||||
def search_and_enrich(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
token: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search X via Xquik and return results.
|
||||
|
||||
Xquik API returns full engagement data by default, so no separate
|
||||
enrichment step is needed.
|
||||
"""
|
||||
return search_xquik(topic, from_date, to_date, depth=depth, token=token)
|
||||
|
||||
|
||||
def parse_xquik_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Extract items from search response.
|
||||
|
||||
Args:
|
||||
response: Response dict from search_xquik()
|
||||
|
||||
Returns:
|
||||
List of normalized item dicts.
|
||||
"""
|
||||
return response.get("items", [])
|
||||
|
||||
|
||||
def _parse_tweet(
|
||||
tweet: Dict[str, Any], index: int, query: str
|
||||
) -> Dict[str, Any] | None:
|
||||
"""Parse a single tweet from the API response into the standard item format."""
|
||||
author = tweet.get("author") or {}
|
||||
username = str(author.get("username", "")).lstrip("@")
|
||||
tweet_id = str(tweet.get("id", ""))
|
||||
|
||||
# Build URL
|
||||
url = ""
|
||||
if username and tweet_id:
|
||||
url = f"https://x.com/{username}/status/{tweet_id}"
|
||||
if not url:
|
||||
return None
|
||||
|
||||
# Parse date
|
||||
date = None
|
||||
created_at = tweet.get("createdAt") or ""
|
||||
if created_at:
|
||||
try:
|
||||
if len(created_at) > 10 and created_at[10] == "T":
|
||||
dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
else:
|
||||
dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
|
||||
date = dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
text = str(tweet.get("text", "")).strip()[:500]
|
||||
|
||||
# Build engagement dict with full metrics
|
||||
engagement = {
|
||||
"likes": _safe_int(tweet.get("likeCount")),
|
||||
"reposts": _safe_int(tweet.get("retweetCount")),
|
||||
"replies": _safe_int(tweet.get("replyCount")),
|
||||
"quotes": _safe_int(tweet.get("quoteCount")),
|
||||
"views": _safe_int(tweet.get("viewCount")),
|
||||
"bookmarks": _safe_int(tweet.get("bookmarkCount")),
|
||||
}
|
||||
|
||||
return {
|
||||
"id": f"XQ{index + 1}",
|
||||
"text": text,
|
||||
"url": url,
|
||||
"author_handle": username,
|
||||
"date": date,
|
||||
"engagement": engagement,
|
||||
"relevance": _compute_relevance(query, text) if query else 0.7,
|
||||
"why_relevant": "",
|
||||
}
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
"""Convert value to int, returning None on failure."""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _url_encode(text: str) -> str:
|
||||
"""URL-encode a string using stdlib."""
|
||||
from urllib.parse import quote
|
||||
return quote(text, safe="")
|
||||
Reference in New Issue
Block a user