Merge pull request #191 from kriptoburak/feat/add-xquik-source

feat: add Xquik as X/Twitter search source

scripts/lib/env.py

@@ -261,6 +261,7 @@ def get_config() -> dict[str, Any]:
         ('SERPER_API_KEY', None),
         ('OPENROUTER_API_KEY', None),
         ('PARALLEL_API_KEY', None),
+        ('XQUIK_API_KEY', None),
         ('FROM_BROWSER', None),
         ('SETUP_COMPLETE', None),
         ('INCLUDE_SOURCES', None),
@@ -613,3 +614,17 @@ def is_pinterest_available(config: dict[str, Any]) -> bool:
 def get_pinterest_token(config: dict[str, Any]) -> str:
     """Get Pinterest API token (same ScrapeCreators key as TikTok/Instagram)."""
     return config.get('SCRAPECREATORS_API_KEY') or ''
+
+
+# Xquik
+def is_xquik_available(config: dict[str, Any]) -> bool:
+    """Check if Xquik X search source is available.
+
+    Requires XQUIK_API_KEY (API key from xquik.com).
+    """
+    return bool(config.get('XQUIK_API_KEY'))
+
+
+def get_xquik_token(config: dict[str, Any]) -> str:
+    """Get Xquik API key."""
+    return config.get('XQUIK_API_KEY') or ''

scripts/lib/normalize.py

@@ -46,6 +46,7 @@ def normalize_source_items(
         "bluesky": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "BS", "Bluesky post"),
         "truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TS", "Truth Social post"),
         "threads": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TH", "Threads post"),
+        "xquik": _normalize_x,
         "pinterest": _normalize_pinterest,
         "polymarket": _normalize_polymarket,
         "grounding": _normalize_grounding,

scripts/lib/pipeline.py

@@ -39,6 +39,7 @@ from . import (
     truthsocial,
     xai_x,
     xiaohongshu_api,
+    xquik,
     youtube_yt,
 )
 from .cluster import cluster_candidates
@@ -56,6 +57,7 @@ SEARCH_ALIAS = {
     "truth": "truthsocial",
     "web": "grounding",
     "xhs": "xiaohongshu",
+    "xquik": "xquik",
 }
 
 MAX_SOURCE_FETCHES: dict[str, int] = {"x": 2}
@@ -74,6 +76,7 @@ MOCK_AVAILABLE_SOURCES = [
     "xiaohongshu",
     "github",
     "perplexity",
+    "xquik",
 ]
 
 
@@ -117,6 +120,8 @@ def available_sources(config: dict[str, Any], requested_sources: list[str] | Non
         available.append("threads")
     if requested_sources and "pinterest" in requested_sources and env.is_pinterest_available(config):
         available.append("pinterest")
+    if env.is_xquik_available(config):
+        available.append("xquik")
     return available
 
 
@@ -935,6 +940,13 @@ def _retrieve_stream(
         ), {}
     if source == "perplexity":
         return perplexity.search(subquery.search_query, date_range, config, deep=config.get("_deep_research", False))
+    if source == "xquik":
+        result = xquik.search_xquik(
+            subquery.search_query, from_date, to_date,
+            depth=depth,
+            token=env.get_xquik_token(config),
+        )
+        return xquik.parse_xquik_response(result), {}
     raise RuntimeError(f"Unsupported source: {source}")
 
 

scripts/lib/xquik.py

@@ -0,0 +1,227 @@
+"""Xquik X search source for the v3.0.0 last30days pipeline.
+
+Uses the Xquik REST API (https://xquik.com/api/v1) to search X/Twitter
+with full engagement metrics (likes, retweets, replies, quotes, views,
+bookmarks). Requires an API key from xquik.com.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, List
+
+from . import http, log
+from .relevance import token_overlap_relevance as _compute_relevance
+
+# Depth configurations: number of results to request per query
+DEPTH_CONFIG = {
+    "quick": {"limit": 10, "queries": 1},
+    "default": {"limit": 20, "queries": 2},
+    "deep": {"limit": 40, "queries": 3},
+}
+
+_BASE_URL = "https://xquik.com/api/v1"
+
+
+def _log(msg: str):
+    log.source_log("Xquik", msg, tty_only=False)
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject for X search queries."""
+    from .query import extract_core_subject
+    return extract_core_subject(topic, max_words=5, strip_suffixes=True)
+
+
+def expand_xquik_queries(topic: str, depth: str) -> List[str]:
+    """Generate query variants based on depth.
+
+    Args:
+        topic: Research topic
+        depth: "quick", "default", or "deep"
+
+    Returns:
+        List of query strings (1 for quick, 2 for default, 3 for deep).
+    """
+    core = _extract_core_subject(topic)
+    queries = [core]
+
+    # Add original topic if meaningfully different
+    if topic.lower().strip() != core.lower().strip():
+        queries.append(topic.strip())
+
+    # Add compound term variant for deep searches
+    if len(queries) < 3:
+        from .query import extract_compound_terms
+        compounds = extract_compound_terms(topic)
+        if compounds:
+            or_parts = " OR ".join(f'"{t}"' for t in compounds[:3])
+            queries.append(f"({or_parts})")
+
+    cap = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])["queries"]
+    return queries[:cap]
+
+
+def search_xquik(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = "",
+) -> Dict[str, Any]:
+    """Search X via Xquik REST API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: Research depth - "quick", "default", or "deep"
+        token: Xquik API key
+
+    Returns:
+        Dict with "items" list and optional "error" string.
+    """
+    if not token:
+        return {"items": [], "error": "No XQUIK_API_KEY configured"}
+
+    cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    queries = expand_xquik_queries(topic, depth)
+    all_items: List[Dict[str, Any]] = []
+    seen_ids: set[str] = set()
+
+    for query_text in queries:
+        try:
+            url = f"{_BASE_URL}/x/tweets/search"
+            # Build query with date filter
+            q = f"{query_text} since:{from_date} until:{to_date}"
+            params = f"q={_url_encode(q)}&queryType=Top&limit={cfg['limit']}"
+            full_url = f"{url}?{params}"
+
+            _log(f"Searching: {query_text}")
+            response = http.get(
+                full_url,
+                headers={"X-Api-Key": token},
+                timeout=30,
+                retries=2,
+            )
+
+            tweets = response.get("tweets", [])
+            if not isinstance(tweets, list):
+                continue
+
+            for i, tweet in enumerate(tweets):
+                if not isinstance(tweet, dict):
+                    continue
+                tweet_id = str(tweet.get("id", ""))
+                if tweet_id in seen_ids:
+                    continue
+                seen_ids.add(tweet_id)
+
+                item = _parse_tweet(tweet, i + len(all_items), query_text)
+                if item:
+                    all_items.append(item)
+
+        except http.HTTPError as exc:
+            status = getattr(exc, "status_code", None)
+            if status in (401, 403):
+                return {"items": [], "error": f"Xquik auth failed ({status})"}
+            _log(f"HTTP error for query '{query_text}': {exc}")
+        except Exception as exc:
+            _log(f"Error for query '{query_text}': {exc}")
+
+    return {"items": all_items}
+
+
+def search_and_enrich(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = "",
+) -> Dict[str, Any]:
+    """Search X via Xquik and return results.
+
+    Xquik API returns full engagement data by default, so no separate
+    enrichment step is needed.
+    """
+    return search_xquik(topic, from_date, to_date, depth=depth, token=token)
+
+
+def parse_xquik_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extract items from search response.
+
+    Args:
+        response: Response dict from search_xquik()
+
+    Returns:
+        List of normalized item dicts.
+    """
+    return response.get("items", [])
+
+
+def _parse_tweet(
+    tweet: Dict[str, Any], index: int, query: str
+) -> Dict[str, Any] | None:
+    """Parse a single tweet from the API response into the standard item format."""
+    author = tweet.get("author") or {}
+    username = str(author.get("username", "")).lstrip("@")
+    tweet_id = str(tweet.get("id", ""))
+
+    # Build URL
+    url = ""
+    if username and tweet_id:
+        url = f"https://x.com/{username}/status/{tweet_id}"
+    if not url:
+        return None
+
+    # Parse date
+    date = None
+    created_at = tweet.get("createdAt") or ""
+    if created_at:
+        try:
+            if len(created_at) > 10 and created_at[10] == "T":
+                dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+            else:
+                dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
+            date = dt.strftime("%Y-%m-%d")
+        except (ValueError, TypeError):
+            pass
+
+    text = str(tweet.get("text", "")).strip()[:500]
+
+    # Build engagement dict with full metrics
+    engagement = {
+        "likes": _safe_int(tweet.get("likeCount")),
+        "reposts": _safe_int(tweet.get("retweetCount")),
+        "replies": _safe_int(tweet.get("replyCount")),
+        "quotes": _safe_int(tweet.get("quoteCount")),
+        "views": _safe_int(tweet.get("viewCount")),
+        "bookmarks": _safe_int(tweet.get("bookmarkCount")),
+    }
+
+    return {
+        "id": f"XQ{index + 1}",
+        "text": text,
+        "url": url,
+        "author_handle": username,
+        "date": date,
+        "engagement": engagement,
+        "relevance": _compute_relevance(query, text) if query else 0.7,
+        "why_relevant": "",
+    }
+
+
+def _safe_int(value: Any) -> int | None:
+    """Convert value to int, returning None on failure."""
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _url_encode(text: str) -> str:
+    """URL-encode a string using stdlib."""
+    from urllib.parse import quote
+    return quote(text, safe="")