refactor(tiktok): replace Apify with ScrapeCreators API

Root cause of empty TikTok results: Apify required monthly subscription.
ScrapeCreators is PAYG with 100 free credits and no subscription.

Key fix: ScrapeCreators nests items under aweme_info wrapper
(search_item_list[].aweme_info.{fields}), which the previous
implementation missed, causing all fields to be empty.

Changes:
- Rewrite tiktok.py to use ScrapeCreators REST API
- Add aweme_info unwrapping for correct field extraction
- Add transcript fetching via /video/transcript endpoint
- Add SCRAPECREATORS_API_KEY to env.py config
- Update last30days.py to use env.get_tiktok_token()
- Delete apify_client_wrapper.py (no longer needed)
- Update tests for new date field format (create_time)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

scripts/last30days.py

@@ -351,7 +351,7 @@ def _search_tiktok(
     depth: str,
     token: str,
 ) -> tuple:
-    """Search TikTok via Apify (runs in thread).
+    """Search TikTok via ScrapeCreators (runs in thread).
 
     Returns:
         Tuple of (tiktok_items, tiktok_error)
@@ -791,7 +791,7 @@ def run_research(
                 progress.start_tiktok()
             tiktok_future = executor.submit(
                 _search_tiktok, topic, from_date, to_date, depth,
-                config.get('APIFY_API_TOKEN', ''),
+                env.get_tiktok_token(config),
             )
 
         if do_hackernews:
@@ -1160,8 +1160,8 @@ def main():
     # Auto-detect yt-dlp for YouTube search
     has_ytdlp = env.is_ytdlp_available()
 
-    # Auto-detect Apify for TikTok
+    # Auto-detect ScrapeCreators/Apify for TikTok
-    has_apify = env.is_apify_available(config)
+    has_tiktok = env.is_tiktok_available(config)
 
     # --diagnose: show source availability and exit
     if args.diagnose:
@@ -1174,7 +1174,7 @@ def main():
             "bird_authenticated": x_source_status["bird_authenticated"],
             "bird_username": x_source_status.get("bird_username"),
             "youtube": has_ytdlp,
-            "tiktok": has_apify,
+            "tiktok": has_tiktok,
             "hackernews": True,
             "polymarket": True,
             "web_search_backend": web_source,
@@ -1204,7 +1204,7 @@ def main():
         "bird_authenticated": x_source_status["bird_authenticated"],
         "bird_username": x_source_status.get("bird_username"),
         "youtube": has_ytdlp,
-        "tiktok": has_apify,
+        "tiktok": has_tiktok,
         "hackernews": True,
         "polymarket": True,
         "web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
@@ -1289,7 +1289,7 @@ def main():
     search_do_hackernews = True
     search_do_polymarket = True
     search_run_youtube = has_ytdlp
-    search_run_tiktok = has_apify
+    search_run_tiktok = has_tiktok
     if args.search:
         search_sources = parse_search_flag(args.search)
         has_reddit = "reddit" in search_sources
@@ -1297,7 +1297,7 @@ def main():
         search_do_hackernews = "hn" in search_sources
         search_do_polymarket = "polymarket" in search_sources
         search_run_youtube = "youtube" in search_sources and has_ytdlp
-        search_run_tiktok = "tiktok" in search_sources and has_apify
+        search_run_tiktok = "tiktok" in search_sources and has_tiktok
         include_search_web = "web" in search_sources
         # Map to existing sources string
         if has_reddit and has_x:
@@ -1449,8 +1449,8 @@ def main():
         source_info["youtube_skip_reason"] = "yt-dlp not installed — fix: brew install yt-dlp"
     elif has_ytdlp and not report.youtube:
         source_info["youtube_skip_reason"] = "0 results (query may be too specific)"
-    if not has_apify:
+    if not has_tiktok:
-        source_info["tiktok_skip_reason"] = "No APIFY_API_TOKEN — sign up free at apify.com"
+        source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)"
     if not web_source:
         source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)"
 

scripts/lib/apify_client_wrapper.py

@@ -1,80 +0,0 @@
-"""Shared Apify client utilities for last30days sources.
-
-Provides a common wrapper around the apify-client SDK so that
-TikTok, Facebook, Instagram (future) all share the same client
-initialization, error handling, and cost-control patterns.
-
-One APIFY_API_TOKEN covers all Apify-backed sources.
-"""
-
-import sys
-from typing import Any, Dict, List, Optional
-
-try:
-    from apify_client import ApifyClient
-except ImportError:
-    ApifyClient = None
-
-
-def is_apify_available() -> bool:
-    """Check if the apify-client library is installed."""
-    return ApifyClient is not None
-
-
-def get_apify_client(token: str) -> "ApifyClient":
-    """Initialize Apify client with token.
-
-    Args:
-        token: Apify API token (from https://console.apify.com)
-
-    Returns:
-        Initialized ApifyClient instance
-
-    Raises:
-        ImportError: If apify-client is not installed
-    """
-    if ApifyClient is None:
-        raise ImportError(
-            "apify-client is not installed. Run: pip install apify-client"
-        )
-    return ApifyClient(token=token)
-
-
-def run_actor_sync(
-    client: "ApifyClient",
-    actor_id: str,
-    run_input: Dict[str, Any],
-    timeout_secs: int = 300,
-    max_items: Optional[int] = None,
-) -> List[Dict[str, Any]]:
-    """Run an Apify actor synchronously and return dataset items.
-
-    Args:
-        client: Initialized ApifyClient
-        actor_id: Actor identifier, e.g. "clockworks/tiktok-scraper"
-        run_input: Actor-specific input dict
-        timeout_secs: Max wait time (default 5 min)
-        max_items: Cap on returned items (cost control)
-
-    Returns:
-        List of result dicts from the actor's default dataset
-    """
-    run = client.actor(actor_id).call(
-        run_input=run_input,
-        timeout_secs=timeout_secs,
-        logger=None,  # Suppress verbose actor log streaming to stderr
-    )
-
-    dataset_id = run["defaultDatasetId"]
-    items = list(client.dataset(dataset_id).iterate_items())
-
-    if max_items and len(items) > max_items:
-        items = items[:max_items]
-    return items
-
-
-def _log(msg: str):
-    """Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
-    if sys.stderr.isatty():
-        sys.stderr.write(f"[Apify] {msg}\n")
-        sys.stderr.flush()

scripts/lib/env.py

@@ -203,6 +203,7 @@ def get_config() -> Dict[str, Any]:
         ('OPENAI_MODEL_PIN', None),
         ('XAI_MODEL_POLICY', 'latest'),
         ('XAI_MODEL_PIN', None),
+        ('SCRAPECREATORS_API_KEY', None),
         ('APIFY_API_TOKEN', None),
         ('AUTH_TOKEN', None),
         ('CT0', None),
@@ -407,13 +408,21 @@ def is_polymarket_available() -> bool:
     return True
 
 
-def is_apify_available(config: Dict[str, Any]) -> bool:
+def is_tiktok_available(config: Dict[str, Any]) -> bool:
-    """Check if Apify token is configured for TikTok/social scraping.
+    """Check if TikTok source is available (ScrapeCreators or legacy Apify).
 
-    Returns True if APIFY_API_TOKEN is set. One token covers
+    Returns True if SCRAPECREATORS_API_KEY or APIFY_API_TOKEN is set.
-    TikTok, Facebook, Instagram (all Apify-backed sources).
     """
-    return bool(config.get('APIFY_API_TOKEN'))
+    return bool(config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN'))
+
+
+def get_tiktok_token(config: Dict[str, Any]) -> str:
+    """Get TikTok API token, preferring ScrapeCreators over legacy Apify."""
+    return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or ''
+
+
+# Backward compat alias
+is_apify_available = is_tiktok_available
 
 
 def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:

scripts/lib/tiktok.py

@@ -1,9 +1,10 @@
-"""TikTok search via Apify clockworks/tiktok-scraper for /last30days.
+"""TikTok search via ScrapeCreators API for /last30days.
 
-Uses the Apify platform to search TikTok by keyword, extract engagement
+Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement
-metrics (views, likes, comments), and optionally pull video captions.
+metrics (views, likes, comments, shares), and fetch video transcripts.
 
-Requires APIFY_API_TOKEN in config. Free tier: $5/month credits.
+Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG.
+API docs: https://scrapecreators.com/docs
 """
 
 import re
@@ -11,9 +12,12 @@ import sys
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Set
 
-from . import apify_client_wrapper
+try:
+    import requests as _requests
+except ImportError:
+    _requests = None
 
-ACTOR_ID = "clockworks/tiktok-scraper"
+SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
 
 # Depth configurations: how many results to fetch / captions to extract
 DEPTH_CONFIG = {
@@ -76,7 +80,7 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo
         combined = f"{text} {' '.join(hashtags)}"
     t_tokens = _tokenize(combined)
 
-    # Split concatenated hashtags (e.g., "claudecode" → "claude", "code")
+    # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code")
     if hashtags:
         for tag in hashtags:
             tag_lower = tag.lower()
@@ -134,20 +138,20 @@ def _log(msg: str):
         sys.stderr.flush()
 
 
+def _sc_headers(token: str) -> Dict[str, str]:
+    """Build ScrapeCreators request headers."""
+    return {
+        "x-api-key": token,
+        "Content-Type": "application/json",
+    }
+
+
 def _parse_date(item: Dict[str, Any]) -> Optional[str]:
-    """Parse date from Apify TikTok item to YYYY-MM-DD.
+    """Parse date from ScrapeCreators TikTok item to YYYY-MM-DD.
 
-    Handles both createTimeISO (ISO string) and createTime (unix timestamp).
+    Handles create_time (unix timestamp).
     """
-    iso = item.get("createTimeISO")
+    ts = item.get("create_time")
-    if iso:
-        try:
-            dt = datetime.fromisoformat(iso.replace("Z", "+00:00"))
-            return dt.strftime("%Y-%m-%d")
-        except (ValueError, TypeError):
-            pass
-
-    ts = item.get("createTime")
     if ts:
         try:
             dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
@@ -158,6 +162,26 @@ def _parse_date(item: Dict[str, Any]) -> Optional[str]:
     return None
 
 
+def _clean_webvtt(text: str) -> str:
+    """Strip WebVTT timestamps and headers from transcript text."""
+    if not text:
+        return ""
+    lines = text.split('\n')
+    cleaned = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        if line.startswith('WEBVTT'):
+            continue
+        if re.match(r'^\d{2}:\d{2}', line):
+            continue
+        if '-->' in line:
+            continue
+        cleaned.append(line)
+    return ' '.join(cleaned)
+
+
 def search_tiktok(
     topic: str,
     from_date: str,
@@ -165,23 +189,23 @@ def search_tiktok(
     depth: str = "default",
     token: str = None,
 ) -> Dict[str, Any]:
-    """Search TikTok via Apify.
+    """Search TikTok via ScrapeCreators API.
 
     Args:
         topic: Search topic
         from_date: Start date (YYYY-MM-DD)
         to_date: End date (YYYY-MM-DD)
         depth: 'quick', 'default', or 'deep'
-        token: Apify API token
+        token: ScrapeCreators API key
 
     Returns:
         Dict with 'items' list and optional 'error'.
     """
     if not token:
-        return {"items": [], "error": "No APIFY_API_TOKEN configured"}
+        return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
 
-    if not apify_client_wrapper.is_apify_available():
+    if not _requests:
-        return {"items": [], "error": "apify-client not installed (pip install apify-client)"}
+        return {"items": [], "error": "requests library not installed"}
 
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     core_topic = _extract_core_subject(topic)
@@ -189,48 +213,61 @@ def search_tiktok(
     _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
 
     try:
-        client = apify_client_wrapper.get_apify_client(token)
+        resp = _requests.get(
-        run_input = {
+            f"{SCRAPECREATORS_BASE}/search/keyword",
-            "searchQueries": [core_topic],
+            params={"query": core_topic, "sort_by": "relevance"},
-            "resultsPerPage": config["results_per_page"],
+            headers=_sc_headers(token),
-            "shouldDownloadSubtitles": False,
+            timeout=30,
-            "shouldDownloadVideos": False,
-            "shouldDownloadCovers": False,
-        }
-        raw_items = apify_client_wrapper.run_actor_sync(
-            client, ACTOR_ID, run_input,
-            timeout_secs=120,
-            max_items=config["results_per_page"],
         )
+        resp.raise_for_status()
+        data = resp.json()
     except Exception as e:
-        _log(f"Apify error: {e}")
+        _log(f"ScrapeCreators error: {e}")
         return {"items": [], "error": f"{type(e).__name__}: {e}"}
 
+    # Items are nested under aweme_info
+    raw_entries = data.get("search_item_list") or data.get("data") or []
+    raw_items = []
+    for entry in raw_entries:
+        if isinstance(entry, dict):
+            info = entry.get("aweme_info", entry)
+            raw_items.append(info)
+
+    # Limit to configured count
+    raw_items = raw_items[:config["results_per_page"]]
+
     # Parse items
     items = []
     for raw in raw_items:
-        video_id = str(raw.get("id", ""))
+        video_id = str(raw.get("aweme_id", ""))
-        text = raw.get("text", "")
+        text = raw.get("desc", "")
-        play_count = raw.get("playCount") or 0
+        stats = raw.get("statistics") or {}
-        digg_count = raw.get("diggCount") or 0
+        play_count = stats.get("play_count") or 0
-        comment_count = raw.get("commentCount") or 0
+        digg_count = stats.get("digg_count") or 0
-        share_count = raw.get("shareCount") or 0
+        comment_count = stats.get("comment_count") or 0
-        author_meta = raw.get("authorMeta") or {}
+        share_count = stats.get("share_count") or 0
-        author_name = author_meta.get("name", "")
+        author = raw.get("author") or {}
-        web_url = raw.get("webVideoUrl", "")
+        author_name = author.get("unique_id", "")
-        hashtags_raw = raw.get("hashtags") or []
+        share_url = raw.get("share_url", "")
-        hashtag_names = [h.get("name", "") for h in hashtags_raw if isinstance(h, dict)]
+        text_extra = raw.get("text_extra") or []
-        duration = (raw.get("videoMeta") or {}).get("duration")
+        hashtag_names = [t.get("hashtag_name", "") for t in text_extra
+                         if isinstance(t, dict) and t.get("hashtag_name")]
+        duration = (raw.get("video") or {}).get("duration")
 
         date_str = _parse_date(raw)
 
         # Compute relevance with hashtag boost
         relevance = _compute_relevance(core_topic, text, hashtag_names)
 
+        # Build URL: prefer share_url, fallback to constructed URL
+        url = share_url.split("?")[0] if share_url else ""
+        if not url and author_name and video_id:
+            url = f"https://www.tiktok.com/@{author_name}/video/{video_id}"
+
         items.append({
             "video_id": video_id,
             "text": text,
-            "url": web_url or f"https://www.tiktok.com/@{author_name}/video/{video_id}",
+            "url": url,
             "author_name": author_name,
             "date": date_str,
             "engagement": {
@@ -268,24 +305,24 @@ def fetch_captions(
     token: str,
     depth: str = "default",
 ) -> Dict[str, str]:
-    """Fetch captions for top N TikTok videos.
+    """Fetch transcripts for top N TikTok videos via ScrapeCreators.
 
     Strategy:
-    1. Primary: Use the 'text' field (video description) — always free
+    1. Use the 'text' field (video description) as baseline caption
-    2. For top N, re-run actor with shouldDownloadSubtitles for spoken-word
+    2. For top N, call /video/transcript for spoken-word captions
 
     Args:
         video_items: Items from search_tiktok()
-        token: Apify API token
+        token: ScrapeCreators API key
         depth: Depth level for caption limit
 
     Returns:
-        Dict mapping video_id → caption text (truncated to 500 words)
+        Dict mapping video_id -> caption text (truncated to 500 words)
     """
     config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
     max_captions = config["max_captions"]
 
-    if not video_items or not token:
+    if not video_items or not token or not _requests:
         return {}
 
     top_items = video_items[:max_captions]
@@ -303,35 +340,33 @@ def fetch_captions(
                 text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
             captions[vid] = text
 
-    # Second pass: try to get spoken-word subtitles for top videos
+    # Second pass: try to get spoken-word transcripts (1 credit each)
-    try:
+    for item in top_items:
-        urls = [item["url"] for item in top_items if item.get("url")]
+        vid = item["video_id"]
-        if urls:
+        url = item.get("url", "")
-            client = apify_client_wrapper.get_apify_client(token)
+        if not url:
-            run_input = {
+            continue
-                "postURLs": urls,
+        try:
-                "shouldDownloadSubtitles": True,
+            resp = _requests.get(
-                "shouldDownloadVideos": False,
+                f"{SCRAPECREATORS_BASE}/video/transcript",
-                "shouldDownloadCovers": False,
+                params={"url": url},
-            }
+                headers=_sc_headers(token),
-            subtitle_items = apify_client_wrapper.run_actor_sync(
+                timeout=15,
-                client, ACTOR_ID, run_input,
-                timeout_secs=60,
-                max_items=max_captions,
             )
-            for raw in subtitle_items:
+            if resp.status_code == 200:
-                vid = str(raw.get("id", ""))
+                data = resp.json()
-                # Check for subtitle text in the response
+                transcript = data.get("transcript")
-                subtitle_text = raw.get("subtitleText") or raw.get("subtitles") or ""
+                if transcript:
-                if isinstance(subtitle_text, list):
+                    if isinstance(transcript, list):
-                    subtitle_text = " ".join(str(s) for s in subtitle_text)
+                        transcript = " ".join(str(s) for s in transcript)
-                if subtitle_text and vid:
+                    transcript = _clean_webvtt(transcript)
-                    words = subtitle_text.split()
+                    if transcript:
-                    if len(words) > CAPTION_MAX_WORDS:
+                        words = transcript.split()
-                        subtitle_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
+                        if len(words) > CAPTION_MAX_WORDS:
-                    captions[vid] = subtitle_text  # Override text with spoken-word
+                            transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
-    except Exception as e:
+                        captions[vid] = transcript
-        _log(f"Subtitle enrichment failed (using text captions): {e}")
+        except Exception as e:
+            _log(f"Transcript fetch failed for {vid}: {e}")
 
     got = sum(1 for v in captions.values() if v)
     _log(f"Got captions for {got}/{len(top_items)} videos")
@@ -352,7 +387,7 @@ def search_and_enrich(
         from_date: Start date (YYYY-MM-DD)
         to_date: End date (YYYY-MM-DD)
         depth: 'quick', 'default', or 'deep'
-        token: Apify API token
+        token: ScrapeCreators API key
 
     Returns:
         Dict with 'items' list. Each item has a 'caption_snippet' field.

tests/test_tiktok.py

@@ -58,14 +58,10 @@ class TestExtractCoreSubject(unittest.TestCase):
 
 
 class TestParseDate(unittest.TestCase):
-    """Test date parsing from Apify items."""
+    """Test date parsing from ScrapeCreators items."""
-
-    def test_iso_date(self):
-        item = {"createTimeISO": "2026-02-28T17:44:35.000Z"}
-        self.assertEqual(tiktok._parse_date(item), "2026-02-28")
 
     def test_unix_timestamp(self):
-        item = {"createTime": 1756403075}
+        item = {"create_time": 1756403075}
         result = tiktok._parse_date(item)
         self.assertIsNotNone(result)
         self.assertRegex(result, r"\d{4}-\d{2}-\d{2}")
@@ -75,6 +71,19 @@ class TestParseDate(unittest.TestCase):
         self.assertIsNone(tiktok._parse_date(item))
 
 
+class TestCleanWebVTT(unittest.TestCase):
+    """Test WebVTT transcript cleaning."""
+
+    def test_strips_timestamps(self):
+        raw = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello world\n\n00:00:02.000 --> 00:00:04.000\nGoodbye"
+        result = tiktok._clean_webvtt(raw)
+        self.assertEqual(result, "Hello world Goodbye")
+
+    def test_empty_input(self):
+        self.assertEqual(tiktok._clean_webvtt(""), "")
+        self.assertEqual(tiktok._clean_webvtt(None), "")
+
+
 class TestNormalizeTikTokItems(unittest.TestCase):
     """Test TikTok normalization."""