diff --git a/scripts/last30days.py b/scripts/last30days.py index 35e52f1..5daf907 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -351,7 +351,7 @@ def _search_tiktok( depth: str, token: str, ) -> tuple: - """Search TikTok via Apify (runs in thread). + """Search TikTok via ScrapeCreators (runs in thread). Returns: Tuple of (tiktok_items, tiktok_error) @@ -791,7 +791,7 @@ def run_research( progress.start_tiktok() tiktok_future = executor.submit( _search_tiktok, topic, from_date, to_date, depth, - config.get('APIFY_API_TOKEN', ''), + env.get_tiktok_token(config), ) if do_hackernews: @@ -1160,8 +1160,8 @@ def main(): # Auto-detect yt-dlp for YouTube search has_ytdlp = env.is_ytdlp_available() - # Auto-detect Apify for TikTok - has_apify = env.is_apify_available(config) + # Auto-detect ScrapeCreators/Apify for TikTok + has_tiktok = env.is_tiktok_available(config) # --diagnose: show source availability and exit if args.diagnose: @@ -1174,7 +1174,7 @@ def main(): "bird_authenticated": x_source_status["bird_authenticated"], "bird_username": x_source_status.get("bird_username"), "youtube": has_ytdlp, - "tiktok": has_apify, + "tiktok": has_tiktok, "hackernews": True, "polymarket": True, "web_search_backend": web_source, @@ -1204,7 +1204,7 @@ def main(): "bird_authenticated": x_source_status["bird_authenticated"], "bird_username": x_source_status.get("bird_username"), "youtube": has_ytdlp, - "tiktok": has_apify, + "tiktok": has_tiktok, "hackernews": True, "polymarket": True, "web_search_backend": "deferred to assistant" if args.no_native_web else web_source, @@ -1289,7 +1289,7 @@ def main(): search_do_hackernews = True search_do_polymarket = True search_run_youtube = has_ytdlp - search_run_tiktok = has_apify + search_run_tiktok = has_tiktok if args.search: search_sources = parse_search_flag(args.search) has_reddit = "reddit" in search_sources @@ -1297,7 +1297,7 @@ def main(): search_do_hackernews = "hn" in search_sources search_do_polymarket = "polymarket" in search_sources search_run_youtube = "youtube" in search_sources and has_ytdlp - search_run_tiktok = "tiktok" in search_sources and has_apify + search_run_tiktok = "tiktok" in search_sources and has_tiktok include_search_web = "web" in search_sources # Map to existing sources string if has_reddit and has_x: @@ -1449,8 +1449,8 @@ def main(): source_info["youtube_skip_reason"] = "yt-dlp not installed — fix: brew install yt-dlp" elif has_ytdlp and not report.youtube: source_info["youtube_skip_reason"] = "0 results (query may be too specific)" - if not has_apify: - source_info["tiktok_skip_reason"] = "No APIFY_API_TOKEN — sign up free at apify.com" + if not has_tiktok: + source_info["tiktok_skip_reason"] = "No SCRAPECREATORS_API_KEY - sign up at scrapecreators.com (100 free credits)" if not web_source: source_info["web_skip_reason"] = "assistant will use WebSearch (add BRAVE_API_KEY for native search)" diff --git a/scripts/lib/apify_client_wrapper.py b/scripts/lib/apify_client_wrapper.py deleted file mode 100644 index b48f75e..0000000 --- a/scripts/lib/apify_client_wrapper.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Shared Apify client utilities for last30days sources. - -Provides a common wrapper around the apify-client SDK so that -TikTok, Facebook, Instagram (future) all share the same client -initialization, error handling, and cost-control patterns. - -One APIFY_API_TOKEN covers all Apify-backed sources. -""" - -import sys -from typing import Any, Dict, List, Optional - -try: - from apify_client import ApifyClient -except ImportError: - ApifyClient = None - - -def is_apify_available() -> bool: - """Check if the apify-client library is installed.""" - return ApifyClient is not None - - -def get_apify_client(token: str) -> "ApifyClient": - """Initialize Apify client with token. - - Args: - token: Apify API token (from https://console.apify.com) - - Returns: - Initialized ApifyClient instance - - Raises: - ImportError: If apify-client is not installed - """ - if ApifyClient is None: - raise ImportError( - "apify-client is not installed. Run: pip install apify-client" - ) - return ApifyClient(token=token) - - -def run_actor_sync( - client: "ApifyClient", - actor_id: str, - run_input: Dict[str, Any], - timeout_secs: int = 300, - max_items: Optional[int] = None, -) -> List[Dict[str, Any]]: - """Run an Apify actor synchronously and return dataset items. - - Args: - client: Initialized ApifyClient - actor_id: Actor identifier, e.g. "clockworks/tiktok-scraper" - run_input: Actor-specific input dict - timeout_secs: Max wait time (default 5 min) - max_items: Cap on returned items (cost control) - - Returns: - List of result dicts from the actor's default dataset - """ - run = client.actor(actor_id).call( - run_input=run_input, - timeout_secs=timeout_secs, - logger=None, # Suppress verbose actor log streaming to stderr - ) - - dataset_id = run["defaultDatasetId"] - items = list(client.dataset(dataset_id).iterate_items()) - - if max_items and len(items) > max_items: - items = items[:max_items] - return items - - -def _log(msg: str): - """Log to stderr (only in interactive terminals; spinner handles non-TTY).""" - if sys.stderr.isatty(): - sys.stderr.write(f"[Apify] {msg}\n") - sys.stderr.flush() diff --git a/scripts/lib/env.py b/scripts/lib/env.py index 313d752..ab72041 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -203,6 +203,7 @@ def get_config() -> Dict[str, Any]: ('OPENAI_MODEL_PIN', None), ('XAI_MODEL_POLICY', 'latest'), ('XAI_MODEL_PIN', None), + ('SCRAPECREATORS_API_KEY', None), ('APIFY_API_TOKEN', None), ('AUTH_TOKEN', None), ('CT0', None), @@ -407,13 +408,21 @@ def is_polymarket_available() -> bool: return True -def is_apify_available(config: Dict[str, Any]) -> bool: - """Check if Apify token is configured for TikTok/social scraping. +def is_tiktok_available(config: Dict[str, Any]) -> bool: + """Check if TikTok source is available (ScrapeCreators or legacy Apify). - Returns True if APIFY_API_TOKEN is set. One token covers - TikTok, Facebook, Instagram (all Apify-backed sources). + Returns True if SCRAPECREATORS_API_KEY or APIFY_API_TOKEN is set. """ - return bool(config.get('APIFY_API_TOKEN')) + return bool(config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN')) + + +def get_tiktok_token(config: Dict[str, Any]) -> str: + """Get TikTok API token, preferring ScrapeCreators over legacy Apify.""" + return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or '' + + +# Backward compat alias +is_apify_available = is_tiktok_available def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/scripts/lib/tiktok.py b/scripts/lib/tiktok.py index 8592ebd..3f5f704 100644 --- a/scripts/lib/tiktok.py +++ b/scripts/lib/tiktok.py @@ -1,9 +1,10 @@ -"""TikTok search via Apify clockworks/tiktok-scraper for /last30days. +"""TikTok search via ScrapeCreators API for /last30days. -Uses the Apify platform to search TikTok by keyword, extract engagement -metrics (views, likes, comments), and optionally pull video captions. +Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement +metrics (views, likes, comments, shares), and fetch video transcripts. -Requires APIFY_API_TOKEN in config. Free tier: $5/month credits. +Requires SCRAPECREATORS_API_KEY in config. 100 free credits, then PAYG. +API docs: https://scrapecreators.com/docs """ import re @@ -11,9 +12,12 @@ import sys from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Set -from . import apify_client_wrapper +try: + import requests as _requests +except ImportError: + _requests = None -ACTOR_ID = "clockworks/tiktok-scraper" +SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok" # Depth configurations: how many results to fetch / captions to extract DEPTH_CONFIG = { @@ -76,7 +80,7 @@ def _compute_relevance(query: str, text: str, hashtags: List[str] = None) -> flo combined = f"{text} {' '.join(hashtags)}" t_tokens = _tokenize(combined) - # Split concatenated hashtags (e.g., "claudecode" → "claude", "code") + # Split concatenated hashtags (e.g., "claudecode" -> "claude", "code") if hashtags: for tag in hashtags: tag_lower = tag.lower() @@ -134,20 +138,20 @@ def _log(msg: str): sys.stderr.flush() +def _sc_headers(token: str) -> Dict[str, str]: + """Build ScrapeCreators request headers.""" + return { + "x-api-key": token, + "Content-Type": "application/json", + } + + def _parse_date(item: Dict[str, Any]) -> Optional[str]: - """Parse date from Apify TikTok item to YYYY-MM-DD. + """Parse date from ScrapeCreators TikTok item to YYYY-MM-DD. - Handles both createTimeISO (ISO string) and createTime (unix timestamp). + Handles create_time (unix timestamp). """ - iso = item.get("createTimeISO") - if iso: - try: - dt = datetime.fromisoformat(iso.replace("Z", "+00:00")) - return dt.strftime("%Y-%m-%d") - except (ValueError, TypeError): - pass - - ts = item.get("createTime") + ts = item.get("create_time") if ts: try: dt = datetime.fromtimestamp(int(ts), tz=timezone.utc) @@ -158,6 +162,26 @@ def _parse_date(item: Dict[str, Any]) -> Optional[str]: return None +def _clean_webvtt(text: str) -> str: + """Strip WebVTT timestamps and headers from transcript text.""" + if not text: + return "" + lines = text.split('\n') + cleaned = [] + for line in lines: + line = line.strip() + if not line: + continue + if line.startswith('WEBVTT'): + continue + if re.match(r'^\d{2}:\d{2}', line): + continue + if '-->' in line: + continue + cleaned.append(line) + return ' '.join(cleaned) + + def search_tiktok( topic: str, from_date: str, @@ -165,23 +189,23 @@ def search_tiktok( depth: str = "default", token: str = None, ) -> Dict[str, Any]: - """Search TikTok via Apify. + """Search TikTok via ScrapeCreators API. Args: topic: Search topic from_date: Start date (YYYY-MM-DD) to_date: End date (YYYY-MM-DD) depth: 'quick', 'default', or 'deep' - token: Apify API token + token: ScrapeCreators API key Returns: Dict with 'items' list and optional 'error'. """ if not token: - return {"items": [], "error": "No APIFY_API_TOKEN configured"} + return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"} - if not apify_client_wrapper.is_apify_available(): - return {"items": [], "error": "apify-client not installed (pip install apify-client)"} + if not _requests: + return {"items": [], "error": "requests library not installed"} config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) core_topic = _extract_core_subject(topic) @@ -189,48 +213,61 @@ def search_tiktok( _log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})") try: - client = apify_client_wrapper.get_apify_client(token) - run_input = { - "searchQueries": [core_topic], - "resultsPerPage": config["results_per_page"], - "shouldDownloadSubtitles": False, - "shouldDownloadVideos": False, - "shouldDownloadCovers": False, - } - raw_items = apify_client_wrapper.run_actor_sync( - client, ACTOR_ID, run_input, - timeout_secs=120, - max_items=config["results_per_page"], + resp = _requests.get( + f"{SCRAPECREATORS_BASE}/search/keyword", + params={"query": core_topic, "sort_by": "relevance"}, + headers=_sc_headers(token), + timeout=30, ) + resp.raise_for_status() + data = resp.json() except Exception as e: - _log(f"Apify error: {e}") + _log(f"ScrapeCreators error: {e}") return {"items": [], "error": f"{type(e).__name__}: {e}"} + # Items are nested under aweme_info + raw_entries = data.get("search_item_list") or data.get("data") or [] + raw_items = [] + for entry in raw_entries: + if isinstance(entry, dict): + info = entry.get("aweme_info", entry) + raw_items.append(info) + + # Limit to configured count + raw_items = raw_items[:config["results_per_page"]] + # Parse items items = [] for raw in raw_items: - video_id = str(raw.get("id", "")) - text = raw.get("text", "") - play_count = raw.get("playCount") or 0 - digg_count = raw.get("diggCount") or 0 - comment_count = raw.get("commentCount") or 0 - share_count = raw.get("shareCount") or 0 - author_meta = raw.get("authorMeta") or {} - author_name = author_meta.get("name", "") - web_url = raw.get("webVideoUrl", "") - hashtags_raw = raw.get("hashtags") or [] - hashtag_names = [h.get("name", "") for h in hashtags_raw if isinstance(h, dict)] - duration = (raw.get("videoMeta") or {}).get("duration") + video_id = str(raw.get("aweme_id", "")) + text = raw.get("desc", "") + stats = raw.get("statistics") or {} + play_count = stats.get("play_count") or 0 + digg_count = stats.get("digg_count") or 0 + comment_count = stats.get("comment_count") or 0 + share_count = stats.get("share_count") or 0 + author = raw.get("author") or {} + author_name = author.get("unique_id", "") + share_url = raw.get("share_url", "") + text_extra = raw.get("text_extra") or [] + hashtag_names = [t.get("hashtag_name", "") for t in text_extra + if isinstance(t, dict) and t.get("hashtag_name")] + duration = (raw.get("video") or {}).get("duration") date_str = _parse_date(raw) # Compute relevance with hashtag boost relevance = _compute_relevance(core_topic, text, hashtag_names) + # Build URL: prefer share_url, fallback to constructed URL + url = share_url.split("?")[0] if share_url else "" + if not url and author_name and video_id: + url = f"https://www.tiktok.com/@{author_name}/video/{video_id}" + items.append({ "video_id": video_id, "text": text, - "url": web_url or f"https://www.tiktok.com/@{author_name}/video/{video_id}", + "url": url, "author_name": author_name, "date": date_str, "engagement": { @@ -268,24 +305,24 @@ def fetch_captions( token: str, depth: str = "default", ) -> Dict[str, str]: - """Fetch captions for top N TikTok videos. + """Fetch transcripts for top N TikTok videos via ScrapeCreators. Strategy: - 1. Primary: Use the 'text' field (video description) — always free - 2. For top N, re-run actor with shouldDownloadSubtitles for spoken-word + 1. Use the 'text' field (video description) as baseline caption + 2. For top N, call /video/transcript for spoken-word captions Args: video_items: Items from search_tiktok() - token: Apify API token + token: ScrapeCreators API key depth: Depth level for caption limit Returns: - Dict mapping video_id → caption text (truncated to 500 words) + Dict mapping video_id -> caption text (truncated to 500 words) """ config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) max_captions = config["max_captions"] - if not video_items or not token: + if not video_items or not token or not _requests: return {} top_items = video_items[:max_captions] @@ -303,35 +340,33 @@ def fetch_captions( text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...' captions[vid] = text - # Second pass: try to get spoken-word subtitles for top videos - try: - urls = [item["url"] for item in top_items if item.get("url")] - if urls: - client = apify_client_wrapper.get_apify_client(token) - run_input = { - "postURLs": urls, - "shouldDownloadSubtitles": True, - "shouldDownloadVideos": False, - "shouldDownloadCovers": False, - } - subtitle_items = apify_client_wrapper.run_actor_sync( - client, ACTOR_ID, run_input, - timeout_secs=60, - max_items=max_captions, + # Second pass: try to get spoken-word transcripts (1 credit each) + for item in top_items: + vid = item["video_id"] + url = item.get("url", "") + if not url: + continue + try: + resp = _requests.get( + f"{SCRAPECREATORS_BASE}/video/transcript", + params={"url": url}, + headers=_sc_headers(token), + timeout=15, ) - for raw in subtitle_items: - vid = str(raw.get("id", "")) - # Check for subtitle text in the response - subtitle_text = raw.get("subtitleText") or raw.get("subtitles") or "" - if isinstance(subtitle_text, list): - subtitle_text = " ".join(str(s) for s in subtitle_text) - if subtitle_text and vid: - words = subtitle_text.split() - if len(words) > CAPTION_MAX_WORDS: - subtitle_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...' - captions[vid] = subtitle_text # Override text with spoken-word - except Exception as e: - _log(f"Subtitle enrichment failed (using text captions): {e}") + if resp.status_code == 200: + data = resp.json() + transcript = data.get("transcript") + if transcript: + if isinstance(transcript, list): + transcript = " ".join(str(s) for s in transcript) + transcript = _clean_webvtt(transcript) + if transcript: + words = transcript.split() + if len(words) > CAPTION_MAX_WORDS: + transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...' + captions[vid] = transcript + except Exception as e: + _log(f"Transcript fetch failed for {vid}: {e}") got = sum(1 for v in captions.values() if v) _log(f"Got captions for {got}/{len(top_items)} videos") @@ -352,7 +387,7 @@ def search_and_enrich( from_date: Start date (YYYY-MM-DD) to_date: End date (YYYY-MM-DD) depth: 'quick', 'default', or 'deep' - token: Apify API token + token: ScrapeCreators API key Returns: Dict with 'items' list. Each item has a 'caption_snippet' field. diff --git a/tests/test_tiktok.py b/tests/test_tiktok.py index 9dbabbb..3307dcc 100644 --- a/tests/test_tiktok.py +++ b/tests/test_tiktok.py @@ -58,14 +58,10 @@ class TestExtractCoreSubject(unittest.TestCase): class TestParseDate(unittest.TestCase): - """Test date parsing from Apify items.""" - - def test_iso_date(self): - item = {"createTimeISO": "2026-02-28T17:44:35.000Z"} - self.assertEqual(tiktok._parse_date(item), "2026-02-28") + """Test date parsing from ScrapeCreators items.""" def test_unix_timestamp(self): - item = {"createTime": 1756403075} + item = {"create_time": 1756403075} result = tiktok._parse_date(item) self.assertIsNotNone(result) self.assertRegex(result, r"\d{4}-\d{2}-\d{2}") @@ -75,6 +71,19 @@ class TestParseDate(unittest.TestCase): self.assertIsNone(tiktok._parse_date(item)) +class TestCleanWebVTT(unittest.TestCase): + """Test WebVTT transcript cleaning.""" + + def test_strips_timestamps(self): + raw = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello world\n\n00:00:02.000 --> 00:00:04.000\nGoodbye" + result = tiktok._clean_webvtt(raw) + self.assertEqual(result, "Hello world Goodbye") + + def test_empty_input(self): + self.assertEqual(tiktok._clean_webvtt(""), "") + self.assertEqual(tiktok._clean_webvtt(None), "") + + class TestNormalizeTikTokItems(unittest.TestCase): """Test TikTok normalization."""