refactor(tiktok): replace Apify with ScrapeCreators API
Root cause of empty TikTok results: Apify required monthly subscription.
ScrapeCreators is PAYG with 100 free credits and no subscription.
Key fix: ScrapeCreators nests items under aweme_info wrapper
(search_item_list[].aweme_info.{fields}), which the previous
implementation missed, causing all fields to be empty.
Changes:
- Rewrite tiktok.py to use ScrapeCreators REST API
- Add aweme_info unwrapping for correct field extraction
- Add transcript fetching via /video/transcript endpoint
- Add SCRAPECREATORS_API_KEY to env.py config
- Update last30days.py to use env.get_tiktok_token()
- Delete apify_client_wrapper.py (no longer needed)
- Update tests for new date field format (create_time)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+14
-5
@@ -203,6 +203,7 @@ def get_config() -> Dict[str, Any]:
|
||||
('OPENAI_MODEL_PIN', None),
|
||||
('XAI_MODEL_POLICY', 'latest'),
|
||||
('XAI_MODEL_PIN', None),
|
||||
('SCRAPECREATORS_API_KEY', None),
|
||||
('APIFY_API_TOKEN', None),
|
||||
('AUTH_TOKEN', None),
|
||||
('CT0', None),
|
||||
@@ -407,13 +408,21 @@ def is_polymarket_available() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def is_apify_available(config: Dict[str, Any]) -> bool:
|
||||
"""Check if Apify token is configured for TikTok/social scraping.
|
||||
def is_tiktok_available(config: Dict[str, Any]) -> bool:
|
||||
"""Check if TikTok source is available (ScrapeCreators or legacy Apify).
|
||||
|
||||
Returns True if APIFY_API_TOKEN is set. One token covers
|
||||
TikTok, Facebook, Instagram (all Apify-backed sources).
|
||||
Returns True if SCRAPECREATORS_API_KEY or APIFY_API_TOKEN is set.
|
||||
"""
|
||||
return bool(config.get('APIFY_API_TOKEN'))
|
||||
return bool(config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN'))
|
||||
|
||||
|
||||
def get_tiktok_token(config: Dict[str, Any]) -> str:
|
||||
"""Get TikTok API token, preferring ScrapeCreators over legacy Apify."""
|
||||
return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or ''
|
||||
|
||||
|
||||
# Backward compat alias
|
||||
is_apify_available = is_tiktok_available
|
||||
|
||||
|
||||
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user