refactor(tiktok): replace Apify with ScrapeCreators API
Root cause of empty TikTok results: Apify required monthly subscription.
ScrapeCreators is PAYG with 100 free credits and no subscription.
Key fix: ScrapeCreators nests items under aweme_info wrapper
(search_item_list[].aweme_info.{fields}), which the previous
implementation missed, causing all fields to be empty.
Changes:
- Rewrite tiktok.py to use ScrapeCreators REST API
- Add aweme_info unwrapping for correct field extraction
- Add transcript fetching via /video/transcript endpoint
- Add SCRAPECREATORS_API_KEY to env.py config
- Update last30days.py to use env.get_tiktok_token()
- Delete apify_client_wrapper.py (no longer needed)
- Update tests for new date field format (create_time)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+15
-6
@@ -58,14 +58,10 @@ class TestExtractCoreSubject(unittest.TestCase):
|
||||
|
||||
|
||||
class TestParseDate(unittest.TestCase):
|
||||
"""Test date parsing from Apify items."""
|
||||
|
||||
def test_iso_date(self):
|
||||
item = {"createTimeISO": "2026-02-28T17:44:35.000Z"}
|
||||
self.assertEqual(tiktok._parse_date(item), "2026-02-28")
|
||||
"""Test date parsing from ScrapeCreators items."""
|
||||
|
||||
def test_unix_timestamp(self):
|
||||
item = {"createTime": 1756403075}
|
||||
item = {"create_time": 1756403075}
|
||||
result = tiktok._parse_date(item)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertRegex(result, r"\d{4}-\d{2}-\d{2}")
|
||||
@@ -75,6 +71,19 @@ class TestParseDate(unittest.TestCase):
|
||||
self.assertIsNone(tiktok._parse_date(item))
|
||||
|
||||
|
||||
class TestCleanWebVTT(unittest.TestCase):
|
||||
"""Test WebVTT transcript cleaning."""
|
||||
|
||||
def test_strips_timestamps(self):
|
||||
raw = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello world\n\n00:00:02.000 --> 00:00:04.000\nGoodbye"
|
||||
result = tiktok._clean_webvtt(raw)
|
||||
self.assertEqual(result, "Hello world Goodbye")
|
||||
|
||||
def test_empty_input(self):
|
||||
self.assertEqual(tiktok._clean_webvtt(""), "")
|
||||
self.assertEqual(tiktok._clean_webvtt(None), "")
|
||||
|
||||
|
||||
class TestNormalizeTikTokItems(unittest.TestCase):
|
||||
"""Test TikTok normalization."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user