082efe03e3
* feat(normalize): pass YouTube top_comments through with Reddit-compatible shape
_normalize_youtube silently dropped top_comments after enrich_with_comments
populated them, so the downstream signals/render/entity layers never saw
YouTube comments. Map likes->score and text->excerpt so the existing
Reddit-compatible readers Just Work.
Shared _remap_comments helper will be reused for TikTok in a later commit.
* feat(tiktok): fetch top comments via ScrapeCreators when opted in
Mirrors the youtube_comments pattern: new env.is_tiktok_comments_available
gate (requires SCRAPECREATORS_API_KEY + tiktok_comments in INCLUDE_SOURCES),
tiktok.enrich_with_comments ranks posts and fetches via
GET /v1/tiktok/video/comments. Vote field is digg_count; text and user.nickname
come across verbatim. Pipeline calls the enricher right after TikTok search
when the gate is open.
Comment-fetch errors never crash the pipeline — the enricher returns an
empty list on 4xx/5xx.
* feat(normalize): pass TikTok top_comments through with digg_count->score mapping
Instagram uses the same shortform normalizer and has no comment fetcher
today, so the key is harmlessly absent there — no Instagram regression.
* feat(signals): add YouTube + TikTok top-comment score to engagement formula
Mirrors Reddit's 10% top-comment slot. Without top_comments present, the
formula reduces to views-dominant weighting; with a high-signal comment,
the item gets a meaningful bump (log1p(10k) ~ 9.2, weighted 0.10 = ~0.92
on the engagement score).
Updated the existing dominant-weight and missing-fields tests to the new
weights (0.45/0.32/0.13 for YT, 0.45/0.27/0.18 for TT). Views still dominate.
* feat(render): source-aware thresholds and vote labels for top comments
10 upvotes on Reddit signals community interest; 10 likes on a viral
TikTok is noise. Introduce per-source minimums (reddit 10, youtube 50,
tiktok 500) and native vote labels ('upvotes' for Reddit, 'likes' for
YT/TT). First-pass numbers — tune after live observation.
* docs: generalize top-comment quoting to YouTube + TikTok, add tiktok_comments opt-in
Synthesis instructions previously called out Reddit top comments only.
Now cover Reddit/YouTube/TikTok uniformly with source-appropriate vote
labels (upvotes vs likes), and explicitly frame YT transcript highlights
and comments as complementary signals. README and setup-wizard copy
document the new tiktok_comments INCLUDE_SOURCES token.
---------
Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
217 lines
8.4 KiB
Python
217 lines
8.4 KiB
Python
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
|
|
|
from lib.tiktok import _parse_items
|
|
|
|
|
|
class TestTikTokAuthorTypeSafety(unittest.TestCase):
|
|
def _make_raw(self, **overrides):
|
|
base = {
|
|
"aweme_id": "1",
|
|
"desc": "test video",
|
|
"share_url": "https://www.tiktok.com/@u/video/1",
|
|
"author": {"unique_id": "testuser"},
|
|
"statistics": {"play_count": 100, "digg_count": 50, "comment_count": 10, "share_count": 5},
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
def test_author_as_dict(self):
|
|
items = _parse_items([self._make_raw()], "test")
|
|
self.assertEqual("testuser", items[0]["author_name"])
|
|
|
|
def test_author_as_string(self):
|
|
items = _parse_items([self._make_raw(author="stringuser")], "test")
|
|
self.assertEqual("stringuser", items[0]["author_name"])
|
|
|
|
def test_author_missing(self):
|
|
raw = self._make_raw()
|
|
del raw["author"]
|
|
items = _parse_items([raw], "test")
|
|
self.assertEqual("", items[0]["author_name"])
|
|
|
|
def test_author_none(self):
|
|
items = _parse_items([self._make_raw(author=None)], "test")
|
|
self.assertEqual("", items[0]["author_name"])
|
|
|
|
|
|
class TestTikTokStatsZeroPreserved(unittest.TestCase):
|
|
def test_zero_play_count(self):
|
|
raw = {
|
|
"aweme_id": "1",
|
|
"desc": "test",
|
|
"share_url": "https://www.tiktok.com/@u/video/1",
|
|
"author": {"unique_id": "u"},
|
|
"statistics": {"play_count": 0, "digg_count": 0, "comment_count": 0, "share_count": 0},
|
|
}
|
|
items = _parse_items([raw], "test")
|
|
self.assertEqual(0, items[0]["engagement"]["views"])
|
|
self.assertEqual(0, items[0]["engagement"]["likes"])
|
|
self.assertEqual(0, items[0]["engagement"]["comments"])
|
|
self.assertEqual(0, items[0]["engagement"]["shares"])
|
|
|
|
def test_stats_missing(self):
|
|
raw = {
|
|
"aweme_id": "1",
|
|
"desc": "test",
|
|
"share_url": "https://www.tiktok.com/@u/video/1",
|
|
"author": {"unique_id": "u"},
|
|
}
|
|
items = _parse_items([raw], "test")
|
|
self.assertEqual(0, items[0]["engagement"]["views"])
|
|
|
|
def test_stats_as_non_dict(self):
|
|
raw = {
|
|
"aweme_id": "1",
|
|
"desc": "test",
|
|
"share_url": "https://www.tiktok.com/@u/video/1",
|
|
"author": {"unique_id": "u"},
|
|
"statistics": "invalid",
|
|
}
|
|
items = _parse_items([raw], "test")
|
|
self.assertEqual(0, items[0]["engagement"]["views"])
|
|
|
|
|
|
class TestExpandTikTokQueries(unittest.TestCase):
|
|
"""Tests for expand_tiktok_queries() multi-query generation."""
|
|
|
|
def test_default_depth_returns_two_plus_queries(self):
|
|
from lib.tiktok import expand_tiktok_queries
|
|
queries = expand_tiktok_queries("Kanye West", "default")
|
|
self.assertGreaterEqual(len(queries), 2)
|
|
# Breaking_news intent should include reaction/edit variant
|
|
variant_found = any(
|
|
"reaction" in q.lower() or "edit" in q.lower() or "trend" in q.lower()
|
|
for q in queries
|
|
)
|
|
self.assertTrue(variant_found, f"Expected reaction/edit/trend variant: {queries}")
|
|
|
|
def test_product_intent_includes_review_variant(self):
|
|
from lib.tiktok import expand_tiktok_queries
|
|
# "best laptop for coding" triggers the product intent (best .* for pattern)
|
|
queries = expand_tiktok_queries("best laptop for coding", "deep")
|
|
variant_found = any(
|
|
"review" in q.lower() or "haul" in q.lower() or "unboxing" in q.lower()
|
|
for q in queries
|
|
)
|
|
self.assertTrue(variant_found, f"Expected review/haul/unboxing variant: {queries}")
|
|
|
|
def test_quick_depth_returns_one_query(self):
|
|
from lib.tiktok import expand_tiktok_queries
|
|
queries = expand_tiktok_queries("Kanye West", "quick")
|
|
self.assertEqual(len(queries), 1)
|
|
|
|
|
|
class TestTikTokCommentsGate(unittest.TestCase):
|
|
def test_gate_requires_key_and_token(self):
|
|
from lib import env
|
|
self.assertFalse(env.is_tiktok_comments_available({}))
|
|
self.assertFalse(env.is_tiktok_comments_available(
|
|
{"SCRAPECREATORS_API_KEY": "k"}
|
|
))
|
|
self.assertFalse(env.is_tiktok_comments_available(
|
|
{"INCLUDE_SOURCES": "tiktok_comments"}
|
|
))
|
|
self.assertTrue(env.is_tiktok_comments_available(
|
|
{"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "tiktok,tiktok_comments"}
|
|
))
|
|
|
|
def test_gate_case_matches_youtube_pattern(self):
|
|
from lib import env
|
|
# Matches the existing youtube_comments behaviour — plain substring match via _parse_include_sources.
|
|
self.assertTrue(env.is_tiktok_comments_available(
|
|
{"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "TIKTOK,TIKTOK_COMMENTS"}
|
|
))
|
|
|
|
|
|
class TestTikTokEnrichWithComments(unittest.TestCase):
|
|
def test_empty_items_returns_empty(self):
|
|
from lib import tiktok
|
|
self.assertEqual([], tiktok.enrich_with_comments([], token="k"))
|
|
|
|
def test_missing_token_is_noop(self):
|
|
from lib import tiktok
|
|
items = [{"video_id": "1", "url": "https://www.tiktok.com/@u/video/1", "engagement": {"views": 100}}]
|
|
result = tiktok.enrich_with_comments(items, token="")
|
|
self.assertNotIn("top_comments", result[0])
|
|
|
|
def test_fetch_post_comments_parses_sc_response(self):
|
|
from unittest.mock import patch
|
|
from lib import tiktok
|
|
|
|
fake_sc_response = {
|
|
"comments": [
|
|
{"text": "loved it", "user": {"nickname": "Alice"},
|
|
"digg_count": 420, "create_time": 1709251200},
|
|
{"text": "meh", "user": {"nickname": "Bob"},
|
|
"digg_count": 3, "create_time": 1709251300},
|
|
{"text": "", "user": {"nickname": "Skip"},
|
|
"digg_count": 999, "create_time": 1709251400},
|
|
],
|
|
"total": 3,
|
|
}
|
|
|
|
class FakeResp:
|
|
def raise_for_status(self):
|
|
pass
|
|
def json(self):
|
|
return fake_sc_response
|
|
|
|
with patch.object(tiktok, "_requests") as mock_req:
|
|
mock_req.get.return_value = FakeResp()
|
|
out = tiktok._fetch_post_comments(
|
|
"https://www.tiktok.com/@u/video/1",
|
|
token="k",
|
|
max_comments=5,
|
|
)
|
|
# Empty-text comment dropped; rest sorted desc by digg_count.
|
|
self.assertEqual(2, len(out))
|
|
self.assertEqual("loved it", out[0]["text"])
|
|
self.assertEqual(420, out[0]["digg_count"])
|
|
self.assertEqual("Alice", out[0]["author"])
|
|
self.assertEqual("2024-03-01", out[0]["date"])
|
|
self.assertEqual(3, out[1]["digg_count"])
|
|
|
|
def test_fetch_post_comments_swallows_http_error(self):
|
|
from unittest.mock import patch
|
|
from lib import tiktok
|
|
|
|
with patch.object(tiktok, "_requests") as mock_req:
|
|
mock_req.get.side_effect = Exception("429 rate limit")
|
|
out = tiktok._fetch_post_comments(
|
|
"https://www.tiktok.com/@u/video/1",
|
|
token="k",
|
|
max_comments=5,
|
|
)
|
|
self.assertEqual([], out)
|
|
|
|
def test_enrich_attaches_top_comments_to_top_ranked_items(self):
|
|
from unittest.mock import patch
|
|
from lib import tiktok
|
|
|
|
items = [
|
|
{"video_id": "low", "url": "https://www.tiktok.com/@u/video/low",
|
|
"engagement": {"views": 10, "likes": 1, "comments": 0}},
|
|
{"video_id": "high", "url": "https://www.tiktok.com/@u/video/high",
|
|
"engagement": {"views": 10000, "likes": 500, "comments": 30}},
|
|
{"video_id": "mid", "url": "https://www.tiktok.com/@u/video/mid",
|
|
"engagement": {"views": 1000, "likes": 50, "comments": 5}},
|
|
]
|
|
with patch.object(tiktok, "_fetch_post_comments") as mock_fetch:
|
|
mock_fetch.return_value = [
|
|
{"author": "A", "text": "fire", "digg_count": 100, "date": "2024-03-01"}
|
|
]
|
|
tiktok.enrich_with_comments(items, token="k", max_posts=2)
|
|
# High and mid get comments; low does not.
|
|
by_id = {i["video_id"]: i for i in items}
|
|
self.assertIn("top_comments", by_id["high"])
|
|
self.assertIn("top_comments", by_id["mid"])
|
|
self.assertNotIn("top_comments", by_id["low"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|