Files
last30days-skill/tests/test_tiktok.py
Matt Van Horn 082efe03e3 feat: surface YouTube + TikTok top comments alongside Reddit (#260)
* feat(normalize): pass YouTube top_comments through with Reddit-compatible shape

_normalize_youtube silently dropped top_comments after enrich_with_comments
populated them, so the downstream signals/render/entity layers never saw
YouTube comments. Map likes->score and text->excerpt so the existing
Reddit-compatible readers Just Work.

Shared _remap_comments helper will be reused for TikTok in a later commit.

* feat(tiktok): fetch top comments via ScrapeCreators when opted in

Mirrors the youtube_comments pattern: new env.is_tiktok_comments_available
gate (requires SCRAPECREATORS_API_KEY + tiktok_comments in INCLUDE_SOURCES),
tiktok.enrich_with_comments ranks posts and fetches via
GET /v1/tiktok/video/comments. Vote field is digg_count; text and user.nickname
come across verbatim. Pipeline calls the enricher right after TikTok search
when the gate is open.

Comment-fetch errors never crash the pipeline — the enricher returns an
empty list on 4xx/5xx.

* feat(normalize): pass TikTok top_comments through with digg_count->score mapping

Instagram uses the same shortform normalizer and has no comment fetcher
today, so the key is harmlessly absent there — no Instagram regression.

* feat(signals): add YouTube + TikTok top-comment score to engagement formula

Mirrors Reddit's 10% top-comment slot. Without top_comments present, the
formula reduces to views-dominant weighting; with a high-signal comment,
the item gets a meaningful bump (log1p(10k) ~ 9.2, weighted 0.10 = ~0.92
on the engagement score).

Updated the existing dominant-weight and missing-fields tests to the new
weights (0.45/0.32/0.13 for YT, 0.45/0.27/0.18 for TT). Views still dominate.

* feat(render): source-aware thresholds and vote labels for top comments

10 upvotes on Reddit signals community interest; 10 likes on a viral
TikTok is noise. Introduce per-source minimums (reddit 10, youtube 50,
tiktok 500) and native vote labels ('upvotes' for Reddit, 'likes' for
YT/TT). First-pass numbers — tune after live observation.

* docs: generalize top-comment quoting to YouTube + TikTok, add tiktok_comments opt-in

Synthesis instructions previously called out Reddit top comments only.
Now cover Reddit/YouTube/TikTok uniformly with source-appropriate vote
labels (upvotes vs likes), and explicitly frame YT transcript highlights
and comments as complementary signals. README and setup-wizard copy
document the new tiktok_comments INCLUDE_SOURCES token.

---------

Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
2026-04-15 08:26:06 -04:00

217 lines
8.4 KiB
Python

import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
from lib.tiktok import _parse_items
class TestTikTokAuthorTypeSafety(unittest.TestCase):
def _make_raw(self, **overrides):
base = {
"aweme_id": "1",
"desc": "test video",
"share_url": "https://www.tiktok.com/@u/video/1",
"author": {"unique_id": "testuser"},
"statistics": {"play_count": 100, "digg_count": 50, "comment_count": 10, "share_count": 5},
}
base.update(overrides)
return base
def test_author_as_dict(self):
items = _parse_items([self._make_raw()], "test")
self.assertEqual("testuser", items[0]["author_name"])
def test_author_as_string(self):
items = _parse_items([self._make_raw(author="stringuser")], "test")
self.assertEqual("stringuser", items[0]["author_name"])
def test_author_missing(self):
raw = self._make_raw()
del raw["author"]
items = _parse_items([raw], "test")
self.assertEqual("", items[0]["author_name"])
def test_author_none(self):
items = _parse_items([self._make_raw(author=None)], "test")
self.assertEqual("", items[0]["author_name"])
class TestTikTokStatsZeroPreserved(unittest.TestCase):
def test_zero_play_count(self):
raw = {
"aweme_id": "1",
"desc": "test",
"share_url": "https://www.tiktok.com/@u/video/1",
"author": {"unique_id": "u"},
"statistics": {"play_count": 0, "digg_count": 0, "comment_count": 0, "share_count": 0},
}
items = _parse_items([raw], "test")
self.assertEqual(0, items[0]["engagement"]["views"])
self.assertEqual(0, items[0]["engagement"]["likes"])
self.assertEqual(0, items[0]["engagement"]["comments"])
self.assertEqual(0, items[0]["engagement"]["shares"])
def test_stats_missing(self):
raw = {
"aweme_id": "1",
"desc": "test",
"share_url": "https://www.tiktok.com/@u/video/1",
"author": {"unique_id": "u"},
}
items = _parse_items([raw], "test")
self.assertEqual(0, items[0]["engagement"]["views"])
def test_stats_as_non_dict(self):
raw = {
"aweme_id": "1",
"desc": "test",
"share_url": "https://www.tiktok.com/@u/video/1",
"author": {"unique_id": "u"},
"statistics": "invalid",
}
items = _parse_items([raw], "test")
self.assertEqual(0, items[0]["engagement"]["views"])
class TestExpandTikTokQueries(unittest.TestCase):
"""Tests for expand_tiktok_queries() multi-query generation."""
def test_default_depth_returns_two_plus_queries(self):
from lib.tiktok import expand_tiktok_queries
queries = expand_tiktok_queries("Kanye West", "default")
self.assertGreaterEqual(len(queries), 2)
# Breaking_news intent should include reaction/edit variant
variant_found = any(
"reaction" in q.lower() or "edit" in q.lower() or "trend" in q.lower()
for q in queries
)
self.assertTrue(variant_found, f"Expected reaction/edit/trend variant: {queries}")
def test_product_intent_includes_review_variant(self):
from lib.tiktok import expand_tiktok_queries
# "best laptop for coding" triggers the product intent (best .* for pattern)
queries = expand_tiktok_queries("best laptop for coding", "deep")
variant_found = any(
"review" in q.lower() or "haul" in q.lower() or "unboxing" in q.lower()
for q in queries
)
self.assertTrue(variant_found, f"Expected review/haul/unboxing variant: {queries}")
def test_quick_depth_returns_one_query(self):
from lib.tiktok import expand_tiktok_queries
queries = expand_tiktok_queries("Kanye West", "quick")
self.assertEqual(len(queries), 1)
class TestTikTokCommentsGate(unittest.TestCase):
def test_gate_requires_key_and_token(self):
from lib import env
self.assertFalse(env.is_tiktok_comments_available({}))
self.assertFalse(env.is_tiktok_comments_available(
{"SCRAPECREATORS_API_KEY": "k"}
))
self.assertFalse(env.is_tiktok_comments_available(
{"INCLUDE_SOURCES": "tiktok_comments"}
))
self.assertTrue(env.is_tiktok_comments_available(
{"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "tiktok,tiktok_comments"}
))
def test_gate_case_matches_youtube_pattern(self):
from lib import env
# Matches the existing youtube_comments behaviour — plain substring match via _parse_include_sources.
self.assertTrue(env.is_tiktok_comments_available(
{"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "TIKTOK,TIKTOK_COMMENTS"}
))
class TestTikTokEnrichWithComments(unittest.TestCase):
def test_empty_items_returns_empty(self):
from lib import tiktok
self.assertEqual([], tiktok.enrich_with_comments([], token="k"))
def test_missing_token_is_noop(self):
from lib import tiktok
items = [{"video_id": "1", "url": "https://www.tiktok.com/@u/video/1", "engagement": {"views": 100}}]
result = tiktok.enrich_with_comments(items, token="")
self.assertNotIn("top_comments", result[0])
def test_fetch_post_comments_parses_sc_response(self):
from unittest.mock import patch
from lib import tiktok
fake_sc_response = {
"comments": [
{"text": "loved it", "user": {"nickname": "Alice"},
"digg_count": 420, "create_time": 1709251200},
{"text": "meh", "user": {"nickname": "Bob"},
"digg_count": 3, "create_time": 1709251300},
{"text": "", "user": {"nickname": "Skip"},
"digg_count": 999, "create_time": 1709251400},
],
"total": 3,
}
class FakeResp:
def raise_for_status(self):
pass
def json(self):
return fake_sc_response
with patch.object(tiktok, "_requests") as mock_req:
mock_req.get.return_value = FakeResp()
out = tiktok._fetch_post_comments(
"https://www.tiktok.com/@u/video/1",
token="k",
max_comments=5,
)
# Empty-text comment dropped; rest sorted desc by digg_count.
self.assertEqual(2, len(out))
self.assertEqual("loved it", out[0]["text"])
self.assertEqual(420, out[0]["digg_count"])
self.assertEqual("Alice", out[0]["author"])
self.assertEqual("2024-03-01", out[0]["date"])
self.assertEqual(3, out[1]["digg_count"])
def test_fetch_post_comments_swallows_http_error(self):
from unittest.mock import patch
from lib import tiktok
with patch.object(tiktok, "_requests") as mock_req:
mock_req.get.side_effect = Exception("429 rate limit")
out = tiktok._fetch_post_comments(
"https://www.tiktok.com/@u/video/1",
token="k",
max_comments=5,
)
self.assertEqual([], out)
def test_enrich_attaches_top_comments_to_top_ranked_items(self):
from unittest.mock import patch
from lib import tiktok
items = [
{"video_id": "low", "url": "https://www.tiktok.com/@u/video/low",
"engagement": {"views": 10, "likes": 1, "comments": 0}},
{"video_id": "high", "url": "https://www.tiktok.com/@u/video/high",
"engagement": {"views": 10000, "likes": 500, "comments": 30}},
{"video_id": "mid", "url": "https://www.tiktok.com/@u/video/mid",
"engagement": {"views": 1000, "likes": 50, "comments": 5}},
]
with patch.object(tiktok, "_fetch_post_comments") as mock_fetch:
mock_fetch.return_value = [
{"author": "A", "text": "fire", "digg_count": 100, "date": "2024-03-01"}
]
tiktok.enrich_with_comments(items, token="k", max_posts=2)
# High and mid get comments; low does not.
by_id = {i["video_id"]: i for i in items}
self.assertIn("top_comments", by_id["high"])
self.assertIn("top_comments", by_id["mid"])
self.assertNotIn("top_comments", by_id["low"])
if __name__ == "__main__":
unittest.main()