feat(tiktok): add TikTok as 7th signal source via Apify

Add TikTok search, scoring, and rendering using the Apify platform
(clockworks/tiktok-scraper actor). Users bring their own APIFY_API_TOKEN
($5/month free credits, no CC required). The shared apify_client_wrapper
module is designed for reuse by future Facebook/Instagram sources.

- New modules: tiktok.py (search + caption extraction), apify_client_wrapper.py
- Schema: TikTokItem dataclass, shares field on Engagement, Report.tiktok
- Pipeline: normalize → filter → score → sort → dedupe → cross-link → render
- Scoring: 0.50*log1p(views) + 0.30*log1p(likes) + 0.20*log1p(comments)
- SKILL.md bumped to v2.7 with TikTok stats, citations, and security docs
- 26 unit tests covering relevance, normalize, score, dedupe, render, round-trip

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-03 05:48:04 -08:00
parent 5e5d586f7d
commit 1db0b6054a
14 changed files with 1619 additions and 36 deletions
+75
View File
@@ -22,6 +22,9 @@ class Engagement:
# YouTube fields
views: Optional[int] = None
# TikTok / Facebook fields
shares: Optional[int] = None
# Polymarket fields
volume: Optional[float] = None
liquidity: Optional[float] = None
@@ -44,6 +47,8 @@ class Engagement:
d['quotes'] = self.quotes
if self.views is not None:
d['views'] = self.views
if self.shares is not None:
d['shares'] = self.shares
if self.volume is not None:
d['volume'] = self.volume
if self.liquidity is not None:
@@ -231,6 +236,45 @@ class YouTubeItem:
return d
@dataclass
class TikTokItem:
"""Normalized TikTok item."""
id: str # video_id
text: str # caption/description
url: str # webVideoUrl
author_name: str # authorMeta.name
date: Optional[str] = None
date_confidence: str = "high" # Apify provides exact timestamps
engagement: Optional[Engagement] = None # views, likes, num_comments, shares
caption_snippet: str = "" # spoken-word caption (if available), else text
hashtags: List[str] = field(default_factory=list)
relevance: float = 0.7
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
d = {
'id': self.id,
'text': self.text,
'url': self.url,
'author_name': self.author_name,
'date': self.date,
'date_confidence': self.date_confidence,
'engagement': self.engagement.to_dict() if self.engagement else None,
'caption_snippet': self.caption_snippet,
'hashtags': self.hashtags,
'relevance': self.relevance,
'why_relevant': self.why_relevant,
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
class HackerNewsItem:
"""Normalized Hacker News item."""
@@ -329,6 +373,7 @@ class Report:
x: List[XItem] = field(default_factory=list)
web: List[WebSearchItem] = field(default_factory=list)
youtube: List[YouTubeItem] = field(default_factory=list)
tiktok: List[TikTokItem] = field(default_factory=list)
hackernews: List[HackerNewsItem] = field(default_factory=list)
polymarket: List[PolymarketItem] = field(default_factory=list)
best_practices: List[str] = field(default_factory=list)
@@ -339,6 +384,7 @@ class Report:
x_error: Optional[str] = None
web_error: Optional[str] = None
youtube_error: Optional[str] = None
tiktok_error: Optional[str] = None
hackernews_error: Optional[str] = None
polymarket_error: Optional[str] = None
# Handle resolution
@@ -362,6 +408,7 @@ class Report:
'x': [x.to_dict() for x in self.x],
'web': [w.to_dict() for w in self.web],
'youtube': [y.to_dict() for y in self.youtube],
'tiktok': [t.to_dict() for t in self.tiktok],
'hackernews': [h.to_dict() for h in self.hackernews],
'polymarket': [p.to_dict() for p in self.polymarket],
'best_practices': self.best_practices,
@@ -378,6 +425,8 @@ class Report:
d['web_error'] = self.web_error
if self.youtube_error:
d['youtube_error'] = self.youtube_error
if self.tiktok_error:
d['tiktok_error'] = self.tiktok_error
if self.hackernews_error:
d['hackernews_error'] = self.hackernews_error
if self.polymarket_error:
@@ -485,6 +534,30 @@ class Report:
cross_refs=y.get('cross_refs', []),
))
# Reconstruct TikTok items
tiktok_items = []
for t in data.get('tiktok', []):
eng = None
if t.get('engagement'):
eng = Engagement(**t['engagement'])
subs = SubScores(**t.get('subs', {})) if t.get('subs') else SubScores()
tiktok_items.append(TikTokItem(
id=t['id'],
text=t.get('text', ''),
url=t['url'],
author_name=t.get('author_name', ''),
date=t.get('date'),
date_confidence=t.get('date_confidence', 'high'),
engagement=eng,
caption_snippet=t.get('caption_snippet', ''),
hashtags=t.get('hashtags', []),
relevance=t.get('relevance', 0.7),
why_relevant=t.get('why_relevant', ''),
subs=subs,
score=t.get('score', 0),
cross_refs=t.get('cross_refs', []),
))
# Reconstruct HackerNews items
hn_items = []
for h in data.get('hackernews', []):
@@ -549,6 +622,7 @@ class Report:
x=x_items,
web=web_items,
youtube=youtube_items,
tiktok=tiktok_items,
hackernews=hn_items,
polymarket=pm_items,
best_practices=data.get('best_practices', []),
@@ -558,6 +632,7 @@ class Report:
x_error=data.get('x_error'),
web_error=data.get('web_error'),
youtube_error=data.get('youtube_error'),
tiktok_error=data.get('tiktok_error'),
hackernews_error=data.get('hackernews_error'),
polymarket_error=data.get('polymarket_error'),
resolved_x_handle=data.get('resolved_x_handle'),