feat(tiktok): add TikTok as 7th signal source via Apify
Add TikTok search, scoring, and rendering using the Apify platform (clockworks/tiktok-scraper actor). Users bring their own APIFY_API_TOKEN ($5/month free credits, no CC required). The shared apify_client_wrapper module is designed for reuse by future Facebook/Instagram sources. - New modules: tiktok.py (search + caption extraction), apify_client_wrapper.py - Schema: TikTokItem dataclass, shares field on Engagement, Report.tiktok - Pipeline: normalize → filter → score → sort → dedupe → cross-link → render - Scoring: 0.50*log1p(views) + 0.30*log1p(likes) + 0.20*log1p(comments) - SKILL.md bumped to v2.7 with TikTok stats, citations, and security docs - 26 unit tests covering relevance, normalize, score, dedupe, render, round-trip Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -200,6 +200,53 @@ def normalize_youtube_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_tiktok_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.TikTokItem]:
|
||||
"""Normalize raw TikTok items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw TikTok items from Apify
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of TikTokItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement") or {}
|
||||
engagement = schema.Engagement(
|
||||
views=eng_raw.get("views"),
|
||||
likes=eng_raw.get("likes"),
|
||||
num_comments=eng_raw.get("comments"),
|
||||
shares=eng_raw.get("shares"),
|
||||
)
|
||||
|
||||
# TikTok dates are reliable (exact timestamps from Apify)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.TikTokItem(
|
||||
id=f"TK{i+1}",
|
||||
text=item.get("text", ""),
|
||||
url=item.get("url", ""),
|
||||
author_name=item.get("author_name", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
caption_snippet=item.get("caption_snippet", ""),
|
||||
hashtags=item.get("hashtags", []),
|
||||
relevance=item.get("relevance", 0.7),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_hackernews_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
|
||||
Reference in New Issue
Block a user