feat: surface YouTube + TikTok top comments alongside Reddit (#260)

* feat(normalize): pass YouTube top_comments through with Reddit-compatible shape _normalize_youtube silently dropped top_comments after enrich_with_comments populated them, so the downstream signals/render/entity layers never saw YouTube comments. Map likes->score and text->excerpt so the existing Reddit-compatible readers Just Work. Shared _remap_comments helper will be reused for TikTok in a later commit. * feat(tiktok): fetch top comments via ScrapeCreators when opted in Mirrors the youtube_comments pattern: new env.is_tiktok_comments_available gate (requires SCRAPECREATORS_API_KEY + tiktok_comments in INCLUDE_SOURCES), tiktok.enrich_with_comments ranks posts and fetches via GET /v1/tiktok/video/comments. Vote field is digg_count; text and user.nickname come across verbatim. Pipeline calls the enricher right after TikTok search when the gate is open. Comment-fetch errors never crash the pipeline — the enricher returns an empty list on 4xx/5xx. * feat(normalize): pass TikTok top_comments through with digg_count->score mapping Instagram uses the same shortform normalizer and has no comment fetcher today, so the key is harmlessly absent there — no Instagram regression. * feat(signals): add YouTube + TikTok top-comment score to engagement formula Mirrors Reddit's 10% top-comment slot. Without top_comments present, the formula reduces to views-dominant weighting; with a high-signal comment, the item gets a meaningful bump (log1p(10k) ~ 9.2, weighted 0.10 = ~0.92 on the engagement score). Updated the existing dominant-weight and missing-fields tests to the new weights (0.45/0.32/0.13 for YT, 0.45/0.27/0.18 for TT). Views still dominate. * feat(render): source-aware thresholds and vote labels for top comments 10 upvotes on Reddit signals community interest; 10 likes on a viral TikTok is noise. Introduce per-source minimums (reddit 10, youtube 50, tiktok 500) and native vote labels ('upvotes' for Reddit, 'likes' for YT/TT). First-pass numbers — tune after live observation. * docs: generalize top-comment quoting to YouTube + TikTok, add tiktok_comments opt-in Synthesis instructions previously called out Reddit top comments only. Now cover Reddit/YouTube/TikTok uniformly with source-appropriate vote labels (upvotes vs likes), and explicitly frame YT transcript highlights and comments as complementary signals. README and setup-wizard copy document the new tiktok_comments INCLUDE_SOURCES token. --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
2026-04-15 08:26:06 -04:00
parent 242e38ef56
commit 082efe03e3
12 changed files with 678 additions and 29 deletions
@@ -441,6 +441,18 @@ def is_youtube_comments_available(config: dict[str, Any]) -> bool:
    return 'youtube_comments' in include


+def is_tiktok_comments_available(config: dict[str, Any]) -> bool:
+    """Check if TikTok comment enrichment is available.
+
+    Requires SCRAPECREATORS_API_KEY AND tiktok_comments in INCLUDE_SOURCES.
+    Mirrors the youtube_comments opt-in pattern.
+    """
+    if not config.get('SCRAPECREATORS_API_KEY'):
+        return False
+    include = _parse_include_sources(config)
+    return 'tiktok_comments' in include
+
+
 def is_youtube_sc_available(config: dict[str, Any]) -> bool:
    """Check if ScrapeCreators YouTube search fallback is available.

@@ -69,6 +69,47 @@ def normalize_source_items(
    return filtered


+def _remap_comments(
+    raw: list[Any],
+    score_keys: tuple[str, ...],
+    excerpt_keys: tuple[str, ...],
+) -> list[dict[str, Any]]:
+    """Normalize comments from any source into the shared Reddit-compatible shape.
+
+    Downstream code (signals._top_comment_score, render._top_comments_list,
+    entity_extract, rerank) all expect `score` and `excerpt`. This helper maps
+    per-source field names (YT: likes/text, TikTok: digg_count/text) onto that
+    shape while preserving author/date/url passthrough.
+    """
+    out: list[dict[str, Any]] = []
+    for raw_c in raw:
+        if not isinstance(raw_c, dict):
+            continue
+        score = _first_present(raw_c, score_keys, default=0)
+        excerpt = _first_present(raw_c, excerpt_keys, default="")
+        try:
+            score_int = int(score or 0)
+        except (TypeError, ValueError):
+            score_int = 0
+        entry: dict[str, Any] = {
+            "score": score_int,
+            "excerpt": str(excerpt or "")[:400],
+            "author": str(raw_c.get("author") or ""),
+            "date": str(raw_c.get("date") or ""),
+        }
+        if raw_c.get("url"):
+            entry["url"] = str(raw_c["url"])
+        out.append(entry)
+    return out
+
+
+def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> Any:
+    for key in keys:
+        if key in d and d[key] not in (None, ""):
+            return d[key]
+    return default
+
+
 def _domain_from_url(url: str) -> str | None:
    if not url:
        return None
@@ -200,6 +241,11 @@ def _normalize_youtube(
    metadata: dict[str, Any] = {}
    if highlights:
        metadata["transcript_highlights"] = highlights
+    metadata["top_comments"] = _remap_comments(
+        item.get("top_comments") or [],
+        score_keys=("score", "likes"),
+        excerpt_keys=("excerpt", "text"),
+    )
    return _source_item(
        item_id=str(item.get("video_id") or item.get("id") or f"YT{index + 1}"),
        source=source,
@@ -242,7 +288,16 @@ def _normalize_shortform_video(
        relevance_hint=item.get("relevance", 0.5),
        why_relevant=str(item.get("why_relevant") or ""),
        snippet=caption,
-        metadata={"hashtags": item.get("hashtags") or []},
+        metadata={
+            "hashtags": item.get("hashtags") or [],
+            "top_comments": _remap_comments(
+                item.get("top_comments") or [],
+                # TikTok uses digg_count as the vote field; Instagram has no
+                # comment fetcher today so the key is harmlessly absent.
+                score_keys=("score", "digg_count", "likes"),
+                excerpt_keys=("excerpt", "text"),
+            ),
+        },
    )


@@ -887,7 +887,11 @@ def _retrieve_stream(
            hashtags=tiktok_hashtags,
            creators=tiktok_creators,
        )
-        return tiktok.parse_tiktok_response(result), {}
+        items = tiktok.parse_tiktok_response(result)
+        if items and env.is_tiktok_comments_available(config):
+            sc_token = config.get("SCRAPECREATORS_API_KEY", "")
+            tiktok.enrich_with_comments(items, token=sc_token)
+        return items, {}
    if source == "instagram":
        # Use raw_topic so expand_instagram_queries() generates diverse variants
        # from the original user topic, not the planner's narrowed search_query.
@@ -152,13 +152,14 @@ def render_full(report: schema.Report) -> str:
                lines.append(f"  *{item.container}*")
            if item.snippet:
                lines.append(f"  {item.snippet[:500]}")
-            # Top comments for Reddit
+            # Top comments for Reddit, YouTube, TikTok, HackerNews.
            top_comments = item.metadata.get("top_comments", [])
            if top_comments and isinstance(top_comments[0], dict):
+                vote_label = _vote_label_for(item.source)
                for tc in top_comments[:3]:
                    excerpt = tc.get("excerpt", tc.get("text", ""))[:200]
                    tc_score = tc.get("score", "")
-                    lines.append(f"  Top comment ({tc_score} upvotes): {excerpt}")
+                    lines.append(f"  Top comment ({tc_score} {vote_label}): {excerpt}")
            # Comment insights for Reddit
            insights = item.metadata.get("comment_insights", [])
            if insights:
@@ -276,7 +277,8 @@ def _render_candidate(candidate: schema.Candidate, prefix: str) -> list[str]:
    for tc in _top_comments_list(primary):
        excerpt = tc.get("excerpt") or tc.get("text") or ""
        score = tc.get("score", "")
-        lines.append(f"   - Comment ({score} upvotes): {_truncate(excerpt.strip(), 240)}")
+        vote_label = _vote_label_for(primary.source) if primary else "upvotes"
+        lines.append(f"   - Comment ({score} {vote_label}): {_truncate(excerpt.strip(), 240)}")
    insight = _comment_insight(primary)
    if insight:
        lines.append(f"   - Insight: {_truncate(insight, 220)}")
@@ -582,13 +584,42 @@ def _format_explanation(candidate: schema.Candidate) -> str | None:
    return candidate.explanation


-def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int = 10) -> list[dict]:
-    """Return up to `limit` top comments with score >= min_score."""
+# Per-source minimum vote counts for showing a top comment in compact emit.
+# Reddit upvotes, YouTube likes, and TikTok likes are not comparable units —
+# 10 upvotes on Reddit signals genuine community interest, 10 likes on a
+# viral TikTok is noise. First-pass values; tune after live observation.
+_TOP_COMMENT_MIN_SCORE: dict[str, int] = {
+    "reddit": 10,
+    "youtube": 50,
+    "tiktok": 500,
+    "hackernews": 5,
+}
+_TOP_COMMENT_VOTE_LABEL: dict[str, str] = {
+    "reddit": "upvotes",
+    "hackernews": "points",
+    "youtube": "likes",
+    "tiktok": "likes",
+}
+
+
+def _vote_label_for(source: str) -> str:
+    return _TOP_COMMENT_VOTE_LABEL.get(source, "votes")
+
+
+def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None) -> list[dict]:
+    """Return up to `limit` top comments with score at or above the source's minimum.
+
+    If `min_score` is passed explicitly it overrides the per-source default;
+    otherwise the source-keyed map is consulted, with an effective default of 0
+    (always show) for unknown sources so new sources don't get silently hidden.
+    """
    if not item:
        return []
    comments = item.metadata.get("top_comments") or []
    if not comments or not isinstance(comments[0], dict):
        return []
+    if min_score is None:
+        min_score = _TOP_COMMENT_MIN_SCORE.get(item.source, 0)
    return [c for c in comments if (c.get("score") or 0) >= min_score][:limit]


@@ -82,12 +82,11 @@ def _top_comment_score(item: schema.SourceItem) -> float:


 # Per-source engagement weights: list of (field_name, weight) tuples.
-# Reddit uses a custom function because upvote_ratio and top_comment_score
-# are not simple log1p fields.
+# Reddit, YouTube, and TikTok use custom functions because they include
+# a dedicated 10% top-comment-score slot (see _reddit_engagement,
+# _youtube_engagement, _tiktok_engagement).
 ENGAGEMENT_WEIGHTS: dict[str, list[tuple[str, float]]] = {
    "x":            [("likes", 0.55), ("reposts", 0.25), ("replies", 0.15), ("quotes", 0.05)],
-    "youtube":      [("views", 0.50), ("likes", 0.35), ("comments", 0.15)],
-    "tiktok":       [("views", 0.50), ("likes", 0.30), ("comments", 0.20)],
    "instagram":    [("views", 0.50), ("likes", 0.30), ("comments", 0.20)],
    "hackernews":   [("points", 0.55), ("comments", 0.45)],
    "bluesky":      [("likes", 0.40), ("reposts", 0.30), ("replies", 0.20), ("quotes", 0.10)],
@@ -113,6 +112,29 @@ def _reddit_engagement(item: schema.SourceItem) -> float | None:
    return (0.50 * score) + (0.35 * comments) + (0.05 * (ratio * 10.0)) + (0.10 * top_comment)


+def _youtube_engagement(item: schema.SourceItem) -> float | None:
+    views = log1p_safe(item.engagement.get("views"))
+    likes = log1p_safe(item.engagement.get("likes"))
+    comments = log1p_safe(item.engagement.get("comments"))
+    top_comment = _top_comment_score(item)
+    if not any([views, likes, comments, top_comment]):
+        return None
+    # Mirrors Reddit: carve out 10% for top-comment signal, keep view-weight
+    # dominant. Without comments, the pre-change weights (0.50/0.35/0.15)
+    # still govern relative ordering.
+    return (0.45 * views) + (0.32 * likes) + (0.13 * comments) + (0.10 * top_comment)
+
+
+def _tiktok_engagement(item: schema.SourceItem) -> float | None:
+    views = log1p_safe(item.engagement.get("views"))
+    likes = log1p_safe(item.engagement.get("likes"))
+    comments = log1p_safe(item.engagement.get("comments"))
+    top_comment = _top_comment_score(item)
+    if not any([views, likes, comments, top_comment]):
+        return None
+    return (0.45 * views) + (0.27 * likes) + (0.18 * comments) + (0.10 * top_comment)
+
+
 def _generic_engagement(item: schema.SourceItem) -> float | None:
    if not item.engagement:
        return None
@@ -125,6 +147,10 @@ def _generic_engagement(item: schema.SourceItem) -> float | None:
 def engagement_raw(item: schema.SourceItem) -> float | None:
    if item.source == "reddit":
        return _reddit_engagement(item)
+    if item.source == "youtube":
+        return _youtube_engagement(item)
+    if item.source == "tiktok":
+        return _tiktok_engagement(item)
    weights = ENGAGEMENT_WEIGHTS.get(item.source)
    if weights:
        return _weighted_engagement(item, weights)
@@ -539,3 +539,137 @@ def parse_tiktok_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
        List of item dicts ready for normalization.
    """
    return response.get("items", [])
+
+
+def _tiktok_total_engagement(item: Dict[str, Any]) -> int:
+    """Total engagement for ranking which posts deserve comment enrichment."""
+    eng = item.get("engagement", {})
+    return (eng.get("views", 0) or 0) + (eng.get("likes", 0) or 0) + (eng.get("comments", 0) or 0)
+
+
+def enrich_with_comments(
+    items: List[Dict[str, Any]],
+    token: str,
+    max_posts: int = 3,
+    max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+    """Enrich top TikTok posts with comment data from ScrapeCreators.
+
+    For the top N posts by engagement, fetches comments via the SC API
+    and attaches them as a ``top_comments`` field on each item. Mirrors
+    youtube_yt.enrich_with_comments.
+
+    Args:
+        items: TikTok items from search_tiktok()
+        token: ScrapeCreators API key
+        max_posts: How many posts to enrich with comments
+        max_comments: Max comments to keep per post
+
+    Returns:
+        Items list (mutated in place) with top_comments added to enriched items.
+    """
+    if not items or not token or max_posts <= 0:
+        return items
+
+    ranked = sorted(items, key=_tiktok_total_engagement, reverse=True)
+    top_items = ranked[:max_posts]
+    _log(f"Enriching comments for {len(top_items)} TikTok posts")
+
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    def _enrich_one(item: dict) -> bool:
+        post_url = item.get("url", "")
+        if not post_url:
+            return False
+        try:
+            comments = _fetch_post_comments(post_url, token, max_comments)
+            if comments:
+                item["top_comments"] = comments
+                return True
+        except Exception as exc:
+            _log(f"Comment enrichment failed for {post_url}: {exc}")
+        return False
+
+    enriched_count = 0
+    with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
+        futures = {executor.submit(_enrich_one, item): item for item in top_items}
+        for future in as_completed(futures):
+            if future.result():
+                enriched_count += 1
+
+    _log(f"Enriched {enriched_count}/{len(top_items)} posts with comments")
+    return items
+
+
+def _fetch_post_comments(
+    post_url: str,
+    token: str,
+    max_comments: int = 5,
+) -> List[Dict[str, Any]]:
+    """Fetch comments for a single TikTok post via ScrapeCreators.
+
+    SC endpoint: GET /v1/tiktok/video/comments?url=<video_url>
+    Response shape: { comments: [{text, user.nickname, digg_count, create_time, ...}], cursor, total }
+
+    Args:
+        post_url: Canonical TikTok post URL (share_url form works)
+        token: ScrapeCreators API key
+        max_comments: Maximum comments to return
+
+    Returns:
+        List of comment dicts with author, text, digg_count (likes), date.
+        Empty list on any error — comment failures never crash the pipeline.
+    """
+    if not _requests:
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"url": post_url, "trim": "true"})
+            url = f"{SCRAPECREATORS_BASE}/video/comments?{params}"
+            headers = http.scrapecreators_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as exc:
+            _log(f"Comment fetch error (urllib) for {post_url}: {exc}")
+            return []
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/video/comments",
+                params={"url": post_url, "trim": "true"},
+                headers=http.scrapecreators_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as exc:
+            _log(f"Comment fetch error for {post_url}: {exc}")
+            return []
+
+    raw_comments = data.get("comments") or data.get("data") or []
+    # Sort by digg_count desc so normalize sees the highest-signal first.
+    raw_comments = sorted(
+        raw_comments,
+        key=lambda c: c.get("digg_count", 0) or 0,
+        reverse=True,
+    )
+    out: List[Dict[str, Any]] = []
+    for c in raw_comments[:max_comments]:
+        text = c.get("text") or ""
+        if not text:
+            continue
+        user = c.get("user") if isinstance(c.get("user"), dict) else {}
+        author = user.get("nickname") or user.get("unique_id") or ""
+        create_time = c.get("create_time")
+        date_str = ""
+        if create_time:
+            try:
+                date_str = dates.timestamp_to_date(int(create_time)) or ""
+            except (ValueError, TypeError):
+                date_str = ""
+        out.append({
+            "author": author,
+            "text": text[:400],
+            "digg_count": c.get("digg_count", 0) or 0,
+            "date": date_str,
+        })
+    return out