diff --git a/SKILL.md b/SKILL.md index 3e52fa0..6b73ab3 100644 --- a/SKILL.md +++ b/SKILL.md @@ -240,7 +240,7 @@ The script will automatically: **Read the ENTIRE output.** It contains EIGHT data sections in this order: Reddit items, X items, YouTube items, TikTok items, Instagram Reels items, Hacker News items, Polymarket items, and WebSearch items. If you miss sections, you will produce incomplete stats. -**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block. +**YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, **transcript highlights** (pre-extracted quotable excerpts from the video), and an optional full transcript in a collapsible section. **Quote the highlights directly in your synthesis** - they are the YouTube equivalent of Reddit top comments. Attribute quotes to the channel name. Count them and include them in your synthesis and stats block. **TikTok items in the output look like:** `**{TK_id}** (score:N) @{creator} [N views, N likes]` followed by a caption, URL, hashtags, and optional caption snippet. Count them and include them in your synthesis and stats block. @@ -303,9 +303,10 @@ The Judge Agent must: 3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal) 4. Weight WebSearch sources LOWER (no engagement data) 5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments. -6. Identify patterns that appear across ALL sources (strongest signals) -7. Note any contradictions between sources -8. Extract the top 3-5 actionable insights +6. **For YouTube: Quote transcript highlights directly in your synthesis.** These are pre-extracted key moments from the video - treat them like Reddit top comments. Attribute to the channel name and include the actual quote. YouTube's value is in what creators SAY, not just their view counts. +7. Identify patterns that appear across ALL sources (strongest signals) +8. Note any contradictions between sources +9. Extract the top 3-5 actionable insights 7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research. diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py index bce1778..a15f202 100644 --- a/scripts/lib/normalize.py +++ b/scripts/lib/normalize.py @@ -193,6 +193,7 @@ def normalize_youtube_items( date_confidence="high", engagement=engagement, transcript_snippet=item.get("transcript_snippet", ""), + transcript_highlights=item.get("transcript_highlights", []), relevance=item.get("relevance", 0.7), why_relevant=item.get("why_relevant", ""), )) diff --git a/scripts/lib/render.py b/scripts/lib/render.py index 19bd7fc..3e5ea43 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -257,8 +257,15 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}") lines.append(f" {item.title}") lines.append(f" {item.url}") + if item.transcript_highlights: + lines.append(" Highlights:") + for hl in item.transcript_highlights[:5]: + lines.append(f' - "{hl}"') if item.transcript_snippet: - lines.append(f" Transcript: {item.transcript_snippet}") + word_count = len(item.transcript_snippet.split()) + lines.append(f"
Full transcript ({word_count} words)") + lines.append(f" {item.transcript_snippet}") + lines.append("
") lines.append(f" *{item.why_relevant}*") lines.append("") diff --git a/scripts/lib/schema.py b/scripts/lib/schema.py index 8805ceb..28f528f 100644 --- a/scripts/lib/schema.py +++ b/scripts/lib/schema.py @@ -210,6 +210,7 @@ class YouTubeItem: date_confidence: str = "high" # YouTube dates are always reliable engagement: Optional[Engagement] = None transcript_snippet: str = "" + transcript_highlights: List[str] = field(default_factory=list) relevance: float = 0.7 why_relevant: str = "" subs: SubScores = field(default_factory=SubScores) @@ -226,6 +227,7 @@ class YouTubeItem: 'date_confidence': self.date_confidence, 'engagement': self.engagement.to_dict() if self.engagement else None, 'transcript_snippet': self.transcript_snippet, + 'transcript_highlights': self.transcript_highlights, 'relevance': self.relevance, 'why_relevant': self.why_relevant, 'subs': self.subs.to_dict(), @@ -655,6 +657,7 @@ class Report: date_confidence=y.get('date_confidence', 'high'), engagement=eng, transcript_snippet=y.get('transcript_snippet', ''), + transcript_highlights=y.get('transcript_highlights', []), relevance=y.get('relevance', 0.7), why_relevant=y.get('why_relevant', ''), subs=subs, diff --git a/scripts/lib/youtube_yt.py b/scripts/lib/youtube_yt.py index b7ac513..4a07463 100644 --- a/scripts/lib/youtube_yt.py +++ b/scripts/lib/youtube_yt.py @@ -38,6 +38,52 @@ TRANSCRIPT_MAX_WORDS = 5000 from .relevance import token_overlap_relevance as _compute_relevance +def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> List[str]: + """Extract quotable highlights from a YouTube transcript. + + Similar to reddit_enrich.extract_comment_insights() but for + continuous speech-to-text rather than threaded comments. + """ + if not transcript: + return [] + + sentences = re.split(r'(?<=[.!?])\s+', transcript) + + filler = [ + r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)", + r"(subscribe|like and comment|hit the bell|check out the link|down below)", + r"^(so |and |but |okay |alright |um |uh )", + r"(thanks for watching|see you (next|in the)|bye)", + ] + + topic_words = [w.lower() for w in topic.lower().split() if len(w) > 2] + + candidates = [] + for sent in sentences: + sent = sent.strip() + words = sent.split() + if len(words) < 8 or len(words) > 50: + continue + if any(re.search(p, sent, re.IGNORECASE) for p in filler): + continue + + score = 0 + if re.search(r'\d', sent): + score += 2 + if re.search(r'[A-Z][a-z]+', sent): + score += 1 + if '?' in sent: + score += 1 + sent_lower = sent.lower() + if any(w in sent_lower for w in topic_words): + score += 2 + + candidates.append((score, sent)) + + candidates.sort(key=lambda x: -x[0]) + return [sent for _, sent in candidates[:limit]] + + def _log(msg: str): """Log to stderr.""" sys.stderr.write(f"[YouTube] {msg}\n") @@ -345,11 +391,15 @@ def search_and_transcribe( top_ids = [item["video_id"] for item in items[:transcript_limit]] transcripts = fetch_transcripts_parallel(top_ids) - # Step 3: Attach transcripts to items + # Step 3: Attach transcripts and extract highlights + core_topic = _extract_core_subject(topic) for item in items: vid = item["video_id"] transcript = transcripts.get(vid) item["transcript_snippet"] = transcript or "" + item["transcript_highlights"] = extract_transcript_highlights( + transcript or "", core_topic, + ) return {"items": items} diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index bb6248b..4fd4705 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -45,5 +45,36 @@ class TestYtDlpFlags(unittest.TestCase): self.assertIn("--no-cookies-from-browser", cmd) +class TestExtractTranscriptHighlights(unittest.TestCase): + def test_extracts_specific_sentences(self): + transcript = ( + "Hey guys welcome back to the channel. " + "In today's video we're looking at something special. " + "The Lego Bugatti Chiron took 13,438 hours to build with over 1 million pieces. " + "Don't forget to subscribe and hit the bell. " + "The tolerance on each brick is 0.002 millimeters which is insane for injection molding. " + "So yeah that's pretty cool. " + "Thanks for watching see you next time." + ) + highlights = youtube_yt.extract_transcript_highlights(transcript, "Lego") + self.assertTrue(len(highlights) > 0) + # Should pick the sentences with numbers and topic relevance, not filler + joined = " ".join(highlights) + self.assertIn("13,438", joined) + self.assertNotIn("subscribe", joined) + self.assertNotIn("welcome back", joined) + + def test_empty_transcript(self): + self.assertEqual(youtube_yt.extract_transcript_highlights("", "test"), []) + + def test_respects_limit(self): + sentences = ". ".join( + f"The model {i} has {i * 100} parameters and runs at {i * 10} tokens per second" + for i in range(20) + ) + "." + highlights = youtube_yt.extract_transcript_highlights(sentences, "model", limit=3) + self.assertEqual(len(highlights), 3) + + if __name__ == "__main__": unittest.main()