sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(quality): YouTube relevance scoring and cross-source linking

Browse Source 
YouTube videos now get real relevance scores based on token overlap
between the search query and video title (was hardcoded at 0.7).
Uses ratio overlap with stopword removal, floored at 0.1.

Cross-source linking annotates items that discuss the same story
across different platforms (e.g., Reddit + HN + X). Items get
bidirectional cross_refs displayed as [xref: R3, HN5] in compact
output so Claude can triangulate multi-platform coverage.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-25 10:46:58 -08:00
parent f60a4359a0
commit 0591f55f0e
8 changed files with 642 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/last30days.py
+5
View File
@@ -1113,6 +1113,11 @@ def main():
by_relevance = sorted(normalized_reddit, key=lambda item: item.relevance, reverse=True)
deduped_reddit = by_relevance[:3]
# Cross-source linking: annotate items that discuss the same story
dedupe.cross_source_link(
deduped_reddit, deduped_x, deduped_youtube, deduped_hn, deduped_web,
)
progress.end_processing()
# Create report
scripts/lib/dedupe.py
+57 -1
View File
@@ -36,7 +36,11 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
return intersection / union if union > 0 else 0.0
def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeItem, schema.HackerNewsItem]) -> str:
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem,
schema.HackerNewsItem, schema.WebSearchItem]
def get_item_text(item: AnyItem) -> str:
"""Get comparable text from an item."""
if isinstance(item, schema.RedditItem):
return item.title
@@ -44,10 +48,23 @@ def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeIte
return item.title
elif isinstance(item, schema.YouTubeItem):
return f"{item.title} {item.channel_name}"
elif isinstance(item, schema.WebSearchItem):
return item.title
else:
return item.text
def _get_cross_source_text(item: AnyItem) -> str:
"""Get text for cross-source comparison.
Same as get_item_text() but truncates X posts to 100 chars
to level the playing field against short Reddit/HN titles.
"""
if isinstance(item, schema.XItem):
return item.text[:100]
return get_item_text(item)
def find_duplicates(
items: List[Union[schema.RedditItem, schema.XItem]],
threshold: float = 0.7,
@@ -138,3 +155,42 @@ def dedupe_hackernews(
) -> List[schema.HackerNewsItem]:
"""Dedupe Hacker News items."""
return dedupe_items(items, threshold)
def cross_source_link(
*source_lists: List[AnyItem],
threshold: float = 0.5,
) -> None:
"""Annotate items with cross-source references.
Compares items across different source types using Jaccard similarity
on char trigrams. When similarity exceeds threshold, adds bidirectional
cross_refs with the related item's ID. Modifies items in-place.
Args:
*source_lists: Variable number of per-source item lists
threshold: Similarity threshold for cross-linking (default 0.5)
"""
all_items = []
for source_list in source_lists:
all_items.extend(source_list)
if len(all_items) <= 1:
return
# Pre-compute trigrams using cross-source text extraction
ngrams = [get_ngrams(_get_cross_source_text(item)) for item in all_items]
for i in range(len(all_items)):
for j in range(i + 1, len(all_items)):
# Skip same-source comparisons (handled by per-source dedupe)
if type(all_items[i]) is type(all_items[j]):
continue
similarity = jaccard_similarity(ngrams[i], ngrams[j])
if similarity >= threshold:
# Bidirectional cross-reference
if all_items[j].id not in all_items[i].cross_refs:
all_items[i].cross_refs.append(all_items[j].id)
if all_items[i].id not in all_items[j].cross_refs:
all_items[j].cross_refs.append(all_items[i].id)
scripts/lib/render.py
+13 -5
View File
@@ -11,6 +11,14 @@ from . import schema
OUTPUT_DIR = Path.home() / ".local" / "share" / "last30days" / "out"
def _xref_tag(item) -> str:
"""Return ' [xref: X1, HN3]' string if item has cross_refs, else ''."""
refs = getattr(item, 'cross_refs', None)
if refs:
return f" [xref: {', '.join(refs)}]"
return ""
def ensure_output_dir():
"""Ensure output directory exists. Supports env override and sandbox fallback."""
global OUTPUT_DIR
@@ -134,7 +142,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
date_str = f" ({item.date})" if item.date else " (date unknown)"
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
lines.append(f"**{item.id}** (score:{item.score}) r/{item.subreddit}{date_str}{conf_str}{eng_str}")
lines.append(f"**{item.id}** (score:{item.score}) r/{item.subreddit}{date_str}{conf_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.title}")
lines.append(f" {item.url}")
lines.append(f" *{item.why_relevant}*")
@@ -176,7 +184,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
date_str = f" ({item.date})" if item.date else " (date unknown)"
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{conf_str}{eng_str}")
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{conf_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.text[:200]}...")
lines.append(f" {item.url}")
lines.append(f" *{item.why_relevant}*")
@@ -205,7 +213,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
date_str = f" ({item.date})" if item.date else ""
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}")
lines.append(f"**{item.id}** (score:{item.score}) {item.channel_name}{date_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.title}")
lines.append(f" {item.url}")
if item.transcript_snippet:
@@ -239,7 +247,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
date_str = f" ({item.date})" if item.date else ""
lines.append(f"**{item.id}** (score:{item.score}) hn/{item.author}{date_str}{eng_str}")
lines.append(f"**{item.id}** (score:{item.score}) hn/{item.author}{date_str}{eng_str}{_xref_tag(item)}")
lines.append(f" {item.title}")
lines.append(f" {item.hn_url}")
lines.append(f" *{item.why_relevant}*")
@@ -265,7 +273,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
date_str = f" ({item.date})" if item.date else " (date unknown)"
conf_str = f" [date:{item.date_confidence}]" if item.date_confidence != "high" else ""
lines.append(f"**{item.id}** [WEB] (score:{item.score}) {item.source_domain}{date_str}{conf_str}")
lines.append(f"**{item.id}** [WEB] (score:{item.score}) {item.source_domain}{date_str}{conf_str}{_xref_tag(item)}")
lines.append(f" {item.title}")
lines.append(f" {item.url}")
lines.append(f" {item.snippet[:150]}...")
scripts/lib/schema.py
+30 -5
View File
@@ -93,9 +93,10 @@ class RedditItem:
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
d = {
'id': self.id,
'title': self.title,
'url': self.url,
@@ -110,6 +111,9 @@ class RedditItem:
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
@@ -126,9 +130,10 @@ class XItem:
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
d = {
'id': self.id,
'text': self.text,
'url': self.url,
@@ -141,6 +146,9 @@ class XItem:
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
@@ -157,9 +165,10 @@ class WebSearchItem:
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
d = {
'id': self.id,
'title': self.title,
'url': self.url,
@@ -172,6 +181,9 @@ class WebSearchItem:
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
@@ -189,9 +201,10 @@ class YouTubeItem:
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
d = {
'id': self.id,
'title': self.title,
'url': self.url,
@@ -205,6 +218,9 @@ class YouTubeItem:
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
@@ -224,9 +240,10 @@ class HackerNewsItem:
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
return {
d = {
'id': self.id,
'title': self.title,
'url': self.url,
@@ -242,6 +259,9 @@ class HackerNewsItem:
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
@@ -338,6 +358,7 @@ class Report:
why_relevant=r.get('why_relevant', ''),
subs=subs,
score=r.get('score', 0),
cross_refs=r.get('cross_refs', []),
))
# Reconstruct X items
@@ -359,6 +380,7 @@ class Report:
why_relevant=x.get('why_relevant', ''),
subs=subs,
score=x.get('score', 0),
cross_refs=x.get('cross_refs', []),
))
# Reconstruct Web items
@@ -377,6 +399,7 @@ class Report:
why_relevant=w.get('why_relevant', ''),
subs=subs,
score=w.get('score', 0),
cross_refs=w.get('cross_refs', []),
))
# Reconstruct YouTube items
@@ -399,6 +422,7 @@ class Report:
why_relevant=y.get('why_relevant', ''),
subs=subs,
score=y.get('score', 0),
cross_refs=y.get('cross_refs', []),
))
# Reconstruct HackerNews items
@@ -424,6 +448,7 @@ class Report:
why_relevant=h.get('why_relevant', ''),
subs=subs,
score=h.get('score', 0),
cross_refs=h.get('cross_refs', []),
))
return cls(
scripts/lib/youtube_yt.py
+35 -3
View File
@@ -17,7 +17,7 @@ import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Set, Tuple
# Depth configurations: how many videos to search / transcribe
DEPTH_CONFIG = {
@@ -35,6 +35,38 @@ TRANSCRIPT_LIMITS = {
# Max words to keep from each transcript
TRANSCRIPT_MAX_WORDS = 500
# Stopwords for relevance computation (common English words that dilute token overlap)
STOPWORDS = frozenset({
'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on',
'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my',
'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can',
'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about',
'all', 'just', 'get', 'has', 'have', 'was', 'will',
})
def _tokenize(text: str) -> Set[str]:
"""Lowercase, strip punctuation, remove stopwords, drop single-char tokens."""
words = re.sub(r'[^\w\s]', ' ', text.lower()).split()
return {w for w in words if w not in STOPWORDS and len(w) > 1}
def _compute_relevance(query: str, title: str) -> float:
"""Compute relevance as ratio of query tokens found in title.
Uses ratio overlap (intersection / query_length) so short queries
score higher when fully represented in the title. Floors at 0.1.
"""
q_tokens = _tokenize(query)
t_tokens = _tokenize(title)
if not q_tokens:
return 0.5 # Neutral fallback for empty/stopword-only queries
overlap = len(q_tokens & t_tokens)
ratio = overlap / len(q_tokens)
return max(0.1, min(1.0, ratio))
def _log(msg: str):
"""Log to stderr."""
@@ -183,8 +215,8 @@ def search_youtube(
"comments": comment_count,
},
"duration": video.get("duration"),
"relevance": 0.7, # Default; no LLM relevance scoring for YouTube
"why_relevant": f"YouTube video about {core_topic}",
"relevance": _compute_relevance(core_topic, video.get("title", "")),
"why_relevant": f"YouTube: {video.get('title', core_topic)[:60]}",
})
# Soft date filter: prefer recent items but fall back to all if too few