feat(reddit): elevate top comments, improve subreddit discovery, default to ScrapeCreators
Three improvements from beta testing: 1. Top comments: 10% scoring weight for comment quality, 💬 top comment rendered prominently in compact/full output, increased insight limits 2. Subreddit discovery: relevance-weighted scoring with topic word matching, utility sub penalties (UTILITY_SUBS blocklist), engagement bonus 3. Default method: SKILL.md primaryEnv → SCRAPECREATORS_API_KEY, web-only banner recommends SC first, security section updated Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
+52
-11
@@ -130,23 +130,62 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]:
|
||||
return queries
|
||||
|
||||
|
||||
def discover_subreddits(results: List[Dict[str, Any]], max_subs: int = 5) -> List[str]:
|
||||
"""Extract top subreddits from global search results by frequency.
|
||||
# Known utility/meta subreddits that match queries but aren't discussion subs.
|
||||
# These get a 0.3x penalty (not banned) in subreddit discovery scoring.
|
||||
UTILITY_SUBS = frozenset({
|
||||
'namethatsong', 'findthatsong', 'tipofmytongue',
|
||||
'whatisthissong', 'helpmefind', 'whatisthisthing',
|
||||
'whatsthissong', 'findareddit', 'subredditdrama',
|
||||
})
|
||||
|
||||
|
||||
def discover_subreddits(
|
||||
results: List[Dict[str, Any]],
|
||||
topic: str = "",
|
||||
max_subs: int = 5,
|
||||
) -> List[str]:
|
||||
"""Extract top subreddits from global search results with relevance weighting.
|
||||
|
||||
Uses frequency + topic-word matching + utility-sub penalties + engagement
|
||||
bonus to find discussion subs rather than utility/meta subs.
|
||||
|
||||
Args:
|
||||
results: List of post dicts from global search
|
||||
topic: Original search topic (for relevance matching)
|
||||
max_subs: Maximum subreddits to return
|
||||
|
||||
Returns:
|
||||
Top subreddit names sorted by post count
|
||||
Top subreddit names sorted by weighted score
|
||||
"""
|
||||
counts = Counter()
|
||||
core = _extract_core_subject(topic) if topic else ""
|
||||
core_words = set(core.lower().split()) if core else set()
|
||||
|
||||
scores = Counter()
|
||||
for post in results:
|
||||
sub = post.get("subreddit", "")
|
||||
if sub:
|
||||
counts[sub] += 1
|
||||
if not sub:
|
||||
continue
|
||||
|
||||
return [sub for sub, _ in counts.most_common(max_subs)]
|
||||
# Base: frequency count
|
||||
base = 1.0
|
||||
|
||||
# Bonus: subreddit name contains a core topic word
|
||||
sub_lower = sub.lower()
|
||||
if core_words and any(w in sub_lower for w in core_words if len(w) > 2):
|
||||
base += 2.0
|
||||
|
||||
# Penalty: known utility/meta subreddits
|
||||
if sub_lower in UTILITY_SUBS:
|
||||
base *= 0.3
|
||||
|
||||
# Bonus: post engagement (high-engagement posts = better sub)
|
||||
ups = post.get("ups") or post.get("score", 0)
|
||||
if ups and ups > 100:
|
||||
base += 0.5
|
||||
|
||||
scores[sub] += base
|
||||
|
||||
return [sub for sub, _ in scores.most_common(max_subs)]
|
||||
|
||||
|
||||
def _parse_date(created_utc) -> Optional[str]:
|
||||
@@ -399,7 +438,7 @@ def search_reddit(
|
||||
all_items.append(item)
|
||||
|
||||
# === Phase 3: Subreddit Discovery + Targeted Search ===
|
||||
discovered_subs = discover_subreddits(all_raw_posts, max_subs=config["subreddit_searches"])
|
||||
discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=config["subreddit_searches"])
|
||||
_log(f"Discovered subreddits: {discovered_subs}")
|
||||
|
||||
core = _extract_core_subject(topic)
|
||||
@@ -484,7 +523,7 @@ def enrich_with_comments(
|
||||
top_comments = []
|
||||
insights = []
|
||||
|
||||
for c in raw_comments[:10]: # Take top 10 comments
|
||||
for ci, c in enumerate(raw_comments[:10]): # Take top 10 comments
|
||||
body = c.get("body", "")
|
||||
if not body or body in ("[deleted]", "[removed]"):
|
||||
continue
|
||||
@@ -494,11 +533,13 @@ def enrich_with_comments(
|
||||
permalink = c.get("permalink", "")
|
||||
comment_url = f"https://reddit.com{permalink}" if permalink else ""
|
||||
|
||||
# Top comment gets more room (400 chars) — funny/clever comments need it
|
||||
max_excerpt = 400 if ci == 0 else 300
|
||||
top_comments.append({
|
||||
"score": score,
|
||||
"date": _parse_date(c.get("created_utc")),
|
||||
"author": author,
|
||||
"excerpt": body[:300],
|
||||
"excerpt": body[:max_excerpt],
|
||||
"url": comment_url,
|
||||
})
|
||||
|
||||
@@ -518,7 +559,7 @@ def enrich_with_comments(
|
||||
top_comments.sort(key=lambda c: c.get("score", 0), reverse=True)
|
||||
|
||||
item["top_comments"] = top_comments[:10]
|
||||
item["comment_insights"] = insights[:7]
|
||||
item["comment_insights"] = insights[:10]
|
||||
|
||||
return items
|
||||
|
||||
|
||||
+22
-5
@@ -108,11 +108,11 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
lines.append("**🌐 WEB SEARCH MODE** - assistant will search blogs, docs & news")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("**⚡ Want better results?** Add API keys or sign in to Codex to unlock Reddit & X data:")
|
||||
lines.append("- `OPENAI_API_KEY` or `codex login` → Reddit threads with real upvotes & comments")
|
||||
lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:")
|
||||
lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — real upvotes, comments, views")
|
||||
lines.append("- `XAI_API_KEY` → X posts with real likes & reposts")
|
||||
lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)")
|
||||
lines.append("- Edit `~/.config/last30days/.env` to add keys")
|
||||
lines.append("- If already signed in but still seeing this, re-run `codex login`")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
|
||||
@@ -137,7 +137,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*")
|
||||
lines.append("")
|
||||
elif report.mode == "x-only" and missing_keys in ("reddit", "none"):
|
||||
lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*")
|
||||
lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) and better triangulation.*")
|
||||
lines.append("")
|
||||
|
||||
# Reddit items
|
||||
@@ -174,7 +174,15 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
lines.append(f" {item.url}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
|
||||
# Top comment insights
|
||||
# Top comment (elevated — Reddit's value IS the comments)
|
||||
if item.top_comments and item.top_comments[0].score >= 10:
|
||||
tc = item.top_comments[0]
|
||||
excerpt = tc.excerpt[:200]
|
||||
if len(tc.excerpt) > 200:
|
||||
excerpt = excerpt.rstrip() + "..."
|
||||
lines.append(f' \U0001f4ac Top comment ({tc.score} upvotes): "{excerpt}"')
|
||||
|
||||
# Comment insights
|
||||
if item.comment_insights:
|
||||
lines.append(" Insights:")
|
||||
for insight in item.comment_insights[:3]:
|
||||
@@ -622,6 +630,15 @@ def render_full_report(report: schema.Report) -> str:
|
||||
eng = item.engagement
|
||||
lines.append(f"- **Engagement:** {eng.score or '?'} points, {eng.num_comments or '?'} comments")
|
||||
|
||||
if item.top_comments and item.top_comments[0].score >= 10:
|
||||
tc = item.top_comments[0]
|
||||
excerpt = tc.excerpt[:200]
|
||||
if len(tc.excerpt) > 200:
|
||||
excerpt = excerpt.rstrip() + "..."
|
||||
lines.append("")
|
||||
lines.append(f'**\U0001f4ac Top Comment** ({tc.score} upvotes, u/{tc.author}):')
|
||||
lines.append(f'> {excerpt}')
|
||||
|
||||
if item.comment_insights:
|
||||
lines.append("")
|
||||
lines.append("**Key Insights from Comments:**")
|
||||
|
||||
+17
-5
@@ -31,10 +31,16 @@ def log1p_safe(x: Optional[int]) -> float:
|
||||
return math.log1p(x)
|
||||
|
||||
|
||||
def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
def compute_reddit_engagement_raw(
|
||||
engagement: Optional[schema.Engagement],
|
||||
top_comment_score: Optional[int] = None,
|
||||
) -> Optional[float]:
|
||||
"""Compute raw engagement score for Reddit item.
|
||||
|
||||
Formula: 0.55*log1p(score) + 0.40*log1p(num_comments) + 0.05*(upvote_ratio*10)
|
||||
Formula: 0.50*log1p(score) + 0.35*log1p(num_comments) + 0.05*(upvote_ratio*10) + 0.10*log1p(top_comment_score)
|
||||
|
||||
The 10% comment quality weight rewards posts where the community engaged deeply
|
||||
— a highly upvoted top comment means the thread sparked real discussion.
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
@@ -45,8 +51,9 @@ def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Op
|
||||
score = log1p_safe(engagement.score)
|
||||
comments = log1p_safe(engagement.num_comments)
|
||||
ratio = (engagement.upvote_ratio or 0.5) * 10
|
||||
top_cmt = log1p_safe(top_comment_score)
|
||||
|
||||
return 0.55 * score + 0.40 * comments + 0.05 * ratio
|
||||
return 0.50 * score + 0.35 * comments + 0.05 * ratio + 0.10 * top_cmt
|
||||
|
||||
|
||||
def compute_x_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
@@ -113,8 +120,13 @@ def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem
|
||||
if not items:
|
||||
return items
|
||||
|
||||
# Compute raw engagement scores
|
||||
eng_raw = [compute_reddit_engagement_raw(item.engagement) for item in items]
|
||||
# Compute raw engagement scores (with top comment quality signal)
|
||||
eng_raw = []
|
||||
for item in items:
|
||||
top_cmt_score = None
|
||||
if item.top_comments:
|
||||
top_cmt_score = item.top_comments[0].score
|
||||
eng_raw.append(compute_reddit_engagement_raw(item.engagement, top_cmt_score))
|
||||
|
||||
# Normalize engagement to 0-100
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
Reference in New Issue
Block a user