From 7048fe7b83abd66be0e849f993ffd38f21f6093e Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Thu, 5 Mar 2026 17:29:18 -0800 Subject: [PATCH] feat(reddit): elevate top comments, improve subreddit discovery, default to ScrapeCreators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three improvements from beta testing: 1. Top comments: 10% scoring weight for comment quality, 💬 top comment rendered prominently in compact/full output, increased insight limits 2. Subreddit discovery: relevance-weighted scoring with topic word matching, utility sub penalties (UTILITY_SUBS blocklist), engagement bonus 3. Default method: SKILL.md primaryEnv → SCRAPECREATORS_API_KEY, web-only banner recommends SC first, security section updated Co-Authored-By: Claude Opus 4.6 --- SKILL.md | 18 +++++++------ scripts/lib/reddit.py | 63 +++++++++++++++++++++++++++++++++++-------- scripts/lib/render.py | 27 +++++++++++++++---- scripts/lib/score.py | 22 +++++++++++---- 4 files changed, 101 insertions(+), 29 deletions(-) diff --git a/SKILL.md b/SKILL.md index 8b5c001..bee4f2c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -11,11 +11,11 @@ metadata: emoji: "📰" requires: env: - - OPENAI_API_KEY + - SCRAPECREATORS_API_KEY bins: - node - python3 - primaryEnv: OPENAI_API_KEY + primaryEnv: SCRAPECREATORS_API_KEY files: - "scripts/*" homepage: https://github.com/mvanhorn/last30days-skill @@ -239,9 +239,10 @@ The Judge Agent must: 2. Weight YouTube sources HIGH (they have views, likes, and transcript content) 3. Weight TikTok sources HIGH (they have views, likes, and caption content — viral signal) 4. Weight WebSearch sources LOWER (no engagement data) -4. Identify patterns that appear across ALL sources (strongest signals) -5. Note any contradictions between sources -6. Extract the top 3-5 actionable insights +5. **For Reddit: Pay special attention to top comments** — they often contain the wittiest, most insightful, or funniest take. When a top comment has high upvotes (shown as `💬 Top comment (N upvotes)`), quote it directly in your synthesis. Reddit's value is in the comments. +6. Identify patterns that appear across ALL sources (strongest signals) +7. Note any contradictions between sources +8. Extract the top 3-5 actionable insights 7. **Cross-platform signals are the strongest evidence.** When items have `[also on: Reddit, HN]` or similar tags, it means the same story appears across multiple platforms. Lead with these cross-platform findings - they're the most important signals in the research. @@ -345,7 +346,7 @@ CITATION RULE: Cite sources sparingly to prove research is real. CITATION PRIORITY (most to least preferred): 1. @handles from X — "per @handle" (these prove the tool's unique value) -2. r/subreddits from Reddit — "per r/subreddit" +2. r/subreddits from Reddit — "per r/subreddit" (when citing Reddit, prefer quoting top comments over just the thread title) 3. YouTube channels — "per [channel name] on YouTube" (transcript-backed insights) 4. TikTok creators — "per @creator on TikTok" (viral/trending signal) 5. Instagram creators — "per @creator on Instagram" (influencer/creator signal) @@ -596,12 +597,13 @@ Want another prompt? Just tell me what you're creating next. ## Security & Permissions **What this skill does:** -- Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery +- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for Reddit search, subreddit discovery, and comment enrichment (requires SCRAPECREATORS_API_KEY — same key as TikTok + Instagram) +- Legacy: Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery (fallback if no SCRAPECREATORS_API_KEY) - Sends search queries to Twitter's GraphQL API (via browser cookie auth) or xAI's API (`api.x.ai`) for X search - Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth) - Sends search queries to Polymarket Gamma API (`gamma-api.polymarket.com`) for prediction market discovery (free, no auth) - Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data) -- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (requires SCRAPECREATORS_API_KEY, PAYG after 100 free credits) +- Sends search queries to ScrapeCreators API (`api.scrapecreators.com`) for TikTok and Instagram search, transcript/caption extraction (same SCRAPECREATORS_API_KEY as Reddit, PAYG after 100 free credits) - Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search - Fetches public Reddit thread data from `reddit.com` for engagement metrics - Stores research findings in local SQLite database (watchlist mode only) diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py index dc4adad..93979d9 100644 --- a/scripts/lib/reddit.py +++ b/scripts/lib/reddit.py @@ -130,23 +130,62 @@ def expand_reddit_queries(topic: str, depth: str) -> List[str]: return queries -def discover_subreddits(results: List[Dict[str, Any]], max_subs: int = 5) -> List[str]: - """Extract top subreddits from global search results by frequency. +# Known utility/meta subreddits that match queries but aren't discussion subs. +# These get a 0.3x penalty (not banned) in subreddit discovery scoring. +UTILITY_SUBS = frozenset({ + 'namethatsong', 'findthatsong', 'tipofmytongue', + 'whatisthissong', 'helpmefind', 'whatisthisthing', + 'whatsthissong', 'findareddit', 'subredditdrama', +}) + + +def discover_subreddits( + results: List[Dict[str, Any]], + topic: str = "", + max_subs: int = 5, +) -> List[str]: + """Extract top subreddits from global search results with relevance weighting. + + Uses frequency + topic-word matching + utility-sub penalties + engagement + bonus to find discussion subs rather than utility/meta subs. Args: results: List of post dicts from global search + topic: Original search topic (for relevance matching) max_subs: Maximum subreddits to return Returns: - Top subreddit names sorted by post count + Top subreddit names sorted by weighted score """ - counts = Counter() + core = _extract_core_subject(topic) if topic else "" + core_words = set(core.lower().split()) if core else set() + + scores = Counter() for post in results: sub = post.get("subreddit", "") - if sub: - counts[sub] += 1 + if not sub: + continue - return [sub for sub, _ in counts.most_common(max_subs)] + # Base: frequency count + base = 1.0 + + # Bonus: subreddit name contains a core topic word + sub_lower = sub.lower() + if core_words and any(w in sub_lower for w in core_words if len(w) > 2): + base += 2.0 + + # Penalty: known utility/meta subreddits + if sub_lower in UTILITY_SUBS: + base *= 0.3 + + # Bonus: post engagement (high-engagement posts = better sub) + ups = post.get("ups") or post.get("score", 0) + if ups and ups > 100: + base += 0.5 + + scores[sub] += base + + return [sub for sub, _ in scores.most_common(max_subs)] def _parse_date(created_utc) -> Optional[str]: @@ -399,7 +438,7 @@ def search_reddit( all_items.append(item) # === Phase 3: Subreddit Discovery + Targeted Search === - discovered_subs = discover_subreddits(all_raw_posts, max_subs=config["subreddit_searches"]) + discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=config["subreddit_searches"]) _log(f"Discovered subreddits: {discovered_subs}") core = _extract_core_subject(topic) @@ -484,7 +523,7 @@ def enrich_with_comments( top_comments = [] insights = [] - for c in raw_comments[:10]: # Take top 10 comments + for ci, c in enumerate(raw_comments[:10]): # Take top 10 comments body = c.get("body", "") if not body or body in ("[deleted]", "[removed]"): continue @@ -494,11 +533,13 @@ def enrich_with_comments( permalink = c.get("permalink", "") comment_url = f"https://reddit.com{permalink}" if permalink else "" + # Top comment gets more room (400 chars) — funny/clever comments need it + max_excerpt = 400 if ci == 0 else 300 top_comments.append({ "score": score, "date": _parse_date(c.get("created_utc")), "author": author, - "excerpt": body[:300], + "excerpt": body[:max_excerpt], "url": comment_url, }) @@ -518,7 +559,7 @@ def enrich_with_comments( top_comments.sort(key=lambda c: c.get("score", 0), reverse=True) item["top_comments"] = top_comments[:10] - item["comment_insights"] = insights[:7] + item["comment_insights"] = insights[:10] return items diff --git a/scripts/lib/render.py b/scripts/lib/render.py index 510135a..1af86cf 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -108,11 +108,11 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append("**🌐 WEB SEARCH MODE** - assistant will search blogs, docs & news") lines.append("") lines.append("---") - lines.append("**⚡ Want better results?** Add API keys or sign in to Codex to unlock Reddit & X data:") - lines.append("- `OPENAI_API_KEY` or `codex login` → Reddit threads with real upvotes & comments") + lines.append("**⚡ Want better results?** Add API keys to unlock Reddit, TikTok, Instagram & X data:") + lines.append("- `SCRAPECREATORS_API_KEY` → Reddit + TikTok + Instagram (one key, all three!) — real upvotes, comments, views") lines.append("- `XAI_API_KEY` → X posts with real likes & reposts") + lines.append("- `OPENAI_API_KEY` (legacy) → Reddit threads (slower, higher cost)") lines.append("- Edit `~/.config/last30days/.env` to add keys") - lines.append("- If already signed in but still seeing this, re-run `codex login`") lines.append("---") lines.append("") @@ -137,7 +137,7 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append("*💡 Tip: Add an xAI key (`XAI_API_KEY`) for X/Twitter data and better triangulation.*") lines.append("") elif report.mode == "x-only" and missing_keys in ("reddit", "none"): - lines.append("*💡 Tip: Add OPENAI_API_KEY or run `codex login` for Reddit data and better triangulation. If already signed in, re-run `codex login`.*") + lines.append("*💡 Tip: Add `SCRAPECREATORS_API_KEY` for Reddit + TikTok + Instagram data (one key, all three) and better triangulation.*") lines.append("") # Reddit items @@ -174,7 +174,15 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append(f" {item.url}") lines.append(f" *{item.why_relevant}*") - # Top comment insights + # Top comment (elevated — Reddit's value IS the comments) + if item.top_comments and item.top_comments[0].score >= 10: + tc = item.top_comments[0] + excerpt = tc.excerpt[:200] + if len(tc.excerpt) > 200: + excerpt = excerpt.rstrip() + "..." + lines.append(f' \U0001f4ac Top comment ({tc.score} upvotes): "{excerpt}"') + + # Comment insights if item.comment_insights: lines.append(" Insights:") for insight in item.comment_insights[:3]: @@ -622,6 +630,15 @@ def render_full_report(report: schema.Report) -> str: eng = item.engagement lines.append(f"- **Engagement:** {eng.score or '?'} points, {eng.num_comments or '?'} comments") + if item.top_comments and item.top_comments[0].score >= 10: + tc = item.top_comments[0] + excerpt = tc.excerpt[:200] + if len(tc.excerpt) > 200: + excerpt = excerpt.rstrip() + "..." + lines.append("") + lines.append(f'**\U0001f4ac Top Comment** ({tc.score} upvotes, u/{tc.author}):') + lines.append(f'> {excerpt}') + if item.comment_insights: lines.append("") lines.append("**Key Insights from Comments:**") diff --git a/scripts/lib/score.py b/scripts/lib/score.py index c3a9158..0b49e20 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -31,10 +31,16 @@ def log1p_safe(x: Optional[int]) -> float: return math.log1p(x) -def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: +def compute_reddit_engagement_raw( + engagement: Optional[schema.Engagement], + top_comment_score: Optional[int] = None, +) -> Optional[float]: """Compute raw engagement score for Reddit item. - Formula: 0.55*log1p(score) + 0.40*log1p(num_comments) + 0.05*(upvote_ratio*10) + Formula: 0.50*log1p(score) + 0.35*log1p(num_comments) + 0.05*(upvote_ratio*10) + 0.10*log1p(top_comment_score) + + The 10% comment quality weight rewards posts where the community engaged deeply + — a highly upvoted top comment means the thread sparked real discussion. """ if engagement is None: return None @@ -45,8 +51,9 @@ def compute_reddit_engagement_raw(engagement: Optional[schema.Engagement]) -> Op score = log1p_safe(engagement.score) comments = log1p_safe(engagement.num_comments) ratio = (engagement.upvote_ratio or 0.5) * 10 + top_cmt = log1p_safe(top_comment_score) - return 0.55 * score + 0.40 * comments + 0.05 * ratio + return 0.50 * score + 0.35 * comments + 0.05 * ratio + 0.10 * top_cmt def compute_x_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: @@ -113,8 +120,13 @@ def score_reddit_items(items: List[schema.RedditItem]) -> List[schema.RedditItem if not items: return items - # Compute raw engagement scores - eng_raw = [compute_reddit_engagement_raw(item.engagement) for item in items] + # Compute raw engagement scores (with top comment quality signal) + eng_raw = [] + for item in items: + top_cmt_score = None + if item.top_comments: + top_cmt_score = item.top_comments[0].score + eng_raw.append(compute_reddit_engagement_raw(item.engagement, top_cmt_score)) # Normalize engagement to 0-100 eng_normalized = normalize_to_100(eng_raw)