From c5be117701cc7cdf12c4ec4b3f81392b966ed0fb Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Wed, 11 Mar 2026 15:21:52 -0700 Subject: [PATCH] Replace hardcoded 0.7 relevance with computed token-overlap scores - bird_x: parse_bird_response now accepts query param and computes token_overlap_relevance against tweet text - reddit: _normalize_post computes relevance from query vs title+selftext - hackernews: blends 60% Algolia rank + 40% token overlap + engagement This makes the 45%-weight relevance factor in score.py actually differentiate results instead of being a constant. --- scripts/last30days.py | 4 ++-- scripts/lib/bird_x.py | 13 ++++++++----- scripts/lib/hackernews.py | 16 ++++++++++++---- scripts/lib/reddit.py | 21 ++++++++++++++------- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/scripts/last30days.py b/scripts/last30days.py index 70f6550..ea3f753 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -353,7 +353,7 @@ def _search_x( raw_response = {"error": str(e)} x_error = f"{type(e).__name__}: {e}" - x_items = bird_x.parse_bird_response(raw_response or {}) + x_items = bird_x.parse_bird_response(raw_response or {}, query=topic) # Check for error in response (Bird returns list on success, dict on error) if raw_response and isinstance(raw_response, dict) and raw_response.get("error") and not x_error: @@ -508,7 +508,7 @@ def _search_hackernews( except Exception as e: return [], f"{type(e).__name__}: {e}" - hn_items = hackernews.parse_hackernews_response(response) + hn_items = hackernews.parse_hackernews_response(response, query=topic) if response.get("error"): hn_error = response["error"] diff --git a/scripts/lib/bird_x.py b/scripts/lib/bird_x.py index 7d541fc..0270a8c 100644 --- a/scripts/lib/bird_x.py +++ b/scripts/lib/bird_x.py @@ -14,6 +14,8 @@ from pathlib import Path from datetime import datetime from typing import Any, Dict, List, Optional, Tuple +from .relevance import token_overlap_relevance as _compute_relevance + # Path to the vendored bird-search wrapper _BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs" @@ -233,7 +235,7 @@ def search_x( response = _run_bird_search(query, count, timeout) # Check if we got results - items = parse_bird_response(response) + items = parse_bird_response(response, query=core_topic) # Retry with fewer keywords if 0 results and query has 3+ words core_words = core_topic.split() @@ -242,7 +244,7 @@ def search_x( _log(f"0 results for '{core_topic}', retrying with '{shorter}'") query = f"{shorter} since:{from_date}" response = _run_bird_search(query, count, timeout) - items = parse_bird_response(response) + items = parse_bird_response(response, query=core_topic) # Last-chance retry: use strongest remaining token (often the product name) if not items and core_words: @@ -329,7 +331,7 @@ def search_handles( continue response = json.loads(output) - items = parse_bird_response(response) + items = parse_bird_response(response, query=core_topic) all_items.extend(items) except json.JSONDecodeError: @@ -340,11 +342,12 @@ def search_handles( return all_items -def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: +def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]: """Parse Bird response to match xai_x output format. Args: response: Raw Bird JSON response + query: Original search query for relevance scoring Returns: List of normalized item dicts matching xai_x.parse_x_response() format. @@ -422,7 +425,7 @@ def parse_bird_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: "date": date, "engagement": engagement if any(v is not None for v in engagement.values()) else None, "why_relevant": "", # Bird doesn't provide relevance explanations - "relevance": 0.7, # Default relevance, let score.py re-rank + "relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7, } items.append(item) diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py index 1ce3792..01f544a 100644 --- a/scripts/lib/hackernews.py +++ b/scripts/lib/hackernews.py @@ -12,6 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, List, Optional from . import http +from .relevance import token_overlap_relevance ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search" ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date" @@ -111,9 +112,13 @@ def search_hackernews( return response -def parse_hackernews_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: +def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]: """Parse Algolia response into normalized item dicts. + Args: + response: Algolia search response + query: Original search query for token-overlap relevance scoring + Returns: List of item dicts ready for normalization. """ @@ -134,11 +139,14 @@ def parse_hackernews_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: article_url = hit.get("url") or "" hn_url = f"https://news.ycombinator.com/item?id={object_id}" - # Relevance: Algolia rank position gives a base, engagement boosts it - # Position 0 = most relevant from Algolia + # Relevance: blend Algolia rank with token-overlap content matching rank_score = max(0.3, 1.0 - (i * 0.02)) # 1.0 -> 0.3 over 35 items engagement_boost = min(0.2, math.log1p(points) / 40) - relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1) + if query: + content_score = token_overlap_relevance(query, hit.get("title", "")) + relevance = min(1.0, 0.6 * rank_score + 0.4 * content_score + engagement_boost) + else: + relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1) items.append({ "object_id": object_id, diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py index f9e56df..8957464 100644 --- a/scripts/lib/reddit.py +++ b/scripts/lib/reddit.py @@ -49,6 +49,7 @@ DEPTH_CONFIG = { } from .query import extract_core_subject as _query_extract +from .relevance import token_overlap_relevance # Reddit-specific noise words (preserves original smaller set) NOISE_WORDS = frozenset({ @@ -184,7 +185,7 @@ def _parse_date(created_utc) -> Optional[str]: return None -def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global") -> Dict[str, Any]: +def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global", query: str = "") -> Dict[str, Any]: """Normalize a ScrapeCreators Reddit post to our internal format.""" permalink = post.get("permalink", "") url = f"https://www.reddit.com{permalink}" if permalink else post.get("url", "") @@ -193,10 +194,16 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global" if url and "reddit.com" not in url: url = "" + title = str(post.get("title", "")).strip() + selftext = str(post.get("selftext", "")) + + # Compute relevance from query-to-content overlap (or default 0.7) + relevance = token_overlap_relevance(query, title + " " + selftext) if query else 0.7 + return { "id": f"R{idx}", "reddit_id": post.get("id", ""), - "title": str(post.get("title", "")).strip(), + "title": title, "url": url, "subreddit": str(post.get("subreddit", "")).strip(), "date": _parse_date(post.get("created_utc")), @@ -205,7 +212,7 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global" "num_comments": post.get("num_comments", 0), "upvote_ratio": post.get("upvote_ratio"), }, - "relevance": 0.7, + "relevance": relevance, "why_relevant": f"Reddit {source_label} search", "selftext": str(post.get("selftext", ""))[:500], } @@ -416,23 +423,23 @@ def search_reddit( _log(f" -> {len(posts)} results") all_raw_posts.extend(posts) - # Normalize all posts + # Normalize all posts (with query for relevance scoring) + core = _extract_core_subject(topic) all_items = [] for i, post in enumerate(all_raw_posts): - item = _normalize_post(post, i + 1, "global") + item = _normalize_post(post, i + 1, "global", query=core) all_items.append(item) # === Phase 3: Subreddit Discovery + Targeted Search === discovered_subs = discover_subreddits(all_raw_posts, topic=topic, max_subs=config["subreddit_searches"]) _log(f"Discovered subreddits: {discovered_subs}") - core = _extract_core_subject(topic) for sub in discovered_subs[:config["subreddit_searches"]]: _log(f"Subreddit search: r/{sub} for '{core}'") sub_posts = _subreddit_search(sub, core, token, sort="relevance", timeframe=timeframe) _log(f" -> {len(sub_posts)} results from r/{sub}") for j, post in enumerate(sub_posts): - item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}") + item = _normalize_post(post, len(all_items) + j + 1, f"r/{sub}", query=core) all_items.append(item) # === Phase 4: Deduplicate ===