From c711e443fe1053f314827560181c36475fd0bb92 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Sat, 14 Mar 2026 00:38:52 -0700 Subject: [PATCH] Reduce Reddit and Polymarket false positives Weight Reddit relevance toward titles, stop Polymarket from expanding low-signal standalone terms, and prevent short binary outcomes from matching unrelated queries. Validation: uv run python -m unittest tests.test_reddit_sc tests.test_polymarket --- scripts/lib/polymarket.py | 37 ++++++++++++++++++++++--- scripts/lib/reddit.py | 21 +++++++++++++-- tests/test_polymarket.py | 57 ++++++++++++++++++++++++++++++++------- tests/test_reddit_sc.py | 19 +++++++++++++ 4 files changed, 118 insertions(+), 16 deletions(-) diff --git a/scripts/lib/polymarket.py b/scripts/lib/polymarket.py index f9da705..de0ee14 100644 --- a/scripts/lib/polymarket.py +++ b/scripts/lib/polymarket.py @@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional from urllib.parse import quote_plus, urlencode from . import http -from .relevance import token_overlap_relevance +from .query_type import detect_query_type +from .relevance import LOW_SIGNAL_QUERY_TOKENS, token_overlap_relevance GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search" @@ -74,7 +75,7 @@ def _expand_queries(topic: str) -> List[str]: words = core.split() if len(words) >= 2: for word in words: - if len(word) > 1: # skip single-char words + if len(word) > 1 and word.lower() not in LOW_SIGNAL_QUERY_TOKENS: queries.append(word) # Add the full topic if different from core @@ -327,19 +328,47 @@ def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None) if core in title_lower: return 1.0 - best_score = token_overlap_relevance(core, title) + query_type = detect_query_type(topic) + title_score = token_overlap_relevance(core, title) + best_score = title_score if outcomes: for outcome_name in outcomes: outcome_lower = outcome_name.lower() outcome_score = token_overlap_relevance(core, outcome_name) - if core in outcome_lower or outcome_lower in core: + if _strong_phrase_match(core, outcome_lower): outcome_score = max(outcome_score, 0.92 if len(outcome_lower.split()) >= 2 else 0.88) + if title_score < 0.3: + outcome_cap = 0.55 if query_type == "prediction" else 0.24 + outcome_score = min(outcome_cap, outcome_score) + else: + outcome_score = max(title_score, 0.75 * title_score + 0.25 * outcome_score) best_score = max(best_score, outcome_score) return round(best_score, 2) +def _strong_phrase_match(core: str, candidate: str) -> bool: + """Require real token matches, not accidental short substrings. + + This prevents binary outcomes like "No" from matching "nano" or similar + short-string accidents. + """ + candidate = " ".join(re.sub(r"[^\w\s]", " ", candidate.lower()).split()) + core = " ".join(re.sub(r"[^\w\s]", " ", core.lower()).split()) + if not candidate or not core: + return False + + candidate_tokens = candidate.split() + core_tokens = set(core.split()) + + if len(candidate_tokens) >= 2: + return candidate in core or core in candidate + + token = candidate_tokens[0] + return len(token) > 2 and token in core_tokens + + def _safe_float(val, default=0.0) -> float: """Safely convert a value to float.""" try: diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py index 6871eb0..19d3edc 100644 --- a/scripts/lib/reddit.py +++ b/scripts/lib/reddit.py @@ -202,8 +202,9 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global" title = str(post.get("title", "")).strip() selftext = str(post.get("selftext", "")) - # Compute relevance from query-to-content overlap (or default 0.7) - relevance = token_overlap_relevance(query, title + " " + selftext) if query else 0.7 + # Score the title first, then let the body provide limited support. + # This keeps long selftexts from overpowering the visible topic signal. + relevance = _compute_post_relevance(query, title, selftext) if query else 0.7 return { "id": f"R{idx}", @@ -223,6 +224,22 @@ def _normalize_post(post: Dict[str, Any], idx: int, source_label: str = "global" } +def _compute_post_relevance(query: str, title: str, selftext: str) -> float: + """Compute Reddit relevance with title-first weighting. + + Title should carry most of the weight because it is the visible summary the + user sees. Selftext can lift a marginal match, but it should not rescue a + weak or ambiguous title into the top ranks. + """ + title_score = token_overlap_relevance(query, title) + if not selftext.strip(): + return title_score + + body_score = token_overlap_relevance(query, selftext) + support_score = max(title_score, body_score) + return round(0.75 * title_score + 0.25 * support_score, 2) + + def _global_search( query: str, token: str, diff --git a/tests/test_polymarket.py b/tests/test_polymarket.py index 0c4e73b..7689e9d 100644 --- a/tests/test_polymarket.py +++ b/tests/test_polymarket.py @@ -78,6 +78,12 @@ class TestExpandQueries(unittest.TestCase): self.assertIn("new", queries) self.assertIn("idea", queries) + def test_low_signal_words_not_expanded_standalone(self): + queries = polymarket._expand_queries("anthropic odds") + self.assertIn("anthropic odds", queries) + self.assertIn("anthropic", queries) + self.assertNotIn("odds", queries) + class TestExtractDomainQueries(unittest.TestCase): def _make_tag(self, label): @@ -195,6 +201,37 @@ class TestFormatPriceMovement(unittest.TestCase): self.assertIsNone(result) +class TestTextSimilarity(unittest.TestCase): + def test_short_binary_outcome_does_not_match_substring(self): + score = polymarket._compute_text_similarity( + "nano banana pro prompting", + "NATO x Russia military clash by...?", + ["No", "Yes"], + ) + self.assertLess(score, 0.3) + + def test_outcome_only_match_is_capped_for_non_prediction_queries(self): + score = polymarket._compute_text_similarity( + "kanye west", + "Top Spotify artist in March?", + ["Kanye West", "Taylor Swift"], + ) + self.assertLess(score, 0.3) + + def test_direct_title_match_beats_outcome_only_prediction_market(self): + direct = polymarket._compute_text_similarity( + "anthropic odds", + "Will Anthropic or OpenAI IPO first?", + [], + ) + generic = polymarket._compute_text_similarity( + "anthropic odds", + "Which company will have the best AI model for coding on March 31", + ["Anthropic", "OpenAI", "Google"], + ) + self.assertGreater(direct, generic) + + class TestParseOutcomePrices(unittest.TestCase): def test_binary_market_json_strings(self): market = { @@ -590,27 +627,27 @@ class TestTextSimilarity(unittest.TestCase): self.assertEqual(score, 1.0) def test_outcome_substring_match(self): - """Topic 'Arizona' should match outcome 'Arizona' even when title has no overlap.""" + """Prediction queries can still use outcome-only entity matches.""" score = polymarket._compute_text_similarity( - "Arizona", + "Arizona odds", "Who will be the #1 overall seed?", outcomes=["Duke", "Arizona", "Houston"], ) - self.assertEqual(score, 1.0) + self.assertEqual(score, 0.55) def test_outcome_bidirectional_match(self): - """Topic 'Arizona Basketball' should match outcome 'Arizona' (outcome in core).""" + """Longer prediction topics keep the same moderated outcome-only cap.""" score = polymarket._compute_text_similarity( - "Arizona Basketball", + "Arizona Basketball odds", "Who will be the #1 overall seed?", outcomes=["Duke", "Arizona", "Houston"], ) - self.assertEqual(score, 0.88) + self.assertEqual(score, 0.55) def test_outcome_token_overlap(self): - """Partial token overlap with outcome gets a moderate score.""" + """Outcome-only prediction matches stay moderate, not dominant.""" score = polymarket._compute_text_similarity( - "Iran War", + "Iran War odds", "Unrelated geopolitics title", outcomes=["War continues", "Peace deal"], ) @@ -630,11 +667,11 @@ class TestTextSimilarity(unittest.TestCase): """Outcomes with price <= 1% should be filtered by the caller, not this function.""" # This function doesn't filter - it trusts the caller to pass only relevant outcomes score = polymarket._compute_text_similarity( - "Arizona", + "Arizona odds", "Unrelated title", outcomes=["Arizona"], ) - self.assertEqual(score, 1.0) + self.assertEqual(score, 0.55) def test_generic_only_odds_match_stays_below_threshold(self): score = polymarket._compute_text_similarity( diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py index a212055..9d38943 100644 --- a/tests/test_reddit_sc.py +++ b/tests/test_reddit_sc.py @@ -158,5 +158,24 @@ class TestDepthConfig(unittest.TestCase): ) +class TestPostRelevance(unittest.TestCase): + def test_body_cannot_rescue_weak_title_too_far(self): + score = reddit._compute_post_relevance( + "anthropic odds", + "President Trump orders agencies to stop using Anthropic technology", + "Long body text eventually mentions odds and other tangential details.", + ) + self.assertLess(score, 0.7) + self.assertGreaterEqual(score, 0.5) + + def test_exact_title_match_stays_high(self): + score = reddit._compute_post_relevance( + "claude code tips", + "Claude Code tips for faster workflows", + "", + ) + self.assertGreater(score, 0.7) + + if __name__ == "__main__": unittest.main()