feat(polymarket): add Polymarket prediction markets as 6th research source

Search Polymarket's free Gamma API for relevant prediction markets on any
topic. Uses smart multi-query expansion to cast a wider net (e.g., "Arizona
Basketball" also searches "Arizona"), merges and dedupes by event ID, and
shows price movement context ("up 22.5% this week"). No API key required.

Also hides sources with zero results from the stats output (all sources).

54 new tests, all passing. Full pipeline integration with scoring, dedupe,
cross-source linking, and rendering.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-02-25 22:27:19 -08:00
parent 52503f8e68
commit 994a4ab2ca
17 changed files with 1699 additions and 43 deletions
+87 -10
View File
@@ -38,9 +38,9 @@ _child_pids: set = set()
_child_pids_lock = threading.Lock()
TIMEOUT_PROFILES = {
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "hackernews_future": 30, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "hackernews_future": 60, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
}
@@ -99,6 +99,7 @@ from lib import (
dates,
dedupe,
hackernews,
polymarket,
entity_extract,
env,
http,
@@ -332,6 +333,34 @@ def _search_hackernews(
return hn_items, hn_error
def _search_polymarket(
topic: str,
from_date: str,
to_date: str,
depth: str,
) -> tuple:
"""Search Polymarket via Gamma API (runs in thread).
Returns:
Tuple of (pm_items, pm_error)
"""
pm_error = None
try:
response = polymarket.search_polymarket(
topic, from_date, to_date, depth=depth,
)
except Exception as e:
return [], f"{type(e).__name__}: {e}"
pm_items = polymarket.parse_polymarket_response(response, topic=topic)
if response.get("error"):
pm_error = response["error"]
return pm_items, pm_error
def _search_web(
topic: str,
config: dict,
@@ -588,6 +617,7 @@ def run_research(
x_items = []
youtube_items = []
hackernews_items = []
polymarket_items = []
web_items = []
raw_openai = None
raw_xai = None
@@ -596,6 +626,7 @@ def run_research(
x_error = None
youtube_error = None
hackernews_error = None
polymarket_error = None
web_error = None
# Determine web search mode
@@ -638,20 +669,22 @@ def run_research(
progress.show_error(f"YouTube error: {e}")
if progress:
progress.end_youtube(len(youtube_items))
return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error
return reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error
# Determine which searches to run
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
do_x = sources in ("both", "x", "all", "x-web")
do_hackernews = True # HN is always available (no API key)
do_polymarket = True # Polymarket is always available (no API key)
# Run Reddit, X, YouTube, HN, and Web searches in parallel
# Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel
reddit_future = None
x_future = None
youtube_future = None
hackernews_future = None
polymarket_future = None
web_future = None
max_workers = 2 + (1 if run_youtube else 0) + (1 if do_hackernews else 0) + (1 if web_backend else 0)
max_workers = 2 + (1 if run_youtube else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit searches
@@ -685,6 +718,13 @@ def run_research(
_search_hackernews, topic, from_date, to_date, depth
)
if do_polymarket:
if progress:
progress.start_polymarket()
polymarket_future = executor.submit(
_search_polymarket, topic, from_date, to_date, depth
)
if web_backend:
sys.stderr.write(f"[web] Searching via {web_backend}\n")
sys.stderr.flush()
@@ -760,6 +800,23 @@ def run_research(
if progress:
progress.end_hackernews(len(hackernews_items))
if polymarket_future:
pm_timeout = timeouts.get("polymarket_future", future_timeout)
try:
polymarket_items, polymarket_error = polymarket_future.result(timeout=pm_timeout)
if polymarket_error and progress:
progress.show_error(f"Polymarket error: {polymarket_error}")
except TimeoutError:
polymarket_error = f"Polymarket search timed out after {pm_timeout}s"
if progress:
progress.show_error(polymarket_error)
except Exception as e:
polymarket_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"Polymarket error: {e}")
if progress:
progress.end_polymarket(len(polymarket_items))
if web_future:
try:
web_items, web_error = web_future.result(timeout=future_timeout)
@@ -868,7 +925,7 @@ def run_research(
if sup_x:
x_items.extend(sup_x)
return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error
return reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error
def main():
@@ -994,6 +1051,7 @@ def main():
"bird_username": x_source_status.get("bird_username"),
"youtube": has_ytdlp,
"hackernews": True,
"polymarket": True,
"web_search_backend": web_source,
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
"brave": bool(config.get("BRAVE_API_KEY")),
@@ -1022,6 +1080,7 @@ def main():
"bird_username": x_source_status.get("bird_username"),
"youtube": has_ytdlp,
"hackernews": True,
"polymarket": True,
"web_search_backend": web_source,
}
ui.show_diagnostic_banner(diag)
@@ -1099,7 +1158,7 @@ def main():
mode = sources
# Run research
reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error = run_research(
reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research(
args.topic,
sources,
config,
@@ -1123,6 +1182,7 @@ def main():
normalized_x = normalize.normalize_x_items(x_items, from_date, to_date)
normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else []
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
# Hard date filter: exclude items with verified dates outside the range
@@ -1134,6 +1194,8 @@ def main():
# YouTube content has a longer shelf life than tweets/posts.
filtered_youtube = normalized_youtube
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
filtered_pm = normalized_pm
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
# Score items
@@ -1141,6 +1203,7 @@ def main():
scored_x = score.score_x_items(filtered_x)
scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else []
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
# Sort items
@@ -1148,6 +1211,7 @@ def main():
sorted_x = score.sort_items(scored_x)
sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else []
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
sorted_web = score.sort_items(scored_web) if scored_web else []
# Dedupe items
@@ -1155,6 +1219,7 @@ def main():
deduped_x = dedupe.dedupe_x(sorted_x)
deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else []
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
# Minimum result guarantee: if all Reddit results were filtered out but
@@ -1166,7 +1231,7 @@ def main():
# Cross-source linking: annotate items that discuss the same story
dedupe.cross_source_link(
deduped_reddit, deduped_x, deduped_youtube, deduped_hn, deduped_web,
deduped_reddit, deduped_x, deduped_youtube, deduped_hn, deduped_pm, deduped_web,
)
progress.end_processing()
@@ -1184,11 +1249,13 @@ def main():
report.x = deduped_x
report.youtube = deduped_youtube
report.hackernews = deduped_hn
report.polymarket = deduped_pm
report.web = deduped_web
report.reddit_error = reddit_error
report.x_error = x_error
report.youtube_error = youtube_error
report.hackernews_error = hackernews_error
report.polymarket_error = polymarket_error
report.web_error = web_error
report.resolved_x_handle = args.x_handle
@@ -1202,7 +1269,7 @@ def main():
if sources == "web":
progress.show_web_only_complete()
else:
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn))
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm))
# Build source info for status footer
source_info = {}
@@ -1270,6 +1337,16 @@ def main():
"engagement_score": item.engagement.score if item.engagement else 0,
"relevance_score": item.relevance,
})
for item in deduped_pm:
findings.append({
"source": "polymarket",
"url": item.url,
"title": item.question,
"author": "polymarket",
"content": item.title,
"engagement_score": item.engagement.volume if item.engagement and item.engagement.volume else 0,
"relevance_score": item.relevance,
})
for item in deduped_web:
findings.append({
"source": "web",
+13 -1
View File
@@ -46,7 +46,7 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem,
schema.HackerNewsItem, schema.WebSearchItem]
schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
def get_item_text(item: AnyItem) -> str:
@@ -57,6 +57,8 @@ def get_item_text(item: AnyItem) -> str:
return item.title
elif isinstance(item, schema.YouTubeItem):
return f"{item.title} {item.channel_name}"
elif isinstance(item, schema.PolymarketItem):
return f"{item.title} {item.question}"
elif isinstance(item, schema.WebSearchItem):
return item.title
else:
@@ -79,6 +81,8 @@ def _get_cross_source_text(item: AnyItem) -> str:
elif title.startswith("Ask HN:"):
title = title[7:].strip()
return title
if isinstance(item, schema.PolymarketItem):
return item.title
return get_item_text(item)
@@ -198,6 +202,14 @@ def dedupe_hackernews(
return dedupe_items(items, threshold)
def dedupe_polymarket(
items: List[schema.PolymarketItem],
threshold: float = 0.7,
) -> List[schema.PolymarketItem]:
"""Dedupe Polymarket items."""
return dedupe_items(items, threshold)
def cross_source_link(
*source_lists: List[AnyItem],
threshold: float = 0.40,
+8
View File
@@ -254,6 +254,14 @@ def is_hackernews_available() -> bool:
return True
def is_polymarket_available() -> bool:
"""Check if Polymarket source is available.
Always returns True - Gamma API is free, no key needed.
"""
return True
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
"""Get detailed X source status for UI decisions.
+45 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
from . import dates, schema
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem)
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem, schema.PolymarketItem)
def filter_by_date_range(
@@ -257,6 +257,50 @@ def normalize_hackernews_items(
return normalized
def normalize_polymarket_items(
items: List[Dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.PolymarketItem]:
"""Normalize raw Polymarket items to schema.
Args:
items: Raw Polymarket items from Gamma API
from_date: Start of date range
to_date: End of date range
Returns:
List of PolymarketItem objects
"""
normalized = []
for i, item in enumerate(items):
engagement = schema.Engagement(
volume=item.get("volume24hr", 0.0),
liquidity=item.get("liquidity", 0.0),
)
date_str = item.get("date")
normalized.append(schema.PolymarketItem(
id=f"PM{i+1}",
title=item.get("title", ""),
question=item.get("question", ""),
url=item.get("url", ""),
outcome_prices=item.get("outcome_prices", []),
outcomes_remaining=item.get("outcomes_remaining", 0),
price_movement=item.get("price_movement"),
date=date_str,
date_confidence="high",
engagement=engagement,
end_date=item.get("end_date"),
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
"""Convert schema items to dicts for JSON serialization."""
return [item.to_dict() for item in items]
+354
View File
@@ -0,0 +1,354 @@
"""Polymarket prediction market search via Gamma API (free, no auth required).
Uses gamma-api.polymarket.com for event/market discovery.
No API key needed - public read-only API with generous rate limits (350 req/10s).
"""
import json
import math
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from urllib.parse import quote_plus, urlencode
from . import http
GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
DEPTH_CONFIG = {
"quick": 5,
"default": 10,
"deep": 20,
}
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[PM] {msg}\n")
sys.stderr.flush()
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from topic string.
Strips common prefixes like 'last 7 days', 'what are people saying about', etc.
"""
topic = topic.strip()
# Remove common leading phrases
prefixes = [
r"^last \d+ days?\s+",
r"^what(?:'s| is| are) (?:people saying about|happening with|going on with)\s+",
r"^how (?:is|are)\s+",
r"^tell me about\s+",
r"^research\s+",
]
for pattern in prefixes:
topic = re.sub(pattern, "", topic, flags=re.IGNORECASE)
return topic.strip()
def _expand_queries(topic: str) -> List[str]:
"""Generate 2-4 search queries to cast a wider net.
Strategy:
- Always include the core subject
- Split multi-word topics into component searches
- Include the full topic if different from core
- Cap at 4 queries, dedupe
"""
core = _extract_core_subject(topic)
queries = [core]
# Split multi-word topics into component searches
words = core.split()
if len(words) >= 2:
# Try the first significant word alone (e.g., "Arizona" from "Arizona Basketball")
queries.append(words[0])
# Add the full topic if different from core
if topic.lower().strip() != core.lower():
queries.append(topic.strip())
# Dedupe while preserving order, cap at 4
seen = set()
unique = []
for q in queries:
q_lower = q.lower().strip()
if q_lower and q_lower not in seen:
seen.add(q_lower)
unique.append(q.strip())
return unique[:4]
def _search_single_query(query: str, limit: int) -> Dict[str, Any]:
"""Run a single search query against Gamma API."""
params = {
"q": query,
"limit": str(limit),
}
url = f"{GAMMA_SEARCH_URL}?{urlencode(params)}"
try:
response = http.request("GET", url, timeout=15, retries=2)
return response
except http.HTTPError as e:
_log(f"Search failed for '{query}': {e}")
return {"events": [], "error": str(e)}
except Exception as e:
_log(f"Search failed for '{query}': {e}")
return {"events": [], "error": str(e)}
def search_polymarket(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> Dict[str, Any]:
"""Search Polymarket via Gamma API with smart query expansion.
Runs 2-4 expanded queries in parallel, merges and dedupes by event ID.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD) - used for activity filtering
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
Returns:
Dict with 'events' list and optional 'error'.
"""
limit_per_query = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
queries = _expand_queries(topic)
_log(f"Searching for '{topic}' with queries: {queries} (limit={limit_per_query})")
# Run all queries in parallel
all_events = {} # event_id -> (event_data, query_index)
errors = []
with ThreadPoolExecutor(max_workers=min(4, len(queries))) as executor:
futures = {
executor.submit(_search_single_query, q, limit_per_query): i
for i, q in enumerate(queries)
}
for future in as_completed(futures):
query_idx = futures[future]
try:
response = future.result(timeout=15)
if response.get("error"):
errors.append(response["error"])
events = response.get("events", [])
for event in events:
event_id = event.get("id", "")
if not event_id:
continue
# Keep the first occurrence (from highest-priority query)
if event_id not in all_events:
all_events[event_id] = (event, query_idx)
elif query_idx < all_events[event_id][1]:
# Replace with higher-priority query result
all_events[event_id] = (event, query_idx)
except Exception as e:
errors.append(str(e))
# Sort by query priority, then by position
merged_events = [ev for ev, _ in sorted(all_events.values(), key=lambda x: x[1])]
_log(f"Found {len(merged_events)} unique events across {len(queries)} queries")
result = {"events": merged_events}
if errors and not merged_events:
result["error"] = "; ".join(errors[:2])
return result
def _format_price_movement(market: Dict[str, Any]) -> Optional[str]:
"""Pick the most significant price change and format it.
Returns string like 'down 11.7% this month' or None if no significant change.
"""
changes = [
(abs(market.get("oneDayPriceChange") or 0), market.get("oneDayPriceChange"), "today"),
(abs(market.get("oneWeekPriceChange") or 0), market.get("oneWeekPriceChange"), "this week"),
(abs(market.get("oneMonthPriceChange") or 0), market.get("oneMonthPriceChange"), "this month"),
]
# Pick the largest absolute change
changes.sort(key=lambda x: x[0], reverse=True)
abs_change, raw_change, period = changes[0]
# Skip if change is less than 1% (noise)
if abs_change < 0.01:
return None
direction = "up" if raw_change > 0 else "down"
pct = abs_change * 100
return f"{direction} {pct:.1f}% {period}"
def _parse_outcome_prices(market: Dict[str, Any]) -> List[tuple]:
"""Parse outcomePrices JSON string into list of (outcome_name, price) tuples."""
outcomes_raw = market.get("outcomes") or []
prices_raw = market.get("outcomePrices")
if not prices_raw:
return []
# Both outcomes and outcomePrices can be JSON-encoded strings
try:
if isinstance(outcomes_raw, str):
outcomes = json.loads(outcomes_raw)
else:
outcomes = outcomes_raw
except (json.JSONDecodeError, TypeError):
outcomes = []
try:
if isinstance(prices_raw, str):
prices = json.loads(prices_raw)
else:
prices = prices_raw
except (json.JSONDecodeError, TypeError):
return []
result = []
for i, price in enumerate(prices):
try:
p = float(price)
except (ValueError, TypeError):
continue
name = outcomes[i] if i < len(outcomes) else f"Outcome {i+1}"
result.append((name, p))
return result
def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List[Dict[str, Any]]:
"""Parse Gamma API response into normalized item dicts.
Each event becomes one item showing its title and top markets.
Args:
response: Raw Gamma API response
topic: Original search topic (for relevance scoring)
Returns:
List of item dicts ready for normalization.
"""
events = response.get("events", [])
items = []
for i, event in enumerate(events):
event_id = event.get("id", "")
title = event.get("title", "")
slug = event.get("slug", "")
# Filter: skip closed/resolved events
if event.get("closed", False):
continue
if not event.get("active", True):
continue
# Get markets for this event
markets = event.get("markets", [])
if not markets:
continue
# Filter to active, open markets with liquidity (excludes resolved markets)
active_markets = []
for m in markets:
if m.get("closed", False):
continue
if not m.get("active", True):
continue
# Must have liquidity (resolved markets have 0 or None)
try:
liq = float(m.get("liquidity", 0) or 0)
except (ValueError, TypeError):
liq = 0
if liq > 0:
active_markets.append(m)
if not active_markets:
continue
# Sort markets by volume (most liquid first)
def market_volume(m):
try:
return float(m.get("volume", 0) or 0)
except (ValueError, TypeError):
return 0
active_markets.sort(key=market_volume, reverse=True)
# Take top market for the event
top_market = active_markets[0]
# Parse outcome prices from top market
outcome_prices = _parse_outcome_prices(top_market)
# Format price movement
price_movement = _format_price_movement(top_market)
# Volume and liquidity
try:
volume24hr = float(top_market.get("volume24hr", 0) or 0)
except (ValueError, TypeError):
volume24hr = 0.0
try:
liquidity = float(top_market.get("liquidity", 0) or 0)
except (ValueError, TypeError):
liquidity = 0.0
# Event URL
url = f"https://polymarket.com/event/{slug}" if slug else f"https://polymarket.com/event/{event_id}"
# Date: use updatedAt from event
updated_at = event.get("updatedAt", "")
date_str = None
if updated_at:
# Parse ISO format: "2026-02-20T15:30:00.000Z"
try:
date_str = updated_at[:10] # YYYY-MM-DD
except (IndexError, TypeError):
pass
# End date for the market
end_date = top_market.get("endDate")
if end_date:
try:
end_date = end_date[:10]
except (IndexError, TypeError):
end_date = None
# Relevance: position-based decay
rank_score = max(0.3, 1.0 - (i * 0.03)) # 1.0 -> 0.3 over ~23 items
engagement_boost = min(0.15, math.log1p(volume24hr) / 60)
relevance = min(1.0, rank_score * 0.75 + engagement_boost + 0.1)
# Top 3 outcomes for multi-outcome markets
top_outcomes = outcome_prices[:3]
remaining = len(outcome_prices) - 3
if remaining < 0:
remaining = 0
items.append({
"event_id": event_id,
"title": title,
"question": top_market.get("question", title),
"url": url,
"outcome_prices": top_outcomes,
"outcomes_remaining": remaining,
"price_movement": price_movement,
"volume24hr": volume24hr,
"liquidity": liquidity,
"date": date_str,
"end_date": end_date,
"relevance": round(relevance, 2),
"why_relevant": f"Prediction market: {title[:60]}",
})
return items
+94 -9
View File
@@ -26,6 +26,8 @@ def _xref_tag(item) -> str:
source_names.add('YouTube')
elif ref_id.startswith('HN'):
source_names.add('HN')
elif ref_id.startswith('PM'):
source_names.add('Polymarket')
elif ref_id.startswith('W'):
source_names.add('Web')
if source_names:
@@ -53,9 +55,10 @@ def _assess_data_freshness(report: schema.Report) -> dict:
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from)
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
total_recent = reddit_recent + x_recent + web_recent + hn_recent
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews)
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket)
return {
"reddit_recent": reddit_recent,
@@ -276,6 +279,59 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append("")
# Polymarket items
if report.polymarket_error:
lines.append("### Prediction Markets (Polymarket)")
lines.append("")
lines.append(f"**ERROR:** {report.polymarket_error}")
lines.append("")
elif report.polymarket:
lines.append("### Prediction Markets (Polymarket)")
lines.append("")
for item in report.polymarket[:limit]:
eng_str = ""
if item.engagement:
eng = item.engagement
parts = []
if eng.volume is not None:
if eng.volume >= 1_000_000:
parts.append(f"${eng.volume/1_000_000:.1f}M vol24h")
elif eng.volume >= 1_000:
parts.append(f"${eng.volume/1_000:.0f}K vol24h")
else:
parts.append(f"${eng.volume:.0f} vol24h")
if eng.liquidity is not None:
if eng.liquidity >= 1_000_000:
parts.append(f"${eng.liquidity/1_000_000:.1f}M liquidity")
elif eng.liquidity >= 1_000:
parts.append(f"${eng.liquidity/1_000:.0f}K liquidity")
else:
parts.append(f"${eng.liquidity:.0f} liquidity")
if parts:
eng_str = f" [{', '.join(parts)}]"
date_str = f" ({item.date})" if item.date else ""
lines.append(f"**{item.id}** (score:{item.score}){eng_str}{_xref_tag(item)}")
lines.append(f" {item.question}")
# Outcome prices with price movement
if item.outcome_prices:
outcomes = []
for name, price in item.outcome_prices:
pct = price * 100
outcomes.append(f"{name}: {pct:.0f}%")
outcome_line = " | ".join(outcomes)
if item.outcomes_remaining > 0:
outcome_line += f" and {item.outcomes_remaining} more"
if item.price_movement:
outcome_line += f" ({item.price_movement})"
lines.append(f" {outcome_line}")
lines.append(f" {item.url}")
lines.append(f" *{item.why_relevant}*")
lines.append("")
# Web items (if any - populated by the assistant)
if report.web_error:
lines.append("### Web Results")
@@ -323,7 +379,7 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
elif report.reddit:
lines.append(f" ✅ Reddit: {len(report.reddit)} threads")
elif report.mode in ("both", "reddit-only", "all", "reddit-web"):
lines.append(" ⚠️ Reddit: 0 threads found")
pass # Hide zero-result sources
else:
reason = source_info.get("reddit_skip_reason", "not configured")
lines.append(f" ⏭️ Reddit: skipped — {reason}")
@@ -337,7 +393,7 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
x_line += f" (via @{report.resolved_x_handle} + keyword search)"
lines.append(x_line)
elif report.mode in ("both", "x-only", "all", "x-web"):
lines.append(" ⚠️ X: 0 posts found")
pass # Hide zero-result sources
else:
reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY")
lines.append(f" ⏭️ X: skipped — {reason}")
@@ -348,17 +404,21 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
elif report.youtube:
with_transcripts = sum(1 for v in report.youtube if getattr(v, 'transcript_snippet', None))
lines.append(f" ✅ YouTube: {len(report.youtube)} videos ({with_transcripts} with transcripts)")
else:
reason = source_info.get("youtube_skip_reason", "yt-dlp not installed (brew install yt-dlp)")
lines.append(f" ⏭️ YouTube: skipped — {reason}")
# Hide when zero results (no skip reason line needed)
# Hacker News
if report.hackernews_error:
lines.append(f" ❌ HN: error - {report.hackernews_error}")
elif report.hackernews:
lines.append(f" ✅ HN: {len(report.hackernews)} stories")
else:
lines.append(" ⏭️ HN: 0 stories found")
# Hide when zero results
# Polymarket
if report.polymarket_error:
lines.append(f" ❌ Polymarket: error - {report.polymarket_error}")
elif report.polymarket:
lines.append(f" ✅ Polymarket: {len(report.polymarket)} markets")
# Hide when zero results
# Web
if report.web_error:
@@ -399,6 +459,8 @@ def render_context_snippet(report: schema.Report) -> str:
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
for item in report.hackernews[:5]:
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
for item in report.polymarket[:5]:
all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url))
for item in report.web[:5]:
all_items.append((item.score, "Web", item.title[:50] + "...", item.url))
@@ -515,6 +577,29 @@ def render_full_report(report: schema.Report) -> str:
lines.append("")
# Polymarket section
if report.polymarket:
lines.append("## Prediction Markets (Polymarket)")
lines.append("")
for item in report.polymarket:
lines.append(f"### {item.id}: {item.question}")
lines.append("")
lines.append(f"- **Event:** {item.title}")
lines.append(f"- **URL:** {item.url}")
lines.append(f"- **Date:** {item.date or 'Unknown'}")
lines.append(f"- **Score:** {item.score}/100")
if item.outcome_prices:
outcomes = [f"{name}: {price*100:.0f}%" for name, price in item.outcome_prices]
lines.append(f"- **Outcomes:** {' | '.join(outcomes)}")
if item.price_movement:
lines.append(f"- **Trend:** {item.price_movement}")
if item.engagement:
eng = item.engagement
lines.append(f"- **Volume:** ${eng.volume or 0:,.0f} | Liquidity: ${eng.liquidity or 0:,.0f}")
lines.append("")
# Web section
if report.web:
lines.append("## Web Results")
+84
View File
@@ -22,6 +22,10 @@ class Engagement:
# YouTube fields
views: Optional[int] = None
# Polymarket fields
volume: Optional[float] = None
liquidity: Optional[float] = None
def to_dict(self) -> Dict[str, Any]:
d = {}
if self.score is not None:
@@ -40,6 +44,10 @@ class Engagement:
d['quotes'] = self.quotes
if self.views is not None:
d['views'] = self.views
if self.volume is not None:
d['volume'] = self.volume
if self.liquidity is not None:
d['liquidity'] = self.liquidity
return d if d else None
@@ -264,6 +272,49 @@ class HackerNewsItem:
return d
@dataclass
class PolymarketItem:
"""Normalized Polymarket prediction market item."""
id: str # "PM1", "PM2", ...
title: str # Event title
question: str # Top market question
url: str # Event page URL
outcome_prices: List[tuple] = field(default_factory=list) # [(name, price), ...]
outcomes_remaining: int = 0
price_movement: Optional[str] = None # "down 11.7% this month"
date: Optional[str] = None
date_confidence: str = "high" # API provides exact timestamps
engagement: Optional[Engagement] = None # volume + liquidity
end_date: Optional[str] = None
relevance: float = 0.5
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
d = {
'id': self.id,
'title': self.title,
'question': self.question,
'url': self.url,
'outcome_prices': self.outcome_prices,
'outcomes_remaining': self.outcomes_remaining,
'price_movement': self.price_movement,
'date': self.date,
'date_confidence': self.date_confidence,
'engagement': self.engagement.to_dict() if self.engagement else None,
'end_date': self.end_date,
'relevance': self.relevance,
'why_relevant': self.why_relevant,
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass
class Report:
"""Full research report."""
@@ -279,6 +330,7 @@ class Report:
web: List[WebSearchItem] = field(default_factory=list)
youtube: List[YouTubeItem] = field(default_factory=list)
hackernews: List[HackerNewsItem] = field(default_factory=list)
polymarket: List[PolymarketItem] = field(default_factory=list)
best_practices: List[str] = field(default_factory=list)
prompt_pack: List[str] = field(default_factory=list)
context_snippet_md: str = ""
@@ -288,6 +340,7 @@ class Report:
web_error: Optional[str] = None
youtube_error: Optional[str] = None
hackernews_error: Optional[str] = None
polymarket_error: Optional[str] = None
# Handle resolution
resolved_x_handle: Optional[str] = None
# Cache info
@@ -310,6 +363,7 @@ class Report:
'web': [w.to_dict() for w in self.web],
'youtube': [y.to_dict() for y in self.youtube],
'hackernews': [h.to_dict() for h in self.hackernews],
'polymarket': [p.to_dict() for p in self.polymarket],
'best_practices': self.best_practices,
'prompt_pack': self.prompt_pack,
'context_snippet_md': self.context_snippet_md,
@@ -326,6 +380,8 @@ class Report:
d['youtube_error'] = self.youtube_error
if self.hackernews_error:
d['hackernews_error'] = self.hackernews_error
if self.polymarket_error:
d['polymarket_error'] = self.polymarket_error
if self.from_cache:
d['from_cache'] = self.from_cache
if self.cache_age_hours is not None:
@@ -455,6 +511,32 @@ class Report:
cross_refs=h.get('cross_refs', []),
))
# Reconstruct Polymarket items (backward compat: key may not exist)
pm_items = []
for p in data.get('polymarket', []):
eng = None
if p.get('engagement'):
eng = Engagement(**p['engagement'])
subs = SubScores(**p.get('subs', {})) if p.get('subs') else SubScores()
pm_items.append(PolymarketItem(
id=p['id'],
title=p['title'],
question=p.get('question', ''),
url=p['url'],
outcome_prices=p.get('outcome_prices', []),
outcomes_remaining=p.get('outcomes_remaining', 0),
price_movement=p.get('price_movement'),
date=p.get('date'),
date_confidence=p.get('date_confidence', 'high'),
engagement=eng,
end_date=p.get('end_date'),
relevance=p.get('relevance', 0.5),
why_relevant=p.get('why_relevant', ''),
subs=subs,
score=p.get('score', 0),
cross_refs=p.get('cross_refs', []),
))
return cls(
topic=data['topic'],
range_from=range_from,
@@ -468,6 +550,7 @@ class Report:
web=web_items,
youtube=youtube_items,
hackernews=hn_items,
polymarket=pm_items,
best_practices=data.get('best_practices', []),
prompt_pack=data.get('prompt_pack', []),
context_snippet_md=data.get('context_snippet_md', ''),
@@ -476,6 +559,7 @@ class Report:
web_error=data.get('web_error'),
youtube_error=data.get('youtube_error'),
hackernews_error=data.get('hackernews_error'),
polymarket_error=data.get('polymarket_error'),
resolved_x_handle=data.get('resolved_x_handle'),
from_cache=data.get('from_cache', False),
cache_age_hours=data.get('cache_age_hours'),
+63 -3
View File
@@ -338,6 +338,64 @@ def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.Ha
return items
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Polymarket item.
Formula: 0.60*log1p(volume) + 0.40*log1p(liquidity)
Volume is the primary signal (money flowing); liquidity indicates market depth.
"""
if engagement is None:
return None
if engagement.volume is None and engagement.liquidity is None:
return None
volume = math.log1p(engagement.volume or 0)
liquidity = math.log1p(engagement.liquidity or 0)
return 0.60 * volume + 0.40 * liquidity
def score_polymarket_items(items: List[schema.PolymarketItem]) -> List[schema.PolymarketItem]:
"""Compute scores for Polymarket items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_polymarket_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
"""Compute scores for WebSearch items WITHOUT engagement metrics.
@@ -395,7 +453,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS
return items
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem]]) -> List:
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem, schema.PolymarketItem]]) -> List:
"""Sort items by score (descending), then date, then source priority.
Args:
@@ -412,7 +470,7 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
date = item.date or "0000-00-00"
date_key = -int(date.replace("-", ""))
# Tertiary: source priority (Reddit > X > YouTube > HN > WebSearch)
# Tertiary: source priority (Reddit > X > YouTube > HN > Polymarket > WebSearch)
if isinstance(item, schema.RedditItem):
source_priority = 0
elif isinstance(item, schema.XItem):
@@ -421,8 +479,10 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
source_priority = 2
elif isinstance(item, schema.HackerNewsItem):
source_priority = 3
else: # WebSearchItem
elif isinstance(item, schema.PolymarketItem):
source_priority = 4
else: # WebSearchItem
source_priority = 5
# Quaternary: title/text for stability
text = getattr(item, "title", "") or getattr(item, "text", "")
+21 -1
View File
@@ -79,6 +79,13 @@ HN_MESSAGES = [
"Discovering developer conversations...",
]
POLYMARKET_MESSAGES = [
"Checking prediction markets...",
"Finding what people are betting on...",
"Scanning Polymarket for odds...",
"Discovering prediction markets...",
]
PROCESSING_MESSAGES = [
"Crunching the data...",
"Scoring and ranking...",
@@ -274,6 +281,15 @@ class ProgressDisplay:
if self.spinner:
self.spinner.stop(f"{Colors.YELLOW}HN{Colors.RESET} Found {count} stories")
def start_polymarket(self):
msg = random.choice(POLYMARKET_MESSAGES)
self.spinner = Spinner(f"{Colors.GREEN}Polymarket{Colors.RESET} {msg}", Colors.GREEN, quiet=True)
self.spinner.start()
def end_polymarket(self, count: int):
if self.spinner:
self.spinner.stop(f"{Colors.GREEN}Polymarket{Colors.RESET} Found {count} markets")
def start_processing(self):
msg = random.choice(PROCESSING_MESSAGES)
self.spinner = Spinner(f"{Colors.PURPLE}Processing{Colors.RESET} {msg}", Colors.PURPLE)
@@ -283,7 +299,7 @@ class ProgressDisplay:
if self.spinner:
self.spinner.stop()
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0):
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0, pm_count: int = 0):
elapsed = time.time() - self.start_time
if IS_TTY:
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
@@ -294,6 +310,8 @@ class ProgressDisplay:
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
if hn_count:
sys.stderr.write(f" {Colors.YELLOW}HN:{Colors.RESET} {hn_count} stories")
if pm_count:
sys.stderr.write(f" {Colors.GREEN}Polymarket:{Colors.RESET} {pm_count} markets")
sys.stderr.write("\n\n")
else:
parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"]
@@ -301,6 +319,8 @@ class ProgressDisplay:
parts.append(f"YouTube: {youtube_count} videos")
if hn_count:
parts.append(f"HN: {hn_count} stories")
if pm_count:
parts.append(f"Polymarket: {pm_count} markets")
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n")
sys.stderr.flush()