feat(polymarket): add Polymarket prediction markets as 6th research source
Search Polymarket's free Gamma API for relevant prediction markets on any
topic. Uses smart multi-query expansion to cast a wider net (e.g., "Arizona
Basketball" also searches "Arizona"), merges and dedupes by event ID, and
shows price movement context ("up 22.5% this week"). No API key required.
Also hides sources with zero results from the stats output (all sources).
54 new tests, all passing. Full pipeline integration with scoring, dedupe,
cross-source linking, and rendering.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+87
-10
@@ -38,9 +38,9 @@ _child_pids: set = set()
|
||||
_child_pids_lock = threading.Lock()
|
||||
|
||||
TIMEOUT_PROFILES = {
|
||||
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "hackernews_future": 30, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "hackernews_future": 60, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||
}
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ from lib import (
|
||||
dates,
|
||||
dedupe,
|
||||
hackernews,
|
||||
polymarket,
|
||||
entity_extract,
|
||||
env,
|
||||
http,
|
||||
@@ -332,6 +333,34 @@ def _search_hackernews(
|
||||
return hn_items, hn_error
|
||||
|
||||
|
||||
def _search_polymarket(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str,
|
||||
) -> tuple:
|
||||
"""Search Polymarket via Gamma API (runs in thread).
|
||||
|
||||
Returns:
|
||||
Tuple of (pm_items, pm_error)
|
||||
"""
|
||||
pm_error = None
|
||||
|
||||
try:
|
||||
response = polymarket.search_polymarket(
|
||||
topic, from_date, to_date, depth=depth,
|
||||
)
|
||||
except Exception as e:
|
||||
return [], f"{type(e).__name__}: {e}"
|
||||
|
||||
pm_items = polymarket.parse_polymarket_response(response, topic=topic)
|
||||
|
||||
if response.get("error"):
|
||||
pm_error = response["error"]
|
||||
|
||||
return pm_items, pm_error
|
||||
|
||||
|
||||
def _search_web(
|
||||
topic: str,
|
||||
config: dict,
|
||||
@@ -588,6 +617,7 @@ def run_research(
|
||||
x_items = []
|
||||
youtube_items = []
|
||||
hackernews_items = []
|
||||
polymarket_items = []
|
||||
web_items = []
|
||||
raw_openai = None
|
||||
raw_xai = None
|
||||
@@ -596,6 +626,7 @@ def run_research(
|
||||
x_error = None
|
||||
youtube_error = None
|
||||
hackernews_error = None
|
||||
polymarket_error = None
|
||||
web_error = None
|
||||
|
||||
# Determine web search mode
|
||||
@@ -638,20 +669,22 @@ def run_research(
|
||||
progress.show_error(f"YouTube error: {e}")
|
||||
if progress:
|
||||
progress.end_youtube(len(youtube_items))
|
||||
return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error
|
||||
return reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error
|
||||
|
||||
# Determine which searches to run
|
||||
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
||||
do_x = sources in ("both", "x", "all", "x-web")
|
||||
do_hackernews = True # HN is always available (no API key)
|
||||
do_polymarket = True # Polymarket is always available (no API key)
|
||||
|
||||
# Run Reddit, X, YouTube, HN, and Web searches in parallel
|
||||
# Run Reddit, X, YouTube, HN, Polymarket, and Web searches in parallel
|
||||
reddit_future = None
|
||||
x_future = None
|
||||
youtube_future = None
|
||||
hackernews_future = None
|
||||
polymarket_future = None
|
||||
web_future = None
|
||||
max_workers = 2 + (1 if run_youtube else 0) + (1 if do_hackernews else 0) + (1 if web_backend else 0)
|
||||
max_workers = 2 + (1 if run_youtube else 0) + (1 if do_hackernews else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
# Submit searches
|
||||
@@ -685,6 +718,13 @@ def run_research(
|
||||
_search_hackernews, topic, from_date, to_date, depth
|
||||
)
|
||||
|
||||
if do_polymarket:
|
||||
if progress:
|
||||
progress.start_polymarket()
|
||||
polymarket_future = executor.submit(
|
||||
_search_polymarket, topic, from_date, to_date, depth
|
||||
)
|
||||
|
||||
if web_backend:
|
||||
sys.stderr.write(f"[web] Searching via {web_backend}\n")
|
||||
sys.stderr.flush()
|
||||
@@ -760,6 +800,23 @@ def run_research(
|
||||
if progress:
|
||||
progress.end_hackernews(len(hackernews_items))
|
||||
|
||||
if polymarket_future:
|
||||
pm_timeout = timeouts.get("polymarket_future", future_timeout)
|
||||
try:
|
||||
polymarket_items, polymarket_error = polymarket_future.result(timeout=pm_timeout)
|
||||
if polymarket_error and progress:
|
||||
progress.show_error(f"Polymarket error: {polymarket_error}")
|
||||
except TimeoutError:
|
||||
polymarket_error = f"Polymarket search timed out after {pm_timeout}s"
|
||||
if progress:
|
||||
progress.show_error(polymarket_error)
|
||||
except Exception as e:
|
||||
polymarket_error = f"{type(e).__name__}: {e}"
|
||||
if progress:
|
||||
progress.show_error(f"Polymarket error: {e}")
|
||||
if progress:
|
||||
progress.end_polymarket(len(polymarket_items))
|
||||
|
||||
if web_future:
|
||||
try:
|
||||
web_items, web_error = web_future.result(timeout=future_timeout)
|
||||
@@ -868,7 +925,7 @@ def run_research(
|
||||
if sup_x:
|
||||
x_items.extend(sup_x)
|
||||
|
||||
return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error
|
||||
return reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error
|
||||
|
||||
|
||||
def main():
|
||||
@@ -994,6 +1051,7 @@ def main():
|
||||
"bird_username": x_source_status.get("bird_username"),
|
||||
"youtube": has_ytdlp,
|
||||
"hackernews": True,
|
||||
"polymarket": True,
|
||||
"web_search_backend": web_source,
|
||||
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
|
||||
"brave": bool(config.get("BRAVE_API_KEY")),
|
||||
@@ -1022,6 +1080,7 @@ def main():
|
||||
"bird_username": x_source_status.get("bird_username"),
|
||||
"youtube": has_ytdlp,
|
||||
"hackernews": True,
|
||||
"polymarket": True,
|
||||
"web_search_backend": web_source,
|
||||
}
|
||||
ui.show_diagnostic_banner(diag)
|
||||
@@ -1099,7 +1158,7 @@ def main():
|
||||
mode = sources
|
||||
|
||||
# Run research
|
||||
reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error = run_research(
|
||||
reddit_items, x_items, youtube_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, polymarket_error, web_error = run_research(
|
||||
args.topic,
|
||||
sources,
|
||||
config,
|
||||
@@ -1123,6 +1182,7 @@ def main():
|
||||
normalized_x = normalize.normalize_x_items(x_items, from_date, to_date)
|
||||
normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else []
|
||||
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
|
||||
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
|
||||
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
|
||||
|
||||
# Hard date filter: exclude items with verified dates outside the range
|
||||
@@ -1134,6 +1194,8 @@ def main():
|
||||
# YouTube content has a longer shelf life than tweets/posts.
|
||||
filtered_youtube = normalized_youtube
|
||||
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
|
||||
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
|
||||
filtered_pm = normalized_pm
|
||||
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
|
||||
|
||||
# Score items
|
||||
@@ -1141,6 +1203,7 @@ def main():
|
||||
scored_x = score.score_x_items(filtered_x)
|
||||
scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else []
|
||||
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
|
||||
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
|
||||
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
|
||||
|
||||
# Sort items
|
||||
@@ -1148,6 +1211,7 @@ def main():
|
||||
sorted_x = score.sort_items(scored_x)
|
||||
sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else []
|
||||
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
|
||||
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
|
||||
sorted_web = score.sort_items(scored_web) if scored_web else []
|
||||
|
||||
# Dedupe items
|
||||
@@ -1155,6 +1219,7 @@ def main():
|
||||
deduped_x = dedupe.dedupe_x(sorted_x)
|
||||
deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else []
|
||||
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
|
||||
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
|
||||
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
|
||||
|
||||
# Minimum result guarantee: if all Reddit results were filtered out but
|
||||
@@ -1166,7 +1231,7 @@ def main():
|
||||
|
||||
# Cross-source linking: annotate items that discuss the same story
|
||||
dedupe.cross_source_link(
|
||||
deduped_reddit, deduped_x, deduped_youtube, deduped_hn, deduped_web,
|
||||
deduped_reddit, deduped_x, deduped_youtube, deduped_hn, deduped_pm, deduped_web,
|
||||
)
|
||||
|
||||
progress.end_processing()
|
||||
@@ -1184,11 +1249,13 @@ def main():
|
||||
report.x = deduped_x
|
||||
report.youtube = deduped_youtube
|
||||
report.hackernews = deduped_hn
|
||||
report.polymarket = deduped_pm
|
||||
report.web = deduped_web
|
||||
report.reddit_error = reddit_error
|
||||
report.x_error = x_error
|
||||
report.youtube_error = youtube_error
|
||||
report.hackernews_error = hackernews_error
|
||||
report.polymarket_error = polymarket_error
|
||||
report.web_error = web_error
|
||||
report.resolved_x_handle = args.x_handle
|
||||
|
||||
@@ -1202,7 +1269,7 @@ def main():
|
||||
if sources == "web":
|
||||
progress.show_web_only_complete()
|
||||
else:
|
||||
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn))
|
||||
progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn), len(deduped_pm))
|
||||
|
||||
# Build source info for status footer
|
||||
source_info = {}
|
||||
@@ -1270,6 +1337,16 @@ def main():
|
||||
"engagement_score": item.engagement.score if item.engagement else 0,
|
||||
"relevance_score": item.relevance,
|
||||
})
|
||||
for item in deduped_pm:
|
||||
findings.append({
|
||||
"source": "polymarket",
|
||||
"url": item.url,
|
||||
"title": item.question,
|
||||
"author": "polymarket",
|
||||
"content": item.title,
|
||||
"engagement_score": item.engagement.volume if item.engagement and item.engagement.volume else 0,
|
||||
"relevance_score": item.relevance,
|
||||
})
|
||||
for item in deduped_web:
|
||||
findings.append({
|
||||
"source": "web",
|
||||
|
||||
+13
-1
@@ -46,7 +46,7 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float:
|
||||
|
||||
|
||||
AnyItem = Union[schema.RedditItem, schema.XItem, schema.YouTubeItem,
|
||||
schema.HackerNewsItem, schema.WebSearchItem]
|
||||
schema.HackerNewsItem, schema.PolymarketItem, schema.WebSearchItem]
|
||||
|
||||
|
||||
def get_item_text(item: AnyItem) -> str:
|
||||
@@ -57,6 +57,8 @@ def get_item_text(item: AnyItem) -> str:
|
||||
return item.title
|
||||
elif isinstance(item, schema.YouTubeItem):
|
||||
return f"{item.title} {item.channel_name}"
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
return f"{item.title} {item.question}"
|
||||
elif isinstance(item, schema.WebSearchItem):
|
||||
return item.title
|
||||
else:
|
||||
@@ -79,6 +81,8 @@ def _get_cross_source_text(item: AnyItem) -> str:
|
||||
elif title.startswith("Ask HN:"):
|
||||
title = title[7:].strip()
|
||||
return title
|
||||
if isinstance(item, schema.PolymarketItem):
|
||||
return item.title
|
||||
return get_item_text(item)
|
||||
|
||||
|
||||
@@ -198,6 +202,14 @@ def dedupe_hackernews(
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def dedupe_polymarket(
|
||||
items: List[schema.PolymarketItem],
|
||||
threshold: float = 0.7,
|
||||
) -> List[schema.PolymarketItem]:
|
||||
"""Dedupe Polymarket items."""
|
||||
return dedupe_items(items, threshold)
|
||||
|
||||
|
||||
def cross_source_link(
|
||||
*source_lists: List[AnyItem],
|
||||
threshold: float = 0.40,
|
||||
|
||||
@@ -254,6 +254,14 @@ def is_hackernews_available() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def is_polymarket_available() -> bool:
|
||||
"""Check if Polymarket source is available.
|
||||
|
||||
Always returns True - Gamma API is free, no key needed.
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Get detailed X source status for UI decisions.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem, schema.PolymarketItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -257,6 +257,50 @@ def normalize_hackernews_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_polymarket_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.PolymarketItem]:
|
||||
"""Normalize raw Polymarket items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw Polymarket items from Gamma API
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of PolymarketItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for i, item in enumerate(items):
|
||||
engagement = schema.Engagement(
|
||||
volume=item.get("volume24hr", 0.0),
|
||||
liquidity=item.get("liquidity", 0.0),
|
||||
)
|
||||
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.PolymarketItem(
|
||||
id=f"PM{i+1}",
|
||||
title=item.get("title", ""),
|
||||
question=item.get("question", ""),
|
||||
url=item.get("url", ""),
|
||||
outcome_prices=item.get("outcome_prices", []),
|
||||
outcomes_remaining=item.get("outcomes_remaining", 0),
|
||||
price_movement=item.get("price_movement"),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
end_date=item.get("end_date"),
|
||||
relevance=item.get("relevance", 0.5),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
|
||||
"""Convert schema items to dicts for JSON serialization."""
|
||||
return [item.to_dict() for item in items]
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
"""Polymarket prediction market search via Gamma API (free, no auth required).
|
||||
|
||||
Uses gamma-api.polymarket.com for event/market discovery.
|
||||
No API key needed - public read-only API with generous rate limits (350 req/10s).
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import quote_plus, urlencode
|
||||
|
||||
from . import http
|
||||
|
||||
GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search"
|
||||
|
||||
DEPTH_CONFIG = {
|
||||
"quick": 5,
|
||||
"default": 10,
|
||||
"deep": 20,
|
||||
}
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
|
||||
if sys.stderr.isatty():
|
||||
sys.stderr.write(f"[PM] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def _extract_core_subject(topic: str) -> str:
|
||||
"""Extract core subject from topic string.
|
||||
|
||||
Strips common prefixes like 'last 7 days', 'what are people saying about', etc.
|
||||
"""
|
||||
topic = topic.strip()
|
||||
# Remove common leading phrases
|
||||
prefixes = [
|
||||
r"^last \d+ days?\s+",
|
||||
r"^what(?:'s| is| are) (?:people saying about|happening with|going on with)\s+",
|
||||
r"^how (?:is|are)\s+",
|
||||
r"^tell me about\s+",
|
||||
r"^research\s+",
|
||||
]
|
||||
for pattern in prefixes:
|
||||
topic = re.sub(pattern, "", topic, flags=re.IGNORECASE)
|
||||
return topic.strip()
|
||||
|
||||
|
||||
def _expand_queries(topic: str) -> List[str]:
|
||||
"""Generate 2-4 search queries to cast a wider net.
|
||||
|
||||
Strategy:
|
||||
- Always include the core subject
|
||||
- Split multi-word topics into component searches
|
||||
- Include the full topic if different from core
|
||||
- Cap at 4 queries, dedupe
|
||||
"""
|
||||
core = _extract_core_subject(topic)
|
||||
queries = [core]
|
||||
|
||||
# Split multi-word topics into component searches
|
||||
words = core.split()
|
||||
if len(words) >= 2:
|
||||
# Try the first significant word alone (e.g., "Arizona" from "Arizona Basketball")
|
||||
queries.append(words[0])
|
||||
|
||||
# Add the full topic if different from core
|
||||
if topic.lower().strip() != core.lower():
|
||||
queries.append(topic.strip())
|
||||
|
||||
# Dedupe while preserving order, cap at 4
|
||||
seen = set()
|
||||
unique = []
|
||||
for q in queries:
|
||||
q_lower = q.lower().strip()
|
||||
if q_lower and q_lower not in seen:
|
||||
seen.add(q_lower)
|
||||
unique.append(q.strip())
|
||||
return unique[:4]
|
||||
|
||||
|
||||
def _search_single_query(query: str, limit: int) -> Dict[str, Any]:
|
||||
"""Run a single search query against Gamma API."""
|
||||
params = {
|
||||
"q": query,
|
||||
"limit": str(limit),
|
||||
}
|
||||
url = f"{GAMMA_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
try:
|
||||
response = http.request("GET", url, timeout=15, retries=2)
|
||||
return response
|
||||
except http.HTTPError as e:
|
||||
_log(f"Search failed for '{query}': {e}")
|
||||
return {"events": [], "error": str(e)}
|
||||
except Exception as e:
|
||||
_log(f"Search failed for '{query}': {e}")
|
||||
return {"events": [], "error": str(e)}
|
||||
|
||||
|
||||
def search_polymarket(
|
||||
topic: str,
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
depth: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""Search Polymarket via Gamma API with smart query expansion.
|
||||
|
||||
Runs 2-4 expanded queries in parallel, merges and dedupes by event ID.
|
||||
|
||||
Args:
|
||||
topic: Search topic
|
||||
from_date: Start date (YYYY-MM-DD) - used for activity filtering
|
||||
to_date: End date (YYYY-MM-DD)
|
||||
depth: 'quick', 'default', or 'deep'
|
||||
|
||||
Returns:
|
||||
Dict with 'events' list and optional 'error'.
|
||||
"""
|
||||
limit_per_query = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||
queries = _expand_queries(topic)
|
||||
|
||||
_log(f"Searching for '{topic}' with queries: {queries} (limit={limit_per_query})")
|
||||
|
||||
# Run all queries in parallel
|
||||
all_events = {} # event_id -> (event_data, query_index)
|
||||
errors = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(4, len(queries))) as executor:
|
||||
futures = {
|
||||
executor.submit(_search_single_query, q, limit_per_query): i
|
||||
for i, q in enumerate(queries)
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
query_idx = futures[future]
|
||||
try:
|
||||
response = future.result(timeout=15)
|
||||
if response.get("error"):
|
||||
errors.append(response["error"])
|
||||
|
||||
events = response.get("events", [])
|
||||
for event in events:
|
||||
event_id = event.get("id", "")
|
||||
if not event_id:
|
||||
continue
|
||||
# Keep the first occurrence (from highest-priority query)
|
||||
if event_id not in all_events:
|
||||
all_events[event_id] = (event, query_idx)
|
||||
elif query_idx < all_events[event_id][1]:
|
||||
# Replace with higher-priority query result
|
||||
all_events[event_id] = (event, query_idx)
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
|
||||
# Sort by query priority, then by position
|
||||
merged_events = [ev for ev, _ in sorted(all_events.values(), key=lambda x: x[1])]
|
||||
_log(f"Found {len(merged_events)} unique events across {len(queries)} queries")
|
||||
|
||||
result = {"events": merged_events}
|
||||
if errors and not merged_events:
|
||||
result["error"] = "; ".join(errors[:2])
|
||||
return result
|
||||
|
||||
|
||||
def _format_price_movement(market: Dict[str, Any]) -> Optional[str]:
|
||||
"""Pick the most significant price change and format it.
|
||||
|
||||
Returns string like 'down 11.7% this month' or None if no significant change.
|
||||
"""
|
||||
changes = [
|
||||
(abs(market.get("oneDayPriceChange") or 0), market.get("oneDayPriceChange"), "today"),
|
||||
(abs(market.get("oneWeekPriceChange") or 0), market.get("oneWeekPriceChange"), "this week"),
|
||||
(abs(market.get("oneMonthPriceChange") or 0), market.get("oneMonthPriceChange"), "this month"),
|
||||
]
|
||||
|
||||
# Pick the largest absolute change
|
||||
changes.sort(key=lambda x: x[0], reverse=True)
|
||||
abs_change, raw_change, period = changes[0]
|
||||
|
||||
# Skip if change is less than 1% (noise)
|
||||
if abs_change < 0.01:
|
||||
return None
|
||||
|
||||
direction = "up" if raw_change > 0 else "down"
|
||||
pct = abs_change * 100
|
||||
return f"{direction} {pct:.1f}% {period}"
|
||||
|
||||
|
||||
def _parse_outcome_prices(market: Dict[str, Any]) -> List[tuple]:
|
||||
"""Parse outcomePrices JSON string into list of (outcome_name, price) tuples."""
|
||||
outcomes_raw = market.get("outcomes") or []
|
||||
prices_raw = market.get("outcomePrices")
|
||||
|
||||
if not prices_raw:
|
||||
return []
|
||||
|
||||
# Both outcomes and outcomePrices can be JSON-encoded strings
|
||||
try:
|
||||
if isinstance(outcomes_raw, str):
|
||||
outcomes = json.loads(outcomes_raw)
|
||||
else:
|
||||
outcomes = outcomes_raw
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
outcomes = []
|
||||
|
||||
try:
|
||||
if isinstance(prices_raw, str):
|
||||
prices = json.loads(prices_raw)
|
||||
else:
|
||||
prices = prices_raw
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return []
|
||||
|
||||
result = []
|
||||
for i, price in enumerate(prices):
|
||||
try:
|
||||
p = float(price)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
name = outcomes[i] if i < len(outcomes) else f"Outcome {i+1}"
|
||||
result.append((name, p))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parse_polymarket_response(response: Dict[str, Any], topic: str = "") -> List[Dict[str, Any]]:
|
||||
"""Parse Gamma API response into normalized item dicts.
|
||||
|
||||
Each event becomes one item showing its title and top markets.
|
||||
|
||||
Args:
|
||||
response: Raw Gamma API response
|
||||
topic: Original search topic (for relevance scoring)
|
||||
|
||||
Returns:
|
||||
List of item dicts ready for normalization.
|
||||
"""
|
||||
events = response.get("events", [])
|
||||
items = []
|
||||
|
||||
for i, event in enumerate(events):
|
||||
event_id = event.get("id", "")
|
||||
title = event.get("title", "")
|
||||
slug = event.get("slug", "")
|
||||
|
||||
# Filter: skip closed/resolved events
|
||||
if event.get("closed", False):
|
||||
continue
|
||||
if not event.get("active", True):
|
||||
continue
|
||||
|
||||
# Get markets for this event
|
||||
markets = event.get("markets", [])
|
||||
if not markets:
|
||||
continue
|
||||
|
||||
# Filter to active, open markets with liquidity (excludes resolved markets)
|
||||
active_markets = []
|
||||
for m in markets:
|
||||
if m.get("closed", False):
|
||||
continue
|
||||
if not m.get("active", True):
|
||||
continue
|
||||
# Must have liquidity (resolved markets have 0 or None)
|
||||
try:
|
||||
liq = float(m.get("liquidity", 0) or 0)
|
||||
except (ValueError, TypeError):
|
||||
liq = 0
|
||||
if liq > 0:
|
||||
active_markets.append(m)
|
||||
|
||||
if not active_markets:
|
||||
continue
|
||||
|
||||
# Sort markets by volume (most liquid first)
|
||||
def market_volume(m):
|
||||
try:
|
||||
return float(m.get("volume", 0) or 0)
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
active_markets.sort(key=market_volume, reverse=True)
|
||||
|
||||
# Take top market for the event
|
||||
top_market = active_markets[0]
|
||||
|
||||
# Parse outcome prices from top market
|
||||
outcome_prices = _parse_outcome_prices(top_market)
|
||||
|
||||
# Format price movement
|
||||
price_movement = _format_price_movement(top_market)
|
||||
|
||||
# Volume and liquidity
|
||||
try:
|
||||
volume24hr = float(top_market.get("volume24hr", 0) or 0)
|
||||
except (ValueError, TypeError):
|
||||
volume24hr = 0.0
|
||||
try:
|
||||
liquidity = float(top_market.get("liquidity", 0) or 0)
|
||||
except (ValueError, TypeError):
|
||||
liquidity = 0.0
|
||||
|
||||
# Event URL
|
||||
url = f"https://polymarket.com/event/{slug}" if slug else f"https://polymarket.com/event/{event_id}"
|
||||
|
||||
# Date: use updatedAt from event
|
||||
updated_at = event.get("updatedAt", "")
|
||||
date_str = None
|
||||
if updated_at:
|
||||
# Parse ISO format: "2026-02-20T15:30:00.000Z"
|
||||
try:
|
||||
date_str = updated_at[:10] # YYYY-MM-DD
|
||||
except (IndexError, TypeError):
|
||||
pass
|
||||
|
||||
# End date for the market
|
||||
end_date = top_market.get("endDate")
|
||||
if end_date:
|
||||
try:
|
||||
end_date = end_date[:10]
|
||||
except (IndexError, TypeError):
|
||||
end_date = None
|
||||
|
||||
# Relevance: position-based decay
|
||||
rank_score = max(0.3, 1.0 - (i * 0.03)) # 1.0 -> 0.3 over ~23 items
|
||||
engagement_boost = min(0.15, math.log1p(volume24hr) / 60)
|
||||
relevance = min(1.0, rank_score * 0.75 + engagement_boost + 0.1)
|
||||
|
||||
# Top 3 outcomes for multi-outcome markets
|
||||
top_outcomes = outcome_prices[:3]
|
||||
remaining = len(outcome_prices) - 3
|
||||
if remaining < 0:
|
||||
remaining = 0
|
||||
|
||||
items.append({
|
||||
"event_id": event_id,
|
||||
"title": title,
|
||||
"question": top_market.get("question", title),
|
||||
"url": url,
|
||||
"outcome_prices": top_outcomes,
|
||||
"outcomes_remaining": remaining,
|
||||
"price_movement": price_movement,
|
||||
"volume24hr": volume24hr,
|
||||
"liquidity": liquidity,
|
||||
"date": date_str,
|
||||
"end_date": end_date,
|
||||
"relevance": round(relevance, 2),
|
||||
"why_relevant": f"Prediction market: {title[:60]}",
|
||||
})
|
||||
|
||||
return items
|
||||
+94
-9
@@ -26,6 +26,8 @@ def _xref_tag(item) -> str:
|
||||
source_names.add('YouTube')
|
||||
elif ref_id.startswith('HN'):
|
||||
source_names.add('HN')
|
||||
elif ref_id.startswith('PM'):
|
||||
source_names.add('Polymarket')
|
||||
elif ref_id.startswith('W'):
|
||||
source_names.add('Web')
|
||||
if source_names:
|
||||
@@ -53,9 +55,10 @@ def _assess_data_freshness(report: schema.Report) -> dict:
|
||||
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
|
||||
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
|
||||
hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from)
|
||||
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
|
||||
|
||||
total_recent = reddit_recent + x_recent + web_recent + hn_recent
|
||||
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews)
|
||||
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent
|
||||
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket)
|
||||
|
||||
return {
|
||||
"reddit_recent": reddit_recent,
|
||||
@@ -276,6 +279,59 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Polymarket items
|
||||
if report.polymarket_error:
|
||||
lines.append("### Prediction Markets (Polymarket)")
|
||||
lines.append("")
|
||||
lines.append(f"**ERROR:** {report.polymarket_error}")
|
||||
lines.append("")
|
||||
elif report.polymarket:
|
||||
lines.append("### Prediction Markets (Polymarket)")
|
||||
lines.append("")
|
||||
for item in report.polymarket[:limit]:
|
||||
eng_str = ""
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
parts = []
|
||||
if eng.volume is not None:
|
||||
if eng.volume >= 1_000_000:
|
||||
parts.append(f"${eng.volume/1_000_000:.1f}M vol24h")
|
||||
elif eng.volume >= 1_000:
|
||||
parts.append(f"${eng.volume/1_000:.0f}K vol24h")
|
||||
else:
|
||||
parts.append(f"${eng.volume:.0f} vol24h")
|
||||
if eng.liquidity is not None:
|
||||
if eng.liquidity >= 1_000_000:
|
||||
parts.append(f"${eng.liquidity/1_000_000:.1f}M liquidity")
|
||||
elif eng.liquidity >= 1_000:
|
||||
parts.append(f"${eng.liquidity/1_000:.0f}K liquidity")
|
||||
else:
|
||||
parts.append(f"${eng.liquidity:.0f} liquidity")
|
||||
if parts:
|
||||
eng_str = f" [{', '.join(parts)}]"
|
||||
|
||||
date_str = f" ({item.date})" if item.date else ""
|
||||
|
||||
lines.append(f"**{item.id}** (score:{item.score}){eng_str}{_xref_tag(item)}")
|
||||
lines.append(f" {item.question}")
|
||||
|
||||
# Outcome prices with price movement
|
||||
if item.outcome_prices:
|
||||
outcomes = []
|
||||
for name, price in item.outcome_prices:
|
||||
pct = price * 100
|
||||
outcomes.append(f"{name}: {pct:.0f}%")
|
||||
outcome_line = " | ".join(outcomes)
|
||||
if item.outcomes_remaining > 0:
|
||||
outcome_line += f" and {item.outcomes_remaining} more"
|
||||
if item.price_movement:
|
||||
outcome_line += f" ({item.price_movement})"
|
||||
lines.append(f" {outcome_line}")
|
||||
|
||||
lines.append(f" {item.url}")
|
||||
lines.append(f" *{item.why_relevant}*")
|
||||
lines.append("")
|
||||
|
||||
# Web items (if any - populated by the assistant)
|
||||
if report.web_error:
|
||||
lines.append("### Web Results")
|
||||
@@ -323,7 +379,7 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
|
||||
elif report.reddit:
|
||||
lines.append(f" ✅ Reddit: {len(report.reddit)} threads")
|
||||
elif report.mode in ("both", "reddit-only", "all", "reddit-web"):
|
||||
lines.append(" ⚠️ Reddit: 0 threads found")
|
||||
pass # Hide zero-result sources
|
||||
else:
|
||||
reason = source_info.get("reddit_skip_reason", "not configured")
|
||||
lines.append(f" ⏭️ Reddit: skipped — {reason}")
|
||||
@@ -337,7 +393,7 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
|
||||
x_line += f" (via @{report.resolved_x_handle} + keyword search)"
|
||||
lines.append(x_line)
|
||||
elif report.mode in ("both", "x-only", "all", "x-web"):
|
||||
lines.append(" ⚠️ X: 0 posts found")
|
||||
pass # Hide zero-result sources
|
||||
else:
|
||||
reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY")
|
||||
lines.append(f" ⏭️ X: skipped — {reason}")
|
||||
@@ -348,17 +404,21 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
|
||||
elif report.youtube:
|
||||
with_transcripts = sum(1 for v in report.youtube if getattr(v, 'transcript_snippet', None))
|
||||
lines.append(f" ✅ YouTube: {len(report.youtube)} videos ({with_transcripts} with transcripts)")
|
||||
else:
|
||||
reason = source_info.get("youtube_skip_reason", "yt-dlp not installed (brew install yt-dlp)")
|
||||
lines.append(f" ⏭️ YouTube: skipped — {reason}")
|
||||
# Hide when zero results (no skip reason line needed)
|
||||
|
||||
# Hacker News
|
||||
if report.hackernews_error:
|
||||
lines.append(f" ❌ HN: error - {report.hackernews_error}")
|
||||
elif report.hackernews:
|
||||
lines.append(f" ✅ HN: {len(report.hackernews)} stories")
|
||||
else:
|
||||
lines.append(" ⏭️ HN: 0 stories found")
|
||||
# Hide when zero results
|
||||
|
||||
# Polymarket
|
||||
if report.polymarket_error:
|
||||
lines.append(f" ❌ Polymarket: error - {report.polymarket_error}")
|
||||
elif report.polymarket:
|
||||
lines.append(f" ✅ Polymarket: {len(report.polymarket)} markets")
|
||||
# Hide when zero results
|
||||
|
||||
# Web
|
||||
if report.web_error:
|
||||
@@ -399,6 +459,8 @@ def render_context_snippet(report: schema.Report) -> str:
|
||||
all_items.append((item.score, "X", item.text[:50] + "...", item.url))
|
||||
for item in report.hackernews[:5]:
|
||||
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
|
||||
for item in report.polymarket[:5]:
|
||||
all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url))
|
||||
for item in report.web[:5]:
|
||||
all_items.append((item.score, "Web", item.title[:50] + "...", item.url))
|
||||
|
||||
@@ -515,6 +577,29 @@ def render_full_report(report: schema.Report) -> str:
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Polymarket section
|
||||
if report.polymarket:
|
||||
lines.append("## Prediction Markets (Polymarket)")
|
||||
lines.append("")
|
||||
for item in report.polymarket:
|
||||
lines.append(f"### {item.id}: {item.question}")
|
||||
lines.append("")
|
||||
lines.append(f"- **Event:** {item.title}")
|
||||
lines.append(f"- **URL:** {item.url}")
|
||||
lines.append(f"- **Date:** {item.date or 'Unknown'}")
|
||||
lines.append(f"- **Score:** {item.score}/100")
|
||||
|
||||
if item.outcome_prices:
|
||||
outcomes = [f"{name}: {price*100:.0f}%" for name, price in item.outcome_prices]
|
||||
lines.append(f"- **Outcomes:** {' | '.join(outcomes)}")
|
||||
if item.price_movement:
|
||||
lines.append(f"- **Trend:** {item.price_movement}")
|
||||
if item.engagement:
|
||||
eng = item.engagement
|
||||
lines.append(f"- **Volume:** ${eng.volume or 0:,.0f} | Liquidity: ${eng.liquidity or 0:,.0f}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Web section
|
||||
if report.web:
|
||||
lines.append("## Web Results")
|
||||
|
||||
@@ -22,6 +22,10 @@ class Engagement:
|
||||
# YouTube fields
|
||||
views: Optional[int] = None
|
||||
|
||||
# Polymarket fields
|
||||
volume: Optional[float] = None
|
||||
liquidity: Optional[float] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {}
|
||||
if self.score is not None:
|
||||
@@ -40,6 +44,10 @@ class Engagement:
|
||||
d['quotes'] = self.quotes
|
||||
if self.views is not None:
|
||||
d['views'] = self.views
|
||||
if self.volume is not None:
|
||||
d['volume'] = self.volume
|
||||
if self.liquidity is not None:
|
||||
d['liquidity'] = self.liquidity
|
||||
return d if d else None
|
||||
|
||||
|
||||
@@ -264,6 +272,49 @@ class HackerNewsItem:
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class PolymarketItem:
|
||||
"""Normalized Polymarket prediction market item."""
|
||||
id: str # "PM1", "PM2", ...
|
||||
title: str # Event title
|
||||
question: str # Top market question
|
||||
url: str # Event page URL
|
||||
outcome_prices: List[tuple] = field(default_factory=list) # [(name, price), ...]
|
||||
outcomes_remaining: int = 0
|
||||
price_movement: Optional[str] = None # "down 11.7% this month"
|
||||
date: Optional[str] = None
|
||||
date_confidence: str = "high" # API provides exact timestamps
|
||||
engagement: Optional[Engagement] = None # volume + liquidity
|
||||
end_date: Optional[str] = None
|
||||
relevance: float = 0.5
|
||||
why_relevant: str = ""
|
||||
subs: SubScores = field(default_factory=SubScores)
|
||||
score: int = 0
|
||||
cross_refs: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
d = {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'question': self.question,
|
||||
'url': self.url,
|
||||
'outcome_prices': self.outcome_prices,
|
||||
'outcomes_remaining': self.outcomes_remaining,
|
||||
'price_movement': self.price_movement,
|
||||
'date': self.date,
|
||||
'date_confidence': self.date_confidence,
|
||||
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||
'end_date': self.end_date,
|
||||
'relevance': self.relevance,
|
||||
'why_relevant': self.why_relevant,
|
||||
'subs': self.subs.to_dict(),
|
||||
'score': self.score,
|
||||
}
|
||||
if self.cross_refs:
|
||||
d['cross_refs'] = self.cross_refs
|
||||
return d
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""Full research report."""
|
||||
@@ -279,6 +330,7 @@ class Report:
|
||||
web: List[WebSearchItem] = field(default_factory=list)
|
||||
youtube: List[YouTubeItem] = field(default_factory=list)
|
||||
hackernews: List[HackerNewsItem] = field(default_factory=list)
|
||||
polymarket: List[PolymarketItem] = field(default_factory=list)
|
||||
best_practices: List[str] = field(default_factory=list)
|
||||
prompt_pack: List[str] = field(default_factory=list)
|
||||
context_snippet_md: str = ""
|
||||
@@ -288,6 +340,7 @@ class Report:
|
||||
web_error: Optional[str] = None
|
||||
youtube_error: Optional[str] = None
|
||||
hackernews_error: Optional[str] = None
|
||||
polymarket_error: Optional[str] = None
|
||||
# Handle resolution
|
||||
resolved_x_handle: Optional[str] = None
|
||||
# Cache info
|
||||
@@ -310,6 +363,7 @@ class Report:
|
||||
'web': [w.to_dict() for w in self.web],
|
||||
'youtube': [y.to_dict() for y in self.youtube],
|
||||
'hackernews': [h.to_dict() for h in self.hackernews],
|
||||
'polymarket': [p.to_dict() for p in self.polymarket],
|
||||
'best_practices': self.best_practices,
|
||||
'prompt_pack': self.prompt_pack,
|
||||
'context_snippet_md': self.context_snippet_md,
|
||||
@@ -326,6 +380,8 @@ class Report:
|
||||
d['youtube_error'] = self.youtube_error
|
||||
if self.hackernews_error:
|
||||
d['hackernews_error'] = self.hackernews_error
|
||||
if self.polymarket_error:
|
||||
d['polymarket_error'] = self.polymarket_error
|
||||
if self.from_cache:
|
||||
d['from_cache'] = self.from_cache
|
||||
if self.cache_age_hours is not None:
|
||||
@@ -455,6 +511,32 @@ class Report:
|
||||
cross_refs=h.get('cross_refs', []),
|
||||
))
|
||||
|
||||
# Reconstruct Polymarket items (backward compat: key may not exist)
|
||||
pm_items = []
|
||||
for p in data.get('polymarket', []):
|
||||
eng = None
|
||||
if p.get('engagement'):
|
||||
eng = Engagement(**p['engagement'])
|
||||
subs = SubScores(**p.get('subs', {})) if p.get('subs') else SubScores()
|
||||
pm_items.append(PolymarketItem(
|
||||
id=p['id'],
|
||||
title=p['title'],
|
||||
question=p.get('question', ''),
|
||||
url=p['url'],
|
||||
outcome_prices=p.get('outcome_prices', []),
|
||||
outcomes_remaining=p.get('outcomes_remaining', 0),
|
||||
price_movement=p.get('price_movement'),
|
||||
date=p.get('date'),
|
||||
date_confidence=p.get('date_confidence', 'high'),
|
||||
engagement=eng,
|
||||
end_date=p.get('end_date'),
|
||||
relevance=p.get('relevance', 0.5),
|
||||
why_relevant=p.get('why_relevant', ''),
|
||||
subs=subs,
|
||||
score=p.get('score', 0),
|
||||
cross_refs=p.get('cross_refs', []),
|
||||
))
|
||||
|
||||
return cls(
|
||||
topic=data['topic'],
|
||||
range_from=range_from,
|
||||
@@ -468,6 +550,7 @@ class Report:
|
||||
web=web_items,
|
||||
youtube=youtube_items,
|
||||
hackernews=hn_items,
|
||||
polymarket=pm_items,
|
||||
best_practices=data.get('best_practices', []),
|
||||
prompt_pack=data.get('prompt_pack', []),
|
||||
context_snippet_md=data.get('context_snippet_md', ''),
|
||||
@@ -476,6 +559,7 @@ class Report:
|
||||
web_error=data.get('web_error'),
|
||||
youtube_error=data.get('youtube_error'),
|
||||
hackernews_error=data.get('hackernews_error'),
|
||||
polymarket_error=data.get('polymarket_error'),
|
||||
resolved_x_handle=data.get('resolved_x_handle'),
|
||||
from_cache=data.get('from_cache', False),
|
||||
cache_age_hours=data.get('cache_age_hours'),
|
||||
|
||||
+63
-3
@@ -338,6 +338,64 @@ def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.Ha
|
||||
return items
|
||||
|
||||
|
||||
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||
"""Compute raw engagement score for Polymarket item.
|
||||
|
||||
Formula: 0.60*log1p(volume) + 0.40*log1p(liquidity)
|
||||
Volume is the primary signal (money flowing); liquidity indicates market depth.
|
||||
"""
|
||||
if engagement is None:
|
||||
return None
|
||||
|
||||
if engagement.volume is None and engagement.liquidity is None:
|
||||
return None
|
||||
|
||||
volume = math.log1p(engagement.volume or 0)
|
||||
liquidity = math.log1p(engagement.liquidity or 0)
|
||||
|
||||
return 0.60 * volume + 0.40 * liquidity
|
||||
|
||||
|
||||
def score_polymarket_items(items: List[schema.PolymarketItem]) -> List[schema.PolymarketItem]:
|
||||
"""Compute scores for Polymarket items.
|
||||
|
||||
Uses same weight structure as Reddit/X (relevance + recency + engagement).
|
||||
"""
|
||||
if not items:
|
||||
return items
|
||||
|
||||
eng_raw = [compute_polymarket_engagement_raw(item.engagement) for item in items]
|
||||
eng_normalized = normalize_to_100(eng_raw)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
rel_score = int(item.relevance * 100)
|
||||
rec_score = dates.recency_score(item.date)
|
||||
|
||||
if eng_normalized[i] is not None:
|
||||
eng_score = int(eng_normalized[i])
|
||||
else:
|
||||
eng_score = DEFAULT_ENGAGEMENT
|
||||
|
||||
item.subs = schema.SubScores(
|
||||
relevance=rel_score,
|
||||
recency=rec_score,
|
||||
engagement=eng_score,
|
||||
)
|
||||
|
||||
overall = (
|
||||
WEIGHT_RELEVANCE * rel_score +
|
||||
WEIGHT_RECENCY * rec_score +
|
||||
WEIGHT_ENGAGEMENT * eng_score
|
||||
)
|
||||
|
||||
if eng_raw[i] is None:
|
||||
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||
|
||||
item.score = max(0, min(100, int(overall)))
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]:
|
||||
"""Compute scores for WebSearch items WITHOUT engagement metrics.
|
||||
|
||||
@@ -395,7 +453,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS
|
||||
return items
|
||||
|
||||
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem]]) -> List:
|
||||
def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem, schema.PolymarketItem]]) -> List:
|
||||
"""Sort items by score (descending), then date, then source priority.
|
||||
|
||||
Args:
|
||||
@@ -412,7 +470,7 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
|
||||
date = item.date or "0000-00-00"
|
||||
date_key = -int(date.replace("-", ""))
|
||||
|
||||
# Tertiary: source priority (Reddit > X > YouTube > HN > WebSearch)
|
||||
# Tertiary: source priority (Reddit > X > YouTube > HN > Polymarket > WebSearch)
|
||||
if isinstance(item, schema.RedditItem):
|
||||
source_priority = 0
|
||||
elif isinstance(item, schema.XItem):
|
||||
@@ -421,8 +479,10 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear
|
||||
source_priority = 2
|
||||
elif isinstance(item, schema.HackerNewsItem):
|
||||
source_priority = 3
|
||||
else: # WebSearchItem
|
||||
elif isinstance(item, schema.PolymarketItem):
|
||||
source_priority = 4
|
||||
else: # WebSearchItem
|
||||
source_priority = 5
|
||||
|
||||
# Quaternary: title/text for stability
|
||||
text = getattr(item, "title", "") or getattr(item, "text", "")
|
||||
|
||||
+21
-1
@@ -79,6 +79,13 @@ HN_MESSAGES = [
|
||||
"Discovering developer conversations...",
|
||||
]
|
||||
|
||||
POLYMARKET_MESSAGES = [
|
||||
"Checking prediction markets...",
|
||||
"Finding what people are betting on...",
|
||||
"Scanning Polymarket for odds...",
|
||||
"Discovering prediction markets...",
|
||||
]
|
||||
|
||||
PROCESSING_MESSAGES = [
|
||||
"Crunching the data...",
|
||||
"Scoring and ranking...",
|
||||
@@ -274,6 +281,15 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.YELLOW}HN{Colors.RESET} Found {count} stories")
|
||||
|
||||
def start_polymarket(self):
|
||||
msg = random.choice(POLYMARKET_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.GREEN}Polymarket{Colors.RESET} {msg}", Colors.GREEN, quiet=True)
|
||||
self.spinner.start()
|
||||
|
||||
def end_polymarket(self, count: int):
|
||||
if self.spinner:
|
||||
self.spinner.stop(f"{Colors.GREEN}Polymarket{Colors.RESET} Found {count} markets")
|
||||
|
||||
def start_processing(self):
|
||||
msg = random.choice(PROCESSING_MESSAGES)
|
||||
self.spinner = Spinner(f"{Colors.PURPLE}Processing{Colors.RESET} {msg}", Colors.PURPLE)
|
||||
@@ -283,7 +299,7 @@ class ProgressDisplay:
|
||||
if self.spinner:
|
||||
self.spinner.stop()
|
||||
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0):
|
||||
def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0, pm_count: int = 0):
|
||||
elapsed = time.time() - self.start_time
|
||||
if IS_TTY:
|
||||
sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ")
|
||||
@@ -294,6 +310,8 @@ class ProgressDisplay:
|
||||
sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos")
|
||||
if hn_count:
|
||||
sys.stderr.write(f" {Colors.YELLOW}HN:{Colors.RESET} {hn_count} stories")
|
||||
if pm_count:
|
||||
sys.stderr.write(f" {Colors.GREEN}Polymarket:{Colors.RESET} {pm_count} markets")
|
||||
sys.stderr.write("\n\n")
|
||||
else:
|
||||
parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"]
|
||||
@@ -301,6 +319,8 @@ class ProgressDisplay:
|
||||
parts.append(f"YouTube: {youtube_count} videos")
|
||||
if hn_count:
|
||||
parts.append(f"HN: {hn_count} stories")
|
||||
if pm_count:
|
||||
parts.append(f"Polymarket: {pm_count} markets")
|
||||
sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user