feat(bluesky): add Bluesky/AT Protocol as social source

Free, no-auth-required search via public.api.bsky.app.
Always-on like HN and Polymarket (no API key needed).

- New scripts/lib/bluesky.py: search + parse via AT Protocol
- BlueskyItem schema, normalization, scoring, deduplication
- Wired into orchestrator ThreadPoolExecutor with timeout config
- Rendering in compact, full, and JSON output modes
- 14 unit tests covering parsing, dates, relevance, edge cases
- --search=bluesky / --search=bsky for bluesky-only mode

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-09 22:54:20 -07:00
parent 4b7087e136
commit 9a1059ee9d
9 changed files with 586 additions and 13 deletions
+86 -10
View File
@@ -38,14 +38,14 @@ _child_pids: set = set()
_child_pids_lock = threading.Lock() _child_pids_lock = threading.Lock()
TIMEOUT_PROFILES = { TIMEOUT_PROFILES = {
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
} }
# Valid source names for the --search flag # Valid source names for the --search flag
VALID_SEARCH_SOURCES = { VALID_SEARCH_SOURCES = {
"reddit", "x", "hn", "youtube", "tiktok", "instagram", "reddit", "x", "hn", "bluesky", "bsky", "youtube", "tiktok", "instagram",
"polymarket", "web", "xiaohongshu", "xhs", "polymarket", "web", "xiaohongshu", "xhs",
} }
@@ -135,6 +135,7 @@ def _install_global_timeout(timeout_seconds: int):
from lib import ( from lib import (
bird_x, bird_x,
bluesky,
dates, dates,
dedupe, dedupe,
hackernews, hackernews,
@@ -513,6 +514,34 @@ def _search_hackernews(
return hn_items, hn_error return hn_items, hn_error
def _search_bluesky(
topic: str,
from_date: str,
to_date: str,
depth: str,
) -> tuple:
"""Search Bluesky via AT Protocol (runs in thread).
Returns:
Tuple of (bsky_items, bsky_error)
"""
bsky_error = None
try:
response = bluesky.search_bluesky(
topic, from_date, to_date, depth=depth,
)
except Exception as e:
return [], f"{type(e).__name__}: {e}"
bsky_items = bluesky.parse_bluesky_response(response)
if response.get("error"):
bsky_error = response["error"]
return bsky_items, bsky_error
def _search_polymarket( def _search_polymarket(
topic: str, topic: str,
from_date: str, from_date: str,
@@ -823,6 +852,7 @@ def run_research(
timeouts: dict = None, timeouts: dict = None,
resolved_handle: str = None, resolved_handle: str = None,
do_hackernews: bool = True, do_hackernews: bool = True,
do_bluesky: bool = True,
do_polymarket: bool = True, do_polymarket: bool = True,
no_native_web: bool = False, no_native_web: bool = False,
) -> tuple: ) -> tuple:
@@ -830,10 +860,10 @@ def run_research(
Returns: Returns:
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items, Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
hackernews_items, polymarket_items, web_items, web_needed, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed,
raw_openai, raw_xai, raw_reddit_enriched, raw_openai, raw_xai, raw_reddit_enriched,
reddit_error, x_error, youtube_error, tiktok_error, instagram_error, reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
hackernews_error, polymarket_error, web_error) hackernews_error, bluesky_error, polymarket_error, web_error)
Note: web_needed is True when web search should be performed by the assistant Note: web_needed is True when web search should be performed by the assistant
(i.e., no native web search API keys are configured). When native web search (i.e., no native web search API keys are configured). When native web search
@@ -849,6 +879,7 @@ def run_research(
tiktok_items = [] tiktok_items = []
instagram_items = [] instagram_items = []
hackernews_items = [] hackernews_items = []
bluesky_items = []
polymarket_items = [] polymarket_items = []
web_items = [] web_items = []
raw_openai = None raw_openai = None
@@ -860,6 +891,7 @@ def run_research(
tiktok_error = None tiktok_error = None
instagram_error = None instagram_error = None
hackernews_error = None hackernews_error = None
bluesky_error = None
polymarket_error = None polymarket_error = None
web_error = None web_error = None
xiaohongshu_error = None xiaohongshu_error = None
@@ -944,7 +976,7 @@ def run_research(
progress.show_error(f"Instagram error: {e}") progress.show_error(f"Instagram error: {e}")
if progress: if progress:
progress.end_instagram(len(instagram_items)) progress.end_instagram(len(instagram_items))
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
# Determine which searches to run # Determine which searches to run
do_reddit = sources in ("both", "reddit", "all", "reddit-web") do_reddit = sources in ("both", "reddit", "all", "reddit-web")
@@ -960,6 +992,7 @@ def run_research(
instagram_future = None instagram_future = None
xiaohongshu_future = None xiaohongshu_future = None
hackernews_future = None hackernews_future = None
bluesky_future = None
polymarket_future = None polymarket_future = None
web_future = None web_future = None
max_workers = ( max_workers = (
@@ -969,6 +1002,7 @@ def run_research(
+ (1 if run_instagram else 0) + (1 if run_instagram else 0)
+ (1 if run_xiaohongshu else 0) + (1 if run_xiaohongshu else 0)
+ (1 if do_hackernews else 0) + (1 if do_hackernews else 0)
+ (1 if do_bluesky else 0)
+ (1 if do_polymarket else 0) + (1 if do_polymarket else 0)
+ (1 if web_backend else 0) + (1 if web_backend else 0)
) )
@@ -1026,6 +1060,11 @@ def run_research(
_search_hackernews, topic, from_date, to_date, depth _search_hackernews, topic, from_date, to_date, depth
) )
if do_bluesky:
bluesky_future = executor.submit(
_search_bluesky, topic, from_date, to_date, depth
)
if do_polymarket: if do_polymarket:
if progress: if progress:
progress.start_polymarket() progress.start_polymarket()
@@ -1158,6 +1197,21 @@ def run_research(
if progress: if progress:
progress.end_hackernews(len(hackernews_items)) progress.end_hackernews(len(hackernews_items))
if bluesky_future:
bsky_timeout = timeouts.get("bluesky_future", future_timeout)
try:
bluesky_items, bluesky_error = bluesky_future.result(timeout=bsky_timeout)
if bluesky_error and progress:
progress.show_error(f"Bluesky error: {bluesky_error}")
except TimeoutError:
bluesky_error = f"Bluesky search timed out after {bsky_timeout}s"
if progress:
progress.show_error(bluesky_error)
except Exception as e:
bluesky_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"Bluesky error: {e}")
if polymarket_future: if polymarket_future:
pm_timeout = timeouts.get("polymarket_future", future_timeout) pm_timeout = timeouts.get("polymarket_future", future_timeout)
try: try:
@@ -1292,7 +1346,7 @@ def run_research(
if sup_x: if sup_x:
x_items.extend(sup_x) x_items.extend(sup_x)
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
def main(): def main():
@@ -1460,6 +1514,7 @@ def main():
"xiaohongshu": has_xiaohongshu, "xiaohongshu": has_xiaohongshu,
"xiaohongshu_api_base": env.get_xiaohongshu_api_base(config), "xiaohongshu_api_base": env.get_xiaohongshu_api_base(config),
"hackernews": True, "hackernews": True,
"bluesky": True,
"polymarket": True, "polymarket": True,
"web_search_backend": web_source, "web_search_backend": web_source,
"parallel_ai": bool(config.get("PARALLEL_API_KEY")), "parallel_ai": bool(config.get("PARALLEL_API_KEY")),
@@ -1493,6 +1548,7 @@ def main():
"instagram": has_instagram, "instagram": has_instagram,
"xiaohongshu": has_xiaohongshu, "xiaohongshu": has_xiaohongshu,
"hackernews": True, "hackernews": True,
"bluesky": True,
"polymarket": True, "polymarket": True,
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source, "web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
} }
@@ -1574,6 +1630,7 @@ def main():
# Apply --search flag: restrict sources to the specified subset # Apply --search flag: restrict sources to the specified subset
search_do_hackernews = True search_do_hackernews = True
search_do_bluesky = True
search_do_polymarket = True search_do_polymarket = True
search_run_youtube = has_ytdlp search_run_youtube = has_ytdlp
search_run_tiktok = has_tiktok search_run_tiktok = has_tiktok
@@ -1584,6 +1641,7 @@ def main():
has_reddit = "reddit" in search_sources has_reddit = "reddit" in search_sources
has_x = "x" in search_sources has_x = "x" in search_sources
search_do_hackernews = "hn" in search_sources search_do_hackernews = "hn" in search_sources
search_do_bluesky = "bluesky" in search_sources or "bsky" in search_sources
search_do_polymarket = "polymarket" in search_sources search_do_polymarket = "polymarket" in search_sources
search_run_youtube = "youtube" in search_sources and has_ytdlp search_run_youtube = "youtube" in search_sources and has_ytdlp
search_run_tiktok = "tiktok" in search_sources and has_tiktok search_run_tiktok = "tiktok" in search_sources and has_tiktok
@@ -1603,7 +1661,7 @@ def main():
sources = "web" # hn/polymarket only; no Reddit/X sources = "web" # hn/polymarket only; no Reddit/X
# Run research # Run research
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error = run_research( reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error = run_research(
args.topic, args.topic,
sources, sources,
config, config,
@@ -1621,6 +1679,7 @@ def main():
timeouts=timeouts, timeouts=timeouts,
resolved_handle=args.x_handle, resolved_handle=args.x_handle,
do_hackernews=search_do_hackernews, do_hackernews=search_do_hackernews,
do_bluesky=search_do_bluesky,
do_polymarket=search_do_polymarket, do_polymarket=search_do_polymarket,
no_native_web=args.no_native_web, no_native_web=args.no_native_web,
) )
@@ -1635,6 +1694,7 @@ def main():
normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else [] normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else []
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else [] normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else [] normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
normalized_bsky = normalize.normalize_bluesky_items(bluesky_items, from_date, to_date) if bluesky_items else []
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else [] normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else [] normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
@@ -1651,6 +1711,7 @@ def main():
# Instagram: hard date filter (instagram.py already pre-filters, but safety net) # Instagram: hard date filter (instagram.py already pre-filters, but safety net)
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else [] filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else [] filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
filtered_bsky = normalize.filter_by_date_range(normalized_bsky, from_date, to_date) if normalized_bsky else []
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine # Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
filtered_pm = normalized_pm filtered_pm = normalized_pm
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else [] filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
@@ -1662,6 +1723,7 @@ def main():
scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else [] scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else []
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else [] scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else [] scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
scored_bsky = score.score_bluesky_items(filtered_bsky) if filtered_bsky else []
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else [] scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
scored_web = score.score_websearch_items(filtered_web) if filtered_web else [] scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
@@ -1672,6 +1734,7 @@ def main():
sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else [] sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else []
sorted_ig = score.sort_items(scored_ig) if scored_ig else [] sorted_ig = score.sort_items(scored_ig) if scored_ig else []
sorted_hn = score.sort_items(scored_hn) if scored_hn else [] sorted_hn = score.sort_items(scored_hn) if scored_hn else []
sorted_bsky = score.sort_items(scored_bsky) if scored_bsky else []
sorted_pm = score.sort_items(scored_pm) if scored_pm else [] sorted_pm = score.sort_items(scored_pm) if scored_pm else []
sorted_web = score.sort_items(scored_web) if scored_web else [] sorted_web = score.sort_items(scored_web) if scored_web else []
@@ -1682,6 +1745,7 @@ def main():
deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else [] deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else []
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else [] deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else [] deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
deduped_bsky = dedupe.dedupe_bluesky(sorted_bsky) if sorted_bsky else []
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else [] deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else [] deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
@@ -1694,7 +1758,7 @@ def main():
# Cross-source linking: annotate items that discuss the same story # Cross-source linking: annotate items that discuss the same story
dedupe.cross_source_link( dedupe.cross_source_link(
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_pm, deduped_web, deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_pm, deduped_web,
) )
progress.end_processing() progress.end_processing()
@@ -1714,6 +1778,7 @@ def main():
report.tiktok = deduped_tiktok report.tiktok = deduped_tiktok
report.instagram = deduped_ig report.instagram = deduped_ig
report.hackernews = deduped_hn report.hackernews = deduped_hn
report.bluesky = deduped_bsky
report.polymarket = deduped_pm report.polymarket = deduped_pm
report.web = deduped_web report.web = deduped_web
report.reddit_error = reddit_error report.reddit_error = reddit_error
@@ -1722,6 +1787,7 @@ def main():
report.tiktok_error = tiktok_error report.tiktok_error = tiktok_error
report.instagram_error = instagram_error report.instagram_error = instagram_error
report.hackernews_error = hackernews_error report.hackernews_error = hackernews_error
report.bluesky_error = bluesky_error
report.polymarket_error = polymarket_error report.polymarket_error = polymarket_error
report.web_error = web_error report.web_error = web_error
report.resolved_x_handle = args.x_handle report.resolved_x_handle = args.x_handle
@@ -1827,6 +1893,16 @@ def main():
"engagement_score": item.engagement.score if item.engagement else 0, "engagement_score": item.engagement.score if item.engagement else 0,
"relevance_score": item.relevance, "relevance_score": item.relevance,
}) })
for item in deduped_bsky:
findings.append({
"source": "bluesky",
"url": item.url,
"title": item.text[:100],
"author": item.author_handle,
"content": item.text,
"engagement_score": item.engagement.likes if item.engagement else 0,
"relevance_score": item.relevance,
})
for item in deduped_pm: for item in deduped_pm:
findings.append({ findings.append({
"source": "polymarket", "source": "polymarket",
+166
View File
@@ -0,0 +1,166 @@
"""Bluesky search via AT Protocol (free, no auth required).
Uses public.api.bsky.app for post discovery.
No API key needed - just HTTP calls via stdlib urllib.
"""
import math
import re
import sys
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from . import http
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
DEPTH_CONFIG = {
"quick": 15,
"default": 30,
"deep": 60,
}
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[Bluesky] {msg}\n")
sys.stderr.flush()
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for Bluesky search."""
text = topic.lower().strip()
prefixes = [
'what are the best', 'what is the best', 'what are the latest',
'what are people saying about', 'what do people think about',
'how do i use', 'how to use', 'how to',
'what are', 'what is', 'tips for', 'best practices for',
]
for p in prefixes:
if text.startswith(p + ' '):
text = text[len(p):].strip()
noise = {
'best', 'top', 'good', 'great', 'awesome',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features', 'recommendations', 'advice',
}
words = text.split()
filtered = [w for w in words if w not in noise]
result = ' '.join(filtered) if filtered else text
return result.rstrip('?!.')
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from Bluesky post to YYYY-MM-DD.
AT Protocol uses ISO 8601 format in indexedAt and createdAt fields.
"""
for key in ("indexedAt", "createdAt"):
val = item.get(key)
if val and isinstance(val, str):
try:
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d")
except (ValueError, TypeError):
pass
return None
def search_bluesky(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
) -> Dict[str, Any]:
"""Search Bluesky via AT Protocol public API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
Returns:
Dict with 'posts' list from AT Protocol response.
"""
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching for '{core_topic}' (depth={depth}, limit={count})")
from urllib.parse import urlencode
params = {
"q": core_topic,
"limit": str(min(count, 100)),
"sort": "top",
}
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
try:
response = http.request("GET", url, timeout=30)
except http.HTTPError as e:
_log(f"Search failed: {e}")
return {"posts": [], "error": str(e)}
except Exception as e:
_log(f"Search failed: {e}")
return {"posts": [], "error": str(e)}
posts = response.get("posts", [])
_log(f"Found {len(posts)} posts")
return response
def parse_bluesky_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse AT Protocol response into normalized item dicts.
Returns:
List of item dicts ready for normalization.
"""
posts = response.get("posts", [])
items = []
for i, post in enumerate(posts):
record = post.get("record") or {}
text = record.get("text") or ""
author = post.get("author") or {}
handle = author.get("handle") or ""
display_name = author.get("displayName") or handle
# Post URI -> URL
# URI format: at://did:plc:xxx/app.bsky.feed.post/rkey
uri = post.get("uri") or ""
rkey = uri.rsplit("/", 1)[-1] if uri else ""
url = f"https://bsky.app/profile/{handle}/post/{rkey}" if handle and rkey else ""
likes = post.get("likeCount") or 0
reposts = post.get("repostCount") or 0
replies = post.get("replyCount") or 0
quotes = post.get("quoteCount") or 0
date_str = _parse_date(post) or _parse_date(record)
# Relevance: position-based (AT Protocol sorts by relevance with sort=top)
rank_score = max(0.3, 1.0 - (i * 0.02))
engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
items.append({
"handle": handle,
"display_name": display_name,
"text": text,
"url": url,
"date": date_str,
"engagement": {
"likes": likes,
"reposts": reposts,
"replies": replies,
"quotes": quotes,
},
"relevance": round(relevance, 2),
"why_relevant": f"Bluesky: @{handle}: {text[:60]}" if text else f"Bluesky: {handle}",
})
return items
+8
View File
@@ -226,6 +226,14 @@ def dedupe_hackernews(
return dedupe_items(items, threshold) return dedupe_items(items, threshold)
def dedupe_bluesky(
items: List[schema.BlueskyItem],
threshold: float = 0.7,
) -> List[schema.BlueskyItem]:
"""Dedupe Bluesky items."""
return dedupe_items(items, threshold)
def dedupe_polymarket( def dedupe_polymarket(
items: List[schema.PolymarketItem], items: List[schema.PolymarketItem],
threshold: float = 0.7, threshold: float = 0.7,
+8
View File
@@ -480,6 +480,14 @@ def is_hackernews_available() -> bool:
return True return True
def is_bluesky_available() -> bool:
"""Check if Bluesky source is available.
Always returns True - AT Protocol search is free, no key needed.
"""
return True
def is_polymarket_available() -> bool: def is_polymarket_available() -> bool:
"""Check if Polymarket source is available. """Check if Polymarket source is available.
+45 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
from . import dates, schema from . import dates, schema
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem) T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.PolymarketItem)
def filter_by_date_range( def filter_by_date_range(
@@ -350,6 +350,50 @@ def normalize_hackernews_items(
return normalized return normalized
def normalize_bluesky_items(
items: List[Dict[str, Any]],
from_date: str,
to_date: str,
) -> List[schema.BlueskyItem]:
"""Normalize raw Bluesky items to schema.
Args:
items: Raw Bluesky items from AT Protocol API
from_date: Start of date range
to_date: End of date range
Returns:
List of BlueskyItem objects
"""
normalized = []
for i, item in enumerate(items):
eng_raw = item.get("engagement") or {}
engagement = schema.Engagement(
likes=eng_raw.get("likes"),
reposts=eng_raw.get("reposts"),
replies=eng_raw.get("replies"),
quotes=eng_raw.get("quotes"),
)
date_str = item.get("date")
normalized.append(schema.BlueskyItem(
id=f"BS{i+1}",
text=item.get("text", ""),
url=item.get("url", ""),
author_handle=item.get("handle", ""),
display_name=item.get("display_name", ""),
date=date_str,
date_confidence="high",
engagement=engagement,
relevance=item.get("relevance", 0.5),
why_relevant=item.get("why_relevant", ""),
))
return normalized
def normalize_polymarket_items( def normalize_polymarket_items(
items: List[Dict[str, Any]], items: List[Dict[str, Any]],
from_date: str, from_date: str,
+68 -2
View File
@@ -59,13 +59,14 @@ def _assess_data_freshness(report: schema.Report) -> dict:
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from) x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from) web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from) hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from)
bsky_recent = sum(1 for b in report.bluesky if b.date and b.date >= report.range_from)
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from) pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from) tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from)
ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from) ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from)
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent + ig_recent total_recent = reddit_recent + x_recent + web_recent + hn_recent + bsky_recent + pm_recent + tiktok_recent + ig_recent
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok) + len(report.instagram) total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.bluesky) + len(report.polymarket) + len(report.tiktok) + len(report.instagram)
return { return {
"reddit_recent": reddit_recent, "reddit_recent": reddit_recent,
@@ -367,6 +368,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
lines.append("") lines.append("")
# Bluesky items
if report.bluesky_error:
lines.append("### Bluesky Posts")
lines.append("")
lines.append(f"**ERROR:** {report.bluesky_error}")
lines.append("")
elif report.bluesky:
lines.append("### Bluesky Posts")
lines.append("")
for item in report.bluesky[:limit]:
eng_str = ""
if item.engagement:
eng = item.engagement
parts = []
if eng.likes is not None:
parts.append(f"{eng.likes}lk")
if eng.reposts is not None:
parts.append(f"{eng.reposts}rp")
if eng.replies is not None:
parts.append(f"{eng.replies}re")
if parts:
eng_str = f" [{', '.join(parts)}]"
date_str = f" ({item.date})" if item.date else ""
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{eng_str}{_xref_tag(item)}")
if item.text:
snippet = item.text[:200]
if len(item.text) > 200:
snippet += "..."
lines.append(f" {snippet}")
if item.url:
lines.append(f" {item.url}")
lines.append(f" *{item.why_relevant}*")
lines.append("")
# Polymarket items # Polymarket items
if report.polymarket_error: if report.polymarket_error:
lines.append("### Prediction Markets (Polymarket)") lines.append("### Prediction Markets (Polymarket)")
@@ -531,6 +568,13 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
lines.append(f" ✅ HN: {len(report.hackernews)} stories") lines.append(f" ✅ HN: {len(report.hackernews)} stories")
# Hide when zero results # Hide when zero results
# Bluesky
if report.bluesky_error:
lines.append(f" ❌ Bluesky: error - {report.bluesky_error}")
elif report.bluesky:
lines.append(f" ✅ Bluesky: {len(report.bluesky)} posts")
# Hide when zero results
# Polymarket # Polymarket
if report.polymarket_error: if report.polymarket_error:
lines.append(f" ❌ Polymarket: error - {report.polymarket_error}") lines.append(f" ❌ Polymarket: error - {report.polymarket_error}")
@@ -581,6 +625,8 @@ def render_context_snippet(report: schema.Report) -> str:
all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url)) all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url))
for item in report.hackernews[:5]: for item in report.hackernews[:5]:
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url)) all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
for item in report.bluesky[:5]:
all_items.append((item.score, "Bluesky", item.text[:50] + "...", item.url))
for item in report.polymarket[:5]: for item in report.polymarket[:5]:
all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url)) all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url))
for item in report.web[:5]: for item in report.web[:5]:
@@ -754,6 +800,26 @@ def render_full_report(report: schema.Report) -> str:
lines.append("") lines.append("")
# Bluesky section
if report.bluesky:
lines.append("## Bluesky Posts")
lines.append("")
for item in report.bluesky:
lines.append(f"### {item.id}: @{item.author_handle}")
lines.append("")
lines.append(f"- **URL:** {item.url}")
lines.append(f"- **Date:** {item.date or 'Unknown'}")
lines.append(f"- **Score:** {item.score}/100")
lines.append(f"- **Relevance:** {item.why_relevant}")
if item.engagement:
eng = item.engagement
lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts, {eng.replies or '?'} replies")
lines.append("")
lines.append(f"> {item.text[:300]}")
lines.append("")
# Polymarket section # Polymarket section
if report.polymarket: if report.polymarket:
lines.append("## Prediction Markets (Polymarket)") lines.append("## Prediction Markets (Polymarket)")
+42
View File
@@ -355,6 +355,43 @@ class HackerNewsItem:
return d return d
@dataclass
class BlueskyItem:
"""Normalized Bluesky post."""
id: str # "BS1", "BS2", ...
text: str
url: str # bsky.app permalink
author_handle: str # user.bsky.social
display_name: str
date: Optional[str] = None
date_confidence: str = "high" # AT Protocol has exact timestamps
engagement: Optional[Engagement] = None # likes, reposts, replies, quotes
relevance: float = 0.5
why_relevant: str = ""
subs: SubScores = field(default_factory=SubScores)
score: int = 0
cross_refs: List[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
d = {
'id': self.id,
'text': self.text,
'url': self.url,
'author_handle': self.author_handle,
'display_name': self.display_name,
'date': self.date,
'date_confidence': self.date_confidence,
'engagement': self.engagement.to_dict() if self.engagement else None,
'relevance': self.relevance,
'why_relevant': self.why_relevant,
'subs': self.subs.to_dict(),
'score': self.score,
}
if self.cross_refs:
d['cross_refs'] = self.cross_refs
return d
@dataclass @dataclass
class PolymarketItem: class PolymarketItem:
"""Normalized Polymarket prediction market item.""" """Normalized Polymarket prediction market item."""
@@ -415,6 +452,7 @@ class Report:
tiktok: List[TikTokItem] = field(default_factory=list) tiktok: List[TikTokItem] = field(default_factory=list)
instagram: List[InstagramItem] = field(default_factory=list) instagram: List[InstagramItem] = field(default_factory=list)
hackernews: List[HackerNewsItem] = field(default_factory=list) hackernews: List[HackerNewsItem] = field(default_factory=list)
bluesky: List[BlueskyItem] = field(default_factory=list)
polymarket: List[PolymarketItem] = field(default_factory=list) polymarket: List[PolymarketItem] = field(default_factory=list)
best_practices: List[str] = field(default_factory=list) best_practices: List[str] = field(default_factory=list)
prompt_pack: List[str] = field(default_factory=list) prompt_pack: List[str] = field(default_factory=list)
@@ -427,6 +465,7 @@ class Report:
tiktok_error: Optional[str] = None tiktok_error: Optional[str] = None
instagram_error: Optional[str] = None instagram_error: Optional[str] = None
hackernews_error: Optional[str] = None hackernews_error: Optional[str] = None
bluesky_error: Optional[str] = None
polymarket_error: Optional[str] = None polymarket_error: Optional[str] = None
# Handle resolution # Handle resolution
resolved_x_handle: Optional[str] = None resolved_x_handle: Optional[str] = None
@@ -452,6 +491,7 @@ class Report:
'tiktok': [t.to_dict() for t in self.tiktok], 'tiktok': [t.to_dict() for t in self.tiktok],
'instagram': [ig.to_dict() for ig in self.instagram], 'instagram': [ig.to_dict() for ig in self.instagram],
'hackernews': [h.to_dict() for h in self.hackernews], 'hackernews': [h.to_dict() for h in self.hackernews],
'bluesky': [b.to_dict() for b in self.bluesky],
'polymarket': [p.to_dict() for p in self.polymarket], 'polymarket': [p.to_dict() for p in self.polymarket],
'best_practices': self.best_practices, 'best_practices': self.best_practices,
'prompt_pack': self.prompt_pack, 'prompt_pack': self.prompt_pack,
@@ -473,6 +513,8 @@ class Report:
d['instagram_error'] = self.instagram_error d['instagram_error'] = self.instagram_error
if self.hackernews_error: if self.hackernews_error:
d['hackernews_error'] = self.hackernews_error d['hackernews_error'] = self.hackernews_error
if self.bluesky_error:
d['bluesky_error'] = self.bluesky_error
if self.polymarket_error: if self.polymarket_error:
d['polymarket_error'] = self.polymarket_error d['polymarket_error'] = self.polymarket_error
if self.from_cache: if self.from_cache:
+60
View File
@@ -468,6 +468,66 @@ def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.Ha
return items return items
def compute_bluesky_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Bluesky item.
Formula: 0.40*log1p(likes) + 0.30*log1p(reposts) + 0.20*log1p(replies) + 0.10*log1p(quotes)
Likes are primary signal; reposts indicate reach; replies indicate discussion depth.
"""
if engagement is None:
return None
if engagement.likes is None and engagement.reposts is None:
return None
likes = log1p_safe(engagement.likes)
reposts = log1p_safe(engagement.reposts)
replies = log1p_safe(engagement.replies)
quotes = log1p_safe(engagement.quotes)
return 0.40 * likes + 0.30 * reposts + 0.20 * replies + 0.10 * quotes
def score_bluesky_items(items: List[schema.BlueskyItem]) -> List[schema.BlueskyItem]:
"""Compute scores for Bluesky items.
Uses same weight structure as Reddit/X (relevance + recency + engagement).
"""
if not items:
return items
eng_raw = [compute_bluesky_engagement_raw(item.engagement) for item in items]
eng_normalized = normalize_to_100(eng_raw)
for i, item in enumerate(items):
rel_score = int(item.relevance * 100)
rec_score = dates.recency_score(item.date)
if eng_normalized[i] is not None:
eng_score = int(eng_normalized[i])
else:
eng_score = DEFAULT_ENGAGEMENT
item.subs = schema.SubScores(
relevance=rel_score,
recency=rec_score,
engagement=eng_score,
)
overall = (
WEIGHT_RELEVANCE * rel_score +
WEIGHT_RECENCY * rec_score +
WEIGHT_ENGAGEMENT * eng_score
)
if eng_raw[i] is None:
overall -= UNKNOWN_ENGAGEMENT_PENALTY
item.score = max(0, min(100, int(overall)))
return items
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
"""Compute raw engagement score for Polymarket item. """Compute raw engagement score for Polymarket item.
+103
View File
@@ -0,0 +1,103 @@
"""Tests for bluesky module."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
from lib import bluesky
class TestExtractCoreSubject(unittest.TestCase):
def test_strips_prefix(self):
result = bluesky._extract_core_subject("what are people saying about claude code")
self.assertEqual(result, "claude code")
def test_strips_noise(self):
result = bluesky._extract_core_subject("latest trending news claude code")
self.assertNotIn("latest", result)
self.assertNotIn("trending", result)
self.assertIn("claude", result)
def test_preserves_core(self):
result = bluesky._extract_core_subject("react native")
self.assertEqual(result, "react native")
class TestParseDate(unittest.TestCase):
def test_indexed_at_iso(self):
item = {"indexedAt": "2024-06-15T12:00:00Z"}
self.assertEqual(bluesky._parse_date(item), "2024-06-15")
def test_created_at_iso(self):
item = {"createdAt": "2024-03-01T08:30:00.000Z"}
self.assertEqual(bluesky._parse_date(item), "2024-03-01")
def test_indexed_at_preferred_over_created_at(self):
item = {"indexedAt": "2024-06-15T12:00:00Z", "createdAt": "2024-06-14T12:00:00Z"}
self.assertEqual(bluesky._parse_date(item), "2024-06-15")
def test_none_returns_none(self):
self.assertIsNone(bluesky._parse_date({}))
def test_invalid_date_returns_none(self):
self.assertIsNone(bluesky._parse_date({"indexedAt": "not-a-date"}))
class TestParseBlueskyResponse(unittest.TestCase):
def test_basic_post(self):
response = {
"posts": [{
"uri": "at://did:plc:abc123/app.bsky.feed.post/xyz789",
"author": {"handle": "alice.bsky.social", "displayName": "Alice"},
"record": {"text": "Hello world", "createdAt": "2024-06-15T12:00:00Z"},
"indexedAt": "2024-06-15T12:01:00Z",
"likeCount": 10,
"repostCount": 5,
"replyCount": 3,
"quoteCount": 1,
}]
}
items = bluesky.parse_bluesky_response(response)
self.assertEqual(len(items), 1)
self.assertEqual(items[0]["handle"], "alice.bsky.social")
self.assertEqual(items[0]["display_name"], "Alice")
self.assertEqual(items[0]["text"], "Hello world")
self.assertEqual(items[0]["url"], "https://bsky.app/profile/alice.bsky.social/post/xyz789")
self.assertEqual(items[0]["engagement"]["likes"], 10)
self.assertEqual(items[0]["engagement"]["reposts"], 5)
self.assertEqual(items[0]["date"], "2024-06-15")
def test_empty_response(self):
items = bluesky.parse_bluesky_response({})
self.assertEqual(items, [])
def test_missing_fields(self):
response = {"posts": [{"uri": "", "author": {}, "record": {}}]}
items = bluesky.parse_bluesky_response(response)
self.assertEqual(len(items), 1)
self.assertEqual(items[0]["handle"], "")
self.assertEqual(items[0]["text"], "")
def test_relevance_decreases_with_position(self):
response = {"posts": [
{"uri": f"at://did/app.bsky.feed.post/{i}", "author": {"handle": f"u{i}"}, "record": {"text": f"post {i}"}}
for i in range(5)
]}
items = bluesky.parse_bluesky_response(response)
self.assertGreater(items[0]["relevance"], items[4]["relevance"])
class TestDepthConfig(unittest.TestCase):
def test_all_depths_exist(self):
for depth in ("quick", "default", "deep"):
self.assertIn(depth, bluesky.DEPTH_CONFIG)
def test_deep_has_more_results(self):
quick = bluesky.DEPTH_CONFIG["quick"]
deep = bluesky.DEPTH_CONFIG["deep"]
self.assertGreater(deep, quick)
if __name__ == "__main__":
unittest.main()