From 9a1059ee9dad3b8033fc130a104ac4d06eb5a232 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Mon, 9 Mar 2026 22:54:20 -0700 Subject: [PATCH] feat(bluesky): add Bluesky/AT Protocol as social source Free, no-auth-required search via public.api.bsky.app. Always-on like HN and Polymarket (no API key needed). - New scripts/lib/bluesky.py: search + parse via AT Protocol - BlueskyItem schema, normalization, scoring, deduplication - Wired into orchestrator ThreadPoolExecutor with timeout config - Rendering in compact, full, and JSON output modes - 14 unit tests covering parsing, dates, relevance, edge cases - --search=bluesky / --search=bsky for bluesky-only mode Co-Authored-By: Claude Opus 4.6 --- scripts/last30days.py | 96 +++++++++++++++++++--- scripts/lib/bluesky.py | 166 +++++++++++++++++++++++++++++++++++++++ scripts/lib/dedupe.py | 8 ++ scripts/lib/env.py | 8 ++ scripts/lib/normalize.py | 46 ++++++++++- scripts/lib/render.py | 70 ++++++++++++++++- scripts/lib/schema.py | 42 ++++++++++ scripts/lib/score.py | 60 ++++++++++++++ tests/test_bluesky.py | 103 ++++++++++++++++++++++++ 9 files changed, 586 insertions(+), 13 deletions(-) create mode 100644 scripts/lib/bluesky.py create mode 100644 tests/test_bluesky.py diff --git a/scripts/last30days.py b/scripts/last30days.py index dc9c926..7f582f2 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -38,14 +38,14 @@ _child_pids: set = set() _child_pids_lock = threading.Lock() TIMEOUT_PROFILES = { - "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, - "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, - "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, + "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, + "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, + "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, } # Valid source names for the --search flag VALID_SEARCH_SOURCES = { - "reddit", "x", "hn", "youtube", "tiktok", "instagram", + "reddit", "x", "hn", "bluesky", "bsky", "youtube", "tiktok", "instagram", "polymarket", "web", "xiaohongshu", "xhs", } @@ -135,6 +135,7 @@ def _install_global_timeout(timeout_seconds: int): from lib import ( bird_x, + bluesky, dates, dedupe, hackernews, @@ -513,6 +514,34 @@ def _search_hackernews( return hn_items, hn_error +def _search_bluesky( + topic: str, + from_date: str, + to_date: str, + depth: str, +) -> tuple: + """Search Bluesky via AT Protocol (runs in thread). + + Returns: + Tuple of (bsky_items, bsky_error) + """ + bsky_error = None + + try: + response = bluesky.search_bluesky( + topic, from_date, to_date, depth=depth, + ) + except Exception as e: + return [], f"{type(e).__name__}: {e}" + + bsky_items = bluesky.parse_bluesky_response(response) + + if response.get("error"): + bsky_error = response["error"] + + return bsky_items, bsky_error + + def _search_polymarket( topic: str, from_date: str, @@ -823,6 +852,7 @@ def run_research( timeouts: dict = None, resolved_handle: str = None, do_hackernews: bool = True, + do_bluesky: bool = True, do_polymarket: bool = True, no_native_web: bool = False, ) -> tuple: @@ -830,10 +860,10 @@ def run_research( Returns: Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items, - hackernews_items, polymarket_items, web_items, web_needed, + hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, - hackernews_error, polymarket_error, web_error) + hackernews_error, bluesky_error, polymarket_error, web_error) Note: web_needed is True when web search should be performed by the assistant (i.e., no native web search API keys are configured). When native web search @@ -849,6 +879,7 @@ def run_research( tiktok_items = [] instagram_items = [] hackernews_items = [] + bluesky_items = [] polymarket_items = [] web_items = [] raw_openai = None @@ -860,6 +891,7 @@ def run_research( tiktok_error = None instagram_error = None hackernews_error = None + bluesky_error = None polymarket_error = None web_error = None xiaohongshu_error = None @@ -944,7 +976,7 @@ def run_research( progress.show_error(f"Instagram error: {e}") if progress: progress.end_instagram(len(instagram_items)) - return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error + return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error # Determine which searches to run do_reddit = sources in ("both", "reddit", "all", "reddit-web") @@ -960,6 +992,7 @@ def run_research( instagram_future = None xiaohongshu_future = None hackernews_future = None + bluesky_future = None polymarket_future = None web_future = None max_workers = ( @@ -969,6 +1002,7 @@ def run_research( + (1 if run_instagram else 0) + (1 if run_xiaohongshu else 0) + (1 if do_hackernews else 0) + + (1 if do_bluesky else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0) ) @@ -1026,6 +1060,11 @@ def run_research( _search_hackernews, topic, from_date, to_date, depth ) + if do_bluesky: + bluesky_future = executor.submit( + _search_bluesky, topic, from_date, to_date, depth + ) + if do_polymarket: if progress: progress.start_polymarket() @@ -1158,6 +1197,21 @@ def run_research( if progress: progress.end_hackernews(len(hackernews_items)) + if bluesky_future: + bsky_timeout = timeouts.get("bluesky_future", future_timeout) + try: + bluesky_items, bluesky_error = bluesky_future.result(timeout=bsky_timeout) + if bluesky_error and progress: + progress.show_error(f"Bluesky error: {bluesky_error}") + except TimeoutError: + bluesky_error = f"Bluesky search timed out after {bsky_timeout}s" + if progress: + progress.show_error(bluesky_error) + except Exception as e: + bluesky_error = f"{type(e).__name__}: {e}" + if progress: + progress.show_error(f"Bluesky error: {e}") + if polymarket_future: pm_timeout = timeouts.get("polymarket_future", future_timeout) try: @@ -1292,7 +1346,7 @@ def run_research( if sup_x: x_items.extend(sup_x) - return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error + return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error def main(): @@ -1460,6 +1514,7 @@ def main(): "xiaohongshu": has_xiaohongshu, "xiaohongshu_api_base": env.get_xiaohongshu_api_base(config), "hackernews": True, + "bluesky": True, "polymarket": True, "web_search_backend": web_source, "parallel_ai": bool(config.get("PARALLEL_API_KEY")), @@ -1493,6 +1548,7 @@ def main(): "instagram": has_instagram, "xiaohongshu": has_xiaohongshu, "hackernews": True, + "bluesky": True, "polymarket": True, "web_search_backend": "deferred to assistant" if args.no_native_web else web_source, } @@ -1574,6 +1630,7 @@ def main(): # Apply --search flag: restrict sources to the specified subset search_do_hackernews = True + search_do_bluesky = True search_do_polymarket = True search_run_youtube = has_ytdlp search_run_tiktok = has_tiktok @@ -1584,6 +1641,7 @@ def main(): has_reddit = "reddit" in search_sources has_x = "x" in search_sources search_do_hackernews = "hn" in search_sources + search_do_bluesky = "bluesky" in search_sources or "bsky" in search_sources search_do_polymarket = "polymarket" in search_sources search_run_youtube = "youtube" in search_sources and has_ytdlp search_run_tiktok = "tiktok" in search_sources and has_tiktok @@ -1603,7 +1661,7 @@ def main(): sources = "web" # hn/polymarket only; no Reddit/X # Run research - reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error = run_research( + reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error = run_research( args.topic, sources, config, @@ -1621,6 +1679,7 @@ def main(): timeouts=timeouts, resolved_handle=args.x_handle, do_hackernews=search_do_hackernews, + do_bluesky=search_do_bluesky, do_polymarket=search_do_polymarket, no_native_web=args.no_native_web, ) @@ -1635,6 +1694,7 @@ def main(): normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else [] normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else [] normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else [] + normalized_bsky = normalize.normalize_bluesky_items(bluesky_items, from_date, to_date) if bluesky_items else [] normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else [] normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else [] @@ -1651,6 +1711,7 @@ def main(): # Instagram: hard date filter (instagram.py already pre-filters, but safety net) filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else [] filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else [] + filtered_bsky = normalize.filter_by_date_range(normalized_bsky, from_date, to_date) if normalized_bsky else [] # Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine filtered_pm = normalized_pm filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else [] @@ -1662,6 +1723,7 @@ def main(): scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else [] scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else [] scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else [] + scored_bsky = score.score_bluesky_items(filtered_bsky) if filtered_bsky else [] scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else [] scored_web = score.score_websearch_items(filtered_web) if filtered_web else [] @@ -1672,6 +1734,7 @@ def main(): sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else [] sorted_ig = score.sort_items(scored_ig) if scored_ig else [] sorted_hn = score.sort_items(scored_hn) if scored_hn else [] + sorted_bsky = score.sort_items(scored_bsky) if scored_bsky else [] sorted_pm = score.sort_items(scored_pm) if scored_pm else [] sorted_web = score.sort_items(scored_web) if scored_web else [] @@ -1682,6 +1745,7 @@ def main(): deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else [] deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else [] deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else [] + deduped_bsky = dedupe.dedupe_bluesky(sorted_bsky) if sorted_bsky else [] deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else [] deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else [] @@ -1694,7 +1758,7 @@ def main(): # Cross-source linking: annotate items that discuss the same story dedupe.cross_source_link( - deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_pm, deduped_web, + deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_pm, deduped_web, ) progress.end_processing() @@ -1714,6 +1778,7 @@ def main(): report.tiktok = deduped_tiktok report.instagram = deduped_ig report.hackernews = deduped_hn + report.bluesky = deduped_bsky report.polymarket = deduped_pm report.web = deduped_web report.reddit_error = reddit_error @@ -1722,6 +1787,7 @@ def main(): report.tiktok_error = tiktok_error report.instagram_error = instagram_error report.hackernews_error = hackernews_error + report.bluesky_error = bluesky_error report.polymarket_error = polymarket_error report.web_error = web_error report.resolved_x_handle = args.x_handle @@ -1827,6 +1893,16 @@ def main(): "engagement_score": item.engagement.score if item.engagement else 0, "relevance_score": item.relevance, }) + for item in deduped_bsky: + findings.append({ + "source": "bluesky", + "url": item.url, + "title": item.text[:100], + "author": item.author_handle, + "content": item.text, + "engagement_score": item.engagement.likes if item.engagement else 0, + "relevance_score": item.relevance, + }) for item in deduped_pm: findings.append({ "source": "polymarket", diff --git a/scripts/lib/bluesky.py b/scripts/lib/bluesky.py new file mode 100644 index 0000000..8f9cbfb --- /dev/null +++ b/scripts/lib/bluesky.py @@ -0,0 +1,166 @@ +"""Bluesky search via AT Protocol (free, no auth required). + +Uses public.api.bsky.app for post discovery. +No API key needed - just HTTP calls via stdlib urllib. +""" + +import math +import re +import sys +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from . import http + +BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts" + +DEPTH_CONFIG = { + "quick": 15, + "default": 30, + "deep": 60, +} + + +def _log(msg: str): + """Log to stderr (only in TTY mode to avoid cluttering Claude Code output).""" + if sys.stderr.isatty(): + sys.stderr.write(f"[Bluesky] {msg}\n") + sys.stderr.flush() + + +def _extract_core_subject(topic: str) -> str: + """Extract core subject from verbose query for Bluesky search.""" + text = topic.lower().strip() + prefixes = [ + 'what are the best', 'what is the best', 'what are the latest', + 'what are people saying about', 'what do people think about', + 'how do i use', 'how to use', 'how to', + 'what are', 'what is', 'tips for', 'best practices for', + ] + for p in prefixes: + if text.startswith(p + ' '): + text = text[len(p):].strip() + noise = { + 'best', 'top', 'good', 'great', 'awesome', + 'latest', 'new', 'news', 'update', 'updates', + 'trending', 'hottest', 'popular', 'viral', + 'practices', 'features', 'recommendations', 'advice', + } + words = text.split() + filtered = [w for w in words if w not in noise] + result = ' '.join(filtered) if filtered else text + return result.rstrip('?!.') + + +def _parse_date(item: Dict[str, Any]) -> Optional[str]: + """Parse date from Bluesky post to YYYY-MM-DD. + + AT Protocol uses ISO 8601 format in indexedAt and createdAt fields. + """ + for key in ("indexedAt", "createdAt"): + val = item.get(key) + if val and isinstance(val, str): + try: + dt = datetime.fromisoformat(val.replace("Z", "+00:00")) + return dt.strftime("%Y-%m-%d") + except (ValueError, TypeError): + pass + return None + + +def search_bluesky( + topic: str, + from_date: str, + to_date: str, + depth: str = "default", +) -> Dict[str, Any]: + """Search Bluesky via AT Protocol public API. + + Args: + topic: Search topic + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) + depth: 'quick', 'default', or 'deep' + + Returns: + Dict with 'posts' list from AT Protocol response. + """ + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + core_topic = _extract_core_subject(topic) + + _log(f"Searching for '{core_topic}' (depth={depth}, limit={count})") + + from urllib.parse import urlencode + params = { + "q": core_topic, + "limit": str(min(count, 100)), + "sort": "top", + } + url = f"{BSKY_SEARCH_URL}?{urlencode(params)}" + + try: + response = http.request("GET", url, timeout=30) + except http.HTTPError as e: + _log(f"Search failed: {e}") + return {"posts": [], "error": str(e)} + except Exception as e: + _log(f"Search failed: {e}") + return {"posts": [], "error": str(e)} + + posts = response.get("posts", []) + _log(f"Found {len(posts)} posts") + return response + + +def parse_bluesky_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse AT Protocol response into normalized item dicts. + + Returns: + List of item dicts ready for normalization. + """ + posts = response.get("posts", []) + items = [] + + for i, post in enumerate(posts): + record = post.get("record") or {} + text = record.get("text") or "" + + author = post.get("author") or {} + handle = author.get("handle") or "" + display_name = author.get("displayName") or handle + + # Post URI -> URL + # URI format: at://did:plc:xxx/app.bsky.feed.post/rkey + uri = post.get("uri") or "" + rkey = uri.rsplit("/", 1)[-1] if uri else "" + url = f"https://bsky.app/profile/{handle}/post/{rkey}" if handle and rkey else "" + + likes = post.get("likeCount") or 0 + reposts = post.get("repostCount") or 0 + replies = post.get("replyCount") or 0 + quotes = post.get("quoteCount") or 0 + + date_str = _parse_date(post) or _parse_date(record) + + # Relevance: position-based (AT Protocol sorts by relevance with sort=top) + rank_score = max(0.3, 1.0 - (i * 0.02)) + engagement_boost = min(0.2, math.log1p(likes + reposts) / 40) + relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1) + + items.append({ + "handle": handle, + "display_name": display_name, + "text": text, + "url": url, + "date": date_str, + "engagement": { + "likes": likes, + "reposts": reposts, + "replies": replies, + "quotes": quotes, + }, + "relevance": round(relevance, 2), + "why_relevant": f"Bluesky: @{handle}: {text[:60]}" if text else f"Bluesky: {handle}", + }) + + return items diff --git a/scripts/lib/dedupe.py b/scripts/lib/dedupe.py index 7497ade..0fbeb22 100644 --- a/scripts/lib/dedupe.py +++ b/scripts/lib/dedupe.py @@ -226,6 +226,14 @@ def dedupe_hackernews( return dedupe_items(items, threshold) +def dedupe_bluesky( + items: List[schema.BlueskyItem], + threshold: float = 0.7, +) -> List[schema.BlueskyItem]: + """Dedupe Bluesky items.""" + return dedupe_items(items, threshold) + + def dedupe_polymarket( items: List[schema.PolymarketItem], threshold: float = 0.7, diff --git a/scripts/lib/env.py b/scripts/lib/env.py index a270439..020684f 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -480,6 +480,14 @@ def is_hackernews_available() -> bool: return True +def is_bluesky_available() -> bool: + """Check if Bluesky source is available. + + Always returns True - AT Protocol search is free, no key needed. + """ + return True + + def is_polymarket_available() -> bool: """Check if Polymarket source is available. diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py index 2c1c626..45ad236 100644 --- a/scripts/lib/normalize.py +++ b/scripts/lib/normalize.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union from . import dates, schema -T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem) +T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.PolymarketItem) def filter_by_date_range( @@ -350,6 +350,50 @@ def normalize_hackernews_items( return normalized +def normalize_bluesky_items( + items: List[Dict[str, Any]], + from_date: str, + to_date: str, +) -> List[schema.BlueskyItem]: + """Normalize raw Bluesky items to schema. + + Args: + items: Raw Bluesky items from AT Protocol API + from_date: Start of date range + to_date: End of date range + + Returns: + List of BlueskyItem objects + """ + normalized = [] + + for i, item in enumerate(items): + eng_raw = item.get("engagement") or {} + engagement = schema.Engagement( + likes=eng_raw.get("likes"), + reposts=eng_raw.get("reposts"), + replies=eng_raw.get("replies"), + quotes=eng_raw.get("quotes"), + ) + + date_str = item.get("date") + + normalized.append(schema.BlueskyItem( + id=f"BS{i+1}", + text=item.get("text", ""), + url=item.get("url", ""), + author_handle=item.get("handle", ""), + display_name=item.get("display_name", ""), + date=date_str, + date_confidence="high", + engagement=engagement, + relevance=item.get("relevance", 0.5), + why_relevant=item.get("why_relevant", ""), + )) + + return normalized + + def normalize_polymarket_items( items: List[Dict[str, Any]], from_date: str, diff --git a/scripts/lib/render.py b/scripts/lib/render.py index 7946b8c..e54ec38 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -59,13 +59,14 @@ def _assess_data_freshness(report: schema.Report) -> dict: x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from) web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from) hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from) + bsky_recent = sum(1 for b in report.bluesky if b.date and b.date >= report.range_from) pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from) tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from) ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from) - total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent + ig_recent - total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok) + len(report.instagram) + total_recent = reddit_recent + x_recent + web_recent + hn_recent + bsky_recent + pm_recent + tiktok_recent + ig_recent + total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.bluesky) + len(report.polymarket) + len(report.tiktok) + len(report.instagram) return { "reddit_recent": reddit_recent, @@ -367,6 +368,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append("") + # Bluesky items + if report.bluesky_error: + lines.append("### Bluesky Posts") + lines.append("") + lines.append(f"**ERROR:** {report.bluesky_error}") + lines.append("") + elif report.bluesky: + lines.append("### Bluesky Posts") + lines.append("") + for item in report.bluesky[:limit]: + eng_str = "" + if item.engagement: + eng = item.engagement + parts = [] + if eng.likes is not None: + parts.append(f"{eng.likes}lk") + if eng.reposts is not None: + parts.append(f"{eng.reposts}rp") + if eng.replies is not None: + parts.append(f"{eng.replies}re") + if parts: + eng_str = f" [{', '.join(parts)}]" + + date_str = f" ({item.date})" if item.date else "" + + lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{eng_str}{_xref_tag(item)}") + if item.text: + snippet = item.text[:200] + if len(item.text) > 200: + snippet += "..." + lines.append(f" {snippet}") + if item.url: + lines.append(f" {item.url}") + lines.append(f" *{item.why_relevant}*") + lines.append("") + # Polymarket items if report.polymarket_error: lines.append("### Prediction Markets (Polymarket)") @@ -531,6 +568,13 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str lines.append(f" ✅ HN: {len(report.hackernews)} stories") # Hide when zero results + # Bluesky + if report.bluesky_error: + lines.append(f" ❌ Bluesky: error - {report.bluesky_error}") + elif report.bluesky: + lines.append(f" ✅ Bluesky: {len(report.bluesky)} posts") + # Hide when zero results + # Polymarket if report.polymarket_error: lines.append(f" ❌ Polymarket: error - {report.polymarket_error}") @@ -581,6 +625,8 @@ def render_context_snippet(report: schema.Report) -> str: all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url)) for item in report.hackernews[:5]: all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url)) + for item in report.bluesky[:5]: + all_items.append((item.score, "Bluesky", item.text[:50] + "...", item.url)) for item in report.polymarket[:5]: all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url)) for item in report.web[:5]: @@ -754,6 +800,26 @@ def render_full_report(report: schema.Report) -> str: lines.append("") + # Bluesky section + if report.bluesky: + lines.append("## Bluesky Posts") + lines.append("") + for item in report.bluesky: + lines.append(f"### {item.id}: @{item.author_handle}") + lines.append("") + lines.append(f"- **URL:** {item.url}") + lines.append(f"- **Date:** {item.date or 'Unknown'}") + lines.append(f"- **Score:** {item.score}/100") + lines.append(f"- **Relevance:** {item.why_relevant}") + + if item.engagement: + eng = item.engagement + lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts, {eng.replies or '?'} replies") + + lines.append("") + lines.append(f"> {item.text[:300]}") + lines.append("") + # Polymarket section if report.polymarket: lines.append("## Prediction Markets (Polymarket)") diff --git a/scripts/lib/schema.py b/scripts/lib/schema.py index fa080f0..d195000 100644 --- a/scripts/lib/schema.py +++ b/scripts/lib/schema.py @@ -355,6 +355,43 @@ class HackerNewsItem: return d +@dataclass +class BlueskyItem: + """Normalized Bluesky post.""" + id: str # "BS1", "BS2", ... + text: str + url: str # bsky.app permalink + author_handle: str # user.bsky.social + display_name: str + date: Optional[str] = None + date_confidence: str = "high" # AT Protocol has exact timestamps + engagement: Optional[Engagement] = None # likes, reposts, replies, quotes + relevance: float = 0.5 + why_relevant: str = "" + subs: SubScores = field(default_factory=SubScores) + score: int = 0 + cross_refs: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + d = { + 'id': self.id, + 'text': self.text, + 'url': self.url, + 'author_handle': self.author_handle, + 'display_name': self.display_name, + 'date': self.date, + 'date_confidence': self.date_confidence, + 'engagement': self.engagement.to_dict() if self.engagement else None, + 'relevance': self.relevance, + 'why_relevant': self.why_relevant, + 'subs': self.subs.to_dict(), + 'score': self.score, + } + if self.cross_refs: + d['cross_refs'] = self.cross_refs + return d + + @dataclass class PolymarketItem: """Normalized Polymarket prediction market item.""" @@ -415,6 +452,7 @@ class Report: tiktok: List[TikTokItem] = field(default_factory=list) instagram: List[InstagramItem] = field(default_factory=list) hackernews: List[HackerNewsItem] = field(default_factory=list) + bluesky: List[BlueskyItem] = field(default_factory=list) polymarket: List[PolymarketItem] = field(default_factory=list) best_practices: List[str] = field(default_factory=list) prompt_pack: List[str] = field(default_factory=list) @@ -427,6 +465,7 @@ class Report: tiktok_error: Optional[str] = None instagram_error: Optional[str] = None hackernews_error: Optional[str] = None + bluesky_error: Optional[str] = None polymarket_error: Optional[str] = None # Handle resolution resolved_x_handle: Optional[str] = None @@ -452,6 +491,7 @@ class Report: 'tiktok': [t.to_dict() for t in self.tiktok], 'instagram': [ig.to_dict() for ig in self.instagram], 'hackernews': [h.to_dict() for h in self.hackernews], + 'bluesky': [b.to_dict() for b in self.bluesky], 'polymarket': [p.to_dict() for p in self.polymarket], 'best_practices': self.best_practices, 'prompt_pack': self.prompt_pack, @@ -473,6 +513,8 @@ class Report: d['instagram_error'] = self.instagram_error if self.hackernews_error: d['hackernews_error'] = self.hackernews_error + if self.bluesky_error: + d['bluesky_error'] = self.bluesky_error if self.polymarket_error: d['polymarket_error'] = self.polymarket_error if self.from_cache: diff --git a/scripts/lib/score.py b/scripts/lib/score.py index 0b49e20..e876330 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -468,6 +468,66 @@ def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.Ha return items +def compute_bluesky_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: + """Compute raw engagement score for Bluesky item. + + Formula: 0.40*log1p(likes) + 0.30*log1p(reposts) + 0.20*log1p(replies) + 0.10*log1p(quotes) + Likes are primary signal; reposts indicate reach; replies indicate discussion depth. + """ + if engagement is None: + return None + + if engagement.likes is None and engagement.reposts is None: + return None + + likes = log1p_safe(engagement.likes) + reposts = log1p_safe(engagement.reposts) + replies = log1p_safe(engagement.replies) + quotes = log1p_safe(engagement.quotes) + + return 0.40 * likes + 0.30 * reposts + 0.20 * replies + 0.10 * quotes + + +def score_bluesky_items(items: List[schema.BlueskyItem]) -> List[schema.BlueskyItem]: + """Compute scores for Bluesky items. + + Uses same weight structure as Reddit/X (relevance + recency + engagement). + """ + if not items: + return items + + eng_raw = [compute_bluesky_engagement_raw(item.engagement) for item in items] + eng_normalized = normalize_to_100(eng_raw) + + for i, item in enumerate(items): + rel_score = int(item.relevance * 100) + rec_score = dates.recency_score(item.date) + + if eng_normalized[i] is not None: + eng_score = int(eng_normalized[i]) + else: + eng_score = DEFAULT_ENGAGEMENT + + item.subs = schema.SubScores( + relevance=rel_score, + recency=rec_score, + engagement=eng_score, + ) + + overall = ( + WEIGHT_RELEVANCE * rel_score + + WEIGHT_RECENCY * rec_score + + WEIGHT_ENGAGEMENT * eng_score + ) + + if eng_raw[i] is None: + overall -= UNKNOWN_ENGAGEMENT_PENALTY + + item.score = max(0, min(100, int(overall))) + + return items + + def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: """Compute raw engagement score for Polymarket item. diff --git a/tests/test_bluesky.py b/tests/test_bluesky.py new file mode 100644 index 0000000..8b0945d --- /dev/null +++ b/tests/test_bluesky.py @@ -0,0 +1,103 @@ +"""Tests for bluesky module.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) +from lib import bluesky + + +class TestExtractCoreSubject(unittest.TestCase): + def test_strips_prefix(self): + result = bluesky._extract_core_subject("what are people saying about claude code") + self.assertEqual(result, "claude code") + + def test_strips_noise(self): + result = bluesky._extract_core_subject("latest trending news claude code") + self.assertNotIn("latest", result) + self.assertNotIn("trending", result) + self.assertIn("claude", result) + + def test_preserves_core(self): + result = bluesky._extract_core_subject("react native") + self.assertEqual(result, "react native") + + +class TestParseDate(unittest.TestCase): + def test_indexed_at_iso(self): + item = {"indexedAt": "2024-06-15T12:00:00Z"} + self.assertEqual(bluesky._parse_date(item), "2024-06-15") + + def test_created_at_iso(self): + item = {"createdAt": "2024-03-01T08:30:00.000Z"} + self.assertEqual(bluesky._parse_date(item), "2024-03-01") + + def test_indexed_at_preferred_over_created_at(self): + item = {"indexedAt": "2024-06-15T12:00:00Z", "createdAt": "2024-06-14T12:00:00Z"} + self.assertEqual(bluesky._parse_date(item), "2024-06-15") + + def test_none_returns_none(self): + self.assertIsNone(bluesky._parse_date({})) + + def test_invalid_date_returns_none(self): + self.assertIsNone(bluesky._parse_date({"indexedAt": "not-a-date"})) + + +class TestParseBlueskyResponse(unittest.TestCase): + def test_basic_post(self): + response = { + "posts": [{ + "uri": "at://did:plc:abc123/app.bsky.feed.post/xyz789", + "author": {"handle": "alice.bsky.social", "displayName": "Alice"}, + "record": {"text": "Hello world", "createdAt": "2024-06-15T12:00:00Z"}, + "indexedAt": "2024-06-15T12:01:00Z", + "likeCount": 10, + "repostCount": 5, + "replyCount": 3, + "quoteCount": 1, + }] + } + items = bluesky.parse_bluesky_response(response) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["handle"], "alice.bsky.social") + self.assertEqual(items[0]["display_name"], "Alice") + self.assertEqual(items[0]["text"], "Hello world") + self.assertEqual(items[0]["url"], "https://bsky.app/profile/alice.bsky.social/post/xyz789") + self.assertEqual(items[0]["engagement"]["likes"], 10) + self.assertEqual(items[0]["engagement"]["reposts"], 5) + self.assertEqual(items[0]["date"], "2024-06-15") + + def test_empty_response(self): + items = bluesky.parse_bluesky_response({}) + self.assertEqual(items, []) + + def test_missing_fields(self): + response = {"posts": [{"uri": "", "author": {}, "record": {}}]} + items = bluesky.parse_bluesky_response(response) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["handle"], "") + self.assertEqual(items[0]["text"], "") + + def test_relevance_decreases_with_position(self): + response = {"posts": [ + {"uri": f"at://did/app.bsky.feed.post/{i}", "author": {"handle": f"u{i}"}, "record": {"text": f"post {i}"}} + for i in range(5) + ]} + items = bluesky.parse_bluesky_response(response) + self.assertGreater(items[0]["relevance"], items[4]["relevance"]) + + +class TestDepthConfig(unittest.TestCase): + def test_all_depths_exist(self): + for depth in ("quick", "default", "deep"): + self.assertIn(depth, bluesky.DEPTH_CONFIG) + + def test_deep_has_more_results(self): + quick = bluesky.DEPTH_CONFIG["quick"] + deep = bluesky.DEPTH_CONFIG["deep"] + self.assertGreater(deep, quick) + + +if __name__ == "__main__": + unittest.main()