feat(bluesky): add Bluesky/AT Protocol as social source
Free, no-auth-required search via public.api.bsky.app. Always-on like HN and Polymarket (no API key needed). - New scripts/lib/bluesky.py: search + parse via AT Protocol - BlueskyItem schema, normalization, scoring, deduplication - Wired into orchestrator ThreadPoolExecutor with timeout config - Rendering in compact, full, and JSON output modes - 14 unit tests covering parsing, dates, relevance, edge cases - --search=bluesky / --search=bsky for bluesky-only mode Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+86
-10
@@ -38,14 +38,14 @@ _child_pids: set = set()
|
|||||||
_child_pids_lock = threading.Lock()
|
_child_pids_lock = threading.Lock()
|
||||||
|
|
||||||
TIMEOUT_PROFILES = {
|
TIMEOUT_PROFILES = {
|
||||||
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
|
||||||
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
|
||||||
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Valid source names for the --search flag
|
# Valid source names for the --search flag
|
||||||
VALID_SEARCH_SOURCES = {
|
VALID_SEARCH_SOURCES = {
|
||||||
"reddit", "x", "hn", "youtube", "tiktok", "instagram",
|
"reddit", "x", "hn", "bluesky", "bsky", "youtube", "tiktok", "instagram",
|
||||||
"polymarket", "web", "xiaohongshu", "xhs",
|
"polymarket", "web", "xiaohongshu", "xhs",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,6 +135,7 @@ def _install_global_timeout(timeout_seconds: int):
|
|||||||
|
|
||||||
from lib import (
|
from lib import (
|
||||||
bird_x,
|
bird_x,
|
||||||
|
bluesky,
|
||||||
dates,
|
dates,
|
||||||
dedupe,
|
dedupe,
|
||||||
hackernews,
|
hackernews,
|
||||||
@@ -513,6 +514,34 @@ def _search_hackernews(
|
|||||||
return hn_items, hn_error
|
return hn_items, hn_error
|
||||||
|
|
||||||
|
|
||||||
|
def _search_bluesky(
|
||||||
|
topic: str,
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
depth: str,
|
||||||
|
) -> tuple:
|
||||||
|
"""Search Bluesky via AT Protocol (runs in thread).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (bsky_items, bsky_error)
|
||||||
|
"""
|
||||||
|
bsky_error = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = bluesky.search_bluesky(
|
||||||
|
topic, from_date, to_date, depth=depth,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
return [], f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
bsky_items = bluesky.parse_bluesky_response(response)
|
||||||
|
|
||||||
|
if response.get("error"):
|
||||||
|
bsky_error = response["error"]
|
||||||
|
|
||||||
|
return bsky_items, bsky_error
|
||||||
|
|
||||||
|
|
||||||
def _search_polymarket(
|
def _search_polymarket(
|
||||||
topic: str,
|
topic: str,
|
||||||
from_date: str,
|
from_date: str,
|
||||||
@@ -823,6 +852,7 @@ def run_research(
|
|||||||
timeouts: dict = None,
|
timeouts: dict = None,
|
||||||
resolved_handle: str = None,
|
resolved_handle: str = None,
|
||||||
do_hackernews: bool = True,
|
do_hackernews: bool = True,
|
||||||
|
do_bluesky: bool = True,
|
||||||
do_polymarket: bool = True,
|
do_polymarket: bool = True,
|
||||||
no_native_web: bool = False,
|
no_native_web: bool = False,
|
||||||
) -> tuple:
|
) -> tuple:
|
||||||
@@ -830,10 +860,10 @@ def run_research(
|
|||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
|
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
|
||||||
hackernews_items, polymarket_items, web_items, web_needed,
|
hackernews_items, bluesky_items, polymarket_items, web_items, web_needed,
|
||||||
raw_openai, raw_xai, raw_reddit_enriched,
|
raw_openai, raw_xai, raw_reddit_enriched,
|
||||||
reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
|
reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
|
||||||
hackernews_error, polymarket_error, web_error)
|
hackernews_error, bluesky_error, polymarket_error, web_error)
|
||||||
|
|
||||||
Note: web_needed is True when web search should be performed by the assistant
|
Note: web_needed is True when web search should be performed by the assistant
|
||||||
(i.e., no native web search API keys are configured). When native web search
|
(i.e., no native web search API keys are configured). When native web search
|
||||||
@@ -849,6 +879,7 @@ def run_research(
|
|||||||
tiktok_items = []
|
tiktok_items = []
|
||||||
instagram_items = []
|
instagram_items = []
|
||||||
hackernews_items = []
|
hackernews_items = []
|
||||||
|
bluesky_items = []
|
||||||
polymarket_items = []
|
polymarket_items = []
|
||||||
web_items = []
|
web_items = []
|
||||||
raw_openai = None
|
raw_openai = None
|
||||||
@@ -860,6 +891,7 @@ def run_research(
|
|||||||
tiktok_error = None
|
tiktok_error = None
|
||||||
instagram_error = None
|
instagram_error = None
|
||||||
hackernews_error = None
|
hackernews_error = None
|
||||||
|
bluesky_error = None
|
||||||
polymarket_error = None
|
polymarket_error = None
|
||||||
web_error = None
|
web_error = None
|
||||||
xiaohongshu_error = None
|
xiaohongshu_error = None
|
||||||
@@ -944,7 +976,7 @@ def run_research(
|
|||||||
progress.show_error(f"Instagram error: {e}")
|
progress.show_error(f"Instagram error: {e}")
|
||||||
if progress:
|
if progress:
|
||||||
progress.end_instagram(len(instagram_items))
|
progress.end_instagram(len(instagram_items))
|
||||||
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
|
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
|
||||||
|
|
||||||
# Determine which searches to run
|
# Determine which searches to run
|
||||||
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
|
||||||
@@ -960,6 +992,7 @@ def run_research(
|
|||||||
instagram_future = None
|
instagram_future = None
|
||||||
xiaohongshu_future = None
|
xiaohongshu_future = None
|
||||||
hackernews_future = None
|
hackernews_future = None
|
||||||
|
bluesky_future = None
|
||||||
polymarket_future = None
|
polymarket_future = None
|
||||||
web_future = None
|
web_future = None
|
||||||
max_workers = (
|
max_workers = (
|
||||||
@@ -969,6 +1002,7 @@ def run_research(
|
|||||||
+ (1 if run_instagram else 0)
|
+ (1 if run_instagram else 0)
|
||||||
+ (1 if run_xiaohongshu else 0)
|
+ (1 if run_xiaohongshu else 0)
|
||||||
+ (1 if do_hackernews else 0)
|
+ (1 if do_hackernews else 0)
|
||||||
|
+ (1 if do_bluesky else 0)
|
||||||
+ (1 if do_polymarket else 0)
|
+ (1 if do_polymarket else 0)
|
||||||
+ (1 if web_backend else 0)
|
+ (1 if web_backend else 0)
|
||||||
)
|
)
|
||||||
@@ -1026,6 +1060,11 @@ def run_research(
|
|||||||
_search_hackernews, topic, from_date, to_date, depth
|
_search_hackernews, topic, from_date, to_date, depth
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if do_bluesky:
|
||||||
|
bluesky_future = executor.submit(
|
||||||
|
_search_bluesky, topic, from_date, to_date, depth
|
||||||
|
)
|
||||||
|
|
||||||
if do_polymarket:
|
if do_polymarket:
|
||||||
if progress:
|
if progress:
|
||||||
progress.start_polymarket()
|
progress.start_polymarket()
|
||||||
@@ -1158,6 +1197,21 @@ def run_research(
|
|||||||
if progress:
|
if progress:
|
||||||
progress.end_hackernews(len(hackernews_items))
|
progress.end_hackernews(len(hackernews_items))
|
||||||
|
|
||||||
|
if bluesky_future:
|
||||||
|
bsky_timeout = timeouts.get("bluesky_future", future_timeout)
|
||||||
|
try:
|
||||||
|
bluesky_items, bluesky_error = bluesky_future.result(timeout=bsky_timeout)
|
||||||
|
if bluesky_error and progress:
|
||||||
|
progress.show_error(f"Bluesky error: {bluesky_error}")
|
||||||
|
except TimeoutError:
|
||||||
|
bluesky_error = f"Bluesky search timed out after {bsky_timeout}s"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(bluesky_error)
|
||||||
|
except Exception as e:
|
||||||
|
bluesky_error = f"{type(e).__name__}: {e}"
|
||||||
|
if progress:
|
||||||
|
progress.show_error(f"Bluesky error: {e}")
|
||||||
|
|
||||||
if polymarket_future:
|
if polymarket_future:
|
||||||
pm_timeout = timeouts.get("polymarket_future", future_timeout)
|
pm_timeout = timeouts.get("polymarket_future", future_timeout)
|
||||||
try:
|
try:
|
||||||
@@ -1292,7 +1346,7 @@ def run_research(
|
|||||||
if sup_x:
|
if sup_x:
|
||||||
x_items.extend(sup_x)
|
x_items.extend(sup_x)
|
||||||
|
|
||||||
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error
|
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -1460,6 +1514,7 @@ def main():
|
|||||||
"xiaohongshu": has_xiaohongshu,
|
"xiaohongshu": has_xiaohongshu,
|
||||||
"xiaohongshu_api_base": env.get_xiaohongshu_api_base(config),
|
"xiaohongshu_api_base": env.get_xiaohongshu_api_base(config),
|
||||||
"hackernews": True,
|
"hackernews": True,
|
||||||
|
"bluesky": True,
|
||||||
"polymarket": True,
|
"polymarket": True,
|
||||||
"web_search_backend": web_source,
|
"web_search_backend": web_source,
|
||||||
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
|
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
|
||||||
@@ -1493,6 +1548,7 @@ def main():
|
|||||||
"instagram": has_instagram,
|
"instagram": has_instagram,
|
||||||
"xiaohongshu": has_xiaohongshu,
|
"xiaohongshu": has_xiaohongshu,
|
||||||
"hackernews": True,
|
"hackernews": True,
|
||||||
|
"bluesky": True,
|
||||||
"polymarket": True,
|
"polymarket": True,
|
||||||
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
|
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
|
||||||
}
|
}
|
||||||
@@ -1574,6 +1630,7 @@ def main():
|
|||||||
|
|
||||||
# Apply --search flag: restrict sources to the specified subset
|
# Apply --search flag: restrict sources to the specified subset
|
||||||
search_do_hackernews = True
|
search_do_hackernews = True
|
||||||
|
search_do_bluesky = True
|
||||||
search_do_polymarket = True
|
search_do_polymarket = True
|
||||||
search_run_youtube = has_ytdlp
|
search_run_youtube = has_ytdlp
|
||||||
search_run_tiktok = has_tiktok
|
search_run_tiktok = has_tiktok
|
||||||
@@ -1584,6 +1641,7 @@ def main():
|
|||||||
has_reddit = "reddit" in search_sources
|
has_reddit = "reddit" in search_sources
|
||||||
has_x = "x" in search_sources
|
has_x = "x" in search_sources
|
||||||
search_do_hackernews = "hn" in search_sources
|
search_do_hackernews = "hn" in search_sources
|
||||||
|
search_do_bluesky = "bluesky" in search_sources or "bsky" in search_sources
|
||||||
search_do_polymarket = "polymarket" in search_sources
|
search_do_polymarket = "polymarket" in search_sources
|
||||||
search_run_youtube = "youtube" in search_sources and has_ytdlp
|
search_run_youtube = "youtube" in search_sources and has_ytdlp
|
||||||
search_run_tiktok = "tiktok" in search_sources and has_tiktok
|
search_run_tiktok = "tiktok" in search_sources and has_tiktok
|
||||||
@@ -1603,7 +1661,7 @@ def main():
|
|||||||
sources = "web" # hn/polymarket only; no Reddit/X
|
sources = "web" # hn/polymarket only; no Reddit/X
|
||||||
|
|
||||||
# Run research
|
# Run research
|
||||||
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, polymarket_error, web_error = run_research(
|
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error = run_research(
|
||||||
args.topic,
|
args.topic,
|
||||||
sources,
|
sources,
|
||||||
config,
|
config,
|
||||||
@@ -1621,6 +1679,7 @@ def main():
|
|||||||
timeouts=timeouts,
|
timeouts=timeouts,
|
||||||
resolved_handle=args.x_handle,
|
resolved_handle=args.x_handle,
|
||||||
do_hackernews=search_do_hackernews,
|
do_hackernews=search_do_hackernews,
|
||||||
|
do_bluesky=search_do_bluesky,
|
||||||
do_polymarket=search_do_polymarket,
|
do_polymarket=search_do_polymarket,
|
||||||
no_native_web=args.no_native_web,
|
no_native_web=args.no_native_web,
|
||||||
)
|
)
|
||||||
@@ -1635,6 +1694,7 @@ def main():
|
|||||||
normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else []
|
normalized_tiktok = normalize.normalize_tiktok_items(tiktok_items, from_date, to_date) if tiktok_items else []
|
||||||
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
|
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
|
||||||
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
|
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
|
||||||
|
normalized_bsky = normalize.normalize_bluesky_items(bluesky_items, from_date, to_date) if bluesky_items else []
|
||||||
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
|
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
|
||||||
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
|
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
|
||||||
|
|
||||||
@@ -1651,6 +1711,7 @@ def main():
|
|||||||
# Instagram: hard date filter (instagram.py already pre-filters, but safety net)
|
# Instagram: hard date filter (instagram.py already pre-filters, but safety net)
|
||||||
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
|
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
|
||||||
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
|
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
|
||||||
|
filtered_bsky = normalize.filter_by_date_range(normalized_bsky, from_date, to_date) if normalized_bsky else []
|
||||||
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
|
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
|
||||||
filtered_pm = normalized_pm
|
filtered_pm = normalized_pm
|
||||||
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
|
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
|
||||||
@@ -1662,6 +1723,7 @@ def main():
|
|||||||
scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else []
|
scored_tiktok = score.score_tiktok_items(filtered_tiktok) if filtered_tiktok else []
|
||||||
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
|
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
|
||||||
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
|
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
|
||||||
|
scored_bsky = score.score_bluesky_items(filtered_bsky) if filtered_bsky else []
|
||||||
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
|
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
|
||||||
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
|
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
|
||||||
|
|
||||||
@@ -1672,6 +1734,7 @@ def main():
|
|||||||
sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else []
|
sorted_tiktok = score.sort_items(scored_tiktok) if scored_tiktok else []
|
||||||
sorted_ig = score.sort_items(scored_ig) if scored_ig else []
|
sorted_ig = score.sort_items(scored_ig) if scored_ig else []
|
||||||
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
|
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
|
||||||
|
sorted_bsky = score.sort_items(scored_bsky) if scored_bsky else []
|
||||||
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
|
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
|
||||||
sorted_web = score.sort_items(scored_web) if scored_web else []
|
sorted_web = score.sort_items(scored_web) if scored_web else []
|
||||||
|
|
||||||
@@ -1682,6 +1745,7 @@ def main():
|
|||||||
deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else []
|
deduped_tiktok = dedupe.dedupe_tiktok(sorted_tiktok) if sorted_tiktok else []
|
||||||
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
|
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
|
||||||
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
|
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
|
||||||
|
deduped_bsky = dedupe.dedupe_bluesky(sorted_bsky) if sorted_bsky else []
|
||||||
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
|
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
|
||||||
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
|
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
|
||||||
|
|
||||||
@@ -1694,7 +1758,7 @@ def main():
|
|||||||
|
|
||||||
# Cross-source linking: annotate items that discuss the same story
|
# Cross-source linking: annotate items that discuss the same story
|
||||||
dedupe.cross_source_link(
|
dedupe.cross_source_link(
|
||||||
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_pm, deduped_web,
|
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_pm, deduped_web,
|
||||||
)
|
)
|
||||||
|
|
||||||
progress.end_processing()
|
progress.end_processing()
|
||||||
@@ -1714,6 +1778,7 @@ def main():
|
|||||||
report.tiktok = deduped_tiktok
|
report.tiktok = deduped_tiktok
|
||||||
report.instagram = deduped_ig
|
report.instagram = deduped_ig
|
||||||
report.hackernews = deduped_hn
|
report.hackernews = deduped_hn
|
||||||
|
report.bluesky = deduped_bsky
|
||||||
report.polymarket = deduped_pm
|
report.polymarket = deduped_pm
|
||||||
report.web = deduped_web
|
report.web = deduped_web
|
||||||
report.reddit_error = reddit_error
|
report.reddit_error = reddit_error
|
||||||
@@ -1722,6 +1787,7 @@ def main():
|
|||||||
report.tiktok_error = tiktok_error
|
report.tiktok_error = tiktok_error
|
||||||
report.instagram_error = instagram_error
|
report.instagram_error = instagram_error
|
||||||
report.hackernews_error = hackernews_error
|
report.hackernews_error = hackernews_error
|
||||||
|
report.bluesky_error = bluesky_error
|
||||||
report.polymarket_error = polymarket_error
|
report.polymarket_error = polymarket_error
|
||||||
report.web_error = web_error
|
report.web_error = web_error
|
||||||
report.resolved_x_handle = args.x_handle
|
report.resolved_x_handle = args.x_handle
|
||||||
@@ -1827,6 +1893,16 @@ def main():
|
|||||||
"engagement_score": item.engagement.score if item.engagement else 0,
|
"engagement_score": item.engagement.score if item.engagement else 0,
|
||||||
"relevance_score": item.relevance,
|
"relevance_score": item.relevance,
|
||||||
})
|
})
|
||||||
|
for item in deduped_bsky:
|
||||||
|
findings.append({
|
||||||
|
"source": "bluesky",
|
||||||
|
"url": item.url,
|
||||||
|
"title": item.text[:100],
|
||||||
|
"author": item.author_handle,
|
||||||
|
"content": item.text,
|
||||||
|
"engagement_score": item.engagement.likes if item.engagement else 0,
|
||||||
|
"relevance_score": item.relevance,
|
||||||
|
})
|
||||||
for item in deduped_pm:
|
for item in deduped_pm:
|
||||||
findings.append({
|
findings.append({
|
||||||
"source": "polymarket",
|
"source": "polymarket",
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""Bluesky search via AT Protocol (free, no auth required).
|
||||||
|
|
||||||
|
Uses public.api.bsky.app for post discovery.
|
||||||
|
No API key needed - just HTTP calls via stdlib urllib.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from . import http
|
||||||
|
|
||||||
|
BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
|
||||||
|
|
||||||
|
DEPTH_CONFIG = {
|
||||||
|
"quick": 15,
|
||||||
|
"default": 30,
|
||||||
|
"deep": 60,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _log(msg: str):
|
||||||
|
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
|
||||||
|
if sys.stderr.isatty():
|
||||||
|
sys.stderr.write(f"[Bluesky] {msg}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_core_subject(topic: str) -> str:
|
||||||
|
"""Extract core subject from verbose query for Bluesky search."""
|
||||||
|
text = topic.lower().strip()
|
||||||
|
prefixes = [
|
||||||
|
'what are the best', 'what is the best', 'what are the latest',
|
||||||
|
'what are people saying about', 'what do people think about',
|
||||||
|
'how do i use', 'how to use', 'how to',
|
||||||
|
'what are', 'what is', 'tips for', 'best practices for',
|
||||||
|
]
|
||||||
|
for p in prefixes:
|
||||||
|
if text.startswith(p + ' '):
|
||||||
|
text = text[len(p):].strip()
|
||||||
|
noise = {
|
||||||
|
'best', 'top', 'good', 'great', 'awesome',
|
||||||
|
'latest', 'new', 'news', 'update', 'updates',
|
||||||
|
'trending', 'hottest', 'popular', 'viral',
|
||||||
|
'practices', 'features', 'recommendations', 'advice',
|
||||||
|
}
|
||||||
|
words = text.split()
|
||||||
|
filtered = [w for w in words if w not in noise]
|
||||||
|
result = ' '.join(filtered) if filtered else text
|
||||||
|
return result.rstrip('?!.')
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||||
|
"""Parse date from Bluesky post to YYYY-MM-DD.
|
||||||
|
|
||||||
|
AT Protocol uses ISO 8601 format in indexedAt and createdAt fields.
|
||||||
|
"""
|
||||||
|
for key in ("indexedAt", "createdAt"):
|
||||||
|
val = item.get(key)
|
||||||
|
if val and isinstance(val, str):
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
|
||||||
|
return dt.strftime("%Y-%m-%d")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def search_bluesky(
|
||||||
|
topic: str,
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
depth: str = "default",
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Search Bluesky via AT Protocol public API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
topic: Search topic
|
||||||
|
from_date: Start date (YYYY-MM-DD)
|
||||||
|
to_date: End date (YYYY-MM-DD)
|
||||||
|
depth: 'quick', 'default', or 'deep'
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'posts' list from AT Protocol response.
|
||||||
|
"""
|
||||||
|
count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
|
||||||
|
core_topic = _extract_core_subject(topic)
|
||||||
|
|
||||||
|
_log(f"Searching for '{core_topic}' (depth={depth}, limit={count})")
|
||||||
|
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
params = {
|
||||||
|
"q": core_topic,
|
||||||
|
"limit": str(min(count, 100)),
|
||||||
|
"sort": "top",
|
||||||
|
}
|
||||||
|
url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = http.request("GET", url, timeout=30)
|
||||||
|
except http.HTTPError as e:
|
||||||
|
_log(f"Search failed: {e}")
|
||||||
|
return {"posts": [], "error": str(e)}
|
||||||
|
except Exception as e:
|
||||||
|
_log(f"Search failed: {e}")
|
||||||
|
return {"posts": [], "error": str(e)}
|
||||||
|
|
||||||
|
posts = response.get("posts", [])
|
||||||
|
_log(f"Found {len(posts)} posts")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def parse_bluesky_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||||
|
"""Parse AT Protocol response into normalized item dicts.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of item dicts ready for normalization.
|
||||||
|
"""
|
||||||
|
posts = response.get("posts", [])
|
||||||
|
items = []
|
||||||
|
|
||||||
|
for i, post in enumerate(posts):
|
||||||
|
record = post.get("record") or {}
|
||||||
|
text = record.get("text") or ""
|
||||||
|
|
||||||
|
author = post.get("author") or {}
|
||||||
|
handle = author.get("handle") or ""
|
||||||
|
display_name = author.get("displayName") or handle
|
||||||
|
|
||||||
|
# Post URI -> URL
|
||||||
|
# URI format: at://did:plc:xxx/app.bsky.feed.post/rkey
|
||||||
|
uri = post.get("uri") or ""
|
||||||
|
rkey = uri.rsplit("/", 1)[-1] if uri else ""
|
||||||
|
url = f"https://bsky.app/profile/{handle}/post/{rkey}" if handle and rkey else ""
|
||||||
|
|
||||||
|
likes = post.get("likeCount") or 0
|
||||||
|
reposts = post.get("repostCount") or 0
|
||||||
|
replies = post.get("replyCount") or 0
|
||||||
|
quotes = post.get("quoteCount") or 0
|
||||||
|
|
||||||
|
date_str = _parse_date(post) or _parse_date(record)
|
||||||
|
|
||||||
|
# Relevance: position-based (AT Protocol sorts by relevance with sort=top)
|
||||||
|
rank_score = max(0.3, 1.0 - (i * 0.02))
|
||||||
|
engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
|
||||||
|
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
|
||||||
|
|
||||||
|
items.append({
|
||||||
|
"handle": handle,
|
||||||
|
"display_name": display_name,
|
||||||
|
"text": text,
|
||||||
|
"url": url,
|
||||||
|
"date": date_str,
|
||||||
|
"engagement": {
|
||||||
|
"likes": likes,
|
||||||
|
"reposts": reposts,
|
||||||
|
"replies": replies,
|
||||||
|
"quotes": quotes,
|
||||||
|
},
|
||||||
|
"relevance": round(relevance, 2),
|
||||||
|
"why_relevant": f"Bluesky: @{handle}: {text[:60]}" if text else f"Bluesky: {handle}",
|
||||||
|
})
|
||||||
|
|
||||||
|
return items
|
||||||
@@ -226,6 +226,14 @@ def dedupe_hackernews(
|
|||||||
return dedupe_items(items, threshold)
|
return dedupe_items(items, threshold)
|
||||||
|
|
||||||
|
|
||||||
|
def dedupe_bluesky(
|
||||||
|
items: List[schema.BlueskyItem],
|
||||||
|
threshold: float = 0.7,
|
||||||
|
) -> List[schema.BlueskyItem]:
|
||||||
|
"""Dedupe Bluesky items."""
|
||||||
|
return dedupe_items(items, threshold)
|
||||||
|
|
||||||
|
|
||||||
def dedupe_polymarket(
|
def dedupe_polymarket(
|
||||||
items: List[schema.PolymarketItem],
|
items: List[schema.PolymarketItem],
|
||||||
threshold: float = 0.7,
|
threshold: float = 0.7,
|
||||||
|
|||||||
@@ -480,6 +480,14 @@ def is_hackernews_available() -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def is_bluesky_available() -> bool:
|
||||||
|
"""Check if Bluesky source is available.
|
||||||
|
|
||||||
|
Always returns True - AT Protocol search is free, no key needed.
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def is_polymarket_available() -> bool:
|
def is_polymarket_available() -> bool:
|
||||||
"""Check if Polymarket source is available.
|
"""Check if Polymarket source is available.
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
|||||||
|
|
||||||
from . import dates, schema
|
from . import dates, schema
|
||||||
|
|
||||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.PolymarketItem)
|
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.TikTokItem, schema.InstagramItem, schema.HackerNewsItem, schema.BlueskyItem, schema.PolymarketItem)
|
||||||
|
|
||||||
|
|
||||||
def filter_by_date_range(
|
def filter_by_date_range(
|
||||||
@@ -350,6 +350,50 @@ def normalize_hackernews_items(
|
|||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_bluesky_items(
|
||||||
|
items: List[Dict[str, Any]],
|
||||||
|
from_date: str,
|
||||||
|
to_date: str,
|
||||||
|
) -> List[schema.BlueskyItem]:
|
||||||
|
"""Normalize raw Bluesky items to schema.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
items: Raw Bluesky items from AT Protocol API
|
||||||
|
from_date: Start of date range
|
||||||
|
to_date: End of date range
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of BlueskyItem objects
|
||||||
|
"""
|
||||||
|
normalized = []
|
||||||
|
|
||||||
|
for i, item in enumerate(items):
|
||||||
|
eng_raw = item.get("engagement") or {}
|
||||||
|
engagement = schema.Engagement(
|
||||||
|
likes=eng_raw.get("likes"),
|
||||||
|
reposts=eng_raw.get("reposts"),
|
||||||
|
replies=eng_raw.get("replies"),
|
||||||
|
quotes=eng_raw.get("quotes"),
|
||||||
|
)
|
||||||
|
|
||||||
|
date_str = item.get("date")
|
||||||
|
|
||||||
|
normalized.append(schema.BlueskyItem(
|
||||||
|
id=f"BS{i+1}",
|
||||||
|
text=item.get("text", ""),
|
||||||
|
url=item.get("url", ""),
|
||||||
|
author_handle=item.get("handle", ""),
|
||||||
|
display_name=item.get("display_name", ""),
|
||||||
|
date=date_str,
|
||||||
|
date_confidence="high",
|
||||||
|
engagement=engagement,
|
||||||
|
relevance=item.get("relevance", 0.5),
|
||||||
|
why_relevant=item.get("why_relevant", ""),
|
||||||
|
))
|
||||||
|
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def normalize_polymarket_items(
|
def normalize_polymarket_items(
|
||||||
items: List[Dict[str, Any]],
|
items: List[Dict[str, Any]],
|
||||||
from_date: str,
|
from_date: str,
|
||||||
|
|||||||
+68
-2
@@ -59,13 +59,14 @@ def _assess_data_freshness(report: schema.Report) -> dict:
|
|||||||
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
|
x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from)
|
||||||
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
|
web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from)
|
||||||
hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from)
|
hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from)
|
||||||
|
bsky_recent = sum(1 for b in report.bluesky if b.date and b.date >= report.range_from)
|
||||||
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
|
pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from)
|
||||||
|
|
||||||
tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from)
|
tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from)
|
||||||
ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from)
|
ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from)
|
||||||
|
|
||||||
total_recent = reddit_recent + x_recent + web_recent + hn_recent + pm_recent + tiktok_recent + ig_recent
|
total_recent = reddit_recent + x_recent + web_recent + hn_recent + bsky_recent + pm_recent + tiktok_recent + ig_recent
|
||||||
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.polymarket) + len(report.tiktok) + len(report.instagram)
|
total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.bluesky) + len(report.polymarket) + len(report.tiktok) + len(report.instagram)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"reddit_recent": reddit_recent,
|
"reddit_recent": reddit_recent,
|
||||||
@@ -367,6 +368,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = "
|
|||||||
|
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
# Bluesky items
|
||||||
|
if report.bluesky_error:
|
||||||
|
lines.append("### Bluesky Posts")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"**ERROR:** {report.bluesky_error}")
|
||||||
|
lines.append("")
|
||||||
|
elif report.bluesky:
|
||||||
|
lines.append("### Bluesky Posts")
|
||||||
|
lines.append("")
|
||||||
|
for item in report.bluesky[:limit]:
|
||||||
|
eng_str = ""
|
||||||
|
if item.engagement:
|
||||||
|
eng = item.engagement
|
||||||
|
parts = []
|
||||||
|
if eng.likes is not None:
|
||||||
|
parts.append(f"{eng.likes}lk")
|
||||||
|
if eng.reposts is not None:
|
||||||
|
parts.append(f"{eng.reposts}rp")
|
||||||
|
if eng.replies is not None:
|
||||||
|
parts.append(f"{eng.replies}re")
|
||||||
|
if parts:
|
||||||
|
eng_str = f" [{', '.join(parts)}]"
|
||||||
|
|
||||||
|
date_str = f" ({item.date})" if item.date else ""
|
||||||
|
|
||||||
|
lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{eng_str}{_xref_tag(item)}")
|
||||||
|
if item.text:
|
||||||
|
snippet = item.text[:200]
|
||||||
|
if len(item.text) > 200:
|
||||||
|
snippet += "..."
|
||||||
|
lines.append(f" {snippet}")
|
||||||
|
if item.url:
|
||||||
|
lines.append(f" {item.url}")
|
||||||
|
lines.append(f" *{item.why_relevant}*")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
# Polymarket items
|
# Polymarket items
|
||||||
if report.polymarket_error:
|
if report.polymarket_error:
|
||||||
lines.append("### Prediction Markets (Polymarket)")
|
lines.append("### Prediction Markets (Polymarket)")
|
||||||
@@ -531,6 +568,13 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str
|
|||||||
lines.append(f" ✅ HN: {len(report.hackernews)} stories")
|
lines.append(f" ✅ HN: {len(report.hackernews)} stories")
|
||||||
# Hide when zero results
|
# Hide when zero results
|
||||||
|
|
||||||
|
# Bluesky
|
||||||
|
if report.bluesky_error:
|
||||||
|
lines.append(f" ❌ Bluesky: error - {report.bluesky_error}")
|
||||||
|
elif report.bluesky:
|
||||||
|
lines.append(f" ✅ Bluesky: {len(report.bluesky)} posts")
|
||||||
|
# Hide when zero results
|
||||||
|
|
||||||
# Polymarket
|
# Polymarket
|
||||||
if report.polymarket_error:
|
if report.polymarket_error:
|
||||||
lines.append(f" ❌ Polymarket: error - {report.polymarket_error}")
|
lines.append(f" ❌ Polymarket: error - {report.polymarket_error}")
|
||||||
@@ -581,6 +625,8 @@ def render_context_snippet(report: schema.Report) -> str:
|
|||||||
all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url))
|
all_items.append((item.score, "Instagram", item.text[:50] + "...", item.url))
|
||||||
for item in report.hackernews[:5]:
|
for item in report.hackernews[:5]:
|
||||||
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
|
all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url))
|
||||||
|
for item in report.bluesky[:5]:
|
||||||
|
all_items.append((item.score, "Bluesky", item.text[:50] + "...", item.url))
|
||||||
for item in report.polymarket[:5]:
|
for item in report.polymarket[:5]:
|
||||||
all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url))
|
all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url))
|
||||||
for item in report.web[:5]:
|
for item in report.web[:5]:
|
||||||
@@ -754,6 +800,26 @@ def render_full_report(report: schema.Report) -> str:
|
|||||||
|
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
# Bluesky section
|
||||||
|
if report.bluesky:
|
||||||
|
lines.append("## Bluesky Posts")
|
||||||
|
lines.append("")
|
||||||
|
for item in report.bluesky:
|
||||||
|
lines.append(f"### {item.id}: @{item.author_handle}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"- **URL:** {item.url}")
|
||||||
|
lines.append(f"- **Date:** {item.date or 'Unknown'}")
|
||||||
|
lines.append(f"- **Score:** {item.score}/100")
|
||||||
|
lines.append(f"- **Relevance:** {item.why_relevant}")
|
||||||
|
|
||||||
|
if item.engagement:
|
||||||
|
eng = item.engagement
|
||||||
|
lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts, {eng.replies or '?'} replies")
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"> {item.text[:300]}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
# Polymarket section
|
# Polymarket section
|
||||||
if report.polymarket:
|
if report.polymarket:
|
||||||
lines.append("## Prediction Markets (Polymarket)")
|
lines.append("## Prediction Markets (Polymarket)")
|
||||||
|
|||||||
@@ -355,6 +355,43 @@ class HackerNewsItem:
|
|||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class BlueskyItem:
|
||||||
|
"""Normalized Bluesky post."""
|
||||||
|
id: str # "BS1", "BS2", ...
|
||||||
|
text: str
|
||||||
|
url: str # bsky.app permalink
|
||||||
|
author_handle: str # user.bsky.social
|
||||||
|
display_name: str
|
||||||
|
date: Optional[str] = None
|
||||||
|
date_confidence: str = "high" # AT Protocol has exact timestamps
|
||||||
|
engagement: Optional[Engagement] = None # likes, reposts, replies, quotes
|
||||||
|
relevance: float = 0.5
|
||||||
|
why_relevant: str = ""
|
||||||
|
subs: SubScores = field(default_factory=SubScores)
|
||||||
|
score: int = 0
|
||||||
|
cross_refs: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
d = {
|
||||||
|
'id': self.id,
|
||||||
|
'text': self.text,
|
||||||
|
'url': self.url,
|
||||||
|
'author_handle': self.author_handle,
|
||||||
|
'display_name': self.display_name,
|
||||||
|
'date': self.date,
|
||||||
|
'date_confidence': self.date_confidence,
|
||||||
|
'engagement': self.engagement.to_dict() if self.engagement else None,
|
||||||
|
'relevance': self.relevance,
|
||||||
|
'why_relevant': self.why_relevant,
|
||||||
|
'subs': self.subs.to_dict(),
|
||||||
|
'score': self.score,
|
||||||
|
}
|
||||||
|
if self.cross_refs:
|
||||||
|
d['cross_refs'] = self.cross_refs
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class PolymarketItem:
|
class PolymarketItem:
|
||||||
"""Normalized Polymarket prediction market item."""
|
"""Normalized Polymarket prediction market item."""
|
||||||
@@ -415,6 +452,7 @@ class Report:
|
|||||||
tiktok: List[TikTokItem] = field(default_factory=list)
|
tiktok: List[TikTokItem] = field(default_factory=list)
|
||||||
instagram: List[InstagramItem] = field(default_factory=list)
|
instagram: List[InstagramItem] = field(default_factory=list)
|
||||||
hackernews: List[HackerNewsItem] = field(default_factory=list)
|
hackernews: List[HackerNewsItem] = field(default_factory=list)
|
||||||
|
bluesky: List[BlueskyItem] = field(default_factory=list)
|
||||||
polymarket: List[PolymarketItem] = field(default_factory=list)
|
polymarket: List[PolymarketItem] = field(default_factory=list)
|
||||||
best_practices: List[str] = field(default_factory=list)
|
best_practices: List[str] = field(default_factory=list)
|
||||||
prompt_pack: List[str] = field(default_factory=list)
|
prompt_pack: List[str] = field(default_factory=list)
|
||||||
@@ -427,6 +465,7 @@ class Report:
|
|||||||
tiktok_error: Optional[str] = None
|
tiktok_error: Optional[str] = None
|
||||||
instagram_error: Optional[str] = None
|
instagram_error: Optional[str] = None
|
||||||
hackernews_error: Optional[str] = None
|
hackernews_error: Optional[str] = None
|
||||||
|
bluesky_error: Optional[str] = None
|
||||||
polymarket_error: Optional[str] = None
|
polymarket_error: Optional[str] = None
|
||||||
# Handle resolution
|
# Handle resolution
|
||||||
resolved_x_handle: Optional[str] = None
|
resolved_x_handle: Optional[str] = None
|
||||||
@@ -452,6 +491,7 @@ class Report:
|
|||||||
'tiktok': [t.to_dict() for t in self.tiktok],
|
'tiktok': [t.to_dict() for t in self.tiktok],
|
||||||
'instagram': [ig.to_dict() for ig in self.instagram],
|
'instagram': [ig.to_dict() for ig in self.instagram],
|
||||||
'hackernews': [h.to_dict() for h in self.hackernews],
|
'hackernews': [h.to_dict() for h in self.hackernews],
|
||||||
|
'bluesky': [b.to_dict() for b in self.bluesky],
|
||||||
'polymarket': [p.to_dict() for p in self.polymarket],
|
'polymarket': [p.to_dict() for p in self.polymarket],
|
||||||
'best_practices': self.best_practices,
|
'best_practices': self.best_practices,
|
||||||
'prompt_pack': self.prompt_pack,
|
'prompt_pack': self.prompt_pack,
|
||||||
@@ -473,6 +513,8 @@ class Report:
|
|||||||
d['instagram_error'] = self.instagram_error
|
d['instagram_error'] = self.instagram_error
|
||||||
if self.hackernews_error:
|
if self.hackernews_error:
|
||||||
d['hackernews_error'] = self.hackernews_error
|
d['hackernews_error'] = self.hackernews_error
|
||||||
|
if self.bluesky_error:
|
||||||
|
d['bluesky_error'] = self.bluesky_error
|
||||||
if self.polymarket_error:
|
if self.polymarket_error:
|
||||||
d['polymarket_error'] = self.polymarket_error
|
d['polymarket_error'] = self.polymarket_error
|
||||||
if self.from_cache:
|
if self.from_cache:
|
||||||
|
|||||||
@@ -468,6 +468,66 @@ def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.Ha
|
|||||||
return items
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def compute_bluesky_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||||
|
"""Compute raw engagement score for Bluesky item.
|
||||||
|
|
||||||
|
Formula: 0.40*log1p(likes) + 0.30*log1p(reposts) + 0.20*log1p(replies) + 0.10*log1p(quotes)
|
||||||
|
Likes are primary signal; reposts indicate reach; replies indicate discussion depth.
|
||||||
|
"""
|
||||||
|
if engagement is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if engagement.likes is None and engagement.reposts is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
likes = log1p_safe(engagement.likes)
|
||||||
|
reposts = log1p_safe(engagement.reposts)
|
||||||
|
replies = log1p_safe(engagement.replies)
|
||||||
|
quotes = log1p_safe(engagement.quotes)
|
||||||
|
|
||||||
|
return 0.40 * likes + 0.30 * reposts + 0.20 * replies + 0.10 * quotes
|
||||||
|
|
||||||
|
|
||||||
|
def score_bluesky_items(items: List[schema.BlueskyItem]) -> List[schema.BlueskyItem]:
|
||||||
|
"""Compute scores for Bluesky items.
|
||||||
|
|
||||||
|
Uses same weight structure as Reddit/X (relevance + recency + engagement).
|
||||||
|
"""
|
||||||
|
if not items:
|
||||||
|
return items
|
||||||
|
|
||||||
|
eng_raw = [compute_bluesky_engagement_raw(item.engagement) for item in items]
|
||||||
|
eng_normalized = normalize_to_100(eng_raw)
|
||||||
|
|
||||||
|
for i, item in enumerate(items):
|
||||||
|
rel_score = int(item.relevance * 100)
|
||||||
|
rec_score = dates.recency_score(item.date)
|
||||||
|
|
||||||
|
if eng_normalized[i] is not None:
|
||||||
|
eng_score = int(eng_normalized[i])
|
||||||
|
else:
|
||||||
|
eng_score = DEFAULT_ENGAGEMENT
|
||||||
|
|
||||||
|
item.subs = schema.SubScores(
|
||||||
|
relevance=rel_score,
|
||||||
|
recency=rec_score,
|
||||||
|
engagement=eng_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
overall = (
|
||||||
|
WEIGHT_RELEVANCE * rel_score +
|
||||||
|
WEIGHT_RECENCY * rec_score +
|
||||||
|
WEIGHT_ENGAGEMENT * eng_score
|
||||||
|
)
|
||||||
|
|
||||||
|
if eng_raw[i] is None:
|
||||||
|
overall -= UNKNOWN_ENGAGEMENT_PENALTY
|
||||||
|
|
||||||
|
item.score = max(0, min(100, int(overall)))
|
||||||
|
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]:
|
||||||
"""Compute raw engagement score for Polymarket item.
|
"""Compute raw engagement score for Polymarket item.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Tests for bluesky module."""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
|
||||||
|
from lib import bluesky
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractCoreSubject(unittest.TestCase):
|
||||||
|
def test_strips_prefix(self):
|
||||||
|
result = bluesky._extract_core_subject("what are people saying about claude code")
|
||||||
|
self.assertEqual(result, "claude code")
|
||||||
|
|
||||||
|
def test_strips_noise(self):
|
||||||
|
result = bluesky._extract_core_subject("latest trending news claude code")
|
||||||
|
self.assertNotIn("latest", result)
|
||||||
|
self.assertNotIn("trending", result)
|
||||||
|
self.assertIn("claude", result)
|
||||||
|
|
||||||
|
def test_preserves_core(self):
|
||||||
|
result = bluesky._extract_core_subject("react native")
|
||||||
|
self.assertEqual(result, "react native")
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseDate(unittest.TestCase):
|
||||||
|
def test_indexed_at_iso(self):
|
||||||
|
item = {"indexedAt": "2024-06-15T12:00:00Z"}
|
||||||
|
self.assertEqual(bluesky._parse_date(item), "2024-06-15")
|
||||||
|
|
||||||
|
def test_created_at_iso(self):
|
||||||
|
item = {"createdAt": "2024-03-01T08:30:00.000Z"}
|
||||||
|
self.assertEqual(bluesky._parse_date(item), "2024-03-01")
|
||||||
|
|
||||||
|
def test_indexed_at_preferred_over_created_at(self):
|
||||||
|
item = {"indexedAt": "2024-06-15T12:00:00Z", "createdAt": "2024-06-14T12:00:00Z"}
|
||||||
|
self.assertEqual(bluesky._parse_date(item), "2024-06-15")
|
||||||
|
|
||||||
|
def test_none_returns_none(self):
|
||||||
|
self.assertIsNone(bluesky._parse_date({}))
|
||||||
|
|
||||||
|
def test_invalid_date_returns_none(self):
|
||||||
|
self.assertIsNone(bluesky._parse_date({"indexedAt": "not-a-date"}))
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseBlueskyResponse(unittest.TestCase):
|
||||||
|
def test_basic_post(self):
|
||||||
|
response = {
|
||||||
|
"posts": [{
|
||||||
|
"uri": "at://did:plc:abc123/app.bsky.feed.post/xyz789",
|
||||||
|
"author": {"handle": "alice.bsky.social", "displayName": "Alice"},
|
||||||
|
"record": {"text": "Hello world", "createdAt": "2024-06-15T12:00:00Z"},
|
||||||
|
"indexedAt": "2024-06-15T12:01:00Z",
|
||||||
|
"likeCount": 10,
|
||||||
|
"repostCount": 5,
|
||||||
|
"replyCount": 3,
|
||||||
|
"quoteCount": 1,
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
items = bluesky.parse_bluesky_response(response)
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
self.assertEqual(items[0]["handle"], "alice.bsky.social")
|
||||||
|
self.assertEqual(items[0]["display_name"], "Alice")
|
||||||
|
self.assertEqual(items[0]["text"], "Hello world")
|
||||||
|
self.assertEqual(items[0]["url"], "https://bsky.app/profile/alice.bsky.social/post/xyz789")
|
||||||
|
self.assertEqual(items[0]["engagement"]["likes"], 10)
|
||||||
|
self.assertEqual(items[0]["engagement"]["reposts"], 5)
|
||||||
|
self.assertEqual(items[0]["date"], "2024-06-15")
|
||||||
|
|
||||||
|
def test_empty_response(self):
|
||||||
|
items = bluesky.parse_bluesky_response({})
|
||||||
|
self.assertEqual(items, [])
|
||||||
|
|
||||||
|
def test_missing_fields(self):
|
||||||
|
response = {"posts": [{"uri": "", "author": {}, "record": {}}]}
|
||||||
|
items = bluesky.parse_bluesky_response(response)
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
self.assertEqual(items[0]["handle"], "")
|
||||||
|
self.assertEqual(items[0]["text"], "")
|
||||||
|
|
||||||
|
def test_relevance_decreases_with_position(self):
|
||||||
|
response = {"posts": [
|
||||||
|
{"uri": f"at://did/app.bsky.feed.post/{i}", "author": {"handle": f"u{i}"}, "record": {"text": f"post {i}"}}
|
||||||
|
for i in range(5)
|
||||||
|
]}
|
||||||
|
items = bluesky.parse_bluesky_response(response)
|
||||||
|
self.assertGreater(items[0]["relevance"], items[4]["relevance"])
|
||||||
|
|
||||||
|
|
||||||
|
class TestDepthConfig(unittest.TestCase):
|
||||||
|
def test_all_depths_exist(self):
|
||||||
|
for depth in ("quick", "default", "deep"):
|
||||||
|
self.assertIn(depth, bluesky.DEPTH_CONFIG)
|
||||||
|
|
||||||
|
def test_deep_has_more_results(self):
|
||||||
|
quick = bluesky.DEPTH_CONFIG["quick"]
|
||||||
|
deep = bluesky.DEPTH_CONFIG["deep"]
|
||||||
|
self.assertGreater(deep, quick)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user