sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(truthsocial): Add Truth Social as opt-in source

Browse Source 
Mastodon-compatible API at truthsocial.com/api/v2/search.
Opt-in via TRUTHSOCIAL_TOKEN env var (bearer token from browser).
Silent when unconfigured. Full pipeline: search, parse, normalize,
score, dedupe, render across all 10 pipeline files.

27 new tests, 440 total passing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-03-10 00:14:39 -07:00
parent b6fd5ff406
commit b38703e53d
12 changed files with 871 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/last30days.py
+80 -10
View File
@@ -38,14 +38,14 @@ _child_pids: set = set()
_child_pids_lock = threading.Lock()
TIMEOUT_PROFILES = {
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
"quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "truthsocial_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10},
"default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "truthsocial_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15},
"deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "truthsocial_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25},
}
# Valid source names for the --search flag
VALID_SEARCH_SOURCES = {
"reddit", "x", "hn", "bluesky", "bsky", "youtube", "tiktok", "instagram",
"reddit", "x", "hn", "bluesky", "bsky", "truthsocial", "truth", "youtube", "tiktok", "instagram",
"polymarket", "web", "xiaohongshu", "xhs",
}
@@ -136,6 +136,7 @@ def _install_global_timeout(timeout_seconds: int):
from lib import (
bird_x,
bluesky,
truthsocial,
dates,
dedupe,
hackernews,
@@ -543,6 +544,35 @@ def _search_bluesky(
return bsky_items, bsky_error
def _search_truthsocial(
topic: str,
from_date: str,
to_date: str,
depth: str,
config: dict = None,
) -> tuple:
"""Search Truth Social via Mastodon API (runs in thread).
Returns:
Tuple of (ts_items, ts_error)
"""
ts_error = None
try:
response = truthsocial.search_truthsocial(
topic, from_date, to_date, depth=depth, config=config,
)
except Exception as e:
return [], f"{type(e).__name__}: {e}"
ts_items = truthsocial.parse_truthsocial_response(response)
if response.get("error"):
ts_error = response["error"]
return ts_items, ts_error
def _search_polymarket(
topic: str,
from_date: str,
@@ -854,6 +884,7 @@ def run_research(
resolved_handle: str = None,
do_hackernews: bool = True,
do_bluesky: bool = True,
do_truthsocial: bool = True,
do_polymarket: bool = True,
no_native_web: bool = False,
) -> tuple:
@@ -861,10 +892,10 @@ def run_research(
Returns:
Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items,
hackernews_items, bluesky_items, polymarket_items, web_items, web_needed,
hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed,
raw_openai, raw_xai, raw_reddit_enriched,
reddit_error, x_error, youtube_error, tiktok_error, instagram_error,
hackernews_error, bluesky_error, polymarket_error, web_error)
hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error)
Note: web_needed is True when web search should be performed by the assistant
(i.e., no native web search API keys are configured). When native web search
@@ -881,6 +912,7 @@ def run_research(
instagram_items = []
hackernews_items = []
bluesky_items = []
truthsocial_items = []
polymarket_items = []
web_items = []
raw_openai = None
@@ -893,6 +925,7 @@ def run_research(
instagram_error = None
hackernews_error = None
bluesky_error = None
truthsocial_error = None
polymarket_error = None
web_error = None
xiaohongshu_error = None
@@ -977,7 +1010,7 @@ def run_research(
progress.show_error(f"Instagram error: {e}")
if progress:
progress.end_instagram(len(instagram_items))
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error
# Determine which searches to run
do_reddit = sources in ("both", "reddit", "all", "reddit-web")
@@ -994,6 +1027,7 @@ def run_research(
xiaohongshu_future = None
hackernews_future = None
bluesky_future = None
truthsocial_future = None
polymarket_future = None
web_future = None
max_workers = (
@@ -1004,6 +1038,7 @@ def run_research(
+ (1 if run_xiaohongshu else 0)
+ (1 if do_hackernews else 0)
+ (1 if do_bluesky else 0)
+ (1 if do_truthsocial else 0)
+ (1 if do_polymarket else 0)
+ (1 if web_backend else 0)
)
@@ -1066,6 +1101,11 @@ def run_research(
_search_bluesky, topic, from_date, to_date, depth, config
)
if do_truthsocial:
truthsocial_future = executor.submit(
_search_truthsocial, topic, from_date, to_date, depth, config
)
if do_polymarket:
if progress:
progress.start_polymarket()
@@ -1213,6 +1253,21 @@ def run_research(
if progress:
progress.show_error(f"Bluesky error: {e}")
if truthsocial_future:
ts_timeout = timeouts.get("truthsocial_future", future_timeout)
try:
truthsocial_items, truthsocial_error = truthsocial_future.result(timeout=ts_timeout)
if truthsocial_error and progress:
progress.show_error(f"Truth Social error: {truthsocial_error}")
except TimeoutError:
truthsocial_error = f"Truth Social search timed out after {ts_timeout}s"
if progress:
progress.show_error(truthsocial_error)
except Exception as e:
truthsocial_error = f"{type(e).__name__}: {e}"
if progress:
progress.show_error(f"Truth Social error: {e}")
if polymarket_future:
pm_timeout = timeouts.get("polymarket_future", future_timeout)
try:
@@ -1347,7 +1402,7 @@ def run_research(
if sup_x:
x_items.extend(sup_x)
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error
return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error
def main():
@@ -1501,6 +1556,9 @@ def main():
# Auto-detect Bluesky (requires BSKY_HANDLE + BSKY_APP_PASSWORD)
has_bluesky = env.is_bluesky_available(config)
# Auto-detect Truth Social (requires TRUTHSOCIAL_TOKEN)
has_truthsocial = env.is_truthsocial_available(config)
# --diagnose: show source availability and exit
if args.diagnose:
web_source = env.get_web_search_source(config)
@@ -1519,6 +1577,7 @@ def main():
"xiaohongshu_api_base": env.get_xiaohongshu_api_base(config),
"hackernews": True,
"bluesky": has_bluesky,
"truthsocial": has_truthsocial,
"polymarket": True,
"web_search_backend": web_source,
"parallel_ai": bool(config.get("PARALLEL_API_KEY")),
@@ -1553,6 +1612,7 @@ def main():
"xiaohongshu": has_xiaohongshu,
"hackernews": True,
"bluesky": True,
"truthsocial": has_truthsocial,
"polymarket": True,
"web_search_backend": "deferred to assistant" if args.no_native_web else web_source,
}
@@ -1635,6 +1695,7 @@ def main():
# Apply --search flag: restrict sources to the specified subset
search_do_hackernews = True
search_do_bluesky = has_bluesky
search_do_truthsocial = has_truthsocial
search_do_polymarket = True
search_run_youtube = has_ytdlp
search_run_tiktok = has_tiktok
@@ -1646,6 +1707,7 @@ def main():
has_x = "x" in search_sources
search_do_hackernews = "hn" in search_sources
search_do_bluesky = ("bluesky" in search_sources or "bsky" in search_sources) and has_bluesky
search_do_truthsocial = ("truthsocial" in search_sources or "truth" in search_sources) and has_truthsocial
search_do_polymarket = "polymarket" in search_sources
search_run_youtube = "youtube" in search_sources and has_ytdlp
search_run_tiktok = "tiktok" in search_sources and has_tiktok
@@ -1665,7 +1727,7 @@ def main():
sources = "web" # hn/polymarket only; no Reddit/X
# Run research
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error = run_research(
reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error = run_research(
args.topic,
sources,
config,
@@ -1684,6 +1746,7 @@ def main():
resolved_handle=args.x_handle,
do_hackernews=search_do_hackernews,
do_bluesky=search_do_bluesky,
do_truthsocial=search_do_truthsocial,
do_polymarket=search_do_polymarket,
no_native_web=args.no_native_web,
)
@@ -1699,6 +1762,7 @@ def main():
normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else []
normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else []
normalized_bsky = normalize.normalize_bluesky_items(bluesky_items, from_date, to_date) if bluesky_items else []
normalized_ts = normalize.normalize_truthsocial_items(truthsocial_items, from_date, to_date) if truthsocial_items else []
normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else []
normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else []
@@ -1716,6 +1780,7 @@ def main():
filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else []
filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else []
filtered_bsky = normalize.filter_by_date_range(normalized_bsky, from_date, to_date) if normalized_bsky else []
filtered_ts = normalize.filter_by_date_range(normalized_ts, from_date, to_date) if normalized_ts else []
# Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine
filtered_pm = normalized_pm
filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else []
@@ -1728,6 +1793,7 @@ def main():
scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else []
scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else []
scored_bsky = score.score_bluesky_items(filtered_bsky) if filtered_bsky else []
scored_ts = score.score_truthsocial_items(filtered_ts) if filtered_ts else []
scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else []
scored_web = score.score_websearch_items(filtered_web) if filtered_web else []
@@ -1739,6 +1805,7 @@ def main():
sorted_ig = score.sort_items(scored_ig) if scored_ig else []
sorted_hn = score.sort_items(scored_hn) if scored_hn else []
sorted_bsky = score.sort_items(scored_bsky) if scored_bsky else []
sorted_ts = score.sort_items(scored_ts) if scored_ts else []
sorted_pm = score.sort_items(scored_pm) if scored_pm else []
sorted_web = score.sort_items(scored_web) if scored_web else []
@@ -1750,6 +1817,7 @@ def main():
deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else []
deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else []
deduped_bsky = dedupe.dedupe_bluesky(sorted_bsky) if sorted_bsky else []
deduped_ts = dedupe.dedupe_truthsocial(sorted_ts) if sorted_ts else []
deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else []
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
@@ -1762,7 +1830,7 @@ def main():
# Cross-source linking: annotate items that discuss the same story
dedupe.cross_source_link(
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_pm, deduped_web,
deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_ts, deduped_pm, deduped_web,
)
progress.end_processing()
@@ -1783,6 +1851,7 @@ def main():
report.instagram = deduped_ig
report.hackernews = deduped_hn
report.bluesky = deduped_bsky
report.truthsocial = deduped_ts
report.polymarket = deduped_pm
report.web = deduped_web
report.reddit_error = reddit_error
@@ -1792,6 +1861,7 @@ def main():
report.instagram_error = instagram_error
report.hackernews_error = hackernews_error
report.bluesky_error = bluesky_error
report.truthsocial_error = truthsocial_error
report.polymarket_error = polymarket_error
report.web_error = web_error
report.resolved_x_handle = args.x_handle