From b38703e53d1b4ab3c68f3426ff0873160e3314df Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Tue, 10 Mar 2026 00:14:39 -0700 Subject: [PATCH] feat(truthsocial): Add Truth Social as opt-in source Mastodon-compatible API at truthsocial.com/api/v2/search. Opt-in via TRUTHSOCIAL_TOKEN env var (bearer token from browser). Silent when unconfigured. Full pipeline: search, parse, normalize, score, dedupe, render across all 10 pipeline files. 27 new tests, 440 total passing. Co-Authored-By: Claude Opus 4.6 --- README.md | 2 +- SKILL.md | 13 +- ...026-03-09-feat-truth-social-source-plan.md | 118 +++++++++ scripts/last30days.py | 90 ++++++- scripts/lib/dedupe.py | 8 + scripts/lib/env.py | 9 + scripts/lib/normalize.py | 43 ++++ scripts/lib/render.py | 74 +++++- scripts/lib/schema.py | 67 ++++++ scripts/lib/score.py | 56 +++++ scripts/lib/truthsocial.py | 183 ++++++++++++++ tests/test_truthsocial.py | 226 ++++++++++++++++++ 12 files changed, 871 insertions(+), 18 deletions(-) create mode 100644 docs/plans/2026-03-09-feat-truth-social-source-plan.md create mode 100644 scripts/lib/truthsocial.py create mode 100644 tests/test_truthsocial.py diff --git a/README.md b/README.md index aa555ab..cf15d84 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Examples: ## What It Does -1. **Researches** - Scans Reddit, X, Bluesky, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web for discussions from the last 30 days +1. **Researches** - Scans Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Instagram, Hacker News, Polymarket, and the web for discussions from the last 30 days 2. **Synthesizes** - Identifies patterns, best practices, and what actually works 3. **Delivers** - Either writes copy-paste-ready prompts for your target tool, or gives you a curated expert-level answer diff --git a/SKILL.md b/SKILL.md index 6703973..a528e16 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,7 +1,7 @@ --- name: last30days version: "2.9.5" -description: "Research a topic from the last 30 days. Also triggered by 'last30'. Sources: Reddit, X, Bluesky, YouTube, TikTok, Instagram, Hacker News, Polymarket, web. Become an expert and write copy-paste-ready prompts." +description: "Research a topic from the last 30 days. Also triggered by 'last30'. Sources: Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Instagram, Hacker News, Polymarket, web. Become an expert and write copy-paste-ready prompts." argument-hint: 'last30 AI video tools, last30 best project management tools' allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch homepage: https://github.com/mvanhorn/last30days-skill @@ -26,6 +26,7 @@ metadata: - CT0 - BSKY_HANDLE - BSKY_APP_PASSWORD + - TRUTHSOCIAL_TOKEN bins: - node - python3 @@ -42,15 +43,16 @@ metadata: - instagram - hackernews - polymarket + - truthsocial - trends - prompts --- # last30days v2.9.5: Research Any Topic from the Last 30 Days -> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section. +> **Permissions overview:** Reads public web/platform data and optionally saves research briefings to `~/Documents/Last30Days/`. X/Twitter search uses optional user-provided tokens (AUTH_TOKEN/CT0 env vars). Bluesky search uses optional app password (BSKY_HANDLE/BSKY_APP_PASSWORD env vars - create at bsky.app/settings/app-passwords). Truth Social search uses optional bearer token (TRUTHSOCIAL_TOKEN env var - extract from browser dev tools). All credential usage and data writes are documented in the [Security & Permissions](#security--permissions) section. -Research ANY topic across Reddit, X, Bluesky, YouTube, TikTok, Hacker News, Polymarket, and the web. Surface what people are actually discussing, recommending, betting on, and debating right now. +Research ANY topic across Reddit, X, Bluesky, Truth Social, YouTube, TikTok, Hacker News, Polymarket, and the web. Surface what people are actually discussing, recommending, betting on, and debating right now. ## CRITICAL: Parse User Intent @@ -87,7 +89,7 @@ Common patterns: **DISPLAY your parsing to the user.** Before running any tools, output: ``` -I'll research {TOPIC} across Reddit, X, Bluesky, TikTok, and the web to find what's been discussed in the last 30 days. +I'll research {TOPIC} across Reddit, X, Bluesky, Truth Social, TikTok, and the web to find what's been discussed in the last 30 days. Parsed intent: - TOPIC = {TOPIC} @@ -151,7 +153,7 @@ Agent mode report format: ``` ## Research Report: {TOPIC} -Generated: {date} | Sources: Reddit, X, Bluesky, YouTube, TikTok, HN, Polymarket, Web +Generated: {date} | Sources: Reddit, X, Bluesky, Truth Social, YouTube, TikTok, HN, Polymarket, Web ### Key Findings [3-5 bullet points, highest-signal insights with citations] @@ -504,6 +506,7 @@ KEY PATTERNS from the research: β”œβ”€ πŸ“Έ Instagram: {N} reels β”‚ {N} views β”‚ {N} likes β”‚ {N} with captions β”œβ”€ 🟑 HN: {N} stories β”‚ {N} points β”‚ {N} comments β”œβ”€ πŸ¦‹ Bluesky: {N} posts β”‚ {N} likes β”‚ {N} reposts +β”œβ”€ πŸ‡ΊπŸ‡Έ Truth Social: {N} posts β”‚ {N} likes β”‚ {N} reposts β”œβ”€ πŸ“Š Polymarket: {N} markets β”‚ {short summary of up to 5 most relevant market odds, e.g. "Championship: 12%, #1 Seed: 28%, Big 12: 64%, vs Kansas: 71%"} β”œβ”€ 🌐 Web: {N} pages β€” Source Name, Source Name, Source Name └─ πŸ—£οΈ Top voices: @{handle1} ({N} likes), @{handle2} β”‚ r/{sub1}, r/{sub2} diff --git a/docs/plans/2026-03-09-feat-truth-social-source-plan.md b/docs/plans/2026-03-09-feat-truth-social-source-plan.md new file mode 100644 index 0000000..9de1ec0 --- /dev/null +++ b/docs/plans/2026-03-09-feat-truth-social-source-plan.md @@ -0,0 +1,118 @@ +--- +title: "feat: Add Truth Social as opt-in source" +type: feat +status: completed +date: 2026-03-09 +--- + +# feat: Add Truth Social as opt-in source + +Add Truth Social (Mastodon fork) as an opt-in social source. When `TRUTHSOCIAL_TOKEN` is set, posts from Truth Social appear alongside other sources in research results. When not configured, completely silent. + +## Problem / Motivation + +Issue #63 requested Truth Social support. Truth Social is a Mastodon fork with ~7M monthly active users. For users who care about that community's perspective on a topic, it's a valuable signal source. Follows the same opt-in pattern as Bluesky. + +## Approach + +Use Truth Social's Mastodon-compatible API directly with `urllib3` (no external dependencies). One env var: `TRUTHSOCIAL_TOKEN` (bearer token). Follow the Bluesky source pattern exactly across all 10 pipeline files. + +**Why bearer token (not username/password):** Truth Social's OAuth uses a non-standard `/oauth/v2/token` endpoint with hardcoded `client_id`/`client_secret` extracted from their JS bundle. These values break when Truth Social updates their frontend. A bearer token is more stable - user extracts it once from browser dev tools (Application > Local Storage > truthsocial.com > `access_token`) or via `truthbrush` CLI. + +**API endpoint:** +``` +GET https://truthsocial.com/api/v2/search +Authorization: Bearer {token} +Params: q={topic}&type=statuses&limit=40 +``` + +**Response format:** Standard Mastodon status objects with `content` (HTML), `created_at`, `url`, `account`, `favourites_count`, `reblogs_count`, `replies_count`. + +## Files to Change + +| # | File | Change | +|---|------|--------| +| 1 | `scripts/lib/truthsocial.py` | **New file.** API client: `search_truthsocial(topic, from_date, to_date, depth, config)` + `parse_truthsocial_response(response)`. Strip HTML tags from `content`. Handle 401/403/429 gracefully. | +| 2 | `scripts/lib/env.py` | Add `('TRUTHSOCIAL_TOKEN', None)` to `get_config()`. Add `is_truthsocial_available(config)`. | +| 3 | `scripts/lib/schema.py` | Add `TruthSocialItem` dataclass (id prefix `"TS"`). Add `truthsocial`/`truthsocial_error` fields to `Report`. Update `to_dict()`/`from_dict()`. | +| 4 | `scripts/lib/normalize.py` | Add `normalize_truthsocial_items()`. Map Mastodon fields: `favourites_count` -> `likes`, `reblogs_count` -> `reposts`, `replies_count` -> `replies`. | +| 5 | `scripts/lib/score.py` | Add `compute_truthsocial_engagement_raw()` + `score_truthsocial_items()`. Same weighted log formula as Bluesky. | +| 6 | `scripts/lib/dedupe.py` | Add `dedupe_truthsocial()` (one-liner wrapping `dedupe_items`). | +| 7 | `scripts/lib/render.py` | Add Truth Social sections to `_xref_tag()`, `_assess_data_freshness()`, `render_compact()`, `render_source_status()`, `render_context_snippet()`, `render_full_report()`. | +| 8 | `scripts/last30days.py` | ~20 touchpoints: TIMEOUT_PROFILES, VALID_SEARCH_SOURCES, import, `_search_truthsocial()`, `run_research()` param + dispatch + collect, `main()` availability + diag + flag + normalize + score + sort + dedupe + report. | +| 9 | `SKILL.md` | Add Truth Social to source lists, optionalEnv (`TRUTHSOCIAL_TOKEN`), stats template, security/privacy section. | +| 10 | `tests/test_truthsocial.py` | **New file.** Tests: HTML stripping, date parsing, response parsing, empty response, missing fields, depth config, auth error handling, successful search with mocked HTTP. | + +## Key Implementation Details + +### HTML stripping (`truthsocial.py`) + +Truth Social returns HTML content (`

Post text

`). Strip tags to plain text: +```python +import re +def _strip_html(html: str) -> str: + text = re.sub(r'', '\n', html) + text = re.sub(r'<[^>]+>', '', text) + return text.strip() +``` + +### Date filtering + +Mastodon `created_at` is ISO 8601 (`2026-03-09T12:00:00.000Z`). Use `[:10]` slice for `YYYY-MM-DD` comparison against `from_date`/`to_date`. + +### Engagement mapping + +| Mastodon field | Internal field | Display | +|---------------|---------------|---------| +| `favourites_count` | `likes` | `{N}lk` | +| `reblogs_count` | `reposts` | `{N}rp` | +| `replies_count` | `replies` | `{N}re` | + +### Error handling + +| HTTP Status | Behavior | +|------------|----------| +| 200 | Parse and return results | +| 401 | Return `{"statuses": [], "error": "Truth Social token expired"}` | +| 403 | Return `{"statuses": [], "error": "Truth Social access denied (Cloudflare)"}` | +| 429 | Return `{"statuses": [], "error": "Truth Social rate limited"}` | +| Other | Return `{"statuses": [], "error": "Truth Social search failed: {status}"}` | + +All errors return empty results gracefully - never crash the research run. + +### DEPTH_CONFIG + +| Depth | Limit | +|-------|-------| +| quick | 15 | +| default | 30 | +| deep | 60 | + +## What NOT to Change + +- `lib/__init__.py` - must stay bare (no eager imports) +- `cross_source_link.py` - works generically on items with `cross_refs` field +- `filter.py` - date filtering is generic +- No new pip dependencies + +## Acceptance Criteria + +- [x] `is_truthsocial_available()` returns False when `TRUTHSOCIAL_TOKEN` not set +- [x] No Truth Social stats line, no error, no mention when unconfigured +- [x] With valid `TRUTHSOCIAL_TOKEN`, posts are returned and rendered +- [x] HTML tags stripped from post content +- [x] Token expiry (401) returns empty results gracefully +- [x] Cloudflare block (403) returns empty results gracefully +- [x] `--diagnose` shows Truth Social availability status +- [x] SKILL.md documents `TRUTHSOCIAL_TOKEN` env var +- [x] All existing tests still pass +- [x] New tests cover: HTML stripping, parsing, auth errors, successful search +- [x] `bash scripts/sync.sh` deploys successfully + +## Sources + +- Issue #63: https://github.com/mvanhorn/last30days-skill/issues/63 +- Truth Social API: Mastodon-compatible at `truthsocial.com/api/v2/search` +- Auth: Bearer token via browser dev tools or `truthbrush` CLI +- Pattern reference: `scripts/lib/bluesky.py` (most recent source addition) +- Bluesky auth plan: `docs/plans/2026-03-09-fix-bluesky-auth-opt-in-plan.md` diff --git a/scripts/last30days.py b/scripts/last30days.py index e57aec6..2532c25 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -38,14 +38,14 @@ _child_pids: set = set() _child_pids_lock = threading.Lock() TIMEOUT_PROFILES = { - "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, - "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, - "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, + "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "tiktok_future": 90, "instagram_future": 90, "hackernews_future": 30, "bluesky_future": 30, "truthsocial_future": 30, "polymarket_future": 15, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, + "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "tiktok_future": 120, "instagram_future": 120, "hackernews_future": 60, "bluesky_future": 60, "truthsocial_future": 60, "polymarket_future": 30, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, + "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "tiktok_future": 150, "instagram_future": 150, "hackernews_future": 90, "bluesky_future": 90, "truthsocial_future": 90, "polymarket_future": 45, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, } # Valid source names for the --search flag VALID_SEARCH_SOURCES = { - "reddit", "x", "hn", "bluesky", "bsky", "youtube", "tiktok", "instagram", + "reddit", "x", "hn", "bluesky", "bsky", "truthsocial", "truth", "youtube", "tiktok", "instagram", "polymarket", "web", "xiaohongshu", "xhs", } @@ -136,6 +136,7 @@ def _install_global_timeout(timeout_seconds: int): from lib import ( bird_x, bluesky, + truthsocial, dates, dedupe, hackernews, @@ -543,6 +544,35 @@ def _search_bluesky( return bsky_items, bsky_error +def _search_truthsocial( + topic: str, + from_date: str, + to_date: str, + depth: str, + config: dict = None, +) -> tuple: + """Search Truth Social via Mastodon API (runs in thread). + + Returns: + Tuple of (ts_items, ts_error) + """ + ts_error = None + + try: + response = truthsocial.search_truthsocial( + topic, from_date, to_date, depth=depth, config=config, + ) + except Exception as e: + return [], f"{type(e).__name__}: {e}" + + ts_items = truthsocial.parse_truthsocial_response(response) + + if response.get("error"): + ts_error = response["error"] + + return ts_items, ts_error + + def _search_polymarket( topic: str, from_date: str, @@ -854,6 +884,7 @@ def run_research( resolved_handle: str = None, do_hackernews: bool = True, do_bluesky: bool = True, + do_truthsocial: bool = True, do_polymarket: bool = True, no_native_web: bool = False, ) -> tuple: @@ -861,10 +892,10 @@ def run_research( Returns: Tuple of (reddit_items, x_items, youtube_items, tiktok_items, instagram_items, - hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, + hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, - hackernews_error, bluesky_error, polymarket_error, web_error) + hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error) Note: web_needed is True when web search should be performed by the assistant (i.e., no native web search API keys are configured). When native web search @@ -881,6 +912,7 @@ def run_research( instagram_items = [] hackernews_items = [] bluesky_items = [] + truthsocial_items = [] polymarket_items = [] web_items = [] raw_openai = None @@ -893,6 +925,7 @@ def run_research( instagram_error = None hackernews_error = None bluesky_error = None + truthsocial_error = None polymarket_error = None web_error = None xiaohongshu_error = None @@ -977,7 +1010,7 @@ def run_research( progress.show_error(f"Instagram error: {e}") if progress: progress.end_instagram(len(instagram_items)) - return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error + return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error # Determine which searches to run do_reddit = sources in ("both", "reddit", "all", "reddit-web") @@ -994,6 +1027,7 @@ def run_research( xiaohongshu_future = None hackernews_future = None bluesky_future = None + truthsocial_future = None polymarket_future = None web_future = None max_workers = ( @@ -1004,6 +1038,7 @@ def run_research( + (1 if run_xiaohongshu else 0) + (1 if do_hackernews else 0) + (1 if do_bluesky else 0) + + (1 if do_truthsocial else 0) + (1 if do_polymarket else 0) + (1 if web_backend else 0) ) @@ -1066,6 +1101,11 @@ def run_research( _search_bluesky, topic, from_date, to_date, depth, config ) + if do_truthsocial: + truthsocial_future = executor.submit( + _search_truthsocial, topic, from_date, to_date, depth, config + ) + if do_polymarket: if progress: progress.start_polymarket() @@ -1213,6 +1253,21 @@ def run_research( if progress: progress.show_error(f"Bluesky error: {e}") + if truthsocial_future: + ts_timeout = timeouts.get("truthsocial_future", future_timeout) + try: + truthsocial_items, truthsocial_error = truthsocial_future.result(timeout=ts_timeout) + if truthsocial_error and progress: + progress.show_error(f"Truth Social error: {truthsocial_error}") + except TimeoutError: + truthsocial_error = f"Truth Social search timed out after {ts_timeout}s" + if progress: + progress.show_error(truthsocial_error) + except Exception as e: + truthsocial_error = f"{type(e).__name__}: {e}" + if progress: + progress.show_error(f"Truth Social error: {e}") + if polymarket_future: pm_timeout = timeouts.get("polymarket_future", future_timeout) try: @@ -1347,7 +1402,7 @@ def run_research( if sup_x: x_items.extend(sup_x) - return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error + return reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error def main(): @@ -1501,6 +1556,9 @@ def main(): # Auto-detect Bluesky (requires BSKY_HANDLE + BSKY_APP_PASSWORD) has_bluesky = env.is_bluesky_available(config) + # Auto-detect Truth Social (requires TRUTHSOCIAL_TOKEN) + has_truthsocial = env.is_truthsocial_available(config) + # --diagnose: show source availability and exit if args.diagnose: web_source = env.get_web_search_source(config) @@ -1519,6 +1577,7 @@ def main(): "xiaohongshu_api_base": env.get_xiaohongshu_api_base(config), "hackernews": True, "bluesky": has_bluesky, + "truthsocial": has_truthsocial, "polymarket": True, "web_search_backend": web_source, "parallel_ai": bool(config.get("PARALLEL_API_KEY")), @@ -1553,6 +1612,7 @@ def main(): "xiaohongshu": has_xiaohongshu, "hackernews": True, "bluesky": True, + "truthsocial": has_truthsocial, "polymarket": True, "web_search_backend": "deferred to assistant" if args.no_native_web else web_source, } @@ -1635,6 +1695,7 @@ def main(): # Apply --search flag: restrict sources to the specified subset search_do_hackernews = True search_do_bluesky = has_bluesky + search_do_truthsocial = has_truthsocial search_do_polymarket = True search_run_youtube = has_ytdlp search_run_tiktok = has_tiktok @@ -1646,6 +1707,7 @@ def main(): has_x = "x" in search_sources search_do_hackernews = "hn" in search_sources search_do_bluesky = ("bluesky" in search_sources or "bsky" in search_sources) and has_bluesky + search_do_truthsocial = ("truthsocial" in search_sources or "truth" in search_sources) and has_truthsocial search_do_polymarket = "polymarket" in search_sources search_run_youtube = "youtube" in search_sources and has_ytdlp search_run_tiktok = "tiktok" in search_sources and has_tiktok @@ -1665,7 +1727,7 @@ def main(): sources = "web" # hn/polymarket only; no Reddit/X # Run research - reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, polymarket_error, web_error = run_research( + reddit_items, x_items, youtube_items, tiktok_items, instagram_items, hackernews_items, bluesky_items, truthsocial_items, polymarket_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, tiktok_error, instagram_error, hackernews_error, bluesky_error, truthsocial_error, polymarket_error, web_error = run_research( args.topic, sources, config, @@ -1684,6 +1746,7 @@ def main(): resolved_handle=args.x_handle, do_hackernews=search_do_hackernews, do_bluesky=search_do_bluesky, + do_truthsocial=search_do_truthsocial, do_polymarket=search_do_polymarket, no_native_web=args.no_native_web, ) @@ -1699,6 +1762,7 @@ def main(): normalized_ig = normalize.normalize_instagram_items(instagram_items, from_date, to_date) if instagram_items else [] normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else [] normalized_bsky = normalize.normalize_bluesky_items(bluesky_items, from_date, to_date) if bluesky_items else [] + normalized_ts = normalize.normalize_truthsocial_items(truthsocial_items, from_date, to_date) if truthsocial_items else [] normalized_pm = normalize.normalize_polymarket_items(polymarket_items, from_date, to_date) if polymarket_items else [] normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else [] @@ -1716,6 +1780,7 @@ def main(): filtered_ig = normalize.filter_by_date_range(normalized_ig, from_date, to_date) if normalized_ig else [] filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else [] filtered_bsky = normalize.filter_by_date_range(normalized_bsky, from_date, to_date) if normalized_bsky else [] + filtered_ts = normalize.filter_by_date_range(normalized_ts, from_date, to_date) if normalized_ts else [] # Polymarket: skip hard date filter - markets are active/traded, updatedAt is fine filtered_pm = normalized_pm filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else [] @@ -1728,6 +1793,7 @@ def main(): scored_ig = score.score_instagram_items(filtered_ig) if filtered_ig else [] scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else [] scored_bsky = score.score_bluesky_items(filtered_bsky) if filtered_bsky else [] + scored_ts = score.score_truthsocial_items(filtered_ts) if filtered_ts else [] scored_pm = score.score_polymarket_items(filtered_pm) if filtered_pm else [] scored_web = score.score_websearch_items(filtered_web) if filtered_web else [] @@ -1739,6 +1805,7 @@ def main(): sorted_ig = score.sort_items(scored_ig) if scored_ig else [] sorted_hn = score.sort_items(scored_hn) if scored_hn else [] sorted_bsky = score.sort_items(scored_bsky) if scored_bsky else [] + sorted_ts = score.sort_items(scored_ts) if scored_ts else [] sorted_pm = score.sort_items(scored_pm) if scored_pm else [] sorted_web = score.sort_items(scored_web) if scored_web else [] @@ -1750,6 +1817,7 @@ def main(): deduped_ig = dedupe.dedupe_instagram(sorted_ig) if sorted_ig else [] deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else [] deduped_bsky = dedupe.dedupe_bluesky(sorted_bsky) if sorted_bsky else [] + deduped_ts = dedupe.dedupe_truthsocial(sorted_ts) if sorted_ts else [] deduped_pm = dedupe.dedupe_polymarket(sorted_pm) if sorted_pm else [] deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else [] @@ -1762,7 +1830,7 @@ def main(): # Cross-source linking: annotate items that discuss the same story dedupe.cross_source_link( - deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_pm, deduped_web, + deduped_reddit, deduped_x, deduped_youtube, deduped_tiktok, deduped_ig, deduped_hn, deduped_bsky, deduped_ts, deduped_pm, deduped_web, ) progress.end_processing() @@ -1783,6 +1851,7 @@ def main(): report.instagram = deduped_ig report.hackernews = deduped_hn report.bluesky = deduped_bsky + report.truthsocial = deduped_ts report.polymarket = deduped_pm report.web = deduped_web report.reddit_error = reddit_error @@ -1792,6 +1861,7 @@ def main(): report.instagram_error = instagram_error report.hackernews_error = hackernews_error report.bluesky_error = bluesky_error + report.truthsocial_error = truthsocial_error report.polymarket_error = polymarket_error report.web_error = web_error report.resolved_x_handle = args.x_handle diff --git a/scripts/lib/dedupe.py b/scripts/lib/dedupe.py index 0fbeb22..ddead58 100644 --- a/scripts/lib/dedupe.py +++ b/scripts/lib/dedupe.py @@ -234,6 +234,14 @@ def dedupe_bluesky( return dedupe_items(items, threshold) +def dedupe_truthsocial( + items: List[schema.TruthSocialItem], + threshold: float = 0.7, +) -> List[schema.TruthSocialItem]: + """Dedupe Truth Social items.""" + return dedupe_items(items, threshold) + + def dedupe_polymarket( items: List[schema.PolymarketItem], threshold: float = 0.7, diff --git a/scripts/lib/env.py b/scripts/lib/env.py index bda9aa3..87f9b7c 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -257,6 +257,7 @@ def get_config() -> Dict[str, Any]: ('CT0', None), ('BSKY_HANDLE', None), ('BSKY_APP_PASSWORD', None), + ('TRUTHSOCIAL_TOKEN', None), ] for key, default in keys: @@ -490,6 +491,14 @@ def is_bluesky_available(config: Dict[str, Any]) -> bool: return bool(config.get('BSKY_HANDLE') and config.get('BSKY_APP_PASSWORD')) +def is_truthsocial_available(config: Dict[str, Any]) -> bool: + """Check if Truth Social source is available. + + Requires TRUTHSOCIAL_TOKEN (bearer token from browser dev tools). + """ + return bool(config.get('TRUTHSOCIAL_TOKEN')) + + def is_polymarket_available() -> bool: """Check if Polymarket source is available. diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py index 45ad236..bce1778 100644 --- a/scripts/lib/normalize.py +++ b/scripts/lib/normalize.py @@ -394,6 +394,49 @@ def normalize_bluesky_items( return normalized +def normalize_truthsocial_items( + items: List[Dict[str, Any]], + from_date: str, + to_date: str, +) -> List[schema.TruthSocialItem]: + """Normalize raw Truth Social items to schema. + + Args: + items: Raw Truth Social items from Mastodon API + from_date: Start of date range + to_date: End of date range + + Returns: + List of TruthSocialItem objects + """ + normalized = [] + + for i, item in enumerate(items): + eng_raw = item.get("engagement") or {} + engagement = schema.Engagement( + likes=eng_raw.get("likes"), + reposts=eng_raw.get("reposts"), + replies=eng_raw.get("replies"), + ) + + date_str = item.get("date") + + normalized.append(schema.TruthSocialItem( + id=f"TS{i+1}", + text=item.get("text", ""), + url=item.get("url", ""), + author_handle=item.get("handle", ""), + display_name=item.get("display_name", ""), + date=date_str, + date_confidence="high", + engagement=engagement, + relevance=item.get("relevance", 0.5), + why_relevant=item.get("why_relevant", ""), + )) + + return normalized + + def normalize_polymarket_items( items: List[Dict[str, Any]], from_date: str, diff --git a/scripts/lib/render.py b/scripts/lib/render.py index e54ec38..5944c3a 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -30,6 +30,10 @@ def _xref_tag(item) -> str: source_names.add('Instagram') elif ref_id.startswith('HN'): source_names.add('HN') + elif ref_id.startswith('BS'): + source_names.add('Bluesky') + elif ref_id.startswith('TS'): + source_names.add('Truth Social') elif ref_id.startswith('PM'): source_names.add('Polymarket') elif ref_id.startswith('W'): @@ -60,13 +64,14 @@ def _assess_data_freshness(report: schema.Report) -> dict: web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from) hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from) bsky_recent = sum(1 for b in report.bluesky if b.date and b.date >= report.range_from) + ts_recent = sum(1 for ts in report.truthsocial if ts.date and ts.date >= report.range_from) pm_recent = sum(1 for p in report.polymarket if p.date and p.date >= report.range_from) tiktok_recent = sum(1 for t in report.tiktok if t.date and t.date >= report.range_from) ig_recent = sum(1 for ig in report.instagram if ig.date and ig.date >= report.range_from) - total_recent = reddit_recent + x_recent + web_recent + hn_recent + bsky_recent + pm_recent + tiktok_recent + ig_recent - total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.bluesky) + len(report.polymarket) + len(report.tiktok) + len(report.instagram) + total_recent = reddit_recent + x_recent + web_recent + hn_recent + bsky_recent + ts_recent + pm_recent + tiktok_recent + ig_recent + total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) + len(report.bluesky) + len(report.truthsocial) + len(report.polymarket) + len(report.tiktok) + len(report.instagram) return { "reddit_recent": reddit_recent, @@ -404,6 +409,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append(f" *{item.why_relevant}*") lines.append("") + # Truth Social items + if report.truthsocial_error: + lines.append("### Truth Social Posts") + lines.append("") + lines.append(f"**ERROR:** {report.truthsocial_error}") + lines.append("") + elif report.truthsocial: + lines.append("### Truth Social Posts") + lines.append("") + for item in report.truthsocial[:limit]: + eng_str = "" + if item.engagement: + eng = item.engagement + parts = [] + if eng.likes is not None: + parts.append(f"{eng.likes}lk") + if eng.reposts is not None: + parts.append(f"{eng.reposts}rp") + if eng.replies is not None: + parts.append(f"{eng.replies}re") + if parts: + eng_str = f" [{', '.join(parts)}]" + + date_str = f" ({item.date})" if item.date else "" + + lines.append(f"**{item.id}** (score:{item.score}) @{item.author_handle}{date_str}{eng_str}{_xref_tag(item)}") + if item.text: + snippet = item.text[:200] + if len(item.text) > 200: + snippet += "..." + lines.append(f" {snippet}") + if item.url: + lines.append(f" {item.url}") + lines.append(f" *{item.why_relevant}*") + lines.append("") + # Polymarket items if report.polymarket_error: lines.append("### Prediction Markets (Polymarket)") @@ -575,6 +616,13 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str lines.append(f" βœ… Bluesky: {len(report.bluesky)} posts") # Hide when zero results + # Truth Social + if report.truthsocial_error: + lines.append(f" ❌ Truth Social: error - {report.truthsocial_error}") + elif report.truthsocial: + lines.append(f" βœ… Truth Social: {len(report.truthsocial)} posts") + # Hide when zero results + # Polymarket if report.polymarket_error: lines.append(f" ❌ Polymarket: error - {report.polymarket_error}") @@ -627,6 +675,8 @@ def render_context_snippet(report: schema.Report) -> str: all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url)) for item in report.bluesky[:5]: all_items.append((item.score, "Bluesky", item.text[:50] + "...", item.url)) + for item in report.truthsocial[:5]: + all_items.append((item.score, "Truth Social", item.text[:50] + "...", item.url)) for item in report.polymarket[:5]: all_items.append((item.score, "Polymarket", item.question[:50] + "...", item.url)) for item in report.web[:5]: @@ -820,6 +870,26 @@ def render_full_report(report: schema.Report) -> str: lines.append(f"> {item.text[:300]}") lines.append("") + # Truth Social section + if report.truthsocial: + lines.append("## Truth Social Posts") + lines.append("") + for item in report.truthsocial: + lines.append(f"### {item.id}: @{item.author_handle}") + lines.append("") + lines.append(f"- **URL:** {item.url}") + lines.append(f"- **Date:** {item.date or 'Unknown'}") + lines.append(f"- **Score:** {item.score}/100") + lines.append(f"- **Relevance:** {item.why_relevant}") + + if item.engagement: + eng = item.engagement + lines.append(f"- **Engagement:** {eng.likes or '?'} likes, {eng.reposts or '?'} reposts, {eng.replies or '?'} replies") + + lines.append("") + lines.append(f"> {item.text[:300]}") + lines.append("") + # Polymarket section if report.polymarket: lines.append("## Prediction Markets (Polymarket)") diff --git a/scripts/lib/schema.py b/scripts/lib/schema.py index d195000..8805ceb 100644 --- a/scripts/lib/schema.py +++ b/scripts/lib/schema.py @@ -392,6 +392,43 @@ class BlueskyItem: return d +@dataclass +class TruthSocialItem: + """Normalized Truth Social post.""" + id: str # "TS1", "TS2", ... + text: str + url: str # truthsocial.com permalink + author_handle: str # username + display_name: str + date: Optional[str] = None + date_confidence: str = "high" # Mastodon API has exact timestamps + engagement: Optional[Engagement] = None # likes, reposts, replies + relevance: float = 0.5 + why_relevant: str = "" + subs: SubScores = field(default_factory=SubScores) + score: int = 0 + cross_refs: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + d = { + 'id': self.id, + 'text': self.text, + 'url': self.url, + 'author_handle': self.author_handle, + 'display_name': self.display_name, + 'date': self.date, + 'date_confidence': self.date_confidence, + 'engagement': self.engagement.to_dict() if self.engagement else None, + 'relevance': self.relevance, + 'why_relevant': self.why_relevant, + 'subs': self.subs.to_dict(), + 'score': self.score, + } + if self.cross_refs: + d['cross_refs'] = self.cross_refs + return d + + @dataclass class PolymarketItem: """Normalized Polymarket prediction market item.""" @@ -453,6 +490,7 @@ class Report: instagram: List[InstagramItem] = field(default_factory=list) hackernews: List[HackerNewsItem] = field(default_factory=list) bluesky: List[BlueskyItem] = field(default_factory=list) + truthsocial: List[TruthSocialItem] = field(default_factory=list) polymarket: List[PolymarketItem] = field(default_factory=list) best_practices: List[str] = field(default_factory=list) prompt_pack: List[str] = field(default_factory=list) @@ -466,6 +504,7 @@ class Report: instagram_error: Optional[str] = None hackernews_error: Optional[str] = None bluesky_error: Optional[str] = None + truthsocial_error: Optional[str] = None polymarket_error: Optional[str] = None # Handle resolution resolved_x_handle: Optional[str] = None @@ -492,6 +531,7 @@ class Report: 'instagram': [ig.to_dict() for ig in self.instagram], 'hackernews': [h.to_dict() for h in self.hackernews], 'bluesky': [b.to_dict() for b in self.bluesky], + 'truthsocial': [ts.to_dict() for ts in self.truthsocial], 'polymarket': [p.to_dict() for p in self.polymarket], 'best_practices': self.best_practices, 'prompt_pack': self.prompt_pack, @@ -515,6 +555,8 @@ class Report: d['hackernews_error'] = self.hackernews_error if self.bluesky_error: d['bluesky_error'] = self.bluesky_error + if self.truthsocial_error: + d['truthsocial_error'] = self.truthsocial_error if self.polymarket_error: d['polymarket_error'] = self.polymarket_error if self.from_cache: @@ -694,6 +736,29 @@ class Report: cross_refs=h.get('cross_refs', []), )) + # Reconstruct Truth Social items (backward compat: key may not exist) + ts_items = [] + for ts in data.get('truthsocial', []): + eng = None + if ts.get('engagement'): + eng = Engagement(**ts['engagement']) + subs = SubScores(**ts.get('subs', {})) if ts.get('subs') else SubScores() + ts_items.append(TruthSocialItem( + id=ts['id'], + text=ts['text'], + url=ts['url'], + author_handle=ts.get('author_handle', ''), + display_name=ts.get('display_name', ''), + date=ts.get('date'), + date_confidence=ts.get('date_confidence', 'high'), + engagement=eng, + relevance=ts.get('relevance', 0.5), + why_relevant=ts.get('why_relevant', ''), + subs=subs, + score=ts.get('score', 0), + cross_refs=ts.get('cross_refs', []), + )) + # Reconstruct Polymarket items (backward compat: key may not exist) pm_items = [] for p in data.get('polymarket', []): @@ -735,6 +800,7 @@ class Report: tiktok=tiktok_items, instagram=ig_items, hackernews=hn_items, + truthsocial=ts_items, polymarket=pm_items, best_practices=data.get('best_practices', []), prompt_pack=data.get('prompt_pack', []), @@ -746,6 +812,7 @@ class Report: tiktok_error=data.get('tiktok_error'), instagram_error=data.get('instagram_error'), hackernews_error=data.get('hackernews_error'), + truthsocial_error=data.get('truthsocial_error'), polymarket_error=data.get('polymarket_error'), resolved_x_handle=data.get('resolved_x_handle'), from_cache=data.get('from_cache', False), diff --git a/scripts/lib/score.py b/scripts/lib/score.py index e876330..f52bb04 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -528,6 +528,62 @@ def score_bluesky_items(items: List[schema.BlueskyItem]) -> List[schema.BlueskyI return items +def compute_truthsocial_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: + """Compute raw engagement score for Truth Social item. + + Formula: 0.45*log1p(likes) + 0.30*log1p(reposts) + 0.25*log1p(replies) + Likes are primary signal; reposts indicate reach; replies indicate discussion. + """ + if engagement is None: + return None + + if engagement.likes is None and engagement.reposts is None: + return None + + likes = log1p_safe(engagement.likes) + reposts = log1p_safe(engagement.reposts) + replies = log1p_safe(engagement.replies) + + return 0.45 * likes + 0.30 * reposts + 0.25 * replies + + +def score_truthsocial_items(items: List[schema.TruthSocialItem]) -> List[schema.TruthSocialItem]: + """Compute scores for Truth Social items.""" + if not items: + return items + + eng_raw = [compute_truthsocial_engagement_raw(item.engagement) for item in items] + eng_normalized = normalize_to_100(eng_raw) + + for i, item in enumerate(items): + rel_score = int(item.relevance * 100) + rec_score = dates.recency_score(item.date) + + if eng_normalized[i] is not None: + eng_score = int(eng_normalized[i]) + else: + eng_score = DEFAULT_ENGAGEMENT + + item.subs = schema.SubScores( + relevance=rel_score, + recency=rec_score, + engagement=eng_score, + ) + + overall = ( + WEIGHT_RELEVANCE * rel_score + + WEIGHT_RECENCY * rec_score + + WEIGHT_ENGAGEMENT * eng_score + ) + + if eng_raw[i] is None: + overall -= UNKNOWN_ENGAGEMENT_PENALTY + + item.score = max(0, min(100, int(overall))) + + return items + + def compute_polymarket_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: """Compute raw engagement score for Polymarket item. diff --git a/scripts/lib/truthsocial.py b/scripts/lib/truthsocial.py new file mode 100644 index 0000000..5aa1b76 --- /dev/null +++ b/scripts/lib/truthsocial.py @@ -0,0 +1,183 @@ +"""Truth Social search via Mastodon-compatible API (requires bearer token). + +Uses truthsocial.com/api/v2/search endpoint. +Requires TRUTHSOCIAL_TOKEN env var (bearer token from browser dev tools). +""" + +import math +import re +import sys +from typing import Any, Dict, List, Optional + +from . import http + +TRUTHSOCIAL_SEARCH_URL = "https://truthsocial.com/api/v2/search" + +DEPTH_CONFIG = { + "quick": 15, + "default": 30, + "deep": 60, +} + + +def _log(msg: str): + """Log to stderr (only in TTY mode to avoid cluttering Claude Code output).""" + if sys.stderr.isatty(): + sys.stderr.write(f"[TruthSocial] {msg}\n") + sys.stderr.flush() + + +def _strip_html(html: str) -> str: + """Strip HTML tags from Truth Social post content.""" + text = re.sub(r'', '\n', html) + text = re.sub(r'<[^>]+>', '', text) + return text.strip() + + +def _extract_core_subject(topic: str) -> str: + """Extract core subject from verbose query for Truth Social search.""" + text = topic.lower().strip() + prefixes = [ + 'what are the best', 'what is the best', 'what are the latest', + 'what are people saying about', 'what do people think about', + 'how do i use', 'how to use', 'how to', + 'what are', 'what is', 'tips for', 'best practices for', + ] + for p in prefixes: + if text.startswith(p + ' '): + text = text[len(p):].strip() + noise = { + 'best', 'top', 'good', 'great', 'awesome', + 'latest', 'new', 'news', 'update', 'updates', + 'trending', 'hottest', 'popular', 'viral', + 'practices', 'features', 'recommendations', 'advice', + } + words = text.split() + filtered = [w for w in words if w not in noise] + result = ' '.join(filtered) if filtered else text + return result.rstrip('?!.') + + +def _parse_date(status: Dict[str, Any]) -> Optional[str]: + """Parse date from Mastodon status to YYYY-MM-DD. + + Mastodon uses ISO 8601 format in created_at field. + """ + val = status.get("created_at") + if val and isinstance(val, str) and len(val) >= 10: + return val[:10] + return None + + +def search_truthsocial( + topic: str, + from_date: str, + to_date: str, + depth: str = "default", + config: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Search Truth Social via Mastodon-compatible API. + + Args: + topic: Search topic + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) + depth: 'quick', 'default', or 'deep' + config: Config dict with TRUTHSOCIAL_TOKEN + + Returns: + Dict with 'statuses' list from Mastodon API response. + """ + config = config or {} + token = config.get("TRUTHSOCIAL_TOKEN", "") + + if not token: + return {"statuses": [], "error": "Truth Social token not configured"} + + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + core_topic = _extract_core_subject(topic) + + _log(f"Searching for '{core_topic}' (depth={depth}, limit={count})") + + from urllib.parse import urlencode + params = { + "q": core_topic, + "type": "statuses", + "limit": str(min(count, 40)), + } + url = f"{TRUTHSOCIAL_SEARCH_URL}?{urlencode(params)}" + + try: + response = http.request( + "GET", url, + headers={"Authorization": f"Bearer {token}"}, + timeout=30, + ) + except http.HTTPError as e: + if e.status_code == 401: + _log("Token expired") + return {"statuses": [], "error": "Truth Social token expired"} + elif e.status_code == 403: + _log("Access denied (Cloudflare)") + return {"statuses": [], "error": "Truth Social access denied (Cloudflare)"} + elif e.status_code == 429: + _log("Rate limited") + return {"statuses": [], "error": "Truth Social rate limited"} + else: + _log(f"Search failed: {e}") + return {"statuses": [], "error": f"Truth Social search failed: {e.status_code}"} + except Exception as e: + _log(f"Search failed: {e}") + return {"statuses": [], "error": str(e)} + + statuses = response.get("statuses", []) + _log(f"Found {len(statuses)} posts") + return response + + +def parse_truthsocial_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse Mastodon API response into normalized item dicts. + + Returns: + List of item dicts ready for normalization. + """ + statuses = response.get("statuses", []) + items = [] + + for i, status in enumerate(statuses): + content_html = status.get("content") or "" + text = _strip_html(content_html) + + account = status.get("account") or {} + handle = account.get("acct") or account.get("username") or "" + display_name = account.get("display_name") or handle + + url = status.get("url") or "" + + likes = status.get("favourites_count") or 0 + reposts = status.get("reblogs_count") or 0 + replies = status.get("replies_count") or 0 + + date_str = _parse_date(status) + + # Relevance: position-based (search results are ranked by relevance) + rank_score = max(0.3, 1.0 - (i * 0.02)) + engagement_boost = min(0.2, math.log1p(likes + reposts) / 40) + relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1) + + items.append({ + "handle": handle, + "display_name": display_name, + "text": text, + "url": url, + "date": date_str, + "engagement": { + "likes": likes, + "reposts": reposts, + "replies": replies, + }, + "relevance": round(relevance, 2), + "why_relevant": f"Truth Social: @{handle}: {text[:60]}" if text else f"Truth Social: {handle}", + }) + + return items diff --git a/tests/test_truthsocial.py b/tests/test_truthsocial.py new file mode 100644 index 0000000..2901e1e --- /dev/null +++ b/tests/test_truthsocial.py @@ -0,0 +1,226 @@ +"""Tests for Truth Social source module.""" +import pytest +from unittest.mock import patch, MagicMock + +from scripts.lib import truthsocial + + +class TestStripHtml: + """Test HTML tag stripping.""" + + def test_basic_paragraph(self): + assert truthsocial._strip_html("

Hello world

") == "Hello world" + + def test_br_tags(self): + assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" + assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" + assert truthsocial._strip_html("Line 1
Line 2") == "Line 1\nLine 2" + + def test_nested_tags(self): + assert truthsocial._strip_html("

Hello world

") == "Hello world" + + def test_empty_string(self): + assert truthsocial._strip_html("") == "" + + def test_no_tags(self): + assert truthsocial._strip_html("plain text") == "plain text" + + def test_entities_preserved(self): + assert truthsocial._strip_html("

& test

") == "& test" + + +class TestExtractCoreSubject: + """Test query preprocessing.""" + + def test_strips_question_prefix(self): + assert truthsocial._extract_core_subject("what are people saying about tariffs") == "tariffs" + + def test_strips_noise_words(self): + assert truthsocial._extract_core_subject("latest trending crypto news") == "crypto" + + def test_preserves_core_topic(self): + assert truthsocial._extract_core_subject("tariffs") == "tariffs" + + def test_strips_trailing_punctuation(self): + assert truthsocial._extract_core_subject("what is bitcoin?") == "bitcoin" + + +class TestParseDate: + """Test date parsing from Mastodon status.""" + + def test_iso_date(self): + assert truthsocial._parse_date({"created_at": "2026-03-09T12:00:00.000Z"}) == "2026-03-09" + + def test_missing_date(self): + assert truthsocial._parse_date({}) is None + + def test_short_date(self): + assert truthsocial._parse_date({"created_at": "short"}) is None + + def test_none_value(self): + assert truthsocial._parse_date({"created_at": None}) is None + + +class TestDepthConfig: + """Test depth configuration.""" + + def test_all_depths_exist(self): + assert "quick" in truthsocial.DEPTH_CONFIG + assert "default" in truthsocial.DEPTH_CONFIG + assert "deep" in truthsocial.DEPTH_CONFIG + + def test_depth_ordering(self): + assert truthsocial.DEPTH_CONFIG["quick"] < truthsocial.DEPTH_CONFIG["default"] + assert truthsocial.DEPTH_CONFIG["default"] < truthsocial.DEPTH_CONFIG["deep"] + + +class TestSearchTruthSocial: + """Test search function auth handling.""" + + def test_no_config_returns_error(self): + result = truthsocial.search_truthsocial("test", "2026-02-09", "2026-03-09") + assert result["statuses"] == [] + assert "not configured" in result["error"] + + def test_empty_token_returns_error(self): + result = truthsocial.search_truthsocial( + "test", "2026-02-09", "2026-03-09", + config={"TRUTHSOCIAL_TOKEN": ""}, + ) + assert result["statuses"] == [] + assert "not configured" in result["error"] + + @patch("scripts.lib.truthsocial.http.request") + def test_401_returns_token_expired(self, mock_request): + from scripts.lib.http import HTTPError + mock_request.side_effect = HTTPError("Unauthorized", status_code=401) + result = truthsocial.search_truthsocial( + "test", "2026-02-09", "2026-03-09", + config={"TRUTHSOCIAL_TOKEN": "expired_token"}, + ) + assert result["statuses"] == [] + assert "expired" in result["error"] + + @patch("scripts.lib.truthsocial.http.request") + def test_403_returns_access_denied(self, mock_request): + from scripts.lib.http import HTTPError + mock_request.side_effect = HTTPError("Forbidden", status_code=403) + result = truthsocial.search_truthsocial( + "test", "2026-02-09", "2026-03-09", + config={"TRUTHSOCIAL_TOKEN": "blocked_token"}, + ) + assert result["statuses"] == [] + assert "Cloudflare" in result["error"] + + @patch("scripts.lib.truthsocial.http.request") + def test_429_returns_rate_limited(self, mock_request): + from scripts.lib.http import HTTPError + mock_request.side_effect = HTTPError("Too Many Requests", status_code=429) + result = truthsocial.search_truthsocial( + "test", "2026-02-09", "2026-03-09", + config={"TRUTHSOCIAL_TOKEN": "rate_limited_token"}, + ) + assert result["statuses"] == [] + assert "rate limited" in result["error"] + + @patch("scripts.lib.truthsocial.http.request") + def test_successful_search(self, mock_request): + mock_request.return_value = { + "statuses": [ + { + "content": "

Test post about tariffs

", + "created_at": "2026-03-09T12:00:00.000Z", + "url": "https://truthsocial.com/@user/123", + "account": {"acct": "user", "display_name": "Test User"}, + "favourites_count": 10, + "reblogs_count": 5, + "replies_count": 3, + } + ] + } + result = truthsocial.search_truthsocial( + "tariffs", "2026-02-09", "2026-03-09", + config={"TRUTHSOCIAL_TOKEN": "valid_token"}, + ) + assert len(result["statuses"]) == 1 + # Verify bearer token was passed + call_args = mock_request.call_args + assert call_args[1]["headers"]["Authorization"] == "Bearer valid_token" + + +class TestParseTruthSocialResponse: + """Test response parsing.""" + + def test_basic_post(self): + response = { + "statuses": [ + { + "content": "

Hello from Truth Social

", + "created_at": "2026-03-09T12:00:00.000Z", + "url": "https://truthsocial.com/@testuser/456", + "account": {"acct": "testuser", "display_name": "Test User"}, + "favourites_count": 100, + "reblogs_count": 50, + "replies_count": 25, + } + ] + } + items = truthsocial.parse_truthsocial_response(response) + assert len(items) == 1 + item = items[0] + assert item["handle"] == "testuser" + assert item["display_name"] == "Test User" + assert item["text"] == "Hello from Truth Social" # HTML stripped + assert item["url"] == "https://truthsocial.com/@testuser/456" + assert item["date"] == "2026-03-09" + assert item["engagement"]["likes"] == 100 + assert item["engagement"]["reposts"] == 50 + assert item["engagement"]["replies"] == 25 + assert item["relevance"] > 0 + + def test_empty_response(self): + items = truthsocial.parse_truthsocial_response({"statuses": []}) + assert items == [] + + def test_missing_fields(self): + response = { + "statuses": [ + { + "content": "", + "account": {}, + } + ] + } + items = truthsocial.parse_truthsocial_response(response) + assert len(items) == 1 + assert items[0]["handle"] == "" + assert items[0]["text"] == "" + assert items[0]["engagement"]["likes"] == 0 + + def test_relevance_ordering(self): + response = { + "statuses": [ + {"content": "

First

", "account": {"acct": "a"}, "favourites_count": 10, "reblogs_count": 0, "replies_count": 0}, + {"content": "

Second

", "account": {"acct": "b"}, "favourites_count": 5, "reblogs_count": 0, "replies_count": 0}, + {"content": "

Third

", "account": {"acct": "c"}, "favourites_count": 1, "reblogs_count": 0, "replies_count": 0}, + ] + } + items = truthsocial.parse_truthsocial_response(response) + assert items[0]["relevance"] >= items[1]["relevance"] + assert items[1]["relevance"] >= items[2]["relevance"] + + def test_html_stripping_in_parse(self): + response = { + "statuses": [ + { + "content": "

Hello @user check this out
New line

", + "account": {"acct": "poster"}, + "favourites_count": 0, + "reblogs_count": 0, + "replies_count": 0, + } + ] + } + items = truthsocial.parse_truthsocial_response(response) + assert "<" not in items[0]["text"] + assert ">" not in items[0]["text"]