diff --git a/.gitignore b/.gitignore index 3f89d8b..7b7b27e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,7 @@ htmlcov/ # Internal planning docs (ce:plan output) — keep local, don't publish docs/plans/ +.context/ /work /print - diff --git a/CONFIGURATION.md b/CONFIGURATION.md index eed672a..41b6e8e 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -57,7 +57,7 @@ The project-scoped file is the cleanest pattern for **per-client setups**: drop | YouTube | `yt-dlp` CLI installed | always on if `yt-dlp` present | yes | | X / Twitter | one of: `AUTH_TOKEN` + `CT0` (browser cookies, Bird CLI), `XAI_API_KEY`, `SCRAPECREATORS_API_KEY`, or `FROM_BROWSER` (cookie-jar auth) | X items in results | cookie-jar / Bird = free; xAI / ScrapeCreators = paid | | TikTok | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `tiktok` | TikTok items | 10K free calls | -| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls | +| Instagram | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `instagram` | Instagram Reels | 10K free calls; raise `LAST30DAYS_TRANSCRIPT_TIMEOUT` (default 30s) if SC is slow on your network | | Threads | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `threads` | Threads items | 10K free calls | | Pinterest | `SCRAPECREATORS_API_KEY` + `INCLUDE_SOURCES` contains `pinterest` | Pinterest items | 10K free calls | | Bluesky | `BSKY_HANDLE` + `BSKY_APP_PASSWORD` | Bluesky items | yes (app password at bsky.app) | @@ -93,6 +93,16 @@ After editing: `chmod 600 ~/.config/last30days/.env` (or `chmod 600 .claude/last **Troubleshooting:** if a source you expected to see isn't appearing in results, run `python3 scripts/last30days.py --diagnose`. It prints a per-source availability report (which keys were detected, which CLIs are installed, which backends are reachable) without running a full search. +### Bluesky app-password format and search host + +`BSKY_APP_PASSWORD` should be a 19-char app password in `xxxx-xxxx-xxxx-xxxx` format (lowercase alphanumeric, three hyphens). Generate one at . The AT Protocol's `createSession` endpoint also accepts your main account login password, but that's bad hygiene — main passwords have no scope (an app password can be limited to non-DM access) and can't be revoked individually. + +The skill defaults to `api.bsky.app` for `searchPosts`, which is the canonical authenticated AppView. The previous default `public.api.bsky.app` is the unauthenticated public mirror and is currently blocked by BunnyCDN for `searchPosts` regardless of auth header (verified 2026-05-04). If Bluesky migrates infrastructure again, override the host without a code change by setting `BSKY_SEARCH_HOST` in your `.env`: + +```bash +BSKY_SEARCH_HOST=api.bsky.app # default — change only if Bluesky moves +``` + --- ## Reasoning provider priority @@ -131,6 +141,8 @@ The default behavior - one slug-named file per topic, overwritten on rerun - is Adding `--store` to any run persists every finding to a SQLite database (default at `~/.local/share/last30days/research.db`). Findings dedupe on the `source_url` column (UNIQUE constraint), so the same URL across runs updates the existing row instead of creating a duplicate. The markdown file still saves; the SQLite is the time-series substrate. +**Always-on alternative:** set `LAST30DAYS_STORE=1` in your `.env` instead of remembering `--store` on every invocation. The flag still works as before; the env var is purely additive. Same hybrid pattern as `LAST30DAYS_DEBUG` — works whether shell-exported or in `.env`. + Relevant tables: `topics`, `research_runs`, `findings`, `settings`. Schema: [`scripts/store.py`](skills/last30days/scripts/store.py). ### `watchlist.py` - recurring topics diff --git a/README.md b/README.md index df6abbc..21adda2 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This README tracks the current v3 pipeline. The runtime skill spec lives in [ski **Claude Code (recommended — auto-updates via marketplace):** ``` /plugin marketplace add mvanhorn/last30days-skill +/plugin install last30days ``` **Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:** diff --git a/skills/last30days/scripts/last30days.py b/skills/last30days/scripts/last30days.py index 7aaa298..4031b9d 100644 --- a/skills/last30days/scripts/last30days.py +++ b/skills/last30days/scripts/last30days.py @@ -908,7 +908,17 @@ def main() -> int: suppress_web_promo=bool(external_plan or comp_plan), ) _write_last_run(topic, report) - if args.store: + # LAST30DAYS_STORE env var = persistence default-on. Read both os.environ + # (for shell-exported users) and config (for users who set it in + # ~/.config/last30days/.env, which env.py loads but does not propagate + # to os.environ). Mirrors the LAST30DAYS_DEBUG / LAST30DAYS_SKIP_PREFLIGHT + # convention; env-var or config wins, with `--store` flag still working. + _store_env = ( + os.environ.get("LAST30DAYS_STORE") + or config.get("LAST30DAYS_STORE") + or "" + ).lower() + if args.store or _store_env in ("1", "true", "yes"): counts = persist_report(report) sys.stderr.write( f"[last30days] Stored {counts['new']} new, {counts['updated']} updated findings\n" @@ -922,6 +932,7 @@ def main() -> int: # degraded-YouTube failure mode (videos returned but transcripts # silently failed - typically a stale yt-dlp binary). youtube_items = report.items_by_source.get("youtube") or [] + instagram_items = report.items_by_source.get("instagram") or [] research_results = { "youtube_videos_count": len(youtube_items), "youtube_transcripts_count": sum( @@ -930,6 +941,17 @@ def main() -> int: ), "youtube_error": report.errors_by_source.get("youtube"), "x_error": report.errors_by_source.get("x"), + # Captions-disabled videos can never produce a transcript regardless + # of yt-dlp version; subtract them from the degraded-ratio + # denominator so a single uploader-disabled video does not trip the + # "stale yt-dlp" nudge. + "youtube_captions_disabled_count": sum( + 1 for it in youtube_items if it.metadata.get("captions_disabled") + ), + # Track Instagram returned-zero-items so quality_nudge can detect + # the silent-failure case (SC configured but the v2 reels endpoint + # 500'd through both the original query and the hashtag retry). + "instagram_items_count": len(instagram_items), } quality = quality_nudge.compute_quality_score(config, research_results) if quality.get("nudge_text"): diff --git a/skills/last30days/scripts/lib/bluesky.py b/skills/last30days/scripts/lib/bluesky.py index 7c3436a..6771052 100644 --- a/skills/last30days/scripts/lib/bluesky.py +++ b/skills/last30days/scripts/lib/bluesky.py @@ -1,10 +1,19 @@ """Bluesky search via AT Protocol (requires app password). -Uses bsky.social for auth and public.api.bsky.app for post search. -Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. +Uses bsky.social for auth and api.bsky.app for post search (the canonical +authenticated AppView). The previous default `public.api.bsky.app` is the +unauthenticated public mirror, which BunnyCDN now blocks for searchPosts +regardless of auth header (verified 2026-05-04). Override the search host +via BSKY_SEARCH_HOST env var if Bluesky migrates infrastructure again. + +Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars. App passwords are +19-char xxxx-xxxx-xxxx-xxxx; generate at bsky.app/settings/app-passwords. +The createSession endpoint accepts main-account passwords too, but they're +bad hygiene (no scope, can't revoke individually). """ import math +import os import re import sys import time @@ -14,7 +23,65 @@ from typing import Any, Dict, List, Optional from . import http, log BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession" -BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts" +_DEFAULT_BSKY_SEARCH_HOST = "api.bsky.app" +BSKY_SEARCH_URL = f"https://{_DEFAULT_BSKY_SEARCH_HOST}/xrpc/app.bsky.feed.searchPosts" + + +def _resolve_search_url(config: Optional[Dict[str, Any]] = None) -> str: + """Resolve the Bluesky search URL with BSKY_SEARCH_HOST override. + + Default is api.bsky.app. Override via BSKY_SEARCH_HOST in shell env or + .env file. The project's env.py loads .env into config but not into + os.environ, so check both — same hybrid pattern as last30days.py for + LAST30DAYS_STORE. + + Hardens user-supplied host values against three common mis-configurations: + whitespace (e.g. " api.bsky.app "), embedded path components (e.g. + "api.bsky.app/xrpc/proxy") that would double the /xrpc/ segment, and + embedded scheme prefixes (e.g. "https://api.bsky.app"). On any of these + we log a warning and fall back to the default rather than building an + invalid URL with an opaque downstream error. + """ + config = config or {} + raw = ( + os.environ.get("BSKY_SEARCH_HOST") + or config.get("BSKY_SEARCH_HOST") + or _DEFAULT_BSKY_SEARCH_HOST + ) + host = raw.strip().rstrip("/") + # Strip embedded scheme so users who paste full URLs do not break the f-string. + for prefix in ("https://", "http://"): + if host.lower().startswith(prefix): + host = host[len(prefix):] + break + if not host or "/" in host or " " in host: + # Embedded path or whitespace remains — don't trust it. Default + log. + if raw != _DEFAULT_BSKY_SEARCH_HOST: + _log( + f"BSKY_SEARCH_HOST={raw!r} is not a bare hostname; " + f"falling back to default {_DEFAULT_BSKY_SEARCH_HOST!r}" + ) + host = _DEFAULT_BSKY_SEARCH_HOST + return f"https://{host}/xrpc/app.bsky.feed.searchPosts" + + +# App-password format: xxxx-xxxx-xxxx-xxxx (19 chars, lowercase alphanumeric +# with three hyphens at fixed positions). +_APP_PASSWORD_RE = re.compile(r"^[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$") + + +def _validate_app_password_format(value) -> bool: + """Return True if value matches Bluesky's 19-char app-password format. + + False for non-strings (None, int, list) so callers passing config dict + values directly don't crash. Detect-but-not-gate: the createSession + endpoint also accepts main-account passwords, so failing this check is + a hygiene smell, not a hard error. + """ + if not isinstance(value, str): + return False + return bool(_APP_PASSWORD_RE.fullmatch(value)) + DEPTH_CONFIG = { "quick": 15, @@ -144,6 +211,20 @@ def search_bluesky( if not handle or not app_password: return {"posts": [], "error": "Bluesky credentials not configured"} + # One-shot hygiene warning if BSKY_APP_PASSWORD is not in app-password + # form. createSession accepts main-account passwords too — but main + # passwords have no scope (full account access), can't be revoked + # individually, and rotating them breaks every service that holds them. + # We warn but do not gate, matching the project's detect-don't-block + # philosophy elsewhere. + if not _validate_app_password_format(app_password): + _log( + "BSKY_APP_PASSWORD does not look like an app password " + "(expected xxxx-xxxx-xxxx-xxxx, 19 chars). It may be a main " + "account password — those work but are bad hygiene. Generate " + "an app password at https://bsky.app/settings/app-passwords" + ) + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) core_topic = _extract_core_subject(topic) @@ -155,7 +236,7 @@ def search_bluesky( "limit": str(min(count, 100)), "sort": "top", } - url = f"{BSKY_SEARCH_URL}?{urlencode(params)}" + url = f"{_resolve_search_url(config)}?{urlencode(params)}" def _auth_and_search() -> tuple[Optional[Dict[str, Any]], Optional[str]]: token = _create_session(handle, app_password) diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index 56fb84e..ec20bb2 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -314,6 +314,7 @@ def get_config() -> dict[str, Any]: ('LAST30DAYS_RERANK_MODEL', None), ('LAST30DAYS_X_MODEL', None), ('LAST30DAYS_X_BACKEND', None), + ('LAST30DAYS_STORE', None), ('OPENAI_MODEL_PIN', None), ('XAI_MODEL_PIN', None), ('SCRAPECREATORS_API_KEY', None), @@ -322,6 +323,7 @@ def get_config() -> dict[str, Any]: ('CT0', None), ('BSKY_HANDLE', None), ('BSKY_APP_PASSWORD', None), + ('BSKY_SEARCH_HOST', None), ('TRUTHSOCIAL_TOKEN', None), ('BRAVE_API_KEY', None), ('EXA_API_KEY', None), @@ -334,11 +336,22 @@ def get_config() -> dict[str, Any]: ('INCLUDE_SOURCES', ''), ('EXCLUDE_SOURCES', ''), ('LAST30DAYS_YOUTUBE_SSH_HOST', None), + ('LAST30DAYS_TRANSCRIPT_TIMEOUT', None), ] for key, default in keys: config[key] = os.environ.get(key) or merged_env.get(key, default) + # Backward-compat: ScrapeCreators' own examples and tutorials use the + # SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and + # CREATORS). Accept that form too so users who follow the vendor's docs + # don't silently end up with has_scrapecreators=False. Canonical name + # wins when both are set. + if not config.get('SCRAPECREATORS_API_KEY'): + legacy = os.environ.get('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY') + if legacy: + config['SCRAPECREATORS_API_KEY'] = legacy + # Track which config source was used (highest-priority file source wins # the label; keychain is only reported when nothing else is configured). if project_env_path: diff --git a/skills/last30days/scripts/lib/instagram.py b/skills/last30days/scripts/lib/instagram.py index 0ffd48b..ab69b6b 100644 --- a/skills/last30days/scripts/lib/instagram.py +++ b/skills/last30days/scripts/lib/instagram.py @@ -7,12 +7,14 @@ Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG. API docs: https://scrapecreators.com/docs """ +import os import re import sys from datetime import datetime from typing import Any, Dict, List, Optional, Set from . import dates, http, log +from .relevance import token_overlap_relevance as _compute_relevance SCRAPECREATORS_BASE = "https://api.scrapecreators.com" @@ -26,7 +28,42 @@ DEPTH_CONFIG = { # Max words to keep from each caption CAPTION_MAX_WORDS = 500 -from .relevance import token_overlap_relevance as _compute_relevance +# Default transcript fetch timeout (seconds). SC's +# /v2/instagram/media/transcript regularly takes >15s on real workloads, +# so the default is generous; override via LAST30DAYS_TRANSCRIPT_TIMEOUT. +DEFAULT_TRANSCRIPT_TIMEOUT = 30 + + +def _resolve_transcript_timeout( + timeout: Optional[float] = None, + config: Optional[Dict[str, Any]] = None, +) -> float: + """Resolve the IG transcript-fetch timeout. + + Priority (highest wins): + 1. Explicit ``timeout`` kwarg + 2. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in os.environ + 3. ``LAST30DAYS_TRANSCRIPT_TIMEOUT`` in caller-supplied config dict + 4. ``DEFAULT_TRANSCRIPT_TIMEOUT`` (30s) + + Mirrors the ``os.environ.get(X) or config.get(X)`` pattern used for + LAST30DAYS_STORE in last30days.py so the env var works whether it's + shell-exported or set in ~/.config/last30days/.env. + """ + if timeout is not None: + try: + return float(timeout) + except (TypeError, ValueError): + pass + raw = os.environ.get("LAST30DAYS_TRANSCRIPT_TIMEOUT") + if not raw and config: + raw = config.get("LAST30DAYS_TRANSCRIPT_TIMEOUT") + if raw: + try: + return float(raw) + except (TypeError, ValueError): + pass + return float(DEFAULT_TRANSCRIPT_TIMEOUT) def _extract_core_subject(topic: str) -> str: @@ -44,6 +81,17 @@ def _extract_core_subject(topic: str) -> str: return extract_core_subject(topic, noise=_INSTAGRAM_NOISE) +def _to_hashtag_form(query: str) -> str: + """Collapse a multi-word query to hashtag form (no spaces, lowercase). + + SC's /v2/instagram/reels/search wraps Google Search and is documented + to be flaky on multi-token queries. Single-token queries map to a + hashtag page lookup which is the stable path. Used as a 500-retry + fallback before the request bubbles up as a silent failure. + """ + return ''.join(query.split()).lower() + + def _infer_query_intent(topic: str) -> str: """Tiny local intent classifier for Instagram query expansion.""" text = topic.lower().strip() @@ -283,6 +331,26 @@ def search_instagram( timeout=30, retries=2, ) + except http.HTTPError as e: + # SC's v2 reels search wraps Google Search and 500s frequently on + # multi-token queries. Single tokens hit the stable hashtag-page + # path. Retry once with hashtag form before bubbling up. + if getattr(e, "status_code", None) == 500 and ' ' in core_topic: + _log(f"IG search 500 on '{core_topic}', retrying with hashtag form") + try: + data = http.get( + f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search", + params={"query": _to_hashtag_form(core_topic)}, + headers=http.scrapecreators_headers(token), + timeout=30, + retries=2, + ) + except Exception as retry_e: + _log(f"IG search retry failed: {retry_e}") + return {"items": [], "error": f"{type(retry_e).__name__}: {retry_e}"} + else: + _log(f"ScrapeCreators error: {e}") + return {"items": [], "error": f"{type(e).__name__}: {e}"} except Exception as e: _log(f"ScrapeCreators error: {e}") return {"items": [], "error": f"{type(e).__name__}: {e}"} @@ -317,6 +385,8 @@ def fetch_captions( video_items: List[Dict[str, Any]], token: str, depth: str = "default", + timeout: Optional[float] = None, + config: Optional[Dict[str, Any]] = None, ) -> Dict[str, str]: """Fetch transcripts for top N Instagram reels via ScrapeCreators. @@ -328,12 +398,19 @@ def fetch_captions( video_items: Items from search_instagram() token: ScrapeCreators API key depth: Depth level for caption limit + timeout: Optional per-request transcript timeout in seconds. When + None, resolves from LAST30DAYS_TRANSCRIPT_TIMEOUT (env or + config), defaulting to DEFAULT_TRANSCRIPT_TIMEOUT (30s). + config: Optional config dict (from env.get_config()) used as a + fallback source for LAST30DAYS_TRANSCRIPT_TIMEOUT when the + value is not exported in os.environ. Returns: Dict mapping video_id -> caption text (truncated to 500 words) """ - config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) - max_captions = config["max_captions"] + depth_cfg = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + max_captions = depth_cfg["max_captions"] + transcript_timeout = _resolve_transcript_timeout(timeout, config) if not video_items or not token: return {} @@ -364,7 +441,7 @@ def fetch_captions( f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript", params={"url": url}, headers=http.scrapecreators_headers(token), - timeout=15, + timeout=transcript_timeout, retries=1, ) transcripts = data.get("transcripts") or [] diff --git a/skills/last30days/scripts/lib/normalize.py b/skills/last30days/scripts/lib/normalize.py index 214119f..b8035d3 100644 --- a/skills/last30days/scripts/lib/normalize.py +++ b/skills/last30days/scripts/lib/normalize.py @@ -251,6 +251,11 @@ def _normalize_youtube( metadata: dict[str, Any] = {} if highlights: metadata["transcript_highlights"] = highlights + if item.get("captions_disabled"): + # Surfaced for quality_nudge: uploader disabled captions, so this + # video should be subtracted from the degraded-transcript-ratio + # denominator (it was never going to produce a transcript). + metadata["captions_disabled"] = True metadata["top_comments"] = _remap_comments( item.get("top_comments") or [], score_keys=("score", "likes"), diff --git a/skills/last30days/scripts/lib/quality_nudge.py b/skills/last30days/scripts/lib/quality_nudge.py index 7ffbb62..4464903 100644 --- a/skills/last30days/scripts/lib/quality_nudge.py +++ b/skills/last30days/scripts/lib/quality_nudge.py @@ -58,12 +58,38 @@ def _is_youtube_degraded(research_results: dict, threshold: float) -> bool: ratio is below threshold. The canonical cause is a stale yt-dlp binary - YouTube's caption format changes frequently and old binaries silently fail every transcript while the search itself still succeeds. + + Captions-disabled videos are subtracted from the denominator: an uploader + who turned off captions can never produce a transcript, so counting that + video toward "fetch failures" produces false positives. A single + captions-disabled video in a small result set was tripping the nudge. """ videos = int(research_results.get("youtube_videos_count") or 0) transcripts = int(research_results.get("youtube_transcripts_count") or 0) + captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0) if videos <= 0: return False - return (transcripts / videos) < threshold + eligible = videos - captions_disabled + if eligible <= 0: + # Every returned video had captions disabled - upstream content fact, + # not a yt-dlp problem. Don't flag. + return False + return (transcripts / eligible) < threshold + + +def _is_instagram_silent_failure(config: dict, research_results: dict) -> bool: + """Instagram is silently failing when SC is configured but the source + returned zero items. The canonical cause is SC's v2 reels endpoint + 500'ing on multi-token queries (it wraps Google Search and is documented + to be flaky there). Pre-fix the user got no signal at all - no Instagram + section in the brief, no error in the footer, just unexplained absence. + """ + if not config.get("SCRAPECREATORS_API_KEY"): + return False # not configured — not a silent failure + count = research_results.get("instagram_items_count") + if count is None: + return False # source not run this invocation + return int(count) == 0 def compute_quality_score(config: dict, research_results: dict) -> dict: @@ -75,6 +101,8 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: reddit_error reflecting what happened this run. Optional keys ``youtube_videos_count`` and ``youtube_transcripts_count`` enable degraded-YouTube detection (transcript-fetch ratio below threshold). + Optional key ``instagram_items_count`` enables silent-failure + detection for the bonus Instagram source. Returns: { @@ -83,13 +111,15 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: "core_missing": ["x", "youtube"], "core_errored": [], # configured but errored at top level "core_degraded": [], # configured and returned items but quality below threshold - "nudge_text": "..." or None if 100% + "bonus_errored": [], # bonus sources (Instagram, etc.) configured but silent + "nudge_text": "..." or None if all sources healthy } """ core_active: List[str] = [] core_missing: List[str] = [] core_errored: List[str] = [] core_degraded: List[str] = [] + bonus_errored: List[str] = [] # HN, Polymarket, and Reddit are always active core_active.append("hn") @@ -127,6 +157,11 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: if has_ytdlp and research_results.get("youtube_error"): core_errored.append("youtube") + # Bonus sources (Instagram, etc.): SC-key holders expect content from + # these but until now the pipeline fell silent on configured-but-zero. + if _is_instagram_silent_failure(config, research_results): + bonus_errored.append("instagram") + score_pct = int(len(core_active) / 5 * 100) has_sc = bool(config.get("SCRAPECREATORS_API_KEY")) @@ -138,7 +173,8 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: research_results, has_sc=has_sc, active_sources=active_sources, - ) if (core_missing or core_degraded) else None + bonus_errored=bonus_errored, + ) if (core_missing or core_degraded or bonus_errored) else None return { "score_pct": score_pct, @@ -146,6 +182,7 @@ def compute_quality_score(config: dict, research_results: dict) -> dict: "core_missing": core_missing, "core_errored": core_errored, "core_degraded": core_degraded, + "bonus_errored": bonus_errored, "nudge_text": nudge_text, } @@ -157,6 +194,7 @@ def _build_nudge_text( research_results: dict = None, has_sc: bool = False, active_sources: list = None, + bonus_errored: List[str] = None, ) -> str: """Build human-readable nudge text describing what was missed or degraded. @@ -165,6 +203,7 @@ def _build_nudge_text( """ lines: List[str] = [] core_degraded = core_degraded or [] + bonus_errored = bonus_errored or [] research_results = research_results or {} # Describe what was missed @@ -183,6 +222,9 @@ def _build_nudge_text( if core_degraded: degraded_labels = ", ".join(SOURCE_LABELS[s] for s in core_degraded) lines.append(f"Degraded: {degraded_labels}.") + if bonus_errored: + bonus_labels = ", ".join(s.capitalize() for s in bonus_errored) + lines.append(f"Bonus source silent: {bonus_labels}.") lines.append("") # Free suggestions @@ -215,12 +257,30 @@ def _build_nudge_text( if "youtube" in core_degraded: videos = int(research_results.get("youtube_videos_count") or 0) transcripts = int(research_results.get("youtube_transcripts_count") or 0) + captions_disabled = int(research_results.get("youtube_captions_disabled_count") or 0) + captions_note = "" + if captions_disabled > 0: + captions_note = ( + f" ({captions_disabled} of those had captions disabled by the " + "uploader, which is a separate cause and not fixable on your end)" + ) free_suggestions.append( f"YouTube returned {videos} videos but only {transcripts} transcripts " - "captured. The most common cause is a stale yt-dlp binary - YouTube's " - "caption format changes frequently and old binaries silently fail every " - "transcript. Update via your package manager: scoop update yt-dlp " - "(Windows), brew upgrade yt-dlp (macOS), or pip install -U yt-dlp." + f"captured{captions_note}. The most common remaining cause is a stale " + "yt-dlp binary - YouTube's caption format changes frequently and old " + "binaries silently fail every transcript. Update via your package " + "manager: scoop update yt-dlp (Windows), brew upgrade yt-dlp (macOS), " + "or pip install -U yt-dlp." + ) + + if "instagram" in bonus_errored: + free_suggestions.append( + "Instagram returned 0 reels despite SC being configured. SC's " + "v2 reels endpoint wraps Google Search and 500's frequently on " + "multi-token queries. The skill now retries with hashtag-form " + "automatically; if zero items still appear, the topic may have " + "no reel coverage on Instagram. Try a single-word topic like " + "the most distinctive noun in your query." ) # Mention bonus opt-in sources when SC key is present diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py index fe033d1..3e46685 100644 --- a/skills/last30days/scripts/lib/youtube_yt.py +++ b/skills/last30days/scripts/lib/youtube_yt.py @@ -385,7 +385,11 @@ def _clean_vtt(vtt_text: str) -> str: _YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" -def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]: +def _fetch_transcript_direct( + video_id: str, + timeout: int = 30, + status: Optional[Dict[str, Any]] = None, +) -> Optional[str]: """Fetch YouTube transcript via direct HTTP without yt-dlp. Scrapes the watch page HTML for the captions track URL in @@ -394,6 +398,9 @@ def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]: Args: video_id: YouTube video ID timeout: HTTP request timeout in seconds + status: Optional dict mutated to record per-video signals. Sets + ``status["no_caption_tracks"] = True`` when the player response + confirms the uploader has no caption tracks (vs. fetch failure). Returns: Raw VTT text, or None if captions are unavailable. @@ -442,6 +449,8 @@ def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]: if not caption_tracks: _log(f"Direct transcript: no caption tracks for {video_id}") + if status is not None: + status["no_caption_tracks"] = True return None # Find English track (prefer exact 'en', then any en variant, then first track) @@ -527,7 +536,11 @@ def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]: return None -def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: +def fetch_transcript( + video_id: str, + temp_dir: str, + status: Optional[Dict[str, Any]] = None, +) -> Optional[str]: """Fetch auto-generated transcript for a YouTube video. Uses yt-dlp when available (preferred, more robust). Falls back to @@ -536,6 +549,10 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: Args: video_id: YouTube video ID temp_dir: Temporary directory for subtitle files + status: Optional dict mutated by the direct-HTTP path to record + per-video signals like ``no_caption_tracks``. Used to surface a + captions-disabled count so the quality nudge avoids false-positive + "stale yt-dlp" flags. Returns: Plaintext transcript string, or None if no captions available. @@ -551,13 +568,13 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir) if not raw_vtt: _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback") - raw_vtt = _fetch_transcript_direct(video_id) + raw_vtt = _fetch_transcript_direct(video_id, status=status) else: if ssh_host: _log("SSH-routing active, using direct HTTP transcript fetch") else: _log("yt-dlp not installed, using direct HTTP transcript fetch") - raw_vtt = _fetch_transcript_direct(video_id) + raw_vtt = _fetch_transcript_direct(video_id, status=status) if not raw_vtt: _log(f"No transcript available for {video_id} (no captions found)") @@ -576,12 +593,16 @@ def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]: def fetch_transcripts_parallel( video_ids: List[str], max_workers: int = 5, + out_captions_disabled: Optional[Set[str]] = None, ) -> Dict[str, Optional[str]]: """Fetch transcripts for multiple videos in parallel. Args: video_ids: List of YouTube video IDs max_workers: Max parallel fetches + out_captions_disabled: Optional set mutated to record video_ids whose + uploader confirmed no caption tracks (vs. transient fetch failures). + Backward-compatible: callers that don't care can omit. Returns: Dict mapping video_id to transcript text (or None). @@ -592,10 +613,11 @@ def fetch_transcripts_parallel( _log(f"Fetching transcripts for {len(video_ids)} videos") results = {} + statuses: Dict[str, Dict[str, Any]] = {vid: {} for vid in video_ids} with tempfile.TemporaryDirectory() as temp_dir: with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { - executor.submit(fetch_transcript, vid, temp_dir): vid + executor.submit(fetch_transcript, vid, temp_dir, statuses[vid]): vid for vid in video_ids } for future in as_completed(futures): @@ -609,6 +631,11 @@ def fetch_transcripts_parallel( _log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}") results[vid] = None + if out_captions_disabled is not None: + for vid, st in statuses.items(): + if st.get("no_caption_tracks"): + out_captions_disabled.add(vid) + got = sum(1 for v in results.values() if v) errors = sum(1 for v in results.values() if v is None) _log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)") @@ -659,15 +686,21 @@ def search_and_transcribe( # good chance of reaching the target number of successful transcripts. transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"]) transcripts: Dict[str, Optional[str]] = {} + captions_disabled_ids: Set[str] = set() if transcript_limit > 0: attempt_count = min(len(items), transcript_limit * 3) candidate_ids = [item["video_id"] for item in items[:attempt_count]] _log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}") - transcripts = fetch_transcripts_parallel(candidate_ids) + transcripts = fetch_transcripts_parallel( + candidate_ids, out_captions_disabled=captions_disabled_ids, + ) else: _log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch") - # Step 3: Attach transcripts and extract highlights + # Step 3: Attach transcripts and extract highlights. Mark captions_disabled + # so quality_nudge can subtract those videos from the degraded-ratio + # denominator (uploader-disabled captions can never produce a transcript; + # counting them was producing false-positive stale-yt-dlp nudges). core_topic = _extract_core_subject(topic) for item in items: vid = item["video_id"] @@ -676,6 +709,7 @@ def search_and_transcribe( item["transcript_highlights"] = extract_transcript_highlights( transcript or "", core_topic, ) + item["captions_disabled"] = vid in captions_disabled_ids return {"items": items} diff --git a/tests/test_bluesky.py b/tests/test_bluesky.py index cb3cd33..c04afd5 100644 --- a/tests/test_bluesky.py +++ b/tests/test_bluesky.py @@ -1,5 +1,6 @@ """Tests for bluesky module.""" +import os import sys import unittest from pathlib import Path @@ -211,5 +212,146 @@ class TestSearchBlueskyAuth(unittest.TestCase): self.assertEqual(mock_request.call_args_list[3].kwargs.get("headers", {}), {"Authorization": "Bearer tok-new"}) +class TestSearchEndpointHostResolution(unittest.TestCase): + """The default search host moved from `public.api.bsky.app` (the + unauthenticated public mirror, now BunnyCDN-blocked for searchPosts) to + `api.bsky.app` (the canonical authenticated AppView). BSKY_SEARCH_HOST + env var or config value can override the default if Bluesky migrates + infrastructure again. Same os.environ-or-config hybrid pattern as + LAST30DAYS_STORE. + """ + + def setUp(self): + # Snapshot env so per-test overrides don't leak + self._saved_env = os.environ.pop("BSKY_SEARCH_HOST", None) + + def tearDown(self): + if self._saved_env is not None: + os.environ["BSKY_SEARCH_HOST"] = self._saved_env + else: + os.environ.pop("BSKY_SEARCH_HOST", None) + + def test_module_constant_uses_canonical_appview(self): + # Regression guard against the public mirror reappearing as the default + self.assertIn("api.bsky.app", bluesky.BSKY_SEARCH_URL) + + def test_module_constant_does_not_use_public_mirror(self): + # Hard regression guard — the exact host that BunnyCDN was blocking + self.assertNotIn("public.api.bsky.app", bluesky.BSKY_SEARCH_URL) + + def test_resolver_default_when_no_override(self): + self.assertEqual( + bluesky._resolve_search_url(), + "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + def test_resolver_env_var_override(self): + os.environ["BSKY_SEARCH_HOST"] = "staging.bsky.app" + self.assertEqual( + bluesky._resolve_search_url(), + "https://staging.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + def test_resolver_config_dict_override(self): + # User has BSKY_SEARCH_HOST only in .env file (project loads .env into + # config, not os.environ). Resolver must read both. + url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "pds.example.com"}) + self.assertEqual(url, "https://pds.example.com/xrpc/app.bsky.feed.searchPosts") + + def test_resolver_env_var_wins_over_config(self): + # When both are set, os.environ takes precedence (matches LAST30DAYS_STORE) + os.environ["BSKY_SEARCH_HOST"] = "shell-host.example" + url = bluesky._resolve_search_url({"BSKY_SEARCH_HOST": "config-host.example"}) + self.assertIn("shell-host.example", url) + self.assertNotIn("config-host.example", url) + + def test_resolver_output_does_not_use_public_mirror(self): + # Regression guard at the resolver level (not just the constant) — + # this is what runtime actually calls. The constant-level guard + # above doesn't catch a regression where the resolver reverts. + self.assertNotIn("public.api.bsky.app", bluesky._resolve_search_url()) + + def test_resolver_strips_surrounding_whitespace(self): + # Pre-fix: " api.bsky.app " produced "https:// api.bsky.app /xrpc/..." + # which urllib raises ValueError on with no hint the env var caused it. + os.environ["BSKY_SEARCH_HOST"] = " api.bsky.app " + self.assertEqual( + bluesky._resolve_search_url(), + "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + def test_resolver_rejects_embedded_path(self): + # "my-proxy.com/xrpc/prefix" would have doubled the /xrpc/ segment. + # We fall back to the default to avoid a guaranteed 404. + os.environ["BSKY_SEARCH_HOST"] = "my-proxy.example.com/xrpc/prefix" + self.assertEqual( + bluesky._resolve_search_url(), + "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + def test_resolver_strips_embedded_scheme(self): + # Users who paste a full URL get a sane outcome, not a malformed URL. + os.environ["BSKY_SEARCH_HOST"] = "https://api.bsky.app" + self.assertEqual( + bluesky._resolve_search_url(), + "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + def test_resolver_empty_string_falls_back_to_default(self): + os.environ["BSKY_SEARCH_HOST"] = "" + self.assertEqual( + bluesky._resolve_search_url(), + "https://api.bsky.app/xrpc/app.bsky.feed.searchPosts", + ) + + +class TestAppPasswordFormat(unittest.TestCase): + """Bluesky app passwords are 19-char xxxx-xxxx-xxxx-xxxx (lowercase + alphanumeric, three hyphens at fixed positions). Main-account passwords + are accepted by createSession but are bad hygiene. The validator detects + the format mismatch without gating any caller. + """ + + def test_accepts_valid_app_password_form(self): + # Use a fake example — never a real password + self.assertTrue(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy5")) + + def test_rejects_length_15_string(self): + # The exact failure mode that triggered the 2026-05-04 investigation: + # user stored their main login password (15 chars) in BSKY_APP_PASSWORD + self.assertFalse(bluesky._validate_app_password_format("mainpassword123")) + + def test_rejects_16_char_no_hyphen_string(self): + # Hex-style API key shape — common confusion with other services + self.assertFalse(bluesky._validate_app_password_format("abcdef0123456789")) + + def test_rejects_uppercase_letters(self): + # Bluesky app passwords are all-lowercase by spec + self.assertFalse(bluesky._validate_app_password_format("WFWP-cq7o-5six-7wy5")) + + def test_rejects_underscore_separator(self): + # Wrong separator + self.assertFalse(bluesky._validate_app_password_format("wfwp_cq7o_5six_7wy5")) + + def test_rejects_special_chars_in_groups(self): + # Special characters are not part of the alphanumeric class + self.assertFalse(bluesky._validate_app_password_format("wfwp-cq7o-5six-7wy@")) + + def test_rejects_empty_string(self): + self.assertFalse(bluesky._validate_app_password_format("")) + + def test_rejects_none(self): + # Callers may pass config.get('BSKY_APP_PASSWORD') which is None when unset + self.assertFalse(bluesky._validate_app_password_format(None)) + + def test_rejects_integer(self): + # Defensive: don't crash if a numeric value sneaks in + self.assertFalse(bluesky._validate_app_password_format(123456789012345)) + + def test_rejects_list(self): + # Defensive: don't crash on iterables + self.assertFalse(bluesky._validate_app_password_format(["wfwp", "cq7o", "5six", "7wy5"])) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_instagram_sc.py b/tests/test_instagram_sc.py index 88ebf7f..2721353 100644 --- a/tests/test_instagram_sc.py +++ b/tests/test_instagram_sc.py @@ -1,8 +1,10 @@ """Tests for instagram.py — ScrapeCreators Instagram search module.""" +import os import sys import unittest from pathlib import Path +from unittest.mock import MagicMock, patch # Add lib to path sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "last30days" / "scripts")) @@ -85,5 +87,223 @@ class TestInstagramDepthConfig(unittest.TestCase): ) +class TestHashtagFormCollapse(unittest.TestCase): + """Tests for _to_hashtag_form() — the multi-word retry workaround.""" + + def test_collapses_spaces(self): + self.assertEqual(instagram._to_hashtag_form("toronto real estate"), "torontorealestate") + + def test_lowercases(self): + self.assertEqual(instagram._to_hashtag_form("Toronto REAL Estate"), "torontorealestate") + + def test_idempotent_on_single_word(self): + self.assertEqual(instagram._to_hashtag_form("ozempic"), "ozempic") + + def test_handles_extra_whitespace(self): + self.assertEqual(instagram._to_hashtag_form(" toronto real estate "), "torontorealestate") + + +class TestSearchRetryOn500(unittest.TestCase): + """Tests for the multi-word -> hashtag retry on SC's flaky 500 path. + + SC's /v2/instagram/reels/search wraps Google Search and is documented + to be unreliable on multi-token queries. The retry collapses to a + hashtag form which hits the stable hashtag-page lookup path. + """ + + def _mock_response(self, status_code, json_payload=None): + m = MagicMock() + m.status_code = status_code + m.json.return_value = json_payload or {} + if status_code >= 400: + m.raise_for_status.side_effect = Exception(f"HTTP {status_code}") + else: + m.raise_for_status.return_value = None + return m + + def test_multiword_500_triggers_retry_with_hashtag_form(self): + """Multi-word query 500 -> retry with collapsed hashtag form.""" + first = self._mock_response(500) + second = self._mock_response(200, {"reels": []}) + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.side_effect = [first, second] + instagram.search_instagram( + "toronto real estate", "2026-04-01", "2026-05-04", + depth="default", token="fake-token", + ) + self.assertEqual(mock_requests.get.call_count, 2) + # First call: original multi-word query + first_params = mock_requests.get.call_args_list[0].kwargs["params"] + self.assertEqual(first_params["query"], "toronto real estate") + # Second call: collapsed hashtag form + second_params = mock_requests.get.call_args_list[1].kwargs["params"] + self.assertEqual(second_params["query"], "torontorealestate") + + def test_singleword_500_does_not_retry(self): + """Single-word query 500 has no spaces to collapse - no retry.""" + only = self._mock_response(500) + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = only + result = instagram.search_instagram( + "ozempic", "2026-04-01", "2026-05-04", + depth="default", token="fake-token", + ) + self.assertEqual(mock_requests.get.call_count, 1) + self.assertIn("error", result) + self.assertEqual(result["items"], []) + + def test_first_call_succeeds_no_retry(self): + """200 on first call -> retry path is never entered.""" + ok = self._mock_response(200, {"reels": []}) + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = ok + instagram.search_instagram( + "toronto real estate", "2026-04-01", "2026-05-04", + depth="default", token="fake-token", + ) + self.assertEqual(mock_requests.get.call_count, 1) + + def test_no_token_short_circuits(self): + """No SCRAPECREATORS_API_KEY -> error returned without HTTP call.""" + with patch.object(instagram, "_requests") as mock_requests: + result = instagram.search_instagram( + "toronto real estate", "2026-04-01", "2026-05-04", + depth="default", token=None, + ) + mock_requests.get.assert_not_called() + self.assertIn("error", result) + self.assertIn("SCRAPECREATORS_API_KEY", result["error"]) + + +class TestTranscriptTimeoutConfig(unittest.TestCase): + """Tests for LAST30DAYS_TRANSCRIPT_TIMEOUT configuration. + + SC's /v2/instagram/media/transcript endpoint regularly takes >15s, + so the timeout must be configurable. Default is DEFAULT_TRANSCRIPT_TIMEOUT + (30s); the env var or per-call kwarg overrides it. + """ + + def setUp(self): + # Snapshot any pre-existing env so we don't leak across tests + self._saved_env = os.environ.pop("LAST30DAYS_TRANSCRIPT_TIMEOUT", None) + + def tearDown(self): + os.environ.pop("LAST30DAYS_TRANSCRIPT_TIMEOUT", None) + if self._saved_env is not None: + os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = self._saved_env + + def _ok_response(self): + m = MagicMock() + m.status_code = 200 + m.json.return_value = {"transcripts": [{"text": "hello world"}]} + return m + + def _video_item(self, vid="abc123"): + return { + "video_id": vid, + "url": f"https://www.instagram.com/reel/{vid}/", + "text": "", + } + + def test_default_timeout_is_30s_when_nothing_set(self): + """No env var, no kwarg -> request uses 30s, not the legacy 15s.""" + items = [self._video_item()] + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = self._ok_response() + instagram.fetch_captions(items, token="fake-token") + kwargs = mock_requests.get.call_args.kwargs + self.assertEqual(kwargs["timeout"], 30.0) + + def test_env_var_override(self): + """LAST30DAYS_TRANSCRIPT_TIMEOUT='60' -> request uses 60s.""" + os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "60" + items = [self._video_item()] + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = self._ok_response() + instagram.fetch_captions(items, token="fake-token") + kwargs = mock_requests.get.call_args.kwargs + self.assertEqual(kwargs["timeout"], 60.0) + + def test_explicit_timeout_kwarg_wins_over_env(self): + """Explicit timeout= kwarg trumps the env var.""" + os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "60" + items = [self._video_item()] + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = self._ok_response() + instagram.fetch_captions(items, token="fake-token", timeout=10) + kwargs = mock_requests.get.call_args.kwargs + self.assertEqual(kwargs["timeout"], 10.0) + + def test_config_dict_fallback_when_env_unset(self): + """config={'LAST30DAYS_TRANSCRIPT_TIMEOUT': '45'} -> request uses 45s.""" + items = [self._video_item()] + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = self._ok_response() + instagram.fetch_captions( + items, + token="fake-token", + config={"LAST30DAYS_TRANSCRIPT_TIMEOUT": "45"}, + ) + kwargs = mock_requests.get.call_args.kwargs + self.assertEqual(kwargs["timeout"], 45.0) + + def test_invalid_env_value_falls_back_to_default(self): + """Garbage env var doesn't crash; falls back to 30s.""" + os.environ["LAST30DAYS_TRANSCRIPT_TIMEOUT"] = "not-a-number" + items = [self._video_item()] + with patch.object(instagram, "_requests") as mock_requests: + mock_requests.get.return_value = self._ok_response() + instagram.fetch_captions(items, token="fake-token") + kwargs = mock_requests.get.call_args.kwargs + self.assertEqual(kwargs["timeout"], 30.0) + + +class TestSearchRetryOn500Urllib(unittest.TestCase): + """Lock in the urllib-path 500-retry. Pre-fix the retry was dead code on + the urllib branch because it checked `getattr(e, 'status', None)` while + `http.HTTPError` exposes the code as `status_code`. Caught by code-review + on 2026-05-04 (REL-001 / ADV-001, two reviewers at 0.97/0.98 confidence). + """ + + def setUp(self): + # Force urllib path by making instagram._requests look absent + self._saved_requests = instagram._requests + instagram._requests = None + + def tearDown(self): + instagram._requests = self._saved_requests + + def test_urllib_500_on_multiword_triggers_retry_with_hashtag_form(self): + from lib import http as http_module + first_error = http_module.HTTPError("HTTP 500: Server Error", 500, "") + second_payload = {"reels": []} + # http.get is called twice: first raises HTTPError(500), second returns dict + with patch.object(http_module, "get") as mock_http_get: + mock_http_get.side_effect = [first_error, second_payload] + instagram.search_instagram( + "toronto real estate", "2026-04-01", "2026-05-04", + depth="default", token="fake-token", + ) + self.assertEqual(mock_http_get.call_count, 2) + # First call URL contains the original multi-word query + first_url = mock_http_get.call_args_list[0].args[0] + self.assertIn("query=toronto+real+estate", first_url) + # Second call URL contains the collapsed hashtag form + second_url = mock_http_get.call_args_list[1].args[0] + self.assertIn("query=torontorealestate", second_url) + + def test_urllib_500_on_singleword_does_not_retry(self): + from lib import http as http_module + only_error = http_module.HTTPError("HTTP 500: Server Error", 500, "") + with patch.object(http_module, "get") as mock_http_get: + mock_http_get.side_effect = only_error + result = instagram.search_instagram( + "ozempic", "2026-04-01", "2026-05-04", + depth="default", token="fake-token", + ) + self.assertEqual(mock_http_get.call_count, 1) + self.assertIn("error", result) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_quality_nudge.py b/tests/test_quality_nudge.py index a2592e9..41476c5 100644 --- a/tests/test_quality_nudge.py +++ b/tests/test_quality_nudge.py @@ -293,3 +293,174 @@ class TestYouTubeDegraded: # But nudge still fires assert q["nudge_text"] is not None assert "Degraded: YouTube" in q["nudge_text"] + + +class TestYouTubeCaptionsDisabledDoesNotFalseFlag: + """Captions-disabled videos must not lower the transcript-fetch ratio. + + A video where the uploader disabled captions can never produce a transcript, + no matter how fresh yt-dlp is. Counting it in the denominator of the + degraded-ratio check produces false positives - one captions-disabled video + in a small result set was triggering a "stale yt-dlp binary" nudge that was + wrong. Fix: subtract captions_disabled from the denominator. + """ + + def test_zero_captions_disabled_preserves_existing_behavior(self): + # Pre-existing case: 0 of 6 transcripts is still degraded (no captions + # disabled to discount). Behavior is unchanged from TestYouTubeDegraded. + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + "youtube_captions_disabled_count": 0, + }, + ) + assert "youtube" in q["core_degraded"] + + def test_all_videos_captions_disabled_does_not_flag(self): + # Every returned video had captions disabled by the uploader. + # That's not a yt-dlp problem - it's an upstream content fact. Must not + # flag degraded. + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 3, + "youtube_transcripts_count": 0, + "youtube_captions_disabled_count": 3, + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_mixed_uses_corrected_denominator(self): + # 6 videos, 3 captions_disabled, 2 transcripts. + # Naive (buggy) ratio: 2/6 = 33% (would flag). + # Corrected ratio: 2/(6-3) = 67% (does NOT flag). + # This case demonstrates the fix changes the verdict. + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 2, + "youtube_captions_disabled_count": 3, + }, + ) + assert "youtube" not in q["core_degraded"] + + def test_mixed_still_flags_when_truly_degraded(self): + # Even after discounting captions-disabled, the ratio is still bad. + # 8 videos, 1 captions_disabled, 1 transcript -> 1/(8-1) = 14% (flags). + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 8, + "youtube_transcripts_count": 1, + "youtube_captions_disabled_count": 1, + }, + ) + assert "youtube" in q["core_degraded"] + # Nudge should still mention the stale yt-dlp possibility but also + # acknowledge that captions-disabled is a separate cause. + assert q["nudge_text"] is not None + assert "captions disabled" in q["nudge_text"].lower() + + def test_missing_count_defaults_to_zero(self): + # Older callers that don't pass the new key still work (default 0). + q = _compute( + ytdlp_installed=True, + result_overrides={ + "youtube_videos_count": 6, + "youtube_transcripts_count": 0, + # youtube_captions_disabled_count intentionally omitted + }, + ) + assert "youtube" in q["core_degraded"] + + +class TestInstagramSilentFailure: + """Instagram is a `bonus` source via SC. Silent-failure detection: if SC + is configured but the source returned zero items, surface a nudge so the + user understands why the brief lacks an Instagram section. + + Pre-fix the user got no signal - SC's /v2/instagram/reels/search 500s + frequently on multi-token queries and the pipeline silently returned + empty without any indication. + """ + + def test_zero_items_with_sc_flags_bonus_errored(self): + q = _compute( + config_overrides={ + "AUTH_TOKEN": "tok123", + "SCRAPECREATORS_API_KEY": "sc_key", + }, + ytdlp_installed=True, + result_overrides={"instagram_items_count": 0}, + ) + assert "instagram" in q["bonus_errored"] + assert q["nudge_text"] is not None + assert "Instagram" in q["nudge_text"] + + def test_zero_items_without_sc_does_not_flag(self): + q = _compute( + config_overrides={"AUTH_TOKEN": "tok123"}, + ytdlp_installed=True, + result_overrides={"instagram_items_count": 0}, + ) + assert "instagram" not in q.get("bonus_errored", []) + + def test_nonzero_items_does_not_flag(self): + q = _compute( + config_overrides={ + "AUTH_TOKEN": "tok123", + "SCRAPECREATORS_API_KEY": "sc_key", + }, + ytdlp_installed=True, + result_overrides={"instagram_items_count": 5}, + ) + assert "instagram" not in q["bonus_errored"] + assert q["nudge_text"] is None + + def test_missing_key_means_source_did_not_run(self): + q = _compute( + config_overrides={ + "AUTH_TOKEN": "tok123", + "SCRAPECREATORS_API_KEY": "sc_key", + }, + ytdlp_installed=True, + ) + assert "instagram" not in q["bonus_errored"] + assert q["nudge_text"] is None + + def test_nudge_text_explains_workaround(self): + q = _compute( + config_overrides={ + "AUTH_TOKEN": "tok123", + "SCRAPECREATORS_API_KEY": "sc_key", + }, + ytdlp_installed=True, + result_overrides={"instagram_items_count": 0}, + ) + assert q["nudge_text"] is not None + text_lower = q["nudge_text"].lower() + assert "instagram" in text_lower + assert ("0 reels" in text_lower or "silent" in text_lower + or "hashtag" in text_lower) + + def test_bonus_errored_does_not_affect_core_score(self): + q = _compute( + config_overrides={ + "AUTH_TOKEN": "tok123", + "SCRAPECREATORS_API_KEY": "sc_key", + }, + ytdlp_installed=True, + result_overrides={"instagram_items_count": 0}, + ) + assert q["score_pct"] == 100 + assert "instagram" in q["bonus_errored"] + assert q["nudge_text"] is not None + assert "Bonus source silent" in q["nudge_text"] + + def test_bonus_errored_field_always_present(self): + q = _compute() + assert q.get("bonus_errored") == [] + diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py index be55b83..6b45eb9 100644 --- a/tests/test_youtube_yt.py +++ b/tests/test_youtube_yt.py @@ -252,7 +252,7 @@ class TestFetchTranscriptFallback(unittest.TestCase): mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=sample_vtt) as direct_mock: result = youtube_yt.fetch_transcript("vid2", "/tmp/test") yt_mock.assert_not_called() - direct_mock.assert_called_once_with("vid2") + direct_mock.assert_called_once_with("vid2", status=None) self.assertIsNotNone(result) self.assertIn("Direct transcript content", result) @@ -363,7 +363,7 @@ class TestSearchAndTranscribe(unittest.TestCase): ] # fetch_transcripts_parallel returns None for music videos, text for talks - def fake_parallel(video_ids, max_workers=5): + def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None): result = {} for vid in video_ids: if vid.startswith("talk"):