diff --git a/SKILL.md b/SKILL.md index 4cbc122..3c06576 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,7 +1,7 @@ --- name: last30days version: "2.1" -description: "Research a topic from the last 30 days. Also triggered by 'last30'. Sources: Reddit, X, YouTube, web. Become an expert and write copy-paste-ready prompts." +description: "Research a topic from the last 30 days. Also triggered by 'last30'. Sources: Reddit, X, YouTube, Hacker News, web. Become an expert and write copy-paste-ready prompts." argument-hint: 'last30 AI video tools, last30 best project management tools' allowed-tools: Bash, Read, Write, AskUserQuestion, WebSearch homepage: https://github.com/mvanhorn/last30days-skill @@ -25,13 +25,14 @@ metadata: - reddit - x - youtube + - hackernews - trends - prompts --- # last30days v2.1: Research Any Topic from the Last 30 Days -Research ANY topic across Reddit, X, YouTube, and the web. Surface what people are actually discussing, recommending, and debating right now. +Research ANY topic across Reddit, X, YouTube, Hacker News, and the web. Surface what people are actually discussing, recommending, and debating right now. ## CRITICAL: Parse User Intent @@ -109,10 +110,10 @@ Use a **timeout of 300000** (5 minutes) on the Bash call. The script typically t The script will automatically: - Detect available API keys -- Run Reddit/X/YouTube searches -- Output ALL results including YouTube transcripts +- Run Reddit/X/YouTube/Hacker News searches +- Output ALL results including YouTube transcripts and HN comments -**Read the ENTIRE output.** It contains THREE data sections in this order: Reddit items, X items, and YouTube items. If you miss the YouTube section, you will produce incomplete stats. +**Read the ENTIRE output.** It contains FOUR data sections in this order: Reddit items, X items, Hacker News items, and YouTube items. If you miss sections, you will produce incomplete stats. **YouTube items in the output look like:** `**{video_id}** (score:N) {channel_name} [N views, N likes]` followed by a title, URL, and optional transcript snippet. Count them and include them in your synthesis and stats block. @@ -253,7 +254,8 @@ CITATION PRIORITY (most to least preferred): 1. @handles from X — "per @handle" (these prove the tool's unique value) 2. r/subreddits from Reddit — "per r/subreddit" 3. YouTube channels — "per [channel name] on YouTube" (transcript-backed insights) -4. Web sources — ONLY when Reddit/X/YouTube don't cover that specific fact +4. HN discussions — "per HN" or "per hn/username" (developer community signal) +5. Web sources — ONLY when Reddit/X/YouTube/HN don't cover that specific fact The tool's value is surfacing what PEOPLE are saying, not what journalists wrote. When both a web article and an X post cover the same fact, cite the X post. @@ -302,6 +304,7 @@ KEY PATTERNS from the research: ✅ All agents reported back! ├─ 🟠 Reddit: {N} threads │ {N} upvotes │ {N} comments ├─ 🔵 X: {N} posts │ {N} likes │ {N} reposts +├─ 🟡 HN: {N} stories │ {N} points │ {N} comments ├─ 🔴 YouTube: {N} videos │ {N} views │ {N} with transcripts ├─ 🌐 Web: {N} pages (supplementary) └─ 🗣️ Top voices: @{handle1} ({N} likes), @{handle2} │ r/{sub1}, r/{sub2} @@ -309,6 +312,7 @@ KEY PATTERNS from the research: ``` If Reddit returned 0 threads, write: "├─ 🟠 Reddit: 0 threads (no results this cycle)" +If HN returned 0 stories, write: "├─ 🟡 HN: 0 stories (no results this cycle)" If YouTube returned 0 videos or yt-dlp is not installed, omit the YouTube line entirely. NEVER use plain text dashes (-) or pipe (|). ALWAYS use ├─ └─ │ and the emoji. @@ -471,7 +475,7 @@ After delivering a prompt, end with: ``` --- 📚 Expert in: {TOPIC} for {TARGET_TOOL} -📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} YouTube videos ({sum} views) + {n} web pages +📊 Based on: {n} Reddit threads ({sum} upvotes) + {n} X posts ({sum} likes) + {n} HN stories ({sum} points) + {n} YouTube videos ({sum} views) + {n} web pages Want another prompt? Just tell me what you're creating next. ``` @@ -483,6 +487,7 @@ Want another prompt? Just tell me what you're creating next. **What this skill does:** - Sends search queries to OpenAI's Responses API (`api.openai.com`) for Reddit discovery - Sends search queries to Twitter's GraphQL API (via browser cookie auth) or xAI's API (`api.x.ai`) for X search +- Sends search queries to Algolia HN Search API (`hn.algolia.com`) for Hacker News story and comment discovery (free, no auth) - Runs `yt-dlp` locally for YouTube search and transcript extraction (no API key, public data) - Optionally sends search queries to Brave Search API, Parallel AI API, or OpenRouter API for web search - Fetches public Reddit thread data from `reddit.com` for engagement metrics @@ -494,6 +499,7 @@ Want another prompt? Just tell me what you're creating next. - Does not share API keys between providers (OpenAI key only goes to api.openai.com, etc.) - Does not log, cache, or write API keys to output files - Does not send data to any endpoint not listed above +- Hacker News source is always available (no API key, no binary dependency) - Cannot be invoked autonomously by the agent (`disable-model-invocation: true`) **Bundled scripts:** `scripts/last30days.py` (main research engine), `scripts/lib/` (search, enrichment, rendering modules), `scripts/lib/vendor/bird-search/` (vendored X search client, MIT licensed) diff --git a/docs/plans/2026-02-24-feat-hacker-news-source-plan.md b/docs/plans/2026-02-24-feat-hacker-news-source-plan.md new file mode 100644 index 0000000..a07ee39 --- /dev/null +++ b/docs/plans/2026-02-24-feat-hacker-news-source-plan.md @@ -0,0 +1,235 @@ +--- +title: "feat: Add Hacker News as a 5th research source" +type: feat +status: completed +date: 2026-02-24 +--- + +# feat: Add Hacker News as a 5th Research Source + +## Overview + +Add Hacker News as a source to the last30days skill, using the free Algolia HN Search API (`hn.algolia.com/api/v1`). HN provides high-signal content from a technical audience — stories with high point counts and active comment threads are strong indicators of what the developer community actually cares about. No API key required. + +## Problem Statement / Motivation + +The skill currently covers Reddit, X, YouTube, and web. Hacker News is missing — and for technical topics, HN often surfaces discussions that don't appear on Reddit or X. HN's upvote system and comment culture produce high-quality signal: a 500-point story with 200 comments means the developer community is genuinely engaged. Community contributor @wkbaran proposed this in PR #26 alongside YouTube and Product Hunt. YouTube shipped in v2.1; now it's time for HN. + +## Proposed Solution + +Add `scripts/lib/hackernews.py` following the exact same pattern as `youtube_yt.py` (the simplest existing source — no API key, just HTTP calls). Use the Algolia HN Search API for discovery, then optionally fetch top comments from high-scoring stories for enrichment (like Reddit enrichment, but using the `/items/:id` endpoint instead of Reddit's JSON API). + +### Two-phase approach (matches existing Reddit pattern): + +1. **Phase 1 — Search**: Query Algolia for stories matching the topic within the date range. Get titles, URLs, points, comment counts. +2. **Phase 2 — Enrichment** (optional, top N stories): Fetch the `/items/:id` endpoint for the highest-scoring stories to get top-level comments. This gives "comment_insights" like Reddit enrichment does. + +## Technical Approach + +### Files to Create + +#### `scripts/lib/hackernews.py` + +The main source module. Pattern matches `youtube_yt.py` (simplest source). + +```python +# scripts/lib/hackernews.py +"""Hacker News search via Algolia API (free, no auth required).""" + +ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search" +ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date" +ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items" + +DEPTH_CONFIG = { + "quick": 15, + "default": 30, + "deep": 60, +} + +ENRICH_LIMITS = { + "quick": 3, + "default": 5, + "deep": 10, +} +``` + +Key functions: + +- `search_hackernews(topic, from_date, to_date, depth="default") -> Dict[str, Any]` + - Calls `hn.algolia.com/api/v1/search?query={topic}&tags=story&numericFilters=created_at_i>{from_ts},created_at_i<{to_ts}&hitsPerPage={count}` + - Uses `http.get()` — stdlib only, matches existing pattern + - Returns raw Algolia response + +- `parse_hackernews_response(response: Dict) -> List[Dict]` + - Extracts hits, maps to raw dicts with fields: `id` (prefix "HN"), `title`, `url`, `hn_url`, `author`, `date`, `engagement` (points, num_comments), `why_relevant`, `relevance` + - `relevance` estimated from Algolia rank + engagement boost (same pattern as Bill's PR) + +- `enrich_top_stories(items, depth="default") -> List[Dict]` + - Fetches `/items/{objectID}` for top N stories (by points) + - Extracts top-level comments (author, text, points) + - Adds `top_comments` and `comment_insights` fields (same structure as Reddit enrichment) + - Uses `ThreadPoolExecutor` for parallel fetching + +- `_date_to_unix(date_str: str) -> int` — Helper, converts YYYY-MM-DD to Unix timestamp + +#### `tests/test_hackernews.py` + +Standard unittest pattern matching existing tests. + +- Test `parse_hackernews_response` with sample Algolia response +- Test `_date_to_unix` conversion +- Test empty response handling +- Test enrichment parsing +- Test score integration with `score.py` + +### Files to Modify + +#### `scripts/lib/schema.py` + +- [x] Add `HackerNewsItem` dataclass: + ```python + @dataclass + class HackerNewsItem: + id: str # "HN1", "HN2", ... + title: str + url: str # Original article URL + hn_url: str # news.ycombinator.com/item?id=... + author: str # HN username + date: Optional[str] + date_confidence: str # Always "high" (Algolia provides exact timestamps) + engagement: Optional[Engagement] # points + num_comments + top_comments: List[Comment] # From enrichment + comment_insights: List[str] # From enrichment + relevance: float + why_relevant: str + subs: SubScores + score: int + ``` +- [x] Add `hackernews: List[HackerNewsItem] = field(default_factory=list)` to `Report` +- [x] Add `hackernews_error: Optional[str] = None` to `Report` +- [x] Update `Report.to_dict()` and `Report.from_dict()` + +#### `scripts/lib/normalize.py` + +- [x] Add `normalize_hackernews_items(items: List[Dict], from_date, to_date) -> List[schema.HackerNewsItem]` + - Maps raw dicts to `HackerNewsItem` dataclass instances + - Sets `date_confidence = "high"` (Algolia provides `created_at_i` exact timestamps) + - Converts `engagement` dict to `schema.Engagement(score=points, num_comments=num_comments)` + +#### `scripts/lib/score.py` + +- [x] Add `compute_hackernews_engagement_raw(engagement) -> float` + - Formula: `0.55 * log1p(points) + 0.45 * log1p(num_comments)` + - Points are the primary signal on HN; comments indicate depth of discussion +- [x] Add `score_hackernews_items(items) -> List[schema.HackerNewsItem]` + - Uses standard 45/25/30 weights (relevance/recency/engagement) — same as Reddit/X/YouTube +- [x] Update `sort_items()` to handle `HackerNewsItem` + - Source priority: Reddit > X > **HN** > YouTube > WebSearch + - HN slots between X and YouTube: higher signal than YouTube (curated upvotes vs raw views), but X has real-time pulse + +#### `scripts/lib/dedupe.py` + +- [x] Add `dedupe_hackernews(items, threshold=0.7) -> List[schema.HackerNewsItem]` +- [x] Update `get_item_text()` to handle `HackerNewsItem` (return `title`) +- [x] Consider cross-source dedup: HN stories often link to the same URLs that appear in web search results. Dedupe by URL match across `hackernews` and `websearch` items. + +#### `scripts/lib/render.py` + +- [x] Add HN section to `render_compact()`: + ``` + ### Hacker News Stories + + **HN1** (score:85) hn/username (2026-02-15) [350pts, 127cmt] + Story title here + https://news.ycombinator.com/item?id=12345 + *Why relevant* + ``` +- [x] Add to `render_source_status()`: + ``` + ✅ HN: {N} stories + ``` +- [x] Add to `render_full_report()` and `render_context_snippet()` +- [x] Update `_assess_data_freshness()` to include HN items + +#### `scripts/last30days.py` + +- [x] Import: `from lib import hackernews` +- [x] Add `_search_hackernews(topic, from_date, to_date, depth) -> (items, error)` wrapper function + - Calls `hackernews.search_hackernews()`, then `hackernews.parse_hackernews_response()` + - Returns `(items, None)` or `([], error_string)` +- [x] Add to `TIMEOUT_PROFILES`: `"hackernews_future": 60` (default), `30` (quick), `90` (deep) +- [x] Add HN to `ThreadPoolExecutor` in `run_research()`: + ```python + if do_hackernews: + hn_future = executor.submit(_search_hackernews, topic, from_date, to_date, depth) + ``` + - Increment `max_workers` by 1 when HN is enabled +- [x] Collect results: `hn_items, hn_error = hn_future.result(timeout=hn_timeout)` +- [x] Add HN enrichment phase (after Reddit enrichment, before Phase 2): + ```python + if hn_items: + hn_items = hackernews.enrich_top_stories(hn_items, depth=depth) + ``` +- [x] Add to processing pipeline: normalize -> filter_by_date_range -> score -> sort -> dedupe +- [x] Assign to `report.hackernews` and `report.hackernews_error` +- [x] Update status UI: add `⏳ 🟡 HN Searching Hacker News...` and `✓ 🟡 HN Found {N} stories` + +#### `scripts/lib/env.py` + +- [x] HN is always available (no API key, no binary dependency) +- [x] Update `get_available_sources()` to include HN in the source list +- [x] Update `validate_sources()` to accept `hn` as a valid source name +- [x] Add `hn` to the `--search` flag documentation + +#### `SKILL.md` + +- [x] Update description: "Sources: Reddit, X, YouTube, **Hacker News**, and web" +- [x] Add HN to stats block template: + ``` + ├─ 🟡 HN: {N} stories │ {N} points │ {N} comments + ``` +- [x] Update `metadata.clawdbot.tags` to include `hackernews` +- [x] Update Security section: add `hn.algolia.com` to endpoints list +- [x] Update citation priority to include HN: Reddit > X > YouTube > HN > Web + +## Acceptance Criteria + +- [x] `python3 scripts/last30days.py "AI coding agents" --emit=compact` shows HN section with stories, points, and comment counts +- [x] HN stories include `hn_url` linking to the HN discussion page (not just the article URL) +- [x] Top stories are enriched with top comments (like Reddit enrichment) +- [x] HN runs in parallel with Reddit/X/YouTube (no serial bottleneck) +- [x] Scoring uses standard 45/25/30 weights with engagement formula tuned for HN metrics +- [x] Stats block shows: `├─ 🟡 HN: {N} stories │ {N} points │ {N} comments` +- [x] `--search=hn` works to run HN only; `--search=reddit,hn` works for combos +- [x] `--quick`, `--deep` flags adjust HN result count (15/30/60) +- [x] All existing tests still pass +- [x] New `tests/test_hackernews.py` with tests for parse, normalize, score, enrichment +- [x] No API key required — works out of the box +- [x] Cross-source URL dedup: HN stories linking to same URL as web results get deduped +- [x] SKILL.md updated with HN in stats block, security section, and citation priority + +## Dependencies & Risks + +**Low risk:** +- Algolia HN API is free, public, no auth, well-established (used since 2014) +- No new dependencies — uses existing `http.py` (stdlib urllib) +- Pattern is identical to YouTube source (simplest existing source) + +**Medium risk:** +- Algolia has no officially documented rate limit, but aggressive use could get throttled + - Mitigation: Existing `http.py` exponential backoff handles 429s + - Default depth only requests 30 items (1 API call for search + N for enrichment) +- Comment enrichment adds N API calls (one per story) which could slow down quick mode + - Mitigation: Limit enrichment to top 3/5/10 stories by depth; use ThreadPoolExecutor + +**Compatibility:** +- HN always available — doesn't break anything when other sources are missing +- Existing `--search` flag needs extension but is backward-compatible + +## Sources & References + +- PR #26 by @wkbaran: [HN implementation reference](https://github.com/mvanhorn/last30days-skill/pull/26) +- Algolia HN API docs: `https://hn.algolia.com/api` +- Existing patterns: `scripts/lib/youtube_yt.py` (simplest source), `scripts/lib/openai_reddit.py` (enrichment pattern) +- Scoring reference: `scripts/lib/score.py:compute_reddit_engagement_raw()` +- Schema reference: `scripts/lib/schema.py:RedditItem` (closest analog to HN) diff --git a/scripts/last30days.py b/scripts/last30days.py index 799965a..13a64bf 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -38,9 +38,9 @@ _child_pids: set = set() _child_pids_lock = threading.Lock() TIMEOUT_PROFILES = { - "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, - "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, - "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, + "quick": {"global": 90, "future": 30, "reddit_future": 60, "youtube_future": 60, "hackernews_future": 30, "http": 15, "enrich_per": 8, "enrich_total": 30, "enrich_max_items": 10}, + "default": {"global": 180, "future": 60, "reddit_future": 90, "youtube_future": 90, "hackernews_future": 60, "http": 30, "enrich_per": 15, "enrich_total": 45, "enrich_max_items": 15}, + "deep": {"global": 300, "future": 90, "reddit_future": 120, "youtube_future": 120, "hackernews_future": 90, "http": 30, "enrich_per": 15, "enrich_total": 60, "enrich_max_items": 25}, } @@ -98,6 +98,7 @@ from lib import ( bird_x, dates, dedupe, + hackernews, entity_extract, env, http, @@ -303,6 +304,34 @@ def _search_youtube( return youtube_items, youtube_error +def _search_hackernews( + topic: str, + from_date: str, + to_date: str, + depth: str, +) -> tuple: + """Search Hacker News via Algolia (runs in thread). + + Returns: + Tuple of (hn_items, hn_error) + """ + hn_error = None + + try: + response = hackernews.search_hackernews( + topic, from_date, to_date, depth=depth, + ) + except Exception as e: + return [], f"{type(e).__name__}: {e}" + + hn_items = hackernews.parse_hackernews_response(response) + + if response.get("error"): + hn_error = response["error"] + + return hn_items, hn_error + + def _search_web( topic: str, config: dict, @@ -516,6 +545,7 @@ def run_research( reddit_items = [] x_items = [] youtube_items = [] + hackernews_items = [] web_items = [] raw_openai = None raw_xai = None @@ -523,6 +553,7 @@ def run_research( reddit_error = None x_error = None youtube_error = None + hackernews_error = None web_error = None # Determine web search mode @@ -565,18 +596,20 @@ def run_research( progress.show_error(f"YouTube error: {e}") if progress: progress.end_youtube(len(youtube_items)) - return reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error + return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error # Determine which searches to run do_reddit = sources in ("both", "reddit", "all", "reddit-web") do_x = sources in ("both", "x", "all", "x-web") + do_hackernews = True # HN is always available (no API key) - # Run Reddit, X, YouTube, and Web searches in parallel + # Run Reddit, X, YouTube, HN, and Web searches in parallel reddit_future = None x_future = None youtube_future = None + hackernews_future = None web_future = None - max_workers = 2 + (1 if run_youtube else 0) + (1 if web_backend else 0) + max_workers = 2 + (1 if run_youtube else 0) + (1 if do_hackernews else 0) + (1 if web_backend else 0) with ThreadPoolExecutor(max_workers=max_workers) as executor: # Submit searches @@ -603,6 +636,13 @@ def run_research( _search_youtube, topic, from_date, to_date, depth ) + if do_hackernews: + if progress: + progress.start_hackernews() + hackernews_future = executor.submit( + _search_hackernews, topic, from_date, to_date, depth + ) + if web_backend: sys.stderr.write(f"[web] Searching via {web_backend}\n") sys.stderr.flush() @@ -661,6 +701,23 @@ def run_research( if progress: progress.end_youtube(len(youtube_items)) + if hackernews_future: + hn_timeout = timeouts.get("hackernews_future", future_timeout) + try: + hackernews_items, hackernews_error = hackernews_future.result(timeout=hn_timeout) + if hackernews_error and progress: + progress.show_error(f"HN error: {hackernews_error}") + except TimeoutError: + hackernews_error = f"HN search timed out after {hn_timeout}s" + if progress: + progress.show_error(hackernews_error) + except Exception as e: + hackernews_error = f"{type(e).__name__}: {e}" + if progress: + progress.show_error(f"HN error: {e}") + if progress: + progress.end_hackernews(len(hackernews_items)) + if web_future: try: web_items, web_error = web_future.result(timeout=future_timeout) @@ -747,6 +804,14 @@ def run_research( if progress: progress.end_reddit_enrich() + # Enrich HN stories with comments + if hackernews_items: + try: + hackernews_items = hackernews.enrich_top_stories(hackernews_items, depth=depth) + except Exception as e: + sys.stderr.write(f"[HN] Enrichment error: {e}\n") + sys.stderr.flush() + # Phase 2: Supplemental search based on entities from Phase 1 # Skip on --quick (speed matters), mock mode, or if Reddit is rate-limiting if depth != "quick" and not mock and (reddit_items or x_items): @@ -760,7 +825,7 @@ def run_research( if sup_x: x_items.extend(sup_x) - return reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error + return reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error def main(): @@ -878,6 +943,7 @@ def main(): "bird_authenticated": x_source_status["bird_authenticated"], "bird_username": x_source_status.get("bird_username"), "youtube": has_ytdlp, + "hackernews": True, "web_search_backend": web_source, "parallel_ai": bool(config.get("PARALLEL_API_KEY")), "brave": bool(config.get("BRAVE_API_KEY")), @@ -905,6 +971,7 @@ def main(): "bird_authenticated": x_source_status["bird_authenticated"], "bird_username": x_source_status.get("bird_username"), "youtube": has_ytdlp, + "hackernews": True, "web_search_backend": web_source, } ui.show_diagnostic_banner(diag) @@ -982,7 +1049,7 @@ def main(): mode = sources # Run research - reddit_items, x_items, youtube_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, web_error = run_research( + reddit_items, x_items, youtube_items, hackernews_items, web_items, web_needed, raw_openai, raw_xai, raw_reddit_enriched, reddit_error, x_error, youtube_error, hackernews_error, web_error = run_research( args.topic, sources, config, @@ -1004,6 +1071,7 @@ def main(): normalized_reddit = normalize.normalize_reddit_items(reddit_items, from_date, to_date) normalized_x = normalize.normalize_x_items(x_items, from_date, to_date) normalized_youtube = normalize.normalize_youtube_items(youtube_items, from_date, to_date) if youtube_items else [] + normalized_hn = normalize.normalize_hackernews_items(hackernews_items, from_date, to_date) if hackernews_items else [] normalized_web = websearch.normalize_websearch_items(web_items, from_date, to_date) if web_items else [] # Hard date filter: exclude items with verified dates outside the range @@ -1014,24 +1082,28 @@ def main(): # that prefers recent videos but keeps older ones for evergreen topics. # YouTube content has a longer shelf life than tweets/posts. filtered_youtube = normalized_youtube + filtered_hn = normalize.filter_by_date_range(normalized_hn, from_date, to_date) if normalized_hn else [] filtered_web = normalize.filter_by_date_range(normalized_web, from_date, to_date) if normalized_web else [] # Score items scored_reddit = score.score_reddit_items(filtered_reddit) scored_x = score.score_x_items(filtered_x) scored_youtube = score.score_youtube_items(filtered_youtube) if filtered_youtube else [] + scored_hn = score.score_hackernews_items(filtered_hn) if filtered_hn else [] scored_web = score.score_websearch_items(filtered_web) if filtered_web else [] # Sort items sorted_reddit = score.sort_items(scored_reddit) sorted_x = score.sort_items(scored_x) sorted_youtube = score.sort_items(scored_youtube) if scored_youtube else [] + sorted_hn = score.sort_items(scored_hn) if scored_hn else [] sorted_web = score.sort_items(scored_web) if scored_web else [] # Dedupe items deduped_reddit = dedupe.dedupe_reddit(sorted_reddit) deduped_x = dedupe.dedupe_x(sorted_x) deduped_youtube = dedupe.dedupe_youtube(sorted_youtube) if sorted_youtube else [] + deduped_hn = dedupe.dedupe_hackernews(sorted_hn) if sorted_hn else [] deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else [] # Minimum result guarantee: if all Reddit results were filtered out but @@ -1055,10 +1127,12 @@ def main(): report.reddit = deduped_reddit report.x = deduped_x report.youtube = deduped_youtube + report.hackernews = deduped_hn report.web = deduped_web report.reddit_error = reddit_error report.x_error = x_error report.youtube_error = youtube_error + report.hackernews_error = hackernews_error report.web_error = web_error # Generate context snippet @@ -1071,7 +1145,7 @@ def main(): if sources == "web": progress.show_web_only_complete() else: - progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube)) + progress.show_complete(len(deduped_reddit), len(deduped_x), len(deduped_youtube), len(deduped_hn)) # Build source info for status footer source_info = {} @@ -1129,6 +1203,16 @@ def main(): "engagement_score": item.engagement.views if item.engagement and item.engagement.views else 0, "relevance_score": item.relevance, }) + for item in deduped_hn: + findings.append({ + "source": "hackernews", + "url": item.hn_url, + "title": item.title, + "author": item.author, + "content": item.title, + "engagement_score": item.engagement.score if item.engagement else 0, + "relevance_score": item.relevance, + }) for item in deduped_web: findings.append({ "source": "web", diff --git a/scripts/lib/dedupe.py b/scripts/lib/dedupe.py index 22a87c9..376edd5 100644 --- a/scripts/lib/dedupe.py +++ b/scripts/lib/dedupe.py @@ -36,10 +36,12 @@ def jaccard_similarity(set1: Set[str], set2: Set[str]) -> float: return intersection / union if union > 0 else 0.0 -def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeItem]) -> str: +def get_item_text(item: Union[schema.RedditItem, schema.XItem, schema.YouTubeItem, schema.HackerNewsItem]) -> str: """Get comparable text from an item.""" if isinstance(item, schema.RedditItem): return item.title + elif isinstance(item, schema.HackerNewsItem): + return item.title elif isinstance(item, schema.YouTubeItem): return f"{item.title} {item.channel_name}" else: @@ -128,3 +130,11 @@ def dedupe_youtube( ) -> List[schema.YouTubeItem]: """Dedupe YouTube items.""" return dedupe_items(items, threshold) + + +def dedupe_hackernews( + items: List[schema.HackerNewsItem], + threshold: float = 0.7, +) -> List[schema.HackerNewsItem]: + """Dedupe Hacker News items.""" + return dedupe_items(items, threshold) diff --git a/scripts/lib/env.py b/scripts/lib/env.py index 81d085e..2c443c4 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -246,6 +246,14 @@ def is_ytdlp_available() -> bool: return youtube_yt.is_ytdlp_installed() +def is_hackernews_available() -> bool: + """Check if Hacker News source is available. + + Always returns True - HN uses free Algolia API, no key needed. + """ + return True + + def get_x_source_status(config: Dict[str, Any]) -> Dict[str, Any]: """Get detailed X source status for UI decisions. diff --git a/scripts/lib/hackernews.py b/scripts/lib/hackernews.py new file mode 100644 index 0000000..f50bbdd --- /dev/null +++ b/scripts/lib/hackernews.py @@ -0,0 +1,252 @@ +"""Hacker News search via Algolia API (free, no auth required). + +Uses hn.algolia.com/api/v1 for story discovery and comment enrichment. +No API key needed - just HTTP calls via stdlib urllib. +""" + +import html +import math +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Dict, List, Optional + +from . import http + +ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search" +ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date" +ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items" + +DEPTH_CONFIG = { + "quick": 15, + "default": 30, + "deep": 60, +} + +ENRICH_LIMITS = { + "quick": 3, + "default": 5, + "deep": 10, +} + + +def _log(msg: str): + """Log to stderr.""" + sys.stderr.write(f"[HN] {msg}\n") + sys.stderr.flush() + + +def _date_to_unix(date_str: str) -> int: + """Convert YYYY-MM-DD to Unix timestamp (start of day UTC).""" + parts = date_str.split("-") + year, month, day = int(parts[0]), int(parts[1]), int(parts[2]) + import calendar + import datetime + dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc) + return int(dt.timestamp()) + + +def _unix_to_date(ts: int) -> str: + """Convert Unix timestamp to YYYY-MM-DD.""" + import datetime + dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc) + return dt.strftime("%Y-%m-%d") + + +def _strip_html(text: str) -> str: + """Strip HTML tags and decode entities from HN comment text.""" + import re + text = html.unescape(text) + text = re.sub(r'

', '\n', text) + text = re.sub(r'<[^>]+>', '', text) + return text.strip() + + +def search_hackernews( + topic: str, + from_date: str, + to_date: str, + depth: str = "default", +) -> Dict[str, Any]: + """Search Hacker News via Algolia API. + + Args: + topic: Search topic + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) + depth: 'quick', 'default', or 'deep' + + Returns: + Dict with Algolia response (contains 'hits' list). + """ + count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) + from_ts = _date_to_unix(from_date) + to_ts = _date_to_unix(to_date) + 86400 # Include the end date + + _log(f"Searching for '{topic}' (since {from_date}, count={count})") + + # Use relevance-sorted search (better for topic matching) + params = { + "query": topic, + "tags": "story", + "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts}", + "hitsPerPage": str(count), + } + + from urllib.parse import urlencode + url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}" + + try: + response = http.request("GET", url, timeout=30) + except http.HTTPError as e: + _log(f"Search failed: {e}") + return {"hits": [], "error": str(e)} + except Exception as e: + _log(f"Search failed: {e}") + return {"hits": [], "error": str(e)} + + hits = response.get("hits", []) + _log(f"Found {len(hits)} stories") + return response + + +def parse_hackernews_response(response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse Algolia response into normalized item dicts. + + Returns: + List of item dicts ready for normalization. + """ + hits = response.get("hits", []) + items = [] + + for i, hit in enumerate(hits): + object_id = hit.get("objectID", "") + points = hit.get("points") or 0 + num_comments = hit.get("num_comments") or 0 + created_at_i = hit.get("created_at_i") + + date_str = None + if created_at_i: + date_str = _unix_to_date(created_at_i) + + # Article URL vs HN discussion URL + article_url = hit.get("url") or "" + hn_url = f"https://news.ycombinator.com/item?id={object_id}" + + # Relevance: Algolia rank position gives a base, engagement boosts it + # Position 0 = most relevant from Algolia + rank_score = max(0.3, 1.0 - (i * 0.02)) # 1.0 -> 0.3 over 35 items + engagement_boost = min(0.2, math.log1p(points) / 40) + relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1) + + items.append({ + "object_id": object_id, + "title": hit.get("title", ""), + "url": article_url, + "hn_url": hn_url, + "author": hit.get("author", ""), + "date": date_str, + "engagement": { + "points": points, + "num_comments": num_comments, + }, + "relevance": round(relevance, 2), + "why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}", + }) + + return items + + +def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any]: + """Fetch top-level comments for a story from Algolia items endpoint. + + Args: + object_id: HN story ID + max_comments: Max comments to return + + Returns: + Dict with 'comments' list and 'comment_insights' list. + """ + url = f"{ALGOLIA_ITEM_URL}/{object_id}" + + try: + data = http.request("GET", url, timeout=15) + except Exception as e: + _log(f"Failed to fetch comments for {object_id}: {e}") + return {"comments": [], "comment_insights": []} + + children = data.get("children", []) + + # Sort by points (highest first), filter to actual comments + real_comments = [ + c for c in children + if c.get("text") and c.get("author") + ] + real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True) + + comments = [] + insights = [] + for c in real_comments[:max_comments]: + text = _strip_html(c.get("text", "")) + excerpt = text[:300] + "..." if len(text) > 300 else text + comments.append({ + "author": c.get("author", ""), + "text": excerpt, + "points": c.get("points") or 0, + }) + # First sentence as insight + first_sentence = text.split(". ")[0].split("\n")[0][:200] + if first_sentence: + insights.append(first_sentence) + + return {"comments": comments, "comment_insights": insights} + + +def enrich_top_stories( + items: List[Dict[str, Any]], + depth: str = "default", +) -> List[Dict[str, Any]]: + """Fetch comments for top N stories by points. + + Args: + items: Parsed HN items + depth: Research depth (controls how many to enrich) + + Returns: + Items with top_comments and comment_insights added. + """ + if not items: + return items + + limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"]) + + # Sort by points to enrich the most popular stories + by_points = sorted( + range(len(items)), + key=lambda i: items[i].get("engagement", {}).get("points", 0), + reverse=True, + ) + to_enrich = by_points[:limit] + + _log(f"Enriching top {len(to_enrich)} stories with comments") + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = { + executor.submit( + _fetch_item_comments, + items[idx]["object_id"], + ): idx + for idx in to_enrich + } + + for future in as_completed(futures): + idx = futures[future] + try: + result = future.result(timeout=15) + items[idx]["top_comments"] = result["comments"] + items[idx]["comment_insights"] = result["comment_insights"] + except Exception: + items[idx]["top_comments"] = [] + items[idx]["comment_insights"] = [] + + return items diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py index 3fd61a9..2f356fc 100644 --- a/scripts/lib/normalize.py +++ b/scripts/lib/normalize.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union from . import dates, schema -T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem) +T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem) def filter_by_date_range( @@ -200,6 +200,63 @@ def normalize_youtube_items( return normalized +def normalize_hackernews_items( + items: List[Dict[str, Any]], + from_date: str, + to_date: str, +) -> List[schema.HackerNewsItem]: + """Normalize raw Hacker News items to schema. + + Args: + items: Raw HN items from Algolia API + from_date: Start of date range + to_date: End of date range + + Returns: + List of HackerNewsItem objects + """ + normalized = [] + + for i, item in enumerate(items): + # Parse engagement + eng_raw = item.get("engagement") or {} + engagement = schema.Engagement( + score=eng_raw.get("points"), + num_comments=eng_raw.get("num_comments"), + ) + + # Parse comments (from enrichment) + top_comments = [] + for c in item.get("top_comments", []): + top_comments.append(schema.Comment( + score=c.get("points", 0), + date=None, + author=c.get("author", ""), + excerpt=c.get("text", ""), + url="", + )) + + # HN dates are always high confidence (exact timestamps from Algolia) + date_str = item.get("date") + + normalized.append(schema.HackerNewsItem( + id=f"HN{i+1}", + title=item.get("title", ""), + url=item.get("url", ""), + hn_url=item.get("hn_url", ""), + author=item.get("author", ""), + date=date_str, + date_confidence="high", + engagement=engagement, + top_comments=top_comments, + comment_insights=item.get("comment_insights", []), + relevance=item.get("relevance", 0.5), + why_relevant=item.get("why_relevant", ""), + )) + + return normalized + + def items_to_dicts(items: List) -> List[Dict[str, Any]]: """Convert schema items to dicts for JSON serialization.""" return [item.to_dict() for item in items] diff --git a/scripts/lib/render.py b/scripts/lib/render.py index c36750e..e51aa80 100644 --- a/scripts/lib/render.py +++ b/scripts/lib/render.py @@ -30,9 +30,10 @@ def _assess_data_freshness(report: schema.Report) -> dict: reddit_recent = sum(1 for r in report.reddit if r.date and r.date >= report.range_from) x_recent = sum(1 for x in report.x if x.date and x.date >= report.range_from) web_recent = sum(1 for w in report.web if w.date and w.date >= report.range_from) + hn_recent = sum(1 for h in report.hackernews if h.date and h.date >= report.range_from) - total_recent = reddit_recent + x_recent + web_recent - total_items = len(report.reddit) + len(report.x) + len(report.web) + total_recent = reddit_recent + x_recent + web_recent + hn_recent + total_items = len(report.reddit) + len(report.x) + len(report.web) + len(report.hackernews) return { "reddit_recent": reddit_recent, @@ -215,6 +216,42 @@ def render_compact(report: schema.Report, limit: int = 15, missing_keys: str = " lines.append(f" *{item.why_relevant}*") lines.append("") + # Hacker News items + if report.hackernews_error: + lines.append("### Hacker News Stories") + lines.append("") + lines.append(f"**ERROR:** {report.hackernews_error}") + lines.append("") + elif report.hackernews: + lines.append("### Hacker News Stories") + lines.append("") + for item in report.hackernews[:limit]: + eng_str = "" + if item.engagement: + eng = item.engagement + parts = [] + if eng.score is not None: + parts.append(f"{eng.score}pts") + if eng.num_comments is not None: + parts.append(f"{eng.num_comments}cmt") + if parts: + eng_str = f" [{', '.join(parts)}]" + + date_str = f" ({item.date})" if item.date else "" + + lines.append(f"**{item.id}** (score:{item.score}) hn/{item.author}{date_str}{eng_str}") + lines.append(f" {item.title}") + lines.append(f" {item.hn_url}") + lines.append(f" *{item.why_relevant}*") + + # Comment insights + if item.comment_insights: + lines.append(f" Insights:") + for insight in item.comment_insights[:3]: + lines.append(f" - {insight}") + + lines.append("") + # Web items (if any - populated by the assistant) if report.web_error: lines.append("### Web Results") @@ -278,6 +315,14 @@ def render_source_status(report: schema.Report, source_info: dict = None) -> str reason = source_info.get("x_skip_reason", "No Bird CLI or XAI_API_KEY") lines.append(f" ⏭️ X: skipped — {reason}") + # Hacker News + if report.hackernews_error: + lines.append(f" ❌ HN: error - {report.hackernews_error}") + elif report.hackernews: + lines.append(f" ✅ HN: {len(report.hackernews)} stories") + else: + lines.append(" ⏭️ HN: 0 stories found") + # YouTube if report.youtube_error: lines.append(f" ❌ YouTube: error — {report.youtube_error}") @@ -325,6 +370,8 @@ def render_context_snippet(report: schema.Report) -> str: all_items.append((item.score, "Reddit", item.title, item.url)) for item in report.x[:5]: all_items.append((item.score, "X", item.text[:50] + "...", item.url)) + for item in report.hackernews[:5]: + all_items.append((item.score, "HN", item.title[:50] + "...", item.hn_url)) for item in report.web[:5]: all_items.append((item.score, "Web", item.title[:50] + "...", item.url)) @@ -414,6 +461,33 @@ def render_full_report(report: schema.Report) -> str: lines.append(f"> {item.text}") lines.append("") + # HN section + if report.hackernews: + lines.append("## Hacker News Stories") + lines.append("") + for item in report.hackernews: + lines.append(f"### {item.id}: {item.title}") + lines.append("") + lines.append(f"- **Author:** {item.author}") + lines.append(f"- **HN URL:** {item.hn_url}") + if item.url: + lines.append(f"- **Article URL:** {item.url}") + lines.append(f"- **Date:** {item.date or 'Unknown'}") + lines.append(f"- **Score:** {item.score}/100") + lines.append(f"- **Relevance:** {item.why_relevant}") + + if item.engagement: + eng = item.engagement + lines.append(f"- **Engagement:** {eng.score or '?'} points, {eng.num_comments or '?'} comments") + + if item.comment_insights: + lines.append("") + lines.append("**Key Insights from Comments:**") + for insight in item.comment_insights: + lines.append(f"- {insight}") + + lines.append("") + # Web section if report.web: lines.append("## Web Results") diff --git a/scripts/lib/schema.py b/scripts/lib/schema.py index da1534f..c93f115 100644 --- a/scripts/lib/schema.py +++ b/scripts/lib/schema.py @@ -207,6 +207,43 @@ class YouTubeItem: } +@dataclass +class HackerNewsItem: + """Normalized Hacker News item.""" + id: str # "HN1", "HN2", ... + title: str + url: str # Original article URL + hn_url: str # news.ycombinator.com/item?id=... + author: str # HN username + date: Optional[str] = None + date_confidence: str = "high" # Algolia provides exact timestamps + engagement: Optional[Engagement] = None # points + num_comments + top_comments: List[Comment] = field(default_factory=list) + comment_insights: List[str] = field(default_factory=list) + relevance: float = 0.5 + why_relevant: str = "" + subs: SubScores = field(default_factory=SubScores) + score: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + 'id': self.id, + 'title': self.title, + 'url': self.url, + 'hn_url': self.hn_url, + 'author': self.author, + 'date': self.date, + 'date_confidence': self.date_confidence, + 'engagement': self.engagement.to_dict() if self.engagement else None, + 'top_comments': [c.to_dict() for c in self.top_comments], + 'comment_insights': self.comment_insights, + 'relevance': self.relevance, + 'why_relevant': self.why_relevant, + 'subs': self.subs.to_dict(), + 'score': self.score, + } + + @dataclass class Report: """Full research report.""" @@ -221,6 +258,7 @@ class Report: x: List[XItem] = field(default_factory=list) web: List[WebSearchItem] = field(default_factory=list) youtube: List[YouTubeItem] = field(default_factory=list) + hackernews: List[HackerNewsItem] = field(default_factory=list) best_practices: List[str] = field(default_factory=list) prompt_pack: List[str] = field(default_factory=list) context_snippet_md: str = "" @@ -229,6 +267,7 @@ class Report: x_error: Optional[str] = None web_error: Optional[str] = None youtube_error: Optional[str] = None + hackernews_error: Optional[str] = None # Cache info from_cache: bool = False cache_age_hours: Optional[float] = None @@ -248,6 +287,7 @@ class Report: 'x': [x.to_dict() for x in self.x], 'web': [w.to_dict() for w in self.web], 'youtube': [y.to_dict() for y in self.youtube], + 'hackernews': [h.to_dict() for h in self.hackernews], 'best_practices': self.best_practices, 'prompt_pack': self.prompt_pack, 'context_snippet_md': self.context_snippet_md, @@ -260,6 +300,8 @@ class Report: d['web_error'] = self.web_error if self.youtube_error: d['youtube_error'] = self.youtube_error + if self.hackernews_error: + d['hackernews_error'] = self.hackernews_error if self.from_cache: d['from_cache'] = self.from_cache if self.cache_age_hours is not None: @@ -359,6 +401,31 @@ class Report: score=y.get('score', 0), )) + # Reconstruct HackerNews items + hn_items = [] + for h in data.get('hackernews', []): + eng = None + if h.get('engagement'): + eng = Engagement(**h['engagement']) + comments = [Comment(**c) for c in h.get('top_comments', [])] + subs = SubScores(**h.get('subs', {})) if h.get('subs') else SubScores() + hn_items.append(HackerNewsItem( + id=h['id'], + title=h['title'], + url=h.get('url', ''), + hn_url=h.get('hn_url', ''), + author=h.get('author', ''), + date=h.get('date'), + date_confidence=h.get('date_confidence', 'high'), + engagement=eng, + top_comments=comments, + comment_insights=h.get('comment_insights', []), + relevance=h.get('relevance', 0.5), + why_relevant=h.get('why_relevant', ''), + subs=subs, + score=h.get('score', 0), + )) + return cls( topic=data['topic'], range_from=range_from, @@ -371,6 +438,7 @@ class Report: x=x_items, web=web_items, youtube=youtube_items, + hackernews=hn_items, best_practices=data.get('best_practices', []), prompt_pack=data.get('prompt_pack', []), context_snippet_md=data.get('context_snippet_md', ''), @@ -378,6 +446,7 @@ class Report: x_error=data.get('x_error'), web_error=data.get('web_error'), youtube_error=data.get('youtube_error'), + hackernews_error=data.get('hackernews_error'), from_cache=data.get('from_cache', False), cache_age_hours=data.get('cache_age_hours'), ) diff --git a/scripts/lib/score.py b/scripts/lib/score.py index 3fca23f..a497b26 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -280,6 +280,64 @@ def score_youtube_items(items: List[schema.YouTubeItem]) -> List[schema.YouTubeI return items +def compute_hackernews_engagement_raw(engagement: Optional[schema.Engagement]) -> Optional[float]: + """Compute raw engagement score for Hacker News item. + + Formula: 0.55*log1p(points) + 0.45*log1p(num_comments) + Points are the primary signal on HN; comments indicate depth of discussion. + """ + if engagement is None: + return None + + if engagement.score is None and engagement.num_comments is None: + return None + + points = log1p_safe(engagement.score) + comments = log1p_safe(engagement.num_comments) + + return 0.55 * points + 0.45 * comments + + +def score_hackernews_items(items: List[schema.HackerNewsItem]) -> List[schema.HackerNewsItem]: + """Compute scores for Hacker News items. + + Uses same weight structure as Reddit/X (relevance + recency + engagement). + """ + if not items: + return items + + eng_raw = [compute_hackernews_engagement_raw(item.engagement) for item in items] + eng_normalized = normalize_to_100(eng_raw) + + for i, item in enumerate(items): + rel_score = int(item.relevance * 100) + rec_score = dates.recency_score(item.date) + + if eng_normalized[i] is not None: + eng_score = int(eng_normalized[i]) + else: + eng_score = DEFAULT_ENGAGEMENT + + item.subs = schema.SubScores( + relevance=rel_score, + recency=rec_score, + engagement=eng_score, + ) + + overall = ( + WEIGHT_RELEVANCE * rel_score + + WEIGHT_RECENCY * rec_score + + WEIGHT_ENGAGEMENT * eng_score + ) + + if eng_raw[i] is None: + overall -= UNKNOWN_ENGAGEMENT_PENALTY + + item.score = max(0, min(100, int(overall))) + + return items + + def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebSearchItem]: """Compute scores for WebSearch items WITHOUT engagement metrics. @@ -337,7 +395,7 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS return items -def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem]]) -> List: +def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem]]) -> List: """Sort items by score (descending), then date, then source priority. Args: @@ -354,15 +412,17 @@ def sort_items(items: List[Union[schema.RedditItem, schema.XItem, schema.WebSear date = item.date or "0000-00-00" date_key = -int(date.replace("-", "")) - # Tertiary: source priority (Reddit > X > YouTube > WebSearch) + # Tertiary: source priority (Reddit > X > HN > YouTube > WebSearch) if isinstance(item, schema.RedditItem): source_priority = 0 elif isinstance(item, schema.XItem): source_priority = 1 - elif isinstance(item, schema.YouTubeItem): + elif isinstance(item, schema.HackerNewsItem): source_priority = 2 - else: # WebSearchItem + elif isinstance(item, schema.YouTubeItem): source_priority = 3 + else: # WebSearchItem + source_priority = 4 # Quaternary: title/text for stability text = getattr(item, "title", "") or getattr(item, "text", "") diff --git a/scripts/lib/ui.py b/scripts/lib/ui.py index 5b95c84..6382757 100644 --- a/scripts/lib/ui.py +++ b/scripts/lib/ui.py @@ -72,6 +72,13 @@ YOUTUBE_MESSAGES = [ "Fetching transcripts...", ] +HN_MESSAGES = [ + "Searching Hacker News...", + "Scanning HN front page stories...", + "Finding technical discussions...", + "Discovering developer conversations...", +] + PROCESSING_MESSAGES = [ "Crunching the data...", "Scoring and ranking...", @@ -257,6 +264,15 @@ class ProgressDisplay: if self.spinner: self.spinner.stop(f"{Colors.RED}YouTube{Colors.RESET} Found {count} videos") + def start_hackernews(self): + msg = random.choice(HN_MESSAGES) + self.spinner = Spinner(f"{Colors.YELLOW}HN{Colors.RESET} {msg}", Colors.YELLOW) + self.spinner.start() + + def end_hackernews(self, count: int): + if self.spinner: + self.spinner.stop(f"{Colors.YELLOW}HN{Colors.RESET} Found {count} stories") + def start_processing(self): msg = random.choice(PROCESSING_MESSAGES) self.spinner = Spinner(f"{Colors.PURPLE}Processing{Colors.RESET} {msg}", Colors.PURPLE) @@ -266,18 +282,22 @@ class ProgressDisplay: if self.spinner: self.spinner.stop() - def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0): + def show_complete(self, reddit_count: int, x_count: int, youtube_count: int = 0, hn_count: int = 0): elapsed = time.time() - self.start_time if IS_TTY: sys.stderr.write(f"\n{Colors.GREEN}{Colors.BOLD}✓ Research complete{Colors.RESET} ") sys.stderr.write(f"{Colors.DIM}({elapsed:.1f}s){Colors.RESET}\n") sys.stderr.write(f" {Colors.YELLOW}Reddit:{Colors.RESET} {reddit_count} threads ") sys.stderr.write(f"{Colors.CYAN}X:{Colors.RESET} {x_count} posts") + if hn_count: + sys.stderr.write(f" {Colors.YELLOW}HN:{Colors.RESET} {hn_count} stories") if youtube_count: sys.stderr.write(f" {Colors.RED}YouTube:{Colors.RESET} {youtube_count} videos") sys.stderr.write("\n\n") else: parts = [f"Reddit: {reddit_count} threads", f"X: {x_count} posts"] + if hn_count: + parts.append(f"HN: {hn_count} stories") if youtube_count: parts.append(f"YouTube: {youtube_count} videos") sys.stderr.write(f"✓ Research complete ({elapsed:.1f}s) - {', '.join(parts)}\n") diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py new file mode 100644 index 0000000..9272e6b --- /dev/null +++ b/tests/test_hackernews.py @@ -0,0 +1,219 @@ +"""Tests for Hacker News source module.""" + +import sys +import unittest +from pathlib import Path + +# Add lib to path +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from lib import hackernews, normalize, schema, score + + +class TestDateToUnix(unittest.TestCase): + def test_known_date(self): + # 2026-01-01 00:00:00 UTC + result = hackernews._date_to_unix("2026-01-01") + self.assertIsInstance(result, int) + self.assertGreater(result, 0) + + def test_roundtrip(self): + ts = hackernews._date_to_unix("2026-02-15") + back = hackernews._unix_to_date(ts) + self.assertEqual(back, "2026-02-15") + + +class TestStripHtml(unittest.TestCase): + def test_basic_html(self): + result = hackernews._strip_html("

Hello world

") + self.assertIn("Hello", result) + self.assertIn("world", result) + self.assertNotIn("<", result) + + def test_html_entities(self): + result = hackernews._strip_html("& test") + self.assertIn("&", result) + self.assertIn("test", result) + + def test_empty(self): + result = hackernews._strip_html("") + self.assertEqual(result, "") + + +class TestParseHackernewsResponse(unittest.TestCase): + SAMPLE_RESPONSE = { + "hits": [ + { + "objectID": "12345", + "title": "Show HN: A new AI coding assistant", + "url": "https://example.com/article", + "author": "pg", + "points": 350, + "num_comments": 127, + "created_at_i": 1739836800, # 2025-02-18 + }, + { + "objectID": "12346", + "title": "Ask HN: Best practices for LLM apps?", + "url": "", + "author": "dang", + "points": 80, + "num_comments": 45, + "created_at_i": 1739750400, # 2025-02-17 + }, + ], + } + + def test_parses_hits(self): + items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE) + self.assertEqual(len(items), 2) + + def test_item_fields(self): + items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE) + item = items[0] + self.assertEqual(item["object_id"], "12345") + self.assertEqual(item["title"], "Show HN: A new AI coding assistant") + self.assertEqual(item["url"], "https://example.com/article") + self.assertEqual(item["hn_url"], "https://news.ycombinator.com/item?id=12345") + self.assertEqual(item["author"], "pg") + self.assertEqual(item["engagement"]["points"], 350) + self.assertEqual(item["engagement"]["num_comments"], 127) + + def test_date_conversion(self): + items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE) + self.assertIsNotNone(items[0]["date"]) + # Should be a valid YYYY-MM-DD string + self.assertRegex(items[0]["date"], r"^\d{4}-\d{2}-\d{2}$") + + def test_hn_url_for_askhn(self): + """Ask HN posts have no article URL, but should have hn_url.""" + items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE) + ask_hn = items[1] + self.assertEqual(ask_hn["url"], "") + self.assertIn("news.ycombinator.com", ask_hn["hn_url"]) + + def test_relevance_range(self): + items = hackernews.parse_hackernews_response(self.SAMPLE_RESPONSE) + for item in items: + self.assertGreaterEqual(item["relevance"], 0.0) + self.assertLessEqual(item["relevance"], 1.0) + + def test_empty_response(self): + items = hackernews.parse_hackernews_response({"hits": []}) + self.assertEqual(items, []) + + def test_missing_hits(self): + items = hackernews.parse_hackernews_response({}) + self.assertEqual(items, []) + + +class TestNormalizeHackernewsItems(unittest.TestCase): + def test_normalize(self): + raw_items = [ + { + "object_id": "99999", + "title": "Test Story", + "url": "https://example.com", + "hn_url": "https://news.ycombinator.com/item?id=99999", + "author": "testuser", + "date": "2026-02-15", + "engagement": {"points": 100, "num_comments": 50}, + "relevance": 0.8, + "why_relevant": "Test", + } + ] + result = normalize.normalize_hackernews_items(raw_items, "2026-01-01", "2026-03-01") + self.assertEqual(len(result), 1) + self.assertIsInstance(result[0], schema.HackerNewsItem) + self.assertEqual(result[0].id, "HN1") + self.assertEqual(result[0].title, "Test Story") + self.assertEqual(result[0].date_confidence, "high") + self.assertEqual(result[0].engagement.score, 100) + self.assertEqual(result[0].engagement.num_comments, 50) + + def test_normalize_with_comments(self): + raw_items = [ + { + "object_id": "99999", + "title": "Test", + "url": "", + "hn_url": "", + "author": "user", + "date": "2026-02-15", + "engagement": {"points": 10, "num_comments": 5}, + "relevance": 0.5, + "why_relevant": "Test", + "top_comments": [ + {"author": "commenter", "text": "Great post!", "points": 5}, + ], + "comment_insights": ["Great post!"], + } + ] + result = normalize.normalize_hackernews_items(raw_items, "2026-01-01", "2026-03-01") + self.assertEqual(len(result[0].top_comments), 1) + self.assertEqual(result[0].top_comments[0].author, "commenter") + self.assertEqual(len(result[0].comment_insights), 1) + + +class TestScoreHackernewsItems(unittest.TestCase): + def test_score_items(self): + items = [ + schema.HackerNewsItem( + id="HN1", title="High engagement", url="", hn_url="", + author="user1", date="2026-02-20", + engagement=schema.Engagement(score=500, num_comments=200), + relevance=0.9, + ), + schema.HackerNewsItem( + id="HN2", title="Low engagement", url="", hn_url="", + author="user2", date="2026-02-18", + engagement=schema.Engagement(score=10, num_comments=3), + relevance=0.5, + ), + ] + scored = score.score_hackernews_items(items) + self.assertEqual(len(scored), 2) + # High engagement + high relevance should score higher + self.assertGreater(scored[0].score, scored[1].score) + + def test_score_empty(self): + result = score.score_hackernews_items([]) + self.assertEqual(result, []) + + def test_engagement_formula(self): + eng = schema.Engagement(score=100, num_comments=50) + result = score.compute_hackernews_engagement_raw(eng) + self.assertIsNotNone(result) + self.assertGreater(result, 0) + + def test_engagement_none(self): + result = score.compute_hackernews_engagement_raw(None) + self.assertIsNone(result) + + def test_engagement_empty(self): + eng = schema.Engagement() + result = score.compute_hackernews_engagement_raw(eng) + self.assertIsNone(result) + + +class TestSortItemsWithHN(unittest.TestCase): + def test_hn_priority_between_x_and_youtube(self): + """HN should sort between X and YouTube at same score.""" + x_item = schema.XItem(id="X1", text="test", url="", author_handle="user") + x_item.score = 50 + + hn_item = schema.HackerNewsItem(id="HN1", title="test", url="", hn_url="", author="user") + hn_item.score = 50 + + yt_item = schema.YouTubeItem(id="YT1", title="test", url="", channel_name="ch") + yt_item.score = 50 + + sorted_items = score.sort_items([yt_item, hn_item, x_item]) + # Same score, so sorted by source priority: X > HN > YouTube + self.assertIsInstance(sorted_items[0], schema.XItem) + self.assertIsInstance(sorted_items[1], schema.HackerNewsItem) + self.assertIsInstance(sorted_items[2], schema.YouTubeItem) + + +if __name__ == "__main__": + unittest.main()