feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources

v3 rewrites the search engine from the ground up:

- Intelligent pre-research: resolves X handles, GitHub repos, subreddits,
  TikTok hashtags, and YouTube channels before searching
- GitHub person-mode: PR velocity, top repos by stars, release notes
- GitHub project-mode: live star counts, README, releases, top issues
- ELI5 mode: plain language synthesis, no jargon
- 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket,
  GitHub, Threads, Pinterest, Perplexity, Bluesky, Web
- Free Reddit comments via public JSON (no API key needed)
- Fun judge v2: humor scoring baked into narrative
- Cookie consent before browser scanning
- 10,000 free ScrapeCreators calls
- 1,012 tests

Thank you to the community contributors whose issues and PRs shaped v3:
@uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116),
@thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78),
@zl190 (#115), @hnshah (#84, #85, #86)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-04-08 10:52:23 -07:00
parent 61904b31e3
commit 0a9ff16dfc
397 changed files with 21427 additions and 53106 deletions
+47 -12
View File
@@ -4,6 +4,7 @@ Uses hn.algolia.com/api/v1 for story discovery and comment enrichment.
No API key needed - just HTTP calls via stdlib urllib.
"""
import datetime
import html
import math
import sys
@@ -11,10 +12,15 @@ import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Dict, List, Optional
from . import http
import re
from . import http, log
from .query import extract_core_subject
from .relevance import token_overlap_relevance
# Common HN prefixes that can cause false-positive keyword matches
_HN_PREFIXES = re.compile(r"^(Tell HN|Show HN|Ask HN|Launch HN)\s*:\s*", re.IGNORECASE)
ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
@@ -33,25 +39,19 @@ ENRICH_LIMITS = {
def _log(msg: str):
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
if sys.stderr.isatty():
sys.stderr.write(f"[HN] {msg}\n")
sys.stderr.flush()
log.source_log("HN", msg)
def _date_to_unix(date_str: str) -> int:
"""Convert YYYY-MM-DD to Unix timestamp (start of day UTC)."""
parts = date_str.split("-")
year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
import calendar
import datetime
dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc)
return int(dt.timestamp())
def _unix_to_date(ts: int) -> str:
"""Convert Unix timestamp to YYYY-MM-DD."""
import datetime
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
return dt.strftime("%Y-%m-%d")
@@ -117,6 +117,30 @@ def search_hackernews(
return response
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
"""Check if the query term appears in the title content, not just an HN prefix or author.
Returns True if the query (or any multi-word token) appears in the title
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
and ignoring the author name. Returns True when query is empty (no filter).
"""
if not query:
return True
stripped = _HN_PREFIXES.sub("", title).strip()
# Also check that the match isn't solely in the author's username
check_text = stripped.lower()
query_lower = query.lower()
# Check each word of the query independently; all must appear somewhere
# in the stripped title (not just the prefix).
query_words = query_lower.split()
for word in query_words:
if word in check_text:
continue
# Word not found in stripped title — reject
return False
return True
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
"""Parse Algolia response into normalized item dicts.
@@ -128,6 +152,16 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
List of item dicts ready for normalization.
"""
hits = response.get("hits", [])
# Post-filter: remove items where query only matched an HN prefix like "Tell HN:"
if query:
before = len(hits)
hits = [
h for h in hits
if _title_matches_query(h.get("title", ""), query, h.get("author", ""))
]
dropped = before - len(hits)
if dropped:
_log(f"Prefix filter removed {dropped}/{before} false-positive hits for '{query}'")
items = []
for i, hit in enumerate(hits):
@@ -154,7 +188,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
items.append({
"object_id": object_id,
"id": object_id,
"title": hit.get("title", ""),
"url": article_url,
"hn_url": hn_url,
@@ -162,7 +196,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
"date": date_str,
"engagement": {
"points": points,
"num_comments": num_comments,
"comments": num_comments,
},
"relevance": round(relevance, 2),
"why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}",
@@ -248,7 +282,7 @@ def enrich_top_stories(
futures = {
executor.submit(
_fetch_item_comments,
items[idx]["object_id"],
items[idx]["id"],
): idx
for idx in to_enrich
}
@@ -259,7 +293,8 @@ def enrich_top_stories(
result = future.result(timeout=15)
items[idx]["top_comments"] = result["comments"]
items[idx]["comment_insights"] = result["comment_insights"]
except Exception:
except (KeyError, TypeError, OSError) as exc:
_log(f"Comment enrichment failed for story {items[idx].get('id', '?')}: {type(exc).__name__}: {exc}")
items[idx]["top_comments"] = []
items[idx]["comment_insights"] = []