feat: v3.0.0 - intelligent search, GitHub person/project mode, ELI5, 13+ sources
v3 rewrites the search engine from the ground up: - Intelligent pre-research: resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching - GitHub person-mode: PR velocity, top repos by stars, release notes - GitHub project-mode: live star counts, README, releases, top issues - ELI5 mode: plain language synthesis, no jargon - 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, Threads, Pinterest, Perplexity, Bluesky, Web - Free Reddit comments via public JSON (no API key needed) - Fun judge v2: humor scoring baked into narrative - Cookie consent before browser scanning - 10,000 free ScrapeCreators calls - 1,012 tests Thank you to the community contributors whose issues and PRs shaped v3: @uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116), @thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78), @zl190 (#115), @hnshah (#84, #85, #86) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+47
-12
@@ -4,6 +4,7 @@ Uses hn.algolia.com/api/v1 for story discovery and comment enrichment.
|
||||
No API key needed - just HTTP calls via stdlib urllib.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import html
|
||||
import math
|
||||
import sys
|
||||
@@ -11,10 +12,15 @@ import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import http
|
||||
import re
|
||||
|
||||
from . import http, log
|
||||
from .query import extract_core_subject
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
# Common HN prefixes that can cause false-positive keyword matches
|
||||
_HN_PREFIXES = re.compile(r"^(Tell HN|Show HN|Ask HN|Launch HN)\s*:\s*", re.IGNORECASE)
|
||||
|
||||
ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
|
||||
ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
|
||||
ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
|
||||
@@ -33,25 +39,19 @@ ENRICH_LIMITS = {
|
||||
|
||||
|
||||
def _log(msg: str):
|
||||
"""Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
|
||||
if sys.stderr.isatty():
|
||||
sys.stderr.write(f"[HN] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
log.source_log("HN", msg)
|
||||
|
||||
|
||||
def _date_to_unix(date_str: str) -> int:
|
||||
"""Convert YYYY-MM-DD to Unix timestamp (start of day UTC)."""
|
||||
parts = date_str.split("-")
|
||||
year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
|
||||
import calendar
|
||||
import datetime
|
||||
dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc)
|
||||
return int(dt.timestamp())
|
||||
|
||||
|
||||
def _unix_to_date(ts: int) -> str:
|
||||
"""Convert Unix timestamp to YYYY-MM-DD."""
|
||||
import datetime
|
||||
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
|
||||
@@ -117,6 +117,30 @@ def search_hackernews(
|
||||
return response
|
||||
|
||||
|
||||
def _title_matches_query(title: str, query: str, author: str = "") -> bool:
|
||||
"""Check if the query term appears in the title content, not just an HN prefix or author.
|
||||
|
||||
Returns True if the query (or any multi-word token) appears in the title
|
||||
after stripping "Tell HN:", "Show HN:", "Ask HN:", "Launch HN:" prefixes
|
||||
and ignoring the author name. Returns True when query is empty (no filter).
|
||||
"""
|
||||
if not query:
|
||||
return True
|
||||
stripped = _HN_PREFIXES.sub("", title).strip()
|
||||
# Also check that the match isn't solely in the author's username
|
||||
check_text = stripped.lower()
|
||||
query_lower = query.lower()
|
||||
# Check each word of the query independently; all must appear somewhere
|
||||
# in the stripped title (not just the prefix).
|
||||
query_words = query_lower.split()
|
||||
for word in query_words:
|
||||
if word in check_text:
|
||||
continue
|
||||
# Word not found in stripped title — reject
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
|
||||
"""Parse Algolia response into normalized item dicts.
|
||||
|
||||
@@ -128,6 +152,16 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
|
||||
List of item dicts ready for normalization.
|
||||
"""
|
||||
hits = response.get("hits", [])
|
||||
# Post-filter: remove items where query only matched an HN prefix like "Tell HN:"
|
||||
if query:
|
||||
before = len(hits)
|
||||
hits = [
|
||||
h for h in hits
|
||||
if _title_matches_query(h.get("title", ""), query, h.get("author", ""))
|
||||
]
|
||||
dropped = before - len(hits)
|
||||
if dropped:
|
||||
_log(f"Prefix filter removed {dropped}/{before} false-positive hits for '{query}'")
|
||||
items = []
|
||||
|
||||
for i, hit in enumerate(hits):
|
||||
@@ -154,7 +188,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
|
||||
relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
|
||||
|
||||
items.append({
|
||||
"object_id": object_id,
|
||||
"id": object_id,
|
||||
"title": hit.get("title", ""),
|
||||
"url": article_url,
|
||||
"hn_url": hn_url,
|
||||
@@ -162,7 +196,7 @@ def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List
|
||||
"date": date_str,
|
||||
"engagement": {
|
||||
"points": points,
|
||||
"num_comments": num_comments,
|
||||
"comments": num_comments,
|
||||
},
|
||||
"relevance": round(relevance, 2),
|
||||
"why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}",
|
||||
@@ -248,7 +282,7 @@ def enrich_top_stories(
|
||||
futures = {
|
||||
executor.submit(
|
||||
_fetch_item_comments,
|
||||
items[idx]["object_id"],
|
||||
items[idx]["id"],
|
||||
): idx
|
||||
for idx in to_enrich
|
||||
}
|
||||
@@ -259,7 +293,8 @@ def enrich_top_stories(
|
||||
result = future.result(timeout=15)
|
||||
items[idx]["top_comments"] = result["comments"]
|
||||
items[idx]["comment_insights"] = result["comment_insights"]
|
||||
except Exception:
|
||||
except (KeyError, TypeError, OSError) as exc:
|
||||
_log(f"Comment enrichment failed for story {items[idx].get('id', '?')}: {type(exc).__name__}: {exc}")
|
||||
items[idx]["top_comments"] = []
|
||||
items[idx]["comment_insights"] = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user