feat(hackernews): add Hacker News as 5th research source
Add HN search via free Algolia API (no key needed). Two-phase approach: search for stories, then enrich top ones with comments. Integrated into the full pipeline (normalize, score, dedupe, render) running in parallel with Reddit/X/YouTube. Source priority: Reddit > X > HN > YouTube > Web. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, TypeVar, Union
|
||||
|
||||
from . import dates, schema
|
||||
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem)
|
||||
T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem, schema.YouTubeItem, schema.HackerNewsItem)
|
||||
|
||||
|
||||
def filter_by_date_range(
|
||||
@@ -200,6 +200,63 @@ def normalize_youtube_items(
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_hackernews_items(
|
||||
items: List[Dict[str, Any]],
|
||||
from_date: str,
|
||||
to_date: str,
|
||||
) -> List[schema.HackerNewsItem]:
|
||||
"""Normalize raw Hacker News items to schema.
|
||||
|
||||
Args:
|
||||
items: Raw HN items from Algolia API
|
||||
from_date: Start of date range
|
||||
to_date: End of date range
|
||||
|
||||
Returns:
|
||||
List of HackerNewsItem objects
|
||||
"""
|
||||
normalized = []
|
||||
|
||||
for i, item in enumerate(items):
|
||||
# Parse engagement
|
||||
eng_raw = item.get("engagement") or {}
|
||||
engagement = schema.Engagement(
|
||||
score=eng_raw.get("points"),
|
||||
num_comments=eng_raw.get("num_comments"),
|
||||
)
|
||||
|
||||
# Parse comments (from enrichment)
|
||||
top_comments = []
|
||||
for c in item.get("top_comments", []):
|
||||
top_comments.append(schema.Comment(
|
||||
score=c.get("points", 0),
|
||||
date=None,
|
||||
author=c.get("author", ""),
|
||||
excerpt=c.get("text", ""),
|
||||
url="",
|
||||
))
|
||||
|
||||
# HN dates are always high confidence (exact timestamps from Algolia)
|
||||
date_str = item.get("date")
|
||||
|
||||
normalized.append(schema.HackerNewsItem(
|
||||
id=f"HN{i+1}",
|
||||
title=item.get("title", ""),
|
||||
url=item.get("url", ""),
|
||||
hn_url=item.get("hn_url", ""),
|
||||
author=item.get("author", ""),
|
||||
date=date_str,
|
||||
date_confidence="high",
|
||||
engagement=engagement,
|
||||
top_comments=top_comments,
|
||||
comment_insights=item.get("comment_insights", []),
|
||||
relevance=item.get("relevance", 0.5),
|
||||
why_relevant=item.get("why_relevant", ""),
|
||||
))
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
|
||||
"""Convert schema items to dicts for JSON serialization."""
|
||||
return [item.to_dict() for item in items]
|
||||
|
||||
Reference in New Issue
Block a user