Matt Van Horn
5ca4829be4
Research topics across Reddit + X from the last 30 days using OpenAI and xAI APIs. Features: - Auto model selection (GPT-5.x, Grok-3) - Popularity-aware scoring (relevance + recency + engagement) - Reddit thread enrichment with real metrics - Near-duplicate detection - Multiple emit modes (compact, json, context, path) - 24h caching with --refresh bypass - NUX for API key setup - 87 passing unit tests Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
119 lines
3.4 KiB
Python
119 lines
3.4 KiB
Python
"""Normalization of raw API data to canonical schema."""
|
|
|
|
from typing import Any, Dict, List
|
|
|
|
from . import dates, schema
|
|
|
|
|
|
def normalize_reddit_items(
|
|
items: List[Dict[str, Any]],
|
|
from_date: str,
|
|
to_date: str,
|
|
) -> List[schema.RedditItem]:
|
|
"""Normalize raw Reddit items to schema.
|
|
|
|
Args:
|
|
items: Raw Reddit items from API
|
|
from_date: Start of date range
|
|
to_date: End of date range
|
|
|
|
Returns:
|
|
List of RedditItem objects
|
|
"""
|
|
normalized = []
|
|
|
|
for item in items:
|
|
# Parse engagement
|
|
engagement = None
|
|
eng_raw = item.get("engagement")
|
|
if isinstance(eng_raw, dict):
|
|
engagement = schema.Engagement(
|
|
score=eng_raw.get("score"),
|
|
num_comments=eng_raw.get("num_comments"),
|
|
upvote_ratio=eng_raw.get("upvote_ratio"),
|
|
)
|
|
|
|
# Parse comments
|
|
top_comments = []
|
|
for c in item.get("top_comments", []):
|
|
top_comments.append(schema.Comment(
|
|
score=c.get("score", 0),
|
|
date=c.get("date"),
|
|
author=c.get("author", ""),
|
|
excerpt=c.get("excerpt", ""),
|
|
url=c.get("url", ""),
|
|
))
|
|
|
|
# Determine date confidence
|
|
date_str = item.get("date")
|
|
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
|
|
|
|
normalized.append(schema.RedditItem(
|
|
id=item.get("id", ""),
|
|
title=item.get("title", ""),
|
|
url=item.get("url", ""),
|
|
subreddit=item.get("subreddit", ""),
|
|
date=date_str,
|
|
date_confidence=date_confidence,
|
|
engagement=engagement,
|
|
top_comments=top_comments,
|
|
comment_insights=item.get("comment_insights", []),
|
|
relevance=item.get("relevance", 0.5),
|
|
why_relevant=item.get("why_relevant", ""),
|
|
))
|
|
|
|
return normalized
|
|
|
|
|
|
def normalize_x_items(
|
|
items: List[Dict[str, Any]],
|
|
from_date: str,
|
|
to_date: str,
|
|
) -> List[schema.XItem]:
|
|
"""Normalize raw X items to schema.
|
|
|
|
Args:
|
|
items: Raw X items from API
|
|
from_date: Start of date range
|
|
to_date: End of date range
|
|
|
|
Returns:
|
|
List of XItem objects
|
|
"""
|
|
normalized = []
|
|
|
|
for item in items:
|
|
# Parse engagement
|
|
engagement = None
|
|
eng_raw = item.get("engagement")
|
|
if isinstance(eng_raw, dict):
|
|
engagement = schema.Engagement(
|
|
likes=eng_raw.get("likes"),
|
|
reposts=eng_raw.get("reposts"),
|
|
replies=eng_raw.get("replies"),
|
|
quotes=eng_raw.get("quotes"),
|
|
)
|
|
|
|
# Determine date confidence
|
|
date_str = item.get("date")
|
|
date_confidence = dates.get_date_confidence(date_str, from_date, to_date)
|
|
|
|
normalized.append(schema.XItem(
|
|
id=item.get("id", ""),
|
|
text=item.get("text", ""),
|
|
url=item.get("url", ""),
|
|
author_handle=item.get("author_handle", ""),
|
|
date=date_str,
|
|
date_confidence=date_confidence,
|
|
engagement=engagement,
|
|
relevance=item.get("relevance", 0.5),
|
|
why_relevant=item.get("why_relevant", ""),
|
|
))
|
|
|
|
return normalized
|
|
|
|
|
|
def items_to_dicts(items: List) -> List[Dict[str, Any]]:
|
|
"""Convert schema items to dicts for JSON serialization."""
|
|
return [item.to_dict() for item in items]
|