diff --git a/plans/fix-strict-date-filtering.md b/plans/fix-strict-date-filtering.md new file mode 100644 index 0000000..2c0cd85 --- /dev/null +++ b/plans/fix-strict-date-filtering.md @@ -0,0 +1,328 @@ +# fix: Enforce Strict 30-Day Date Filtering + +## Overview + +The `/last30days` skill is returning content older than 30 days, violating its core promise. Analysis shows: +- **Reddit**: Only 40% of results within 30 days (9/15 were older, some from 2022!) +- **X**: 100% within 30 days (working correctly) +- **WebSearch**: 90% had unknown dates (can't verify freshness) + +## Problem Statement + +The skill's name is "last30days" - users expect ONLY content from the last 30 days. Currently: + +1. **Reddit search prompt** says "prefer recent threads, but include older relevant ones if recent ones are scarce" - this is too permissive +2. **X search prompt** explicitly includes `from_date` and `to_date` - this is why it works +3. **WebSearch** returns pages without publication dates - we can't verify they're recent +4. **Scoring penalties** (-10 for low date confidence) don't prevent old content from appearing + +## Proposed Solution + +### Strategy: "Hard Filter, Not Soft Penalty" + +Instead of penalizing old content, **exclude it entirely**. If it's not from the last 30 days, it shouldn't appear. + +| Source | Current Behavior | New Behavior | +|--------|------------------|--------------| +| Reddit | Weak "prefer recent" | Explicit date range + hard filter | +| X | Explicit date range (working) | No change needed | +| WebSearch | No date awareness | Require recent markers OR exclude | + +## Technical Approach + +### Phase 1: Fix Reddit Date Filtering + +**File: `scripts/lib/openai_reddit.py`** + +Current prompt (line 33): +``` +Find {min_items}-{max_items} relevant Reddit discussion threads. +Prefer recent threads, but include older relevant ones if recent ones are scarce. +``` + +New prompt: +``` +Find {min_items}-{max_items} relevant Reddit discussion threads from {from_date} to {to_date}. + +CRITICAL: Only include threads posted within the last 30 days (after {from_date}). +Do NOT include threads older than {from_date}, even if they seem relevant. +If you cannot find enough recent threads, return fewer results rather than older ones. +``` + +**Changes needed:** +1. Add `from_date` and `to_date` parameters to `search_reddit()` function +2. Inject dates into `REDDIT_SEARCH_PROMPT` like X does +3. Update caller in `last30days.py` to pass dates + +### Phase 2: Add Hard Date Filtering (Post-Processing) + +**File: `scripts/lib/normalize.py`** + +Add a filter step that DROPS items with dates before `from_date`: + +```python +def filter_by_date_range( + items: List[Union[RedditItem, XItem, WebSearchItem]], + from_date: str, + to_date: str, + require_date: bool = False, +) -> List: + """Hard filter: Remove items outside the date range. + + Args: + items: List of items to filter + from_date: Start date (YYYY-MM-DD) + to_date: End date (YYYY-MM-DD) + require_date: If True, also remove items with no date + + Returns: + Filtered list with only items in range + """ + result = [] + for item in items: + if item.date is None: + if not require_date: + result.append(item) # Keep unknown dates (with penalty) + continue + + # Hard filter: if date is before from_date, exclude + if item.date < from_date: + continue # DROP - too old + + if item.date > to_date: + continue # DROP - future date (likely parsing error) + + result.append(item) + + return result +``` + +### Phase 3: WebSearch Date Intelligence + +WebSearch CAN find recent content - Medium posts have dates, GitHub has commit timestamps, news sites have publication dates. We should **extract and prioritize** these signals. + +**Strategy: "Date Detective"** + +1. **Extract dates from URLs**: Many sites embed dates in URLs + - Medium: `medium.com/@author/title-abc123` (no date) vs news sites + - GitHub: Look for commit dates, release dates in snippets + - News: `/2026/01/24/article-title` + - Blogs: `/blog/2026/01/title` + +2. **Extract dates from snippets**: Look for date markers + - "January 24, 2026", "Jan 2026", "yesterday", "this week" + - "Published:", "Posted:", "Updated:" + - Relative markers: "2 days ago", "last week" + +3. **Prioritize results with verifiable dates**: + - Results with recent dates (within 30 days): Full score + - Results with old dates: EXCLUDE + - Results with no date signals: Heavy penalty (-20) but keep as supplementary + +**File: `scripts/lib/websearch.py`** + +Add date extraction functions: + +```python +import re +from datetime import datetime, timedelta + +# Patterns for date extraction +URL_DATE_PATTERNS = [ + r'/(\d{4})/(\d{2})/(\d{2})/', # /2026/01/24/ + r'/(\d{4})-(\d{2})-(\d{2})/', # /2026-01-24/ + r'/(\d{4})(\d{2})(\d{2})/', # /20260124/ +] + +SNIPPET_DATE_PATTERNS = [ + r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (\d{1,2}),? (\d{4})', + r'(\d{1,2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (\d{4})', + r'(\d{4})-(\d{2})-(\d{2})', + r'Published:?\s*(\d{4}-\d{2}-\d{2})', + r'(\d{1,2}) (days?|hours?|minutes?) ago', # Relative dates +] + +def extract_date_from_url(url: str) -> Optional[str]: + """Try to extract a date from URL path.""" + for pattern in URL_DATE_PATTERNS: + match = re.search(pattern, url) + if match: + # Parse and return YYYY-MM-DD format + ... + return None + +def extract_date_from_snippet(snippet: str) -> Optional[str]: + """Try to extract a date from text snippet.""" + for pattern in SNIPPET_DATE_PATTERNS: + match = re.search(pattern, snippet, re.IGNORECASE) + if match: + # Parse and return YYYY-MM-DD format + ... + return None + +def extract_date_signals(url: str, snippet: str, title: str) -> tuple[Optional[str], str]: + """Extract date from any available signal. + + Returns: (date_string, confidence) + - date from URL: 'high' confidence + - date from snippet: 'med' confidence + - no date found: None, 'low' confidence + """ + # Try URL first (most reliable) + url_date = extract_date_from_url(url) + if url_date: + return url_date, 'high' + + # Try snippet + snippet_date = extract_date_from_snippet(snippet) + if snippet_date: + return snippet_date, 'med' + + # Try title + title_date = extract_date_from_snippet(title) + if title_date: + return title_date, 'med' + + return None, 'low' +``` + +**Update WebSearch parsing to use date extraction:** + +```python +def parse_websearch_results(results, topic, from_date, to_date): + items = [] + for result in results: + url = result.get('url', '') + snippet = result.get('snippet', '') + title = result.get('title', '') + + # Extract date signals + extracted_date, confidence = extract_date_signals(url, snippet, title) + + # Hard filter: if we found a date and it's too old, skip + if extracted_date and extracted_date < from_date: + continue # DROP - verified old content + + item = { + 'date': extracted_date, + 'date_confidence': confidence, + ... + } + items.append(item) + + return items +``` + +**File: `scripts/lib/score.py`** + +Update WebSearch scoring to reward date-verified results: + +```python +# WebSearch date confidence adjustments +WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date (was 10) +WEBSEARCH_VERIFIED_BONUS = 10 # Bonus for URL-verified recent date + +def score_websearch_items(items): + for item in items: + ... + # Date confidence adjustments + if item.date_confidence == 'high': + overall += WEBSEARCH_VERIFIED_BONUS # Reward verified dates + elif item.date_confidence == 'low': + overall -= WEBSEARCH_NO_DATE_PENALTY # Heavy penalty for unknown + ... +``` + +**Result**: WebSearch results with verifiable recent dates rank well. Results with no dates are heavily penalized but still appear as supplementary context. Old verified content is excluded entirely. + +### Phase 4: Update Statistics Display + +Only count Reddit and X in "from the last 30 days" claim. WebSearch should be clearly labeled as supplementary. + +## Acceptance Criteria + +### Functional Requirements + +- [x] Reddit search prompt includes explicit `from_date` and `to_date` +- [x] Items with dates before `from_date` are EXCLUDED, not just penalized +- [x] X search continues working (no regression) +- [x] WebSearch extracts dates from URLs (e.g., `/2026/01/24/`) +- [x] WebSearch extracts dates from snippets (e.g., "January 24, 2026") +- [x] WebSearch with verified recent dates gets +10 bonus +- [x] WebSearch with no date signals gets -20 penalty (but still appears) +- [x] WebSearch with verified OLD dates is EXCLUDED + +### Non-Functional Requirements + +- [ ] No increase in API latency +- [ ] Graceful handling when few recent results exist (return fewer, not older) +- [ ] Clear user messaging when results are limited due to strict filtering + +### Quality Gates + +- [ ] Test: Reddit search returns 0% results older than 30 days +- [ ] Test: X search continues to return 100% recent results +- [ ] Test: WebSearch is clearly differentiated in output +- [ ] Test: Edge case - topic with no recent content shows helpful message + +## Implementation Order + +1. **Phase 1**: Fix Reddit prompt (highest impact, simple change) +2. **Phase 2**: Add hard date filter in normalize.py (safety net) +3. **Phase 3**: Add WebSearch date extraction (URL + snippet parsing) +4. **Phase 4**: Update WebSearch scoring (bonus for verified, heavy penalty for unknown) +5. **Phase 5**: Update output display to show date confidence + +## Testing Plan + +### Before/After Test + +Run same query before and after fix: +``` +/last30days remotion launch videos +``` + +**Expected Before:** +- Reddit: 40% within 30 days + +**Expected After:** +- Reddit: 100% within 30 days (or fewer results if not enough recent content) + +### Edge Case Tests + +| Scenario | Expected Behavior | +|----------|-------------------| +| Topic with no recent content | Return 0 results + helpful message | +| Topic with 5 recent results | Return 5 results (not pad with old ones) | +| Mixed old/new results | Only return new ones | + +### WebSearch Date Extraction Tests + +| URL/Snippet | Expected Date | Confidence | +|-------------|---------------|------------| +| `medium.com/blog/2026/01/15/title` | 2026-01-15 | high | +| `github.com/repo` + "Released Jan 20, 2026" | 2026-01-20 | med | +| `docs.example.com/guide` (no date signals) | None | low | +| `news.site.com/2024/05/old-article` | 2024-05-XX | EXCLUDE (too old) | +| Snippet: "Updated 3 days ago" | calculated | med | + +## Risk Analysis + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Fewer results for niche topics | High | Medium | Explain why in output | +| User confusion about reduced results | Medium | Low | Clear messaging | +| Date parsing errors exclude valid content | Low | Medium | Keep items with unknown dates, just label clearly | + +## References + +### Internal References +- Reddit search: `scripts/lib/openai_reddit.py:25-63` +- X search (working example): `scripts/lib/xai_x.py:26-55` +- Date confidence: `scripts/lib/dates.py:62-90` +- Scoring penalties: `scripts/lib/score.py:149-153` +- Normalization: `scripts/lib/normalize.py:49,99` + +### External References +- OpenAI Responses API lacks native date filtering +- Must rely on prompt engineering + post-processing diff --git a/scripts/last30days.py b/scripts/last30days.py index ca83f9d..0a6215d 100644 --- a/scripts/last30days.py +++ b/scripts/last30days.py @@ -95,6 +95,8 @@ def run_research( config["OPENAI_API_KEY"], selected_models["openai"], topic, + from_date, + to_date, depth=depth, ) except http.HTTPError as e: @@ -319,9 +321,14 @@ def main(): normalized_reddit = normalize.normalize_reddit_items(reddit_items, from_date, to_date) normalized_x = normalize.normalize_x_items(x_items, from_date, to_date) + # Hard date filter: exclude items with verified dates outside the range + # This is the safety net - even if prompts let old content through, this filters it + filtered_reddit = normalize.filter_by_date_range(normalized_reddit, from_date, to_date) + filtered_x = normalize.filter_by_date_range(normalized_x, from_date, to_date) + # Score items - scored_reddit = score.score_reddit_items(normalized_reddit) - scored_x = score.score_x_items(normalized_x) + scored_reddit = score.score_reddit_items(filtered_reddit) + scored_x = score.score_x_items(filtered_x) # Sort items sorted_reddit = score.sort_items(scored_reddit) diff --git a/scripts/lib/normalize.py b/scripts/lib/normalize.py index 62c216a..0d2577e 100644 --- a/scripts/lib/normalize.py +++ b/scripts/lib/normalize.py @@ -1,9 +1,51 @@ """Normalization of raw API data to canonical schema.""" -from typing import Any, Dict, List +from typing import Any, Dict, List, TypeVar, Union from . import dates, schema +T = TypeVar("T", schema.RedditItem, schema.XItem, schema.WebSearchItem) + + +def filter_by_date_range( + items: List[T], + from_date: str, + to_date: str, + require_date: bool = False, +) -> List[T]: + """Hard filter: Remove items outside the date range. + + This is the safety net - even if the prompt lets old content through, + this filter will exclude it. + + Args: + items: List of items to filter + from_date: Start date (YYYY-MM-DD) - exclude items before this + to_date: End date (YYYY-MM-DD) - exclude items after this + require_date: If True, also remove items with no date + + Returns: + Filtered list with only items in range (or unknown dates if not required) + """ + result = [] + for item in items: + if item.date is None: + if not require_date: + result.append(item) # Keep unknown dates (with scoring penalty) + continue + + # Hard filter: if date is before from_date, exclude + if item.date < from_date: + continue # DROP - too old + + # Hard filter: if date is after to_date, exclude (likely parsing error) + if item.date > to_date: + continue # DROP - future date + + result.append(item) + + return result + def normalize_reddit_items( items: List[Dict[str, Any]], diff --git a/scripts/lib/openai_reddit.py b/scripts/lib/openai_reddit.py index 305750c..6f90dde 100644 --- a/scripts/lib/openai_reddit.py +++ b/scripts/lib/openai_reddit.py @@ -24,21 +24,27 @@ DEPTH_CONFIG = { REDDIT_SEARCH_PROMPT = """Search Reddit for DISCUSSION THREADS about: {topic} +DATE RANGE: Only include threads from {from_date} to {to_date} (last 30 days). + SEARCH GUIDANCE: - Search for "site:reddit.com/r/ {topic}" to find subreddit discussions -- Look in subreddits like r/design, r/UI_Design, r/iOSProgramming, r/SwiftUI, r/Figma, r/webdev, r/userexperience, r/graphic_design +- Look in subreddits like r/design, r/UI_Design, r/iOSProgramming, r/SwiftUI, r/Figma, r/webdev, r/userexperience, r/graphic_design, r/ClaudeAI, r/ClaudeCode - ONLY include URLs containing "/r/" and "/comments/" (actual discussion threads) - IGNORE: developers.reddit.com, business.reddit.com, reddit.com/user/ -Find {min_items}-{max_items} relevant Reddit discussion threads. Prefer recent threads, but include older relevant ones if recent ones are scarce. +CRITICAL DATE REQUIREMENT: +- ONLY include threads posted AFTER {from_date} +- Do NOT include threads older than {from_date}, even if they seem relevant +- If you cannot find enough recent threads, return FEWER results rather than older ones +- It is better to return 3 recent threads than 15 old ones -CRITICAL: Return ALL discussion threads you find as JSON. Do NOT return errors or empty results. +Find {min_items}-{max_items} relevant Reddit discussion threads from the last 30 days. For EACH Reddit thread URL (containing /r/subreddit/comments/), extract: - Thread title - Full Reddit URL - Subreddit name -- Date (if visible, otherwise null) +- Date (MUST be after {from_date}, otherwise do not include) - Why it's relevant Return ONLY valid JSON: @@ -57,16 +63,18 @@ Return ONLY valid JSON: Rules: - ONLY URLs matching: reddit.com/r/*/comments/* -- MUST return threads found - NEVER return empty items or errors -- If threads are older than 30 days, still include them with accurate dates +- ONLY threads from {from_date} to {to_date} - relevance: 0.0-1.0 -- Diverse subreddits preferred""" +- Diverse subreddits preferred +- Fewer recent results is better than many old results""" def search_reddit( api_key: str, model: str, topic: str, + from_date: str, + to_date: str, depth: str = "default", mock_response: Optional[Dict] = None, ) -> Dict[str, Any]: @@ -76,6 +84,8 @@ def search_reddit( api_key: OpenAI API key model: Model to use topic: Search topic + from_date: Start date (YYYY-MM-DD) - only include threads after this + to_date: End date (YYYY-MM-DD) - only include threads before this depth: Research depth - "quick", "default", or "deep" mock_response: Mock response for testing @@ -106,7 +116,13 @@ def search_reddit( } ], "include": ["web_search_call.action.sources"], - "input": REDDIT_SEARCH_PROMPT.format(topic=topic, min_items=min_items, max_items=max_items), + "input": REDDIT_SEARCH_PROMPT.format( + topic=topic, + from_date=from_date, + to_date=to_date, + min_items=min_items, + max_items=max_items, + ), } return http.post(OPENAI_RESPONSES_URL, payload, headers=headers, timeout=timeout) diff --git a/scripts/lib/score.py b/scripts/lib/score.py index 54a51c8..0f9eb69 100644 --- a/scripts/lib/score.py +++ b/scripts/lib/score.py @@ -15,6 +15,10 @@ WEBSEARCH_WEIGHT_RELEVANCE = 0.55 WEBSEARCH_WEIGHT_RECENCY = 0.45 WEBSEARCH_SOURCE_PENALTY = 15 # Points deducted for lacking engagement +# WebSearch date confidence adjustments +WEBSEARCH_VERIFIED_BONUS = 10 # Bonus for URL-verified recent date (high confidence) +WEBSEARCH_NO_DATE_PENALTY = 20 # Heavy penalty for no date signals (low confidence) + # Default engagement score for unknown DEFAULT_ENGAGEMENT = 35 UNKNOWN_ENGAGEMENT_PENALTY = 10 @@ -223,6 +227,11 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS Uses reweighted formula: 55% relevance + 45% recency - 15pt source penalty. This ensures WebSearch items rank below comparable Reddit/X items. + Date confidence adjustments: + - High confidence (URL-verified date): +10 bonus + - Med confidence (snippet-extracted date): no change + - Low confidence (no date signals): -20 penalty + Args: items: List of WebSearch items @@ -255,11 +264,14 @@ def score_websearch_items(items: List[schema.WebSearchItem]) -> List[schema.WebS # Apply source penalty (WebSearch < Reddit/X for same relevance/recency) overall -= WEBSEARCH_SOURCE_PENALTY - # Apply penalty for low date confidence - if item.date_confidence == "low": - overall -= 10 - elif item.date_confidence == "med": - overall -= 5 + # Apply date confidence adjustments + # High confidence (URL-verified): reward with bonus + # Med confidence (snippet-extracted): neutral + # Low confidence (no date signals): heavy penalty + if item.date_confidence == "high": + overall += WEBSEARCH_VERIFIED_BONUS # Reward verified recent dates + elif item.date_confidence == "low": + overall -= WEBSEARCH_NO_DATE_PENALTY # Heavy penalty for unknown item.score = max(0, min(100, int(overall))) diff --git a/scripts/lib/websearch.py b/scripts/lib/websearch.py index 89a1320..fe87654 100644 --- a/scripts/lib/websearch.py +++ b/scripts/lib/websearch.py @@ -11,12 +11,196 @@ The typical flow is: """ import re -from typing import Any, Dict, List, Optional +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse from . import schema +# Month name mappings for date parsing +MONTH_MAP = { + "jan": 1, "january": 1, + "feb": 2, "february": 2, + "mar": 3, "march": 3, + "apr": 4, "april": 4, + "may": 5, + "jun": 6, "june": 6, + "jul": 7, "july": 7, + "aug": 8, "august": 8, + "sep": 9, "sept": 9, "september": 9, + "oct": 10, "october": 10, + "nov": 11, "november": 11, + "dec": 12, "december": 12, +} + + +def extract_date_from_url(url: str) -> Optional[str]: + """Try to extract a date from URL path. + + Many sites embed dates in URLs like: + - /2026/01/24/article-title + - /2026-01-24/article + - /blog/20260124/title + + Args: + url: URL to parse + + Returns: + Date string in YYYY-MM-DD format, or None + """ + # Pattern 1: /YYYY/MM/DD/ (most common) + match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', url) + if match: + year, month, day = match.groups() + if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31: + return f"{year}-{month}-{day}" + + # Pattern 2: /YYYY-MM-DD/ or /YYYY-MM-DD- + match = re.search(r'/(\d{4})-(\d{2})-(\d{2})[-/]', url) + if match: + year, month, day = match.groups() + if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31: + return f"{year}-{month}-{day}" + + # Pattern 3: /YYYYMMDD/ (compact) + match = re.search(r'/(\d{4})(\d{2})(\d{2})/', url) + if match: + year, month, day = match.groups() + if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31: + return f"{year}-{month}-{day}" + + return None + + +def extract_date_from_snippet(text: str) -> Optional[str]: + """Try to extract a date from text snippet or title. + + Looks for patterns like: + - January 24, 2026 or Jan 24, 2026 + - 24 January 2026 + - 2026-01-24 + - "3 days ago", "yesterday", "last week" + + Args: + text: Text to parse + + Returns: + Date string in YYYY-MM-DD format, or None + """ + if not text: + return None + + text_lower = text.lower() + + # Pattern 1: Month DD, YYYY (e.g., "January 24, 2026") + match = re.search( + r'\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|' + r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)' + r'\s+(\d{1,2})(?:st|nd|rd|th)?,?\s*(\d{4})\b', + text_lower + ) + if match: + month_str, day, year = match.groups() + month = MONTH_MAP.get(month_str[:3]) + if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31: + return f"{year}-{month:02d}-{int(day):02d}" + + # Pattern 2: DD Month YYYY (e.g., "24 January 2026") + match = re.search( + r'\b(\d{1,2})(?:st|nd|rd|th)?\s+' + r'(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|' + r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)' + r'\s+(\d{4})\b', + text_lower + ) + if match: + day, month_str, year = match.groups() + month = MONTH_MAP.get(month_str[:3]) + if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31: + return f"{year}-{month:02d}-{int(day):02d}" + + # Pattern 3: YYYY-MM-DD (ISO format) + match = re.search(r'\b(\d{4})-(\d{2})-(\d{2})\b', text) + if match: + year, month, day = match.groups() + if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31: + return f"{year}-{month}-{day}" + + # Pattern 4: Relative dates ("3 days ago", "yesterday", etc.) + today = datetime.now() + + if "yesterday" in text_lower: + date = today - timedelta(days=1) + return date.strftime("%Y-%m-%d") + + if "today" in text_lower: + return today.strftime("%Y-%m-%d") + + # "N days ago" + match = re.search(r'\b(\d+)\s*days?\s*ago\b', text_lower) + if match: + days = int(match.group(1)) + if days <= 60: # Reasonable range + date = today - timedelta(days=days) + return date.strftime("%Y-%m-%d") + + # "N hours ago" -> today + match = re.search(r'\b(\d+)\s*hours?\s*ago\b', text_lower) + if match: + return today.strftime("%Y-%m-%d") + + # "last week" -> ~7 days ago + if "last week" in text_lower: + date = today - timedelta(days=7) + return date.strftime("%Y-%m-%d") + + # "this week" -> ~3 days ago (middle of week) + if "this week" in text_lower: + date = today - timedelta(days=3) + return date.strftime("%Y-%m-%d") + + return None + + +def extract_date_signals( + url: str, + snippet: str, + title: str, +) -> Tuple[Optional[str], str]: + """Extract date from any available signal. + + Tries URL first (most reliable), then snippet, then title. + + Args: + url: Page URL + snippet: Page snippet/description + title: Page title + + Returns: + Tuple of (date_string, confidence) + - date from URL: 'high' confidence + - date from snippet/title: 'med' confidence + - no date found: None, 'low' confidence + """ + # Try URL first (most reliable) + url_date = extract_date_from_url(url) + if url_date: + return url_date, "high" + + # Try snippet + snippet_date = extract_date_from_snippet(snippet) + if snippet_date: + return snippet_date, "med" + + # Try title + title_date = extract_date_from_snippet(title) + if title_date: + return title_date, "med" + + return None, "low" + + # Domains to exclude (Reddit and X are handled separately) EXCLUDED_DOMAINS = { "reddit.com", @@ -70,15 +254,25 @@ def is_excluded_domain(url: str) -> bool: def parse_websearch_results( results: List[Dict[str, Any]], topic: str, + from_date: str = "", + to_date: str = "", ) -> List[Dict[str, Any]]: """Parse WebSearch results into normalized format. This function expects results from Claude's WebSearch tool. Each result should have: title, url, snippet, and optionally date/relevance. + Uses "Date Detective" approach: + 1. Extract dates from URLs (high confidence) + 2. Extract dates from snippets/titles (med confidence) + 3. Hard filter: exclude items with verified old dates + 4. Keep items with no date signals (with low confidence penalty) + Args: results: List of WebSearch result dicts topic: Original search topic (for context) + from_date: Start date for filtering (YYYY-MM-DD) + to_date: End date for filtering (YYYY-MM-DD) Returns: List of normalized item dicts ready for WebSearchItem creation @@ -103,15 +297,27 @@ def parse_websearch_results( if not title and not snippet: continue - # Parse date if provided - date = result.get("date") + # Use Date Detective to extract date signals + date = result.get("date") # Use provided date if available date_confidence = "low" - if date: - # Validate date format - if re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)): - date_confidence = "med" # WebSearch dates are often approximate - else: - date = None + + if date and re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)): + # Provided date is valid + date_confidence = "med" + else: + # Try to extract date from URL/snippet/title + extracted_date, confidence = extract_date_signals(url, snippet, title) + if extracted_date: + date = extracted_date + date_confidence = confidence + + # Hard filter: if we found a date and it's too old, skip + if date and from_date and date < from_date: + continue # DROP - verified old content + + # Hard filter: if date is in the future, skip (parsing error) + if date and to_date and date > to_date: + continue # DROP - future date # Get relevance if provided, default to 0.5 relevance = result.get("relevance", 0.5)