sawz/last30days-skill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Enforce strict 30-day date filtering

Browse Source 
Previously Reddit was returning ~60% old content (some from 2022).
This commit adds multiple layers of date enforcement:

- Reddit prompt: Explicit from_date/to_date with "fewer results > older results"
- Hard filter: filter_by_date_range() in normalize.py excludes old content
- WebSearch Date Detective: Extracts dates from URLs (/2026/01/24/) and
  snippets ("January 24, 2026", "3 days ago")
- WebSearch scoring: +10 bonus for verified dates, -20 penalty for unknown

The skill now guarantees only content from the last 30 days.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
2026-01-24 12:23:34 -08:00
parent 354e8bad03
commit 18696e7b05
6 changed files with 636 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/websearch.py
+215 -9
View File
@@ -11,12 +11,196 @@ The typical flow is:
"""
import re
from typing import Any, Dict, List, Optional
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import urlparse
from . import schema
# Month name mappings for date parsing
MONTH_MAP = {
"jan": 1, "january": 1,
"feb": 2, "february": 2,
"mar": 3, "march": 3,
"apr": 4, "april": 4,
"may": 5,
"jun": 6, "june": 6,
"jul": 7, "july": 7,
"aug": 8, "august": 8,
"sep": 9, "sept": 9, "september": 9,
"oct": 10, "october": 10,
"nov": 11, "november": 11,
"dec": 12, "december": 12,
}
def extract_date_from_url(url: str) -> Optional[str]:
"""Try to extract a date from URL path.
Many sites embed dates in URLs like:
- /2026/01/24/article-title
- /2026-01-24/article
- /blog/20260124/title
Args:
url: URL to parse
Returns:
Date string in YYYY-MM-DD format, or None
"""
# Pattern 1: /YYYY/MM/DD/ (most common)
match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', url)
if match:
year, month, day = match.groups()
if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
return f"{year}-{month}-{day}"
# Pattern 2: /YYYY-MM-DD/ or /YYYY-MM-DD-
match = re.search(r'/(\d{4})-(\d{2})-(\d{2})[-/]', url)
if match:
year, month, day = match.groups()
if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
return f"{year}-{month}-{day}"
# Pattern 3: /YYYYMMDD/ (compact)
match = re.search(r'/(\d{4})(\d{2})(\d{2})/', url)
if match:
year, month, day = match.groups()
if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
return f"{year}-{month}-{day}"
return None
def extract_date_from_snippet(text: str) -> Optional[str]:
"""Try to extract a date from text snippet or title.
Looks for patterns like:
- January 24, 2026 or Jan 24, 2026
- 24 January 2026
- 2026-01-24
- "3 days ago", "yesterday", "last week"
Args:
text: Text to parse
Returns:
Date string in YYYY-MM-DD format, or None
"""
if not text:
return None
text_lower = text.lower()
# Pattern 1: Month DD, YYYY (e.g., "January 24, 2026")
match = re.search(
r'\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
r'\s+(\d{1,2})(?:st|nd|rd|th)?,?\s*(\d{4})\b',
text_lower
)
if match:
month_str, day, year = match.groups()
month = MONTH_MAP.get(month_str[:3])
if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
return f"{year}-{month:02d}-{int(day):02d}"
# Pattern 2: DD Month YYYY (e.g., "24 January 2026")
match = re.search(
r'\b(\d{1,2})(?:st|nd|rd|th)?\s+'
r'(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|'
r'jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)'
r'\s+(\d{4})\b',
text_lower
)
if match:
day, month_str, year = match.groups()
month = MONTH_MAP.get(month_str[:3])
if month and 2020 <= int(year) <= 2030 and 1 <= int(day) <= 31:
return f"{year}-{month:02d}-{int(day):02d}"
# Pattern 3: YYYY-MM-DD (ISO format)
match = re.search(r'\b(\d{4})-(\d{2})-(\d{2})\b', text)
if match:
year, month, day = match.groups()
if 2020 <= int(year) <= 2030 and 1 <= int(month) <= 12 and 1 <= int(day) <= 31:
return f"{year}-{month}-{day}"
# Pattern 4: Relative dates ("3 days ago", "yesterday", etc.)
today = datetime.now()
if "yesterday" in text_lower:
date = today - timedelta(days=1)
return date.strftime("%Y-%m-%d")
if "today" in text_lower:
return today.strftime("%Y-%m-%d")
# "N days ago"
match = re.search(r'\b(\d+)\s*days?\s*ago\b', text_lower)
if match:
days = int(match.group(1))
if days <= 60: # Reasonable range
date = today - timedelta(days=days)
return date.strftime("%Y-%m-%d")
# "N hours ago" -> today
match = re.search(r'\b(\d+)\s*hours?\s*ago\b', text_lower)
if match:
return today.strftime("%Y-%m-%d")
# "last week" -> ~7 days ago
if "last week" in text_lower:
date = today - timedelta(days=7)
return date.strftime("%Y-%m-%d")
# "this week" -> ~3 days ago (middle of week)
if "this week" in text_lower:
date = today - timedelta(days=3)
return date.strftime("%Y-%m-%d")
return None
def extract_date_signals(
url: str,
snippet: str,
title: str,
) -> Tuple[Optional[str], str]:
"""Extract date from any available signal.
Tries URL first (most reliable), then snippet, then title.
Args:
url: Page URL
snippet: Page snippet/description
title: Page title
Returns:
Tuple of (date_string, confidence)
- date from URL: 'high' confidence
- date from snippet/title: 'med' confidence
- no date found: None, 'low' confidence
"""
# Try URL first (most reliable)
url_date = extract_date_from_url(url)
if url_date:
return url_date, "high"
# Try snippet
snippet_date = extract_date_from_snippet(snippet)
if snippet_date:
return snippet_date, "med"
# Try title
title_date = extract_date_from_snippet(title)
if title_date:
return title_date, "med"
return None, "low"
# Domains to exclude (Reddit and X are handled separately)
EXCLUDED_DOMAINS = {
"reddit.com",
@@ -70,15 +254,25 @@ def is_excluded_domain(url: str) -> bool:
def parse_websearch_results(
results: List[Dict[str, Any]],
topic: str,
from_date: str = "",
to_date: str = "",
) -> List[Dict[str, Any]]:
"""Parse WebSearch results into normalized format.
This function expects results from Claude's WebSearch tool.
Each result should have: title, url, snippet, and optionally date/relevance.
Uses "Date Detective" approach:
1. Extract dates from URLs (high confidence)
2. Extract dates from snippets/titles (med confidence)
3. Hard filter: exclude items with verified old dates
4. Keep items with no date signals (with low confidence penalty)
Args:
results: List of WebSearch result dicts
topic: Original search topic (for context)
from_date: Start date for filtering (YYYY-MM-DD)
to_date: End date for filtering (YYYY-MM-DD)
Returns:
List of normalized item dicts ready for WebSearchItem creation
@@ -103,15 +297,27 @@ def parse_websearch_results(
if not title and not snippet:
continue
# Parse date if provided
date = result.get("date")
# Use Date Detective to extract date signals
date = result.get("date") # Use provided date if available
date_confidence = "low"
if date:
# Validate date format
if re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)):
date_confidence = "med" # WebSearch dates are often approximate
else:
date = None
if date and re.match(r'^\d{4}-\d{2}-\d{2}$', str(date)):
# Provided date is valid
date_confidence = "med"
else:
# Try to extract date from URL/snippet/title
extracted_date, confidence = extract_date_signals(url, snippet, title)
if extracted_date:
date = extracted_date
date_confidence = confidence
# Hard filter: if we found a date and it's too old, skip
if date and from_date and date < from_date:
continue # DROP - verified old content
# Hard filter: if date is in the future, skip (parsing error)
if date and to_date and date > to_date:
continue # DROP - future date
# Get relevance if provided, default to 0.5
relevance = result.get("relevance", 0.5)