fix(github): reject garbage in _parse_date; consolidate date parsing
github.py _parse_date used naive string slicing (return iso_str[:10])
which accepted any 10+ character string as a "date." For input
"hello world" it returned "hello worl". Now delegates to
dates.parse_date() which validates the format and returns None for
non-dates.
Also migrated reddit.py and threads.py _parse_date to the shared
dates.parse_date(). Both previously reimplemented ISO-with-trailing-
offset handling (the .replace("Z", "+00:00") dance) and reddit.py
also had its own Unix timestamp branch. dates.parse_date() already
handles all of this, including the +0000 no-colon variant Reddit emits.
Preserved reddit.py's original falsy-check so 0 still returns None
(epoch 0 would otherwise parse as "1970-01-01", breaking an existing
test and changing long-standing behavior).
Added 4 new github tests for garbage rejection and offset variants.
All 1026 existing tests pass (15 pre-existing failures unchanged).
This commit is contained in:
@@ -17,7 +17,7 @@ import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import log
|
||||
from . import dates, log
|
||||
from .query import extract_core_subject
|
||||
from .relevance import token_overlap_relevance
|
||||
|
||||
@@ -106,13 +106,14 @@ def _parse_repo_from_url(html_url: str) -> str:
|
||||
|
||||
|
||||
def _parse_date(iso_str: Optional[str]) -> Optional[str]:
|
||||
"""Extract YYYY-MM-DD from ISO 8601 datetime string."""
|
||||
if not iso_str:
|
||||
return None
|
||||
try:
|
||||
return iso_str[:10]
|
||||
except (IndexError, TypeError):
|
||||
return None
|
||||
"""Parse a GitHub ISO 8601 datetime string and return YYYY-MM-DD.
|
||||
|
||||
Returns None for non-date input. GitHub's API always emits ISO 8601
|
||||
(e.g. "2026-02-26T16:00:00Z"), but we defer to dates.parse_date() so
|
||||
garbage input gets rejected instead of silently sliced.
|
||||
"""
|
||||
dt = dates.parse_date(iso_str)
|
||||
return dt.strftime("%Y-%m-%d") if dt else None
|
||||
|
||||
|
||||
def _compute_relevance(
|
||||
|
||||
+8
-20
@@ -12,7 +12,6 @@ import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_wait
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
try:
|
||||
@@ -28,7 +27,7 @@ def _first_of(*values, default=None):
|
||||
return v
|
||||
return default
|
||||
|
||||
from . import http, log
|
||||
from . import dates, http, log
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/reddit"
|
||||
|
||||
@@ -212,27 +211,16 @@ def _parse_date(value) -> Optional[str]:
|
||||
|
||||
Global search returns ``created_at`` as an ISO string
|
||||
(e.g. "2018-05-03T01:09:17.620000+0000"); subreddit search returns
|
||||
``created_utc`` as a Unix timestamp. Handle both.
|
||||
``created_utc`` as a Unix timestamp. dates.parse_date() handles both,
|
||||
plus edge cases like Z suffix and +0000 (no colon) offset.
|
||||
|
||||
Falsy inputs (None, "", 0) return None, matching the original behavior
|
||||
where a Unix timestamp of 0 meant "no date" rather than epoch 0.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
# ISO-8601 string (contains 'T' or '-')
|
||||
if isinstance(value, str) and ("T" in value or "-" in value):
|
||||
try:
|
||||
# Strip trailing offset variations (+0000, Z) for fromisoformat
|
||||
clean = value.replace("Z", "+00:00")
|
||||
if clean.endswith("+0000"):
|
||||
clean = clean[:-5] + "+00:00"
|
||||
dt = datetime.fromisoformat(clean)
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
# Unix timestamp (int or float or numeric string)
|
||||
try:
|
||||
dt = datetime.fromtimestamp(float(value), tz=timezone.utc)
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError, OSError):
|
||||
return None
|
||||
dt = dates.parse_date(str(value))
|
||||
return dt.strftime("%Y-%m-%d") if dt else None
|
||||
|
||||
|
||||
def _extract_subreddit_name(value: Any) -> str:
|
||||
|
||||
+10
-23
@@ -9,10 +9,9 @@ API docs: https://scrapecreators.com/docs
|
||||
|
||||
import math
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import http, log
|
||||
from . import dates, http, log
|
||||
from .relevance import token_overlap_relevance as _compute_relevance
|
||||
|
||||
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/threads"
|
||||
@@ -52,29 +51,17 @@ def _extract_core_subject(topic: str) -> str:
|
||||
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Parse date from Threads item to YYYY-MM-DD.
|
||||
|
||||
Tries common timestamp fields: taken_at (unix), created_at (ISO),
|
||||
and falls back to any date-like string field.
|
||||
Tries common timestamp fields in order: taken_at and create_time
|
||||
(unix timestamps in Meta APIs), then created_at, published_at, and
|
||||
date (ISO 8601 strings). dates.parse_date() handles both.
|
||||
"""
|
||||
# Unix timestamp (taken_at is common in Meta APIs)
|
||||
for key in ("taken_at", "create_time"):
|
||||
ts = item.get(key)
|
||||
if ts:
|
||||
try:
|
||||
from . import dates
|
||||
return dates.timestamp_to_date(int(ts))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# ISO 8601 string
|
||||
for key in ("created_at", "published_at", "date"):
|
||||
for key in ("taken_at", "create_time", "created_at", "published_at", "date"):
|
||||
val = item.get(key)
|
||||
if val and isinstance(val, str):
|
||||
try:
|
||||
dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if val is None:
|
||||
continue
|
||||
dt = dates.parse_date(str(val))
|
||||
if dt:
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user