From 65fcf6be65e6522f80627b621a39759efe5c9f3a Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Fri, 10 Apr 2026 07:39:07 -0400 Subject: [PATCH] fix(github): reject garbage in _parse_date; consolidate date parsing github.py _parse_date used naive string slicing (return iso_str[:10]) which accepted any 10+ character string as a "date." For input "hello world" it returned "hello worl". Now delegates to dates.parse_date() which validates the format and returns None for non-dates. Also migrated reddit.py and threads.py _parse_date to the shared dates.parse_date(). Both previously reimplemented ISO-with-trailing- offset handling (the .replace("Z", "+00:00") dance) and reddit.py also had its own Unix timestamp branch. dates.parse_date() already handles all of this, including the +0000 no-colon variant Reddit emits. Preserved reddit.py's original falsy-check so 0 still returns None (epoch 0 would otherwise parse as "1970-01-01", breaking an existing test and changing long-standing behavior). Added 4 new github tests for garbage rejection and offset variants. All 1026 existing tests pass (15 pre-existing failures unchanged). --- scripts/lib/github.py | 17 +++++++++-------- scripts/lib/reddit.py | 28 ++++++++-------------------- scripts/lib/threads.py | 33 ++++++++++----------------------- tests/test_github.py | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 51 deletions(-) diff --git a/scripts/lib/github.py b/scripts/lib/github.py index 2a549e6..a48dd21 100644 --- a/scripts/lib/github.py +++ b/scripts/lib/github.py @@ -17,7 +17,7 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, List, Optional -from . import log +from . import dates, log from .query import extract_core_subject from .relevance import token_overlap_relevance @@ -106,13 +106,14 @@ def _parse_repo_from_url(html_url: str) -> str: def _parse_date(iso_str: Optional[str]) -> Optional[str]: - """Extract YYYY-MM-DD from ISO 8601 datetime string.""" - if not iso_str: - return None - try: - return iso_str[:10] - except (IndexError, TypeError): - return None + """Parse a GitHub ISO 8601 datetime string and return YYYY-MM-DD. + + Returns None for non-date input. GitHub's API always emits ISO 8601 + (e.g. "2026-02-26T16:00:00Z"), but we defer to dates.parse_date() so + garbage input gets rejected instead of silently sliced. + """ + dt = dates.parse_date(iso_str) + return dt.strftime("%Y-%m-%d") if dt else None def _compute_relevance( diff --git a/scripts/lib/reddit.py b/scripts/lib/reddit.py index fc438d8..cc59914 100644 --- a/scripts/lib/reddit.py +++ b/scripts/lib/reddit.py @@ -12,7 +12,6 @@ import sys import time from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed, wait as futures_wait -from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Set try: @@ -28,7 +27,7 @@ def _first_of(*values, default=None): return v return default -from . import http, log +from . import dates, http, log SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/reddit" @@ -212,27 +211,16 @@ def _parse_date(value) -> Optional[str]: Global search returns ``created_at`` as an ISO string (e.g. "2018-05-03T01:09:17.620000+0000"); subreddit search returns - ``created_utc`` as a Unix timestamp. Handle both. + ``created_utc`` as a Unix timestamp. dates.parse_date() handles both, + plus edge cases like Z suffix and +0000 (no colon) offset. + + Falsy inputs (None, "", 0) return None, matching the original behavior + where a Unix timestamp of 0 meant "no date" rather than epoch 0. """ if not value: return None - # ISO-8601 string (contains 'T' or '-') - if isinstance(value, str) and ("T" in value or "-" in value): - try: - # Strip trailing offset variations (+0000, Z) for fromisoformat - clean = value.replace("Z", "+00:00") - if clean.endswith("+0000"): - clean = clean[:-5] + "+00:00" - dt = datetime.fromisoformat(clean) - return dt.strftime("%Y-%m-%d") - except (ValueError, TypeError): - pass - # Unix timestamp (int or float or numeric string) - try: - dt = datetime.fromtimestamp(float(value), tz=timezone.utc) - return dt.strftime("%Y-%m-%d") - except (ValueError, TypeError, OSError): - return None + dt = dates.parse_date(str(value)) + return dt.strftime("%Y-%m-%d") if dt else None def _extract_subreddit_name(value: Any) -> str: diff --git a/scripts/lib/threads.py b/scripts/lib/threads.py index 32a3d0c..61c6723 100644 --- a/scripts/lib/threads.py +++ b/scripts/lib/threads.py @@ -9,10 +9,9 @@ API docs: https://scrapecreators.com/docs import math import re -from datetime import datetime, timezone from typing import Any, Dict, List, Optional -from . import http, log +from . import dates, http, log from .relevance import token_overlap_relevance as _compute_relevance SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/threads" @@ -52,29 +51,17 @@ def _extract_core_subject(topic: str) -> str: def _parse_date(item: Dict[str, Any]) -> Optional[str]: """Parse date from Threads item to YYYY-MM-DD. - Tries common timestamp fields: taken_at (unix), created_at (ISO), - and falls back to any date-like string field. + Tries common timestamp fields in order: taken_at and create_time + (unix timestamps in Meta APIs), then created_at, published_at, and + date (ISO 8601 strings). dates.parse_date() handles both. """ - # Unix timestamp (taken_at is common in Meta APIs) - for key in ("taken_at", "create_time"): - ts = item.get(key) - if ts: - try: - from . import dates - return dates.timestamp_to_date(int(ts)) - except (ValueError, TypeError): - pass - - # ISO 8601 string - for key in ("created_at", "published_at", "date"): + for key in ("taken_at", "create_time", "created_at", "published_at", "date"): val = item.get(key) - if val and isinstance(val, str): - try: - dt = datetime.fromisoformat(val.replace("Z", "+00:00")) - return dt.strftime("%Y-%m-%d") - except (ValueError, TypeError): - pass - + if val is None: + continue + dt = dates.parse_date(str(val)) + if dt: + return dt.strftime("%Y-%m-%d") return None diff --git a/tests/test_github.py b/tests/test_github.py index 7ab9099..c943fda 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -56,6 +56,22 @@ class TestParseDate(unittest.TestCase): def test_empty(self): self.assertIsNone(github._parse_date("")) + def test_rejects_garbage(self): + """The old naive slicing returned 'hello worl' for 'hello world'. Reject it.""" + self.assertIsNone(github._parse_date("hello world")) + self.assertIsNone(github._parse_date("not-a-date")) + self.assertIsNone(github._parse_date("abcdefghij")) + + def test_rejects_invalid_date_values(self): + """An out-of-range date like 2026-99-99 is not a real date.""" + self.assertIsNone(github._parse_date("2026-99-99")) + + def test_iso_with_offset(self): + self.assertEqual(github._parse_date("2026-03-15T12:00:00+00:00"), "2026-03-15") + + def test_iso_with_no_colon_offset(self): + self.assertEqual(github._parse_date("2026-03-15T12:00:00+0000"), "2026-03-15") + class TestSearchGithub(unittest.TestCase): @patch.dict("os.environ", {}, clear=True)