From 2acbf8a8698a22aa81c5f952a82dcd61ef1f3f96 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sat, 25 Apr 2026 17:17:17 -0400 Subject: [PATCH] perf: batch store_findings, dedup source_items in O(1), remove dead code (#206) 1. N+1 queries in store.store_findings() The old loop ran one SELECT per finding to check existence, then one INSERT or UPDATE. 100 findings cost 200 serial SQLite roundtrips. Now: one batch SELECT with WHERE source_url IN (...) builds a lookup dict, then executemany() handles all inserts and updates. Query count stays constant regardless of batch size. Benchmark on 500 findings: ~30ms to ~20ms; gap widens on slower storage. 2. O(n^2) source_items dedup in fusion.weighted_rrf() Merging an item into an existing candidate ran any(existing.source == ... for existing in candidate.source_items), linearly scanning a list that grew with each merge. At 40 candidates with 20 source_items each, fusion went quadratic. Now tracks (source, item_id) tuples in a per-candidate set for O(1) lookup. The source_items list itself is unchanged since other code iterates it. 3. Dead code removal - providers.GeminiClient.ground_search() and .url_context_json(): zero callers. Deleted. - render._top_comment_excerpt(): zero callers. Deleted. - env.is_reddit_available(): one-line wrapper around get_reddit_source. Callers can check get_reddit_source(config) is not None directly. --- skills/last30days/scripts/lib/env.py | 8 -- skills/last30days/scripts/lib/fusion.py | 7 +- skills/last30days/scripts/lib/providers.py | 7 -- skills/last30days/scripts/lib/render.py | 10 -- skills/last30days/scripts/store.py | 118 ++++++++++++--------- 5 files changed, 76 insertions(+), 74 deletions(-) diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py index f864d7e..e78f012 100644 --- a/skills/last30days/scripts/lib/env.py +++ b/skills/last30days/scripts/lib/env.py @@ -372,14 +372,6 @@ def config_exists() -> bool: return False -def is_reddit_available(config: dict[str, Any]) -> bool: - """Check if Reddit search is available. - - v3 uses ScrapeCreators only. - """ - return bool(config.get('SCRAPECREATORS_API_KEY')) - - def get_reddit_source(config: dict[str, Any]) -> str | None: """Determine which Reddit backend to use. diff --git a/skills/last30days/scripts/lib/fusion.py b/skills/last30days/scripts/lib/fusion.py index 7d902f0..80959d5 100644 --- a/skills/last30days/scripts/lib/fusion.py +++ b/skills/last30days/scripts/lib/fusion.py @@ -116,6 +116,8 @@ def weighted_rrf( """Fuse ranked lists into a single candidate pool.""" subqueries = {subquery.label: subquery for subquery in plan.subqueries} candidates: dict[str, schema.Candidate] = {} + # Track (source, item_id) pairs already attached to each candidate for O(1) dedup. + seen_source_items: dict[str, set[tuple[str, str]]] = {} for (label, source), items in streams.items(): subquery = subqueries[label] @@ -154,6 +156,7 @@ def weighted_rrf( ] }, ) + seen_source_items[key] = {(item.source, item.item_id)} continue candidate = candidates[key] @@ -179,7 +182,9 @@ def weighted_rrf( candidate.subquery_labels.append(label) if item.source not in candidate.sources: candidate.sources.append(item.source) - if not any(existing.source == item.source and existing.item_id == item.item_id for existing in candidate.source_items): + source_item_key = (item.source, item.item_id) + if source_item_key not in seen_source_items[key]: + seen_source_items[key].add(source_item_key) candidate.source_items.append(item) candidate.metadata.setdefault("provenance", []).append( { diff --git a/skills/last30days/scripts/lib/providers.py b/skills/last30days/scripts/lib/providers.py index 3516731..0a5e88c 100644 --- a/skills/last30days/scripts/lib/providers.py +++ b/skills/last30days/scripts/lib/providers.py @@ -93,13 +93,6 @@ class GeminiClient(ReasoningClient): ) return extract_gemini_text(payload) - def ground_search(self, model: str, prompt: str) -> dict[str, Any]: - return self._generate_content(model, prompt, tools=[{"google_search": {}}]) - - def url_context_json(self, model: str, prompt: str) -> dict[str, Any]: - return self.generate_json(model, prompt, tools=[{"url_context": {}}]) - - class OpenAIClient(ReasoningClient): name = "openai" diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.py index c2f336c..3614979 100644 --- a/skills/last30days/scripts/lib/render.py +++ b/skills/last30days/scripts/lib/render.py @@ -1505,16 +1505,6 @@ def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score return [c for c in comments if (c.get("score") or 0) >= min_score][:limit] -def _top_comment_excerpt(item: schema.SourceItem | None) -> str | None: - if not item: - return None - comments = item.metadata.get("top_comments") or [] - if not comments or not isinstance(comments[0], dict): - return None - top = comments[0] - return str(top.get("excerpt") or top.get("text") or "").strip() or None - - def _comment_insight(item: schema.SourceItem | None) -> str | None: if not item: return None diff --git a/skills/last30days/scripts/store.py b/skills/last30days/scripts/store.py index bb2687e..6cf77e6 100644 --- a/skills/last30days/scripts/store.py +++ b/skills/last30days/scripts/store.py @@ -346,61 +346,83 @@ def store_findings( findings: List[Dict[str, Any]], ) -> Dict[str, int]: """Store findings with URL-based dedup. Returns counts of new/updated.""" + # Collect findings that have a URL, preserving order. + with_urls: List[tuple[str, Dict[str, Any]]] = [] + for f in findings: + url = f.get("source_url") or f.get("url") + if url: + with_urls.append((url, f)) + + if not with_urls: + conn = _connect() + try: + conn.execute( + "UPDATE research_runs SET findings_new = 0, findings_updated = 0 WHERE id = ?", + (run_id,), + ) + conn.commit() + finally: + conn.close() + return {"new": 0, "updated": 0} + conn = _connect() - new_count = 0 - updated_count = 0 - try: - for f in findings: - url = f.get("source_url") or f.get("url") - if not url: - continue + # Single batch SELECT to find existing findings by URL. + urls = [url for url, _ in with_urls] + placeholders = ",".join("?" for _ in urls) + rows = conn.execute( + f"SELECT id, source_url, engagement_score FROM findings WHERE source_url IN ({placeholders})", + urls, + ).fetchall() + existing_by_url = {row["source_url"]: row for row in rows} - existing = conn.execute( - "SELECT id, engagement_score, sighting_count FROM findings WHERE source_url = ?", - (url,), - ).fetchone() + update_rows: List[tuple] = [] + insert_rows: List[tuple] = [] + for url, f in with_urls: + existing = existing_by_url.get(url) + new_engagement = f.get("engagement_score", 0) if existing: - # Update engagement and re-sighting info - new_engagement = f.get("engagement_score", 0) - conn.execute( - """UPDATE findings SET - last_seen = datetime('now'), - sighting_count = sighting_count + 1, - engagement_score = ?, - run_id = ? - WHERE id = ?""", - ( - max(new_engagement, existing["engagement_score"] or 0), - run_id, - existing["id"], - ), - ) - updated_count += 1 + update_rows.append(( + max(new_engagement, existing["engagement_score"] or 0), + run_id, + existing["id"], + )) else: - # New finding - conn.execute( - """INSERT INTO findings - (run_id, topic_id, source, source_url, source_title, - author, content, summary, engagement_score, relevance_score) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - run_id, - topic_id, - f.get("source", "unknown"), - url, - f.get("source_title") or f.get("title", ""), - f.get("author", ""), - f.get("content") or f.get("text", ""), - f.get("summary", ""), - f.get("engagement_score", 0), - f.get("relevance_score", 0), - ), - ) - new_count += 1 + insert_rows.append(( + run_id, + topic_id, + f.get("source", "unknown"), + url, + f.get("source_title") or f.get("title", ""), + f.get("author", ""), + f.get("content") or f.get("text", ""), + f.get("summary", ""), + new_engagement, + f.get("relevance_score", 0), + )) - # Update run stats + if update_rows: + conn.executemany( + """UPDATE findings SET + last_seen = datetime('now'), + sighting_count = sighting_count + 1, + engagement_score = ?, + run_id = ? + WHERE id = ?""", + update_rows, + ) + if insert_rows: + conn.executemany( + """INSERT INTO findings + (run_id, topic_id, source, source_url, source_title, + author, content, summary, engagement_score, relevance_score) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + insert_rows, + ) + + new_count = len(insert_rows) + updated_count = len(update_rows) conn.execute( "UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?", (new_count, updated_count, run_id),