perf: batch store_findings, dedup source_items in O(1), remove dead code (#206)
1. N+1 queries in store.store_findings()
The old loop ran one SELECT per finding to check existence, then one
INSERT or UPDATE. 100 findings cost 200 serial SQLite roundtrips.
Now: one batch SELECT with WHERE source_url IN (...) builds a lookup
dict, then executemany() handles all inserts and updates. Query count
stays constant regardless of batch size. Benchmark on 500 findings:
~30ms to ~20ms; gap widens on slower storage.
2. O(n^2) source_items dedup in fusion.weighted_rrf()
Merging an item into an existing candidate ran any(existing.source ==
... for existing in candidate.source_items), linearly scanning a list
that grew with each merge. At 40 candidates with 20 source_items each,
fusion went quadratic. Now tracks (source, item_id) tuples in a
per-candidate set for O(1) lookup. The source_items list itself is
unchanged since other code iterates it.
3. Dead code removal
- providers.GeminiClient.ground_search() and .url_context_json(): zero
callers. Deleted.
- render._top_comment_excerpt(): zero callers. Deleted.
- env.is_reddit_available(): one-line wrapper around get_reddit_source.
Callers can check get_reddit_source(config) is not None directly.
This commit is contained in:
Ilia Alshanetsky
@@ -346,61 +346,83 @@ def store_findings(
|
||||
findings: List[Dict[str, Any]],
|
||||
) -> Dict[str, int]:
|
||||
"""Store findings with URL-based dedup. Returns counts of new/updated."""
|
||||
# Collect findings that have a URL, preserving order.
|
||||
with_urls: List[tuple[str, Dict[str, Any]]] = []
|
||||
for f in findings:
|
||||
url = f.get("source_url") or f.get("url")
|
||||
if url:
|
||||
with_urls.append((url, f))
|
||||
|
||||
if not with_urls:
|
||||
conn = _connect()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE research_runs SET findings_new = 0, findings_updated = 0 WHERE id = ?",
|
||||
(run_id,),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
return {"new": 0, "updated": 0}
|
||||
|
||||
conn = _connect()
|
||||
new_count = 0
|
||||
updated_count = 0
|
||||
|
||||
try:
|
||||
for f in findings:
|
||||
url = f.get("source_url") or f.get("url")
|
||||
if not url:
|
||||
continue
|
||||
# Single batch SELECT to find existing findings by URL.
|
||||
urls = [url for url, _ in with_urls]
|
||||
placeholders = ",".join("?" for _ in urls)
|
||||
rows = conn.execute(
|
||||
f"SELECT id, source_url, engagement_score FROM findings WHERE source_url IN ({placeholders})",
|
||||
urls,
|
||||
).fetchall()
|
||||
existing_by_url = {row["source_url"]: row for row in rows}
|
||||
|
||||
existing = conn.execute(
|
||||
"SELECT id, engagement_score, sighting_count FROM findings WHERE source_url = ?",
|
||||
(url,),
|
||||
).fetchone()
|
||||
update_rows: List[tuple] = []
|
||||
insert_rows: List[tuple] = []
|
||||
|
||||
for url, f in with_urls:
|
||||
existing = existing_by_url.get(url)
|
||||
new_engagement = f.get("engagement_score", 0)
|
||||
if existing:
|
||||
# Update engagement and re-sighting info
|
||||
new_engagement = f.get("engagement_score", 0)
|
||||
conn.execute(
|
||||
"""UPDATE findings SET
|
||||
last_seen = datetime('now'),
|
||||
sighting_count = sighting_count + 1,
|
||||
engagement_score = ?,
|
||||
run_id = ?
|
||||
WHERE id = ?""",
|
||||
(
|
||||
max(new_engagement, existing["engagement_score"] or 0),
|
||||
run_id,
|
||||
existing["id"],
|
||||
),
|
||||
)
|
||||
updated_count += 1
|
||||
update_rows.append((
|
||||
max(new_engagement, existing["engagement_score"] or 0),
|
||||
run_id,
|
||||
existing["id"],
|
||||
))
|
||||
else:
|
||||
# New finding
|
||||
conn.execute(
|
||||
"""INSERT INTO findings
|
||||
(run_id, topic_id, source, source_url, source_title,
|
||||
author, content, summary, engagement_score, relevance_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
run_id,
|
||||
topic_id,
|
||||
f.get("source", "unknown"),
|
||||
url,
|
||||
f.get("source_title") or f.get("title", ""),
|
||||
f.get("author", ""),
|
||||
f.get("content") or f.get("text", ""),
|
||||
f.get("summary", ""),
|
||||
f.get("engagement_score", 0),
|
||||
f.get("relevance_score", 0),
|
||||
),
|
||||
)
|
||||
new_count += 1
|
||||
insert_rows.append((
|
||||
run_id,
|
||||
topic_id,
|
||||
f.get("source", "unknown"),
|
||||
url,
|
||||
f.get("source_title") or f.get("title", ""),
|
||||
f.get("author", ""),
|
||||
f.get("content") or f.get("text", ""),
|
||||
f.get("summary", ""),
|
||||
new_engagement,
|
||||
f.get("relevance_score", 0),
|
||||
))
|
||||
|
||||
# Update run stats
|
||||
if update_rows:
|
||||
conn.executemany(
|
||||
"""UPDATE findings SET
|
||||
last_seen = datetime('now'),
|
||||
sighting_count = sighting_count + 1,
|
||||
engagement_score = ?,
|
||||
run_id = ?
|
||||
WHERE id = ?""",
|
||||
update_rows,
|
||||
)
|
||||
if insert_rows:
|
||||
conn.executemany(
|
||||
"""INSERT INTO findings
|
||||
(run_id, topic_id, source, source_url, source_title,
|
||||
author, content, summary, engagement_score, relevance_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
insert_rows,
|
||||
)
|
||||
|
||||
new_count = len(insert_rows)
|
||||
updated_count = len(update_rows)
|
||||
conn.execute(
|
||||
"UPDATE research_runs SET findings_new = ?, findings_updated = ? WHERE id = ?",
|
||||
(new_count, updated_count, run_id),
|
||||
|
||||
Reference in New Issue
Block a user