perf: optimize dedup, parallelize handle searches and enrichment

The dedup hot path recomputed normalize_text() 4 times per comparison and recomputed item_text() on every inner-loop iteration. Pre-computing n-gram sets and token sets into a _PreparedText cache cuts dedup time by 6x (2.16s to 0.39s on 300 unique items). Bird handle searches spawned one Node process per handle sequentially. Now uses ThreadPoolExecutor so N handles run concurrently. Same pattern applied to YouTube comment enrichment (was serial, Reddit was already parallel) and the retry-thin-sources phase in the pipeline. Clustering now pre-computes candidate text and uses prepared_similarity for the O(n^2) grouping and MMR representative selection loops. Minor: _is_wsl() cached with lru_cache, Bundle.add_items() uses extend() instead of list concatenation. End-to-end: 5.2s -> 3.7s (29% faster) on a typical 4-source query.
2026-04-09 19:00:30 -04:00
parent 252c8222f1
commit eef3547c37
7 changed files with 128 additions and 59 deletions
@@ -719,44 +719,49 @@ def _retry_thin_sources(
        weight=0.3,
    )

-    for source in thin_sources:
-        if source in rate_limited_sources:
-            continue
-        try:
-            raw_items, _artifact = _retrieve_stream(
-                topic=topic,
-                subquery=retry_subquery,
-                source=source,
-                config=config,
-                depth=depth,
-                date_range=date_range,
-                runtime=runtime,
-                mock=mock,
-                rate_limited_sources=rate_limited_sources,
-                rate_limit_lock=rate_limit_lock,
-                web_backend=web_backend,
-                raw_topic=topic,
-            )
-            normalized = _normalize_score_dedupe(
-                source,
-                raw_items,
-                from_date,
-                to_date,
-                freshness_mode=plan.freshness_mode,
-                ranking_query=retry_subquery.ranking_query,
-            )
-            normalized = normalized[:settings["per_stream_limit"]]
+    def _retry_one_source(source: str) -> tuple[str, list[schema.SourceItem]]:
+        raw_items, _artifact = _retrieve_stream(
+            topic=topic,
+            subquery=retry_subquery,
+            source=source,
+            config=config,
+            depth=depth,
+            date_range=date_range,
+            runtime=runtime,
+            mock=mock,
+            rate_limited_sources=rate_limited_sources,
+            rate_limit_lock=rate_limit_lock,
+            web_backend=web_backend,
+            raw_topic=topic,
+        )
+        normalized = _normalize_score_dedupe(
+            source,
+            raw_items,
+            from_date,
+            to_date,
+            freshness_mode=plan.freshness_mode,
+            ranking_query=retry_subquery.ranking_query,
+        )
+        return source, normalized[:settings["per_stream_limit"]]

-            existing_urls = {item.url for item in bundle.items_by_source.get(source, []) if item.url}
-            new_items = [item for item in normalized if item.url not in existing_urls]
+    retryable = [s for s in thin_sources if s not in rate_limited_sources]

-            if new_items:
-                bundle.items_by_source.setdefault(source, []).extend(new_items)
-                primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
-                existing = bundle.items_by_source_and_query.get((primary_label, source), [])
-                bundle.items_by_source_and_query[(primary_label, source)] = existing + new_items
-        except Exception as exc:
-            print(f"[Pipeline] Retry failed for {source}: {type(exc).__name__}: {exc}", file=sys.stderr)
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    with ThreadPoolExecutor(max_workers=min(4, len(retryable) or 1)) as executor:
+        futures = {executor.submit(_retry_one_source, s): s for s in retryable}
+        for future in as_completed(futures):
+            source = futures[future]
+            try:
+                source, normalized = future.result()
+                existing_urls = {item.url for item in bundle.items_by_source.get(source, []) if item.url}
+                new_items = [item for item in normalized if item.url not in existing_urls]
+
+                if new_items:
+                    bundle.items_by_source.setdefault(source, []).extend(new_items)
+                    primary_label = plan.subqueries[0].label if plan.subqueries else "primary"
+                    bundle.items_by_source_and_query.setdefault((primary_label, source), []).extend(new_items)
+            except Exception as exc:
+                print(f"[Pipeline] Retry failed for {source}: {type(exc).__name__}: {exc}", file=sys.stderr)


 def _retrieve_stream(