perf: optimize dedup, parallelize handle searches and enrichment
The dedup hot path recomputed normalize_text() 4 times per comparison and recomputed item_text() on every inner-loop iteration. Pre-computing n-gram sets and token sets into a _PreparedText cache cuts dedup time by 6x (2.16s to 0.39s on 300 unique items). Bird handle searches spawned one Node process per handle sequentially. Now uses ThreadPoolExecutor so N handles run concurrently. Same pattern applied to YouTube comment enrichment (was serial, Reddit was already parallel) and the retry-thin-sources phase in the pipeline. Clustering now pre-computes candidate text and uses prepared_similarity for the O(n^2) grouping and MMR representative selection loops. Minor: _is_wsl() cached with lru_cache, Bundle.add_items() uses extend() instead of list concatenation. End-to-end: 5.2s -> 3.7s (29% faster) on a typical 4-source query.
This commit is contained in:
+14
-7
@@ -312,10 +312,9 @@ def search_handles(
|
||||
Returns:
|
||||
List of raw item dicts (same format as parse_bird_response output).
|
||||
"""
|
||||
all_items = []
|
||||
core_topic = _extract_core_subject(topic) if topic else None
|
||||
|
||||
for handle in handles:
|
||||
def _search_one_handle(handle: str) -> List[Dict[str, Any]]:
|
||||
handle = handle.lstrip("@")
|
||||
if core_topic:
|
||||
query = f"from:{handle} {core_topic} since:{from_date}"
|
||||
@@ -350,24 +349,32 @@ def search_handles(
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
_log(f"Handle search timed out for @{handle}")
|
||||
continue
|
||||
return []
|
||||
|
||||
if proc.returncode != 0:
|
||||
_log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
|
||||
continue
|
||||
return []
|
||||
|
||||
output = (stdout or "").strip()
|
||||
if not output:
|
||||
continue
|
||||
return []
|
||||
|
||||
response = json.loads(output)
|
||||
items = parse_bird_response(response, query=core_topic)
|
||||
all_items.extend(items)
|
||||
return parse_bird_response(response, query=core_topic)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
_log(f"Invalid JSON from handle search for @{handle}")
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
_log(f"Handle search error for @{handle}: {e}")
|
||||
return []
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
all_items: List[Dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=min(5, len(handles))) as executor:
|
||||
futures = {executor.submit(_search_one_handle, h): h for h in handles}
|
||||
for future in as_completed(futures):
|
||||
all_items.extend(future.result())
|
||||
|
||||
return all_items
|
||||
|
||||
|
||||
Reference in New Issue
Block a user