Matt Van Horn
0a9ff16dfc
v3 rewrites the search engine from the ground up: - Intelligent pre-research: resolves X handles, GitHub repos, subreddits, TikTok hashtags, and YouTube channels before searching - GitHub person-mode: PR velocity, top repos by stars, release notes - GitHub project-mode: live star counts, README, releases, top issues - ELI5 mode: plain language synthesis, no jargon - 13+ sources: Reddit, X, YouTube, TikTok, Instagram, HN, Polymarket, GitHub, Threads, Pinterest, Perplexity, Bluesky, Web - Free Reddit comments via public JSON (no API key needed) - Fun judge v2: humor scoring baked into narrative - Cookie consent before browser scanning - 10,000 free ScrapeCreators calls - 1,012 tests Thank you to the community contributors whose issues and PRs shaped v3: @uppinote20 (#143), @zerone0x (#134, #136), @thinkun (#116), @thomasmktong (#124), @fanispoulinakisai-boop (#100), @pejmanjohn (#78), @zl190 (#115), @hnshah (#84, #85, #86) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
51 lines
1.4 KiB
Python
51 lines
1.4 KiB
Python
"""Best-window extraction for rerankable evidence snippets."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from . import relevance, schema
|
|
|
|
|
|
def _truncate_words(text: str, max_words: int) -> str:
|
|
words = text.split()
|
|
if len(words) <= max_words:
|
|
return text.strip()
|
|
return " ".join(words[:max_words]).strip() + "..."
|
|
|
|
|
|
def _windows(words: list[str], size: int, overlap: int) -> list[str]:
|
|
if not words:
|
|
return []
|
|
if len(words) <= size:
|
|
return [" ".join(words)]
|
|
step = max(1, size - overlap)
|
|
return [
|
|
" ".join(words[start:start + size])
|
|
for start in range(0, len(words), step)
|
|
]
|
|
|
|
|
|
def extract_best_snippet(
|
|
item: schema.SourceItem,
|
|
ranking_query: str,
|
|
max_words: int = 120,
|
|
) -> str:
|
|
"""Prefer existing snippets, else extract the best matching evidence window."""
|
|
preferred = item.snippet.strip()
|
|
if preferred:
|
|
return _truncate_words(preferred, max_words)
|
|
|
|
body = item.body.strip()
|
|
if not body:
|
|
return _truncate_words(item.title, max_words)
|
|
|
|
words = body.split()
|
|
candidates = _windows(words, size=min(max_words, 110), overlap=30)
|
|
if not candidates:
|
|
return _truncate_words(body, max_words)
|
|
|
|
best = max(
|
|
candidates,
|
|
key=lambda candidate: relevance.token_overlap_relevance(ranking_query, candidate),
|
|
)
|
|
return _truncate_words(best, max_words)
|