Extract relevance_filter, add Bluesky/TruthSocial type hint + test coverage
- Extract _relevance_filter from last30days.py closure to score.relevance_filter() for testability - Add BlueskyItem/TruthSocialItem to sort_items() type hint (was missing despite being in _ITEM_SOURCE_MAP) - Add tests: Bluesky/TruthSocial engagement scoring, sort_items mixed sources, relevance_filter behavior (threshold, minimum-result guarantee, missing attr), select_openai_model HTTP 401/403 error paths
This commit is contained in:
+8
-21
@@ -1830,27 +1830,14 @@ def main():
|
||||
deduped_web = websearch.dedupe_websearch(sorted_web) if sorted_web else []
|
||||
|
||||
# Post-retrieval relevance filter: drop low-relevance items per source
|
||||
# Only filter when there are enough items (>3) to avoid empty results
|
||||
def _relevance_filter(items, source_name, threshold=0.3):
|
||||
"""Filter items below relevance threshold with minimum-result guarantee."""
|
||||
if len(items) <= 3:
|
||||
return items
|
||||
passed = [i for i in items if getattr(i, 'relevance', 0.0) >= threshold]
|
||||
if not passed:
|
||||
# Keep top 3 by relevance if all filtered
|
||||
print(f"[{source_name} WARNING] All results below relevance {threshold}, keeping top 3", file=sys.stderr)
|
||||
by_rel = sorted(items, key=lambda x: getattr(x, 'relevance', 0.0), reverse=True)
|
||||
return by_rel[:3]
|
||||
return passed
|
||||
|
||||
deduped_reddit = _relevance_filter(deduped_reddit, "REDDIT")
|
||||
deduped_x = _relevance_filter(deduped_x, "X")
|
||||
deduped_youtube = _relevance_filter(deduped_youtube, "YOUTUBE")
|
||||
deduped_tiktok = _relevance_filter(deduped_tiktok, "TIKTOK")
|
||||
deduped_ig = _relevance_filter(deduped_ig, "INSTAGRAM")
|
||||
deduped_hn = _relevance_filter(deduped_hn, "HN")
|
||||
deduped_bsky = _relevance_filter(deduped_bsky, "BLUESKY")
|
||||
deduped_ts = _relevance_filter(deduped_ts, "TRUTHSOCIAL")
|
||||
deduped_reddit = score.relevance_filter(deduped_reddit, "REDDIT")
|
||||
deduped_x = score.relevance_filter(deduped_x, "X")
|
||||
deduped_youtube = score.relevance_filter(deduped_youtube, "YOUTUBE")
|
||||
deduped_tiktok = score.relevance_filter(deduped_tiktok, "TIKTOK")
|
||||
deduped_ig = score.relevance_filter(deduped_ig, "INSTAGRAM")
|
||||
deduped_hn = score.relevance_filter(deduped_hn, "HN")
|
||||
deduped_bsky = score.relevance_filter(deduped_bsky, "BLUESKY")
|
||||
deduped_ts = score.relevance_filter(deduped_ts, "TRUTHSOCIAL")
|
||||
|
||||
# Cross-source linking: annotate items that discuss the same story
|
||||
dedupe.cross_source_link(
|
||||
|
||||
Reference in New Issue
Block a user