refactor(normalize): extract _join_comment_excerpts helper (#283)
_normalize_reddit, _normalize_hackernews, and _normalize_github inlined the same 5-line comprehension to stringify and space-join the first 3 top_comments' excerpt field. Extract one helper, call it from all three. The comment field name varies per source (Reddit/GitHub use 'excerpt', HN uses 'text'), so it's passed as a parameter. Behavior unchanged.
This commit is contained in:
@@ -110,6 +110,19 @@ def _first_present(d: dict[str, Any], keys: tuple[str, ...], default: Any) -> An
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _join_comment_excerpts(
|
||||||
|
top_comments: list[Any],
|
||||||
|
key: str,
|
||||||
|
limit: int = 3,
|
||||||
|
) -> str:
|
||||||
|
"""Space-join the `key` field from the first `limit` dict-shaped comments."""
|
||||||
|
return " ".join(
|
||||||
|
str(comment.get(key) or "").strip()
|
||||||
|
for comment in top_comments[:limit]
|
||||||
|
if isinstance(comment, dict)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _domain_from_url(url: str) -> str | None:
|
def _domain_from_url(url: str) -> str | None:
|
||||||
if not url:
|
if not url:
|
||||||
return None
|
return None
|
||||||
@@ -169,11 +182,7 @@ def _normalize_reddit(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = " ".join(
|
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||||
str(comment.get("excerpt") or "").strip()
|
|
||||||
for comment in top_comments[:3]
|
|
||||||
if isinstance(comment, dict)
|
|
||||||
)
|
|
||||||
body = "\n".join(
|
body = "\n".join(
|
||||||
part
|
part
|
||||||
for part in [
|
for part in [
|
||||||
@@ -338,11 +347,7 @@ def _normalize_hackernews(
|
|||||||
to_date: str,
|
to_date: str,
|
||||||
) -> schema.SourceItem:
|
) -> schema.SourceItem:
|
||||||
top_comments = item.get("top_comments") or []
|
top_comments = item.get("top_comments") or []
|
||||||
comment_text = " ".join(
|
comment_text = _join_comment_excerpts(top_comments, "text")
|
||||||
str(comment.get("text") or "").strip()
|
|
||||||
for comment in top_comments[:3]
|
|
||||||
if isinstance(comment, dict)
|
|
||||||
)
|
|
||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)
|
||||||
return _source_item(
|
return _source_item(
|
||||||
@@ -441,11 +446,7 @@ def _normalize_github(
|
|||||||
title = str(item.get("title") or "").strip()
|
title = str(item.get("title") or "").strip()
|
||||||
snippet_text = str(item.get("snippet") or "").strip()
|
snippet_text = str(item.get("snippet") or "").strip()
|
||||||
top_comments = item.get("metadata", {}).get("top_comments") or []
|
top_comments = item.get("metadata", {}).get("top_comments") or []
|
||||||
comment_text = " ".join(
|
comment_text = _join_comment_excerpts(top_comments, "excerpt")
|
||||||
str(comment.get("excerpt") or "").strip()
|
|
||||||
for comment in top_comments[:3]
|
|
||||||
if isinstance(comment, dict)
|
|
||||||
)
|
|
||||||
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
body = "\n".join(part for part in [title, snippet_text, comment_text] if part)
|
||||||
metadata = item.get("metadata") or {}
|
metadata = item.get("metadata") or {}
|
||||||
return _source_item(
|
return _source_item(
|
||||||
|
|||||||
Reference in New Issue
Block a user