fix: Instagram 429 限速时 15 秒超时后自动 fallback 到 Jina
instaloader 遇到 429 会自动等 30 分钟重试导致卡住。 改为在 executor 里跑,15 秒超时后自动切 Jina Reader fallback。
This commit is contained in:
@@ -53,7 +53,10 @@ class InstagramChannel(Channel):
|
|||||||
|
|
||||||
async def _read_instaloader(self, url: str, config=None) -> ReadResult:
|
async def _read_instaloader(self, url: str, config=None) -> ReadResult:
|
||||||
"""Read Instagram content using instaloader Python API."""
|
"""Read Instagram content using instaloader Python API."""
|
||||||
try:
|
import asyncio
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def _sync_read():
|
||||||
import instaloader
|
import instaloader
|
||||||
L = instaloader.Instaloader(
|
L = instaloader.Instaloader(
|
||||||
download_pictures=False,
|
download_pictures=False,
|
||||||
@@ -63,6 +66,7 @@ class InstagramChannel(Channel):
|
|||||||
download_comments=False,
|
download_comments=False,
|
||||||
save_metadata=False,
|
save_metadata=False,
|
||||||
compress_json=False,
|
compress_json=False,
|
||||||
|
max_connection_attempts=1, # Don't retry on rate limit
|
||||||
)
|
)
|
||||||
|
|
||||||
# Try to load session if available
|
# Try to load session if available
|
||||||
@@ -74,26 +78,32 @@ class InstagramChannel(Channel):
|
|||||||
|
|
||||||
path = urlparse(url).path.strip("/")
|
path = urlparse(url).path.strip("/")
|
||||||
|
|
||||||
# Detect URL type
|
|
||||||
if "/p/" in url or "/reel/" in url:
|
if "/p/" in url or "/reel/" in url:
|
||||||
return await self._read_post(L, url, path)
|
return self._read_post_sync(L, url, path)
|
||||||
else:
|
else:
|
||||||
return await self._read_profile(L, url, path)
|
return self._read_profile_sync(L, url, path)
|
||||||
|
|
||||||
except ImportError:
|
try:
|
||||||
return await self._read_jina(url)
|
# Run with 15s timeout to avoid instaloader's 30-min retry
|
||||||
except Exception as e:
|
loop = asyncio.get_event_loop()
|
||||||
# Fallback to Jina on any error
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
||||||
|
result = await asyncio.wait_for(
|
||||||
|
loop.run_in_executor(pool, _sync_read),
|
||||||
|
timeout=15,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
except (asyncio.TimeoutError, Exception):
|
||||||
|
# Any error or timeout → Jina fallback
|
||||||
return await self._read_jina(url)
|
return await self._read_jina(url)
|
||||||
|
|
||||||
async def _read_post(self, L, url: str, path: str) -> ReadResult:
|
def _read_post_sync(self, L, url: str, path: str) -> ReadResult:
|
||||||
"""Read a single Instagram post."""
|
"""Read a single Instagram post (sync, runs in executor)."""
|
||||||
import instaloader
|
import instaloader
|
||||||
|
|
||||||
# Extract shortcode from URL
|
# Extract shortcode from URL
|
||||||
match = re.search(r"/(?:p|reel)/([A-Za-z0-9_-]+)", url)
|
match = re.search(r"/(?:p|reel)/([A-Za-z0-9_-]+)", url)
|
||||||
if not match:
|
if not match:
|
||||||
return await self._read_jina(url)
|
raise ValueError("Cannot extract shortcode from URL")
|
||||||
|
|
||||||
shortcode = match.group(1)
|
shortcode = match.group(1)
|
||||||
try:
|
try:
|
||||||
@@ -123,16 +133,16 @@ class InstagramChannel(Channel):
|
|||||||
extra={"likes": post.likes, "comments": post.comments},
|
extra={"likes": post.likes, "comments": post.comments},
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
return await self._read_jina(url)
|
raise # Let executor timeout handle fallback
|
||||||
|
|
||||||
async def _read_profile(self, L, url: str, path: str) -> ReadResult:
|
def _read_profile_sync(self, L, url: str, path: str) -> ReadResult:
|
||||||
"""Read an Instagram profile."""
|
"""Read an Instagram profile (sync, runs in executor)."""
|
||||||
import instaloader
|
import instaloader
|
||||||
|
|
||||||
# Extract username from path
|
# Extract username from path
|
||||||
username = path.split("/")[0] if path else ""
|
username = path.split("/")[0] if path else ""
|
||||||
if not username or username in ("p", "reel", "stories", "explore"):
|
if not username or username in ("p", "reel", "stories", "explore"):
|
||||||
return await self._read_jina(url)
|
raise ValueError("Cannot extract username from URL")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
profile = instaloader.Profile.from_username(L.context, username)
|
profile = instaloader.Profile.from_username(L.context, username)
|
||||||
@@ -175,7 +185,7 @@ class InstagramChannel(Channel):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
return await self._read_jina(url)
|
raise # Let executor timeout handle fallback
|
||||||
|
|
||||||
async def _read_jina(self, url: str) -> ReadResult:
|
async def _read_jina(self, url: str) -> ReadResult:
|
||||||
"""Fallback: use Jina Reader."""
|
"""Fallback: use Jina Reader."""
|
||||||
|
|||||||
Reference in New Issue
Block a user