Matt Van Horn
32992834ee
- Xiaohongshu search via local MCP service (opt-in, zero impact if service not running) - Reddit public JSON fallback (works with zero API keys) - Reddit priority: ScrapeCreators -> OpenAI -> public fallback - Updated env.py: Reddit always available via public fallback Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
37 lines
1.2 KiB
JavaScript
37 lines
1.2 KiB
JavaScript
export async function paginateCursor(opts) {
|
|
const { maxPages, pageDelayMs = 1000 } = opts;
|
|
const seen = new Set();
|
|
const items = [];
|
|
let cursor = opts.cursor;
|
|
let pagesFetched = 0;
|
|
while (true) {
|
|
if (pagesFetched > 0 && pageDelayMs > 0) {
|
|
await opts.sleep(pageDelayMs);
|
|
}
|
|
const page = await opts.fetchPage(cursor);
|
|
if (!page.success) {
|
|
if (items.length > 0) {
|
|
return { success: false, error: page.error, items, nextCursor: cursor };
|
|
}
|
|
return page;
|
|
}
|
|
pagesFetched += 1;
|
|
for (const item of page.items) {
|
|
const key = opts.getKey(item);
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
seen.add(key);
|
|
items.push(item);
|
|
}
|
|
const pageCursor = page.cursor;
|
|
if (!pageCursor || pageCursor === cursor) {
|
|
return { success: true, items, nextCursor: undefined };
|
|
}
|
|
if (maxPages !== undefined && pagesFetched >= maxPages) {
|
|
return { success: true, items, nextCursor: pageCursor };
|
|
}
|
|
cursor = pageCursor;
|
|
}
|
|
}
|
|
//# sourceMappingURL=paginate-cursor.js.map
|