Merge PR #48: feat: add Xiaohongshu source + Reddit public fallback
- Xiaohongshu search via local MCP service (opt-in, zero impact if service not running) - Reddit public JSON fallback (works with zero API keys) - Reddit priority: ScrapeCreators -> OpenAI -> public fallback - Updated env.py: Reddit always available via public fallback Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt Van Horn
+358
@@ -0,0 +1,358 @@
|
||||
import { SETTINGS_NAME_REGEX, SETTINGS_SCREEN_NAME_REGEX, SETTINGS_USER_ID_REGEX, TWITTER_API_BASE, } from './twitter-client-constants.js';
|
||||
import { buildFollowingFeatures } from './twitter-client-features.js';
|
||||
import { extractCursorFromInstructions, parseUsersFromInstructions } from './twitter-client-utils.js';
|
||||
export function withUsers(Base) {
|
||||
class TwitterClientUsers extends Base {
|
||||
// biome-ignore lint/complexity/noUselessConstructor lint/suspicious/noExplicitAny: TS mixin constructor requirement.
|
||||
constructor(...args) {
|
||||
super(...args);
|
||||
}
|
||||
async getFollowingQueryIds() {
|
||||
const primary = await this.getQueryId('Following');
|
||||
return Array.from(new Set([primary, 'BEkNpEt5pNETESoqMsTEGA']));
|
||||
}
|
||||
async getFollowersQueryIds() {
|
||||
const primary = await this.getQueryId('Followers');
|
||||
return Array.from(new Set([primary, 'kuFUYP9eV1FPoEy4N-pi7w']));
|
||||
}
|
||||
parseUsersFromRestResponse(users) {
|
||||
return (users ?? [])
|
||||
.map((u) => {
|
||||
const id = typeof u.id_str === 'string' ? u.id_str : typeof u.id === 'number' ? String(u.id) : null;
|
||||
const username = typeof u.screen_name === 'string' ? u.screen_name : null;
|
||||
if (!id || !username) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id,
|
||||
username,
|
||||
name: typeof u.name === 'string' && u.name.length > 0 ? u.name : username,
|
||||
description: typeof u.description === 'string' ? u.description : undefined,
|
||||
followersCount: typeof u.followers_count === 'number' ? u.followers_count : undefined,
|
||||
followingCount: typeof u.friends_count === 'number' ? u.friends_count : undefined,
|
||||
isBlueVerified: typeof u.verified === 'boolean' ? u.verified : undefined,
|
||||
profileImageUrl: typeof u.profile_image_url_https === 'string' ? u.profile_image_url_https : undefined,
|
||||
createdAt: typeof u.created_at === 'string' ? u.created_at : undefined,
|
||||
};
|
||||
})
|
||||
.filter((u) => u !== null);
|
||||
}
|
||||
async getFollowersViaRest(userId, count, cursor) {
|
||||
const params = new URLSearchParams({
|
||||
user_id: userId,
|
||||
count: String(count),
|
||||
skip_status: 'true',
|
||||
include_user_entities: 'false',
|
||||
});
|
||||
if (cursor) {
|
||||
params.set('cursor', cursor);
|
||||
}
|
||||
const urls = [
|
||||
`https://x.com/i/api/1.1/followers/list.json?${params.toString()}`,
|
||||
`https://api.twitter.com/1.1/followers/list.json?${params.toString()}`,
|
||||
];
|
||||
let lastError;
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
lastError = `HTTP ${response.status}: ${text.slice(0, 200)}`;
|
||||
continue;
|
||||
}
|
||||
const data = (await response.json());
|
||||
const users = this.parseUsersFromRestResponse(data.users);
|
||||
const nextCursor = data.next_cursor_str && data.next_cursor_str !== '0' ? data.next_cursor_str : undefined;
|
||||
return { success: true, users, nextCursor };
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
return { success: false, error: lastError ?? 'Unknown error fetching followers' };
|
||||
}
|
||||
async getFollowingViaRest(userId, count, cursor) {
|
||||
const params = new URLSearchParams({
|
||||
user_id: userId,
|
||||
count: String(count),
|
||||
skip_status: 'true',
|
||||
include_user_entities: 'false',
|
||||
});
|
||||
if (cursor) {
|
||||
params.set('cursor', cursor);
|
||||
}
|
||||
const urls = [
|
||||
`https://x.com/i/api/1.1/friends/list.json?${params.toString()}`,
|
||||
`https://api.twitter.com/1.1/friends/list.json?${params.toString()}`,
|
||||
];
|
||||
let lastError;
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
lastError = `HTTP ${response.status}: ${text.slice(0, 200)}`;
|
||||
continue;
|
||||
}
|
||||
const data = (await response.json());
|
||||
const users = this.parseUsersFromRestResponse(data.users);
|
||||
const nextCursor = data.next_cursor_str && data.next_cursor_str !== '0' ? data.next_cursor_str : undefined;
|
||||
return { success: true, users, nextCursor };
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
return { success: false, error: lastError ?? 'Unknown error fetching following' };
|
||||
}
|
||||
/**
|
||||
* Fetch the account associated with the current cookies
|
||||
*/
|
||||
async getCurrentUser() {
|
||||
const candidateUrls = [
|
||||
'https://x.com/i/api/account/settings.json',
|
||||
'https://api.twitter.com/1.1/account/settings.json',
|
||||
'https://x.com/i/api/account/verify_credentials.json?skip_status=true&include_entities=false',
|
||||
'https://api.twitter.com/1.1/account/verify_credentials.json?skip_status=true&include_entities=false',
|
||||
];
|
||||
let lastError;
|
||||
for (const url of candidateUrls) {
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
lastError = `HTTP ${response.status}: ${text.slice(0, 200)}`;
|
||||
continue;
|
||||
}
|
||||
// biome-ignore lint/suspicious/noExplicitAny: Twitter API response is dynamic here
|
||||
let data;
|
||||
try {
|
||||
data = await response.json();
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
continue;
|
||||
}
|
||||
const username = typeof data?.screen_name === 'string'
|
||||
? data.screen_name
|
||||
: typeof data?.user?.screen_name === 'string'
|
||||
? data.user.screen_name
|
||||
: null;
|
||||
const name = typeof data?.name === 'string'
|
||||
? data.name
|
||||
: typeof data?.user?.name === 'string'
|
||||
? data.user.name
|
||||
: (username ?? '');
|
||||
const userId = typeof data?.user_id === 'string'
|
||||
? data.user_id
|
||||
: typeof data?.user_id_str === 'string'
|
||||
? data.user_id_str
|
||||
: typeof data?.user?.id_str === 'string'
|
||||
? data.user.id_str
|
||||
: typeof data?.user?.id === 'string'
|
||||
? data.user.id
|
||||
: null;
|
||||
if (username && userId) {
|
||||
this.clientUserId = userId;
|
||||
return {
|
||||
success: true,
|
||||
user: {
|
||||
id: userId,
|
||||
username,
|
||||
name: name || username,
|
||||
},
|
||||
};
|
||||
}
|
||||
lastError = 'Could not determine current user from response';
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
// Fallback: scrape the authenticated settings page (HTML) for screen_name/user_id
|
||||
const profilePages = ['https://x.com/settings/account', 'https://twitter.com/settings/account'];
|
||||
for (const page of profilePages) {
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(page, {
|
||||
headers: {
|
||||
cookie: this.cookieHeader,
|
||||
'user-agent': this.userAgent,
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
lastError = `HTTP ${response.status} (settings page)`;
|
||||
continue;
|
||||
}
|
||||
const html = await response.text();
|
||||
const usernameMatch = SETTINGS_SCREEN_NAME_REGEX.exec(html);
|
||||
const idMatch = SETTINGS_USER_ID_REGEX.exec(html);
|
||||
const nameMatch = SETTINGS_NAME_REGEX.exec(html);
|
||||
const username = usernameMatch?.[1];
|
||||
const userId = idMatch?.[1];
|
||||
const name = nameMatch?.[1]?.replace(/\\"/g, '"');
|
||||
if (username && userId) {
|
||||
return {
|
||||
success: true,
|
||||
user: {
|
||||
id: userId,
|
||||
username,
|
||||
name: name || username,
|
||||
},
|
||||
};
|
||||
}
|
||||
lastError = 'Could not parse settings page for user info';
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
error: lastError ?? 'Unknown error fetching current user',
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get users that a user is following
|
||||
*/
|
||||
async getFollowing(userId, count = 20, cursor) {
|
||||
const variables = {
|
||||
userId,
|
||||
count,
|
||||
includePromotedContent: false,
|
||||
};
|
||||
if (cursor) {
|
||||
variables.cursor = cursor;
|
||||
}
|
||||
const features = buildFollowingFeatures();
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
const tryOnce = async () => {
|
||||
let lastError;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getFollowingQueryIds();
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/Following?${params.toString()}`;
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
if (response.status === 404) {
|
||||
had404 = true;
|
||||
lastError = `HTTP ${response.status}`;
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||
}
|
||||
const data = (await response.json());
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return { success: false, error: data.errors.map((e) => e.message).join(', '), had404 };
|
||||
}
|
||||
const instructions = data.data?.user?.result?.timeline?.timeline?.instructions;
|
||||
const users = parseUsersFromInstructions(instructions);
|
||||
const nextCursor = extractCursorFromInstructions(instructions);
|
||||
return { success: true, users, nextCursor, had404 };
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
return { success: false, error: lastError ?? 'Unknown error fetching following', had404 };
|
||||
};
|
||||
const { result, refreshed } = await this.withRefreshedQueryIdsOn404(tryOnce);
|
||||
if (result.success) {
|
||||
return { success: true, users: result.users, nextCursor: result.nextCursor };
|
||||
}
|
||||
if (refreshed) {
|
||||
// GraphQL Following can also return 404 (queryId churn / endpoint flakiness).
|
||||
// Fallback to the internal v1.1 REST endpoint used by the web client (cookie-auth; no dev API key).
|
||||
const restAttempt = await this.getFollowingViaRest(userId, count, cursor);
|
||||
if (restAttempt.success) {
|
||||
return restAttempt;
|
||||
}
|
||||
}
|
||||
return { success: false, error: result.error };
|
||||
}
|
||||
/**
|
||||
* Get users that follow a user
|
||||
*/
|
||||
async getFollowers(userId, count = 20, cursor) {
|
||||
const variables = {
|
||||
userId,
|
||||
count,
|
||||
includePromotedContent: false,
|
||||
};
|
||||
if (cursor) {
|
||||
variables.cursor = cursor;
|
||||
}
|
||||
const features = buildFollowingFeatures();
|
||||
const params = new URLSearchParams({
|
||||
variables: JSON.stringify(variables),
|
||||
features: JSON.stringify(features),
|
||||
});
|
||||
const tryOnce = async () => {
|
||||
let lastError;
|
||||
let had404 = false;
|
||||
const queryIds = await this.getFollowersQueryIds();
|
||||
for (const queryId of queryIds) {
|
||||
const url = `${TWITTER_API_BASE}/${queryId}/Followers?${params.toString()}`;
|
||||
try {
|
||||
const response = await this.fetchWithTimeout(url, {
|
||||
method: 'GET',
|
||||
headers: this.getHeaders(),
|
||||
});
|
||||
if (response.status === 404) {
|
||||
had404 = true;
|
||||
lastError = `HTTP ${response.status}`;
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
return { success: false, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
|
||||
}
|
||||
const data = (await response.json());
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
return { success: false, error: data.errors.map((e) => e.message).join(', '), had404 };
|
||||
}
|
||||
const instructions = data.data?.user?.result?.timeline?.timeline?.instructions;
|
||||
const users = parseUsersFromInstructions(instructions);
|
||||
const nextCursor = extractCursorFromInstructions(instructions);
|
||||
return { success: true, users, nextCursor, had404 };
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
return { success: false, error: lastError ?? 'Unknown error fetching followers', had404 };
|
||||
};
|
||||
const { result, refreshed } = await this.withRefreshedQueryIdsOn404(tryOnce);
|
||||
if (result.success) {
|
||||
return { success: true, users: result.users, nextCursor: result.nextCursor };
|
||||
}
|
||||
if (refreshed) {
|
||||
// GraphQL Followers regularly returns 404 (queryId churn / endpoint flakiness).
|
||||
// Fallback to the internal v1.1 REST endpoint used by the web client (cookie-auth; no dev API key).
|
||||
const restAttempt = await this.getFollowersViaRest(userId, count, cursor);
|
||||
if (restAttempt.success) {
|
||||
return restAttempt;
|
||||
}
|
||||
}
|
||||
return { success: false, error: result.error };
|
||||
}
|
||||
}
|
||||
return TwitterClientUsers;
|
||||
}
|
||||
//# sourceMappingURL=twitter-client-users.js.map
|
||||
Reference in New Issue
Block a user