twitter-scraper
twitter-scraper copied to clipboard
Implementation of search of communities
There should probably be a way to search for twitter communities. There doesn't seem to be any, but to make it easier to implement this functionality, here's two functions I wrote for fetching and searching communities:
function searchTwitterCommunities(query, count=3) {
const urlParams = querystring.encode({variables: JSON.stringify({query})})
return new Promise(async (resolve, reject) => {
const response = await fetch(`https://x.com/i/api/graphql/daVUkhfHn7-Z8llpYVKJSw/CommunitiesSearchQuery?${urlParams}`, {
headers: {
cookie: `auth_token=${twitterAuthenticationHeaders.cookie.auth_token}; ct0=${twitterAuthenticationHeaders.cookie.ct0}`,
"x-csrf-token": twitterAuthenticationHeaders["x-csrf-token"],
authorization: "Bearer " + twitterAuthenticationHeaders.authorization
}
})
const json = await response.json()
let communities = json.data.communities_search_slice.items_results
communities = communities.map(x => x.result)
communities.length = count
communities = communities.map(async community => {
// add on to this returned object the keys: description and joinPolicy. need to use fetchTwitterCommunity for it
const fetchedCommunity = await fetchTwitterCommunity(community.rest_id);
return {
id: community.rest_id,
name: community.name,
members: community.member_count,
topic: community.primary_community_topic.topic_name,
nsfw: community.is_nsfw,
picture: community.custom_banner_media ? community.custom_banner_media.media_info.original_img_url : community.default_banner_media.media_info.original_img_url,
joinPolicy: fetchedCommunity.joinPolicy,
description: fetchedCommunity.description
}
})
communities = await Promise.all(communities)
resolve(communities)
})
}
async function fetchTwitterCommunity(communityId) {
const urlParams = querystring.encode({
variables: JSON.stringify({communityId: communityId}),
features: JSON.stringify({"c9s_list_members_action_api_enabled":false,"c9s_superc9s_indication_enabled":false})
})
const response = await fetch(`https://x.com/i/api/graphql/YDYGxdoPEu0zNC2eWP_0MQ/CommunityQuery?${urlParams}`, {
headers: {
cookie: `auth_token=${twitterAuthenticationHeaders.cookie.auth_token}; ct0=${twitterAuthenticationHeaders.cookie.ct0}`,
"x-csrf-token": twitterAuthenticationHeaders["x-csrf-token"],
authorization: "Bearer " + twitterAuthenticationHeaders.authorization
}
})
const json = await response.json()
let community = json.data.communityResults.result
return {
id: community.rest_id,
name: community.name,
joinPolicy: community.join_policy, // this
description: community.description // and this are only used
}
}