twitter-scraper icon indicating copy to clipboard operation
twitter-scraper copied to clipboard

Implementation of search of communities

Open witherBattler opened this issue 1 year ago • 0 comments

There should probably be a way to search for twitter communities. There doesn't seem to be any, but to make it easier to implement this functionality, here's two functions I wrote for fetching and searching communities:

function searchTwitterCommunities(query, count=3) {
  
  const urlParams = querystring.encode({variables: JSON.stringify({query})})
  
  return new Promise(async (resolve, reject) => {
    const response = await fetch(`https://x.com/i/api/graphql/daVUkhfHn7-Z8llpYVKJSw/CommunitiesSearchQuery?${urlParams}`, {
      headers: {
        cookie: `auth_token=${twitterAuthenticationHeaders.cookie.auth_token}; ct0=${twitterAuthenticationHeaders.cookie.ct0}`,
        "x-csrf-token": twitterAuthenticationHeaders["x-csrf-token"],
        authorization: "Bearer " + twitterAuthenticationHeaders.authorization
      }
    })
    const json = await response.json()
    let communities = json.data.communities_search_slice.items_results
    communities = communities.map(x => x.result)
    communities.length = count
    communities = communities.map(async community => {
      // add on to this returned object the keys: description and joinPolicy. need to use fetchTwitterCommunity for it 
      const fetchedCommunity = await fetchTwitterCommunity(community.rest_id);

      return {
        id: community.rest_id,
        name: community.name,
        members: community.member_count,
        topic: community.primary_community_topic.topic_name,
        nsfw: community.is_nsfw,
        picture: community.custom_banner_media ? community.custom_banner_media.media_info.original_img_url : community.default_banner_media.media_info.original_img_url,
        joinPolicy: fetchedCommunity.joinPolicy,
        description: fetchedCommunity.description
      }
    })
    communities = await Promise.all(communities)

    resolve(communities)
  })
  
}

async function fetchTwitterCommunity(communityId) {
  const urlParams = querystring.encode({
    variables: JSON.stringify({communityId: communityId}),
    features: JSON.stringify({"c9s_list_members_action_api_enabled":false,"c9s_superc9s_indication_enabled":false})
  })

  const response = await fetch(`https://x.com/i/api/graphql/YDYGxdoPEu0zNC2eWP_0MQ/CommunityQuery?${urlParams}`, {
    headers: {
      cookie: `auth_token=${twitterAuthenticationHeaders.cookie.auth_token}; ct0=${twitterAuthenticationHeaders.cookie.ct0}`,
      "x-csrf-token": twitterAuthenticationHeaders["x-csrf-token"],
      authorization: "Bearer " + twitterAuthenticationHeaders.authorization
    }
  })

  const json = await response.json()
  let community = json.data.communityResults.result

  return {
    id: community.rest_id,
    name: community.name,
    joinPolicy: community.join_policy, // this
    description: community.description // and this are only used
  }
  
}

witherBattler avatar Jul 29 '24 17:07 witherBattler