diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 189203bd..1ba00c66 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -210,6 +210,16 @@ EOD } } + private function getFullText($id) + { + $url = sprintf( + 'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en', + $id + ); + + return json_decode(getContents($url), false); + } + public function collectData() { // $data will contain an array of all found tweets (unfiltered) @@ -220,13 +230,11 @@ EOD $tweets = []; // Get authentication information - + $cache = RssBridge::getCache(); + $api = new TwitterClient($cache); // Try to get all tweets switch ($this->queriedContext) { case 'By username': - $cache = RssBridge::getCache(); - $api = new TwitterClient($cache); - $screenName = $this->getInput('u'); $screenName = trim($screenName); $screenName = ltrim($screenName, '@'); @@ -238,35 +246,45 @@ EOD case 'By keyword or hashtag': // Does not work with the recent twitter changes $params = [ - 'q' => urlencode($this->getInput('q')), - 'tweet_mode' => 'extended', - 'tweet_search_mode' => 'live', + 'q' => urlencode($this->getInput('q')), + 'tweet_mode' => 'extended', + 'tweet_search_mode' => 'live', ]; - $data = $this->makeApiCall('/1.1/search/tweets.json', $params)->statuses; + $tweets = $api->search($params)->statuses; + $data = (object) [ + 'tweets' => $tweets + ]; break; case 'By list': // Does not work with the recent twitter changes - $params = [ - 'slug' => strtolower($this->getInput('list')), - 'owner_screen_name' => strtolower($this->getInput('user')), - 'tweet_mode' => 'extended', + // $params = [ + // 'slug' => strtolower($this->getInput('list')), + // 'owner_screen_name' => strtolower($this->getInput('user')), + // 'tweet_mode' => 'extended', + // ]; + $query = [ + 'screenName' => strtolower($this->getInput('user')), + 'listSlug' => strtolower($this->getInput('list')) ]; - $data = $this->makeApiCall('/1.1/lists/statuses.json', $params); + $data = $api->fetchListTweets($query, $this->queriedContext); break; case 'By list ID': // Does not work with the recent twitter changes - $params = [ - 'list_id' => $this->getInput('listid'), - 'tweet_mode' => 'extended', + // $params = [ + // 'list_id' => $this->getInput('listid'), + // 'tweet_mode' => 'extended', + // ]; + + $query = [ + 'listId' => $this->getInput('listid') ]; - $data = $this->makeApiCall('/1.1/lists/statuses.json', $params); + $data = $api->fetchListTweets($query, $this->queriedContext); break; - default: returnServerError('Invalid query context !'); } @@ -314,6 +332,7 @@ EOD $this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null; } + $i = 0; foreach ($tweets as $tweet) { // Skip own Retweets... if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) { @@ -325,14 +344,6 @@ EOD continue; } - switch ($this->queriedContext) { - case 'By username': - if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) { - continue 2; - } - break; - } - $item = []; $realtweet = $tweet; @@ -341,11 +352,40 @@ EOD $realtweet = $tweet->retweeted_status; } - $item['username'] = $data->user_info->legacy->screen_name; - $item['fullname'] = $data->user_info->legacy->name; - $item['avatar'] = $data->user_info->legacy->profile_image_url_https; + if (isset($realtweet->truncated) && $realtweet->truncated) { + try { + $realtweet = $this->getFullText($realtweet->id_str); + } catch (HttpException $e) { + $realtweet = $tweet; + } + } + + switch ($this->queriedContext) { + case 'By username': + if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) { + continue 2; + } + $item['username'] = $data->user_info->legacy->screen_name; + $item['fullname'] = $data->user_info->legacy->name; + $item['avatar'] = $data->user_info->legacy->profile_image_url_https; + $item['id'] = $realtweet->id_str; + break; + case 'By list': + case 'By list ID': + $item['username'] = $data->userIds[$i]->legacy->screen_name; + $item['fullname'] = $data->userIds[$i]->legacy->name; + $item['avatar'] = $data->userIds[$i]->legacy->profile_image_url_https; + $item['id'] = $realtweet->conversation_id_str; + break; + case 'By keyword or hashtag': + $item['username'] = $realtweet->user->screen_name; + $item['fullname'] = $realtweet->user->name; + $item['avatar'] = $realtweet->user->profile_image_url_https; + $item['id'] = $realtweet->id_str; + break; + } + $item['timestamp'] = $realtweet->created_at; - $item['id'] = $realtweet->id_str; $item['uri'] = self::URI . $item['username'] . '/status/' . $item['id']; $item['author'] = (isset($tweet->retweeted_status) ? 'RT: ' : '') . $item['fullname'] @@ -353,7 +393,11 @@ EOD . $item['username'] . ')'; // Convert plain text URLs into HTML hyperlinks - $fulltext = $realtweet->full_text; + if (isset($realtweet->full_text)) { + $fulltext = $realtweet->full_text; + } else { + $fulltext = $realtweet->text; + } $cleanedTweet = $fulltext; $foundUrls = false; @@ -385,7 +429,7 @@ EOD if ($foundUrls === false) { // fallback to regex'es $reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/'; - if (preg_match($reg_ex, $realtweet->full_text, $url)) { + if (preg_match($reg_ex, $fulltext, $url)) { $cleanedTweet = preg_replace( $reg_ex, "{$url[0]} ", @@ -410,10 +454,17 @@ EOD EOD; } + $medias = []; + if (isset($realtweet->extended_entities->media)) { + $medias = $realtweet->extended_entities->media; + } else if (isset($realtweet->mediaDetails)) { + $medias = $realtweet->mediaDetails; + } + // Get images $media_html = ''; - if (isset($realtweet->extended_entities->media) && !$this->getInput('noimg')) { - foreach ($realtweet->extended_entities->media as $media) { + if (!$this->getInput('noimg')) { + foreach ($medias as $media) { switch ($media->type) { case 'photo': $image = $media->media_url_https . '?name=orig'; @@ -496,6 +547,7 @@ EOD; EOD; // put out + $i++; $this->items[] = $item; } diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 341610a4..92c797d6 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -20,6 +20,88 @@ class TwitterClient $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; } + private function extractTweetAndUsersFromGraphQL($timeline) + { + if (isset($timeline->data->user)) { + $result = $timeline->data->user->result; + $instructions = $result->timeline_v2->timeline->instructions; + } else { + $result = $timeline->data->list->timeline_response; + $instructions = $result->timeline->instructions; + } + if (isset($result->__typename) && $result->__typename === 'UserUnavailable') { + throw new \Exception('UserUnavailable'); + } + $instructionTypes = [ + 'TimelineAddEntries', + 'TimelineClearCache', + 'TimelinePinEntry', // unclear purpose, maybe pinned tweet? + ]; + if (!isset($instructions[1]) && isset($timeline->data->user)) { + throw new \Exception('The account exists but has not tweeted yet?'); + } + + $entries = null; + foreach ($instructions as $instruction) { + $instructionType = ''; + if (isset($instruction->type)) { + $instructionType = $instruction->type; + } else { + $instructionType = $instruction->__typename; + } + + if ($instructionType === 'TimelineAddEntries') { + $entries = $instruction->entries; + break; + } + } + if (!$entries) { + throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type')))); + } + + $tweets = []; + $userIds = []; + foreach ($entries as $entry) { + $entryType = ''; + + if (isset($entry->content->entryType)) { + $entryType = $entry->content->entryType; + } else { + $entryType = $entry->content->__typename; + } + + if ($entryType !== 'TimelineTimelineItem') { + continue; + } + + if (isset($timeline->data->user)) { + if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { + continue; + } + $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; + + $userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result; + } else { + if (!isset($entry->content->content->tweetResult->result->legacy)) { + continue; + } + $tweets[] = $entry->content->content->tweetResult->result->legacy; + + $userIds[] = $entry->content->content->tweetResult->result->core->user_result->result; + } + } + + return (object) [ + 'userIds' => $userIds, + 'tweets' => $tweets, + ]; + } + + private function extractTweetFromSearch($searchResult) + { + return $searchResult->statuses; + } + public function fetchUserTweets(string $screenName): \stdClass { $this->fetchGuestToken(); @@ -36,58 +118,66 @@ class TwitterClient } try { - $timeline = $this->fetchTimeline($userInfo->rest_id); + $timeline = $this->fetchTimelineUsingSearch($screenName); } catch (HttpException $e) { if ($e->getCode() === 403) { $this->data['guest_token'] = null; $this->fetchGuestToken(); - $timeline = $this->fetchTimeline($userInfo->rest_id); + $timeline = $this->fetchTimelineUsingSearch($screenName); } else { throw $e; } } - $result = $timeline->data->user->result; - if ($result->__typename === 'UserUnavailable') { - throw new \Exception('UserUnavailable'); - } - $instructionTypes = [ - 'TimelineAddEntries', - 'TimelineClearCache', - 'TimelinePinEntry', // unclear purpose, maybe pinned tweet? - ]; - $instructions = $result->timeline_v2->timeline->instructions; - if (!isset($instructions[1])) { - throw new \Exception('The account exists but has not tweeted yet?'); - } + $tweets = $this->extractTweetFromSearch($timeline); - $entries = null; - foreach ($instructions as $instruction) { - if ($instruction->type === 'TimelineAddEntries') { - $entries = $instruction->entries; - break; - } - } - if (!$entries) { - throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type')))); - } - - $tweets = []; - foreach ($entries as $entry) { - if ($entry->content->entryType !== 'TimelineTimelineItem') { - continue; - } - if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { - continue; - } - $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; - } return (object) [ 'user_info' => $userInfo, 'tweets' => $tweets, ]; } + public function fetchListTweets($query, $operation = '') + { + $id = ''; + $this->fetchGuestToken(); + if ($operation == 'By list') { + try { + $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']); + $id = $listInfo->id_str; + } catch (HttpException $e) { + if ($e->getCode() === 403) { + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']); + $id = $listInfo->id_str; + } else { + throw $e; + } + } + } else if ($operation == 'By list ID') { + $id = $query['listId']; + } else { + throw new \Exception('Unknown operation to make list tweets'); + } + + try { + $timeline = $this->fetchListTimeline($id); + } catch (HttpException $e) { + if ($e->getCode() === 403) { + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $timeline = $this->fetchListTimeline($id); + } else { + throw $e; + } + } + + $data = $this->extractTweetAndUsersFromGraphQL($timeline); + + return $data; + } + private function fetchGuestToken(): void { if (isset($this->data['guest_token'])) { @@ -173,6 +263,158 @@ class TwitterClient return $response; } + private function fetchTimelineUsingSearch($screenName) + { + $params = [ + 'q' => 'from:' . $screenName, + 'modules' => 'status', + 'result_type' => 'recent' + ]; + $response = $this->search($params); + return $response; + } + + public function search($queryParam) + { + $url = sprintf( + 'https://api.twitter.com/1.1/search/tweets.json?%s', + http_build_query($queryParam) + ); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + return $response; + } + + private function fetchListInfoBySlug($screenName, $listSlug) + { + if (isset($this->data[$screenName . '-' . $listSlug])) { + return $this->data[$screenName . '-' . $listSlug]; + } + + $features = [ + 'android_graphql_skip_api_media_color_palette' => false, + 'blue_business_profile_image_shape_enabled' => false, + 'creator_subscriptions_subscription_count_enabled' => false, + 'creator_subscriptions_tweet_preview_api_enabled' => true, + 'freedom_of_speech_not_reach_fetch_enabled' => false, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false, + 'hidden_profile_likes_enabled' => false, + 'highlights_tweets_tab_ui_enabled' => false, + 'interactive_text_enabled' => false, + 'longform_notetweets_consumption_enabled' => true, + 'longform_notetweets_inline_media_enabled' => false, + 'longform_notetweets_richtext_consumption_enabled' => true, + 'longform_notetweets_rich_text_read_enabled' => false, + 'responsive_web_edit_tweet_api_enabled' => false, + 'responsive_web_enhance_cards_enabled' => false, + 'responsive_web_graphql_exclude_directive_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'responsive_web_graphql_timeline_navigation_enabled' => false, + 'responsive_web_media_download_video_enabled' => false, + 'responsive_web_text_conversations_enabled' => false, + 'responsive_web_twitter_article_tweet_consumption_enabled' => false, + 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, + 'rweb_lists_timeline_redesign_enabled' => true, + 'spaces_2022_h2_clipping' => true, + 'spaces_2022_h2_spaces_communities' => true, + 'standardized_nudges_misinfo' => false, + 'subscriptions_verification_info_enabled' => true, + 'subscriptions_verification_info_reason_enabled' => true, + 'subscriptions_verification_info_verified_since_enabled' => true, + 'super_follow_badge_privacy_enabled' => false, + 'super_follow_exclusive_tweet_notifications_enabled' => false, + 'super_follow_tweet_api_enabled' => false, + 'super_follow_user_api_enabled' => false, + 'tweet_awards_web_tipping_enabled' => false, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, + 'tweetypie_unmention_optimization_enabled' => false, + 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false, + 'verified_phone_label_enabled' => false, + 'vibe_api_enabled' => false, + 'view_counts_everywhere_api_enabled' => false + ]; + $variables = [ + 'screenName' => $screenName, + 'listSlug' => $listSlug + ]; + + $url = sprintf( + 'https://twitter.com/i/api/graphql/-kmqNvm5Y-cVrfvBy6docg/ListBySlug?variables=%s&features=%s', + urlencode(json_encode($variables)), + urlencode(json_encode($features)) + ); + + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + if (isset($response->errors)) { + // Grab the first error message + throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); + } + $listInfo = $response->data->user_by_screen_name->list; + $this->data[$screenName . '-' . $listSlug] = $listInfo; + + $this->cache->setScope('twitter'); + $this->cache->setKey(['cache']); + $this->cache->saveData($this->data); + return $listInfo; + } + + private function fetchListTimeline($listId) + { + $features = [ + 'android_graphql_skip_api_media_color_palette' => false, + 'blue_business_profile_image_shape_enabled' => false, + 'creator_subscriptions_subscription_count_enabled' => false, + 'creator_subscriptions_tweet_preview_api_enabled' => true, + 'freedom_of_speech_not_reach_fetch_enabled' => false, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false, + 'hidden_profile_likes_enabled' => false, + 'highlights_tweets_tab_ui_enabled' => false, + 'interactive_text_enabled' => false, + 'longform_notetweets_consumption_enabled' => true, + 'longform_notetweets_inline_media_enabled' => false, + 'longform_notetweets_richtext_consumption_enabled' => true, + 'longform_notetweets_rich_text_read_enabled' => false, + 'responsive_web_edit_tweet_api_enabled' => false, + 'responsive_web_enhance_cards_enabled' => false, + 'responsive_web_graphql_exclude_directive_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'responsive_web_graphql_timeline_navigation_enabled' => false, + 'responsive_web_media_download_video_enabled' => false, + 'responsive_web_text_conversations_enabled' => false, + 'responsive_web_twitter_article_tweet_consumption_enabled' => false, + 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, + 'rweb_lists_timeline_redesign_enabled' => true, + 'spaces_2022_h2_clipping' => true, + 'spaces_2022_h2_spaces_communities' => true, + 'standardized_nudges_misinfo' => false, + 'subscriptions_verification_info_enabled' => true, + 'subscriptions_verification_info_reason_enabled' => true, + 'subscriptions_verification_info_verified_since_enabled' => true, + 'super_follow_badge_privacy_enabled' => false, + 'super_follow_exclusive_tweet_notifications_enabled' => false, + 'super_follow_tweet_api_enabled' => false, + 'super_follow_user_api_enabled' => false, + 'tweet_awards_web_tipping_enabled' => false, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, + 'tweetypie_unmention_optimization_enabled' => false, + 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false, + 'verified_phone_label_enabled' => false, + 'vibe_api_enabled' => false, + 'view_counts_everywhere_api_enabled' => false + ]; + $variables = [ + 'rest_id' => $listId, + 'count' => 20 + ]; + + $url = sprintf( + 'https://twitter.com/i/api/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s', + urlencode(json_encode($variables)), + urlencode(json_encode($features)) + ); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + return $response; + } + private function createHttpHeaders(): array { $headers = [