danolo
/
rss-bridge
mirror of https://github.com/danoloan10/rss-bridge
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TwitterBridge] Fix search, user, list ID (#3566) 

* Add ability to fetch user, list tweet

* Fix user, search, list ID although list still broke

* clear whitespace

* Revert CACHE_TIMEOUT

* clear whitespace, change single quote

* Clear PHP warning, add ability to get full-text if truncated

* Clear PHP warning

* clear warning

* clear whitespace

* Add check condition for mediaDetails.

* Add whitespace

* Add try catch exception for get full-text tweet

* clear warning

* clear warning
This commit is contained in:
csisoap 2023-07-26 03:36:41 +07:00 committed by GitHub
parent 2cc89b767c
commit 556bca58cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 364 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
bridges/TwitterBridge.php
View File

@ -210,6 +210,16 @@ EOD
}
}
private function getFullText($id)
{
$url = sprintf(
'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en',
$id
);
return json_decode(getContents($url), false);
}
public function collectData()
{
// $data will contain an array of all found tweets (unfiltered)
@ -220,13 +230,11 @@ EOD
$tweets = [];
// Get authentication information
$cache = RssBridge::getCache();
$api = new TwitterClient($cache);
// Try to get all tweets
switch ($this->queriedContext) {
case 'By username':
$cache = RssBridge::getCache();
$api = new TwitterClient($cache);
$screenName = $this->getInput('u');
$screenName = trim($screenName);
$screenName = ltrim($screenName, '@');
@ -243,30 +251,40 @@ EOD
'tweet_search_mode' => 'live',
];
$data = $this->makeApiCall('/1.1/search/tweets.json', $params)->statuses;
$tweets = $api->search($params)->statuses;
$data = (object) [
'tweets' => $tweets
];
break;
case 'By list':
// Does not work with the recent twitter changes
$params = [
'slug' => strtolower($this->getInput('list')),
'owner_screen_name' => strtolower($this->getInput('user')),
'tweet_mode' => 'extended',
// $params = [
// 'slug' => strtolower($this->getInput('list')),
// 'owner_screen_name' => strtolower($this->getInput('user')),
// 'tweet_mode' => 'extended',
// ];
$query = [
'screenName' => strtolower($this->getInput('user')),
'listSlug' => strtolower($this->getInput('list'))
];
$data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
$data = $api->fetchListTweets($query, $this->queriedContext);
break;
case 'By list ID':
// Does not work with the recent twitter changes
$params = [
'list_id' => $this->getInput('listid'),
'tweet_mode' => 'extended',
// $params = [
// 'list_id' => $this->getInput('listid'),
// 'tweet_mode' => 'extended',
// ];
$query = [
'listId' => $this->getInput('listid')
];
$data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
$data = $api->fetchListTweets($query, $this->queriedContext);
break;
default:
returnServerError('Invalid query context !');
}
@ -314,6 +332,7 @@ EOD
$this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null;
}
$i = 0;
foreach ($tweets as $tweet) {
// Skip own Retweets...
if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) {
@ -325,14 +344,6 @@ EOD
continue;
}
switch ($this->queriedContext) {
case 'By username':
if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) {
continue 2;
}
break;
}
$item = [];
$realtweet = $tweet;
@ -341,11 +352,40 @@ EOD
$realtweet = $tweet->retweeted_status;
}
if (isset($realtweet->truncated) && $realtweet->truncated) {
try {
$realtweet = $this->getFullText($realtweet->id_str);
} catch (HttpException $e) {
$realtweet = $tweet;
}
}
switch ($this->queriedContext) {
case 'By username':
if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) {
continue 2;
}
$item['username'] = $data->user_info->legacy->screen_name;
$item['fullname'] = $data->user_info->legacy->name;
$item['avatar'] = $data->user_info->legacy->profile_image_url_https;
$item['timestamp'] = $realtweet->created_at;
$item['id'] = $realtweet->id_str;
break;
case 'By list':
case 'By list ID':
$item['username'] = $data->userIds[$i]->legacy->screen_name;
$item['fullname'] = $data->userIds[$i]->legacy->name;
$item['avatar'] = $data->userIds[$i]->legacy->profile_image_url_https;
$item['id'] = $realtweet->conversation_id_str;
break;
case 'By keyword or hashtag':
$item['username'] = $realtweet->user->screen_name;
$item['fullname'] = $realtweet->user->name;
$item['avatar'] = $realtweet->user->profile_image_url_https;
$item['id'] = $realtweet->id_str;
break;
}
$item['timestamp'] = $realtweet->created_at;
$item['uri'] = self::URI . $item['username'] . '/status/' . $item['id'];
$item['author'] = (isset($tweet->retweeted_status) ? 'RT: ' : '')
. $item['fullname']
@ -353,7 +393,11 @@ EOD
. $item['username'] . ')';
// Convert plain text URLs into HTML hyperlinks
if (isset($realtweet->full_text)) {
$fulltext = $realtweet->full_text;
} else {
$fulltext = $realtweet->text;
}
$cleanedTweet = $fulltext;
$foundUrls = false;
@ -385,7 +429,7 @@ EOD
if ($foundUrls === false) {
// fallback to regex'es
$reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/';
if (preg_match($reg_ex, $realtweet->full_text, $url)) {
if (preg_match($reg_ex, $fulltext, $url)) {
$cleanedTweet = preg_replace(
$reg_ex,
"<a href='{$url[0]}' target='_blank'>{$url[0]}</a> ",
@ -410,10 +454,17 @@ EOD
EOD;
}
$medias = [];
if (isset($realtweet->extended_entities->media)) {
$medias = $realtweet->extended_entities->media;
} else if (isset($realtweet->mediaDetails)) {
$medias = $realtweet->mediaDetails;
}
// Get images
$media_html = '';
if (isset($realtweet->extended_entities->media) && !$this->getInput('noimg')) {
foreach ($realtweet->extended_entities->media as $media) {
if (!$this->getInput('noimg')) {
foreach ($medias as $media) {
switch ($media->type) {
case 'photo':
$image = $media->media_url_https . '?name=orig';
@ -496,6 +547,7 @@ EOD;
EOD;
// put out
$i++;
$this->items[] = $item;
}

314
lib/TwitterClient.php
View File

@ -20,6 +20,88 @@ class TwitterClient
$this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR';
}
private function extractTweetAndUsersFromGraphQL($timeline)
{
if (isset($timeline->data->user)) {
$result = $timeline->data->user->result;
$instructions = $result->timeline_v2->timeline->instructions;
} else {
$result = $timeline->data->list->timeline_response;
$instructions = $result->timeline->instructions;
}
if (isset($result->__typename) && $result->__typename === 'UserUnavailable') {
throw new \Exception('UserUnavailable');
}
$instructionTypes = [
'TimelineAddEntries',
'TimelineClearCache',
'TimelinePinEntry', // unclear purpose, maybe pinned tweet?
];
if (!isset($instructions[1]) && isset($timeline->data->user)) {
throw new \Exception('The account exists but has not tweeted yet?');
}
$entries = null;
foreach ($instructions as $instruction) {
$instructionType = '';
if (isset($instruction->type)) {
$instructionType = $instruction->type;
} else {
$instructionType = $instruction->__typename;
}
if ($instructionType === 'TimelineAddEntries') {
$entries = $instruction->entries;
break;
}
}
if (!$entries) {
throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type'))));
}
$tweets = [];
$userIds = [];
foreach ($entries as $entry) {
$entryType = '';
if (isset($entry->content->entryType)) {
$entryType = $entry->content->entryType;
} else {
$entryType = $entry->content->__typename;
}
if ($entryType !== 'TimelineTimelineItem') {
continue;
}
if (isset($timeline->data->user)) {
if (!isset($entry->content->itemContent->tweet_results->result->legacy)) {
continue;
}
$tweets[] = $entry->content->itemContent->tweet_results->result->legacy;
$userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result;
} else {
if (!isset($entry->content->content->tweetResult->result->legacy)) {
continue;
}
$tweets[] = $entry->content->content->tweetResult->result->legacy;
$userIds[] = $entry->content->content->tweetResult->result->core->user_result->result;
}
}
return (object) [
'userIds' => $userIds,
'tweets' => $tweets,
];
}
private function extractTweetFromSearch($searchResult)
{
return $searchResult->statuses;
}
public function fetchUserTweets(string $screenName): \stdClass
{
$this->fetchGuestToken();
@ -36,58 +118,66 @@ class TwitterClient
}
try {
$timeline = $this->fetchTimeline($userInfo->rest_id);
$timeline = $this->fetchTimelineUsingSearch($screenName);
} catch (HttpException $e) {
if ($e->getCode() === 403) {
$this->data['guest_token'] = null;
$this->fetchGuestToken();
$timeline = $this->fetchTimeline($userInfo->rest_id);
$timeline = $this->fetchTimelineUsingSearch($screenName);
} else {
throw $e;
}
}
$result = $timeline->data->user->result;
if ($result->__typename === 'UserUnavailable') {
throw new \Exception('UserUnavailable');
}
$instructionTypes = [
'TimelineAddEntries',
'TimelineClearCache',
'TimelinePinEntry', // unclear purpose, maybe pinned tweet?
];
$instructions = $result->timeline_v2->timeline->instructions;
if (!isset($instructions[1])) {
throw new \Exception('The account exists but has not tweeted yet?');
}
$tweets = $this->extractTweetFromSearch($timeline);
$entries = null;
foreach ($instructions as $instruction) {
if ($instruction->type === 'TimelineAddEntries') {
$entries = $instruction->entries;
break;
}
}
if (!$entries) {
throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type'))));
}
$tweets = [];
foreach ($entries as $entry) {
if ($entry->content->entryType !== 'TimelineTimelineItem') {
continue;
}
if (!isset($entry->content->itemContent->tweet_results->result->legacy)) {
continue;
}
$tweets[] = $entry->content->itemContent->tweet_results->result->legacy;
}
return (object) [
'user_info' => $userInfo,
'tweets' => $tweets,
];
}
public function fetchListTweets($query, $operation = '')
{
$id = '';
$this->fetchGuestToken();
if ($operation == 'By list') {
try {
$listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']);
$id = $listInfo->id_str;
} catch (HttpException $e) {
if ($e->getCode() === 403) {
$this->data['guest_token'] = null;
$this->fetchGuestToken();
$listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']);
$id = $listInfo->id_str;
} else {
throw $e;
}
}
} else if ($operation == 'By list ID') {
$id = $query['listId'];
} else {
throw new \Exception('Unknown operation to make list tweets');
}
try {
$timeline = $this->fetchListTimeline($id);
} catch (HttpException $e) {
if ($e->getCode() === 403) {
$this->data['guest_token'] = null;
$this->fetchGuestToken();
$timeline = $this->fetchListTimeline($id);
} else {
throw $e;
}
}
$data = $this->extractTweetAndUsersFromGraphQL($timeline);
return $data;
}
private function fetchGuestToken(): void
{
if (isset($this->data['guest_token'])) {
@ -173,6 +263,158 @@ class TwitterClient
return $response;
}
private function fetchTimelineUsingSearch($screenName)
{
$params = [
'q' => 'from:' . $screenName,
'modules' => 'status',
'result_type' => 'recent'
];
$response = $this->search($params);
return $response;
}
public function search($queryParam)
{
$url = sprintf(
'https://api.twitter.com/1.1/search/tweets.json?%s',
http_build_query($queryParam)
);
$response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
return $response;
}
private function fetchListInfoBySlug($screenName, $listSlug)
{
if (isset($this->data[$screenName . '-' . $listSlug])) {
return $this->data[$screenName . '-' . $listSlug];
}
$features = [
'android_graphql_skip_api_media_color_palette' => false,
'blue_business_profile_image_shape_enabled' => false,
'creator_subscriptions_subscription_count_enabled' => false,
'creator_subscriptions_tweet_preview_api_enabled' => true,
'freedom_of_speech_not_reach_fetch_enabled' => false,
'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false,
'hidden_profile_likes_enabled' => false,
'highlights_tweets_tab_ui_enabled' => false,
'interactive_text_enabled' => false,
'longform_notetweets_consumption_enabled' => true,
'longform_notetweets_inline_media_enabled' => false,
'longform_notetweets_richtext_consumption_enabled' => true,
'longform_notetweets_rich_text_read_enabled' => false,
'responsive_web_edit_tweet_api_enabled' => false,
'responsive_web_enhance_cards_enabled' => false,
'responsive_web_graphql_exclude_directive_enabled' => true,
'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false,
'responsive_web_graphql_timeline_navigation_enabled' => false,
'responsive_web_media_download_video_enabled' => false,
'responsive_web_text_conversations_enabled' => false,
'responsive_web_twitter_article_tweet_consumption_enabled' => false,
'responsive_web_twitter_blue_verified_badge_is_enabled' => true,
'rweb_lists_timeline_redesign_enabled' => true,
'spaces_2022_h2_clipping' => true,
'spaces_2022_h2_spaces_communities' => true,
'standardized_nudges_misinfo' => false,
'subscriptions_verification_info_enabled' => true,
'subscriptions_verification_info_reason_enabled' => true,
'subscriptions_verification_info_verified_since_enabled' => true,
'super_follow_badge_privacy_enabled' => false,
'super_follow_exclusive_tweet_notifications_enabled' => false,
'super_follow_tweet_api_enabled' => false,
'super_follow_user_api_enabled' => false,
'tweet_awards_web_tipping_enabled' => false,
'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false,
'tweetypie_unmention_optimization_enabled' => false,
'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false,
'verified_phone_label_enabled' => false,
'vibe_api_enabled' => false,
'view_counts_everywhere_api_enabled' => false
];
$variables = [
'screenName' => $screenName,
'listSlug' => $listSlug
];
$url = sprintf(
'https://twitter.com/i/api/graphql/-kmqNvm5Y-cVrfvBy6docg/ListBySlug?variables=%s&features=%s',
urlencode(json_encode($variables)),
urlencode(json_encode($features))
);
$response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
if (isset($response->errors)) {
// Grab the first error message
throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message));
}
$listInfo = $response->data->user_by_screen_name->list;
$this->data[$screenName . '-' . $listSlug] = $listInfo;
$this->cache->setScope('twitter');
$this->cache->setKey(['cache']);
$this->cache->saveData($this->data);
return $listInfo;
}
private function fetchListTimeline($listId)
{
$features = [
'android_graphql_skip_api_media_color_palette' => false,
'blue_business_profile_image_shape_enabled' => false,
'creator_subscriptions_subscription_count_enabled' => false,
'creator_subscriptions_tweet_preview_api_enabled' => true,
'freedom_of_speech_not_reach_fetch_enabled' => false,
'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false,
'hidden_profile_likes_enabled' => false,
'highlights_tweets_tab_ui_enabled' => false,
'interactive_text_enabled' => false,
'longform_notetweets_consumption_enabled' => true,
'longform_notetweets_inline_media_enabled' => false,
'longform_notetweets_richtext_consumption_enabled' => true,
'longform_notetweets_rich_text_read_enabled' => false,
'responsive_web_edit_tweet_api_enabled' => false,
'responsive_web_enhance_cards_enabled' => false,
'responsive_web_graphql_exclude_directive_enabled' => true,
'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false,
'responsive_web_graphql_timeline_navigation_enabled' => false,
'responsive_web_media_download_video_enabled' => false,
'responsive_web_text_conversations_enabled' => false,
'responsive_web_twitter_article_tweet_consumption_enabled' => false,
'responsive_web_twitter_blue_verified_badge_is_enabled' => true,
'rweb_lists_timeline_redesign_enabled' => true,
'spaces_2022_h2_clipping' => true,
'spaces_2022_h2_spaces_communities' => true,
'standardized_nudges_misinfo' => false,
'subscriptions_verification_info_enabled' => true,
'subscriptions_verification_info_reason_enabled' => true,
'subscriptions_verification_info_verified_since_enabled' => true,
'super_follow_badge_privacy_enabled' => false,
'super_follow_exclusive_tweet_notifications_enabled' => false,
'super_follow_tweet_api_enabled' => false,
'super_follow_user_api_enabled' => false,
'tweet_awards_web_tipping_enabled' => false,
'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false,
'tweetypie_unmention_optimization_enabled' => false,
'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false,
'verified_phone_label_enabled' => false,
'vibe_api_enabled' => false,
'view_counts_everywhere_api_enabled' => false
];
$variables = [
'rest_id' => $listId,
'count' => 20
];
$url = sprintf(
'https://twitter.com/i/api/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s',
urlencode(json_encode($variables)),
urlencode(json_encode($features))
);
$response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
return $response;
}
private function createHttpHeaders(): array
{
$headers = [