danolo
/
rss-bridge
mirror of https://github.com/danoloan10/rss-bridge
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(youtube): reduce excessive network calls (#3757)

This commit is contained in:
Dag 2023-10-15 03:15:47 +02:00 committed by GitHub
parent 2aa52aa99a
commit 611fabe46c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 59 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
bridges/YoutubeBridge.php
View File

@ -104,6 +104,14 @@ class YoutubeBridge extends BridgeAbstract
$username = $this->getInput('u'); $username = $this->getInput('u');
$channel = $this->getInput('c'); $channel = $this->getInput('c');
$custom = $this->getInput('custom'); $custom = $this->getInput('custom');
$playlist = $this->getInput('p');
$search = $this->getInput('s');
$durationMin = $this->getInput('duration_min');
$durationMax = $this->getInput('duration_max');
// Whether to discriminate videos by duration
$filterByDuration = $durationMin || $durationMax;
if ($username) { if ($username) {
// user and channel // user and channel
@ -119,15 +127,6 @@ class YoutubeBridge extends BridgeAbstract
$url_listing = self::URI . '/' . urlencode($request) . '/videos'; $url_listing = self::URI . '/' . urlencode($request) . '/videos';
} }
$playlist = $this->getInput('p');
$search = $this->getInput('s');
$durationMin = $this->getInput('duration_min');
$durationMax = $this->getInput('duration_max');
// Whether to discriminate videos by duration
$filterByDuration = $durationMin || $durationMax;
if ($url_feed || $url_listing) { if ($url_feed || $url_listing) {
// user, channel or custom // user, channel or custom
$this->feeduri = $url_listing; $this->feeduri = $url_listing;
@ -217,9 +216,9 @@ class YoutubeBridge extends BridgeAbstract
} }
} }
private function fetchVideoDetails($vid, &$author, &$desc, &$time) private function fetchVideoDetails($videoId, &$author, &$description, &$timestamp)
{ {
$url = self::URI . "/watch?v=$vid"; $url = self::URI . "/watch?v=$videoId";
$html = $this->fetch($url, true); $html = $this->fetch($url, true);
// Skip unavailable videos // Skip unavailable videos
@ -234,7 +233,7 @@ class YoutubeBridge extends BridgeAbstract
$elDatePublished = $html->find('meta[itemprop=datePublished]', 0); $elDatePublished = $html->find('meta[itemprop=datePublished]', 0);
if (!is_null($elDatePublished)) { if (!is_null($elDatePublished)) {
$time = strtotime($elDatePublished->getAttribute('content')); $timestamp = strtotime($elDatePublished->getAttribute('content'));
} }
$jsonData = $this->extractJsonFromHtml($html); $jsonData = $this->extractJsonFromHtml($html);
@ -254,30 +253,28 @@ class YoutubeBridge extends BridgeAbstract
} }
} }
if (!$videoSecondaryInfo) { if (!$videoSecondaryInfo) {
returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $vid); returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $videoId);
} }
$desc = $videoSecondaryInfo->attributedDescription->content ?? ''; $description = $videoSecondaryInfo->attributedDescription->content ?? '';
// Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space) // Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space)
$whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}"; $whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}";
$descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $desc, self::URI, $whitespaceChars); $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $description, self::URI, $whitespaceChars);
foreach ($descEnhancements as $descEnhancement) { foreach ($descEnhancements as $descEnhancement) {
if (isset($descEnhancement['url'])) { if (isset($descEnhancement['url'])) {
$descBefore = mb_substr($desc, 0, $descEnhancement['pos']); $descBefore = mb_substr($description, 0, $descEnhancement['pos']);
$descValue = mb_substr($desc, $descEnhancement['pos'], $descEnhancement['len']); $descValue = mb_substr($description, $descEnhancement['pos'], $descEnhancement['len']);
$descAfter = mb_substr($desc, $descEnhancement['pos'] + $descEnhancement['len'], null); $descAfter = mb_substr($description, $descEnhancement['pos'] + $descEnhancement['len'], null);
// Extended trim for the display value of internal links, e.g.: // Extended trim for the display value of internal links, e.g.:
// FAVICON • Video Name // FAVICON • Video Name
// FAVICON / @ChannelName // FAVICON / @ChannelName
$descValue = trim($descValue, $whitespaceChars . '•/'); $descValue = trim($descValue, $whitespaceChars . '•/');
$desc = sprintf('%s<a href="%s" target="_blank">%s</a>%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter); $description = sprintf('%s<a href="%s" target="_blank">%s</a>%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter);
} }
} }
$desc = nl2br($desc);
} }
private function ytBridgeGetVideoDescriptionEnhancements( private function ytBridgeGetVideoDescriptionEnhancements(
@ -425,7 +422,7 @@ class YoutubeBridge extends BridgeAbstract
private function fetch($url, bool $cache = false) private function fetch($url, bool $cache = false)
{ {
$header = ['Accept-Language: en-US']; $header = ['Accept-Language: en-US'];
$ttl = 86400; $ttl = 86400 * 3; // 3d
$stripNewlines = false; $stripNewlines = false;
if ($cache) { if ($cache) {
return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines);
@ -447,15 +444,9 @@ class YoutubeBridge extends BridgeAbstract
private function fetchItemsFromFromJsonData($jsonData) private function fetchItemsFromFromJsonData($jsonData)
{ {
$duration_min = $this->getInput('duration_min') ?: -1; $minimumDurationSeconds = ($this->getInput('duration_min') ?: -1) * 60;
$duration_min = $duration_min * 60; $maximumDurationSeconds = ($this->getInput('duration_max') ?: INF) * 60;
$duration_max = $this->getInput('duration_max') ?: INF;
$duration_max = $duration_max * 60;
if ($duration_max < $duration_min) {
returnClientError('Max duration must be greater than min duration!');
}
foreach ($jsonData as $item) { foreach ($jsonData as $item) {
$wrapper = null; $wrapper = null;
if (isset($item->gridVideoRenderer)) { if (isset($item->gridVideoRenderer)) {
@ -469,18 +460,33 @@ class YoutubeBridge extends BridgeAbstract
} else { } else {
continue; continue;
} }
$videoId = $wrapper->videoId;
$title = $wrapper->title->runs[0]->text;
$author = '';
$desc = '';
$time = '';
// The duration comes in one of the formats: // 01:03:30 | 15:06 | 1:24
// hh:mm:ss / mm:ss / m:ss $lengthText = $wrapper->lengthText->simpleText ?? null;
// 01:03:30 / 15:06 / 1:24 // 6,875 views
$viewCount = $wrapper->viewCountText->simpleText ?? null;
// Dc645M8Het8
$videoId = $wrapper->videoId;
// Jumbo frames - transfer more data faster!
$title = $wrapper->title->runs[0]->text ?? $wrapper->title->accessibility->accessibilityData->label ?? null;
$author = null;
$description = $wrapper->descriptionSnippet->runs[0]->text ?? null;
// 5 days ago | 1 month ago
$publishedTimeText = $wrapper->publishedTimeText->simpleText ?? $wrapper->videoInfo->runs[2]->text ?? null;
$timestamp = null;
if ($publishedTimeText) {
try {
$publicationDate = new \DateTimeImmutable($publishedTimeText);
// Hard-code hour, minute and second
$publicationDate = $publicationDate->setTime(0, 0, 0);
$timestamp = $publicationDate->getTimestamp();
} catch (\Exception $e) {
}
}
$durationText = 0; $durationText = 0;
if (isset($wrapper->lengthText)) { if ($lengthText) {
$durationText = $wrapper->lengthText->simpleText; $durationText = $lengthText;
} else { } else {
foreach ($wrapper->thumbnailOverlays as $overlay) { foreach ($wrapper->thumbnailOverlays as $overlay) {
if (isset($overlay->thumbnailOverlayTimeStatusRenderer)) { if (isset($overlay->thumbnailOverlayTimeStatusRenderer)) {
@ -497,35 +503,37 @@ class YoutubeBridge extends BridgeAbstract
} }
sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds);
$duration = $hours * 3600 + $minutes * 60 + $seconds; $duration = $hours * 3600 + $minutes * 60 + $seconds;
if ($duration < $duration_min || $duration > $duration_max) { if ($duration < $minimumDurationSeconds || $duration > $maximumDurationSeconds) {
continue; continue;
} }
} }
if (!$description || !$timestamp) {
//$durationSeconds = (int) $wrapper->lengthSeconds; $this->fetchVideoDetails($videoId, $author, $description, $timestamp);
if ($duration < $duration_min || $duration > $duration_max) { }
continue; $this->addItem($videoId, $title, $author, $description, $timestamp);
if (count($this->items) >= 99) {
break;
} }
$this->fetchVideoDetails($videoId, $author, $desc, $time);
$this->addItem($videoId, $title, $author, $desc, $time);
} }
} }
private function addItem($videoId, $title, $author, $desc, $time, $thumbnail = '') private function addItem($videoId, $title, $author, $description, $timestamp, $thumbnail = '')
{ {
$description = nl2br($description);
$item = []; $item = [];
// This should probably be uid? // This should probably be uid?
$item['id'] = $videoId; $item['id'] = $videoId;
$item['title'] = $title; $item['title'] = $title;
$item['author'] = $author; $item['author'] = $author ?? '';
$item['timestamp'] = $time; $item['timestamp'] = $timestamp;
$item['uri'] = self::URI . '/watch?v=' . $videoId; $item['uri'] = self::URI . '/watch?v=' . $videoId;
if (!$thumbnail) { if (!$thumbnail) {
// Fallback to default thumbnail if there aren't any provided. // Fallback to default thumbnail if there aren't any provided.
$thumbnail = '0'; $thumbnail = '0';
} }
$thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg'; $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg';
$item['content'] = sprintf('<a href="%s"><img src="%s" /></a><br />%s', $item['uri'], $thumbnailUri, $desc); $item['content'] = sprintf('<a href="%s"><img src="%s" /></a><br />%s', $item['uri'], $thumbnailUri, $description);
$this->items[] = $item; $this->items[] = $item;
} }