danolo
/
rss-bridge
mirror of https://github.com/danoloan10/rss-bridge
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add limit options to the slowest bridges

This commit is contained in:
dag 2022-04-10 18:56:24 +02:00 committed by GitHub
parent 0b40f51c01
commit 5a733b3d82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 80 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bridges/DarkReadingBridge.php
View File

@ -33,7 +33,8 @@ class DarkReadingBridge extends FeedExpander {
'Insider Threats' => '663_Insider%20Threats',
'Vulnerability Management' => '664_Vulnerability%20Management',
)
)
),
'limit' => self::LIMIT,
));
public function collectData(){
@ -48,7 +49,8 @@ class DarkReadingBridge extends FeedExpander {
if ($feed_id != '000') {
$feed_url .= '?f_n=' . $feed_id . '&f_ln=' . $feed_name;
}
$this->collectExpandableDatas($feed_url, 20);
$limit = $this->getInput('limit') ?? 10;
$this->collectExpandableDatas($feed_url, $limit);
}
protected function parseItem($newsItem){

6
bridges/FindACrewBridge.php
View File

@ -26,7 +26,8 @@ class FindACrewBridge extends BridgeAbstract {
'distance' => array(
'name' => 'Limit boundary of search in KM',
'title' => 'Boundary of the search in kilometers when using longitude and latitude'
)
),
'limit' => self::LIMIT,
)
);
@ -59,7 +60,8 @@ class FindACrewBridge extends BridgeAbstract {
$html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.');
$annonces = $html->find('.css_SrhRst');
foreach ($annonces as $annonce) {
$limit = $this->getInput('limit') ?? 10;
foreach (array_slice($annonces, 0, $limit) as $annonce) {
$item = array();
$link = parent::getURI() . $annonce->find('.lstsum-btn-con a', 0)->href;

3
bridges/FolhaDeSaoPauloBridge.php
View File

@ -63,6 +63,7 @@ class FolhaDeSaoPauloBridge extends FeedExpander {
$feed_url = self::URI . '/' . $this->getInput('feed');
}
Debug::log('URL: ' . $feed_url);
$this->collectExpandableDatas($feed_url, $this->getInput('amount'));
$limit = $this->getInput('amount');
$this->collectExpandableDatas($feed_url, $limit);
}
}

6
bridges/GQMagazineBridge.php
View File

@ -32,6 +32,7 @@ class GQMagazineBridge extends BridgeAbstract
'required' => true,
'exampleValue' => 'sexe/news'
),
'limit' => self::LIMIT,
));
const REPLACED_ATTRIBUTES = array(
@ -76,7 +77,12 @@ class GQMagazineBridge extends BridgeAbstract
// Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content !
$main = $html->find('main', 0);
$limit = $this->getInput('limit') ?? 10;
foreach ($main->find('a') as $link) {
if (count($this->items) >= $limit) {
break;
}
$uri = $link->href;
$date = $link->parent()->find('time', 0);

13
bridges/InternetArchiveBridge.php
View File

@ -23,7 +23,8 @@ class InternetArchiveBridge extends BridgeAbstract {
'Web Archives' => 'web-archive',
),
'defaultValue' => 'uploads',
)
),
'limit' => self::LIMIT,
)
);
@ -72,7 +73,8 @@ class InternetArchiveBridge extends BridgeAbstract {
if ($this->getInput('content') !== 'posts') {
$detailsDivNumber = 0;
foreach ($html->find('div.results > div[data-id]') as $index => $result) {
$results = $html->find('div.results > div[data-id]');
foreach ($results as $index => $result) {
$item = array();
if (in_array($result->class, $this->skipClasses)) {
@ -110,6 +112,11 @@ class InternetArchiveBridge extends BridgeAbstract {
}
$detailsDivNumber++;
$limit = $this->getInput('limit') ?? 10;
if (count($this->items) >= $limit) {
break;
}
}
}
@ -302,7 +309,7 @@ EOD;
$items[] = $item;
if (count($items) >= 10) {
if (count($items) >= $this->getInput('limit') ?? 10) {
break;
}
}

7
bridges/NextInpactBridge.php
View File

@ -55,7 +55,8 @@ class NextInpactBridge extends FeedExpander {
'Hide Brief' => '1',
'Only Brief' => '2'
)
)
),
'limit' => self::LIMIT,
));
public function collectData(){
@ -80,7 +81,9 @@ class NextInpactBridge extends FeedExpander {
$feed = 'params';
}
$this->collectExpandableDatas($base_uri . 'rss/' . $feed . '.xml' . $args);
$url = sprintf('%srss/%s.xml%s', $base_uri, $feed, $args);
$limit = $this->getInput('limit') ?? 10;
$this->collectExpandableDatas($url, $limit);
}
protected function parseItem($newsItem){

4
bridges/OpenlyBridge.php
View File

@ -130,7 +130,9 @@ class OpenlyBridge extends BridgeAbstract {
$this->feedTitle = $html->find('a.tooltipitem', 0)->plaintext;
}
foreach($html->find('div.item') as $div) {
$items = $html->find('div.item');
$limit = 5;
foreach(array_slice($items, 0, $limit) as $div) {
$this->items[] = $this->getArticle($div->find('a', 0)->href);
if (count($this->items) >= $this->itemLimit) {

9
bridges/PcGamerBridge.php
View File

@ -7,11 +7,18 @@ class PcGamerBridge extends BridgeAbstract
updates and news on all your favorite PC gaming franchises.';
const MAINTAINER = 'IceWreck, mdemoss';
const PARAMETERS = [
[
'limit' => self::LIMIT,
]
];
public function collectData()
{
$html = getSimpleHTMLDOMCached($this->getURI(), 300);
$stories = $html->find('a.article-link');
foreach ($stories as $element) {
$limit = $this->getInput('limit') ?? 10;
foreach (array_slice($stories, 0, $limit) as $element) {
$item = array();
$item['uri'] = $element->href;
$articleHtml = getSimpleHTMLDOMCached($item['uri']);

16
bridges/UnogsBridge.php
View File

@ -17,7 +17,8 @@ class UnogsBridge extends BridgeAbstract {
'What\'s New' => 'new last 7 days',
'Expiring' => 'expiring'
)
)
),
'limit' => self::LIMIT,
),
'Global' => array(),
'Country' => array(
@ -160,8 +161,17 @@ EOD;
break;
}
$api_url = self::URI . '/api/search?query=' . urlencode($feed)
. ($country_code ? '&countrylist=' . $country_code : '') . '&limit=30';
$limit = $this->getInput('limit') ?? 30;
// https://rapidapi.com/unogs/api/unogsng/details
$api_url = sprintf(
'%s/api/search?query=%s%s&limit=%s',
self::URI,
urlencode($feed),
$country_code ? '&countrylist=' . $country_code : '',
$limit
);
$json_data = $this->getJSON($api_url);
$movies = $json_data['results'];

8
bridges/WeLiveSecurityBridge.php
View File

@ -5,6 +5,11 @@ class WeLiveSecurityBridge extends FeedExpander {
const NAME = 'We Live Security';
const URI = 'https://www.welivesecurity.com/';
const DESCRIPTION = 'Returns the newest articles.';
const PARAMETERS = [
[
'limit' => self::LIMIT,
],
];
protected function parseItem($item){
$item = parent::parseItem($item);
@ -27,6 +32,7 @@ class WeLiveSecurityBridge extends FeedExpander {
public function collectData(){
$feed = static::URI . 'feed/';
$this->collectExpandableDatas($feed);
$limit = $this->getInput('limit') ?? 10;
$this->collectExpandableDatas($feed, $limit);
}
}

6
bridges/WiredBridge.php
View File

@ -22,7 +22,8 @@ class WiredBridge extends FeedExpander {
'WIRED Guides' => 'wired-guide', // /feed/tag/wired-guide/latest/rss
'Photo' => 'photo' // /feed/category/photo/latest/rss
)
)
),
'limit' => self::LIMIT,
));
public function collectData(){
@ -42,7 +43,8 @@ class WiredBridge extends FeedExpander {
}
$feed_url .= 'rss';
$this->collectExpandableDatas($feed_url);
$limit = $this->getInput('limit') ?? -1;
$this->collectExpandableDatas($feed_url, $limit);
}
protected function parseItem($newsItem){

6
bridges/ZDNetBridge.php
View File

@ -156,7 +156,8 @@ class ZDNetBridge extends FeedExpander {
'ZDNet Government' => 'blog/government'
)
)
)
),
'limit' => self::LIMIT,
));
public function collectData(){
@ -167,7 +168,8 @@ class ZDNetBridge extends FeedExpander {
$baseUri = str_replace('www.', 'downloads.', $baseUri);
}
$url = $baseUri . trim($feed, '/') . '/rss.xml';
$this->collectExpandableDatas($url);
$limit = $this->getInput('limit') ?? 10;
$this->collectExpandableDatas($url, $limit);
}
protected function parseItem($item){

10
lib/BridgeAbstract.php
View File

@ -87,6 +87,16 @@ abstract class BridgeAbstract implements BridgeInterface {
*/
const TEST_DETECT_PARAMETERS = array();
/**
* This is a convenient const for the limit option in bridge contexts.
* Can be inlined and modified if necessary.
*/
protected const LIMIT = [
'name' => 'Limit',
'type' => 'number',
'title' => 'Maximum number of items to return',
];
/**
* Holds the list of items collected by the bridge
*