diff --git a/.github/prtester.py b/.github/prtester.py
index df6cc1ff..a2a7ab84 100644
--- a/.github/prtester.py
+++ b/.github/prtester.py
@@ -1,113 +1,159 @@
+import argparse
import requests
-import itertools
from bs4 import BeautifulSoup
from datetime import datetime
+from typing import Iterable
import os.path
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
#
# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
-# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing
+# It also add a tag with the url of em's public instance, so viewing
# the HTML file locally will actually work as designed.
-def testBridges(bridges,status):
- for bridge in bridges:
- if bridge.get('data-ref'): # Some div entries are empty, this ignores those
- bridgeid = bridge.get('id')
- bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
- print(bridgeid + "\n")
- bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
- forms = bridge.find_all("form")
- formid = 1
- for form in forms:
- # a bridge can have multiple contexts, named 'forms' in html
- # this code will produce a fully working formstring that should create a working feed when called
- # this will create an example feed for every single context, to test them all
- formstring = ''
- errormessages = []
- parameters = form.find_all("input")
- lists = form.find_all("select")
- # this for/if mess cycles through all available input parameters, checks if it required, then pulls
- # the default or examplevalue and then combines it all together into the formstring
- # if an example or default value is missing for a required attribute, it will throw an error
- # any non-required fields are not tested!!!
- for parameter in parameters:
- if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
- cleanvalue = parameter.get('value').replace(" ","+")
- formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
- if parameter.get('type') == 'number' or parameter.get('type') == 'text':
- if parameter.has_attr('required'):
- if parameter.get('placeholder') == '':
- if parameter.get('value') == '':
- errormessages.append(parameter.get('name'))
- else:
- formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
+class Instance:
+ name = ''
+ url = ''
+
+def main(instances: Iterable[Instance], with_upload: bool, comment_title: str):
+ start_date = datetime.now()
+ table_rows = []
+ for instance in instances:
+ page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
+ soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
+ bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
+ table_rows += testBridges(instance, bridge_cards, with_upload) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version
+ with open(file=os.getcwd() + '/comment.txt', mode='w+', encoding='utf-8') as file:
+ table_rows_value = '\n'.join(sorted(table_rows))
+ file.write(f'''
+## {comment_title}
+| Bridge | Context | Status |
+| - | - | - |
+{table_rows_value}
+
+*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
+ '''.strip())
+
+def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) -> Iterable:
+ instance_suffix = ''
+ if instance.name:
+ instance_suffix = f' ({instance.name})'
+ table_rows = []
+ for bridge_card in bridge_cards:
+ bridgeid = bridge_card.get('id')
+ bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
+ print(f'{bridgeid}{instance_suffix}\n')
+ bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
+ bridge_name = bridgeid.replace('Bridge', '')
+ context_forms = bridge_card.find_all("form")
+ form_number = 1
+ for context_form in context_forms:
+ # a bridge can have multiple contexts, named 'forms' in html
+ # this code will produce a fully working formstring that should create a working feed when called
+ # this will create an example feed for every single context, to test them all
+ formstring = ''
+ error_messages = []
+ context_name = '*untitled*'
+ context_name_element = context_form.find_previous_sibling('h5')
+ if context_name_element and context_name_element.text.strip() != '':
+ context_name = context_name_element.text
+ parameters = context_form.find_all("input")
+ lists = context_form.find_all("select")
+ # this for/if mess cycles through all available input parameters, checks if it required, then pulls
+ # the default or examplevalue and then combines it all together into the formstring
+ # if an example or default value is missing for a required attribute, it will throw an error
+ # any non-required fields are not tested!!!
+ for parameter in parameters:
+ if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
+ cleanvalue = parameter.get('value').replace(" ","+")
+ formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
+ if parameter.get('type') == 'number' or parameter.get('type') == 'text':
+ if parameter.has_attr('required'):
+ if parameter.get('placeholder') == '':
+ if parameter.get('value') == '':
+ name_value = parameter.get('name')
+ error_messages.append(f'Missing example or default value for parameter "{name_value}"')
else:
- formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
- # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
- if parameter.get('type') == 'checkbox':
- if parameter.has_attr('checked'):
- formstring = formstring + '&' + parameter.get('name') + '=on'
- for listing in lists:
- selectionvalue = ''
- listname = listing.get('name')
- cleanlist = []
- for option in listing.contents:
- if 'optgroup' in option.name:
- cleanlist.extend(option)
+ formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
else:
- cleanlist.append(option)
- firstselectionentry = 1
- for selectionentry in cleanlist:
- if firstselectionentry:
+ formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
+ # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
+ if parameter.get('type') == 'checkbox':
+ if parameter.has_attr('checked'):
+ formstring = formstring + '&' + parameter.get('name') + '=on'
+ for listing in lists:
+ selectionvalue = ''
+ listname = listing.get('name')
+ cleanlist = []
+ for option in listing.contents:
+ if 'optgroup' in option.name:
+ cleanlist.extend(option)
+ else:
+ cleanlist.append(option)
+ firstselectionentry = 1
+ for selectionentry in cleanlist:
+ if firstselectionentry:
+ selectionvalue = selectionentry.get('value')
+ firstselectionentry = 0
+ else:
+ if 'selected' in selectionentry.attrs:
selectionvalue = selectionentry.get('value')
- firstselectionentry = 0
- else:
- if 'selected' in selectionentry.attrs:
- selectionvalue = selectionentry.get('value')
- break
- formstring = formstring + '&' + listname + '=' + selectionvalue
- if not errormessages:
- # if all example/default values are present, form the full request string, run the request, replace the static css
- # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site.
- r = requests.get(URL + bridgestring + formstring)
- pagetext = r.text.replace('static/style.css','https://rss-bridge.org/bridge01/static/style.css')
- pagetext = pagetext.encode("utf_8")
- termpad = requests.post(url="https://termpad.com/", data=pagetext)
- termpadurl = termpad.text
- termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
- termpadurl = termpadurl.replace('\n','')
- with open(os.getcwd() + '/comment.txt', 'a+') as file:
- file.write("\n")
- file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+ break
+ formstring = formstring + '&' + listname + '=' + selectionvalue
+ termpad_url = 'about:blank'
+ if error_messages:
+ status = '
'.join(map(lambda m: f'❌ `{m}`', error_messages))
+ else:
+ # if all example/default values are present, form the full request string, run the request, add a tag with
+ # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
+ # then upload it to termpad.com, a pastebin-like-site.
+ response = requests.get(instance.url + bridgestring + formstring)
+ page_text = response.text.replace('','')
+ page_text = page_text.encode("utf_8")
+ soup = BeautifulSoup(page_text, "html.parser")
+ status_messages = list(map(lambda e: f'⚠️ `{e.text.strip().splitlines()[0]}`', soup.find_all('pre')))
+ if response.status_code != 200:
+ status_messages = [f'❌ `HTTP status {response.status_code} {response.reason}`'] + status_messages
else:
- # if there are errors (which means that a required value has no example or default value), log out which error appeared
- termpad = requests.post(url="https://termpad.com/", data=str(errormessages))
- termpadurl = termpad.text
- termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
- termpadurl = termpadurl.replace('\n','')
- with open(os.getcwd() + '/comment.txt', 'a+') as file:
- file.write("\n")
- file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
- formid += 1
+ feed_items = soup.select('.feeditem')
+ feed_items_length = len(feed_items)
+ if feed_items_length <= 0:
+ status_messages += [f'⚠️ `The feed has no items`']
+ elif feed_items_length == 1 and len(soup.select('.error')) > 0:
+ status_messages = [f'❌ `{feed_items[0].text.strip().splitlines()[0]}`'] + status_messages
+ status = '
'.join(status_messages)
+ if status.strip() == '':
+ status = '✔️'
+ if with_upload:
+ termpad = requests.post(url="https://termpad.com/", data=page_text)
+ termpad_url = termpad.text.strip()
+ termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/')
+ table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |')
+ form_number += 1
+ return table_rows
-gitstatus = ["current", "pr"]
-now = datetime.now()
-date_time = now.strftime("%Y-%m-%d, %H:%M:%S")
-
-with open(os.getcwd() + '/comment.txt', 'w+') as file:
- file.write(''' ## Pull request artifacts
-| file | last change |
-| ---- | ------ |''')
-
-for status in gitstatus: # run this twice, once for the current version, once for the PR version
- if status == "current":
- port = "3000" # both ports are defined in the corresponding workflow .yml file
- elif status == "pr":
- port = "3001"
- URL = "http://localhost:" + port
- page = requests.get(URL) # Use python requests to grab the rss-bridge main page
- soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
- bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page
- testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-i', '--instances', nargs='+')
+ parser.add_argument('-nu', '--no-upload', action='store_true')
+ parser.add_argument('-t', '--comment-title', default='Pull request artifacts')
+ args = parser.parse_args()
+ instances = []
+ if args.instances:
+ for instance_arg in args.instances:
+ instance_arg_parts = instance_arg.split('::')
+ instance = Instance()
+ instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else ''
+ instance.url = instance_arg_parts[0]
+ instances.append(instance)
+ else:
+ instance = Instance()
+ instance.name = 'current'
+ instance.url = 'http://localhost:3000'
+ instances.append(instance)
+ instance = Instance()
+ instance.name = 'pr'
+ instance.url = 'http://localhost:3001'
+ instances.append(instance)
+ main(instances=instances, with_upload=not args.no_upload, comment_title=args.comment_title);
\ No newline at end of file