danolo
/
rss-bridge
mirror of https://github.com/danoloan10/rss-bridge
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[core] improve pull request artifacts comment (#3705)

This commit is contained in:
User123698745 2023-09-24 21:13:01 +02:00 committed by GitHub
parent 857e908929
commit 09f3c1532a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 143 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

240
.github/prtester.py vendored
View File

@ -1,113 +1,159 @@
import argparse
import requests import requests
import itertools
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from datetime import datetime from datetime import datetime
from typing import Iterable
import os.path import os.path
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
# #
# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of # This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
# RSS-Bridge, generate a feed for each of the bridges and save the output as html files. # RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing # It also add a <base> tag with the url of em's public instance, so viewing
# the HTML file locally will actually work as designed. # the HTML file locally will actually work as designed.
def testBridges(bridges,status): class Instance:
for bridge in bridges: name = ''
if bridge.get('data-ref'): # Some div entries are empty, this ignores those url = ''
bridgeid = bridge.get('id')
bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata def main(instances: Iterable[Instance], with_upload: bool, comment_title: str):
print(bridgeid + "\n") start_date = datetime.now()
bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' table_rows = []
forms = bridge.find_all("form") for instance in instances:
formid = 1 page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
for form in forms: soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
# a bridge can have multiple contexts, named 'forms' in html bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
# this code will produce a fully working formstring that should create a working feed when called table_rows += testBridges(instance, bridge_cards, with_upload) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version
# this will create an example feed for every single context, to test them all with open(file=os.getcwd() + '/comment.txt', mode='w+', encoding='utf-8') as file:
formstring = '' table_rows_value = '\n'.join(sorted(table_rows))
errormessages = [] file.write(f'''
parameters = form.find_all("input") ## {comment_title}
lists = form.find_all("select") | Bridge | Context | Status |
# this for/if mess cycles through all available input parameters, checks if it required, then pulls | - | - | - |
# the default or examplevalue and then combines it all together into the formstring {table_rows_value}
# if an example or default value is missing for a required attribute, it will throw an error
# any non-required fields are not tested!!! *last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
for parameter in parameters: '''.strip())
if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
cleanvalue = parameter.get('value').replace(" ","+") def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) -> Iterable:
formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue instance_suffix = ''
if parameter.get('type') == 'number' or parameter.get('type') == 'text': if instance.name:
if parameter.has_attr('required'): instance_suffix = f' ({instance.name})'
if parameter.get('placeholder') == '': table_rows = []
if parameter.get('value') == '': for bridge_card in bridge_cards:
errormessages.append(parameter.get('name')) bridgeid = bridge_card.get('id')
else: bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') print(f'{bridgeid}{instance_suffix}\n')
bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
bridge_name = bridgeid.replace('Bridge', '')
context_forms = bridge_card.find_all("form")
form_number = 1
for context_form in context_forms:
# a bridge can have multiple contexts, named 'forms' in html
# this code will produce a fully working formstring that should create a working feed when called
# this will create an example feed for every single context, to test them all
formstring = ''
error_messages = []
context_name = '*untitled*'
context_name_element = context_form.find_previous_sibling('h5')
if context_name_element and context_name_element.text.strip() != '':
context_name = context_name_element.text
parameters = context_form.find_all("input")
lists = context_form.find_all("select")
# this for/if mess cycles through all available input parameters, checks if it required, then pulls
# the default or examplevalue and then combines it all together into the formstring
# if an example or default value is missing for a required attribute, it will throw an error
# any non-required fields are not tested!!!
for parameter in parameters:
if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
cleanvalue = parameter.get('value').replace(" ","+")
formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
if parameter.get('type') == 'number' or parameter.get('type') == 'text':
if parameter.has_attr('required'):
if parameter.get('placeholder') == '':
if parameter.get('value') == '':
name_value = parameter.get('name')
error_messages.append(f'Missing example or default value for parameter "{name_value}"')
else: else:
formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
# same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
if parameter.get('type') == 'checkbox':
if parameter.has_attr('checked'):
formstring = formstring + '&' + parameter.get('name') + '=on'
for listing in lists:
selectionvalue = ''
listname = listing.get('name')
cleanlist = []
for option in listing.contents:
if 'optgroup' in option.name:
cleanlist.extend(option)
else: else:
cleanlist.append(option) formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
firstselectionentry = 1 # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
for selectionentry in cleanlist: if parameter.get('type') == 'checkbox':
if firstselectionentry: if parameter.has_attr('checked'):
formstring = formstring + '&' + parameter.get('name') + '=on'
for listing in lists:
selectionvalue = ''
listname = listing.get('name')
cleanlist = []
for option in listing.contents:
if 'optgroup' in option.name:
cleanlist.extend(option)
else:
cleanlist.append(option)
firstselectionentry = 1
for selectionentry in cleanlist:
if firstselectionentry:
selectionvalue = selectionentry.get('value')
firstselectionentry = 0
else:
if 'selected' in selectionentry.attrs:
selectionvalue = selectionentry.get('value') selectionvalue = selectionentry.get('value')
firstselectionentry = 0 break
else: formstring = formstring + '&' + listname + '=' + selectionvalue
if 'selected' in selectionentry.attrs: termpad_url = 'about:blank'
selectionvalue = selectionentry.get('value') if error_messages:
break status = '<br>'.join(map(lambda m: f'❌ `{m}`', error_messages))
formstring = formstring + '&' + listname + '=' + selectionvalue else:
if not errormessages: # if all example/default values are present, form the full request string, run the request, add a <base> tag with
# if all example/default values are present, form the full request string, run the request, replace the static css # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
# file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site. # then upload it to termpad.com, a pastebin-like-site.
r = requests.get(URL + bridgestring + formstring) response = requests.get(instance.url + bridgestring + formstring)
pagetext = r.text.replace('static/style.css','https://rss-bridge.org/bridge01/static/style.css') page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">')
pagetext = pagetext.encode("utf_8") page_text = page_text.encode("utf_8")
termpad = requests.post(url="https://termpad.com/", data=pagetext) soup = BeautifulSoup(page_text, "html.parser")
termpadurl = termpad.text status_messages = list(map(lambda e: f'⚠️ `{e.text.strip().splitlines()[0]}`', soup.find_all('pre')))
termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') if response.status_code != 200:
termpadurl = termpadurl.replace('\n','') status_messages = [f'❌ `HTTP status {response.status_code} {response.reason}`'] + status_messages
with open(os.getcwd() + '/comment.txt', 'a+') as file:
file.write("\n")
file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
else: else:
# if there are errors (which means that a required value has no example or default value), log out which error appeared feed_items = soup.select('.feeditem')
termpad = requests.post(url="https://termpad.com/", data=str(errormessages)) feed_items_length = len(feed_items)
termpadurl = termpad.text if feed_items_length <= 0:
termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') status_messages += [f'⚠️ `The feed has no items`']
termpadurl = termpadurl.replace('\n','') elif feed_items_length == 1 and len(soup.select('.error')) > 0:
with open(os.getcwd() + '/comment.txt', 'a+') as file: status_messages = [f'❌ `{feed_items[0].text.strip().splitlines()[0]}`'] + status_messages
file.write("\n") status = '<br>'.join(status_messages)
file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |") if status.strip() == '':
formid += 1 status = '✔️'
if with_upload:
termpad = requests.post(url="https://termpad.com/", data=page_text)
termpad_url = termpad.text.strip()
termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/')
table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |')
form_number += 1
return table_rows
gitstatus = ["current", "pr"] if __name__ == '__main__':
now = datetime.now() parser = argparse.ArgumentParser()
date_time = now.strftime("%Y-%m-%d, %H:%M:%S") parser.add_argument('-i', '--instances', nargs='+')
parser.add_argument('-nu', '--no-upload', action='store_true')
with open(os.getcwd() + '/comment.txt', 'w+') as file: parser.add_argument('-t', '--comment-title', default='Pull request artifacts')
file.write(''' ## Pull request artifacts args = parser.parse_args()
| file | last change | instances = []
| ---- | ------ |''') if args.instances:
for instance_arg in args.instances:
for status in gitstatus: # run this twice, once for the current version, once for the PR version instance_arg_parts = instance_arg.split('::')
if status == "current": instance = Instance()
port = "3000" # both ports are defined in the corresponding workflow .yml file instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else ''
elif status == "pr": instance.url = instance_arg_parts[0]
port = "3001" instances.append(instance)
URL = "http://localhost:" + port else:
page = requests.get(URL) # Use python requests to grab the rss-bridge main page instance = Instance()
soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup instance.name = 'current'
bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page instance.url = 'http://localhost:3000'
testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version instances.append(instance)
instance = Instance()
instance.name = 'pr'
instance.url = 'http://localhost:3001'
instances.append(instance)
main(instances=instances, with_upload=not args.no_upload, comment_title=args.comment_title);