From 09f3c1532adc20c5850e3f0751c24d2b402386d0 Mon Sep 17 00:00:00 2001 From: User123698745 Date: Sun, 24 Sep 2023 21:13:01 +0200 Subject: [PATCH] [core] improve pull request artifacts comment (#3705) --- .github/prtester.py | 240 ++++++++++++++++++++++++++------------------ 1 file changed, 143 insertions(+), 97 deletions(-) diff --git a/.github/prtester.py b/.github/prtester.py index df6cc1ff..a2a7ab84 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -1,113 +1,159 @@ +import argparse import requests -import itertools from bs4 import BeautifulSoup from datetime import datetime +from typing import Iterable import os.path # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge # # This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of # RSS-Bridge, generate a feed for each of the bridges and save the output as html files. -# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing +# It also add a tag with the url of em's public instance, so viewing # the HTML file locally will actually work as designed. -def testBridges(bridges,status): - for bridge in bridges: - if bridge.get('data-ref'): # Some div entries are empty, this ignores those - bridgeid = bridge.get('id') - bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata - print(bridgeid + "\n") - bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' - forms = bridge.find_all("form") - formid = 1 - for form in forms: - # a bridge can have multiple contexts, named 'forms' in html - # this code will produce a fully working formstring that should create a working feed when called - # this will create an example feed for every single context, to test them all - formstring = '' - errormessages = [] - parameters = form.find_all("input") - lists = form.find_all("select") - # this for/if mess cycles through all available input parameters, checks if it required, then pulls - # the default or examplevalue and then combines it all together into the formstring - # if an example or default value is missing for a required attribute, it will throw an error - # any non-required fields are not tested!!! - for parameter in parameters: - if parameter.get('type') == 'hidden' and parameter.get('name') == 'context': - cleanvalue = parameter.get('value').replace(" ","+") - formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue - if parameter.get('type') == 'number' or parameter.get('type') == 'text': - if parameter.has_attr('required'): - if parameter.get('placeholder') == '': - if parameter.get('value') == '': - errormessages.append(parameter.get('name')) - else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') +class Instance: + name = '' + url = '' + +def main(instances: Iterable[Instance], with_upload: bool, comment_title: str): + start_date = datetime.now() + table_rows = [] + for instance in instances: + page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page + soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup + bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page + table_rows += testBridges(instance, bridge_cards, with_upload) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version + with open(file=os.getcwd() + '/comment.txt', mode='w+', encoding='utf-8') as file: + table_rows_value = '\n'.join(sorted(table_rows)) + file.write(f''' +## {comment_title} +| Bridge | Context | Status | +| - | - | - | +{table_rows_value} + +*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}* + '''.strip()) + +def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) -> Iterable: + instance_suffix = '' + if instance.name: + instance_suffix = f' ({instance.name})' + table_rows = [] + for bridge_card in bridge_cards: + bridgeid = bridge_card.get('id') + bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata + print(f'{bridgeid}{instance_suffix}\n') + bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' + bridge_name = bridgeid.replace('Bridge', '') + context_forms = bridge_card.find_all("form") + form_number = 1 + for context_form in context_forms: + # a bridge can have multiple contexts, named 'forms' in html + # this code will produce a fully working formstring that should create a working feed when called + # this will create an example feed for every single context, to test them all + formstring = '' + error_messages = [] + context_name = '*untitled*' + context_name_element = context_form.find_previous_sibling('h5') + if context_name_element and context_name_element.text.strip() != '': + context_name = context_name_element.text + parameters = context_form.find_all("input") + lists = context_form.find_all("select") + # this for/if mess cycles through all available input parameters, checks if it required, then pulls + # the default or examplevalue and then combines it all together into the formstring + # if an example or default value is missing for a required attribute, it will throw an error + # any non-required fields are not tested!!! + for parameter in parameters: + if parameter.get('type') == 'hidden' and parameter.get('name') == 'context': + cleanvalue = parameter.get('value').replace(" ","+") + formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue + if parameter.get('type') == 'number' or parameter.get('type') == 'text': + if parameter.has_attr('required'): + if parameter.get('placeholder') == '': + if parameter.get('value') == '': + name_value = parameter.get('name') + error_messages.append(f'Missing example or default value for parameter "{name_value}"') else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') - # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring - if parameter.get('type') == 'checkbox': - if parameter.has_attr('checked'): - formstring = formstring + '&' + parameter.get('name') + '=on' - for listing in lists: - selectionvalue = '' - listname = listing.get('name') - cleanlist = [] - for option in listing.contents: - if 'optgroup' in option.name: - cleanlist.extend(option) + formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') else: - cleanlist.append(option) - firstselectionentry = 1 - for selectionentry in cleanlist: - if firstselectionentry: + formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') + # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring + if parameter.get('type') == 'checkbox': + if parameter.has_attr('checked'): + formstring = formstring + '&' + parameter.get('name') + '=on' + for listing in lists: + selectionvalue = '' + listname = listing.get('name') + cleanlist = [] + for option in listing.contents: + if 'optgroup' in option.name: + cleanlist.extend(option) + else: + cleanlist.append(option) + firstselectionentry = 1 + for selectionentry in cleanlist: + if firstselectionentry: + selectionvalue = selectionentry.get('value') + firstselectionentry = 0 + else: + if 'selected' in selectionentry.attrs: selectionvalue = selectionentry.get('value') - firstselectionentry = 0 - else: - if 'selected' in selectionentry.attrs: - selectionvalue = selectionentry.get('value') - break - formstring = formstring + '&' + listname + '=' + selectionvalue - if not errormessages: - # if all example/default values are present, form the full request string, run the request, replace the static css - # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site. - r = requests.get(URL + bridgestring + formstring) - pagetext = r.text.replace('static/style.css','https://rss-bridge.org/bridge01/static/style.css') - pagetext = pagetext.encode("utf_8") - termpad = requests.post(url="https://termpad.com/", data=pagetext) - termpadurl = termpad.text - termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') - termpadurl = termpadurl.replace('\n','') - with open(os.getcwd() + '/comment.txt', 'a+') as file: - file.write("\n") - file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |") + break + formstring = formstring + '&' + listname + '=' + selectionvalue + termpad_url = 'about:blank' + if error_messages: + status = '
'.join(map(lambda m: f'❌ `{m}`', error_messages)) + else: + # if all example/default values are present, form the full request string, run the request, add a tag with + # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and + # then upload it to termpad.com, a pastebin-like-site. + response = requests.get(instance.url + bridgestring + formstring) + page_text = response.text.replace('','') + page_text = page_text.encode("utf_8") + soup = BeautifulSoup(page_text, "html.parser") + status_messages = list(map(lambda e: f'⚠️ `{e.text.strip().splitlines()[0]}`', soup.find_all('pre'))) + if response.status_code != 200: + status_messages = [f'❌ `HTTP status {response.status_code} {response.reason}`'] + status_messages else: - # if there are errors (which means that a required value has no example or default value), log out which error appeared - termpad = requests.post(url="https://termpad.com/", data=str(errormessages)) - termpadurl = termpad.text - termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') - termpadurl = termpadurl.replace('\n','') - with open(os.getcwd() + '/comment.txt', 'a+') as file: - file.write("\n") - file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |") - formid += 1 + feed_items = soup.select('.feeditem') + feed_items_length = len(feed_items) + if feed_items_length <= 0: + status_messages += [f'⚠️ `The feed has no items`'] + elif feed_items_length == 1 and len(soup.select('.error')) > 0: + status_messages = [f'❌ `{feed_items[0].text.strip().splitlines()[0]}`'] + status_messages + status = '
'.join(status_messages) + if status.strip() == '': + status = '✔️' + if with_upload: + termpad = requests.post(url="https://termpad.com/", data=page_text) + termpad_url = termpad.text.strip() + termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/') + table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |') + form_number += 1 + return table_rows -gitstatus = ["current", "pr"] -now = datetime.now() -date_time = now.strftime("%Y-%m-%d, %H:%M:%S") - -with open(os.getcwd() + '/comment.txt', 'w+') as file: - file.write(''' ## Pull request artifacts -| file | last change | -| ---- | ------ |''') - -for status in gitstatus: # run this twice, once for the current version, once for the PR version - if status == "current": - port = "3000" # both ports are defined in the corresponding workflow .yml file - elif status == "pr": - port = "3001" - URL = "http://localhost:" + port - page = requests.get(URL) # Use python requests to grab the rss-bridge main page - soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup - bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page - testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--instances', nargs='+') + parser.add_argument('-nu', '--no-upload', action='store_true') + parser.add_argument('-t', '--comment-title', default='Pull request artifacts') + args = parser.parse_args() + instances = [] + if args.instances: + for instance_arg in args.instances: + instance_arg_parts = instance_arg.split('::') + instance = Instance() + instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else '' + instance.url = instance_arg_parts[0] + instances.append(instance) + else: + instance = Instance() + instance.name = 'current' + instance.url = 'http://localhost:3000' + instances.append(instance) + instance = Instance() + instance.name = 'pr' + instance.url = 'http://localhost:3001' + instances.append(instance) + main(instances=instances, with_upload=not args.no_upload, comment_title=args.comment_title); \ No newline at end of file