PR#84: QA Landing page code cleanup - fedora-qa/oraculum

fedora-qa / oraculum

#84 QA Landing page code cleanup

Merged 3 years ago by frantisekz. Opened 3 years ago by frantisekz.

ladning_page_cleanup into master

František Zatloukal • 3 years ago

959e8af

conf/settings.py.example

file modified

		`@@ -38,6 +38,8 @@`
		`HEALTH_CHECK_URL = "https://repochecker.decathorpe.com/data/"`
		`KOJI_URL = "https://koji.fedoraproject.org/koji/"`
		`KOJIHUB_URL = "https://koji.fedoraproject.org/kojihub/"`
		`+ BLOCKERBUGS_URL = "https://qa.fedoraproject.org/blockerbugs/"`
		`+ SCHEDULE_URL = "https://fedorapeople.org/groups/schedule/f-%d/f-%d-key.ics"`

		`EPEL_RELEASES = [6, 7, 8]`

oraculum/action_providers/fedora_manual_testing.py

file modified

+9 -5

		`@@ -1,9 +1,7 @@`
		`import pprint`
		`import re`
		`- import requests`

		`from oraculum.utils import Action, ACTION_TIMES`
		`- from oraculum.utils import schedule`

		`INFO = {`
		`'provider': 'fedora_manual_testing',`
		`@@ -27,10 +25,16 @@`


		`def load_stats():`
		`- resp = requests.get("https://www.happyassassin.net/testcase_stats/%d/data.json" % schedule.current_devel())`
		`- if not resp.ok:`
		`+ # Work around cyclic import`
		`+ from oraculum import CACHE`
		`+ from oraculum.utils.dashboard_helpers import get_json`
		`+`
		`+ releases = CACHE.get("fedora_releases")`
		`+ release = releases["branched"] or releases["rawhide"]`
		`+`
		`+ data = get_json("https://www.happyassassin.net/testcase_stats/%d/data.json" % release)`
		`+ if not data:`
		`return []`
		`- data = resp.json()`

		`composes_rev = list(reversed(data['allcomposes']))`
		`composes_len = len(composes_rev)`

oraculum/config.py

file modified

		`@@ -67,6 +67,8 @@`
		`HEALTH_CHECK_URL = 'https://repochecker.decathorpe.com/data/'`
		`KOJI_URL = 'https://koji.fedoraproject.org/koji/'`
		`KOJIHUB_URL = 'https://koji.fedoraproject.org/kojihub/'`
		`+ BLOCKERBUGS_URL = 'https://qa.fedoraproject.org/blockerbugs/'`
		`+ SCHEDULE_URL = 'https://fedorapeople.org/groups/schedule/f-%d/f-%d-key.ics'`

		`EPEL_RELEASES = [6, 7, 8]`

oraculum/controllers/landing_page.py

file modified

+3 -2

		`@@ -28,13 +28,14 @@`


		`def api_v1_landing_page():`
		`+ releases = CACHE.get("fedora_releases")`
		`return {`
		`'blockerbugs': blockerbugs.get_blockerbugs(),`
		`'meetings': fedocal.get_qa_meetings(),`
		`'last_qa_meeting': meetbot.get_last_qa_meeting(),`
		`'schedule': schedule.get_schedule(),`
		`- 'stable': schedule.current_stable(),`
		`- 'devel': schedule.current_devel(),`
		`+ 'stable': releases["stable"],`
		`+ 'devel': releases["branched"] or releases["rawhide"],`
		`}`

oraculum/utils/blockerbugs.py

file modified

+33 -18

		`@@ -1,10 +1,32 @@`
		`- import requests`
		`- import json`
		`- from oraculum.utils import schedule`
		`+ #`
		`+ # blockerbugs.py - Blockerbugs parsers and utilities for QA Ladning Page`
		`+ #`
		`+ # Copyright 2020, Red Hat, Inc`
		`+ #`
		`+ # This program is free software; you can redistribute it and/or modify`
		`+ # it under the terms of the GNU General Public License as published by`
		`+ # the Free Software Foundation; either version 2 of the License, or`
		`+ # (at your option) any later version.`
		`+ #`
		`+ # This program is distributed in the hope that it will be useful,`
		`+ # but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
		`+ # GNU General Public License for more details.`
		`+ #`
		`+ # You should have received a copy of the GNU General Public License along`
		`+ # with this program; if not, write to the Free Software Foundation, Inc.,`
		`+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.`
		`+ #`
		`+ # Authors:`
		`+ # Frantisek Zatloukal <fzatlouk@redhat.com>`
		`+ # Josef Skladanka <jskladan@redhat.com>`

		`- # FIXME - make it possible to use qa.fp.o/blockerbugs directly`
		`- BLOCKERBUGS_URL = "http://37.205.11.174:8080/api/v0/milestones/%d/%s/bugs"`
		`+ import urllib`

		`+ from oraculum import app, CACHE`
		`+ from oraculum.utils.dashboard_helpers import get_json`
		`+`
		`+ BLOCKERBUGS_URL = urllib.parse.urljoin(app.config["BLOCKERBUGS_URL"], "api/v0/milestones/%d/%s/bugs")`

		`def get_blockerbugs():`
		`data = {`
		`@@ -17,22 +39,16 @@`
		`"final_fe": "NaN",`
		`"final_fe_proposed": "NaN"`
		`}`
		`- release = schedule.current_devel()`
		`+ releases = CACHE.get("fedora_releases")`
		`+ release = releases["branched"] or releases["rawhide"]`

		`- try:`
		`- resp_beta = requests.get(BLOCKERBUGS_URL % (release, "beta"))`
		`- resp_final = requests.get(BLOCKERBUGS_URL % (release, "final"))`
		`- except:`
		`- return data`
		`+ resp_beta = get_json(BLOCKERBUGS_URL % (release, "beta"))`
		`+ resp_final = get_json(BLOCKERBUGS_URL % (release, "final"))`

		`beta_blockers, beta_fe, final_blockers, final_fe = (0, 0, 0, 0)`
		`beta_blockers_prop, beta_fe_prop, final_blockers_prop, final_fe_prop = (0, 0, 0, 0)`

		`- if resp_beta.status_code != 200 or resp_final.status_code != 200:`
		`- return data`
		`-`
		`- bugs_list = json.loads(resp_beta.text)`
		`- for bug in bugs_list:`
		`+ for bug in resp_beta:`
		`if not (bug["active"]):`
		`continue`
		`if "accepted_blocker" in bug["type"]:`
		`@@ -44,8 +60,7 @@`
		`if "proposed_fe" in bug["type"]:`
		`beta_fe_prop += 1`

		`- bugs_list = json.loads(resp_final.text)`
		`- for bug in bugs_list:`
		`+ for bug in resp_final:`
		`if not (bug["active"]):`
		`continue`
		`if "accepted_blocker" in bug["type"]:`

oraculum/utils/schedule.py

file modified

+52 -32

		`@@ -1,48 +1,68 @@`
		`+ #`
		`+ # schedule.py - Fedora Release Schedule parsers and utilities for QA Ladning Page`
		`+ #`
		`+ # Copyright 2020, Red Hat, Inc`
		`+ #`
		`+ # This program is free software; you can redistribute it and/or modify`
		`+ # it under the terms of the GNU General Public License as published by`
		`+ # the Free Software Foundation; either version 2 of the License, or`
		`+ # (at your option) any later version.`
		`+ #`
		`+ # This program is distributed in the hope that it will be useful,`
		`+ # but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
		`+ # GNU General Public License for more details.`
		`+ #`
		`+ # You should have received a copy of the GNU General Public License along`
		`+ # with this program; if not, write to the Free Software Foundation, Inc.,`
		`+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.`
		`+ #`
		`+ # Authors:`
		`+ # Frantisek Zatloukal <fzatlouk@redhat.com>`
		`+ # Josef Skladanka <jskladan@redhat.com>`
		`+ # Lukas Brabec <lbrabec@redhat.com>`
		`+`
		`import icalendar`
		`- import requests`
		`import datetime`
		`import pytz`

		`- SCHEDULE_URL = "https://fedorapeople.org/groups/schedule/f-%d/f-%d-key.ics"`
		`- RELEASES_URL = "https://getfedora.org/releases.json"`
		`-`
		`-`
		`- def current_stable():`
		`- resp = requests.get(RELEASES_URL)`
		`- if resp.status_code == 200:`
		`- releases = resp.json()`
		`- return max([int(r['version']) for r in releases if 'Beta' not in r['version']]) # FIXME`
		`- else:`
		`- return -1 # FIXME`
		`-`
		`-`
		`- def current_devel():`
		`- return current_stable() + 1`
		`-`
		`+ from oraculum import app`
		`+ from oraculum.utils.dashboard_helpers import get_json`

		`def _is_relevant(summary):`
		`- relevant = ['Branch Fedora', 'Beta Freeze starts', 'Beta Release', 'Final Freeze starts', 'Final Release']`
		`+ relevant = ['Branch Fedora', 'Beta Freeze', 'Beta Release Public Availability', 'Final Freeze', 'Final Release Public Availability']`
		`return any([word in summary for word in relevant])`


		`def get_schedule():`
		`+`
		`+ # Work around cyclic import`
		`+ from oraculum import CACHE`
		`+`
		`today = datetime.datetime.utcnow()`
		`today = today.replace(tzinfo=pytz.UTC)`
		`- resp = requests.get(SCHEDULE_URL % (current_devel(), current_devel())) # FIXME`
		`+`
		`+ releases = CACHE.get("fedora_releases")`
		`+ release = releases["branched"] or releases["rawhide"]`
		`+`
		`+ resp = get_json(app.config["SCHEDULE_URL"] % (release, release), ignore_json_errors=True)`
		`data = []`
		`- if resp.status_code == 200: # FIXME`
		`- ical_raw = icalendar.Calendar.from_ical(resp.text)`
		`- for c in ical_raw.walk():`
		`- if c.name == "VEVENT":`
		`- summary = c.get('summary')`
		`- date = c.get('dtstart').dt.astimezone(pytz.UTC)`
		`- if _is_relevant(summary):`
		`- data.append({`
		`- 'summary': summary,`
		`- 'date': date.strftime("%d %b %Y"),`
		`- 'dtdate': date,`
		`- 'current': date <= today # mark all until now as current and clean up later`
		`- })`
		`+`
		`+ if not resp:`
		`+ return []`
		`+`
		`+ ical_raw = icalendar.Calendar.from_ical(resp)`
jskladan commented 3 years ago So, what happens now, if the response was not 200?
		`+ for c in ical_raw.walk():`
		`+ if c.name == "VEVENT":`
		`+ summary = c.get('summary')`
		`+ date = c.get('dtstart').dt.astimezone(pytz.UTC)`
		`+ if _is_relevant(summary):`
		`+ data.append({`
		`+ 'summary': summary,`
		`+ 'date': date.strftime("%d %b %Y"),`
		`+ 'dtdate': date,`
		`+ 'current': date <= today # mark all until now as current and clean up later`
		`+ })`

		`data = sorted(data, key=lambda c: c['dtdate'])`