From 21837b94c5b83667d821b37c48f47a6cf8b03b9c Mon Sep 17 00:00:00 2001 From: FrantiĊĦek Zatloukal Date: Feb 19 2021 11:29:44 +0000 Subject: Try to be more clever with bugzilla handling Such speed, much wow! --- diff --git a/oraculum/utils/bugzilla.py b/oraculum/utils/bugzilla.py index c1524f0..0eca08c 100644 --- a/oraculum/utils/bugzilla.py +++ b/oraculum/utils/bugzilla.py @@ -71,17 +71,38 @@ def get_package_bugs(package): Returns all open Bugs for a single package """ data = [] - query = BUGZILLA.url_to_query("https://bugzilla.redhat.com/buglist.cgi?bug_status=NEW&bug_status=__open__&" - "classification=Fedora&product=Fedora&product=Fedora EPEL&component=%s" % urllib.parse.quote(package)) + # Try to get the since value from db if we already have something + since = CACHE.get_refreshed_time("packager-dashboard_bugs", package) + if since: + # Let's have a faster BZ if we have something in db + since = since.strftime("%Y-%m-%d") + query = BUGZILLA.url_to_query("https://bugzilla.redhat.com/buglist.cgi?&" + "classification=Fedora&product=Fedora&chfieldfrom=%s&chfieldto=Now&" + "product=Fedora EPEL&component=%s" % (since, urllib.parse.quote(package))) + else: + query = BUGZILLA.url_to_query("https://bugzilla.redhat.com/buglist.cgi?bug_status=NEW&bug_status=__open__&" + "classification=Fedora&product=Fedora&product=Fedora EPEL&" + "component=%s" % urllib.parse.quote(package)) query["include_fields"] = ['blocks', 'comments', 'creation_time', 'creator', 'id', 'keywords', 'last_change_time', 'severity', 'priority', 'status', 'summary', 'version', 'whiteboard'] bugs = BUGZILLA.query(query) - if len(bugs) == 0: + # If we don't have anything in the db (since == None) and we didn't receive anything, return [] + if not since and len(bugs) == 0: return [] + # If we have something in db and received no changes, return the old data from db + if since and len(bugs) == 0: + return CACHE.get("packager-dashboard_bugs", package) + ftbfs_trackers = set(CACHE.get('ftbfs_trackers')) fti_trackers = set(CACHE.get('fti_trackers')) blocker_trackers = set(CACHE.get('blocker_trackers')) + closed_ids = set() + for bug in bugs: + # We need to track ids of closed bugs since time limited query doesn't fetch just the open bugs + if "CLOSED" in bug.status: + closed_ids.add(bug.id) + continue if bug.creator == 'Upstream Release Monitoring': bug.keywords.append("ReleaseMonitoring") for blocks in bug.blocks: @@ -114,4 +135,15 @@ def get_package_bugs(package): "comments": (len(bug.comments) - 1), # Let's ignore the first comment that every bug has "url": "https://bugzilla.redhat.com/%s" % bug.id }) - return data + if not since: + # Just return what we have received if we didn't limit our query by time + return data + + # And fnally combine stuff we have with stuff we just got if we limited our query with time + old_data = CACHE.get("packager-dashboard_bugs", package) + # IDs from new data (we need to join received and closed ids here, because both come from new data) + received_ids = {bug["bug_id"] for bug in data}.union(closed_ids) + + # and return data in old and not in new query and not closed in new query + data from new query (not closed) + return ([bug for bug in old_data if bug["bug_id"] not in received_ids] + + [bug for bug in data if bug["bug_id"] not in closed_ids]) diff --git a/tests/test_packager_dashboard_bugzilla.py b/tests/test_packager_dashboard_bugzilla.py index 1025f4b..e20fff8 100644 --- a/tests/test_packager_dashboard_bugzilla.py +++ b/tests/test_packager_dashboard_bugzilla.py @@ -1,3 +1,4 @@ +from datetime import date from unittest import mock from oraculum.utils import bugzilla @@ -11,7 +12,7 @@ class dotdict(dict): __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ -def bz_advanced_releases_stub(what): +def bz_cache_get_stub(what, param=None): if "fedora_releases" in what: return { "pre_eol": None, @@ -27,6 +28,24 @@ def bz_advanced_releases_stub(what): return [3,4] if "blocker_trackers" in what: return [5,6] + if "package_g" in param: + # Mock pre-cached bugs + dct = {'bug_id': 1, + 'comments': 1, + 'keywords': [], + 'modified': '2020-10-01 12:21:44', + 'priority': 'unspecified', + 'priority_severity': 'unspecified', + 'release': 'Fedora 97', + 'reported': '2020-01-24 22:08:29', + 'severity': 'unspecified', + 'status': 'NEW', + 'title': 'Some bug description', + 'url': 'https://bugzilla.redhat.com/1'} + return_data = [{**dct}, {**dct}] + return_data[1]["bug_id"] = 3 + return_data[1]["url"] = 'https://bugzilla.redhat.com/3' + return return_data def bz_query(query): dct = { @@ -67,6 +86,16 @@ def bz_query(query): dct["priority"] = "high" dct["severity"] = "urgent" + if query["pkg_name"] == "package_g": + dct["id"] = 2 + dct["last_change_time"] = "2021-10-01 12:21:44" + dct_new_bug = {**dct} + dct_new_bug["id"] = 2 + dct_closed_now = {**dct} + dct_closed_now["id"] = 3 + dct_closed_now["status"] = "CLOSED CAUSE REASONS" + return [dotdict(dct_closed_now), dotdict(dct_new_bug)] + return [dotdict(dct)] @@ -135,7 +164,8 @@ class TestBugzilla(object): expected_result_f[0]["title"] = "[fedora-all]Some bug description" expected_result_f[0]["priority_severity"] = "urgent" - monkeypatch.setattr(CACHE, 'get', bz_advanced_releases_stub) + monkeypatch.setattr(CACHE, 'get', bz_cache_get_stub) + monkeypatch.setattr(CACHE, 'get_refreshed_time', mock.MagicMock(return_value=None)) monkeypatch.setattr(BUGZILLA, 'url_to_query', lambda url: {"pkg_name": url.split("component=")[1]}) monkeypatch.setattr(BUGZILLA, 'query', bz_query) @@ -145,3 +175,50 @@ class TestBugzilla(object): assert bugzilla.get_package_bugs("package_d") == expected_result_d # F97 FTBFS assert bugzilla.get_package_bugs("package_e") == expected_result_e # F96 FTI assert bugzilla.get_package_bugs("package_f") == expected_result_f # Fedora CVE, High priotiy, Urgent severity + + def test_package_bugs_fast_path(self, monkeypatch): + expected_result = [{ + 'bug_id': 1, + 'comments': 1, + 'keywords': [], + 'modified': '2020-10-01 12:21:44', + 'priority': 'unspecified', + 'priority_severity': 'unspecified', + 'release': 'Fedora 97', + 'reported': '2020-01-24 22:08:29', + 'severity': 'unspecified', + 'status': 'NEW', + 'title': 'Some bug description', + 'url': 'https://bugzilla.redhat.com/1' + }, + { + 'bug_id': 2, + 'comments': 1, + 'keywords': [], + 'modified': '2021-10-01 12:21:44', + 'priority': 'unspecified', + 'priority_severity': 'unspecified', + 'release': 'Fedora 97', + 'reported': '2020-01-24 22:08:29', + 'severity': 'unspecified', + 'status': 'NEW', + 'title': 'Some bug description', + 'url': 'https://bugzilla.redhat.com/2' + } + # Bug with bug_id == 3 shouldn't be here because it's CLOSED ... + ] + + monkeypatch.setattr(CACHE, 'get', bz_cache_get_stub) + monkeypatch.setattr(CACHE, 'get_refreshed_time', mock.MagicMock(return_value=date(day=1, month=10, year=2021))) + monkeypatch.setattr(BUGZILLA, 'url_to_query', lambda url: {"pkg_name": url.split("component=")[1], + "url": url}) + monkeypatch.setattr(BUGZILLA, 'query', bz_query) + + assert bugzilla.get_package_bugs("package_g") == expected_result + + # Let's try a scenario where bugzilla doesn't return anything new and we have something in cache + monkeypatch.setattr(BUGZILLA, 'query', mock.MagicMock(return_value=[])) + expected_result[1]["modified"] = "2020-10-01 12:21:44" + expected_result[1]["bug_id"] = 3 + expected_result[1]["url"] = "https://bugzilla.redhat.com/3" + assert bugzilla.get_package_bugs("package_g") == [expected_result[0], expected_result[1]]