From 5c91e5e533ccd77bfbc7b1942ec3b143f9b7660f Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Oct 12 2020 21:43:17 +0000 Subject: [PATCH 1/2] Attempt to reduce the amount of mdapi queries --- diff --git a/oraculum/utils/pagure.py b/oraculum/utils/pagure.py index 6aa6caf..ea122bb 100644 --- a/oraculum/utils/pagure.py +++ b/oraculum/utils/pagure.py @@ -153,7 +153,7 @@ def get_package_versions(package): versions = {} releases = CACHE.get("fedora_releases") for release in releases["values"][:-1]: # Skip Rawhide - versions[release_from_dist(str(release))] = {"stable": None, "testing": None} + versions["Fedora %d" % release] = {"stable": None, "testing": None} # Append Rawhide in standardized naming format versions["Fedora Rawhide"] = {"stable": None, "testing": None} for release in app.config["EPEL_RELEASES"]: @@ -182,6 +182,12 @@ def get_package_versions(package): for release in versions: if "Fedora" in release and not versions[release]["stable"]: if "Rawhide" in release: + # Before the mass rebuild, it'll be pretty common for packages to not have any Rawhide build + # Let's copy out version from previous Fedora for cases when Rawhide does not have any build + if versions["Fedora %d" % releases["values"][-2]]["stable"]: + versions[release]["stable"] = versions["Fedora %d" % releases["values"][-2]]["stable"] + # Such speed, much wow! + continue mdapi_response = get_json("https://mdapi.fedoraproject.org/%s/srcpkg/%s" % ("rawhide", package), False, False) else: mdapi_response = get_json("https://mdapi.fedoraproject.org/%s/srcpkg/%s" % ("f" + release[-2:], package), False, False) From d9e04854175972ffcf976c7c673a49b5db262de8 Mon Sep 17 00:00:00 2001 From: František Zatloukal Date: Oct 13 2020 06:53:55 +0000 Subject: [PATCH 2/2] Don't retry requests to mdapi at all --- diff --git a/oraculum/utils/dashboard_helpers.py b/oraculum/utils/dashboard_helpers.py index 84876e2..9d002c4 100644 --- a/oraculum/utils/dashboard_helpers.py +++ b/oraculum/utils/dashboard_helpers.py @@ -42,10 +42,14 @@ def get_json(json_url, attempt_retry=True, log_errors=True, ignore_json_errors=F Returns json data from provided url """ session = requests.Session() - retries = Retry(total=3, - backoff_factor=0.1, - status_forcelist=[429, 500, 502, 503, 504]) - session.mount('https://', HTTPAdapter(max_retries=retries)) + + if attempt_retry: + retries = Retry(total=3, + backoff_factor=0.1, + status_forcelist=[429, 500, 502, 503, 504]) + session.mount('https://', HTTPAdapter(max_retries=retries)) + else: + session.mount('https://', HTTPAdapter()) try: resp = session.get(json_url) except ConnectionError: