#35 Report last completed build when FTBFS
Merged 3 years ago by frantisekz. Opened 3 years ago by frantisekz.

@@ -30,6 +30,8 @@ 

  ORPHANS_JSON_URL = "https://churchyard.fedorapeople.org/orphans.json"

  KOSCHEI_API_URL = "https://koschei.fedoraproject.org/api/v1/packages"

  HEALTH_CHECK_URL = "https://repochecker.decathorpe.com/data/"

+ KOJI_URL = "https://koji.fedoraproject.org/koji/"

+ KOJIHUB_URL = "https://koji.fedoraproject.org/kojihub/"

  

  EPEL_RELEASES = [7, 8]

  

file modified
+2
@@ -58,6 +58,8 @@ 

      ORPHANS_JSON_URL = 'https://churchyard.fedorapeople.org/orphans.json'

      KOSCHEI_API_URL = 'https://koschei.fedoraproject.org/api/v1/packages'

      HEALTH_CHECK_URL = 'https://repochecker.decathorpe.com/data/'

+     KOJI_URL = 'https://koji.fedoraproject.org/koji/'

+     KOJIHUB_URL = 'https://koji.fedoraproject.org/kojihub/'

  

      EPEL_RELEASES = [7, 8]

  

file modified
+54 -4
@@ -29,6 +29,7 @@ 

  import datetime

  import requests

  import bugzilla

+ import koji

  

  from bodhi.client.bindings import BodhiClient

  from urllib3.util.retry import Retry
@@ -99,7 +100,6 @@ 

          return "Unknown"

      return release_from_dist(split)

  

- 

  def release_from_dist(dist):

      """

      Returns Fedora XX or EPEL X from dist (fcXX/elX or fXX/epelX)
@@ -112,6 +112,18 @@ 

          return "Fedora Rawhide"

      return "Fedora %s" % dist[-2:]

  

+ def release_is_active(release_string):

+     """

+     Gets release string (Fedora XX/EPEL X), returns False if the release is EOL, return True otherwise

+     """

+     releases = CACHE.get("fedora_releases")

+     if "Rawhide" in release_string:

+         return True

+     release_string = int(release_string.replace("Fedora ", "").replace("EPEL ", "")) # Get just the number

+     if release_string in releases["values"] or release_string in app.config["EPEL_RELEASES"]:

+         return True

+     return False

+ 

  def process_update(update):

      """

      Cleans up single update dictionary to contain only data frontend needs
@@ -468,6 +480,34 @@ 

                  data[package].append(process_override(override))

      return data

  

+ def process_koji_queue(session, data):

+     koji_results = session.multiCall()

+     for element in koji_results:

+         try:

+             # koji multicall returns results packed inside lists of one element, redefine element for better readability later

+             element = element[0][0]

+         except IndexError:

+             # Sometimes, koji returns incomplete results

+             continue

+         for entry in data[element["name"]]:

+             """

+             We need to iterate throughout all members of data["package_name"] to find and alter only the architecture we

+             now have in the element.

+             Example of element (only keys we use are listed below):

+                {'tag_name': 'fXX',

+                 'build_id': XYZ,

+                 'completion_time': 'YYYY-MM-DD HH:MM:SS.FFFFFF',

+                 'name': 'package_name',

+                 ...}

+             """

+             # Since dict doesn't guarantee order, we must pair koji result and data entry

+             if release_from_dist(element["tag_name"]) == entry["release"]:

+                 data[element["name"]][data[element["name"]].index(entry)]["last_success"] = {

+                     "time": element["completion_time"],

+                     "url": app.config["KOJI_URL"] + "buildinfo?buildID=" + str(element["build_id"])}

+                 # And we can jump to another element early if we found what we needed, yay!

+                 continue

+     return data

  

  def parse_koschei_data():

      """
@@ -480,16 +520,26 @@ 

      """

      data = defaultdict(list)

      koschei_resp = get_json(app.config['KOSCHEI_API_URL'])

+     # Set up koji API

+     koji_session = koji.ClientSession(app.config['KOJIHUB_URL'])

+     koji_session.multicall = True

+ 

      for item in koschei_resp:

          if "playground" in item["collection"]:

              # We don't care about EPEL 8 Playground

              continue

+         if not release_is_active(release_from_dist(item["collection"])):

+             # koschei might contain data for EOL Fedora releases, we don't care about those

+             continue

          data[item["name"]].append({

              "release": release_from_dist(item["collection"]),

              "status": item["state"],

-             "url": "https://koschei.fedoraproject.org/package/%s?collection=%s" % (item["name"], item["collection"])

-         })

-     return data

+             "url": "https://koschei.fedoraproject.org/package/%s?collection=%s" % (item["name"], item["collection"]),

+             "last_success": {"time": None, "url": None}})

+         if item["state"] == "failing":

+             koji_session.getLatestBuilds(item["collection"], package=item["name"])

+ 

+     return process_koji_queue(koji_session, data)

  

  

  def get_user_koschei_data(packages):

no initial comment

The only problem (more like possible problem) I see here is koji throttling our requests. Maybe have a look at how request batching was implemented in taskotron?

Other than that, LGTM

rebased onto 32bdd2f

3 years ago

1 new commit added

  • Introduce release_is_active() to get active/EOL status from release string
3 years ago

In Taskotron we used koji.session.multiCall(), which increases the speed of requests by an order of magnitude and also lowers server load.
https://pagure.io/taskotron/libtaskotron/blob/develop/f/libtaskotron/ext/fedora/koji_utils.py

rebased onto 550d265

3 years ago

1 new commit added

  • Make use of koji multicall
3 years ago

So, koji multicall is now implemented and being used here.

Thanks @kparal and @jskladan for tips/hints!

I'm not really sure what's happening here, could you add a comment detailing the structure of the data received from koji, so it is easily visible for the "future Josef and Frantisek who stare at this code"? Thanks!

Apart of ^^^ I guess this is OK. Thanks!

rebased onto 1ef58ca

3 years ago

1 new commit added

  • Improve documentation
3 years ago

Added comment, I am not sure it's great way of / enough explaining, wdyt @jskladan ?

rebased onto 300cfd9

3 years ago

rebased onto 7c27fae

3 years ago

Pull-Request has been merged by frantisekz

3 years ago