#35 If we fail to call bugzilla, retry a few times before raising the exception
Merged 4 years ago by nphilipp. Opened 4 years ago by pingou.
fedora-infra/ pingou/distgit-bugzilla-sync retry_bz  into  master

@@ -35,6 +35,7 @@ 

  from email.message import EmailMessage

  import itertools

  import json

+ import math

  from operator import itemgetter

  import os

  import re
@@ -157,6 +158,20 @@ 

      def build_product_cache(self, pagure_projects):

          """ Cache the bugzilla info about each package in each product.

          """

+         def _query_component(query, num_attempts=5):

+             for i in range(num_attempts):

+                 try:

+                     raw_data = self.server._proxy.Component.get({'names': query})

+                     break

+                 except Exception as e:

+                     if i >= num_attempts - 1:

+                         raise

+                     if self.config['verbose']:

+                         print(f"    ERROR {e}")

+                         print("    - Query failed, going to try again in 20 seconds")

+                     # Wait 20 seconds and try again

+                     time.sleep(20)

+             return raw_data

  

          if self.config['bugzilla']['compat_api'] == 'getcomponentsdetails':

              # Old API -- in python-bugzilla.  But with current server, this
@@ -167,6 +182,7 @@ 

          elif self.config['bugzilla']['compat_api'] == 'component.get':

              # Way that's undocumented in the partner-bugzilla api but works

              # currently

+             chunk = self.config['bugzilla']['req_segment']

              for collection, product in self.config["products"].items():

                  bz_product_name = product.get('bz_product_name', collection)

                  # restrict the list of info returned to only the packages of
@@ -177,7 +193,8 @@ 

                      if bz_product_name in project["products"]

                  ]

                  product_info_by_pkg = {}

-                 for pkg_segment in segment(pkglist, self.config['bugzilla']['req_segment']):

+                 estimated_size = math.ceil(len(pkglist) / chunk)

+                 for cnt, pkg_segment in enumerate(segment(pkglist, chunk), start=1):

                      # Format that bugzilla will understand.  Strip None's that

                      # segment() pads out the final data segment() with

                      query = [
@@ -185,7 +202,11 @@ 

                          for p in pkg_segment

                          if p is not None

                      ]

-                     raw_data = self.server._proxy.Component.get({'names': query})

+                     if self.config['verbose']:

+                         print(

+                             f"  - Querying product `{bz_product_name}`, "

+                             f"query {cnt} of {estimated_size}")

+                     raw_data = _query_component(query)

                      for package in raw_data['components']:

                          # Reformat data to be the same as what's returned from

                          # getcomponentsdetails
@@ -233,12 +254,14 @@ 

          bz_email = email_overrides.get(bz_email, bz_email)

          return bz_email

  

-     def update_open_bugs(self, new_poc, prev_poc, product, name):

+     def update_open_bugs(self, new_poc, prev_poc, product, name, print_fas_names=False):

          '''Change the package owner

          :arg new_poc: email of the new point of contact.

          :arg prev_poc: Username of the previous point of contact

          :arg product: The product of the package to change in bugzilla

          :arg name: Name of the package to change the owner.

+         :kwarg print_fas_names: Boolean specifying wether to print email or FAS names

+             (if these could be found).

          '''

          bz_query = {}

          bz_query['product'] = product
@@ -254,6 +277,16 @@ 

          for bug in query_results:

              if bug.assigned_to == prev_poc and bug.assigned_to != new_poc:

                  if self.config["verbose"]:

+                     old_poc = bug.assigned_to

+                     if print_fas_names:

+                         if old_poc in self.inverted_user_cache:

+                             old_poc = self.inverted_user_cache[old_poc]

+                         else:

+                             old_poc = old_poc.split('@', 1)[0] + "@..."

+                         if new_poc in self.inverted_user_cache:

+                             new_poc = self.inverted_user_cache[new_poc]

+                         else:

+                             new_poc = newpoc.split('@', 1)[0] + "@..."

                      print(

                          f'    - reassigning bug #{bug.bug_id} '

                          f'from {bug.assigned_to} to {new_poc}'
@@ -414,6 +447,7 @@ 

                          prev_poc=product[pkg_key]['initialowner'],

                          name=package,

                          product=bz_product_name,

+                         print_fas_names=print_fas_names,

                      )

              else:

                  if self.config.get("print-no-change"):

This allows for temporary network glitches to be ignored.

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr

Metadata Update from @nphilipp:
- Request assigned

4 years ago

Hmm, cnt shouldn't have to be initialized to 0 when calling the function. How about just specifying the number of attempts if you want to deviate from the default 5 and get rid of the recursion in the process?

         def _query_component(query, num_attempts=5):
            for i in range(num_attempts):
                try:
                    raw_data = self.server._proxy.Component.get({'names': query})
                except Exception as e:
                    if i >= num_attempts - 1:
                        raise
                    if self.config['verbose']:
                        print(f"    ERROR {e}")
                        print("    - Query failed, going to try again in 20 seconds")
                    # Wait 20 seconds and try again
                    time.sleep(20)
            return raw_data
...
            raw_data = _query_component(query)

I guess this could be easily made generic enough so we can reuse it for other "repeat N times after failure" situations, too (just musing, out of scope for this PR).

You don't have to manually set and increment cnt here, just use enumerate():

    for cnt, pkg_segment in enumerate(segment(pkglist, chunk), start=1):
        ...

I've tried this first but it doesn't work with what is returned by segment()

rebased onto 2fd1d0115de9d647d1c1206784f04309bb50d36b

4 years ago

2 new commits added

  • When updating bugs do not print email address if using --print-fas-names
  • If we fail to call bugzilla, retry a few times before raising the exception
4 years ago

Looks good to me, thanks!

Pull-Request has been merged by nphilipp

4 years ago
Metadata