#3 Make the code run with Python 3, add setup.py and small cleanups
Merged 8 months ago by pingou. Opened 8 months ago by nphilipp.
fedora-infra/ nphilipp/distgit-bugzilla-sync master--python3-cleanup-setup.py  into  master

pagure_sync_bugzilla.py pagure-sync-bugzilla.py
file renamed
+49 -49

@@ -1,4 +1,4 @@ 

- #!/usr/bin/python -tt

+ #!/usr/bin/python3 -tt

  # -*- coding: utf-8 -*-

  #

  # Copyright © 2013-2019  Red Hat, Inc.

@@ -29,35 +29,29 @@ 

  This ... script takes information about package onwership and imports it

  into bugzilla.

  '''

- from __future__ import print_function

- import re

+ 

  import argparse

  import datetime

- import time

- import sys

- import os

+ from email.message import EmailMessage

So we are py3 only?

Well, we can run this script from practically anywhere given the right credentials to retrieve the list of users and to do changes in Bugzilla. Then, Python 2 is officially dead in about 6 weeks. Unless there is a really good reason to support Python 2, I don't want to support it, or do you?

wfm and your setup.py made it clear

  import itertools

  import json

- import xmlrpclib

- import codecs

+ import multiprocessing.pool

+ import os

+ import re

  import smtplib

+ import sys

+ import time

  import traceback

- import multiprocessing.pool

- try:

-     from email.Message import Message

- except ImportError:

-     from email.message import EmailMessage as Message

+ import xmlrpc.client

  

- import bugzilla as bugzilla_lib

+ from bugzilla import Bugzilla

  import dogpile.cache

- import requests

- import yaml

- from six import string_types

  import fedora.client

  from fedora.client.fas2 import AccountSystem

- 

+ import requests

  from requests.adapters import HTTPAdapter

- from requests.packages.urllib3.util.retry import Retry

+ from urllib3.util import Retry

+ import yaml

  

  

  env = 'staging'

@@ -151,7 +145,7 @@ 

      'modules': 'module',

      'container': 'container',

  }

- INVERSE_PDC_TYPES = dict([(v, k) for k, v in PDC_TYPES.items()])

+ INVERSE_PDC_TYPES = {v: k for k, v in PDC_TYPES.items()}

  

  

  # When querying for current info, take segments of 1000 packages a time

@@ -231,7 +225,7 @@ 

  def segment(iterable, chunk, fill=None):

      '''Collect data into `chunk` sized block'''

      args = [iter(iterable)] * chunk

-     return itertools.izip_longest(*args, fillvalue=fill)

+     return itertools.zip_longest(*args, fillvalue=fill)

  

  

  class ProductCache(dict):

@@ -256,7 +250,7 @@ 

          elif BZCOMPAPI == 'component.get':

              # Way that's undocumented in the partner-bugzilla api but works

              # currently

-             pkglist = projects_dict[key].keys()

+             pkglist = list(projects_dict[key])

              products = {}

              for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT):

                  # Format that bugzilla will understand.  Strip None's that

@@ -280,14 +274,14 @@ 

          return super(ProductCache, self).__getitem__(key)

  

  

- class BugzillaProxy(object):

+ class BugzillaProxy:

  

      def __init__(self, bzServer, username, password, acls):

          self.bzXmlRpcServer = bzServer

          self.username = username

          self.password = password

  

-         self.server = bugzilla_lib.Bugzilla(

+         self.server = Bugzilla(

              url=self.bzXmlRpcServer,

              user=self.username,

              password=self.password)

@@ -355,11 +349,11 @@ 

          # Lookup product

          try:

              product = self.productCache[collection]

-         except xmlrpclib.Fault as e:

+         except xmlrpc.client.Fault as e:

              # Output something useful in args

              e.args = (e.faultCode, e.faultString)

              raise

-         except xmlrpclib.ProtocolError as e:

+         except xmlrpc.client.ProtocolError as e:

              e.args = ('ProtocolError', e.errcode, e.errmsg)

              raise

  

@@ -411,11 +405,11 @@ 

                  if not DRYRUN:

                      try:

                          self.server.editcomponent(data)

-                     except xmlrpclib.Fault as e:

+                     except xmlrpc.client.Fault as e:

                          # Output something useful in args

                          e.args = (data, e.faultCode, e.faultString)

                          raise

-                     except xmlrpclib.ProtocolError as e:

+                     except xmlrpc.client.ProtocolError as e:

                          e.args = ('ProtocolError', e.errcode, e.errmsg)

                          raise

          else:

@@ -441,7 +435,7 @@ 

              if not DRYRUN:

                  try:

                      self.server.addcomponent(data)

-                 except xmlrpclib.Fault as e:

+                 except xmlrpc.client.Fault as e:

                      # Output something useful in args

                      e.args = (data, e.faultCode, e.faultString)

                      raise

@@ -456,7 +450,7 @@ 

          # Send no email in staging...

          pass

      else:

-         msg = Message()

+         msg = EmailMessage()

          msg.add_header('To', ','.join(toAddress))

          msg.add_header('From', fromAddress)

          msg.add_header('Subject', subject)

@@ -528,7 +522,7 @@ 

  

  

  @cache.cache_on_arguments()

- def _get_override_yaml(project):

+ def _get_override_yaml(project, session):

      pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(

          PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],

          project['name'])

@@ -593,6 +587,7 @@ 

      data = rv.json()

      return [branch['name'] for branch in data['results']]

  

+ 

  def _is_retired(product, project):

      branches = project['branches']

      if product == 'Fedora EPEL':

@@ -636,9 +631,9 @@ 

          owner = 'orphan'

  

      # Check if the Bugzilla ticket assignee has been manually overridden

-     override_yaml = _get_override_yaml(project)

+     override_yaml = _get_override_yaml(project, session)

      if override_yaml.get(product) \

-             and isinstance(override_yaml[product], string_types):

+             and isinstance(override_yaml[product], str):

          owner = override_yaml[product]

  

      return {

@@ -660,8 +655,9 @@ 

      }

  

  

- if __name__ == '__main__':

-     sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

+ def main():

+     """The entrypoint to the script."""

+     global VERBOSE, DRYRUN, projects_dict

  

      parser = argparse.ArgumentParser(

          description='Script syncing information between Pagure and bugzilla'

@@ -758,8 +754,8 @@ 

                  products.add(NAMESPACE_TO_PRODUCT[project['namespace']])

          project['products'] = list(products)

  

-     ## Now, we must transform the data we collected into something that PkgDB

-     ## would have returned

+     # Now, we must transform the data we collected into something that PkgDB

+     # would have returned

      p_to_legacy_schema = resilient_partial(_to_legacy_schema, session=session)

      items = [

          (product, project)

@@ -775,21 +771,21 @@ 

      # Initialize the connection to bugzilla

      bugzilla = BugzillaProxy(BZSERVER, BZUSER, BZPASS, projects_dict)

  

-     for product in projects_dict.keys():

+     for product, pkgs in projects_dict.items():

          if product not in PRODUCTS:

              continue

-         for pkg in sorted(projects_dict[product]):

+         for pkgname, pkginfo in sorted(projects_dict[product].items(),

+                                        key=lambda x: x[0]):

              if VERBOSE:

-                 print("Assesssing bugzilla status for %r" % pkg)

-             pkgInfo = projects_dict[product][pkg]

+                 print("Assessing bugzilla status for %r" % pkgname)

              try:

                  bugzilla.add_edit_component(

-                     pkg,

+                     pkgname,

                      product,

-                     pkgInfo['owner'],

-                     pkgInfo['summary'],

-                     pkgInfo['qacontact'],

-                     pkgInfo['cclist']

+                     pkginfo['owner'],

+                     pkginfo['summary'],

+                     pkginfo['qacontact'],

+                     pkginfo['cclist']

                  )

              except ValueError as e:

                  # A username didn't have a bugzilla address

@@ -798,15 +794,15 @@ 

                  # A Package or Collection was returned via xmlrpc but wasn't

                  # present when we tried to change it

                  errors.append(str(e.args))

-             except xmlrpclib.ProtocolError as e:

+             except xmlrpc.client.ProtocolError as e:

                  # Unrecoverable and likely means that nothing is going to

                  # succeed.

                  errors.append(str(e.args))

                  break

-             except xmlrpclib.Error as e:

+             except xmlrpc.client.Error as e:

                  # An error occurred in the xmlrpc call.  Shouldn't happen but

                  # we better see what it is

-                 errors.append('%s -- %s' % (pkg, e.args[-1]))

+                 errors.append('%s -- %s' % (pkgname, e.args[-1]))

  

      # Send notification of errors

      if errors:

@@ -824,3 +820,7 @@ 

              json.dump({}, stream)

  

      sys.exit(0)

+ 

+ 

+ if __name__ == '__main__':

+     main()

file added
+6

@@ -0,0 +1,6 @@ 

+ bugzilla

+ dogpile.cache

+ python-fedora

+ PyYAML

+ requests

+ urllib3

file added
+9

@@ -0,0 +1,9 @@ 

+ [flake8]

+ show-source = True

+ max-line-length = 100

+ exclude = .git,.tox,dist,*egg,build,tools

+ #ignore =

+ 

+ # Configure flake8-import-order

+ #application-import-names =

+ import-order-style = google

file added
+46

@@ -0,0 +1,46 @@ 

+ import os.path

+ 

+ from setuptools import setup

+ 

+ 

+ HERE = os.path.dirname(__file__)

+ with open(os.path.join(HERE, 'requirements.txt'), 'r') as f:

+     INSTALL_REQUIRES = [x.strip() for x in f.readlines()]

+ with open(os.path.join(HERE, 'test_requirements.txt'), 'r') as f:

+     TESTS_REQUIRE = [x.strip() for x in f.readlines()]

+ 

+ 

+ setup(

+     name='distgit-bugzilla-sync',

+     version='0.1',

+     description='script to set default assignee, CC list from component owners',

+     # Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers

+     classifiers=[

+         'Development Status :: 3 - Alpha',

+         'Intended Audience :: Developers',

+         'Intended Audience :: System Administrators',

+         'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)',

+         'Operating System :: POSIX :: Linux',

+         'Programming Language :: Python :: 3',

+         'Programming Language :: Python :: 3.6',

+         'Programming Language :: Python :: 3.7',

+         'Programming Language :: Python :: 3.8',

+         'Topic :: Software Development :: Bug Tracking',

+     ],

+     license='GPLv2+',

+     maintainer='Fedora Infrastructure Team',

+     maintainer_email='infrastructure@lists.fedoraproject.org',

+     platforms=['Fedora', 'GNU/Linux'],

+     url='https://pagure.io/Fedora-Infra/distgit-bugzilla-sync',

+     keywords='fedora',

+     packages=[],

+     include_package_data=True,

+     zip_safe=False,

+     install_requires=INSTALL_REQUIRES,

+     tests_require=TESTS_REQUIRE,

+     entry_points={

+         'console_scripts': [

+             'distgit-bugzilla-sync = pagure_sync_bugzilla:main',

+         ],

+     },

+ )

@@ -0,0 +1,2 @@ 

+ flake8

+ pytest

no initial comment

rebased onto 15d20a860a67f57af672b5194d5375b18897d758

8 months ago

Is there an interest in putting these there vs just defining them in the setup() below?

Well, we can run this script from practically anywhere given the right credentials to retrieve the list of users and to do changes in Bugzilla. Then, Python 2 is officially dead in about 6 weeks. Unless there is a really good reason to support Python 2, I don't want to support it, or do you?

I don't care really, it came with the blurb I copied from Bodhi's setup.py. ;)

wfm and your setup.py made it clear

In bodhi it makes sense to not repeat the info since there are a few setup() but hear I find it more confusing this it makes me wonder: why did we use variables here?

Makes sense, I'll change it.

5 new commits added

  • add setup.py and related files
  • wrap main code path in its own function
  • appease flake8
  • fix typo
  • convert to Python 3 (only)
8 months ago

Looks good to me, thanks! :)

Pull-Request has been merged by pingou

8 months ago