#1184 rest of python3 support for koji lib
Merged 3 years ago by mikem. Opened 3 years ago by julian8628.
julian8628/koji py3-lib  into  master

file modified
-2
@@ -7,8 +7,6 @@ 

      tests/*

      hub/*

      util/*

-     koji/daemon.py

-     koji/tasks.py

  

  [report]

  exclude_lines =

file modified
+8 -8
@@ -65,21 +65,21 @@ 

  	@git clean -d -q -x

  

  test:

- 	coverage erase

- 	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/.:www/lib coverage run \

+ 	coverage2 erase

+ 	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/.:www/lib coverage2 run \

  	    --source . /usr/bin/nosetests

- 	coverage report

- 	coverage html

+ 	coverage2 report

+ 	coverage2 html

  	@echo Full coverage report in htmlcov/index.html

  

  test3:

- 	coverage erase

+ 	coverage3 erase

  	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/. coverage3 run \

  	    --rcfile .coveragerc3 --source . \

- 	    /usr/bin/nosetests-3 \

+ 	    /usr/bin/nosetests \

  	    tests/test_lib tests/test_cli

- 	coverage report --rcfile .coveragerc3

- 	coverage html --rcfile .coveragerc3

+ 	coverage3 report --rcfile .coveragerc3

+ 	coverage3 html --rcfile .coveragerc3

  	@echo Full coverage report at file://${PWD}/htmlcov/index.html

  

  test-tarball:

file modified
+3 -10
@@ -1,15 +1,8 @@ 

  PYTHON=python

  PACKAGE = $(shell basename `pwd`)

- ifeq ($(PYTHON), python3)

-     # for python3 we fully support only basic library + CLI

-     PYFILES = __init__.py util.py plugin.py xmlrpcplus.py

-     PYSCRIPTS =

-     SUBDIRS =

- else

-     PYFILES = $(wildcard *.py)

-     PYSCRIPTS = context.py

-     SUBDIRS =

- endif

+ PYFILES = $(wildcard *.py)

+ PYSCRIPTS =

+ SUBDIRS =

  PYVER := $(shell $(PYTHON) -c 'import sys; print("%.3s" % (sys.version))')

  PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print(sys.prefix)')

  PYLIBDIR = $(PYSYSDIR)/lib/python$(PYVER)

file modified
+2 -2
@@ -29,9 +29,9 @@ 

  except ImportError:

      krbV = None

  import koji

- import urlparse      #for parse_qs

  from .context import context

  from six.moves import range

+ from six.moves import urllib

  from six.moves import zip

  import six

  from .util import to_list
@@ -83,7 +83,7 @@ 

              if not args:

                  self.message = 'no session args'

                  return

-             args = urlparse.parse_qs(args, strict_parsing=True)

+             args = urllib.parse.parse_qs(args, strict_parsing=True)

          hostip = self.get_remote_ip(override=hostip)

          try:

              id = int(args['session-id'][0])

file modified
+3 -3
@@ -31,7 +31,7 @@ 

  import os

  import signal

  import logging

- import urlparse

+ from six.moves import urllib

  from fnmatch import fnmatch

  import base64

  import time
@@ -258,7 +258,7 @@ 

  

          # replace the scheme with http:// so that the urlparse works in all cases

          dummyurl = self.url.replace(scheme, 'http://', 1)

-         dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)

+         dummyscheme, netloc, path, params, query, fragment = urllib.parse.urlparse(dummyurl)

  

          user = None

          userhost = netloc.split('@')
@@ -532,7 +532,7 @@ 

              netloc = self.host

              path = self.repository

              query = self.module

-             r['source'] = urlparse.urlunsplit([scheme, netloc, path, query, fragment])

+             r['source'] = urllib.parse.urlunsplit([scheme, netloc, path, query, fragment])

          else:

              # just use the same url

              r['source'] = self.url

file modified
+3 -2
@@ -25,6 +25,7 @@ 

  import os

  import itertools

  import six

+ from six.moves import zip

  

  class Rpmdiff:

  
@@ -179,8 +180,8 @@ 

          if not isinstance(oldflags, list): oldflags = [ oldflags ]

          if not isinstance(newflags, list): newflags = [ newflags ]

  

-         o = zip(old[name], oldflags, old[name[:-1]+'VERSION'])

-         n = zip(new[name], newflags, new[name[:-1]+'VERSION'])

+         o = list(zip(old[name], oldflags, old[name[:-1]+'VERSION']))

+         n = list(zip(new[name], newflags, new[name[:-1]+'VERSION']))

  

          if name == 'PROVIDES': # filter our self provide

              oldNV = (old['name'], rpm.RPMSENSE_EQUAL,

file modified
+10 -4
@@ -504,7 +504,8 @@ 

          #  c) is canonical

          host_arches = host['arches']

          if not host_arches:

-             raise koji.BuildError("No arch list for this host: %s" % host['name'])

+             raise koji.BuildError("No arch list for this host: %s" %

+                                   host['name'])

          tag_arches = tag['arches']

          if not tag_arches:

              raise koji.BuildError("No arch list for tag: %s" % tag['name'])
@@ -520,12 +521,17 @@ 

              # because we just forked from a common parent

              random.seed()

              arch = random.choice(common_arches)

-             self.logger.info('Valid arches: %s, using: %s' % (' '.join(common_arches), arch))

+             self.logger.info('Valid arches: %s, using: %s' %

+                              (' '.join(sorted(common_arches)), arch))

              return arch

          else:

              # no overlap

-             raise koji.BuildError("host %s (%s) does not support any arches of tag %s (%s)" % \

-                 (host['name'], ', '.join(host_arches), tag['name'], ', '.join(tag_arches)))

+             raise koji.BuildError("host %s (%s) does not support any arches"

+                                   " of tag %s (%s)" %

+                                   (host['name'],

+                                    ', '.join(sorted(host_arches)),

+                                    tag['name'],

+                                    ', '.join(sorted(tag_arches))))

  

      def getRepo(self, tag):

          """

file modified
+22
@@ -39,6 +39,7 @@ 

  from six.moves import range

  import six

  import warnings

+ from six.moves import zip

  

  # imported from kojiweb and kojihub

  try:
@@ -77,6 +78,7 @@ 

  DATE_RE = re.compile(r'(\d+)-(\d+)-(\d+)')

  TIME_RE = re.compile(r'(\d+):(\d+):(\d+)')

  

+ 

  def parseTime(val):

      """

      Parse a string time in either "YYYY-MM-DD HH24:MI:SS" or "YYYY-MM-DD"
@@ -98,6 +100,7 @@ 

      return calendar.timegm(

              datetime.datetime(*(date + time)).timetuple())

  

+ 

  def checkForBuilds(session, tag, builds, event, latest=False):

      """Check that the builds existed in tag at the time of the event.

         If latest=True, check that the builds are the latest in tag."""
@@ -114,6 +117,7 @@ 

  

      return True

  

+ 

  def duration(start):

      """Return the duration between start and now in MM:SS format"""

      elapsed = time.time() - start
@@ -121,6 +125,7 @@ 

      secs = int(elapsed % 60)

      return '%s:%02i' % (mins, secs)

  

+ 

  def printList(l):

      """Print the contents of the list comma-separated"""

      if len(l) == 0:
@@ -135,6 +140,7 @@ 

          ret += l[-1]

          return ret

  

+ 

  def multi_fnmatch(s, patterns):

      """Returns true if s matches any pattern in the list

  
@@ -147,6 +153,7 @@ 

              return True

      return False

  

+ 

  def dslice(dict, keys, strict=True):

      """Returns a new dictionary containing only the specified keys"""

      ret = {}
@@ -156,6 +163,7 @@ 

              ret[key] = dict[key]

      return ret

  

+ 

  def dslice_ex(dict, keys, strict=True):

      """Returns a new dictionary with only the specified keys removed"""

      ret = dict.copy()
@@ -488,6 +496,7 @@ 

                      'ts' : rinfo['create_ts']}

      return None

  

+ 

  def filedigestAlgo(hdr):

      """

      Get the file digest algorithm used in hdr.
@@ -505,6 +514,7 @@ 

      digest_algo = koji.RPM_FILEDIGESTALGO_IDS.get(digest_algo_id, 'unknown')

      return digest_algo.lower()

  

+ 

  def parseStatus(rv, prefix):

      if isinstance(prefix, (list, tuple)):

          prefix = ' '.join(prefix)
@@ -515,6 +525,7 @@ 

      else:

          return '%s terminated for unknown reasons' % prefix

  

+ 

  def isSuccess(rv):

      """Return True if rv indicates successful completion

      (exited with status 0), False otherwise."""
@@ -523,6 +534,7 @@ 

      else:

          return False

  

+ 

  def setup_rlimits(opts, logger=None):

      logger = logger or logging.getLogger("koji")

      for key in opts:
@@ -548,6 +560,7 @@ 

          except ValueError as e:

              logger.error("Unable to set %s: %s", key, e)

  

+ 

  class adler32_constructor(object):

  

      #mimicing the hashlib constructors
@@ -577,6 +590,7 @@ 

      digest_size = 4

      block_size = 1      #I think

  

+ 

  def tsort(parts):

      """Given a partial ordering, return a totally ordered list.

  
@@ -598,6 +612,7 @@ 

          raise ValueError('total ordering not possible')

      return result

  

+ 

  class MavenConfigOptAdapter(object):

      """

      Wrap a ConfigParser so it looks like a optparse.Values instance
@@ -621,6 +636,7 @@ 

              return value

          raise AttributeError(name)

  

+ 

  def maven_opts(values, chain=False, scratch=False):

      """

      Convert the argument (an optparse.Values object) to a dict of build options
@@ -656,10 +672,12 @@ 

          opts['scratch'] = True

      return opts

  

+ 

  def maven_params(config, package, chain=False, scratch=False):

      values = MavenConfigOptAdapter(config, package)

      return maven_opts(values, chain=chain, scratch=scratch)

  

+ 

  def wrapper_params(config, package, chain=False, scratch=False):

      params = {}

      values = MavenConfigOptAdapter(config, package)
@@ -670,6 +688,7 @@ 

          params['create_build'] = True

      return params

  

+ 

  def parse_maven_params(confs, chain=False, scratch=False):

      """

      Parse .ini files that contain parameters to launch a Maven build.
@@ -703,6 +722,7 @@ 

          raise ValueError("No sections found in: %s" % ', '.join(confs))

      return builds

  

+ 

  def parse_maven_param(confs, chain=False, scratch=False, section=None):

      """

      Parse .ini files that contain parameters to launch a Maven build.
@@ -723,6 +743,7 @@ 

          raise ValueError("Multiple sections in: %s, you must specify the section" % ', '.join(confs))

      return builds

  

+ 

  def parse_maven_chain(confs, scratch=False):

      """

      Parse maven-chain config.
@@ -741,6 +762,7 @@ 

          raise ValueError('No possible build order, missing/circular dependencies')

      return builds

  

+ 

  def to_list(l):

      """

      Helper function for py2/py3 compatibility used e.g. in

@@ -62,8 +62,8 @@ 

          unlock-tag                Unlock a tag

          write-signed-rpm          Write signed RPMs to disk

  

- Try "koji --help" for help about global options

- Try "koji help" to get all available commands

- Try "koji <command> --help" for help about the options of a particular command

- Try "koji help <category>" to get commands under a particular category

+ Try "{progname} --help" for help about global options

+ Try "{progname} help" to get all available commands

+ Try "{progname} <command> --help" for help about the options of a particular command

+ Try "{progname} help <category>" to get commands under a particular category

  Available categories are: admin, all, bind, build, download, info, misc, monitor, search

@@ -134,8 +134,8 @@ 

  search commands:

          search                    Search the system

  

- Try "koji --help" for help about global options

- Try "koji help" to get all available commands

- Try "koji <command> --help" for help about the options of a particular command

- Try "koji help <category>" to get commands under a particular category

+ Try "{progname} --help" for help about global options

+ Try "{progname} help" to get all available commands

+ Try "{progname} <command> --help" for help about the options of a particular command

+ Try "{progname} help <category>" to get commands under a particular category

  Available categories are: admin, all, bind, build, download, info, misc, monitor, search

@@ -7,7 +7,7 @@ 

  except ImportError:

      import unittest

  

- from . import loadcli

+ from . import loadcli, utils

  cli = loadcli.cli

  

  
@@ -31,13 +31,9 @@ 

      def test_list_commands(self, stdout):

          cli.list_commands()

          actual = stdout.getvalue()

-         if six.PY2:

-             actual = actual.replace('nosetests', 'koji')

-         else:

-             actual = actual.replace('nosetests-3', 'koji')

          filename = os.path.dirname(__file__) + '/data/list-commands.txt'

          with open(filename, 'rb') as f:

-             expected = f.read().decode('ascii')

+             expected = f.read().decode('ascii').format(progname=utils.PROGNAME)

          self.assertMultiLineEqual(actual, expected)

  

      @mock.patch('sys.stdout', new_callable=six.StringIO)
@@ -47,11 +43,7 @@ 

          self.parser.parse_args.return_value = [options, arguments]

          cli.handle_help(self.options, self.session, self.args)

          actual = stdout.getvalue()

-         if six.PY2:

-             actual = actual.replace('nosetests', 'koji')

-         else:

-             actual = actual.replace('nosetests-3', 'koji')

          filename = os.path.dirname(__file__) + '/data/list-commands-admin.txt'

          with open(filename, 'rb') as f:

-             expected = f.read().decode('ascii')

+             expected = f.read().decode('ascii').format(progname=utils.PROGNAME)

          self.assertMultiLineEqual(actual, expected)

file modified
+2 -1
@@ -11,6 +11,7 @@ 

      import unittest

  

  

+ PROGNAME = os.path.basename(sys.argv[0]) or 'koji'

  

  """

    Classes
@@ -40,7 +41,7 @@ 

  class CliTestCase(unittest.TestCase):

  

      # public attribute

-     progname = os.path.basename(sys.argv[0]) or 'koji'

+     progname = PROGNAME

      error_format = None

      STDOUT = sys.stdout

      STDERR = sys.stderr

tests/test_lib/test_auth.py tests/test_lib_py2only/test_auth.py
file renamed
+51 -7
@@ -1,13 +1,17 @@ 

  from __future__ import absolute_import

+ 

  import mock

+ 

  try:

      import unittest2 as unittest

  except ImportError:

      import unittest

+ import six

  

  import koji

  import koji.auth

  

+ 

  class TestAuthSession(unittest.TestCase):

      def test_instance(self):

          """Simple auth.Session instance"""
@@ -27,7 +31,8 @@ 

          context.cnx.cursor.return_value = cursor

          cursor.fetchone.side_effect = [

              # get session

-             [koji.AUTHTYPE_NORMAL, 344, False, False, 'master', 'start_time', 'start_ts', 'update_time', 'update_ts', 'user_id'],

+             [koji.AUTHTYPE_NORMAL, 344, False, False, 'master', 'start_time',

+              'start_ts', 'update_time', 'update_ts', 'user_id'],

              # get user

              ['name', koji.USER_STATUS['NORMAL'], koji.USERTYPES['NORMAL']],

              # get excl.session
@@ -107,7 +112,8 @@ 

  

          s.makeShared()

          c = cursor.execute.call_args[0]

-         self.assertEqual(c[0], 'UPDATE sessions SET "exclusive"=NULL WHERE id=%(session_id)s')

+         self.assertEqual(c[0],

+                          'UPDATE sessions SET "exclusive"=NULL WHERE id=%(session_id)s')

          self.assertEqual(c[1]['session_id'], 123)

  

      @mock.patch('socket.gethostbyname')
@@ -165,17 +171,54 @@ 

              s.login('user', 'password')

  

      @mock.patch('koji.auth.context')

-     def test_krbLogin(self, context):

+     @mock.patch('koji.auth.socket')

+     @mock.patch('koji.auth.base64')

+     def test_krbLogin(self, base64, socket, context):

          # TODO

          s, cntext, cursor = self.get_session()

          context.cnx = cntext.cnx

  

-         with self.assertRaises(koji.AuthError):

+         with self.assertRaises(koji.AuthError) as cm:

              s.krbLogin('krb_req', 'proxyuser')

+         self.assertEqual(cm.exception.args[0], 'Already logged in')

  

          s.logged_in = False

-         with self.assertRaises(TypeError):

-             s.krbLogin('krb_req', 'proxyuser')

+         if six.PY3:

+             with self.assertRaises(koji.AuthError) as cm:

+                 s.krbLogin('krb_req', 'proxyuser')

+             self.assertEqual(cm.exception.args[0], 'krbV module not installed')

+         else:

+             with mock.patch('koji.auth.krbV', create=True) as krbV:

+                 princ = mock.MagicMock()

+                 princ.name = 'princ_name'

+                 krbV.default_context.return_value \

+                     .rd_req.return_value = (mock.MagicMock(), 2, 3,

+                                             [1, 2, princ])

+                 with self.assertRaises(koji.AuthError) as cm:

+                     s.krbLogin('krb_req', 'proxyuser')

+                 self.assertEqual(cm.exception.args[0],

+                                  'Kerberos principal princ_name is'

+                                  ' not authorized to log in other users')

+                 context.opts = {'ProxyPrincipals': 'anyothers,' + princ.name,

+                                 'AuthPrincipal': 'authprinc',

+                                 'AuthKeytab': 'authkeytab',

+                                 'LoginCreatesUser': False,

+                                 'CheckClientIP': False}

+                 with self.assertRaises(koji.AuthError) as cm:

+                     s.krbLogin('krb_req', 'proxyuser@realm.com')

+                 self.assertEqual(cm.exception.args[0],

+                                  'Unknown Kerberos principal:'

+                                  ' proxyuser@realm.com')

+                 context.opts['LoginCreatesUser'] = True

+                 context.cnx.cursor.return_value. \

+                     fetchone.side_effect = [None,

+                                             None,

+                                             None,

+                                             (1,),

+                                             ('name', 'type',

+                                              koji.USER_STATUS['NORMAL']),

+                                             ('session-id',)]

+                 s.krbLogin('krb_req', 'proxyuser@realm.com')

  

      # functions outside Session object

  
@@ -187,7 +230,8 @@ 

          cursor.fetchone.return_value = ['name', 'status', 'usertype']

  

          self.assertEqual(sorted(koji.auth.get_user_data(1).items()),

-                          sorted({'name': 'name', 'status': 'status', 'usertype': 'usertype'}.items()))

+                          sorted({'name': 'name', 'status': 'status',

+                                  'usertype': 'usertype'}.items()))

  

          cursor.fetchone.return_value = None

          self.assertEqual(koji.auth.get_user_data(1), None)

@@ -2,6 +2,7 @@ 

  import six

  import time

  import random

+ from six.moves import range

  try:

      import unittest2 as unittest

  except ImportError:

tests/test_lib/test_krbv.py tests/test_lib_py2only/test_krbv.py
file renamed
+2
@@ -22,6 +22,8 @@ 

          with self.assertRaises(ImportError):

              session.krb_login()

  

+     # this case should work on python3, but skipped still

+     @unittest.skipIf(six.PY3, "skipped on python3 since missing of python-krbV")

      @mock.patch('koji.krbV', create=True)

      @mock.patch('requests_kerberos.__version__', new='0.7.0')

      @mock.patch('koji.ClientSession._serverPrincipal')

tests/test_lib/test_restart_tasks.py tests/test_lib_py2only/test_restart_tasks.py
file renamed
file was moved with no change to the file
tests/test_lib/test_tasks.py tests/test_lib_py2only/test_tasks.py
file renamed
file was moved with no change to the file