#374 Python 2.4/3.x compatible CLI
Closed 6 years ago by tkopecek. Opened 7 years ago by tkopecek.
tkopecek/koji python3-cli  into  master

file modified
+107 -71
@@ -40,27 +40,23 @@ 

          import simplejson as json

      except ImportError:

          json = None

- import ConfigParser

  import base64

  import dateutil.parser

- import errno

  import koji

  import koji.util

  import fnmatch

- from koji.util import md5_constructor

  import logging

+ import optparse

  import os

  import re

  import pprint

+ import pycurl

  import random

  import socket

  import stat

  import string

  import time

  import traceback

- import urlgrabber.grabber as grabber

- import urlgrabber.progress as progress

- import xmlrpclib

  try:

      import libcomps

  except ImportError:  # pragma: no cover
@@ -69,7 +65,12 @@ 

          import yum.comps as yumcomps

      except ImportError:

          yumcomps = None

- import optparse

+ if sys.version_info[0] < 3:

+     import ConfigParser

+     import xmlrpclib

+ else: # pragma: no cover

+     import configparser as ConfigParser

+     import xmlrpc.client as xmlrpclib

  

  

  # fix OptionParser for python 2.3 (optparse verion 1.4.1+)
@@ -106,6 +107,13 @@ 

              u'नमस्कार',

              u'안녕하세요')

  

+ def printable_unicode(s):

+     # python3 unicode handling

+     if sys.version_info[0] < 3:

+         return s.encode('utf-8')

+     else: # pragma: no cover

+         return s

+ 

  def _(args):

      """Stub function for translation"""

      return args
@@ -142,7 +150,7 @@ 

  def get_epilog_str(progname=None):

      if progname is None:

          progname = os.path.basename(sys.argv[0]) or 'koji'

-     categories_ordered=', '.join(sorted(['all'] + categories.keys()))

+     categories_ordered=', '.join(sorted(['all'] + list(categories.keys())))

      epilog_str = '''

  Try "%(progname)s --help" for help about global options

  Try "%(progname)s help" to get all available commands
@@ -237,12 +245,13 @@ 

      # load local config

      try:

          result = koji.read_config(options.profile, user_config=options.configFile)

-     except koji.ConfigurationError, e:

+     except koji.ConfigurationError:

+         e = sys.exc_info()[1]

          parser.error(e.args[0])

          assert False  # pragma: no cover

  

      # update options according to local config

-     for name, value in result.iteritems():

+     for name, value in result.items():

          if getattr(options, name, None) is None:

              setattr(options, name, value)

  
@@ -339,8 +348,8 @@ 

          error = None

          try:

              result = self.session.getTaskResult(self.id)

-         except (xmlrpclib.Fault,koji.GenericError),e:

-             error = e

+         except (xmlrpclib.Fault, koji.GenericError):

+             error = sys.exc_info()[1]

          if error is None:

              # print("%s: complete" % self.str())

              # We already reported this task as complete in update()
@@ -407,7 +416,7 @@ 

      open = 0

      failed = 0

      done = 0

-     for task_id in tasks.keys():

+     for task_id in list(tasks.keys()):

          status = tasks[task_id].info['state']

          if status == koji.TASK_STATES['FAILED']:

              failed += 1
@@ -420,7 +429,7 @@ 

      print("  %d free  %d open  %d done  %d failed" % (free, open, done, failed))

  

  def display_task_results(tasks):

-     for task in [task for task in tasks.values() if task.level == 0]:

+     for task in [task for task in list(tasks.values()) if task.level == 0]:

          state = task.info['state']

          task_label = task.str()

  
@@ -448,7 +457,7 @@ 

              tasks[task_id] = TaskWatcher(task_id,session,quiet=quiet)

          while True:

              all_done = True

-             for task_id,task in tasks.items():

+             for task_id,task in list(tasks.items()):

                  changed = task.update()

                  if not task.is_done():

                      all_done = False
@@ -461,7 +470,7 @@ 

                          rv = 1

                  for child in session.getTaskChildren(task_id):

                      child_id = child['id']

-                     if not child_id in tasks.keys():

+                     if not child_id in list(tasks.keys()):

                          tasks[child_id] = TaskWatcher(child_id, session, task.level + 1, quiet=quiet)

                          tasks[child_id].update()

                          # If we found new children, go through the list again,
@@ -479,7 +488,7 @@ 

          if tasks and not quiet:

              progname = os.path.basename(sys.argv[0]) or 'koji'

              tlist = ['%s: %s' % (t.str(), t.display_state(t.info))

-                             for t in tasks.values() if not t.is_done()]

+                             for t in list(tasks.values()) if not t.is_done()]

              print( \

  """Tasks still running. You can continue to watch with the '%s watch-task' command.

  Running Tasks:
@@ -511,7 +520,7 @@ 

              output = list_task_output_all_volumes(session, task_id)

              # convert to list of (file, volume)

              files = []

-             for filename, volumes in output.iteritems():

+             for filename, volumes in output.items():

                  files += [(filename, volume) for volume in volumes]

  

              if opts.log:
@@ -547,7 +556,8 @@ 

      """List task output with all volumes, or fake it"""

      try:

          return session.listTaskOutput(task_id, all_volumes=True)

-     except koji.GenericError, e:

+     except koji.GenericError:

+         e = sys.exc_info()[1]

          if 'got an unexpected keyword argument' not in str(e):

              raise

      # otherwise leave off the option and fake it
@@ -664,7 +674,7 @@ 

      activate_session(session)

  

      vals = {}

-     for key, val in subopts.__dict__.items():

+     for key, val in list(subopts.__dict__.items()):

          if val is not None:

              vals[key] = val

      if 'arches' in vals:
@@ -941,7 +951,7 @@ 

  def _running_in_bg():

      try:

          return (not os.isatty(0)) or (os.getpgrp() != os.tcgetpgrp(0))

-     except OSError, e:

+     except OSError:

          return True

  

  def handle_build(options, session, args):
@@ -1164,11 +1174,12 @@ 

          try:

              params = koji.util.parse_maven_param(build_opts.inis, scratch=build_opts.scratch,

                                                   section=build_opts.section)

-         except ValueError, e:

+         except ValueError:

+             e = sys.exc_info()[1]

              parser.error(e.args[0])

-         opts = params.values()[0]

+         opts = list(params.values())[0]

          if opts.pop('type', 'maven') != 'maven':

-             parser.error(_("Section %s does not contain a maven-build config") % params.keys()[0])

+             parser.error(_("Section %s does not contain a maven-build config") % list(params.keys())[0])

          source = opts.pop('scmurl')

      else:

          source = args[1]
@@ -1223,11 +1234,12 @@ 

          try:

              params = koji.util.parse_maven_param(build_opts.inis, scratch=build_opts.scratch,

                                                   section=build_opts.section)

-         except ValueError, e:

+         except ValueError:

+             e = sys.exc_info()[1]

              parser.error(e.args[0])

-         opts = params.values()[0]

+         opts = list(params.values())[0]

          if opts.get('type') != 'wrapper':

-             parser.error(_("Section %s does not contain a wrapper-rpm config") % params.keys()[0])

+             parser.error(_("Section %s does not contain a wrapper-rpm config") % list(params.keys())[0])

          url = opts['scmurl']

          package = opts['buildrequires'][0]

          target_info = session.getBuildTarget(target, strict=True)
@@ -1297,7 +1309,8 @@ 

              opts[key] = val

      try:

          builds = koji.util.parse_maven_chain(args[1:], scratch=opts.get('scratch'))

-     except ValueError, e:

+     except ValueError:

+         e = sys.exc_info()[1]

          parser.error(e.args[0])

      priority = None

      if build_opts.background:
@@ -1569,7 +1582,7 @@ 

  

  def linked_upload(localfile, path, name=None):

      """Link a file into the (locally writable) workdir, bypassing upload"""

-     old_umask = os.umask(002)

+     old_umask = os.umask(2) # o002

      try:

          if name is None:

              name = os.path.basename(localfile)
@@ -1618,7 +1631,7 @@ 

              nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm'])

          to_import.setdefault(nvr,[]).append((path,data))

      builds_missing = False

-     nvrs = to_import.keys()

+     nvrs = list(to_import.keys())

      nvrs.sort()

      for nvr in nvrs:

          to_import[nvr].sort()
@@ -1665,7 +1678,8 @@ 

          sys.stdout.flush()

          try:

              session.importRPM(serverdir, os.path.basename(path))

-         except koji.GenericError, e:

+         except koji.GenericError:

+             e = sys.exc_info()[1]

              print(_("\nError importing: %s" % str(e).splitlines()[-1]))

              sys.stdout.flush()

          else:
@@ -1924,7 +1938,7 @@ 

          previous = session.queryRPMSigs(rpm_id=rinfo['id'], sigkey=sigkey)

          assert len(previous) <= 1

          if previous:

-             sighash = md5_constructor(sighdr).hexdigest()

+             sighash = koji.util.md5_constructor(sighdr).hexdigest()

              if previous[0]['sighash'] == sighash:

                  print(_("Signature already imported: %s") % path)

                  continue
@@ -2080,7 +2094,7 @@ 

              #that the build was recently untagged from

              tags.setdefault(entry['tag_name'], 1)

          if options.debug:

-             print("Tags: %s" % tags.keys())

+             print("Tags: %s" % list(tags.keys()))

          for tag_name in tags:

              if tag_name == options.trashcan_tag:

                  if options.debug:
@@ -2250,7 +2264,7 @@ 

          build_space = 0

          if not by_sig and options.debug:

              print("(build has no signatures)")

-         for sigkey, rpms in by_sig.iteritems():

+         for sigkey, rpms in by_sig.items():

              mycount = 0

              archdirs = {}

              sigdirs = {}
@@ -2274,7 +2288,8 @@ 

                          print("Unlinking: %s" % signedpath)

                      try:

                          os.unlink(signedpath)

-                     except OSError, e:

+                     except OSError:

+                         e = sys.exc_info()[1]

                          print("Error removing %s: %s" % (signedpath, e))

                          print("This script needs write access to %s" % koji.BASEDIR)

                          continue
@@ -2294,10 +2309,11 @@ 

                          print("Removing dir: %s" % dir)

                      try:

                          os.rmdir(dir)

-                     except OSError, e:

+                     except OSError:

+                         e = sys.exc_info()[1]

                          print("Error removing %s: %s" % (signedpath, e))

              if len(sigdirs) == 1:

-                 dir = sigdirs.keys()[0]

+                 dir = list(sigdirs.keys())[0]

                  if options.test:

                      print("Would have removed dir: %s" % dir)

                  else:
@@ -2305,7 +2321,8 @@ 

                          print("Removing dir: %s" % dir)

                      try:

                          os.rmdir(dir)

-                     except OSError, e:

+                     except OSError:

+                         e = sys.exc_info()[1]

                          print("Error removing %s: %s" % (signedpath, e))

              elif len(sigdirs) > 1:

                  print("Warning: more than one signature dir for %s: %r" % (sigkey, sigdirs))
@@ -2539,7 +2556,8 @@ 

          sys.stdout.write(_("importing %s... ") % nvr)

          try:

              session.importBuildInPlace(data)

-         except koji.GenericError, e:

+         except koji.GenericError:

+             e = sys.exc_info()[1]

              print(_("\nError importing: %s" % str(e).splitlines()[-1]))

              sys.stdout.flush()

          else:
@@ -3610,35 +3628,35 @@ 

                  dstgroups[group['name']] = group

          #construct to-do lists.

          paddlist = [] # list containing new packages to be added from src tag

-         for (package_name, pkg) in srcpkgs.iteritems():

+         for (package_name, pkg) in srcpkgs.items():

              if package_name not in dstpkgs:

                  paddlist.append(pkg)

          paddlist.sort(key = lambda x: x['package_name'])

          pdellist = [] # list containing packages no more present in dst tag

-         for (package_name, pkg) in dstpkgs.iteritems():

+         for (package_name, pkg) in dstpkgs.items():

              if package_name not in srcpkgs:

                  pdellist.append(pkg)

          pdellist.sort(key = lambda x: x['package_name'])

          baddlist = [] # list containing new builds to be added from src tag

-         for (nvr, lbld) in srclblds.iteritems():

+         for (nvr, lbld) in srclblds.items():

              if nvr not in dstlblds:

                  baddlist.append(lbld)

          baddlist.sort(key = lambda x: x['package_name'])

          bdellist = [] # list containing new builds to be removed from src tag

-         for (nvr, lbld) in dstlblds.iteritems():

+         for (nvr, lbld) in dstlblds.items():

              if nvr not in srclblds:

                  bdellist.append(lbld)

          bdellist.sort(key = lambda x: x['package_name'])

          gaddlist = [] # list containing new groups to be added from src tag

-         for (grpname, group) in srcgroups.iteritems():

+         for (grpname, group) in srcgroups.items():

              if grpname not in dstgroups:

                  gaddlist.append(group)

          gdellist = [] # list containing groups to be removed from src tag

-         for (grpname, group) in dstgroups.iteritems():

+         for (grpname, group) in dstgroups.items():

              if grpname not in srcgroups:

                  gdellist.append(group)

          grpchanges = {} # dict of changes to make in shared groups

-         for (grpname, group) in srcgroups.iteritems():

+         for (grpname, group) in srcgroups.items():

              if grpname in dstgroups:

                  grpchanges[grpname] = {'adds':[], 'dels':[]}

                  # Store whether group is inherited or not
@@ -3980,15 +3998,15 @@ 

          if depth < currtag['currdepth']:

              outspacing = depth - outdepth

              sys.stdout.write(' ' * (outspacing * 3 - 1))

-             sys.stdout.write(u'\u2502'.encode('UTF-8'))

+             sys.stdout.write(printable_unicode(u'\u2502'))

              outdepth = depth

  

      sys.stdout.write(' ' * ((currtag['currdepth'] - outdepth) * 3 - 1))

      if siblings:

-         sys.stdout.write(u'\u251c'.encode('UTF-8'))

+         sys.stdout.write(printable_unicode(u'\u251c'))

      else:

-         sys.stdout.write(u'\u2514'.encode('UTF-8'))

-     sys.stdout.write(u'\u2500'.encode('UTF-8'))

+         sys.stdout.write(printable_unicode(u'\u2514'))

+     sys.stdout.write(printable_unicode(u'\u2500'))

      if reverse:

          sys.stdout.write('%(name)s (%(tag_id)i)\n' % currtag)

      else:
@@ -4332,7 +4350,7 @@ 

          else:

              return '%s.name' % key

      if edit:

-         keys = x.keys()

+         keys = list(x.keys())

          keys.sort()

          y = other[-1]

          for key in keys:
@@ -4347,7 +4365,7 @@ 

                  continue

              print("    %s: %s -> %s" % (key, x[key], y[key]))

      elif create and options.verbose and table != 'tag_listing':

-         keys = x.keys()

+         keys = list(x.keys())

          keys.sort()

          # the table keys have already been represented in the base format string

          also_hidden = list(_table_keys[table])
@@ -4516,7 +4534,7 @@ 

                  kwargs['afterEvent'] = last_event

  

  def _handleMap(lines, data, prefix=''):

-     for key, val in data.items():

+     for key, val in list(data.items()):

          if key != '__starstar':

              lines.append('  %s%s: %s' % (prefix, key, val))

  
@@ -4590,7 +4608,7 @@ 

              _handleOpts(lines, params[4])

      elif method == 'chainmaven':

          lines.append("Builds:")

-         for package, opts in params[0].items():

+         for package, opts in list(params[0].items()):

              lines.append("  " + package)

              _handleMap(lines, opts, prefix="  ")

          lines.append("Build Target: %s" % params[1])
@@ -4823,7 +4841,7 @@ 

              print("Include all Maven archives?: %s" % (info['maven_include_all'] and 'yes' or 'no'))

          if 'extra' in info:

              print("Tag options:")

-             keys = info['extra'].keys()

+             keys = list(info['extra'].keys())

              keys.sort()

              for key in keys:

                  print("  %s : %s" % (key, pprint.pformat(info['extra'][key])))
@@ -6671,7 +6689,8 @@ 

          # We want the latest build, not a specific build

          try:

              builds = session.listTagged(suboptions.latestfrom, latest=True, package=build, type=suboptions.type)

-         except koji.GenericError, data:

+         except koji.GenericError:

+             data = sys.exc_info()[1]

              print("Error finding latest build: %s" % data)

              return 1

          if not builds:
@@ -6752,15 +6771,29 @@ 

              url = pathinfo.build(info) + '/' + fname

              urls.append((url, os.path.basename(fname)))

  

-     if suboptions.quiet:

-         pg = None

-     else:

-         pg = progress.TextMeter()

+     def _progress(download_t, download_d, upload_t, upload_d):

+         if download_t == 0:

+             percent_done = 0.0

+         else:

+             percent_done = float(download_d)/float(download_t)

+         percent_done_str = "%02d%%" % (percent_done * 100)

+         data_done = _format_size(download_d)

+ 

+         sys.stdout.write("[% -36s] % 4s % 10s\r" % ('='*(int(percent_done * 36)), percent_done_str, data_done))

+         sys.stdout.flush()

  

      for url, relpath in urls:

          if '/' in relpath:

              koji.ensuredir(os.path.dirname(relpath))

-         grabber.urlgrab(url, filename=relpath, progress_obj=pg, text=relpath)

+         print(relpath)

+         c = pycurl.Curl()

+         c.setopt(c.URL, url)

+         c.setopt(c.WRITEDATA, open(relpath, 'wb'))

+         if not suboptions.quiet:

+             c.setopt(c.NOPROGRESS, False)

+             c.setopt(c.XFERINFOFUNCTION, _progress)

+         c.perform()

+         print('')

  

  

  def anon_handle_download_logs(options, session, args):
@@ -7003,7 +7036,7 @@ 

              targets = session.getBuildTargets(destTagID=tag_info['id'])

              if targets:

                  maybe = {}.fromkeys([t['build_tag_name'] for t in targets])

-                 maybe = maybe.keys()

+                 maybe = list(maybe.keys())

                  maybe.sort()

                  print("Suggested tags: %s" % ', '.join(maybe))

              return 1
@@ -7260,7 +7293,7 @@ 

      if not u:

          print("Not authenticated")

          u = {'name' : 'anonymous user'}

-     print("%s, %s!" % (random.choice(greetings).encode('utf-8'), u["name"]))

+     print("%s, %s!" % (printable_unicode(random.choice(greetings)), u["name"]))

      print("")

      print("You are using the hub at %s" % session.baseurl)

      authtype = u.get('authtype', getattr(session, 'authtype', None))
@@ -7318,7 +7351,8 @@ 

              kwargs['new_chroot'] = True

  

          task_id = session.runroot(tag, arch, command, **kwargs)

-     except koji.GenericError, e:

+     except koji.GenericError:

+         e = sys.exc_info()[1]

          if 'Invalid method' in str(e):

              print("* The runroot plugin appears to not be installed on the"

                    " koji hub.  Please contact the administrator.")
@@ -7394,8 +7428,8 @@ 

  

      try:

          task_id = session.saveFailedTree(br_id, opts.full)

-     except koji.GenericError as e:

-         m = str(e)

+     except koji.GenericError:

+         m = str(sys.exc_info()[1])

          if 'Invalid method' in m:

              print(_("* The save_failed_tree plugin appears to not be "

                      "installed on the koji hub.  Please contact the "
@@ -7428,7 +7462,7 @@ 

      chosen = set(args)

      if options.admin:

          chosen.add('admin')

-     avail = set(categories.keys() + ['all'])

+     avail = set(list(categories.keys()) + ['all'])

      unavail = chosen - avail

      for arg in unavail:

          print("No such help category: %s" % arg)
@@ -7441,13 +7475,13 @@ 

  

  def list_commands(categories_chosen=None):

      if categories_chosen is None or "all" in categories_chosen:

-         categories_chosen = categories.keys()

+         categories_chosen = list(categories.keys())

      else:

          # copy list since we're about to modify it

          categories_chosen = list(categories_chosen)

      categories_chosen.sort()

      handlers = []

-     for name,value in globals().items():

+     for name,value in list(globals().items()):

          if name.startswith('handle_'):

              alias = name.replace('handle_','')

              alias = alias.replace('_','-')
@@ -7509,9 +7543,11 @@ 

                  session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas)

              else:

                  session.krb_login(proxyuser=options.runas)

-         except socket.error, e:

+         except socket.error:

+             e = sys.exc_info()[1]

              warn(_("Could not connect to Kerberos authentication service: %s") % e.args[1])

-         except Exception, e:

+         except Exception:

+             e = sys.exc_info()[1]

              if krbV is not None and isinstance(e, krbV.Krb5Error):

                  error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0]))

              else:

@@ -667,7 +667,7 @@ 

   * ``python-krbV``

   * ``python-mock``

   * ``python-simplejson``

-  * ``python-urlgrabber``

+  * ``python-pycurl``

   * ``python-psycopg2``

   * ``python-requests``

   * ``python-qpid-proton``

file modified
+1 -1
@@ -30,7 +30,7 @@ 

  Requires: pyOpenSSL

  Requires: python-requests

  Requires: python-requests-kerberos

- Requires: python-urlgrabber

+ Requires: python-pycurl

  Requires: python-dateutil

  BuildRequires: python

  %if %{use_systemd}

file modified
+64 -44
@@ -25,20 +25,18 @@ 

  try:

      import krbV

  except ImportError:  # pragma: no cover

+     krbV = None

      sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n")

      sys.stderr.flush()

  import base64

  import datetime

- import ConfigParser

  import errno

- import exceptions

  from fnmatch import fnmatch

- import httplib

  import imp

  import logging

  import logging.handlers

- from koji.util import md5_constructor

  SSL_Error = None

+ PythonImportError = ImportError # will be masked by koji.ImportError

  try:

      from OpenSSL.SSL import Error as SSL_Error

  except Exception:  #pragma: no cover
@@ -55,11 +53,11 @@ 

  import re

  try:

      import requests

- except ImportError:  #pragma: no cover

+ except ImportError:  # pragma: no cover

      requests = None

  try:

      from requests_kerberos import HTTPKerberosAuth

- except ImportError:  #pragma: no cover

+ except ImportError:  # pragma: no cover

      HTTPKerberosAuth = None

  import rpm

  import shutil
@@ -70,14 +68,32 @@ 

  import time

  import traceback

  import urllib

- import urllib2

- import urlparse

- import util

  import warnings

- import xmlrpclib

  import xml.sax

  import xml.sax.handler

- from xmlrpclib import loads, dumps, Fault

+ if sys.version_info[0] < 3:

+     import ConfigParser

+     import httplib

+     import urllib2

+     import urlparse

+     urllib_urlopen = urllib2.urlopen

+     urllib_splitport = urllib.splitport

+     urllib_urlencode = urllib.urlencode

+     urlparse_urlsplit = urlparse.urlsplit

+     import xmlrpclib

+     from xmlrpclib import loads, dumps, Fault

+ else: # pragma: no cover

+     import configparser as ConfigParser

+     import http.client as httplib

+     urllib_urlopen = urllib.request.urlopen

+     urllib_splitport = urllib.parse.splitport

+     urllib_urlencode = urllib.parse.urlencode

+     urlparse_urlsplit = urllib.parse.urlsplit

+     import xmlrpc.client as xmlrpclib

+     loads = xmlrpclib.loads

+     dumps = xmlrpclib.dumps

+     Fault = xmlrpclib.Fault

+ from koji.util import md5_constructor, adler32_constructor

  

  PROFILE_MODULES = {}  # {module_name: module_instance}

  
@@ -380,7 +396,7 @@ 

      code = getattr(fault, 'faultCode', None)

      if code is None:

          return fault

-     for v in globals().values():

+     for v in list(globals().values()):

          if isinstance(v, type(Exception)) and issubclass(v, GenericError) and \

                  code == getattr(v, 'faultCode', None):

              ret = v(fault.faultString)
@@ -398,7 +414,7 @@ 

          desc: the description of the exception (docstring)

      """

      ret = []

-     for n, v in globals().items():

+     for n, v in list(globals().items()):

          if isinstance(v, type(Exception)) and issubclass(v, GenericError):

              code = getattr(v, 'faultCode', None)

              if code is None:
@@ -471,8 +487,8 @@ 

      """Load xmlrpc data from a string, but catch faults"""

      try:

          return loads(s)

-     except Fault, f:

-         return f

+     except Fault:

+         return sys.exc_info()[1]

  

  ## BEGIN kojikamid dup

  
@@ -627,11 +643,11 @@ 

          print("Store at offset %d (%0x)" % (store, store))

          #sort entries by offset, dtype

          #also rearrange: tag, dtype, offset, count -> offset, dtype, tag, count

-         order = sorted([(x[2], x[1], x[0], x[3]) for x in self.index.itervalues()])

+         order = sorted([(x[2], x[1], x[0], x[3]) for x in list(self.index.values())])

          next = store

          #map some rpmtag codes

          tags = {}

-         for name, code in rpm.__dict__.iteritems():

+         for name, code in list(rpm.__dict__.items()):

              if name.startswith('RPMTAG_') and isinstance(code, int):

                  tags[code] = name[7:].lower()

          for entry in order:
@@ -1106,7 +1122,7 @@ 

          xml.sax.parseString(contents, handler)

  

      for field in fields:

-         if field not in values.keys():

+         if field not in list(values.keys()):

              raise GenericError('could not extract %s from POM: %s' % (field, (path or '<contents>')))

      return values

  
@@ -1407,7 +1423,7 @@ 

      if opts.get('maven_opts'):

          mavenrc = 'export MAVEN_OPTS="%s"\n' % ' '.join(opts['maven_opts'])

      if opts.get('maven_envs'):

-         for name, val in opts['maven_envs'].iteritems():

+         for name, val in list(opts['maven_envs'].items()):

              mavenrc += 'export %s="%s"\n' % (name, val)

      if mavenrc:

          files['etc/mavenrc'] = mavenrc
@@ -1470,25 +1486,25 @@ 

  """ % locals())

  

      parts.append("\n")

-     for key, value in config_opts.iteritems():

+     for key, value in list(config_opts.items()):

          parts.append("config_opts[%r] = %r\n" % (key, value))

      parts.append("\n")

-     for key, value in plugin_conf.iteritems():

+     for key, value in list(plugin_conf.items()):

          parts.append("config_opts['plugin_conf'][%r] = %r\n" % (key, value))

      parts.append("\n")

  

      if bind_opts:

          # This line is REQUIRED for mock to work if bind_opts defined.

          parts.append("config_opts['internal_dev_setup'] = False\n")

-         for key in bind_opts.keys():

-             for mnt_src, mnt_dest in bind_opts.get(key).iteritems():

+         for key in list(bind_opts.keys()):

+             for mnt_src, mnt_dest in list(bind_opts.get(key).items()):

                  parts.append("config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % (key, mnt_src, mnt_dest))

          parts.append("\n")

  

-     for key, value in macros.iteritems():

+     for key, value in list(macros.items()):

          parts.append("config_opts['macros'][%r] = %r\n" % (key, value))

      parts.append("\n")

-     for key, value in files.iteritems():

+     for key, value in list(files.items()):

          parts.append("config_opts['files'][%r] = %r\n" % (key, value))

  

      return ''.join(parts)
@@ -1517,7 +1533,7 @@ 

          rv += "Frame %s in %s at line %s\n" % (frame.f_code.co_name,

                                                 frame.f_code.co_filename,

                                                 frame.f_lineno)

-         for key, value in frame.f_locals.items():

+         for key, value in list(frame.f_locals.items()):

              rv += "  %20s = " % key

              # we must _absolutely_ avoid propagating eceptions, and str(value)

              # COULD cause any exception, so we MUST catch any...:
@@ -1534,7 +1550,7 @@ 

      on options"""

      if topurl:

          url = "%s/%s" % (topurl, relpath)

-         src = urllib2.urlopen(url)

+         src = urllib_urlopen(url)

          fo = tempfile.TemporaryFile(dir=tempdir)

          shutil.copyfileobj(src, fo)

          src.close()
@@ -1551,7 +1567,8 @@ 

      configs = []

      try:

          conf_dir_contents = os.listdir(dir_name)

-     except OSError, exception:

+     except OSError:

+         exception = sys.exc_info()[1]

          if exception.errno != errno.ENOENT:

              raise

      else:
@@ -1728,7 +1745,7 @@ 

  

  class PathInfo(object):

      # ASCII numbers and upper- and lower-case letter for use in tmpdir()

-     ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]

+     ASCII_CHARS = [chr(i) for i in list(range(48, 58)) + list(range(65, 91)) + list(range(97, 123))]

  

      def __init__(self, topdir=None):

          self._topdir = topdir
@@ -2060,7 +2077,7 @@ 

                  pass

  

          if not krbV:

-             raise exceptions.ImportError(

+             raise PythonImportError(

                  "Please install python-krbV to use kerberos."

              )

  
@@ -2125,8 +2142,8 @@ 

          """Get the Kerberos principal of the server we're connecting

          to, based on baseurl."""

  

-         uri = urlparse.urlsplit(self.baseurl)

-         host, port = urllib.splitport(uri[1])

+         uri = urlparse_urlsplit(self.baseurl)

+         host, port = urllib_splitport(uri[1])

          if self.opts.get('krb_rdns', True):

              servername = socket.getfqdn(host)

          else:
@@ -2138,13 +2155,13 @@ 

  

      def gssapi_login(self, proxyuser=None):

          if not HTTPKerberosAuth:

-             raise exceptions.ImportError(

+             raise PythonImportError(

                  "Please install python-requests-kerberos to use GSSAPI."

              )

  

          # force https

          old_baseurl = self.baseurl

-         uri = urlparse.urlsplit(self.baseurl)

+         uri = urlparse_urlsplit(self.baseurl)

          if uri[0] != 'https':

              self.baseurl = 'https://%s%s' % (uri[1], uri[2])

  
@@ -2188,7 +2205,7 @@ 

          # when API is changed

  

          # force https

-         uri = urlparse.urlsplit(self.baseurl)

+         uri = urlparse_urlsplit(self.baseurl)

          if uri[0] != 'https':

              self.baseurl = 'https://%s%s' % (uri[1], uri[2])

  
@@ -2263,7 +2280,7 @@ 

              sinfo = self.sinfo.copy()

              sinfo['callnum'] = self.callnum

              self.callnum += 1

-             handler = "%s?%s" % (self.baseurl, urllib.urlencode(sinfo))

+             handler = "%s?%s" % (self.baseurl, urllib_urlencode(sinfo))

          elif name == 'sslLogin':

              handler = self.baseurl + '/ssllogin'

          else:
@@ -2282,7 +2299,8 @@ 

          for i in (0, 1):

              try:

                  return self._sendOneCall(handler, headers, request)

-             except Exception, e:

+             except Exception:

+                 e = sys.exc_info()[1]

                  if i or not is_conn_error(e):

                      raise

                  self.logger.debug("Connection Error: %s", e)
@@ -2375,7 +2393,8 @@ 

                  # note that, for logged-in sessions the server should tell us (via a RetryError fault)

                  # if the call cannot be retried. For non-logged-in sessions, all calls should be read-only

                  # and hence retryable.

-                 except Fault, fault:

+                 except Fault:

+                     fault = sys.exc_info()[1]

                      #try to convert the fault to a known exception

                      err = convertFault(fault)

                      if isinstance(err, ServerOffline):
@@ -2391,7 +2410,8 @@ 

                  except (SystemExit, KeyboardInterrupt):

                      #(depending on the python version, these may or may not be subclasses of Exception)

                      raise

-                 except Exception, e:

+                 except Exception:

+                     e = sys.exc_info()[1]

                      tb_str = ''.join(traceback.format_exception(*sys.exc_info()))

                      self.new_session()

  
@@ -2471,7 +2491,7 @@ 

          if callback:

              callback(0, size, 0, 0, 0)

          problems = False

-         full_chksum = util.adler32_constructor()

+         full_chksum = adler32_constructor()

          # cycle is need to run at least once (for empty files)

          first_cycle = True

          callopts = {'overwrite': overwrite}
@@ -2486,7 +2506,7 @@ 

              result = self._callMethod('rawUpload', (chunk, ofs, path, name), callopts)

              if self.retries > 1:

                  problems = True

-             hexdigest = util.adler32_constructor(chunk).hexdigest()

+             hexdigest = adler32_constructor(chunk).hexdigest()

              full_chksum.update(chunk)

              if result['size'] != len(chunk):

                  raise GenericError("server returned wrong chunk size: %s != %s" % (result['size'], len(chunk)))
@@ -2534,7 +2554,7 @@ 

              args['volume'] = volume

          size = len(chunk)

          self.callnum += 1

-         handler = "%s?%s" % (self.baseurl, urllib.urlencode(args))

+         handler = "%s?%s" % (self.baseurl, urllib_urlencode(args))

          headers = [

              ('User-Agent', 'koji/1.7'),  #XXX

              ("Content-Type", "application/octet-stream"),
@@ -2669,7 +2689,7 @@ 

              values = []

              data = {}

              record.message = record.getMessage()

-             for key, value in self.mapping.iteritems():

+             for key, value in list(self.mapping.items()):

                  value = str(value)

                  if value.find("%(asctime)") >= 0:

                      if self.formatter:
@@ -2915,7 +2935,7 @@ 

          # decode it using the fallback encoding.

          try:

              return value.decode('utf8').encode('utf8')

-         except UnicodeDecodeError, err:

+         except UnicodeDecodeError:

              return value.decode(fallback).encode('utf8')

      else:

          return value

file modified
+19 -14
@@ -30,7 +30,10 @@ 

  import stat

  import sys

  import time

- import ConfigParser

+ if sys.version_info[0] < 3:

+     import ConfigParser

+ else: # pragma: no cover

+     import configparser as ConfigParser

  from zlib import adler32

  

  # imported from kojiweb and kojihub
@@ -154,12 +157,13 @@ 

          kwargs = {}

      try:

          return func(*args, **kwargs)

-     except TypeError, e:

-         if sys.exc_info()[2].tb_next is None:

+     except TypeError:

+         ei = sys.exc_info()

+         if ei[2].tb_next is None:

              # The stack is only one high, so the error occurred in this function.

              # Therefore, we assume the TypeError is due to a parameter mismatch

              # in the above function call.

-             raise koji.ParameterError(str(e))

+             raise koji.ParameterError(str(ei[1]))

          raise

  

  
@@ -232,17 +236,17 @@ 

          return LazyDict(self)

  

      def values(self):

-         return [lazy_eval(val) for val in super(LazyDict, self).values()]

+         return [lazy_eval(val) for val in list(super(LazyDict, self).values())]

  

      def items(self):

-         return [(key, lazy_eval(val)) for key, val in super(LazyDict, self).items()]

+         return [(key, lazy_eval(val)) for key, val in list(super(LazyDict, self).items())]

  

      def itervalues(self):

-         for val in super(LazyDict, self).itervalues():

+         for val in super(LazyDict, self).values():

              yield lazy_eval(val)

  

      def iteritems(self):

-         for key, val in super(LazyDict, self).iteritems():

+         for key, val in super(LazyDict, self).items():

              yield key, lazy_eval(val)

  

      def pop(self, key, *args, **kwargs):
@@ -457,19 +461,20 @@ 

          logger.warn('Setting resource limit: %s = %r', key, limits)

          try:

              resource.setrlimit(rcode, tuple(limits))

-         except ValueError, e:

+         except ValueError:

+             e = sys.exc_info()[1]

              logger.error("Unable to set %s: %s", key, e)

  

  class adler32_constructor(object):

  

      #mimicing the hashlib constructors

      def __init__(self, arg=''):

-         self._value = adler32(arg) & 0xffffffffL

+         self._value = adler32(arg) & 0xffffffff

          #the bitwise and works around a bug in some versions of python

          #see: https://bugs.python.org/issue1202

  

      def update(self, arg):

-         self._value = adler32(arg, self._value) & 0xffffffffL

+         self._value = adler32(arg, self._value) & 0xffffffff

  

      def digest(self):

          return self._value
@@ -496,11 +501,11 @@ 

      parts = parts.copy()

      result = []

      while True:

-         level = set([name for name, deps in parts.iteritems() if not deps])

+         level = set([name for name, deps in parts.items() if not deps])

          if not level:

              break

          result.append(level)

-         parts = dict([(name, deps - level) for name, deps in parts.iteritems()

+         parts = dict([(name, deps - level) for name, deps in parts.items()

                        if name not in level])

      if parts:

          raise ValueError('total ordering not possible')
@@ -641,7 +646,7 @@ 

      """

      builds = parse_maven_params(confs, chain=True, scratch=scratch)

      depmap = {}

-     for package, params in builds.items():

+     for package, params in list(builds.items()):

          depmap[package] = set(params.get('buildrequires', []))

      try:

          tsort(depmap)

file modified
+1 -1
@@ -81,7 +81,7 @@ 

          islink.assert_called_once_with(dst)

          move.assert_not_called()

  

-     @mock.patch('urllib2.urlopen')

+     @mock.patch('koji.urllib_urlopen')

      @mock.patch('tempfile.TemporaryFile')

      @mock.patch('shutil.copyfileobj')

      @mock.patch('__builtin__.open')

file modified
-1
@@ -39,7 +39,6 @@ 

  import sys

  import time

  import urllib2

- import urlgrabber.grabber as grabber

  import xmlrpclib  # for ProtocolError and Fault

  import rpm

  

file modified
+5 -2
@@ -40,8 +40,8 @@ 

  import threading

  import base64

  import pwd

- import urlgrabber

  import fnmatch

+ import pycurl

  from ConfigParser import ConfigParser

  from optparse import OptionParser

  try:
@@ -665,7 +665,10 @@ 

              else:

                  raise koji.BuildError('unsupported file type: %s' % type)

              koji.ensuredir(os.path.dirname(localpath))

-             urlgrabber.urlgrab(remote_url, filename=localpath)

+             c = pycurl.Curl()

+             c.setopt(c.URL, remote_url)

+             c.setopt(c.WRITEDATA, open(localpath, 'wb'))

+             c.perform()

  

          return file(localpath, 'r')

  

I've gone once more through and have some 2.4 compatible solution. It has quite a few ugly parts, but it seems to work. If there is anybody willing to test the functionality I would be glad. (CLI tests are already broken for a while for 2.4).

  • krbV support is not there - anyway gssapi should solve all the problems - works for me with Fedora
  • urlgrabber not avaiable in python3 variant - substituted with pycurl and uglier progress bar
  • try/except Exception, e vs Exception as e - I had to remove all of these and reacquire exception via sys.exc_info - ugly
  • Not sure if unicode works correctly everywhere

This should be removed if we're not using it anymore.

1 new commit added

  • remove urlgrabber completely
7 years ago

@ngompa It was used in one more place (kojivmd), but it is oneliner, so it doesn't hurt to remove dependency completely.

1 new commit added

  • group python3 imports
7 years ago

Glad to see things are moving. I hoped that 2.4 would be dropped and wasn't ready to do this kind of hacks. Let me know if I can still help in any way.

With py24 legacy on a separate branch, we are now free to target 2.6+ here, which should help us avoid some of painful workarounds.

Dropping this one, as we have legacy branch. Discussion continues in #151.

Pull-Request has been closed by tkopecek

6 years ago