From 147f781183a9e9710d155a5dc17fc5066b0c4b72 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 11 2018 21:37:45 +0000 Subject: PR#685: Rest of automated conversion from py3 changes Merges #685 https://pagure.io/koji/pull-request/685 --- diff --git a/builder/kojid b/builder/kojid index 2a74993..7ff9c49 100755 --- a/builder/kojid +++ b/builder/kojid @@ -21,6 +21,9 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import +from __future__ import division +import six try: import krbV except ImportError: # pragma: no cover @@ -36,7 +39,7 @@ import logging import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask -from koji.util import parseStatus, isSuccess, dslice, dslice_ex +from koji.util import parseStatus, isSuccess, dslice, dslice_ex, to_list import multilib.multilib as multilib import os import pwd @@ -53,11 +56,11 @@ import sys import time import traceback import xml.dom.minidom -import xmlrpclib +import six.moves.xmlrpc_client import zipfile import copy import Cheetah.Template -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser from fnmatch import fnmatch from gzip import GzipFile from optparse import OptionParser, SUPPRESS_HELP @@ -257,7 +260,7 @@ class BuildRoot(object): output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) #write config - fo = file(configfile,'w') + fo = open(configfile,'w') fo.write(output) fo.close() @@ -270,7 +273,7 @@ class BuildRoot(object): id_suffix = 'repo' name_prefix = 'Repository for Koji' for dep in self.deps: - if isinstance(dep, (int, long)): + if isinstance(dep, six.integer_types): # dep is a task ID, the url points to the task output directory repo_type = 'task' dep_url = pi.task(dep) @@ -357,7 +360,7 @@ class BuildRoot(object): """ settings = settings % locals() - fo = file(self.rootdir() + destfile, 'w') + fo = open(self.rootdir() + destfile, 'w') fo.write(settings) fo.close() @@ -1054,7 +1057,7 @@ class BuildTask(BaseTaskHandler): archdict[a] = 1 if not archdict: raise koji.BuildError("No matching arches were found") - return archdict.keys() + return to_list(archdict.keys()) def choose_taskarch(self, arch, srpm, build_tag): @@ -1115,7 +1118,7 @@ class BuildTask(BaseTaskHandler): self.logger.debug("Waiting on subtasks...") # wait for subtasks to finish - results = self.wait(subtasks.values(), all=True, failany=failany) + results = self.wait(to_list(subtasks.values()), all=True, failany=failany) # finalize import # merge data into needed args for completeBuild call @@ -1123,7 +1126,7 @@ class BuildTask(BaseTaskHandler): brmap = {} logs = {} built_srpm = None - for (arch, task_id) in subtasks.iteritems(): + for (arch, task_id) in six.iteritems(subtasks): result = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch,result,)) brootid = result['brootid'] @@ -1203,7 +1206,7 @@ class BuildArchTask(BaseBuildTask): self.logger.warn("Negative average build duration for %s: %s", name, avg) return # increase the task weight by 0.75 for every hour of build duration - adj = (avg / 4800.0) + adj = avg / 4800.0 # cap the adjustment at +4.5 weight = self.weight() + min(4.5, adj) self.session.host.setTaskWeight(self.id, weight) @@ -1631,7 +1634,7 @@ class BuildMavenTask(BaseBuildTask): for filepath in logs: self.uploadFile(os.path.join(outputdir, filepath), relPath=os.path.dirname(filepath)) - for relpath, files in output_files.iteritems(): + for relpath, files in six.iteritems(output_files): for filename in files: self.uploadFile(os.path.join(outputdir, relpath, filename), relPath=relpath) @@ -1867,7 +1870,7 @@ class WrapperRPMTask(BaseBuildTask): contents = contents.encode('utf-8') specfile = spec_template[:-5] - specfd = file(specfile, 'w') + specfd = open(specfile, 'w') specfd.write(contents) specfd.close() @@ -2078,7 +2081,7 @@ class ChainMavenTask(MultiPlatformTask): pkg_to_wrap = params['buildrequires'][0] to_wrap = self.done[pkg_to_wrap] - if isinstance(to_wrap, (int, long)): + if isinstance(to_wrap, six.integer_types): task_to_wrap = self.session.getTaskInfo(to_wrap, request=True) build_to_wrap = None else: @@ -2097,8 +2100,8 @@ class ChainMavenTask(MultiPlatformTask): running[task_id] = package del todo[package] try: - results = self.wait(running.keys()) - except (xmlrpclib.Fault, koji.GenericError) as e: + results = self.wait(to_list(running.keys())) + except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: # One task has failed, wait for the rest to complete before the # chainmaven task fails. self.wait(all=True) should thrown an exception. self.wait(all=True) @@ -2150,8 +2153,8 @@ class ChainMavenTask(MultiPlatformTask): have the same keys and those keys have the same values. If a value is list, it will be considered equal to a list with the same values in a different order.""" - akeys = a.keys() - bkeys = b.keys() + akeys = to_list(a.keys()) + bkeys = to_list(b.keys()) if sorted(akeys) != sorted(bkeys): return False for key in akeys: @@ -2312,7 +2315,7 @@ class BuildBaseImageTask(BuildImageTask): canfail.append(subtasks[arch]) self.logger.debug("Got image subtasks: %r" % (subtasks)) self.logger.debug("Waiting on image subtasks (%s can fail)..." % canfail) - results = self.wait(subtasks.values(), all=True, failany=True, canfail=canfail) + results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail) # if everything failed, fail even if all subtasks are in canfail self.logger.debug('subtask results: %r', results) @@ -2621,7 +2624,7 @@ class BuildLiveMediaTask(BuildImageTask): self.logger.debug("Got image subtasks: %r", subtasks) self.logger.debug("Waiting on livemedia subtasks...") - results = self.wait(subtasks.values(), all=True, failany=True, canfail=canfail) + results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail) # if everything failed, fail even if all subtasks are in canfail self.logger.debug('subtask results: %r', results) @@ -2657,7 +2660,7 @@ class BuildLiveMediaTask(BuildImageTask): wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist, label='wrapper %s' % arch, arch='noarch') - results2 = self.wait(wrapper_tasks.values(), all=True, failany=True) + results2 = self.wait(to_list(wrapper_tasks.values()), all=True, failany=True) self.logger.debug('wrapper results: %r', results2) # add wrapper rpm results into main results @@ -2835,7 +2838,7 @@ class ImageTask(BaseTaskHandler): self.ks.handler.repo.repoList = [] # delete whatever the ks file told us if opts.get('repo'): user_repos = opts['repo'] - if isinstance(user_repos, basestring): + if isinstance(user_repos, six.string_types): user_repos = user_repos.split(',') index = 0 for user_repo in user_repos: @@ -2919,7 +2922,7 @@ class ImageTask(BaseTaskHandler): } # Duplicated with pungi/util.py _apply_substitutions - for k, v in sorted(substitutions.items(), key=lambda x: len(x[0]), reverse=True): + for k, v in sorted(to_list(substitutions.items()), key=lambda x: len(x[0]), reverse=True): if k in name: name = name.replace(k, v) if k in version: @@ -4955,7 +4958,7 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r def uniq(self, items): """Remove duplicates from the list of items, and sort the list.""" m = dict(zip(items, [1] * len(items))) - l = m.keys() + l = to_list(m.keys()) l.sort() return l @@ -5000,8 +5003,8 @@ class NewRepoTask(BaseTaskHandler): # gather subtask results data = {} if subtasks: - results = self.wait(subtasks.values(), all=True, failany=True) - for (arch, task_id) in subtasks.iteritems(): + results = self.wait(to_list(subtasks.values()), all=True, failany=True) + for (arch, task_id) in six.iteritems(subtasks): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch,data[arch],)) @@ -5042,7 +5045,7 @@ class CreaterepoTask(BaseTaskHandler): if external_repos: self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: - fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') + fo = open(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() @@ -5154,7 +5157,7 @@ class NewDistRepoTask(BaseTaskHandler): method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') if len(subtasks) > 0 and task_opts['multilib']: - self.wait(subtasks.values(), all=True, failany=True) + self.wait(to_list(subtasks.values()), all=True, failany=True) for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib @@ -5168,8 +5171,8 @@ class NewDistRepoTask(BaseTaskHandler): method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') # wait for 64-bit subtasks to finish - self.wait(subtasks.values(), all=True, failany=True) - for (arch, task_id) in subtasks.iteritems(): + self.wait(to_list(subtasks.values()), all=True, failany=True) + for (arch, task_id) in six.iteritems(subtasks): if task_opts['multilib'] and arch in arch32s: # already moved above continue @@ -5505,7 +5508,7 @@ enabled=1 # select our rpms selected = {} for rpm_id in rpm_idx: - avail_keys = rpm_idx[rpm_id].keys() + avail_keys = to_list(rpm_idx[rpm_id].keys()) best_key = self.pick_key(keys, avail_keys) if best_key is None: # we lack a matching key for this rpm @@ -5569,7 +5572,7 @@ enabled=1 fmt = '%(name)s-%(version)s-%(release)s.%(arch)s' filenames = [[fmt % selected[r], r] for r in sig_missing] for fname, rpm_id in sorted(filenames): - avail = rpm_idx.get(rpm_id, {}).keys() + avail = to_list(rpm_idx.get(rpm_id, {}).keys()) outfile.write('%s: %r\n' % (fname, avail)) outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) @@ -5620,7 +5623,7 @@ enabled=1 def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') - datafile = file(filename, 'w') + datafile = open(filename, 'w') try: json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True) finally: @@ -5656,9 +5659,9 @@ class WaitrepoTask(BaseTaskHandler): if not targets: raise koji.GenericError("No build target for tag: %s" % taginfo['name']) - if isinstance(newer_than, basestring) and newer_than.lower() == "now": + if isinstance(newer_than, six.string_types) and newer_than.lower() == "now": newer_than = start - if not isinstance(newer_than, (type(None), int, long, float)): + if not isinstance(newer_than, to_list(six.integer_types) + [type(None), float]): raise koji.GenericError("Invalid value for newer_than: %s" % newer_than) if newer_than and nvrs: @@ -5823,7 +5826,7 @@ def get_options(): defaults[name] = config.getboolean('kojid', name) elif name in ['plugin', 'plugins']: defaults['plugin'] = value.split() - elif name in defaults.keys(): + elif name in to_list(defaults.keys()): defaults[name] = value elif name.upper().startswith('RLIMIT_'): defaults[name.upper()] = value @@ -5913,7 +5916,7 @@ if __name__ == "__main__": options.serverca) except koji.AuthError as e: quit("Error: Unable to log in: %s" % e) - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) elif options.user: try: @@ -5921,7 +5924,7 @@ if __name__ == "__main__": session.login() except koji.AuthError: quit("Error: Unable to log in. Bad credentials?") - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) elif krbV: krb_principal = options.krb_principal @@ -5947,7 +5950,7 @@ if __name__ == "__main__": #make sure it works try: ret = session.echo("OK") - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) if ret != ["OK"]: quit("Error: incorrect server response: %r" % (ret)) diff --git a/builder/mergerepos b/builder/mergerepos index bb3ee6d..5e4b357 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -21,6 +21,7 @@ # Largely borrowed from the mergerepo script included in createrepo and # written by Seth Vidal +from __future__ import absolute_import import createrepo import os.path import rpmUtils.miscutils @@ -238,7 +239,7 @@ class RepoMerge(object): include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = file(pkgorigins, 'w') + origins = open(pkgorigins, 'w') seen_rpms = {} for repo in repos: @@ -283,7 +284,7 @@ def main(args): opts = parse_args(args) if opts.blocked: - blocked_fo = file(opts.blocked) + blocked_fo = open(opts.blocked) blocked_list = blocked_fo.readlines() blocked_fo.close() blocked = dict([(b.strip(), 1) for b in blocked_list]) diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 77bc342..7341490 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -23,6 +23,7 @@ import six.moves.xmlrpc_client from six.moves import filter from six.moves import map from six.moves import zip +from six.moves import range try: import libcomps @@ -34,7 +35,7 @@ except ImportError: # pragma: no cover yumcomps = None import koji -from koji.util import md5_constructor +from koji.util import md5_constructor, to_list from koji_cli.lib import _, OptionParser, activate_session, parse_arches, \ _unique_path, _running_in_bg, _progress_callback, watch_tasks, \ arg_filter, linked_upload, list_task_output_all_volumes, \ @@ -653,9 +654,9 @@ def handle_maven_build(options, session, args): section=build_opts.section) except ValueError as e: parser.error(e.args[0]) - opts = list(params.values())[0] + opts = to_list(params.values())[0] if opts.pop('type', 'maven') != 'maven': - parser.error(_("Section %s does not contain a maven-build config") % list(params.keys())[0]) + parser.error(_("Section %s does not contain a maven-build config") % to_list(params.keys())[0]) source = opts.pop('scmurl') else: source = args[1] @@ -714,9 +715,9 @@ def handle_wrapper_rpm(options, session, args): section=build_opts.section) except ValueError as e: parser.error(e.args[0]) - opts = list(params.values())[0] + opts = to_list(params.values())[0] if opts.get('type') != 'wrapper': - parser.error(_("Section %s does not contain a wrapper-rpm config") % list(params.keys())[0]) + parser.error(_("Section %s does not contain a wrapper-rpm config") % to_list(params.keys())[0]) url = opts['scmurl'] package = opts['buildrequires'][0] target_info = session.getBuildTarget(target, strict=True) @@ -1145,7 +1146,7 @@ def handle_import(goptions, session, args): nvr = "%(name)s-%(version)s-%(release)s" % koji.parse_NVRA(data['sourcerpm']) to_import.setdefault(nvr,[]).append((path,data)) builds_missing = False - nvrs = list(to_import.keys()) + nvrs = to_list(to_import.keys()) nvrs.sort() for nvr in nvrs: to_import[nvr].sort() @@ -1373,7 +1374,7 @@ def _import_comps(session, filename, tag, options): for k in pkgopts.keys(): if six.PY2 and isinstance(pkgopts[k], unicode): pkgopts[k] = str(pkgopts[k]) - s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(list(pkgopts.keys()))]) + s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(pkgopts.keys())]) print(" Package: %s: {%s}" % (pkg.name, s_opts)) session.groupPackageListAdd(tag, group.id, pkg.name, force=force, **pkgopts) # libcomps does not support group dependencies @@ -1406,7 +1407,7 @@ def _import_comps_alt(session, filename, tag, options): # no cover 3.x for k in pkgopts.keys(): if six.PY2 and isinstance(pkgopts[k], unicode): pkgopts[k] = str(pkgopts[k]) - s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(list(pkgopts.keys()))]) + s_opts = ', '.join(["'%s': %r" % (k, pkgopts[k]) for k in sorted(pkgopts.keys())]) print(" Package: %s: {%s}" % (pkg, s_opts)) session.groupPackageListAdd(tag, group.groupid, pkg, force=force, **pkgopts) #yum.comps does not support group dependencies @@ -1616,7 +1617,7 @@ def handle_prune_signed_copies(options, session, args): #that the build was recently untagged from tags.setdefault(entry['tag_name'], 1) if options.debug: - print("Tags: %s" % list(tags.keys())) + print("Tags: %s" % to_list(tags.keys())) for tag_name in tags: if tag_name == options.trashcan_tag: if options.debug: @@ -1833,7 +1834,7 @@ def handle_prune_signed_copies(options, session, args): except OSError as e: print("Error removing %s: %s" % (signedpath, e)) if len(sigdirs) == 1: - dir = list(sigdirs.keys())[0] + dir = to_list(sigdirs.keys())[0] if options.test: print("Would have removed dir: %s" % dir) else: @@ -4133,7 +4134,7 @@ def _print_histline(entry, **kwargs): else: return '%s.name' % key if edit: - keys = list(x.keys()) + keys = to_list(x.keys()) keys.sort() y = other[-1] for key in keys: @@ -4148,7 +4149,7 @@ def _print_histline(entry, **kwargs): continue print(" %s: %s -> %s" % (key, x[key], y[key])) elif create and options.verbose and table != 'tag_listing': - keys = list(x.keys()) + keys = to_list(x.keys()) keys.sort() # the table keys have already been represented in the base format string also_hidden = list(_table_keys[table]) @@ -4635,9 +4636,7 @@ def anon_handle_taginfo(goptions, session, args): print("Include all Maven archives?: %s" % (info['maven_include_all'] and 'yes' or 'no')) if 'extra' in info: print("Tag options:") - keys = list(info['extra'].keys()) - keys.sort() - for key in keys: + for key in sorted(info['extra'].keys()): print(" %s : %s" % (key, pprint.pformat(info['extra'][key]))) dest_targets = session.getBuildTargets(destTagID=info['id'], **event_opts) build_targets = session.getBuildTargets(buildTagID=info['id'], **event_opts) @@ -6829,8 +6828,7 @@ def anon_handle_wait_repo(options, session, args): targets = session.getBuildTargets(destTagID=tag_info['id']) if targets: maybe = {}.fromkeys([t['build_tag_name'] for t in targets]) - maybe = list(maybe.keys()) - maybe.sort() + maybe = sorted(maybe.keys()) print("Suggested tags: %s" % ', '.join(maybe)) return 1 tag_id = tag_info['id'] diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index 0bc86b0..02c6fd2 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -19,6 +19,7 @@ except ImportError: # pragma: no cover krbV = None import koji +from koji.util import to_list # fix OptionParser for python 2.3 (optparse verion 1.4.1+) # code taken from optparse version 1.5a2 @@ -94,7 +95,7 @@ categories = { def get_epilog_str(progname=None): if progname is None: progname = os.path.basename(sys.argv[0]) or 'koji' - categories_ordered=', '.join(sorted(['all'] + list(categories.keys()))) + categories_ordered=', '.join(sorted(['all'] + to_list(categories.keys()))) epilog_str = ''' Try "%(progname)s --help" for help about global options Try "%(progname)s help" to get all available commands @@ -301,7 +302,7 @@ def watch_tasks(session, tasklist, quiet=False, poll_interval=60): rv = 1 for child in session.getTaskChildren(task_id): child_id = child['id'] - if not child_id in list(tasks.keys()): + if not child_id in tasks.keys(): tasks[child_id] = TaskWatcher(child_id, session, task.level + 1, quiet=quiet) tasks[child_id].update() # If we found new children, go through the list again, diff --git a/hub/kojihub.py b/hub/kojihub.py index f37fe1a..22dd484 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -22,6 +22,7 @@ # Mike Bonnet # Cristian Balint +from __future__ import absolute_import import base64 import calendar import koji.rpmdiff @@ -42,7 +43,7 @@ import tempfile import time import traceback import urlparse -import xmlrpclib +import six.moves.xmlrpc_client import zipfile import rpm @@ -61,6 +62,8 @@ from koji.util import md5_constructor from koji.util import multi_fnmatch from koji.util import safer_move from koji.util import sha1_constructor +from koji.util import to_list +from six.moves import range logger = logging.getLogger('koji.hub') def log_error(msg): @@ -395,7 +398,7 @@ class Task(object): if xml_request.find(' %(after)s OR ev2.time > %(after)s') @@ -6864,7 +6867,7 @@ def query_history(tables=None, **kwargs): fields[c_test] = '_created_after_event' fields[r_test] = '_revoked_after_event' elif arg == 'before': - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): value = datetime.datetime.fromtimestamp(value).isoformat(' ') data['before'] = value clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s') @@ -7019,7 +7022,7 @@ def build_references(build_id, limit=None): idx.setdefault(row['id'], row) if limit is not None and len(idx) > limit: break - ret['rpms'] = idx.values() + ret['rpms'] = to_list(idx.values()) ret['component_of'] = [] # find images/archives that contain the build rpms @@ -7050,7 +7053,7 @@ def build_references(build_id, limit=None): idx.setdefault(row['id'], row) if limit is not None and len(idx) > limit: break - ret['archives'] = idx.values() + ret['archives'] = to_list(idx.values()) # find images/archives that contain the build archives fields = ['archive_id'] @@ -7392,7 +7395,7 @@ def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_s from_tag = get_tag(from_id) for email in get_notification_recipients(build, from_tag['id'], state): recipients[email] = 1 - recipients_uniq = recipients.keys() + recipients_uniq = to_list(recipients.keys()) if len(recipients_uniq) > 0 and not (is_successful and ignore_success): task_id = make_task('tagNotification', [recipients_uniq, is_successful, tag_id, from_id, build_id, user_id, ignore_success, failure_msg]) return task_id @@ -7620,8 +7623,8 @@ class InsertProcessor(object): if not self.data and not self.rawdata: return "-- incomplete update: no assigns" parts = ['INSERT INTO %s ' % self.table] - columns = self.data.keys() - columns.extend(self.rawdata.keys()) + columns = to_list(self.data.keys()) + columns.extend(to_list(self.rawdata.keys())) parts.append("(%s) " % ', '.join(columns)) values = [] for key in columns: @@ -7664,7 +7667,7 @@ class InsertProcessor(object): del data['create_event'] del data['creator_id'] clauses = ["%s = %%(%s)s" % (k, k) for k in data] - query = QueryProcessor(columns=data.keys(), tables=[self.table], + query = QueryProcessor(columns=to_list(data.keys()), tables=[self.table], clauses=clauses, values=data) if query.execute(): return True @@ -8043,7 +8046,7 @@ def policy_get_pkg(data): if not pkginfo: #for some operations (e.g. adding a new package), the package #entry may not exist yet - if isinstance(data['package'], basestring): + if isinstance(data['package'], six.string_types): return {'id' : None, 'name' : data['package']} else: raise koji.GenericError("Invalid package: %s" % data['package']) @@ -8331,7 +8334,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): return False groups = koji.auth.get_user_groups(user['id']) args = self.str.split()[1:] - for group_id, group in groups.iteritems(): + for group_id, group in six.iteritems(groups): for pattern in args: if fnmatch.fnmatch(group, pattern): return True @@ -8931,7 +8934,7 @@ class RootExports(object): values = {} q = """SELECT id, EXTRACT(EPOCH FROM time) FROM events""" if before is not None: - if not isinstance(before, (int, long, float)): + if not isinstance(before, list(six.integer_types) + [float]): raise koji.GenericError('invalid type for before: %s' % type(before)) # use the repr() conversion because it retains more precision than the # string conversion @@ -8963,7 +8966,7 @@ class RootExports(object): # we will accept offset and size as strings to work around xmlrpc limits offset = koji.decode_int(offset) size = koji.decode_int(size) - if isinstance(md5sum, basestring): + if isinstance(md5sum, six.string_types): # this case is for backwards compatibility verify = "md5" digest = md5sum @@ -9551,9 +9554,9 @@ class RootExports(object): if before: if isinstance(before, datetime.datetime): before = calendar.timegm(before.utctimetuple()) - elif isinstance(before, (str, unicode)): + elif isinstance(before, six.string_types): before = koji.util.parseTime(before) - elif isinstance(before, (int, long)): + elif isinstance(before, six.integer_types): pass else: raise koji.GenericError('invalid type for before: %s' % type(before)) @@ -9561,9 +9564,9 @@ class RootExports(object): if after: if isinstance(after, datetime.datetime): after = calendar.timegm(after.utctimetuple()) - elif isinstance(after, (str, unicode)): + elif isinstance(after, six.string_types): after = koji.util.parseTime(after) - elif isinstance(after, (int, long)): + elif isinstance(after, six.integer_types): pass else: raise koji.GenericError('invalid type for after: %s' % type(after)) @@ -9654,7 +9657,7 @@ class RootExports(object): def listTagged(self, tag, event=None, inherit=False, prefix=None, latest=False, package=None, owner=None, type=None): """List builds tagged with tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) results = readTaggedBuilds(tag, event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) @@ -9665,14 +9668,14 @@ class RootExports(object): def listTaggedRPMS(self, tag, event=None, inherit=False, latest=False, package=None, arch=None, rpmsigs=False, owner=None, type=None): """List rpms and builds within tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, event=event, inherit=inherit, latest=latest, package=package, arch=arch, rpmsigs=rpmsigs, owner=owner, type=type) def listTaggedArchives(self, tag, event=None, inherit=False, latest=False, package=None, type=None): """List archives and builds within a tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): tag = get_tag_id(tag, strict=True) return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) @@ -9851,14 +9854,14 @@ class RootExports(object): def getLatestBuilds(self, tag, event=None, package=None, type=None): """List latest builds for tag (inheritance enabled)""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedBuilds(tag, event, inherit=True, latest=True, package=package, type=type) def getLatestRPMS(self, tag, package=None, arch=None, event=None, rpmsigs=False, type=None): """List latest RPMS for tag (inheritance enabled)""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readTaggedRPMS(tag, package=package, arch=arch, event=event, inherit=True, latest=True, rpmsigs=rpmsigs, type=type) @@ -9918,13 +9921,13 @@ class RootExports(object): def getInheritanceData(self, tag, event=None): """Return inheritance data for tag""" - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) return readInheritanceData(tag, event) def setInheritanceData(self, tag, data, clear=False): - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) context.session.assertPerm('admin') @@ -9935,7 +9938,7 @@ class RootExports(object): stops = {} if jumps is None: jumps = {} - if not isinstance(tag, (int, long)): + if not isinstance(tag, six.integer_types): #lookup tag id tag = get_tag_id(tag, strict=True) for mapping in [stops, jumps]: @@ -9962,7 +9965,7 @@ class RootExports(object): - buildroot_id If no build has the given ID, or the build generated no RPMs, an empty list is returned.""" - if not isinstance(build, (int, long)): + if not isinstance(build, six.integer_types): #lookup build id build = self.findBuildID(build, strict=True) return self.listRPMs(buildID=build) @@ -10187,9 +10190,9 @@ class RootExports(object): userID = get_user(userID, strict=True)['id'] if pkgID is not None: pkgID = get_package_id(pkgID, strict=True) - result_list = readPackageList(tagID=tagID, userID=userID, pkgID=pkgID, + result_list = list(readPackageList(tagID=tagID, userID=userID, pkgID=pkgID, inherit=inherited, with_dups=with_dups, - event=event).values() + event=event).values()) if with_dups: # when with_dups=True, readPackageList returns a list of list of dicts # convert it to a list of dicts for consistency @@ -10353,7 +10356,7 @@ class RootExports(object): return taginfo def getRepo(self, tag, state=None, event=None, dist=False): - if isinstance(tag, (int, long)): + if isinstance(tag, six.integer_types): id = tag else: id = get_tag_id(tag, strict=True) @@ -10626,8 +10629,8 @@ class RootExports(object): if val.find(' -from ConfigParser import RawConfigParser +from __future__ import absolute_import +from __future__ import division +from six.moves.configparser import RawConfigParser import datetime import inspect import logging @@ -38,6 +40,8 @@ import koji.util # import xmlrpclib functions from koji to use tweaked Marshaller from koji.xmlrpcplus import getparser, dumps, Fault, ExtendedMarshaller from koji.context import context +from six.moves import range +import six class Marshaller(ExtendedMarshaller): @@ -96,7 +100,7 @@ class HandlerRegistry(object): Handlers are functions marked with one of the decorators defined in koji.plugin """ - for v in vars(plugin).itervalues(): + for v in six.itervalues(vars(plugin)): if isinstance(v, type): #skip classes continue @@ -116,7 +120,7 @@ class HandlerRegistry(object): if ret: return ret ret = tuple(inspect.getargspec(func)) - if inspect.ismethod(func) and func.im_self: + if inspect.ismethod(func) and func.__self__: # bound method, remove first arg args, varargs, varkw, defaults = ret if args: @@ -143,17 +147,17 @@ class HandlerRegistry(object): def _getFuncArgs(self, func): args = [] - for x in range(0, func.func_code.co_argcount): - if x == 0 and func.func_code.co_varnames[x] == "self": + for x in range(0, func.__code__.co_argcount): + if x == 0 and func.__code__.co_varnames[x] == "self": continue - if func.func_defaults and func.func_code.co_argcount - x <= len(func.func_defaults): - args.append((func.func_code.co_varnames[x], func.func_defaults[x - func.func_code.co_argcount + len(func.func_defaults)])) + if func.__defaults__ and func.__code__.co_argcount - x <= len(func.__defaults__): + args.append((func.__code__.co_varnames[x], func.__defaults__[x - func.__code__.co_argcount + len(func.__defaults__)])) else: - args.append(func.func_code.co_varnames[x]) + args.append(func.__code__.co_varnames[x]) return args def system_listMethods(self): - return self.funcs.keys() + return koji.util.to_list(self.funcs.keys()) def system_methodSignature(self, method): #it is not possible to autogenerate this data @@ -475,7 +479,7 @@ def load_config(environ): opts['policy'] = dict(config.items('policy')) else: opts['policy'] = {} - for pname, text in _default_policies.iteritems(): + for pname, text in six.iteritems(_default_policies): opts['policy'].setdefault(pname, text) # use configured KojiDir if opts.get('KojiDir') is not None: @@ -543,14 +547,14 @@ def get_policy(opts, plugins): continue alltests.append(koji.policy.findSimpleTests(vars(plugin))) policy = {} - for pname, text in opts['policy'].iteritems(): + for pname, text in six.iteritems(opts['policy']): #filter/merge tests merged = {} for tests in alltests: # tests can be limited to certain policies by setting a class variable - for name, test in tests.iteritems(): + for name, test in six.iteritems(tests): if hasattr(test, 'policy'): - if isinstance(test.policy, basestring): + if isinstance(test.policy, six.string_types): if pname != test.policy: continue elif pname not in test.policy: @@ -647,7 +651,7 @@ def load_scripts(environ): def get_memory_usage(): pagesize = resource.getpagesize() - statm = [pagesize*int(y)/1024 for y in "".join(open("/proc/self/statm").readlines()).strip().split()] + statm = [pagesize * int(y) // 1024 for y in "".join(open("/proc/self/statm").readlines()).strip().split()] size, res, shr, text, lib, data, dirty = statm return res - shr diff --git a/koji/__init__.py b/koji/__init__.py index 9a084a3..05428af 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -23,6 +23,7 @@ from __future__ import absolute_import +from __future__ import division import sys from six.moves import range from six.moves import zip @@ -449,7 +450,7 @@ def decode_args2(args, names, strict=True): args, opts = decode_args(*args) if strict and len(names) < len(args): raise TypeError("Expecting at most %i arguments" % len(names)) - ret = dict(list(zip(names, args))) + ret = dict(zip(names, args)) ret.update(opts) return ret @@ -1165,7 +1166,7 @@ def parse_pom(path=None, contents=None): xml.sax.parseString(contents, handler) for field in fields: - if field not in list(values.keys()): + if field not in util.to_list(values.keys()): raise GenericError('could not extract %s from POM: %s' % (field, (path or ''))) return values @@ -2178,7 +2179,7 @@ class ClientSession(object): # decode and decrypt the login info sinfo_priv = base64.decodestring(sinfo_enc) sinfo_str = ac.rd_priv(sinfo_priv) - sinfo = dict(list(zip(['session-id', 'session-key'], sinfo_str.split()))) + sinfo = dict(zip(['session-id', 'session-key'], sinfo_str.split())) if not sinfo: self.logger.warn('No session info received') @@ -2569,7 +2570,7 @@ class ClientSession(object): ret = [] callgrp = (calls[i:i + batch] for i in range(0, len(calls), batch)) self.logger.debug("MultiCall with batch size %i, calls/groups(%i/%i)", - batch, len(calls), round(len(calls) / batch)) + batch, len(calls), round(len(calls) // batch)) for c in callgrp: ret.extend(self._callMethod('multiCall', (c,), {})) else: @@ -2750,9 +2751,9 @@ class ClientSession(object): if t2 <= 0: t2 = 1 if debug: - self.logger.debug("Uploaded %d bytes in %f seconds (%f kbytes/sec)" % (size, t1, size/t1/1024)) + self.logger.debug("Uploaded %d bytes in %f seconds (%f kbytes/sec)" % (size, t1, size / t1 / 1024.0)) if debug: - self.logger.debug("Total: %d bytes in %f seconds (%f kbytes/sec)" % (ofs, t2, ofs/t2/1024)) + self.logger.debug("Total: %d bytes in %f seconds (%f kbytes/sec)" % (ofs, t2, ofs / t2 / 1024.0)) if callback: callback(ofs, totalsize, size, t1, t2) fo.close() @@ -3060,7 +3061,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False): if isinstance(value, tuple): return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value]) elif isinstance(value, list): - return list([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value]) + return [fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value] elif isinstance(value, dict): ret = {} for k in value: diff --git a/koji/auth.py b/koji/auth.py index 69b64ee..de03e03 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -34,6 +34,7 @@ from .context import context from six.moves import range from six.moves import zip import six +from .util import to_list # 1 - load session if provided # - check uri for session id @@ -108,7 +109,7 @@ class Session(object): 'user_id': 'user_id', } # sort for stability (unittests) - fields, aliases = list(zip(*list(sorted(fields.items(), key=lambda x: x[1])))) + fields, aliases = zip(*sorted(fields.items(), key=lambda x: x[1])) q = """ SELECT %s FROM sessions WHERE id = %%(id)i @@ -120,7 +121,7 @@ class Session(object): row = c.fetchone() if not row: raise koji.AuthError('Invalid session or bad credentials') - session_data = dict(list(zip(aliases, row))) + session_data = dict(zip(aliases, row)) #check for expiration if session_data['expired']: raise koji.AuthExpired('session "%i" has expired' % id) @@ -158,7 +159,7 @@ class Session(object): fields = ('name', 'status', 'usertype') q = """SELECT %s FROM users WHERE id=%%(user_id)s""" % ','.join(fields) c.execute(q, session_data) - user_data = dict(list(zip(fields, c.fetchone()))) + user_data = dict(zip(fields, c.fetchone())) if user_data['status'] != koji.USER_STATUS['NORMAL']: raise koji.AuthError('logins by %s are not allowed' % user_data['name']) @@ -537,7 +538,7 @@ class Session(object): def getPerms(self): if not self.logged_in: return [] - return list(self.perms.keys()) + return to_list(self.perms.keys()) def hasPerm(self, name): if not self.logged_in: @@ -709,7 +710,7 @@ def get_user_data(user_id): row = c.fetchone() if not row: return None - return dict(list(zip(fields, row))) + return dict(zip(fields, row)) def login(*args, **opts): return context.session.login(*args, **opts) diff --git a/koji/context.py b/koji/context.py index 811c6c1..3bcff4e 100755 --- a/koji/context.py +++ b/koji/context.py @@ -24,7 +24,10 @@ # - request data # - auth data -import thread +from __future__ import absolute_import +import six.moves._thread +from six.moves import range +import six class _data(object): pass @@ -35,7 +38,7 @@ class ThreadLocal(object): # should probably be getattribute, but easier to debug this way def __getattr__(self, key): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: raise AttributeError(key) @@ -43,7 +46,7 @@ class ThreadLocal(object): return object.__getattribute__(data, key) def __setattr__(self, key, value): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: tdict[id] = _data() @@ -51,7 +54,7 @@ class ThreadLocal(object): return object.__setattr__(data, key, value) def __delattr__(self, key): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: raise AttributeError(key) @@ -62,14 +65,14 @@ class ThreadLocal(object): return ret def __str__(self): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') return "(current thread: %s) {" % id + \ - ", ".join(["%s : %s" %(k, v.__dict__) for (k, v) in tdict.iteritems()]) + \ + ", ".join(["%s : %s" %(k, v.__dict__) for (k, v) in six.iteritems(tdict)]) + \ "}" def _threadclear(self): - id = thread.get_ident() + id = six.moves._thread.get_ident() tdict = object.__getattribute__(self, '_tdict') if id not in tdict: return @@ -97,8 +100,8 @@ if __name__ == '__main__': context._threadclear() print(context) - for x in xrange(1, 10): - thread.start_new_thread(test, ()) + for x in range(1, 10): + six.moves._thread.start_new_thread(test, ()) time.sleep(4) print('') diff --git a/koji/daemon.py b/koji/daemon.py index decee91..697efd9 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -20,11 +20,14 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import +from __future__ import division import koji import koji.tasks import koji.xmlrpcplus from koji.tasks import safe_rmtree -from koji.util import md5_constructor, adler32_constructor, parseStatus, dslice +from koji.util import md5_constructor, adler32_constructor, parseStatus, \ + dslice, to_list import os import signal import logging @@ -36,6 +39,8 @@ import subprocess import sys import traceback import errno +from six.moves import range +import six def incremental_upload(session, fname, fd, path, retries=5, logger=None): @@ -580,7 +585,7 @@ class TaskManager(object): """Attempt to shut down cleanly""" for task_id in self.pids.keys(): self.cleanupTask(task_id) - self.session.host.freeTasks(self.tasks.keys()) + self.session.host.freeTasks(to_list(self.tasks.keys())) self.session.host.updateHost(task_load=0.0, ready=False) def updateBuildroots(self, nolocal=False): @@ -611,14 +616,14 @@ class TaskManager(object): #task not running - expire the buildroot #TODO - consider recycling hooks here (with strong sanity checks) self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br) - self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, self.tasks.keys())) + self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id, to_list(self.tasks.keys()))) self.session.host.setBuildRootState(id, st_expired) continue if nolocal: return local_br = self._scanLocalBuildroots() # get info on local_only buildroots (most likely expired) - local_only = [id for id in local_br.iterkeys() if id not in db_br] + local_only = [id for id in six.iterkeys(local_br) if id not in db_br] if local_only: missed_br = self.session.listBuildroots(buildrootID=tuple(local_only)) #get all the task info in one call @@ -719,7 +724,7 @@ class TaskManager(object): fo = open(fn, 'r') id = None name = None - for n in xrange(10): + for n in range(10): # data should be in first few lines line = fo.readline() if line.startswith('# Koji buildroot id:'): @@ -850,7 +855,7 @@ class TaskManager(object): # Note: we may still take an assigned task below #sort available capacities for each of our bins avail = {} - for bin in bins.iterkeys(): + for bin in six.iterkeys(bins): avail[bin] = [host['capacity'] - host['task_load'] for host in bin_hosts[bin]] avail[bin].sort() avail[bin].reverse() @@ -882,7 +887,7 @@ class TaskManager(object): #accept this task) bin_avail = avail.get(bin, [0]) self.logger.debug("available capacities for bin: %r" % bin_avail) - median = bin_avail[(len(bin_avail)-1)/2] + median = bin_avail[(len(bin_avail)-1)//2] self.logger.debug("ours: %.2f, median: %.2f" % (our_avail, median)) if not self.checkRelAvail(bin_avail, our_avail): #decline for now and give the upper half a chance @@ -900,7 +905,7 @@ class TaskManager(object): Check our available capacity against the capacity of other hosts in this bin. Return True if we should take a task, False otherwise. """ - median = bin_avail[(len(bin_avail)-1)/2] + median = bin_avail[(len(bin_avail)-1)//2] self.logger.debug("ours: %.2f, median: %.2f" % (avail, median)) if avail >= median: return True @@ -1093,7 +1098,7 @@ class TaskManager(object): raise IOError("No such directory: %s" % br_path) fs_stat = os.statvfs(br_path) available = fs_stat.f_bavail * fs_stat.f_bsize - availableMB = available / 1024 / 1024 + availableMB = available // 1024 // 1024 self.logger.debug("disk space available in '%s': %i MB", br_path, availableMB) if availableMB < self.options.minspace: self.status = "Insufficient disk space: %i MB, %i MB required" % (availableMB, self.options.minspace) diff --git a/koji/db.py b/koji/db.py index 9e92191..c4e9ab3 100644 --- a/koji/db.py +++ b/koji/db.py @@ -21,6 +21,7 @@ # Mike McLean +from __future__ import absolute_import import logging import sys import psycopg2 @@ -33,7 +34,7 @@ import psycopg2 # del psycopg2.extensions.string_types[1266] import time import traceback -import context +from . import context import re POSITIONAL_RE = re.compile(r'%[a-z]') diff --git a/koji/plugin.py b/koji/plugin.py index 396b811..5250e87 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -19,6 +19,7 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import import imp import koji import logging diff --git a/koji/policy.py b/koji/policy.py index 0842d84..3a7733e 100644 --- a/koji/policy.py +++ b/koji/policy.py @@ -17,10 +17,14 @@ # Authors: # Mike McLean +from __future__ import absolute_import import fnmatch import logging import koji +import six + +from koji.util import to_list class BaseSimpleTest(object): @@ -292,7 +296,7 @@ class SimpleRuleSet(object): index[name] = 1 index = {} _recurse(self.ruleset, index) - return index.keys() + return to_list(index.keys()) def _apply(self, rules, data, top=False): for tests, negate, action in rules: @@ -361,7 +365,7 @@ def findSimpleTests(namespace): namespace = (namespace,) ret = {} for ns in namespace: - for key, value in ns.iteritems(): + for key, value in six.iteritems(ns): if value is BaseSimpleTest: # skip this abstract base class if we encounter it # this module contains generic tests, so it is valid to include it diff --git a/koji/rpmdiff.py b/koji/rpmdiff.py index 2e18583..2ab6e81 100644 --- a/koji/rpmdiff.py +++ b/koji/rpmdiff.py @@ -18,11 +18,13 @@ # This library and program is heavily based on rpmdiff from the rpmlint package # It was modified to be used as standalone library for the Koji project. +from __future__ import absolute_import import hashlib import json import rpm import os import itertools +import six class Rpmdiff: @@ -113,8 +115,8 @@ class Rpmdiff: old_files_dict = self.__fileIteratorToDict(old.fiFromHeader()) new_files_dict = self.__fileIteratorToDict(new.fiFromHeader()) - files = list(set(itertools.chain(old_files_dict.iterkeys(), - new_files_dict.iterkeys()))) + files = list(set(itertools.chain(six.iterkeys(old_files_dict), + six.iterkeys(new_files_dict)))) files.sort() self.old_data['files'] = old_files_dict self.new_data['files'] = new_files_dict diff --git a/koji/tasks.py b/koji/tasks.py index bdc045b..84cc8d2 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -20,18 +20,20 @@ # Mike McLean # Mike Bonnet +from __future__ import absolute_import import koji import koji.plugin import koji.util import os import logging -import xmlrpclib +import six.moves.xmlrpc_client import signal import shutil import random import time import pprint import six.moves.urllib.request +from six.moves import range def scan_mounts(topdir): """Search path for mountpoints""" @@ -390,7 +392,7 @@ class BaseTaskHandler(object): try: self.session.getTaskResult(task) checked.add(task) - except (koji.GenericError, xmlrpclib.Fault): + except (koji.GenericError, six.moves.xmlrpc_client.Fault): self.logger.info("task %s failed or was canceled, cancelling unfinished tasks" % task) self.session.cancelTaskChildren(self.id) # reraise the original error now, rather than waiting for @@ -577,7 +579,7 @@ class SleepTask(BaseTaskHandler): class ForkTask(BaseTaskHandler): Methods = ['fork'] def handler(self, n=5, m=37): - for i in xrange(n): + for i in range(n): os.spawnvp(os.P_NOWAIT, 'sleep', ['sleep', str(m)]) class WaitTestTask(BaseTaskHandler): @@ -592,7 +594,7 @@ class WaitTestTask(BaseTaskHandler): _taskWeight = 0.1 def handler(self, count, seconds=10): tasks = [] - for i in xrange(count): + for i in range(count): task_id = self.subtask(method='sleep', arglist=[seconds], label=str(i)) tasks.append(task_id) bad_task = self.subtask('sleep', ['BAD_ARG'], label='bad') diff --git a/koji/util.py b/koji/util.py index eaf7813..a0d033c 100644 --- a/koji/util.py +++ b/koji/util.py @@ -19,6 +19,7 @@ # Mike Bonnet from __future__ import absolute_import +from __future__ import division import calendar import datetime from koji.xmlrpcplus import DateTime @@ -116,7 +117,7 @@ def checkForBuilds(session, tag, builds, event, latest=False): def duration(start): """Return the duration between start and now in MM:SS format""" elapsed = time.time() - start - mins = int(elapsed / 60) + mins = int(elapsed // 60) secs = int(elapsed % 60) return '%s:%02i' % (mins, secs) @@ -181,7 +182,7 @@ class DataWalker(object): if isinstance(value, tuple): value = tuple([self._walk(x) for x in value]) elif isinstance(value, list): - value = list([self._walk(x) for x in value]) + value = [self._walk(x) for x in value] elif isinstance(value, dict): ret = {} for k in value: @@ -729,3 +730,17 @@ def parse_maven_chain(confs, scratch=False): except ValueError: raise ValueError('No possible build order, missing/circular dependencies') return builds + +def to_list(l): + """ + Helper function for py2/py3 compatibility used e.g. in + list(dict.keys()) + + Don't use it for structures like list(zip(x, y)), where six.moves.zip is + used, so it is always an iterator. + """ + + if isinstance(l, list): + return l + else: + return list(l) diff --git a/koji/xmlrpcplus.py b/koji/xmlrpcplus.py index 9164cd7..06284b4 100644 --- a/koji/xmlrpcplus.py +++ b/koji/xmlrpcplus.py @@ -2,6 +2,7 @@ Custom xmlrpc handling for Koji """ +from __future__ import absolute_import import six import six.moves.xmlrpc_client as xmlrpc_client import types diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index cc7bdbb..f2f0bff 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -1,8 +1,9 @@ # kojid plugin +from __future__ import absolute_import import commands import koji -import ConfigParser +import six.moves.configparser import os import platform import re @@ -48,7 +49,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): return res def _read_config(self): - cp = ConfigParser.SafeConfigParser() + cp = six.moves.configparser.SafeConfigParser() cp.read(CONFIG_FILE) self.config = { 'default_mounts': [], @@ -89,7 +90,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): 'fstype': cp.get(section_name, 'fstype'), 'options': cp.get(section_name, 'options'), }) - except ConfigParser.NoOptionError: + except six.moves.configparser.NoOptionError: raise koji.GenericError("bad config: missing options in %s section" % section_name) for path in self.config['default_mounts'] + self.config['safe_roots'] + [x[0] for x in self.config['path_subs']]: diff --git a/plugins/builder/save_failed_tree.py b/plugins/builder/save_failed_tree.py index 4c202c9..7c579a7 100644 --- a/plugins/builder/save_failed_tree.py +++ b/plugins/builder/save_failed_tree.py @@ -1,8 +1,9 @@ +from __future__ import absolute_import import fnmatch import os import sys import tarfile -import ConfigParser +import six.moves.configparser import koji import koji.tasks as tasks @@ -27,7 +28,7 @@ def omit_paths3(tarinfo): def read_config(): global config - cp = ConfigParser.SafeConfigParser() + cp = six.moves.configparser.SafeConfigParser() cp.read(CONFIG_FILE) config = { 'path_filters': [], diff --git a/plugins/cli/runroot.py b/plugins/cli/runroot.py index 942e64c..9f0a463 100644 --- a/plugins/cli/runroot.py +++ b/plugins/cli/runroot.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import sys import time diff --git a/plugins/cli/save_failed_tree.py b/plugins/cli/save_failed_tree.py index 46b5fa3..5bf1a6d 100644 --- a/plugins/cli/save_failed_tree.py +++ b/plugins/cli/save_failed_tree.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import koji from koji.plugin import export_cli from koji_cli.lib import _, activate_session, OptionParser, watch_tasks diff --git a/plugins/hub/echo.py b/plugins/hub/echo.py index 6727d41..859e05c 100644 --- a/plugins/hub/echo.py +++ b/plugins/hub/echo.py @@ -5,10 +5,12 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import from koji.plugin import callbacks, callback, ignore_error +from koji.util import to_list import logging -@callback(*callbacks.keys()) +@callback(*to_list(callbacks.keys())) @ignore_error def echo(cbtype, *args, **kws): logging.getLogger('koji.plugin.echo').info('Called the %s callback, args: %s; kws: %s', diff --git a/plugins/hub/messagebus.py b/plugins/hub/messagebus.py index a45561c..615f77a 100644 --- a/plugins/hub/messagebus.py +++ b/plugins/hub/messagebus.py @@ -4,10 +4,11 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import from koji import PluginError from koji.context import context from koji.plugin import callbacks, callback, ignore_error, convert_datetime -import ConfigParser +import six.moves.configparser import logging import qpid.messaging import qpid.messaging.transports @@ -78,7 +79,7 @@ def get_config(): if config: return config - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) if not config.has_option('broker', 'timeout'): config.set('broker', 'timeout', '60') diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 85165ba..3de56fc 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -5,10 +5,11 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import import koji from koji.plugin import callback, ignore_error, convert_datetime from koji.context import context -import ConfigParser +import six.moves.configparser import logging import json import random @@ -269,7 +270,7 @@ def send_queued_msgs(cbtype, *args, **kws): log = logging.getLogger('koji.plugin.protonmsg') global CONFIG if not CONFIG: - conf = ConfigParser.SafeConfigParser() + conf = six.moves.configparser.SafeConfigParser() with open(CONFIG_FILE) as conffile: conf.readfp(conffile) CONFIG = conf diff --git a/plugins/hub/rpm2maven.py b/plugins/hub/rpm2maven.py index ad78b8f..717df54 100644 --- a/plugins/hub/rpm2maven.py +++ b/plugins/hub/rpm2maven.py @@ -5,11 +5,12 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import import koji from koji.context import context from koji.plugin import callback from koji.util import rmtree -import ConfigParser +import six.moves.configparser import fnmatch import os import shutil @@ -31,7 +32,7 @@ def maven_import(cbtype, *args, **kws): filepath = kws['filepath'] if not config: - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) name_patterns = config.get('patterns', 'rpm_names').split() for pattern in name_patterns: diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py index e6ee0b3..708aede 100644 --- a/plugins/hub/runroot_hub.py +++ b/plugins/hub/runroot_hub.py @@ -3,6 +3,7 @@ # plugin has a config file. This hub plugin has no config file. +from __future__ import absolute_import from koji.context import context from koji.plugin import export import koji diff --git a/plugins/hub/save_failed_tree.py b/plugins/hub/save_failed_tree.py index 5b34c2d..103bb95 100644 --- a/plugins/hub/save_failed_tree.py +++ b/plugins/hub/save_failed_tree.py @@ -1,5 +1,6 @@ +from __future__ import absolute_import import sys -import ConfigParser +import six.moves.configparser import koji from koji.context import context from koji.plugin import export @@ -28,7 +29,7 @@ def saveFailedTree(buildrootID, full=False, **opts): # read configuration only once if config is None: - config = ConfigParser.SafeConfigParser() + config = six.moves.configparser.SafeConfigParser() config.read(CONFIG_FILE) allowed_methods = config.get('permissions', 'allowed_methods').split() if len(allowed_methods) == 1 and allowed_methods[0] == '*': diff --git a/setup.py b/setup.py index d51a428..13f8f41 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import sys from setuptools import setup diff --git a/tests/test_cli/data/plugins/plugin1.py b/tests/test_cli/data/plugins/plugin1.py index 3e405f5..41d876e 100644 --- a/tests/test_cli/data/plugins/plugin1.py +++ b/tests/test_cli/data/plugins/plugin1.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from koji.plugin import export_cli, export_as @export_as('foobar') diff --git a/tests/test_cli/fakeclient.py b/tests/test_cli/fakeclient.py index 6c652c8..d0b07d0 100644 --- a/tests/test_cli/fakeclient.py +++ b/tests/test_cli/fakeclient.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import koji diff --git a/tests/test_cli/test_activate_session.py b/tests/test_cli/test_activate_session.py index a97e6f8..707d43d 100644 --- a/tests/test_cli/test_activate_session.py +++ b/tests/test_cli/test_activate_session.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import shutil import tempfile diff --git a/tests/test_cli/test_import_sig.py b/tests/test_cli/test_import_sig.py index 332a708..c29fa54 100644 --- a/tests/test_cli/test_import_sig.py +++ b/tests/test_cli/test_import_sig.py @@ -5,6 +5,7 @@ import hashlib import mock import random import six +from six.moves import range try: import unittest2 as unittest except ImportError: diff --git a/tests/test_cli/test_list_notifications.py b/tests/test_cli/test_list_notifications.py index ca173b9..53893d3 100644 --- a/tests/test_cli/test_list_notifications.py +++ b/tests/test_cli/test_list_notifications.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_cli/test_list_tasks.py b/tests/test_cli/test_list_tasks.py index 497a799..8daebc1 100644 --- a/tests/test_cli/test_list_tasks.py +++ b/tests/test_cli/test_list_tasks.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import six try: diff --git a/tests/test_cli/test_taskinfo.py b/tests/test_cli/test_taskinfo.py index 684365c..5c342be 100644 --- a/tests/test_cli/test_taskinfo.py +++ b/tests/test_cli/test_taskinfo.py @@ -487,7 +487,7 @@ Finished: Thu Jan 1 00:50:00 1970 } # need ordered dict to get same results - files = collections.OrderedDict(sorted(files.items(), + files = collections.OrderedDict(sorted(list(files.items()), key=lambda t: t[0])) list_task_output_mock.side_effect = [[], files, {}] diff --git a/tests/test_cli/utils.py b/tests/test_cli/utils.py index 2e243ed..e01639e 100644 --- a/tests/test_cli/utils.py +++ b/tests/test_cli/utils.py @@ -1,8 +1,10 @@ from __future__ import print_function +from __future__ import absolute_import import mock import os import six import sys +from six.moves import map try: import unittest2 as unittest except ImportError: diff --git a/tests/test_docs_version.py b/tests/test_docs_version.py index db8e617..e53c7f3 100644 --- a/tests/test_docs_version.py +++ b/tests/test_docs_version.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import subprocess try: diff --git a/tests/test_hub/test_add_btype.py b/tests/test_hub/test_add_btype.py index eef3d2d..64f19e1 100644 --- a/tests/test_hub/test_add_btype.py +++ b/tests/test_hub/test_add_btype.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: import unittest2 as unittest except ImportError: diff --git a/tests/test_hub/test_add_external_rpm.py b/tests/test_hub/test_add_external_rpm.py index 874aff1..a869256 100644 --- a/tests/test_hub/test_add_external_rpm.py +++ b/tests/test_hub/test_add_external_rpm.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_add_host.py b/tests/test_hub/test_add_host.py index 34c51b1..3746b2e 100644 --- a/tests/test_hub/test_add_host.py +++ b/tests/test_hub/test_add_host.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_add_host_to_channel.py b/tests/test_hub/test_add_host_to_channel.py index 3ca1036..68f2fc4 100644 --- a/tests/test_hub/test_add_host_to_channel.py +++ b/tests/test_hub/test_add_host_to_channel.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_apply_query_opts.py b/tests/test_hub/test_apply_query_opts.py index 8e78090..d8652d6 100644 --- a/tests/test_hub/test_apply_query_opts.py +++ b/tests/test_hub/test_apply_query_opts.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy try: import unittest2 as unittest diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index 8c82551..032658c 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import os import shutil diff --git a/tests/test_hub/test_check_volume_policy.py b/tests/test_hub/test_check_volume_policy.py index 8271cec..9531d28 100644 --- a/tests/test_hub/test_check_volume_policy.py +++ b/tests/test_hub/test_check_volume_policy.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_complete_image_build.py b/tests/test_hub/test_complete_image_build.py index 646032a..1bdde53 100644 --- a/tests/test_hub/test_complete_image_build.py +++ b/tests/test_hub/test_complete_image_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import json import mock diff --git a/tests/test_hub/test_complete_maven_build.py b/tests/test_hub/test_complete_maven_build.py index 2bd7419..42ba161 100644 --- a/tests/test_hub/test_complete_maven_build.py +++ b/tests/test_hub/test_complete_maven_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import json import mock diff --git a/tests/test_hub/test_create_tag.py b/tests/test_hub/test_create_tag.py index d0a3765..f483211 100644 --- a/tests/test_hub/test_create_tag.py +++ b/tests/test_hub/test_create_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_delete_build.py b/tests/test_hub/test_delete_build.py index c46207b..d4b79f8 100644 --- a/tests/test_hub/test_delete_build.py +++ b/tests/test_hub/test_delete_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_delete_tag.py b/tests/test_hub/test_delete_tag.py index 832cece..ef182d8 100644 --- a/tests/test_hub/test_delete_tag.py +++ b/tests/test_hub/test_delete_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index 272b3f5..e0637b1 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: import unittest2 as unittest except ImportError: diff --git a/tests/test_hub/test_edit_host.py b/tests/test_hub/test_edit_host.py index 8b81547..a4b61d3 100644 --- a/tests/test_hub/test_edit_host.py +++ b/tests/test_hub/test_edit_host.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_edit_tag.py b/tests/test_hub/test_edit_tag.py index e011267..cb32059 100644 --- a/tests/test_hub/test_edit_tag.py +++ b/tests/test_hub/test_edit_tag.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_active_repos.py b/tests/test_hub/test_get_active_repos.py index eb67a57..54db43d 100644 --- a/tests/test_hub/test_get_active_repos.py +++ b/tests/test_hub/test_get_active_repos.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_archive_file.py b/tests/test_hub/test_get_archive_file.py index b3a4b9b..cec5ef0 100644 --- a/tests/test_hub/test_get_archive_file.py +++ b/tests/test_hub/test_get_archive_file.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_build_logs.py b/tests/test_hub/test_get_build_logs.py index f0b8ebc..c03ff29 100644 --- a/tests/test_hub/test_get_build_logs.py +++ b/tests/test_hub/test_get_build_logs.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import os import shutil diff --git a/tests/test_hub/test_get_build_type.py b/tests/test_hub/test_get_build_type.py index 7cd03f1..325386c 100644 --- a/tests/test_hub/test_get_build_type.py +++ b/tests/test_hub/test_get_build_type.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_host.py b/tests/test_hub/test_get_host.py index f5ea774..420e7e2 100644 --- a/tests/test_hub/test_get_host.py +++ b/tests/test_hub/test_get_host.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_next_release.py b/tests/test_hub/test_get_next_release.py index 7b69dec..1578899 100644 --- a/tests/test_hub/test_get_next_release.py +++ b/tests/test_hub/test_get_next_release.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_upload_path.py b/tests/test_hub/test_get_upload_path.py index 9958f27..374b968 100644 --- a/tests/test_hub/test_get_upload_path.py +++ b/tests/test_hub/test_get_upload_path.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import mock import shutil diff --git a/tests/test_hub/test_get_user_perms.py b/tests/test_hub/test_get_user_perms.py index ffad733..81ff47d 100644 --- a/tests/test_hub/test_get_user_perms.py +++ b/tests/test_hub/test_get_user_perms.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_get_verify_class.py b/tests/test_hub/test_get_verify_class.py index 84adc82..721c992 100644 --- a/tests/test_hub/test_get_verify_class.py +++ b/tests/test_hub/test_get_verify_class.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: import unittest2 as unittest except ImportError: diff --git a/tests/test_hub/test_group_operations.py b/tests/test_hub/test_group_operations.py index 90b525e..aee0095 100644 --- a/tests/test_hub/test_group_operations.py +++ b/tests/test_hub/test_group_operations.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_import_build.py b/tests/test_hub/test_import_build.py index 783eea0..22f1186 100644 --- a/tests/test_hub/test_import_build.py +++ b/tests/test_hub/test_import_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_import_image_internal.py b/tests/test_hub/test_import_image_internal.py index 62ad108..2527d4d 100644 --- a/tests/test_hub/test_import_image_internal.py +++ b/tests/test_hub/test_import_image_internal.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import os import shutil diff --git a/tests/test_hub/test_insert_processor.py b/tests/test_hub/test_insert_processor.py index b661bf5..bd0a5ee 100644 --- a/tests/test_hub/test_insert_processor.py +++ b/tests/test_hub/test_insert_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_list_archive_files.py b/tests/test_hub/test_list_archive_files.py index 46d95d9..a8bcc63 100644 --- a/tests/test_hub/test_list_archive_files.py +++ b/tests/test_hub/test_list_archive_files.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_list_archives.py b/tests/test_hub/test_list_archives.py index 677755d..1f529ee 100644 --- a/tests/test_hub/test_list_archives.py +++ b/tests/test_hub/test_list_archives.py @@ -1,5 +1,6 @@ +from __future__ import absolute_import import mock -from utils import DBQueryTestCase +from .utils import DBQueryTestCase import koji import kojihub diff --git a/tests/test_hub/test_list_btypes.py b/tests/test_hub/test_list_btypes.py index 147444b..512a672 100644 --- a/tests/test_hub/test_list_btypes.py +++ b/tests/test_hub/test_list_btypes.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_list_channels.py b/tests/test_hub/test_list_channels.py index 3746b86..595e05a 100644 --- a/tests/test_hub/test_list_channels.py +++ b/tests/test_hub/test_list_channels.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_list_hosts.py b/tests/test_hub/test_list_hosts.py index e13e38f..60161e7 100644 --- a/tests/test_hub/test_list_hosts.py +++ b/tests/test_hub/test_list_hosts.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_list_task_output.py b/tests/test_hub/test_list_task_output.py index 7da39ea..4a5bcd3 100644 --- a/tests/test_hub/test_list_task_output.py +++ b/tests/test_hub/test_list_task_output.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_listing.py b/tests/test_hub/test_listing.py index 9587fd1..b622be0 100644 --- a/tests/test_hub/test_listing.py +++ b/tests/test_hub/test_listing.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_models/test_host.py b/tests/test_hub/test_models/test_host.py index e94ac0d..1a39a2e 100644 --- a/tests/test_hub/test_models/test_host.py +++ b/tests/test_hub/test_models/test_host.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_new_typed_build.py b/tests/test_hub/test_new_typed_build.py index f93f84e..a0e93ba 100644 --- a/tests/test_hub/test_new_typed_build.py +++ b/tests/test_hub/test_new_typed_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_notifications.py b/tests/test_hub/test_notifications.py index 269ef8a..0cea57c 100644 --- a/tests/test_hub/test_notifications.py +++ b/tests/test_hub/test_notifications.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_pkglist.py b/tests/test_hub/test_pkglist.py index 1dfe6d4..731abb6 100644 --- a/tests/test_hub/test_pkglist.py +++ b/tests/test_hub/test_pkglist.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_policy_tests.py b/tests/test_hub/test_policy_tests.py index 7d114fc..ad9e5c7 100644 --- a/tests/test_hub/test_policy_tests.py +++ b/tests/test_hub/test_policy_tests.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_query_processor.py b/tests/test_hub/test_query_processor.py index 8e1464f..b807d21 100644 --- a/tests/test_hub/test_query_processor.py +++ b/tests/test_hub/test_query_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_recycle_build.py b/tests/test_hub/test_recycle_build.py index 119814f..d5b81c4 100644 --- a/tests/test_hub/test_recycle_build.py +++ b/tests/test_hub/test_recycle_build.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import koji diff --git a/tests/test_hub/test_remove_host_from_channel.py b/tests/test_hub/test_remove_host_from_channel.py index cb3d111..db26349 100644 --- a/tests/test_hub/test_remove_host_from_channel.py +++ b/tests/test_hub/test_remove_host_from_channel.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_repos.py b/tests/test_hub/test_repos.py index 8bda536..670bcfa 100644 --- a/tests/test_hub/test_repos.py +++ b/tests/test_hub/test_repos.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_rpmdiff.py b/tests/test_hub/test_rpmdiff.py index fcc1280..c7d00f4 100644 --- a/tests/test_hub/test_rpmdiff.py +++ b/tests/test_hub/test_rpmdiff.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock try: diff --git a/tests/test_hub/test_savepoint.py b/tests/test_hub/test_savepoint.py index 279729e..a698721 100644 --- a/tests/test_hub/test_savepoint.py +++ b/tests/test_hub/test_savepoint.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import unittest diff --git a/tests/test_hub/test_set_host_enabled.py b/tests/test_hub/test_set_host_enabled.py index 698e451..b239080 100644 --- a/tests/test_hub/test_set_host_enabled.py +++ b/tests/test_hub/test_set_host_enabled.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_tag_operations.py b/tests/test_hub/test_tag_operations.py index fbd5cfb..e412389 100644 --- a/tests/test_hub/test_tag_operations.py +++ b/tests/test_hub/test_tag_operations.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import mock import shutil diff --git a/tests/test_hub/test_task_wait_results.py b/tests/test_hub/test_task_wait_results.py index a454038..0dc72da 100644 --- a/tests/test_hub/test_task_wait_results.py +++ b/tests/test_hub/test_task_wait_results.py @@ -1,5 +1,6 @@ +from __future__ import absolute_import import mock -import xmlrpclib +import six.moves.xmlrpc_client try: import unittest2 as unittest except ImportError: @@ -65,8 +66,8 @@ class TestTaskWaitResults(unittest.TestCase): task = self.getTask(t) task.getResult.return_value = "OK" task.isCanceled.return_value = False - self.tasks[6].getResult.side_effect = xmlrpclib.Fault(1, "error") - with self.assertRaises(xmlrpclib.Fault): + self.tasks[6].getResult.side_effect = six.moves.xmlrpc_client.Fault(1, "error") + with self.assertRaises(six.moves.xmlrpc_client.Fault): results = self.host_exports.taskWaitResults(parent, task_ids) self.assertEqual(results, []) self.tasks[6].getResult.side_effect = koji.GenericError('problem') diff --git a/tests/test_hub/test_update_processor.py b/tests/test_hub/test_update_processor.py index 83fcaa6..9537884 100644 --- a/tests/test_hub/test_update_processor.py +++ b/tests/test_hub/test_update_processor.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/test_user_groups.py b/tests/test_hub/test_user_groups.py index 0d06e15..42ad16c 100644 --- a/tests/test_hub/test_user_groups.py +++ b/tests/test_hub/test_user_groups.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_hub/utils.py b/tests/test_hub/utils.py index b1cf635..8a4fae9 100644 --- a/tests/test_hub/utils.py +++ b/tests/test_hub/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import six try: diff --git a/tests/test_kojira/test_repo_manager.py b/tests/test_kojira/test_repo_manager.py index e4c34cd..928a244 100644 --- a/tests/test_kojira/test_repo_manager.py +++ b/tests/test_kojira/test_repo_manager.py @@ -86,7 +86,7 @@ class RepoManagerTest(unittest.TestCase): self.mgr.checkTasks() # should have removed the close tasks - self.assertEqual(self.mgr.tasks.keys(), [101, 102]) + self.assertEqual(list(self.mgr.tasks.keys()), [101, 102]) @mock.patch('time.sleep') def test_regen_loop(self, sleep): diff --git a/tests/test_lib/test_argspec.py b/tests/test_lib/test_argspec.py index 29df33c..a565a64 100644 --- a/tests/test_lib/test_argspec.py +++ b/tests/test_lib/test_argspec.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import try: import unittest2 as unittest except ImportError: diff --git a/tests/test_lib/test_encode_datetime.py b/tests/test_lib/test_encode_datetime.py index 8d931df..e2a2909 100644 --- a/tests/test_lib/test_encode_datetime.py +++ b/tests/test_lib/test_encode_datetime.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import datetime try: import unittest2 as unittest diff --git a/tests/test_lib/test_gen_mock_config.py b/tests/test_lib/test_gen_mock_config.py index 35bae4f..b6d822a 100644 --- a/tests/test_lib/test_gen_mock_config.py +++ b/tests/test_lib/test_gen_mock_config.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import ast import os import os.path diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index 6c69cd5..425b048 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import import copy import datetime import mock +from six.moves import range try: import unittest2 as unittest except ImportError: diff --git a/tests/test_lib/test_utils.py b/tests/test_lib/test_utils.py index 23e1bbb..65c9377 100644 --- a/tests/test_lib/test_utils.py +++ b/tests/test_lib/test_utils.py @@ -7,6 +7,7 @@ import os import resource import six.moves.configparser import time +import six try: import unittest2 as unittest except ImportError: @@ -795,24 +796,24 @@ class MavenUtilTestCase(unittest.TestCase): name, release, date = 'fedora', 26, datetime.now().strftime('%Y%m%d') data = {'name': name, 'release': release, 'date': date} - six.assertCountEqual(self, data.items(), ldict.items()) - six.assertCountEqual(self, data.items(), [v for v in ldict.iteritems()]) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) + six.assertCountEqual(self, list(data.items()), [v for v in six.iteritems(ldict)]) name, release, date = 'rhel', 7, '20171012' - six.assertCountEqual(self, [name, release, date], ldict.values()) - six.assertCountEqual(self, [name, release, date], [v for v in ldict.itervalues()]) + six.assertCountEqual(self, [name, release, date], list(ldict.values())) + six.assertCountEqual(self, [name, release, date], [v for v in six.itervalues(ldict)]) data = {'name': name, 'release': release, 'date': date} self.assertEqual(name, ldict.pop('name')) data.pop('name') - six.assertCountEqual(self, data.items(), ldict.items()) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) (key, value) = ldict.popitem() data.pop(key) - six.assertCountEqual(self, data.items(), ldict.items()) + six.assertCountEqual(self, list(data.items()), list(ldict.items())) ldict_copy = ldict.copy() - six.assertCountEqual(self, data.items(), ldict_copy.items()) + six.assertCountEqual(self, list(data.items()), list(ldict_copy.items())) def test_LazyRecord(self): """Test LazyRecord object""" @@ -878,7 +879,7 @@ class MavenUtilTestCase(unittest.TestCase): actual = koji.util.eventFromOpts(session, opts) self.assertNotEqual(None, actual) - six.assertCountEqual(self, expect.items(), actual.items()) + six.assertCountEqual(self, list(expect.items()), list(actual.items())) # no event is matched case opts = mock.MagicMock(event=0, ts=0, repo=0) @@ -967,6 +968,19 @@ class MavenUtilTestCase(unittest.TestCase): self.assertNotEqual(copy.digest(), chksum.digest()) self.assertEqual(614401368, chksum.digest()) + def test_to_list(self): + l = [1, 2, 3] + + r = koji.util.to_list(l) + self.assertEqual(l, r) + + it = iter(l) + r = koji.util.to_list(it) + self.assertEqual(l, r) + + with self.assertRaises(TypeError): + koji.util.to_list(1) + class TestRmtree(unittest.TestCase): @patch('koji.util._rmtree') diff --git a/tests/test_lib/test_xmlrpcplus.py b/tests/test_lib/test_xmlrpcplus.py index 239900c..7ea5424 100644 --- a/tests/test_lib/test_xmlrpcplus.py +++ b/tests/test_lib/test_xmlrpcplus.py @@ -1,4 +1,6 @@ # coding=utf-8 +from __future__ import absolute_import +from six.moves import range try: import unittest2 as unittest except ImportError: diff --git a/tests/test_lib_py2only/test_restart_tasks.py b/tests/test_lib_py2only/test_restart_tasks.py index ac8d1e4..f43df03 100644 --- a/tests/test_lib_py2only/test_restart_tasks.py +++ b/tests/test_lib_py2only/test_restart_tasks.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import shutil import tempfile diff --git a/tests/test_lib_py2only/test_tasks.py b/tests/test_lib_py2only/test_tasks.py index e321d59..e196453 100644 --- a/tests/test_lib_py2only/test_tasks.py +++ b/tests/test_lib_py2only/test_tasks.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import random import shutil import six +from six.moves import range try: import unittest2 as unittest except ImportError: diff --git a/tests/test_plugins/load_plugin.py b/tests/test_plugins/load_plugin.py index dcc3f25..ac20795 100644 --- a/tests/test_plugins/load_plugin.py +++ b/tests/test_plugins/load_plugin.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os import sys diff --git a/tests/test_plugins/test_protonmsg.py b/tests/test_plugins/test_protonmsg.py index ed26f64..1414bee 100644 --- a/tests/test_plugins/test_protonmsg.py +++ b/tests/test_plugins/test_protonmsg.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import six import protonmsg import tempfile @@ -8,7 +9,7 @@ except ImportError: from mock import patch, MagicMock from koji.context import context -from ConfigParser import SafeConfigParser +from six.moves.configparser import SafeConfigParser class TestProtonMsg(unittest.TestCase): def tearDown(self): diff --git a/tests/test_plugins/test_runroot_builder.py b/tests/test_plugins/test_runroot_builder.py index 496a5ad..d200277 100644 --- a/tests/test_plugins/test_runroot_builder.py +++ b/tests/test_plugins/test_runroot_builder.py @@ -80,7 +80,7 @@ class FakeConfigParser(object): return def sections(self): - return self.CONFIG.keys() + return list(self.CONFIG.keys()) def has_option(self, section, key): return section in self.CONFIG and key in self.CONFIG[section] diff --git a/tests/test_plugins/test_runroot_hub.py b/tests/test_plugins/test_runroot_hub.py index 6c64f3c..637634e 100644 --- a/tests/test_plugins/test_runroot_hub.py +++ b/tests/test_plugins/test_runroot_hub.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock try: import unittest2 as unittest diff --git a/tests/test_plugins/test_save_failed_tree_cli.py b/tests/test_plugins/test_save_failed_tree_cli.py index ec6fb7f..452c7f8 100644 --- a/tests/test_plugins/test_save_failed_tree_cli.py +++ b/tests/test_plugins/test_save_failed_tree_cli.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import mock import six try: @@ -7,7 +8,7 @@ except ImportError: import koji -import load_plugin +from . import load_plugin save_failed_tree = load_plugin.load_plugin('cli', 'save_failed_tree') diff --git a/tests/test_scm.py b/tests/test_scm.py index 6eb3910..c97f057 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging import mock import shutil diff --git a/tests/test_www/test_util.py b/tests/test_www/test_util.py index 0ead026..8cc25b9 100644 --- a/tests/test_www/test_util.py +++ b/tests/test_www/test_util.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import unittest from kojiweb.util import formatMode diff --git a/util/koji-gc b/util/koji-gc index e128180..027aea7 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -6,14 +6,16 @@ # Authors: # Mike McLean +from __future__ import absolute_import +import six try: import krbV except ImportError: # pragma: no cover krbV = None import koji -from koji.util import LazyDict, LazyValue +from koji.util import LazyDict, LazyValue, to_list import koji.policy -import ConfigParser +import six.moves.configparser from email.MIMEText import MIMEText import fnmatch import optparse @@ -23,7 +25,7 @@ import smtplib import socket # for socket.error import sys import time -import xmlrpclib # for ProtocolError and Fault +import six.moves.xmlrpc_client # for ProtocolError and Fault OptionParser = optparse.OptionParser @@ -114,7 +116,7 @@ def get_options(): defaults = parser.get_default_values() - config = ConfigParser.ConfigParser() + config = six.moves.configparser.ConfigParser() cf = getattr(options, 'config_file', None) if cf: if not os.access(cf, os.F_OK): @@ -198,7 +200,7 @@ def get_options(): if len(parts) < 2: continue options.key_aliases[parts[0].upper()] = parts[1] - except ValueError, e: + except ValueError as e: print(e) parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases')) @@ -339,7 +341,7 @@ def warn(msg): def ensure_connection(session): try: ret = session.getAPIVersion() - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: error(_("Error: Unable to connect to server")) if ret != koji.API_VERSION: warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) @@ -373,9 +375,9 @@ def activate_session(session): session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) else: session.krb_login(proxyuser=options.runas) - except krbV.Krb5Error, e: + except krbV.Krb5Error as e: error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0])) - except socket.error, e: + except socket.error as e: warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1]) if not options.noauth and not session.logged_in: error(_("Error: unable to log in, no authentication methods available")) @@ -454,7 +456,7 @@ def handle_trash(): continue try: refs = session.buildReferences(binfo['id'], limit=10) - except xmlrpclib.Fault: + except six.moves.xmlrpc_client.Fault: print("[%i/%i] Error checking references for %s. Skipping" % (i, N, nvr)) continue #XXX - this is more data than we need @@ -535,7 +537,7 @@ def handle_trash(): by_owner = {} for binfo in to_trash: by_owner.setdefault(binfo['owner_name'], []).append(binfo) - owners = by_owner.keys() + owners = to_list(by_owner.keys()) owners.sort() for owner_name in owners: builds = [(b['nvr'], b) for b in by_owner[owner_name]] @@ -557,7 +559,7 @@ def handle_trash(): #best we can do currently owner = binfo['owner_id'] else: - owner = max([(n, k) for k, n in count.iteritems()])[1] + owner = max([(n, k) for k, n in six.iteritems(count)])[1] session.packageListAdd(trashcan_tag, binfo['name'], owner) session.tagBuildBypass(trashcan_tag, binfo['id'], force=True) @@ -656,7 +658,7 @@ def handle_delete(just_salvage=False): session.untagBuildBypass(trashcan_tag, binfo['id']) try: session.deleteBuild(binfo['id']) - except (xmlrpclib.Fault, koji.GenericError), e: + except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: print("Warning: deletion failed: %s" % e) #server issue pass @@ -735,7 +737,7 @@ def read_policies(fn=None): The expected format as follows test [params] [&& test [params] ...] :: (keep|untag|skip) """ - fo = file(fn, 'r') + fo = open(fn, 'r') tests = koji.policy.findSimpleTests(globals()) ret = koji.policy.SimpleRuleSet(fo, tests) fo.close() @@ -770,7 +772,7 @@ def get_build_sigs(build, cache=False): for sig in sigs: if sig['sigkey']: keys.setdefault(sig['sigkey'], 1) - ret = build_sig_cache[build] = keys.keys() + ret = build_sig_cache[build] = to_list(keys.keys()) return ret def handle_prune(): @@ -830,7 +832,7 @@ def handle_prune(): pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h) else: pkghist.setdefault(h['name'], []).append(h) - pkgs = pkghist.keys() + pkgs = to_list(pkghist.keys()) pkgs.sort() for pkg in pkgs: if not check_package(pkg): @@ -875,7 +877,7 @@ def handle_prune(): try: session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass) untagged.setdefault(nvr, {})[tagname] = 1 - except (xmlrpclib.Fault, koji.GenericError), e: + except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: print("Warning: untag operation failed: %s" % e) pass # if action == 'keep' do nothing @@ -909,7 +911,7 @@ def handle_prune(): print("Deleting untagged build: %s" % nvr) try: session.deleteBuild(build_id, strict=False) - except (xmlrpclib.Fault, koji.GenericError), e: + except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e: print("Warning: deletion failed: %s" % e) #server issue pass diff --git a/util/koji-shadow b/util/koji-shadow index 563f8b0..ec079f4 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -22,12 +22,16 @@ # Dennis Gilmore # Karsten Hopp +from __future__ import absolute_import +from six.moves import range +import six try: import krbV except ImportError: # pragma: no cover krbV = None import koji -import ConfigParser +from koji.util import to_list +import six.moves.configparser import fnmatch import optparse import os @@ -38,7 +42,7 @@ import string import sys import time import urllib2 -import xmlrpclib # for ProtocolError and Fault +import six.moves.xmlrpc_client # for ProtocolError and Fault import rpm # koji.fp.o keeps stalling, probably network errors... @@ -160,7 +164,7 @@ def get_options(): (options, args) = parser.parse_args() defaults = parser.get_default_values() - config = ConfigParser.ConfigParser() + config = six.moves.configparser.ConfigParser() cf = getattr(options, 'config_file', None) if cf: if not os.access(cf, os.F_OK): @@ -296,7 +300,7 @@ def warn(msg): def ensure_connection(session): try: ret = session.getAPIVersion() - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: error(_("Error: Unable to connect to server")) if ret != koji.API_VERSION: warn(_("WARNING: The server is at API version %d and the client is at " @@ -326,9 +330,9 @@ def activate_session(session): session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) else: session.krb_login(proxyuser=options.runas) - except krbV.Krb5Error, e: + except krbV.Krb5Error as e: error(_("Kerberos authentication failed: '%s' (%s)") % (e.args[1], e.args[0])) - except socket.error, e: + except socket.error as e: warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1]) if not options.noauth and not session.logged_in: error(_("Error: unable to log in")) @@ -448,7 +452,7 @@ class TrackedBuild(object): fsrc = urllib2.urlopen(url) fn = "%s/%s.src.rpm" % (options.workpath, self.nvr) koji.ensuredir(os.path.dirname(fn)) - fdst = file(fn, 'w') + fdst = open(fn, 'w') shutil.copyfileobj(fsrc, fdst) fsrc.close() fdst.close() @@ -495,7 +499,7 @@ class TrackedBuild(object): log("Warning: some rpms for %s lacked buildroots:" % self.nvr) for rinfo in bad: log(" %(name)s-%(version)s-%(release)s.%(arch)s" % rinfo) - return brs.keys() + return to_list(brs.keys()) def getDeps(self): buildroots = self.getBuildroots() @@ -544,7 +548,7 @@ class TrackedBuild(object): # changes happened during the build startup and some subtasks got the old # repo and others the new one. base = [] - for name, brlist in bases.iteritems(): + for name, brlist in six.iteritems(bases): #We want to determine for each name if that package was present #in /all/ the buildroots or just some. #Because brlist is constructed only from elements of buildroots, we @@ -554,12 +558,11 @@ class TrackedBuild(object): #each buildroot had this as a base package base.append(name) if len(tags) > 1: - log("Warning: found multiple buildroot tags for %s: %s" % (self.nvr, tags.keys())) - counts = [(n, tag) for tag, n in tags.iteritems()] - sort(counts) + log("Warning: found multiple buildroot tags for %s: %s" % (self.nvr, to_list(tags.keys()))) + counts = sorted([(n, tag) for tag, n in six.iteritems(tags)]) tag = counts[-1][1] else: - tag = tags.keys()[0] + tag = to_list(tags.keys())[0] # due bugs in used tools mainline koji instance could store empty buildroot infos for builds if len(builds) == 0: self.setState("noroot") @@ -659,9 +662,9 @@ class BuildTracker(object): return grey return default - def rpmvercmp(self, (e1, v1, r1), (e2, v2, r2)): + def rpmvercmp(self, nvr1, nvr2): """find out which build is newer""" - rc = rpm.labelCompare((e1, v1, r1), (e2, v2, r2)) + rc = rpm.labelCompare(nvr1, nvr2) if rc == 1: #first evr wins return 1 @@ -861,7 +864,7 @@ class BuildTracker(object): taginfo = remote.getTag(tag) builds = remote.listTagged(taginfo['id'], latest=True) for build in builds: - for retry in xrange(10): + for retry in range(10): try: self.scanBuild(build['id'], tag=tag) if options.first_one: @@ -880,13 +883,13 @@ class BuildTracker(object): if options.link_imports: #bit of a hack, but faster than uploading dst = "%s/%s/%s" % (koji.pathinfo.work(), serverdir, fn) - old_umask = os.umask(002) + old_umask = os.umask(0o02) try: koji.ensuredir(os.path.dirname(dst)) os.chown(os.path.dirname(dst), 48, 48) #XXX - hack log ("Downloading %s to %s" % (url, dst)) fsrc = urllib2.urlopen(url) - fdst = file(fn, 'w') + fdst = open(fn, 'w') shutil.copyfileobj(fsrc, fdst) fsrc.close() fdst.close() @@ -899,7 +902,7 @@ class BuildTracker(object): dst = "%s/%s" % (options.workpath, fn) log ("Downloading %s to %s..." % (url, dst)) fsrc = urllib2.urlopen(url) - fdst = file(dst, 'w') + fdst = open(dst, 'w') shutil.copyfileobj(fsrc, fdst) fsrc.close() fdst.close() @@ -1003,7 +1006,7 @@ class BuildTracker(object): for pkg in session.listPackages(pkgID=name): owners.setdefault(pkg['owner_id'], []).append(pkg) if owners: - order = [(len(v), k) for k, v in owners.iteritems()] + order = [(len(v), k) for k, v in six.iteritems(owners)] order.sort() owner = order[-1][1] else: @@ -1119,7 +1122,7 @@ class BuildTracker(object): log("-- %s --" % time.asctime()) self.report_brief() for state in ('broken', 'noroot', 'blocked'): - builds = self.state_idx[state].values() + builds = to_list(self.state_idx[state].values()) not_replaced = [b for b in builds if not b.substitute] n_replaced = len(builds) - len(not_replaced) log("%s: %i (+%i replaced)" % (state, len(not_replaced), n_replaced)) @@ -1149,7 +1152,7 @@ class BuildTracker(object): nvr = dep.substitute problem_counts.setdefault(nvr, 0) problem_counts[nvr] += 1 - order = [(c, nvr) for (nvr, c) in problem_counts.iteritems()] + order = [(c, nvr) for (nvr, c) in six.iteritems(problem_counts)] if order: order.sort() order.reverse() @@ -1160,8 +1163,7 @@ class BuildTracker(object): def report_brief(self): N = len(self.builds) - states = self.state_idx.keys() - states.sort() + states = sorted(self.state_idx.keys()) parts = ["%s: %i" % (s, len(self.state_idx[s])) for s in states] parts.append("total: %i" % N) log (' '.join(parts)) @@ -1232,7 +1234,7 @@ class BuildTracker(object): ret = False if options.max_jobs and len(self.state_idx['pending']) >= options.max_jobs: return ret - missing = [(b.order, b.id, b) for b in self.state_idx['missing'].itervalues()] + missing = [(b.order, b.id, b) for b in six.itervalues(self.state_idx['missing'])] missing.sort() for order, build_id, build in missing: if not self.checkBuildDeps(build): @@ -1300,7 +1302,7 @@ def main(args): if options.logfile: filename = options.logfile try: - logfile = os.open(filename,os.O_CREAT|os.O_RDWR|os.O_APPEND, 0777) + logfile = os.open(filename,os.O_CREAT|os.O_RDWR|os.O_APPEND, 0o777) except: logfile = None if logfile is not None: diff --git a/util/kojira b/util/kojira index 5245373..51ef84d 100755 --- a/util/kojira +++ b/util/kojira @@ -20,12 +20,14 @@ # Authors: # Mike McLean +from __future__ import absolute_import +from __future__ import division import sys import os import koji -from koji.util import rmtree, parseStatus +from koji.util import rmtree, parseStatus, to_list from optparse import OptionParser -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser import errno import logging import logging.handlers @@ -34,6 +36,7 @@ import signal import time import threading import traceback +import six @@ -83,7 +86,7 @@ class ManagedRepo(object): tags = {self.tag_id : 1} for x in order: tags[x['parent_id']] = 1 - self.taglist = tags.keys() + self.taglist = to_list(tags.keys()) @property def dist(self): @@ -152,7 +155,7 @@ class ManagedRepo(object): #also check dir age. We do this because a repo can be created from an older event #and should not be removed based solely on that event's timestamp. mtime = os.stat(path).st_mtime - except OSError, e: + except OSError as e: if e.errno == 2: # No such file or directory, so the repo either never existed, # or has already been deleted, so allow it to be marked deleted. @@ -216,9 +219,9 @@ class RepoManager(object): def printState(self): self.logger.debug('Tracking %i repos, %i child processes', len(self.repos), len(self.delete_pids)) - for tag_id, task_id in self.tasks.iteritems(): + for tag_id, task_id in six.iteritems(self.tasks): self.logger.debug("Tracking task %s for tag %s", task_id, tag_id) - for pid, desc in self.delete_pids.iteritems(): + for pid, desc in six.iteritems(self.delete_pids): self.logger.debug("Delete job %s: %r", pid, desc) def rmtree(self, path): @@ -244,7 +247,7 @@ class RepoManager(object): prefix = "pid %i (%s)" % (pid, self.delete_pids.get(pid)) try: (childpid, status) = os.waitpid(pid, os.WNOHANG) - except OSError, e: + except OSError as e: if e.errno != errno.ECHILD: #should not happen raise @@ -279,7 +282,7 @@ class RepoManager(object): for pid in self.delete_pids: try: os.kill(pid, sig) - except OSError, e: + except OSError as e: if e.errno != errno.ESRCH: logger.error("Unable to kill process %s", pid) @@ -312,7 +315,7 @@ class RepoManager(object): def checkCurrentRepos(self): """Determine which repos are current""" to_check = [] - repo_ids = self.repos.keys() + repo_ids = to_list(self.repos.keys()) for repo_id in repo_ids: repo = self.repos.get(repo_id) if repo is None: @@ -473,7 +476,7 @@ class RepoManager(object): if max_n == 0: # no recent use or missing data max_n = 1 - adj = stats['n_recent'] * 9.0 / max_n + 1 # 1.0 to 10.0 + adj = stats['n_recent'] * 9.0 // max_n + 1 # 1.0 to 10.0 ts = entry['expire_ts'] age = time.time() - ts # XXX - need to make sure our times aren't far off, otherwise this @@ -658,8 +661,7 @@ class RepoManager(object): debuginfo_pat = self.options.debuginfo_tags.split() src_pat = self.options.source_tags.split() - order = self.needed_tags.values() - order.sort(key=lambda t:t['score']) + order = sorted(self.needed_tags.values(), key=lambda t: t['score']) for tag in order: if running_tasks >= self.options.max_repo_tasks: self.logger.info("Maximum number of repo tasks reached") diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 271dcaa..94619a0 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -26,14 +26,15 @@ # kojiwind --install # in a cygwin shell. +from __future__ import absolute_import from optparse import OptionParser -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser import os import subprocess import sys import tempfile import time -import xmlrpclib +import six.moves.xmlrpc_client import base64 import hashlib import logging @@ -42,6 +43,7 @@ import threading import re import glob import zipfile +import six MANAGER_PORT = 7000 @@ -586,13 +588,13 @@ def get_mgmt_server(): macaddr, gateway = find_net_info() logger.debug('found MAC address %s, connecting to %s:%s', macaddr, gateway, MANAGER_PORT) - server = xmlrpclib.ServerProxy('http://%s:%s/' % + server = six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, MANAGER_PORT), allow_none=True) # we would set a timeout on the socket here, but that is apparently not # supported by python/cygwin/Windows task_port = server.getPort(macaddr) logger.debug('found task-specific port %s', task_port) - return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + return six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) def get_options(): """handle usage and parse options""" @@ -639,7 +641,7 @@ def stream_logs(server, handler, builds): logpath = os.path.join(build.source_dir, relpath) if logpath not in logs: logs[logpath] = (relpath, None) - for log, (relpath, fd) in logs.iteritems(): + for log, (relpath, fd) in six.iteritems(logs): if not fd: if os.path.isfile(log): try: diff --git a/vm/kojivmd b/vm/kojivmd index e50aad0..7e1dc7f 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -20,6 +20,8 @@ # Authors: # Mike Bonnet +from __future__ import absolute_import +from __future__ import division import koji import koji.util from koji.daemon import SCM, TaskManager @@ -28,7 +30,7 @@ from koji.tasks import RestartTask, RestartVerifyTask import sys import logging import os -import xmlrpclib +import six.moves.xmlrpc_client import signal import time import subprocess @@ -36,13 +38,13 @@ import libvirt import libxml2 import random import socket -import SimpleXMLRPCServer +import six.moves.xmlrpc_server import threading import base64 import pwd import requests import fnmatch -from ConfigParser import ConfigParser +from six.moves.configparser import ConfigParser from contextlib import closing from optparse import OptionParser try: @@ -245,14 +247,14 @@ def main(options, session): # Tasks for handling VM lifecycle #################### -class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): +class DaemonXMLRPCServer(six.moves.xmlrpc_server.SimpleXMLRPCServer): allow_reuse_address = True def __init__(self, addr, port): if sys.version_info[:2] <= (2, 4): - SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False) + six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False) else: - SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False, + six.moves.xmlrpc_server.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False, allow_none=True) self.logger = logging.getLogger('koji.vm.DaemonXMLRPCServer') self.socket.settimeout(5) @@ -260,7 +262,7 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): def server_close(self): self.active = False - SimpleXMLRPCServer.SimpleXMLRPCServer.server_close(self) + six.moves.xmlrpc_server.SimpleXMLRPCServer.server_close(self) def handle_while_active(self): while self.active: @@ -281,20 +283,20 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): # Copy and paste from SimpleXMLRPCServer, with the addition of passing # allow_none=True to xmlrpclib.dumps() def _marshaled_dispatch(self, data, dispatch_method = None): - params, method = xmlrpclib.loads(data) + params, method = six.moves.xmlrpc_client.loads(data) try: if dispatch_method is not None: response = dispatch_method(method, params) else: response = self._dispatch(method, params) response = (response,) - response = xmlrpclib.dumps(response, methodresponse=1, allow_none=True) - except xmlrpclib.Fault, fault: - response = xmlrpclib.dumps(fault) + response = six.moves.xmlrpc_client.dumps(response, methodresponse=1, allow_none=True) + except six.moves.xmlrpc_client.Fault as fault: + response = six.moves.xmlrpc_client.dumps(fault) except: # report exception back to server - response = xmlrpclib.dumps( - xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) + response = six.moves.xmlrpc_client.dumps( + six.moves.xmlrpc_client.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) ) return response @@ -314,7 +316,7 @@ class WinBuildTask(MultiPlatformTask): strict=False) # winspec and patches options are urls # verify the urls before passing them to the VM - for url in [source_url] + subopts.values(): + for url in [source_url] + koji.util.to_list(subopts.values()): scm = SCM(url) scm.assert_allowed(self.options.allowed_scms) @@ -416,7 +418,7 @@ class VMExecTask(BaseTaskHandler): def __init__(self, *args, **kw): super(VMExecTask, self).__init__(*args, **kw) - self.task_manager = xmlrpclib.ServerProxy('http://%s:%s/' % (self.options.privaddr, self.options.portbase), + self.task_manager = six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (self.options.privaddr, self.options.portbase), allow_none=True) self.port = None self.server = None @@ -688,7 +690,7 @@ class VMExecTask(BaseTaskHandler): f.close() - return file(localpath, 'r') + return open(localpath, 'r') def getFile(self, buildinfo, archiveinfo, offset, length, type): """ @@ -717,7 +719,7 @@ class VMExecTask(BaseTaskHandler): if offset == 0: if os.path.exists(local_path): raise koji.BuildError('cannot overwrite %s' % local_path) - fobj = file(local_path, 'w') + fobj = open(local_path, 'w') else: if not os.path.isfile(local_path): raise koji.BuildError('% does not exist' % local_path) @@ -725,7 +727,7 @@ class VMExecTask(BaseTaskHandler): if offset != size: raise koji.BuildError('cannot write to %s at offset %s, size is %s' % \ (local_path, offset, size)) - fobj = file(local_path, 'r+') + fobj = open(local_path, 'r+') fobj.seek(offset) data = base64.b64decode(contents) fobj.write(data) @@ -755,7 +757,7 @@ class VMExecTask(BaseTaskHandler): else: raise koji.BuildError('unsupported checksum algorithm: %s' % algo) - fobj = file(local_path, 'r') + fobj = open(local_path, 'r') while True: data = fobj.read(1048576) if not data: @@ -828,7 +830,7 @@ class VMExecTask(BaseTaskHandler): self.setupTaskServer() vm.create() self.logger.info('Started VM %s', clone_name) - except libvirt.libvirtError, e: + except libvirt.libvirtError as e: self.logger.error('error starting VM %s', clone_name, exc_info=True) raise koji.PreBuildError('error starting VM %s, error was: %s' % \ (clone_name, e)) @@ -844,7 +846,7 @@ class VMExecTask(BaseTaskHandler): if self.success is None: # task is still running # make sure it hasn't exceeded the timeout - mins = (time.time() - start) / 60 + mins = (time.time() - start) // 60 if mins > timeout: vm.destroy() self.server.server_close() @@ -929,7 +931,7 @@ class VMTaskManager(TaskManager): raise IOError('No such directory: %s' % self.options.imagedir) fs_stat = os.statvfs(self.options.imagedir) available = fs_stat.f_bavail * fs_stat.f_bsize - availableMB = available / 1024 / 1024 + availableMB = available // 1024 // 1024 self.logger.debug('disk space available in %s: %i MB', self.options.imagedir, availableMB) if availableMB < self.options.minspace: self.status = 'Insufficient disk space: %i MB, %i MB required' % (availableMB, self.options.minspace) @@ -938,7 +940,7 @@ class VMTaskManager(TaskManager): return True def checkMem(self): - phys_mem = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE') / 1024 + phys_mem = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE') // 1024 vm_mem = 0 for vm_id in self.libvirt_conn.listDomainsID(): vm = self.libvirt_conn.lookupByID(vm_id) @@ -1087,9 +1089,9 @@ if __name__ == "__main__": # authenticate using SSL client certificates session.ssl_login(options.cert, None, options.serverca) - except koji.AuthError, e: + except koji.AuthError as e: quit("Error: Unable to log in: %s" % e) - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) elif options.user: try: @@ -1097,7 +1099,7 @@ if __name__ == "__main__": session.login() except koji.AuthError: quit("Error: Unable to log in. Bad credentials?") - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) elif krbV: krb_principal = options.krb_principal @@ -1107,9 +1109,9 @@ if __name__ == "__main__": session.krb_login(principal=krb_principal, keytab=options.keytab, ccache=options.ccache) - except krbV.Krb5Error, e: + except krbV.Krb5Error as e: quit("Kerberos authentication failed: '%s' (%s)" % (e.args[1], e.args[0])) - except socket.error, e: + except socket.error as e: quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1]) else: quit("No username/password supplied and Kerberos missing or not configured") @@ -1123,7 +1125,7 @@ if __name__ == "__main__": #make sure it works try: ret = session.echo("OK") - except xmlrpclib.ProtocolError: + except six.moves.xmlrpc_client.ProtocolError: quit("Error: Unable to connect to server %s" % (options.server)) if ret != ["OK"]: quit("Error: incorrect server response: %r" % (ret)) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 6b6659e..fb6c4a3 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -20,22 +20,27 @@ # Mike Bonnet # Mike McLean +from __future__ import absolute_import +from __future__ import division import os import os.path import re import sys import mimetypes -import Cookie +import six.moves.http_cookies import datetime import logging import time import koji import kojiweb.util +from koji.util import to_list from koji.server import ServerRedirect from kojiweb.util import _initValues from kojiweb.util import _genHTML from kojiweb.util import _getValidTokens from koji.util import sha1_constructor +from six.moves import range +import six # Convenience definition of a commonly-used sort function _sortbyname = kojiweb.util.sortByKeyFunc('name') @@ -53,7 +58,7 @@ def _setUserCookie(environ, user): shasum = sha1_constructor(value) shasum.update(options['Secret'].value) value = "%s:%s" % (shasum.hexdigest(), value) - cookies = Cookie.SimpleCookie() + cookies = six.moves.http_cookies.SimpleCookie() cookies['user'] = value c = cookies['user'] #morsel instance c['secure'] = True @@ -66,7 +71,7 @@ def _setUserCookie(environ, user): environ['koji.headers'].append(['Cache-Control', 'no-cache="set-cookie"']) def _clearUserCookie(environ): - cookies = Cookie.SimpleCookie() + cookies = six.moves.http_cookies.SimpleCookie() cookies['user'] = '' c = cookies['user'] #morsel instance c['path'] = os.path.dirname(environ['SCRIPT_NAME']) @@ -76,7 +81,7 @@ def _clearUserCookie(environ): def _getUserCookie(environ): options = environ['koji.options'] - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) + cookies = six.moves.http_cookies.SimpleCookie(environ.get('HTTP_COOKIE', '')) if 'user' not in cookies: return None value = cookies['user'].value @@ -681,7 +686,7 @@ def taskinfo(environ, taskID): values['pathinfo'] = pathinfo paths = [] # (volume, relpath) tuples - for relname, volumes in server.listTaskOutput(task['id'], all_volumes=True).iteritems(): + for relname, volumes in six.iteritems(server.listTaskOutput(task['id'], all_volumes=True)): paths += [(volume, relname) for volume in volumes] values['output'] = sorted(paths, key = _sortByExtAndName) if environ['koji.currentUser']: @@ -700,8 +705,8 @@ def taskstatus(environ, taskID): return '' files = server.listTaskOutput(taskID, stat=True, all_volumes=True) output = '%i:%s\n' % (task['id'], koji.TASK_STATES[task['state']]) - for filename, volumes_data in files.iteritems(): - for volume, file_stats in volumes_data.iteritems(): + for filename, volumes_data in six.iteritems(files): + for volume, file_stats in six.iteritems(volumes_data): output += '%s:%s:%s\n' % (volume, filename, file_stats['st_size']) return output @@ -809,7 +814,7 @@ def tags(environ, start=None, order=None, childID=None): return _genHTML(environ, 'tags.chtml') -_PREFIX_CHARS = [chr(char) for char in range(48, 58) + range(97, 123)] +_PREFIX_CHARS = [chr(char) for char in list(range(48, 58)) + list(range(97, 123))] def packages(environ, tagID=None, userID=None, order='package_name', start=None, prefix=None, inherited='1'): values = _initValues(environ, 'Packages', 'packages') @@ -2120,7 +2125,7 @@ def buildsbytarget(environ, days='7', start=None, order='-builds'): if builds > maxBuilds: maxBuilds = builds - kojiweb.util.paginateList(values, targets.values(), start, 'targets', 'target', order) + kojiweb.util.paginateList(values, to_list(targets.values()), start, 'targets', 'target', order) values['order'] = order diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 778d992..df17a1d 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -19,6 +19,7 @@ # Authors: # Mike McLean +from __future__ import absolute_import import cgi import inspect import koji @@ -29,9 +30,10 @@ import pprint import sys import traceback -from ConfigParser import RawConfigParser +from six.moves.configparser import RawConfigParser from koji.server import ServerError, ServerRedirect -from koji.util import dslice +from koji.util import dslice, to_list +import six class URLNotFound(ServerError): @@ -396,14 +398,14 @@ class Dispatcher(object): else: # last one wins headers[key] = (name, value) - if isinstance(result, basestring): + if isinstance(result, six.string_types): headers.setdefault('content-length', ('Content-Length', str(len(result)))) headers.setdefault('content-type', ('Content-Type', 'text/html')) - headers = headers.values() + extra + headers = to_list(headers.values()) + extra self.logger.debug("Headers:") self.logger.debug(koji.util.LazyString(pprint.pformat, [headers])) start_response(status, headers) - if isinstance(result, basestring): + if isinstance(result, six.string_types): result = [result] return result diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index adea48b..c32f7f9 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -20,6 +20,8 @@ # Mike Bonnet # Mike McLean +from __future__ import absolute_import +from __future__ import division import Cheetah.Template import datetime import koji @@ -29,9 +31,10 @@ import stat #a bunch of exception classes that explainError needs from socket import error as socket_error from socket import sslerror as socket_sslerror -from xmlrpclib import ProtocolError +from six.moves.xmlrpc_client import ProtocolError from xml.parsers.expat import ExpatError import cgi +from six.moves import range class NoSuchException(Exception): pass @@ -149,7 +152,7 @@ def _genHTML(environ, fileName): def _truncTime(): now = datetime.datetime.now() # truncate to the nearest 15 minutes - return now.replace(minute=(now.minute / 15 * 15), second=0, microsecond=0) + return now.replace(minute=(now.minute // 15 * 15), second=0, microsecond=0) def _genToken(environ, tstamp=None): if 'koji.currentLogin' in environ and environ['koji.currentLogin']: @@ -354,9 +357,9 @@ def _populateValues(values, dataName, prefix, data, totalRows, start, count, pag values[(prefix and prefix + 'Count' or 'count')] = count values[(prefix and prefix + 'Range' or 'range')] = pageSize values[(prefix and prefix + 'Order' or 'order')] = order - currentPage = start / pageSize + currentPage = start // pageSize values[(prefix and prefix + 'CurrentPage' or 'currentPage')] = currentPage - totalPages = totalRows / pageSize + totalPages = totalRows // pageSize if totalRows % pageSize > 0: totalPages += 1 pages = [page for page in range(0, totalPages) if (abs(page - currentPage) < 100 or ((page + 1) % 100 == 0))]