PR#20: Outstanding work from koji-devel - koji

koji

#20 Outstanding work from koji-devel

Merged 10 years ago by mikeb. Opened 10 years ago by mikem.

https://github.com/mikem23/koji-playground.git master into master

cli: if given an rpm name without --rpm, download the containing build

Mike McLean • 10 years ago

709558e

cli: handle rpms in download-build

Mike McLean • 10 years ago

393dba2

add-pkg: use multicall and ignore existing packages

Mike McLean • 10 years ago

4365699

wsdl archivetype

Mike McLean • 10 years ago

a621081

avoid traceback when cli fails to parse task parameters

Mike McLean • 10 years ago

a6c218d

avoid masking exceptions in retry code

Mike McLean • 10 years ago

33cf1ab

handle Unexpected EOF exceptions on reads

Mike McLean • 10 years ago

7e56b13

clarify error message for noarch rpmdiff mismatches

Mike McLean • 10 years ago

de92c54

add rc archivetype

Mike McLean • 10 years ago

8168da1

cli/koji

file modified

+45 -14

		`@@ -766,19 +766,23 @@`
		`print "No such tag: %s" % tag`
		`sys.exit(1)`
		`pkglist = dict([(p['package_name'], p['package_id']) for p in session.listPackages(tagID=dsttag['id'])])`
		`- ret = 0`
		`+ to_add = []`
		`for package in args[1:]:`
		`package_id = pkglist.get(package, None)`
		`if not package_id is None:`
		`print "Package %s already exists in tag %s" % (package, tag)`
		`- ret = 1`
		`- if ret:`
		`- return ret`
		`+ continue`
		`+ to_add.append(package)`
		`if options.extra_arches:`
		`opts['extra_arches'] = ' '.join(options.extra_arches.replace(',',' ').split())`
		`- for package in args[1:]:`
		`- #really should implement multicall...`
		`- session.packageListAdd(tag,package,options.owner,**opts)`
		`+`
		`+ # add the packages`
		`+ print "Adding %i packages to tag %s" % (len(to_add), dsttag['name'])`
		`+ session.multicall = True`
		`+ for package in to_add:`
		`+ session.packageListAdd(tag, package, options.owner, **opts)`
		`+ session.multiCall(strict=True)`
		`+`

		`def handle_block_pkg(options, session, args):`
		`"[admin] Block a package in the listing for tag"`
		`@@ -4224,7 +4228,18 @@`
		`lines.append("%sOptions:" % prefix)`
		`_handleMap(lines, opts, prefix)`

		`+`
		`def _parseTaskParams(session, method, task_id):`
		`+ try:`
		`+ return _do_parseTaskParams(session, method, task_id)`
		`+ except Exception:`
		`+ if logger.isEnabledFor(logging.DEBUG):`
		`+ tb_str = ''.join(traceback.format_exception(*sys.exc_info()))`
		`+ logger.debug(tb_str)`
		`+ return ['Unable to parse task parameters']`
		`+`
		`+`
		`+ def _do_parseTaskParams(session, method, task_id):`
		`"""Parse the return of getTaskRequest()"""`
		`params = session.getTaskRequest(task_id)`

		`@@ -4301,11 +4316,10 @@`
		`if len(params) > 2:`
		`_handleOpts(lines, params[2])`
		`elif method in ('createLiveCD', 'createAppliance'):`
		`- lines.append("Arch: %s" % params[0])`
		`- lines.append("Build Target: %s" % params[1])`
		`- lines.append("Kickstart File: %s" % params[2])`
		`- if len(params) > 3:`
		`- _handleOpts(lines, params[3])`
		`+ lines.append("Arch: %s" % params[3])`
		`+ lines.append("Kickstart File: %s" % params[7])`
		`+ if len(params) > 8:`
		`+ _handleOpts(lines, params[8])`
		`elif method == 'newRepo':`
		`tag = session.getTag(params[0])`
		`lines.append("Tag: %s" % tag['name'])`
		`@@ -6192,12 +6206,13 @@`
		`usage = _("usage: %prog download-build [options] <n-v-r \| build_id \| package>")`
		`usage += _("\n(Specify the --help global option for a list of other help options)")`
		`parser = OptionParser(usage=usage)`
		`- parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[],`
		`+ parser.add_option("--arch", "-a", dest="arches", metavar="ARCH", action="append", default=[],`
		`help=_("Only download packages for this arch (may be used multiple times)"))`
		`parser.add_option("--type", help=_("Download archives of the given type, rather than rpms (maven, win, or image)"))`
		`parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag"))`
		`parser.add_option("--debuginfo", action="store_true", help=_("Also download -debuginfo rpms"))`
		`parser.add_option("--task-id", action="store_true", help=_("Interperet id as a task id"))`
		`+ parser.add_option("--rpm", action="store_true", help=_("Download the given rpm"))`
		`parser.add_option("--key", help=_("Download rpms signed with the given key"))`
		`parser.add_option("--topurl", metavar="URL", default=options.topurl,`
		`help=_("URL under which Koji files are accessible"))`
		`@@ -6237,7 +6252,20 @@`
		`print "%s has no builds of %s" % (suboptions.latestfrom, build)`
		`return 1`
		`info = builds[0]`
		`+ elif suboptions.rpm:`
		`+ rpminfo = session.getRPM(build)`
		`+ if rpminfo is None:`
		`+ print "No such rpm: %s" % build`
		`+ return 1`
		`+ info = session.getBuild(rpminfo['build_id'])`
		`else:`
		`+ # if we're given an rpm name without --rpm, download the containing build`
		`+ try:`
		`+ nvra = koji.parse_NVRA(build)`
		`+ rpminfo = session.getRPM(build)`
		`+ build = rpminfo['build_id']`
		`+ except Exception:`
		`+ pass`
		`info = session.getBuild(build)`

		`if info is None:`
		`@@ -6278,7 +6306,10 @@`
		`arches = suboptions.arches`
		`if len(arches) == 0:`
		`arches = None`
		`- rpms = session.listRPMs(buildID=info['id'], arches=arches)`
		`+ if suboptions.rpm:`
		`+ rpms = [rpminfo]`
		`+ else:`
		`+ rpms = session.listRPMs(buildID=info['id'], arches=arches)`
		`if not rpms:`
		`if arches:`
		`print "No %s packages available for %s" % (" or ".join(arches), koji.buildLabel(info))`

docs/schema.sql

file modified

		`@@ -821,6 +821,8 @@`
		`insert into archivetypes (name, description, extensions) values ('groovy', 'Groovy script file', 'groovy gvy');`
		`insert into archivetypes (name, description, extensions) values ('batch', 'Batch file', 'bat');`
		`insert into archivetypes (name, description, extensions) values ('shell', 'Shell script', 'sh');`
		`+ insert into archivetypes (name, description, extensions) values ('rc', 'Resource file', 'rc');`
		`+ insert into archivetypes (name, description, extensions) values ('wsdl', 'Web Services Description Language', 'wsdl');`


		`-- Do we want to enforce a constraint that a build can only generate one`

hub/kojihub.py

file modified

+3 -2

		`@@ -7752,8 +7752,9 @@`
		`status = proc.wait()`
		`if os.WIFSIGNALED(status) or \`
		`(os.WEXITSTATUS(status) != 0):`
		`- raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \`
		`- (os.path.basename(first_rpm), output)`
		`+ raise koji.BuildError(`
		`+ 'The following noarch package built differently on different architectures: %s\n'`
		`+ 'rpmdiff output was:\n%s' % (os.path.basename(first_rpm), output))`

		`def importImageInternal(task_id, build_id, imgdata):`
		`"""`

koji/__init__.py

file modified

+5 -29

		`@@ -58,7 +58,6 @@`
		`import xml.sax`
		`import xml.sax.handler`
		`from xmlrpclib import loads, dumps, Fault`
		`- import OpenSSL`
		`import zipfile`

		`def _(args):`
		`@@ -1959,34 +1958,11 @@`
		`raise`
		`except Exception, e:`
		`self._close_connection()`
		`- if isinstance(e, OpenSSL.SSL.Error):`
		`- # pyOpenSSL doesn't use different exception`
		`- # subclasses, we have to actually parse the args`
		`- for arg in e.args:`
		`- # First, check to see if 'arg' is iterable because`
		`- # it can be anything..`
		`- try:`
		`- iter(arg)`
		`- except TypeError:`
		`- continue`
		`-`
		`- # We do all this so that we can detect cert expiry`
		`- # so we can avoid retrying those over and over.`
		`- for items in arg:`
		`- try:`
		`- iter(items)`
		`- except TypeError:`
		`- continue`
		`-`
		`- if len(items) != 3:`
		`- continue`
		`-`
		`- _, _, ssl_reason = items`
		`-`
		`- if ('certificate revoked' in ssl_reason or`
		`- 'certificate expired' in ssl_reason):`
		`- # There's no point in retrying for this`
		`- raise`
		`+`
		`+ if ssl.SSLCommon.is_cert_error(e):`
		`+ # There's no point in retrying for this`
		`+ raise`
		`+`
		`if not self.logged_in:`
		`#in the past, non-logged-in sessions did not retry. For compatibility purposes`
		`#this behavior is governed by the anon_retry opt.`

koji/ssl/SSLCommon.py

file modified

+37

		`@@ -29,6 +29,43 @@`
		`return preverifyOK`


		`+ def is_cert_error(e):`
		`+ """Determine if an OpenSSL error is due to a bad cert"""`
		`+`
		`+ if not isinstance(e, SSL.Error):`
		`+ return False`
		`+`
		`+ # pyOpenSSL doesn't use different exception`
		`+ # subclasses, we have to actually parse the args`
		`+ for arg in e.args:`
		`+ # First, check to see if 'arg' is iterable because`
		`+ # it can be anything..`
		`+ try:`
		`+ iter(arg)`
		`+ except TypeError:`
		`+ continue`
		`+`
		`+ # We do all this so that we can detect cert expiry`
		`+ # so we can avoid retrying those over and over.`
		`+ for items in arg:`
		`+ try:`
		`+ iter(items)`
		`+ except TypeError:`
		`+ continue`
		`+`
		`+ if len(items) != 3:`
		`+ continue`
		`+`
		`+ _, _, ssl_reason = items`
		`+`
		`+ if ('certificate revoked' in ssl_reason or`
		`+ 'certificate expired' in ssl_reason):`
		`+ return True`
		`+`
		`+ #otherwise`
		`+ return False`
		`+`
		`+`
		`def CreateSSLContext(certs):`
		`key_and_cert = certs['key_and_cert']`
		`peer_ca_cert = certs['peer_ca_cert']`

koji/ssl/SSLConnection.py

file modified

		`@@ -142,6 +142,10 @@`
		`return None`
		`except SSL.WantReadError:`
		`time.sleep(0.2)`
		`+ except SSL.SysCallError, e:`
		`+ if e.args == (-1, 'Unexpected EOF'):`
		`+ break`
		`+ raise`
		`return None`

		`class PlgFileObject(socket._fileobject):`