#1117 python3 kojid
Merged 5 months ago by mikem. Opened 9 months ago by tkopecek.
tkopecek/koji issue1116  into  master

file modified
+1 -1

@@ -81,7 +81,7 @@ 

  	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/. coverage3 run \

  	    --rcfile .coveragerc3 --source . \

  	    /usr/bin/nosetests \

- 	    tests/test_lib tests/test_cli

+ 	    tests/test_lib tests/test_cli tests/test_builder

  	coverage3 report --rcfile .coveragerc3

  	coverage3 html --rcfile .coveragerc3

  	@echo Full coverage report at file://${PWD}/htmlcov/index.html

file modified
+313 -167

@@ -23,64 +23,81 @@ 

  

  from __future__ import absolute_import

  from __future__ import division

- import six

- try:

-     import krbV

- except ImportError:  # pragma: no cover

-     krbV = None

- import koji

- import koji.plugin

- import koji.rpmdiff

- import koji.util

- import koji.tasks

+ 

+ import Cheetah.Template

+ import copy

  import glob

+ import grp

  import json

  import logging

  import logging.handlers

- from koji.daemon import incremental_upload, log_output, TaskManager, SCM

- from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask

- from koji.util import parseStatus, isSuccess, dslice, dslice_ex, to_list

- import multilib.multilib as multilib

  import os

  import pwd

- import grp

  import random

  import re

  import rpm

- import rpmUtils.arch

  import shutil

  import signal

+ import six

+ import six.moves.xmlrpc_client

  import smtplib

  import socket

  import sys

  import time

  import traceback

  import xml.dom.minidom

- import six.moves.xmlrpc_client

  import zipfile

- import copy

- import Cheetah.Template

  from six.moves.configparser import ConfigParser

  from fnmatch import fnmatch

  from gzip import GzipFile

  from optparse import OptionParser, SUPPRESS_HELP

- from yum import repoMDObject

- import yum.packages

- import yum.Errors

  

- #imports for LiveCD, LiveMedia, and Appliance handler

- image_enabled = False

+ from multilib import multilib

+ import koji

+ import koji.arch

+ import koji.plugin

+ import koji.rpmdiff

+ import koji.util

+ import koji.tasks

+ from koji.daemon import incremental_upload, log_output, TaskManager, SCM

+ from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask

+ from koji.util import parseStatus, isSuccess, dslice, dslice_ex, to_list

+ 

+ try:

+     import krbV

+ except ImportError:  # pragma: no cover

+     krbV = None

+ 

+ try:

+     import librepo

+     import io

+ except ImportError:

+     librepo = None

+ 

+ try:

+     import dnf

+ except ImportError:

+     dnf = None

+ 

+ try:

+     # yum

+     from yum import repoMDObject

+     import yum.packages

+     import yum.Errors

+     yum_available = True

+ except ImportError:

+     yum_available = False

+ 

+ # imports for LiveCD, LiveMedia, and Appliance handler

  try:

      import pykickstart.parser as ksparser

      import pykickstart.handlers.control as kscontrol

      import pykickstart.errors as kserrors

-     import hashlib

      import iso9660 # from pycdio

      image_enabled = True

  except ImportError:  # pragma: no cover

-     pass

+     image_enabled = False

  

- ozif_enabled = False

  try:

      from imgfac.BuildDispatcher import BuildDispatcher

      from imgfac.Builder import Builder

@@ -96,7 +113,7 @@ 

      from imgfac.FactoryUtils import qemu_convert_cmd

      ozif_enabled = True

  except ImportError:  # pragma: no cover

-     pass

+     ozif_enabled = False

  

  def main(options, session):

      logger = logging.getLogger("koji.build")

@@ -260,9 +277,8 @@ 

          output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)

  

          #write config

-         fo = open(configfile,'w')

-         fo.write(output)

-         fo.close()

+         with open(configfile,'w') as fo:

+             fo.write(output)

  

      def _repositoryEntries(self, pi, plugin=False):

          entries = []

@@ -360,9 +376,8 @@ 

  </settings>

  """

          settings = settings % locals()

-         fo = open(self.rootdir() + destfile, 'w')

-         fo.write(settings)

-         fo.close()

+         with open(self.rootdir() + destfile, 'w') as fo:

+             fo.write(settings)

  

      def mock(self, args):

          """Run mock"""

@@ -605,7 +620,7 @@ 

          try:

              ts = rpm.TransactionSet()

              for h in ts.dbMatch():

-                 pkg = koji.get_header_fields(h,fields)

+                 pkg = koji.get_header_fields(h, fields)

                  #skip our fake packages

                  if pkg['name'] in ['buildsys-build', 'gpg-pubkey']:

                      #XXX config

@@ -703,35 +718,59 @@ 

              ext_url = erepo['url'].replace('$arch', self.br_arch)

              erepo_idx[ext_url] = erepo

          pathinfo = koji.PathInfo(topdir='')

-         #XXX - cheap hack to get relative paths

-         repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name'])

-         repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml')

  

+         repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name'])

          opts = dict([(k, getattr(self.options, k)) for k in ('topurl','topdir')])

          opts['tempdir'] = self.options.workdir

-         fo = koji.openRemoteFile(repomdpath, **opts)

-         try:

-             repodata = repoMDObject.RepoMD('ourrepo', fo)

-         except:

-             raise koji.BuildError("Unable to parse repomd.xml file for %s" % os.path.join(repodir, self.br_arch))

-         data  = repodata.getData('origin')

-         pkgorigins  = data.location[1]

+ 

+         # prefer librepo

+         if librepo is not None:

+             repo_url = os.path.join(repodir, self.br_arch)

+             # repo_url can start with '/', don't use os.path.join

+             if self.options.topurl:

+                 repo_url = '%s/%s' % (self.options.topurl, repo_url)

+             elif self.options.topdir:

+                 repo_url = '%s/%s' % (self.options.topdir, repo_url)

+             logging.error(repo_url)

+             tmpdir = os.path.join(self.options.workdir, 'librepo-markExternalRPMs')

+             koji.ensuredir(tmpdir)

+             h = librepo.Handle()

+             r = librepo.Result()

+             h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)

+             h.setopt(librepo.LRO_URLS, [repo_url])

+             h.setopt(librepo.LRO_DESTDIR, tmpdir)

+             h.perform(r)

+             pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href']

+             koji.util.rmtree(tmpdir)

+         elif yum_available:

+             #XXX - cheap hack to get relative paths

+             repomdpath = os.path.join(repodir, self.br_arch, 'repodata', 'repomd.xml')

+             with koji.openRemoteFile(repomdpath, **opts) as fo:

+                 try:

+                     repodata = repoMDObject.RepoMD('ourrepo', fo)

+                 except:

+                     raise koji.BuildError("Unable to parse repomd.xml file for %s" % os.path.join(repodir, self.br_arch))

+             data  = repodata.getData('origin')

+             pkgorigins  = data.location[1]

+         else:

+             # shouldn't occur

+             raise koji.GenericError("install librepo or yum")

  

          relpath = os.path.join(repodir, self.br_arch, pkgorigins)

-         fo = koji.openRemoteFile(relpath, **opts)

-         #at this point we know there were external repos at the create event,

-         #so there should be an origins file.

-         origin_idx = {}

-         fo2 = GzipFile(fileobj=fo, mode='r')

-         for line in fo2:

-             parts=line.split(None, 2)

-             if len(parts) < 2:

-                 continue

-             #first field is formated by yum as [e:]n-v-r.a

-             nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" %  koji.parse_NVRA(parts[0])

-             origin_idx[nvra] = parts[1]

-         fo2.close()

-         fo.close()

+         with koji.openRemoteFile(relpath, **opts) as fo:

+             #at this point we know there were external repos at the create event,

+             #so there should be an origins file.

+             origin_idx = {}

+             with GzipFile(fileobj=fo, mode='r') as fo2:

+                 if six.PY3:

+                     fo2 = io.TextIOWrapper(fo2, encoding='utf-8')

+                 for line in fo2:

+                     parts=line.split(None, 2)

+                     if len(parts) < 2:

+                         continue

+                     #first field is formated by yum as [e:]n-v-r.a

+                     nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" %  koji.parse_NVRA(parts[0])

+                     origin_idx[nvra] = parts[1]

          # mergerepo starts from a local repo in the task workdir, so internal

          # rpms have an odd-looking origin that we need to look for

          localtail = '/repo_%s_premerge/' % self.repo_info['id']

@@ -922,7 +961,7 @@ 

              self.event_id = self.session.getLastEvent()['id']

          srpm = self.getSRPM(src, build_tag, repo_info['id'])

          h = self.readSRPMHeader(srpm)

-         data = koji.get_header_fields(h,['name','version','release','epoch'])

+         data = koji.get_header_fields(h, ['name','version','release','epoch'])

          data['task_id'] = self.id

          if getattr(self, 'source', False):

              data['source'] = self.source['source']

@@ -1006,10 +1045,9 @@ 

          relpath = "work/%s" % srpm

          opts = dict([(k, getattr(self.options, k)) for k in ('topurl','topdir')])

          opts['tempdir'] = self.workdir

-         fo = koji.openRemoteFile(relpath, **opts)

-         h = koji.get_rpm_header(fo)

-         fo.close()

-         if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:

+         with koji.openRemoteFile(relpath, **opts) as fo:

+             h = koji.get_rpm_header(fo)

+         if not koji.get_header_field(h, 'sourcepackage'):

              raise koji.BuildError("%s is not a source package" % srpm)

          return h

  

@@ -1028,9 +1066,9 @@ 

          archlist = arches.split()

          self.logger.debug('base archlist: %r' % archlist)

          # - adjust arch list based on srpm macros

-         buildarchs = h[rpm.RPMTAG_BUILDARCHS]

-         exclusivearch = h[rpm.RPMTAG_EXCLUSIVEARCH]

-         excludearch = h[rpm.RPMTAG_EXCLUDEARCH]

+         buildarchs = koji.get_header_field(h, 'buildarchs')

+         exclusivearch = koji.get_header_field(h, 'exclusivearch')

+         excludearch = koji.get_header_field(h, 'excludearch')

          if buildarchs:

              archlist = buildarchs

              self.logger.debug('archlist after buildarchs: %r' % archlist)

@@ -1071,8 +1109,8 @@ 

          # see https://pagure.io/koji/issue/19

  

          h = self.readSRPMHeader(srpm)

-         exclusivearch = h[rpm.RPMTAG_EXCLUSIVEARCH]

-         excludearch = h[rpm.RPMTAG_EXCLUDEARCH]

+         exclusivearch = koji.get_header_field(h, 'exclusivearch')

+         excludearch = koji.get_header_field(h, 'excludearch')

  

          if exclusivearch or excludearch:

              # if one of the tag arches is filtered out, then we can't use a

@@ -1216,13 +1254,13 @@ 

          return self.checkHostArch(tag, hostdata)

  

      def srpm_sanity_checks(self, filename):

-         header = koji.get_rpm_header(filename)

+         h_fields = koji.get_header_fields(filename, ['packager', 'vendor', 'distribution'])

  

-         if not header[rpm.RPMTAG_PACKAGER]:

+         if not h_fields['packager']:

              raise koji.BuildError("The build system failed to set the packager tag")

-         if not header[rpm.RPMTAG_VENDOR]:

+         if not h_fields['vendor']:

              raise koji.BuildError("The build system failed to set the vendor tag")

-         if not header[rpm.RPMTAG_DISTRIBUTION]:

+         if not h_fields['distribution']:

              raise koji.BuildError("The build system failed to set the distribution tag")

  

      def handler(self, pkg, root, arch, keep_srpm, opts=None):

@@ -1243,15 +1281,12 @@ 

              raise koji.BuildError("SRPM file missing: %s" % fn)

          # peel E:N-V-R from package

          h = koji.get_rpm_header(fn)

-         name = h[rpm.RPMTAG_NAME]

-         ver = h[rpm.RPMTAG_VERSION]

-         rel = h[rpm.RPMTAG_RELEASE]

-         epoch = h[rpm.RPMTAG_EPOCH]

-         if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:

+         name = koji.get_header_field(h, 'name')

+         if not koji.get_header_field(h, 'sourcepackage'):

              raise koji.BuildError("not a source package")

          # Disable checking for distribution in the initial SRPM because it

          # might have been built outside of the build system

-         # if not h[rpm.RPMTAG_DISTRIBUTION]:

+         # if not koji.get_header_field(h, 'distribution'):

          #    raise koji.BuildError, "the distribution tag is not set in the original srpm"

  

          self.updateWeight(name)

@@ -1870,9 +1905,8 @@ 

          contents = contents.encode('utf-8')

  

          specfile = spec_template[:-5]

-         specfd = open(specfile, 'w')

-         specfd.write(contents)

-         specfd.close()

+         with open(specfile, 'w') as specfd:

+             specfd.write(contents)

  

          # Run spec file sanity checks.  Any failures will throw a BuildError

          self.spec_sanity_checks(specfile)

@@ -1959,8 +1993,8 @@ 

              raise koji.BuildError('no rpms found')

  

          try:

-             for rpm in [srpm] + rpms:

-                 self.uploadFile(os.path.join(resultdir, rpm))

+             for rpm_fn in [srpm] + rpms:

+                 self.uploadFile(os.path.join(resultdir, rpm_fn))

          except (SystemExit, ServerExit, KeyboardInterrupt):

              raise

          except:

@@ -2228,8 +2262,8 @@ 

          task = self.session.getTaskInfo(self.id)

          user_id = task['owner']

          try:

-             build = self.session.getBuild(build_id, strict=True)

-             tag = self.session.getTag(tag_id, strict=True)

+             self.session.getBuild(build_id, strict=True)

+             self.session.getTag(tag_id, strict=True)

  

              #several basic sanity checks have already been run (and will be run

              #again when we make the final call). Our job is to perform the more

@@ -2337,7 +2371,6 @@ 

                          ignored_arches.add(arch)

  

              # wrap in an RPM if asked

-             rpm_results = None

              spec_url = opts.get('specfile')

              for arch in arches:

                  # get around an xmlrpc limitation, use arches for keys instead

@@ -2354,7 +2387,7 @@ 

              # 1 results hash from the subtasks

              if 'kickstart' in opts:

                  saw_ks = False

-                 for arch in results.keys():

+                 for arch in results:

                      if arch in ignored_arches:

                          continue

                      ks = os.path.basename(opts.get('kickstart'))

@@ -2442,7 +2475,6 @@ 

              self.logger.info('results: %s' % results)

  

              # wrap in an RPM if asked

-             rpm_results = None

              spec_url = opts.get('specfile')

              if spec_url:

                  results[create_task_id]['rpmresults'] = self.buildWrapperRPM(

@@ -2528,7 +2560,6 @@ 

  

              # wrap in an RPM if needed

              spec_url = opts.get('specfile')

-             rpm_results = None

              if spec_url:

                  results[create_task_id]['rpmresults'] = self.buildWrapperRPM(

                      spec_url, create_task_id,

@@ -2860,9 +2891,8 @@ 

          kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' %

                                (target_info['build_tag_name'], self.id))

          koji.ensuredir(broot.tmpdir())

-         outfile = open(kskoji, 'w')

-         outfile.write(str(self.ks.handler))

-         outfile.close()

+         with open(kskoji, 'w') as outfile:

+             outfile.write(str(self.ks.handler))

  

          # put the new ksfile in the output directory

          if not os.path.exists(kskoji):

@@ -3444,11 +3474,10 @@ 

          else:

              tops = dict([(k, getattr(self.options, k)) for k in ('topurl','topdir')])

              tops['tempdir'] = self.workdir

-             ks_src = koji.openRemoteFile(ksfile, **tops)

-             kspath = os.path.join(self.workdir, os.path.basename(ksfile))

-             ks_dest = open(kspath, 'w')

-             ks_dest.write(ks_src.read())

-             ks_dest.close()

+             with koji.openRemoteFile(ksfile, **tops) as ks_src:

+                 kspath = os.path.join(self.workdir, os.path.basename(ksfile))

+                 with open(kspath, 'w') as ks_dest:

+                     ks_dest.write(ks_src.read())

          self.logger.debug('uploading kickstart from here: %s' % kspath)

          self.uploadFile(kspath) # upload the original ks file

          return kspath # absolute path to the ks file

@@ -3534,9 +3563,8 @@ 

              an absolute path to the kickstart file we wrote

          """

          kspath = os.path.join(self.workdir, ksname)

-         outfile = open(kspath, 'w')

-         outfile.write(str(ksobj.handler))

-         outfile.close()

+         with open(kspath, 'w') as outfile:

+             outfile.write(str(ksobj.handler))

  

          # put the new ksfile in the output directory

          if not os.path.exists(kspath):

@@ -3668,9 +3696,8 @@ 

          edriver = newxml.getElementsByTagName('driver')[0]

          edriver.setAttribute('type', format)

          xml_path = os.path.join(self.workdir, filename)

-         xmlfd = open(xml_path, 'w')

-         xmlfd.write(newxml.toprettyxml())

-         xmlfd.close()

+         with open(xml_path, 'w') as xmlfd:

+             xmlfd.write(newxml.toprettyxml())

          return xml_path

  

      def getScreenshot(self):

@@ -3719,7 +3746,7 @@ 

          if len(formats) == 0:

              # we only want a raw disk image (no format option given)

              f_dict['raw'] = True

-         elif 'raw' not in f_dict.keys():

+         elif 'raw' not in f_dict:

              f_dict['raw'] = False

          self.logger.debug('Image delivery plan: %s' % f_dict)

          return f_dict

@@ -3785,7 +3812,7 @@ 

              that points to the path of the XML file for that image

          """

          imginfo = {}

-         for fmt in images.keys():

+         for fmt in images:

              imginfo[fmt] = images[fmt]

              lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch),

                  self.base_img.base_image.parameters['libvirt_xml'])

@@ -4108,9 +4135,8 @@ 

          ApplicationConfiguration(configuration=config)

  

          tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)

-         tdl = open(tdl_path, 'w')

-         tdl.write(template)

-         tdl.close()

+         with open(tdl_path, 'w') as tdl:

+             tdl.write(template)

          self.uploadFile(tdl_path)

  

          # ImageFactory picks a port to the guest VM using a rolling integer.

@@ -4141,8 +4167,8 @@ 

          }

          # record the RPMs that were installed

          if not opts.get('scratch'):

-             fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',

-                 'payloadhash', 'buildtime')

+             #fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',

+             #    'payloadhash', 'buildtime')

              icicle = xml.dom.minidom.parseString(images['raw']['icicle'])

              self.logger.debug('ICICLE: %s' % images['raw']['icicle'])

              for p in icicle.getElementsByTagName('extra'):

@@ -4170,7 +4196,7 @@ 

              br.markExternalRPMs(imgdata['rpmlist'])

  

          # upload the results

-         for format in (f for f in self.formats.keys() if self.formats[f]):

+         for format in (f for f in self.formats if self.formats[f]):

              newimg = images[format]['image']

              if ('ova' in format or format in ('raw-xz', 'liveimg-squashfs', 'tar-gz')):

                  newname = self.imgname + '.' + format.replace('-', '.')

@@ -4257,11 +4283,10 @@ 

          else:

              tops = dict([(k, getattr(self.options, k)) for k in ('topurl','topdir')])

              tops['tempdir'] = self.workdir

-             remote_fileobj = koji.openRemoteFile(filepath, **tops)

              final_path = os.path.join(self.workdir, os.path.basename(filepath))

-             final_fileobj = open(final_path, 'w')

-             final_fileobj.write(remote_fileobj.read())

-             final_fileobj.close()

+             with koji.openRemoteFile(filepath, **tops) as remote_fileobj:

+                 with open(final_path, 'w') as final_fileobj:

+                     shutil.copyfileobj(remote_fileobj, final_fileobj)

          self.logger.debug('uploading retrieved file from here: %s' % final_path)

          self.uploadFile(final_path) # upload the original ks file

          return final_path # absolute path to the ks file

@@ -4386,8 +4411,6 @@ 

  

          # TODO: Copy-paste from BaseImage - refactor

          target_info = self.session.getBuildTarget(opts['target'], strict=True)

-         build_tag = target_info['build_tag']

-         repo_info = self.getRepo(build_tag)

  

          name = opts['name']

          version = opts['version']

@@ -4471,8 +4494,8 @@ 

                      pim = PersistentImageManager.default_manager()

                      pim.add_image(target_image)

                      target.target_image = target_image

-                     open(target_image.data, "w").write("Mock build from task ID: %s" %

-                                                  (str(self.id)))

+                     with open(target_image.data, "w") as f:

+                         f.write("Mock build from task ID: %s" % self.id)

                      target_image.status='COMPLETE'

                  else:

                      target = bd.builder_for_target_image('indirection',

@@ -4641,9 +4664,9 @@ 

  

          # check srpm name

          h = koji.get_rpm_header(srpm)

-         name = h[rpm.RPMTAG_NAME]

-         version = h[rpm.RPMTAG_VERSION]

-         release = h[rpm.RPMTAG_RELEASE]

+         name = koji.get_header_field(h, 'name')

+         version = koji.get_header_field(h, 'version')

+         release = koji.get_header_field(h, 'release')

          srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()

          if srpm_name != os.path.basename(srpm):

              raise koji.BuildError('srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm)))

@@ -5045,9 +5068,8 @@ 

          if external_repos:

              self.merge_repos(external_repos, arch, groupdata)

          elif pkglist is None:

-             fo = open(os.path.join(self.datadir, "EMPTY_REPO"), 'w')

-             fo.write("This repo is empty because its tag has no content for this arch\n")

-             fo.close()

+             with open(os.path.join(self.datadir, "EMPTY_REPO"), 'w') as fo:

+                 fo.write("This repo is empty because its tag has no content for this arch\n")

  

          uploadpath = self.getUploadDir()

          files = []

@@ -5169,7 +5191,7 @@ 

              canonArches.add(koji.canonArch(arch))

          arch32s = set()

          for arch in canonArches:

-             if not rpmUtils.arch.isMultiLibArch(arch):

+             if not koji.arch.isMultiLibArch(arch):

                  arch32s.add(arch)

          for arch in arch32s:

              # we do 32-bit multilib arches first so the 64-bit ones can

@@ -5265,8 +5287,11 @@ 

          # sort out our package list(s)

          self.uploadpath = self.getUploadDir()

          self.get_rpms(tag, arch, keys, opts)

-         if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):

-             self.do_multilib(arch, self.archmap[arch], opts['multilib'])

+         if opts['multilib'] and koji.arch.isMultiLibArch(arch):

+             if dnf is not None:

+                 self.do_multilib_dnf(arch, self.archmap[arch], opts['multilib'])

+             else:

+                 self.do_multilib_yum(arch, self.archmap[arch], opts['multilib'])

          self.split_pkgs(opts)

          self.write_kojipkgs()

          self.write_pkglist()

@@ -5359,7 +5384,135 @@ 

              raise koji.GenericError('failed to create repo: %s' \

                      % parseStatus(status, ' '.join(cmd)))

  

-     def do_multilib(self, arch, ml_arch, conf):

+ 

+     def do_multilib_dnf(self, arch, ml_arch, conf):

+         repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])

+         mldir = os.path.join(repodir, koji.canonArch(ml_arch))

+         ml_true = set()  # multilib packages we need to include before depsolve

+         ml_conf = os.path.join(koji.pathinfo.work(), conf)

+ 

+         # read pkgs data from multilib repo

+         ml_pkgfile = os.path.join(mldir, 'kojipkgs')

+         ml_pkgs = json.load(open(ml_pkgfile, 'r'))

+ 

+         # step 1: figure out which packages are multilib (should already exist)

+         dnfbase = dnf.Base()

+         mlm = multilib.DevelMultilibMethod(ml_conf)

+         fs_missing = set()

+         for bnp in self.kojipkgs:

+                 rpminfo = self.kojipkgs[bnp]

+                 ppath = rpminfo['_pkgpath']

+                 dnfbase.fill_sack(load_system_repo=False, load_available_repos=False)

+                 po = dnfbase.sack.add_cmdline_package(ppath)

+                 if mlm.select(po):

+                     # we need a multilib package to be included

+                     ml_bnp = bnp.replace(arch, self.archmap[arch])

+                     ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)

+                     # ^ XXX - should actually generate this

+                     if ml_bnp not in ml_pkgs:

+                         # not in our multilib repo

+                         self.logger.error('%s (multilib) is not on the filesystem' % ml_path)

+                         fs_missing.add(ml_path)

+                         # we defer failure so can report all the missing deps

+                         continue

+                     ml_true.add(ml_path)

+ 

+         # step 2: set up architectures for dnf configuration

+         self.logger.info("Resolving multilib for %s using method devel" % arch)

+         dnfdir = os.path.join(self.workdir, 'dnf')

+         # TODO: unwind this arch mess

+         archlist = (arch, 'noarch')

+         transaction_arch = arch

+         archlist = archlist + self.compat[self.biarch[arch]]

+         best_compat = self.compat[self.biarch[arch]][0]

+         if koji.arch.archDifference(best_compat, arch) > 0:

+             transaction_arch = best_compat

+         dnfconfig = """

+ [main]

+ debuglevel=2

+ #pkgpolicy=newest

+ #exactarch=1

+ gpgcheck=0

+ #reposdir=/dev/null

+ #cachedir=/dnfcache

+ installroot=%s

+ #logfile=/dnf.log

+ 

+ [koji-%s]

+ name=koji multilib task

+ baseurl=file://%s

+ enabled=1

+ 

+ """ % (dnfdir, self.id, mldir)

+         os.makedirs(os.path.join(dnfdir, "dnfcache"))

+         os.makedirs(os.path.join(dnfdir, 'var/lib/rpm'))

+ 

+         # step 3: proceed with dnf config and set up

+         yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch)

+         with open(yconfig_path, 'w') as f:

+             f.write(dnfconfig)

+         self.session.uploadWrapper(yconfig_path, self.uploadpath,

+             os.path.basename(yconfig_path))

+         conf = dnf.conf.Conf()

+         conf.reposdir = [] # don't use system repos at all

+         conf.read(yconfig_path)

+         dnfbase = dnf.Base(conf)

+         if hasattr(koji.arch, 'ArchStorage'):

+             dnfbase.conf.arch = transaction_arch

+         else:

+             koji.arch.canonArch = transaction_arch

+         dnfbase.read_all_repos()

+         dnfbase.fill_sack(load_system_repo=False, load_available_repos=True)

+         for pkg in ml_true:

+             dnfbase.install(pkg)

+ 

+         # step 4: execute dnf transaction to get dependencies

+         self.logger.info("Resolving dependencies for arch %s" % arch)

+ 

+         ml_needed = {}

+         try:

+             dnfbase.resolve()

+             self.logger.info('dnf depsolve successfully finished')

+             for po in dnfbase.transaction.install_set:

+                 bnp = os.path.basename(po.localPkg())

+                 dep_path = os.path.join(mldir, bnp[0].lower(), bnp)

+                 ml_needed[dep_path] = po

+                 if not os.path.exists(dep_path):

+                     self.logger.error('%s (multilib dep) not on filesystem' % dep_path)

+                     fs_missing.add(dep_path)

+         except dnf.exceptions.DepsolveError:

+             self.logger.error('dnf depsolve was unsuccessful')

+             raise

+ 

+         if len(fs_missing) > 0:

+             missing_log = os.path.join(self.workdir, 'missing_multilib.log')

+             with open(missing_log, 'w') as outfile:

+                 outfile.write('The following multilib files were missing:\n')

+                 for ml_path in fs_missing:

+                     outfile.write(ml_path + '\n')

+             self.session.uploadWrapper(missing_log, self.uploadpath)

+             raise koji.GenericError('multilib packages missing. '

+                     'See missing_multilib.log')

+ 

+         # step 5: update kojipkgs

+         for dep_path in ml_needed:

+             tspkg = ml_needed[dep_path]

+             bnp = os.path.basename(dep_path)

+             if bnp in self.kojipkgs:

+                 # we expect duplication with noarch, but not other arches

+                 if tspkg.arch != 'noarch':

+                     self.logger.warning("Multilib duplicate: %s", bnp)

+                 continue

+             rpminfo = ml_pkgs[bnp].copy()

+             # fix _pkgpath, which comes from another task and could be wrong

+             # for us

+             # TODO: would be better if we could use the proper path here

+             rpminfo['_pkgpath'] = dep_path

+             rpminfo['_multilib'] = True

+             self.kojipkgs[bnp] = rpminfo

+ 

+ 

+     def do_multilib_yum(self, arch, ml_arch, conf):

          repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])

          mldir = os.path.join(repodir, koji.canonArch(ml_arch))

          ml_true = set()  # multilib packages we need to include before depsolve

@@ -5399,12 +5552,12 @@ 

          transaction_arch = arch

          archlist = archlist + self.compat[self.biarch[arch]]

          best_compat = self.compat[self.biarch[arch]][0]

-         if rpmUtils.arch.archDifference(best_compat, arch) > 0:

+         if koji.arch.archDifference(best_compat, arch) > 0:

              transaction_arch = best_compat

-         if hasattr(rpmUtils.arch, 'ArchStorage'):

+         if hasattr(koji.arch, 'ArchStorage'):

              yumbase.preconf.arch = transaction_arch

          else:

-             rpmUtils.arch.canonArch = transaction_arch

+             koji.arch.canonArch = transaction_arch

  

          yconfig = """

  [main]

@@ -5428,9 +5581,8 @@ 

  

          # step 3: proceed with yum config and set up

          yconfig_path = os.path.join(yumdir, 'yum.conf-koji-%s' % arch)

-         f = open(yconfig_path, 'w')

-         f.write(yconfig)

-         f.close()

+         with open(yconfig_path, 'w') as f:

+             f.write(yconfig)

          self.session.uploadWrapper(yconfig_path, self.uploadpath,

              os.path.basename(yconfig_path))

          yumbase.doConfigSetup(fn=yconfig_path)

@@ -5467,12 +5619,11 @@ 

              raise koji.GenericError(errors)

          if len(fs_missing) > 0:

              missing_log = os.path.join(self.workdir, 'missing_multilib.log')

-             outfile = open(missing_log, 'w')

-             outfile.write('The following multilib files were missing:\n')

-             for ml_path in fs_missing:

-                 outfile.write(ml_path)

-                 outfile.write('\n')

-             outfile.close()

+             with open(missing_log, 'w') as outfile:

+                 outfile.write('The following multilib files were missing:\n')

+                 for ml_path in fs_missing:

+                     outfile.write(ml_path)

+                     outfile.write('\n')

              self.session.uploadWrapper(missing_log, self.uploadpath)

              raise koji.GenericError('multilib packages missing. '

                      'See missing_multilib.log')

@@ -5570,33 +5721,31 @@ 

          # report problems

          if len(fs_missing) > 0:

              missing_log = os.path.join(self.workdir, 'missing_files.log')

-             outfile = open(missing_log, 'w')

-             outfile.write('Some rpm files were missing.\n'

-                 'Most likely, you want to create these signed copies.\n\n'

-                 'Missing files:\n')

-             for pkgpath in sorted(fs_missing):

-                 outfile.write(pkgpath)

-                 outfile.write('\n')

-             outfile.close()

+             with open(missing_log, 'w') as outfile:

+                 outfile.write('Some rpm files were missing.\n'

+                     'Most likely, you want to create these signed copies.\n\n'

+                     'Missing files:\n')

+                 for pkgpath in sorted(fs_missing):

+                     outfile.write(pkgpath)

+                     outfile.write('\n')

              self.session.uploadWrapper(missing_log, self.uploadpath)

              raise koji.GenericError('Packages missing from the filesystem. '

                      'See missing_files.log.')

          if sig_missing:

              # log missing signatures and possibly error

              missing_log = os.path.join(self.workdir, 'missing_signatures.log')

-             outfile = open(missing_log, 'w')

-             outfile.write('Some rpms were missing requested signatures.\n')

-             if opts['skip_missing_signatures']:

-                 outfile.write('The skip_missing_signatures option was specified, so '

-                         'these files were excluded.\n')

-             outfile.write('Acceptable keys: %r\n\n' % keys)

-             outfile.write('# RPM name: available keys\n')

-             fmt = '%(name)s-%(version)s-%(release)s.%(arch)s'

-             filenames = [[fmt % selected[r], r] for r in sig_missing]

-             for fname, rpm_id in sorted(filenames):

-                 avail = to_list(rpm_idx.get(rpm_id, {}).keys())

-                 outfile.write('%s: %r\n' % (fname, avail))

-             outfile.close()

+             with open(missing_log, 'w') as outfile:

+                 outfile.write('Some rpms were missing requested signatures.\n')

+                 if opts['skip_missing_signatures']:

+                     outfile.write('The skip_missing_signatures option was specified, so '

+                             'these files were excluded.\n')

+                 outfile.write('Acceptable keys: %r\n\n' % keys)

+                 outfile.write('# RPM name: available keys\n')

+                 fmt = '%(name)s-%(version)s-%(release)s.%(arch)s'

+                 filenames = [[fmt % selected[r], r] for r in sig_missing]

+                 for fname, rpm_id in sorted(filenames):

+                     avail = to_list(rpm_idx.get(rpm_id, {}).keys())

+                     outfile.write('%s: %r\n' % (fname, avail))

              self.session.uploadWrapper(missing_log, self.uploadpath)

              if (not opts['skip_missing_signatures']

                          and not opts['allow_missing_signatures']):

@@ -5645,11 +5794,8 @@ 

  

      def write_kojipkgs(self):

          filename = os.path.join(self.repodir, 'kojipkgs')

-         datafile = open(filename, 'w')

-         try:

+         with open(filename, 'w') as datafile:

              json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True)

-         finally:

-             datafile.close()

  

  

  class WaitrepoTask(BaseTaskHandler):

file modified
+9 -7

@@ -168,13 +168,9 @@ 

  Group: Applications/System

  License: LGPLv2 and GPLv2+

  #mergerepos (from createrepo) is GPLv2+

- Requires: %{name} = %{version}-%{release}

- # we need the python2 lib here

- Requires: python2-%{name} = %{version}-%{release}

  Requires: mock >= 0.9.14

  Requires(pre): /usr/sbin/useradd

  Requires: squashfs-tools

- Requires: python2-multilib

  %if %{use_systemd}

  Requires(post): systemd

  Requires(preun): systemd

@@ -188,9 +184,15 @@ 

  Requires: /usr/bin/cvs

  Requires: /usr/bin/svn

  Requires: /usr/bin/git

- Requires: python-cheetah

- %if 0%{?fedora} >= 9

  Requires: createrepo >= 0.9.2

+ %if 0%{with python3}

+ Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release}

+ Requires: python%{python3_pkgversion}-multilib

+ Requires: python%{python3_pkgversion}-cheetah

+ %else

+ Requires: python2-%{name} = %{version}-%{release}

+ Requires: python2-multilib

+ Requires: python-cheetah

  %endif

  

  %description builder

@@ -201,7 +203,6 @@ 

  Summary: Koji virtual machine management daemon

  Group: Applications/System

  License: LGPLv2

- Requires: %{name} = %{version}-%{release}

  # we need the python2 lib here

  Requires: python2-%{name} = %{version}-%{release}

  %if %{use_systemd}

@@ -278,6 +279,7 @@ 

  make DESTDIR=$RPM_BUILD_ROOT PYTHON=%{__python3} %{?install_opt} install

  # alter python interpreter in koji CLI

  sed -i 's/\#\!\/usr\/bin\/python2/\#\!\/usr\/bin\/python3/' $RPM_BUILD_ROOT/usr/bin/koji

+ sed -i 's/\#\!\/usr\/bin\/python2/\#\!\/usr\/bin\/python3/' $RPM_BUILD_ROOT/usr/sbin/kojid

  %endif

  

  %clean

file modified
+33 -6

@@ -890,6 +890,22 @@ 

      return hdr

  

  

+ def _decode_item(item):

+     """Decode rpm header byte strings to str in py3"""

+     if six.PY2:

+         return item

+     elif isinstance(item, bytes):

+         try:

+             return item.decode()

+         except UnicodeDecodeError:

+             # typically signatures

+             return item

+     elif isinstance(item, list):

+         return [_decode_item(x) for x in item]

+     else:

+         return item

+ 

+ 

  def get_header_field(hdr, name, src_arch=False):

      """Extract named field from an rpm header"""

      name = name.upper()

@@ -905,6 +921,7 @@ 

              return "nosrc"

          return "src"

  

+     # REMOVED?

      result = _get_header_field(hdr, name)

  

      if name in ("NOSOURCE", "NOPATCH"):

@@ -913,12 +930,7 @@ 

              result = []

          elif isinstance(result, six.integer_types):

              result = [result]

-     if six.PY3 and isinstance(result, bytes):

-         try:

-             result = result.decode('utf-8')

-         except UnicodeDecodeError:

-             # typically signatures

-             pass

+     result = _decode_item(result)

  

      sizetags = ('SIZE', 'ARCHIVESIZE', 'FILESIZES', 'SIGSIZE')

      if name in sizetags and (result is None or result == []):

@@ -929,6 +941,8 @@ 

              pass

  

      return result

+     # REPLACED

+     # return _get_header_field(hdr, name)

  

  

  def _get_header_field(hdr, name):

@@ -2389,6 +2403,14 @@ 

          else:

              handler = self.baseurl

          request = dumps(args, name, allow_none=1)

+         try:

+             request.encode('latin-1')

+         except (UnicodeEncodeError, UnicodeDecodeError):

+             # py2 string throws UnicodeDecodeError

+             # py3 string throws UnicodeEncodeError

+             # if string is not converted to UTF, requests will raise an error

+             # on identical check before sending data

+             request = request.encode('utf-8')

          headers = [

              # connection class handles Host

              ('User-Agent', 'koji/1'),

@@ -2682,6 +2704,11 @@ 

              ("Content-length", str(size)),

          ]

          request = chunk

+         if six.PY3 and isinstance(chunk, str):

+             request = chunk.encode('utf-8')

+         else:

+             # py2 or bytes

+             request = chunk

          return handler, headers, request

  

      def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=True, volume=None):

file added
+512

@@ -0,0 +1,512 @@ 

+ #!/usr/bin/python

+ #

+ 

+ import os

+ import rpm

+ import ctypes

+ import struct

+ 

+ _ppc64_native_is_best = True

+ 

+ # dict mapping arch -> ( multicompat, best personality, biarch personality )

+ multilibArches = { "x86_64":  ( "athlon", "x86_64", "athlon" ),

+                    "sparc64v": ( "sparcv9v", "sparcv9v", "sparc64v" ),

+                    "sparc64": ( "sparcv9", "sparcv9", "sparc64" ),

+                    "ppc64":   ( "ppc", "ppc", "ppc64" ),

+                    "s390x":   ( "s390", "s390x", "s390" ),

+                    }

+ if _ppc64_native_is_best:

+     multilibArches["ppc64"] = ( "ppc", "ppc64", "ppc64" )

+ 

+ arches = {

+     # ia32

+     "athlon": "i686",

+     "i686": "i586",

+     "geode": "i686",

+     "i586": "i486",

+     "i486": "i386",

+     "i386": "noarch",

+ 

+     # amd64

+     "x86_64": "athlon",

+     "amd64": "x86_64",

+     "ia32e": "x86_64",

+ 

+     #ppc64le

+     "ppc64le":  "noarch",

+ 

+     # ppc

+     "ppc64p7": "ppc64",

+     "ppc64pseries": "ppc64",

+     "ppc64iseries": "ppc64",

+     "ppc64": "ppc",

+     "ppc": "noarch",

+ 

+     # s390{,x}

+     "s390x": "s390",

+     "s390": "noarch",

+ 

+     # sparc

+     "sparc64v": "sparcv9v",

+     "sparc64": "sparcv9",

+     "sparcv9v": "sparcv9",

+     "sparcv9": "sparcv8",

+     "sparcv8": "sparc",

+     "sparc": "noarch",

+ 

+     # alpha

+     "alphaev7":   "alphaev68",

+     "alphaev68":  "alphaev67",

+     "alphaev67":  "alphaev6",

+     "alphaev6":   "alphapca56",

+     "alphapca56": "alphaev56",

+     "alphaev56":  "alphaev5",

+     "alphaev5":   "alphaev45",

+     "alphaev45":  "alphaev4",

+     "alphaev4":   "alpha",

+     "alpha":      "noarch",

+ 

+     # arm

+     "armv7l": "armv6l",

+     "armv6l": "armv5tejl",

+     "armv5tejl": "armv5tel",

+     "armv5tel": "noarch",

+ 

+     #arm hardware floating point

+     "armv7hnl": "armv7hl",

+     "armv7hl": "armv6hl",

+     "armv6hl": "noarch",

+ 

+     # arm64

+     "arm64": "noarch",

+ 

+     # super-h

+     "sh4a": "sh4",

+     "sh4": "noarch",

+     "sh3": "noarch",

+ 

+     #itanium

+     "ia64": "noarch",

+     }

+ 

+ #  Will contain information parsed from /proc/self/auxv via _parse_auxv().

+ # Should move into rpm really.

+ _aux_vector = {

+     "platform": "",

+     "hwcap": 0,

+     }

+ 

+ def legitMultiArchesInSameLib(arch=None):

+     # this is completely crackrock - if anyone has a better way I

+     # am all ears

+ 

+     arch = getBestArch(arch)

+     if isMultiLibArch(arch):

+         arch = getBaseArch(myarch=arch)

+ 

+     results = [arch]

+ 

+     if arch in ('x86_64', 'ppc64') or arch.startswith('sparcv9'):

+         for (k, v) in arches.items():

+             if v == arch:

+                 results.append(k)

+     return results

+ 

+ 

+ def canCoinstall(arch1, arch2):

+     """Take two arches and return True if it is possible that they can be

+        installed together with the same nevr. Ex: arch1=i386 and arch2=i686 then

+        it will return False. arch1=i386 and arch2=x86_64 will return True.

+        It does not determine whether or not the arches make any sense. Just whether

+        they could possibly install w/o conflict"""

+ 

+     # if both are a multlibarch then we can't coinstall  (x86_64, ia32e)

+     # if both are not multilibarches then we can't coinstall (i386, i686)

+ 

+     if 'noarch' in [arch1, arch2]: # noarch can never coinstall

+         return False

+ 

+     if isMultiLibArch(arch=arch1) == isMultiLibArch(arch=arch2):

+         return False

+     # this section keeps arch1=x86_64 arch2=ppc from returning True

+     if arch1 in getArchList(arch2) or arch2 in getArchList(arch1):

+         return True

+     return False

+ 

+ # this computes the difference between myarch and targetarch

+ def archDifference(myarch, targetarch):

+     if myarch == targetarch:

+         return 1

+     if myarch in arches:

+         ret = archDifference(arches[myarch], targetarch)

+         if ret != 0:

+             return ret + 1

+         return 0

+     return 0

+ 

+ def score(arch):

+     return archDifference(canonArch, arch)

+ 

+ def isMultiLibArch(arch=None):

+     """returns true if arch is a multilib arch, false if not"""

+     if arch is None:

+         arch = canonArch

+ 

+     if arch not in arches: # or we could check if it is noarch

+         return 0

+ 

+     if arch in multilibArches:

+         return 1

+ 

+     if arches[arch] in multilibArches:

+         return 1

+ 

+     return 0

+ 

+ def getBestArchFromList(archlist, myarch=None):

+     """

+         return the best arch from the list for myarch if - myarch is not given,

+         then return the best arch from the list for the canonArch.

+     """

+ 

+     if len(archlist) == 0:

+         return None

+ 

+     if myarch is None:

+         myarch = canonArch

+ 

+     mybestarch = getBestArch(myarch)

+ 

+     bestarch = getBestArch(myarch)

+     if bestarch != myarch:

+         bestarchchoice = getBestArchFromList(archlist, bestarch)

+         if bestarchchoice != None and bestarchchoice != "noarch":

+             return bestarchchoice

+ 

+     thisarch = archlist[0]

+     for arch in archlist[1:]:

+         val1 = archDifference(myarch, thisarch)

+         val2 = archDifference(myarch, arch)

+         if val1 == 0 and val2 == 0:

+             continue

+         if val1 < val2:

+             if val1 == 0:

+                 thisarch = arch

+         if val2 < val1:

+             if val2 != 0:

+                 thisarch = arch

+         if val1 == val2:

+             pass

+ 

+     # thisarch should now be our bestarch

+     # one final check to make sure we're not returning a bad arch

+     val = archDifference(myarch, thisarch)

+     if val == 0:

+         return None

+ 

+     return thisarch

+ 

+ 

+ def getArchList(thisarch=None):

+     # this returns a list of archs that are compatible with arch given

+     if not thisarch:

+         thisarch = canonArch

+ 

+     archlist = [thisarch]

+     while thisarch in arches:

+         thisarch = arches[thisarch]

+         archlist.append(thisarch)

+ 

+     # hack hack hack

+     # sparc64v is also sparc64 compat

+     if archlist[0] == "sparc64v":

+         archlist.insert(1,"sparc64")

+ 

+     # if we're a weirdo arch - add noarch on there.

+     if len(archlist) == 1 and archlist[0] == thisarch:

+         archlist.append('noarch')

+     return archlist

+ 

+ def _try_read_cpuinfo():

+     """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not

+         mounted). """

+     try:

+         return open("/proc/cpuinfo", "r")

+     except:

+         return []

+ 

+ def _parse_auxv():

+     """ Read /proc/self/auxv and parse it into global dict for easier access

+         later on, very similar to what rpm does. """

+     # In case we can't open and read /proc/self/auxv, just return

+     try:

+         data = open("/proc/self/auxv", "rb").read()

+     except:

+         return

+ 

+     # Define values from /usr/include/elf.h

+     AT_PLATFORM = 15

+     AT_HWCAP = 16

+     fmtlen = struct.calcsize("LL")

+     offset = 0

+     platform = ctypes.c_char_p()

+ 

+     # Parse the data and fill in _aux_vector dict

+     while offset <= len(data) - fmtlen:

+         at_type, at_val = struct.unpack_from("LL", data, offset)

+         if at_type == AT_PLATFORM:

+             platform.value = at_val

+             _aux_vector["platform"] = platform.value

+         if at_type == AT_HWCAP:

+             _aux_vector["hwcap"] = at_val

+         offset = offset + fmtlen

+ 

+ def getCanonX86Arch(arch):

+     #

+     if arch == "i586":

+         for line in _try_read_cpuinfo():

+             if line.startswith("model name"):

+                 if line.find("Geode(TM)") != -1:

+                     return "geode"

+                 break

+         return arch

+     # only athlon vs i686 isn't handled with uname currently

+     if arch != "i686":

+         return arch

+ 

+     # if we're i686 and AuthenticAMD, then we should be an athlon

+     for line in _try_read_cpuinfo():

+         if line.startswith("vendor") and line.find("AuthenticAMD") != -1:

+             return "athlon"

+         # i686 doesn't guarantee cmov, but we depend on it

+         elif line.startswith("flags"):

+             if line.find("cmov") == -1:

+                 return "i586"

+             break

+ 

+     return arch

+ 

+ def getCanonARMArch(arch):

+     # the %{_target_arch} macro in rpm will let us know the abi we are using

+     target = rpm.expandMacro('%{_target_cpu}')

+     if target.startswith('armv6h'):

+         return target

+     if target.startswith('armv7h'):

+         return target

+     return arch

+ 

+ def getCanonPPCArch(arch):

+     # FIXME: should I do better handling for mac, etc?

+     if arch != "ppc64":

+         return arch

+ 

+     machine = None

+     for line in _try_read_cpuinfo():

+         if line.find("machine") != -1:

+             machine = line.split(':')[1]

+             break

+ 

+     platform = _aux_vector["platform"]

+     if machine is None and not platform:

+         return arch

+ 

+     try:

+         if platform.startswith("power") and int(platform[5:].rstrip('+')) >= 7:

+             return "ppc64p7"

+     except:

+         pass

+ 

+     if machine is None:

+         return arch

+ 

+     if machine.find("CHRP IBM") != -1:

+         return "ppc64pseries"

+     if machine.find("iSeries") != -1:

+         return "ppc64iseries"

+     return arch

+ 

+ def getCanonSPARCArch(arch):

+     # Deal with sun4v, sun4u, sun4m cases

+     SPARCtype = None

+     for line in _try_read_cpuinfo():

+         if line.startswith("type"):

+             SPARCtype = line.split(':')[1]

+             break

+     if SPARCtype is None:

+         return arch

+ 

+     if SPARCtype.find("sun4v") != -1:

+         if arch.startswith("sparc64"):

+             return "sparc64v"

+         else:

+             return "sparcv9v"

+     if SPARCtype.find("sun4u") != -1:

+         if arch.startswith("sparc64"):

+             return "sparc64"

+         else:

+             return "sparcv9"

+     if SPARCtype.find("sun4m") != -1:

+         return "sparcv8"

+     return arch

+ 

+ def getCanonX86_64Arch(arch):

+     if arch != "x86_64":

+         return arch

+ 

+     vendor = None

+     for line in _try_read_cpuinfo():

+         if line.startswith("vendor_id"):

+             vendor = line.split(':')[1]

+             break

+     if vendor is None:

+         return arch

+ 

+     if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1:

+         return "amd64"

+     if vendor.find("GenuineIntel") != -1:

+         return "ia32e"

+     return arch

+ 

+ def getCanonArch(skipRpmPlatform = 0):

+     if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):

+         try:

+             f = open("/etc/rpm/platform", "r")

+             line = f.readline()

+             f.close()

+             (arch, vendor, opersys) = line.split("-", 2)

+             return arch

+         except:

+             pass

+ 

+     arch = os.uname()[4]

+ 

+     _parse_auxv()

+ 

+     if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"):

+         return getCanonX86Arch(arch)

+ 

+     if arch.startswith("arm"):

+         return getCanonARMArch(arch)

+     if arch.startswith("ppc"):

+         return getCanonPPCArch(arch)

+     if arch.startswith("sparc"):

+         return getCanonSPARCArch(arch)

+     if arch == "x86_64":

+         return getCanonX86_64Arch(arch)

+ 

+     return arch

+ 

+ canonArch = getCanonArch()

+ 

+ # this gets you the "compat" arch of a biarch pair

+ def getMultiArchInfo(arch = canonArch):

+     if arch in multilibArches:

+         return multilibArches[arch]

+     if arch in arches and arches[arch] != "noarch":

+         return getMultiArchInfo(arch = arches[arch])

+     return None

+ 

+ # get the best usual userspace arch for the arch we're on.  this is

+ # our arch unless we're on an arch that uses the secondary as its

+ # userspace (eg ppc64, sparc64)

+ def getBestArch(myarch=None):

+     if myarch:

+         arch = myarch

+     else:

+         arch = canonArch

+ 

+     if arch.startswith("sparc64"):

+         arch = multilibArches[arch][1]

+ 

+     if arch.startswith("ppc64") and not _ppc64_native_is_best and arch != "ppc64le":

+         arch = 'ppc'

+ 

+     return arch

+ 

+ def getBaseArch(myarch=None):

+     """returns 'base' arch for myarch, if specified, or canonArch if not.

+        base arch is the arch before noarch in the arches dict if myarch is not

+        a key in the multilibArches."""

+ 

+     if not myarch:

+         myarch = canonArch

+ 

+     if myarch not in arches: # this is dumb, but <shrug>

+         return myarch

+ 

+     if myarch.startswith("sparc64"):

+         return "sparc"

+     elif myarch == "ppc64le":

+         return "ppc64le"

+     elif myarch.startswith("ppc64") and not _ppc64_native_is_best:

+         return "ppc"

+     elif myarch.startswith("arm64"):

+         return "arm64"

+     elif myarch.startswith("armv6h"):

+         return "armhfp"

+     elif myarch.startswith("armv7h"):

+         return "armhfp"

+     elif myarch.startswith("arm"):

+         return "arm"

+ 

+     if isMultiLibArch(arch=myarch):

+         if myarch in multilibArches:

+             return myarch

+         else:

+             return arches[myarch]

+ 

+     if myarch in arches:

+         basearch = myarch

+         value = arches[basearch]

+         while value != 'noarch':

+             basearch = value

+             value = arches[basearch]

+ 

+         return basearch

+ 

+ 

+ class ArchStorage(object):

+     """class for keeping track of what arch we have set and doing various

+        permutations based on it"""

+     def __init__(self):

+         self.canonarch = None

+         self.basearch = None

+         self.bestarch = None

+         self.compatarches = []

+         self.archlist = []

+         self.multilib = False

+         self.setup_arch()

+ 

+     def setup_arch(self, arch=None, archlist_includes_compat_arch=True):

+         if arch:

+             self.canonarch = arch

+         else:

+             self.canonarch = canonArch

+ 

+         self.basearch = getBaseArch(myarch=self.canonarch)

+         self.archlist = getArchList(thisarch=self.canonarch)

+ 

+         if not archlist_includes_compat_arch: # - do we bother including i686 and below on x86_64

+             limit_archlist = []

+             for a in self.archlist:

+                 if isMultiLibArch(a) or a == 'noarch':

+                     limit_archlist.append(a)

+             self.archlist = limit_archlist

+ 

+         self.bestarch = getBestArch(myarch=self.canonarch)

+         self.compatarches = getMultiArchInfo(arch=self.canonarch)

+         self.multilib = isMultiLibArch(arch=self.canonarch)

+         self.legit_multi_arches = legitMultiArchesInSameLib(arch = self.canonarch)

+ 

+     def get_best_arch_from_list(self, archlist, fromarch=None):

+         if not fromarch:

+             fromarch = self.canonarch

+         return getBestArchFromList(archlist, myarch=fromarch)

+ 

+     def score(self, arch):

+         return archDifference(self.canonarch, arch)

+ 

+     def get_arch_list(self, arch):

+         if not arch:

+             return self.archlist

+         return getArchList(thisarch=arch)

file modified
+16 -10

@@ -31,7 +31,6 @@ 

  import os

  import signal

  import logging

- from six.moves import urllib

  from fnmatch import fnmatch

  import base64

  import time

@@ -40,6 +39,7 @@ 

  import traceback

  import errno

  from six.moves import range

+ from six.moves import urllib

  import six

  

  

@@ -122,7 +122,10 @@ 

              if logerror:

                  os.dup2(fd, 2)

              # echo the command we're running into the logfile

-             os.write(fd, '$ %s\n' % ' '.join(args))

+             msg = '$ %s\n' % ' '.join(args)

+             if six.PY3:

+                 msg = msg.encode('utf-8')

+             os.write(fd, msg)

              environ = os.environ.copy()

              if env:

                  environ.update(env)

@@ -131,7 +134,10 @@ 

              msg = ''.join(traceback.format_exception(*sys.exc_info()))

              if fd:

                  try:

-                     os.write(fd, msg)

+                     if six.PY3:

+                         os.write(fs, msg.encode('utf-8'))

+                     else:

+                         os.write(fd, msg)

                      os.close(fd)

                  except:

                      pass

@@ -588,7 +594,7 @@ 

  

      def shutdown(self):

          """Attempt to shut down cleanly"""

-         for task_id in self.pids.keys():

+         for task_id in self.pids:

              self.cleanupTask(task_id)

          self.session.host.freeTasks(to_list(self.tasks.keys()))

          self.session.host.updateHost(task_load=0.0, ready=False)

@@ -628,7 +634,7 @@ 

              return

          local_br = self._scanLocalBuildroots()

          # get info on local_only buildroots (most likely expired)

-         local_only = [id for id in six.iterkeys(local_br) if id not in db_br]

+         local_only = [id for id in local_br if id not in db_br]

          if local_only:

              missed_br = self.session.listBuildroots(buildrootID=tuple(local_only))

              #get all the task info in one call

@@ -796,14 +802,14 @@ 

              #about). This will happen after a daemon restart, for example.

              self.logger.info("freeing stale tasks: %r" % stale)

              self.session.host.freeTasks(stale)

-         for id, pid in self.pids.items():

+         for id, pid in list(self.pids.items()):

              if self._waitTask(id, pid):

                  # the subprocess handles most everything, we just need to clear things out

                  if self.cleanupTask(id, wait=False):

                      del self.pids[id]

                  if id in self.tasks:

                      del self.tasks[id]

-         for id, pid in self.pids.items():