f116d93
# -*- coding: utf-8 -*-
5451453
5451453
1f85a11
# This program is free software; you can redistribute it and/or modify
1f85a11
# it under the terms of the GNU General Public License as published by
1f85a11
# the Free Software Foundation; version 2 of the License.
1f85a11
#
1f85a11
# This program is distributed in the hope that it will be useful,
1f85a11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
1f85a11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
1f85a11
# GNU Library General Public License for more details.
1f85a11
#
1f85a11
# You should have received a copy of the GNU General Public License
1f85a11
# along with this program; if not, write to the Free Software
1f85a11
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
1f85a11
5451453
Jesse Keating 3b89e45
import yum
1f85a11
import os
Jesse Keating 3b89e45
import re
Jesse Keating a621cdc
import shutil
Jesse Keating 6748133
import sys
f5c6d44
import pungi.util
94235b0
import pprint
2244630
import lockfile
Jesse Keating 3b89e45
import logging
Jesse Keating 3b89e45
import urlgrabber.progress
Jesse Keating 3b89e45
import subprocess
Jesse Keating 3b89e45
import createrepo
Jesse Keating 74e0147
import ConfigParser
9d7f82d
from fnmatch import fnmatch
9d7f82d
5451453
import arch as arch_module
30c0f35
import multilib
5451453
1f85a11
2244630
class ReentrantYumLock(object):
2244630
    """ A lock that can be acquired multiple times by the same process. """
2244630
2244630
    def __init__(self, lock, log):
2244630
        self.lock = lock
2244630
        self.log = log
2244630
        self.count = 0
2244630
2244630
    def __enter__(self):
2244630
        if not self.count:
2244630
            self.log.info("Waiting on %r" % self.lock.lock_file)
2244630
            self.lock.acquire()
2244630
            self.log.info("Got %r" % self.lock.lock_file)
2244630
        self.count = self.count + 1
2244630
        self.log.info("Lock count upped to %i" % self.count)
2244630
2244630
    def __exit__(self, type, value, tb):
2244630
        self.count = self.count - 1
2244630
        self.log.info("Lock count downed to %i" % self.count)
2244630
        self.log.info("%r %r %r" % (type, value, tb))
2244630
        if not self.count:
2244630
            self.lock.release()
2244630
            self.log.info("Released %r" % self.lock.lock_file)
2244630
2244630
2244630
def yumlocked(method):
2244630
    """ A locking decorator. """
2244630
    def wrapper(self, *args, **kwargs):
2244630
        with self.yumlock:
2244630
            return method(self, *args, **kwargs)
2244630
    # TODO - replace argspec, signature, etc..
2244630
    return wrapper
2244630
2244630
c047fe5
def is_debug(po):
c047fe5
    if "debuginfo" in po.name:
c047fe5
        return True
c047fe5
    return False
c047fe5
c047fe5
c047fe5
def is_source(po):
c047fe5
    if po.arch in ("src", "nosrc"):
c047fe5
        return True
c047fe5
    return False
c047fe5
c047fe5
30c0f35
def is_noarch(po):
30c0f35
    if po.arch == "noarch":
30c0f35
        return True
30c0f35
    return False
30c0f35
30c0f35
c047fe5
def is_package(po):
c047fe5
    if is_debug(po):
c047fe5
        return False
c047fe5
    if is_source(po):
c047fe5
        return False
c047fe5
    return True
c047fe5
c047fe5
Jesse Keating dcf4d90
class MyConfigParser(ConfigParser.ConfigParser):
Jesse Keating dcf4d90
    """A subclass of ConfigParser which does not lowercase options"""
Jesse Keating dcf4d90
Jesse Keating dcf4d90
    def optionxform(self, optionstr):
Jesse Keating dcf4d90
        return optionstr
Jesse Keating dcf4d90
Jesse Keating dcf4d90
Jesse Keating 28c0eca
class PungiBase(object):
1f85a11
    """The base Pungi class.  Set up config items and logging here"""
1f85a11
1f85a11
    def __init__(self, config):
1f85a11
        self.config = config
320724e
        multilib.init(self.config.get('pungi', 'multilibconf'))
Jesse Keating 5c70f43
5451453
        # ARCH setup
5451453
        self.tree_arch = self.config.get('pungi', 'arch')
17221a3
        self.yum_arch = arch_module.tree_arch_to_yum_arch(self.tree_arch)
5451453
        full_archlist = self.config.getboolean('pungi', 'full_archlist')
5451453
        self.valid_arches = arch_module.get_valid_arches(self.tree_arch, multilib=full_archlist)
5451453
        self.valid_arches.append("src") # throw source in there, filter it later
a9581e2
        self.valid_native_arches = arch_module.get_valid_arches(self.tree_arch, multilib=False)
5451453
        self.valid_multilib_arches = arch_module.get_valid_multilib_arches(self.tree_arch)
1f85a11
c047fe5
        # arch: compatible arches
c047fe5
        self.compatible_arches = {}
c047fe5
        for i in self.valid_arches:
c047fe5
            self.compatible_arches[i] = arch_module.get_compatible_arches(i)
c047fe5
5451453
        self.doLoggerSetup()
f57a4ac
        self.workdir = os.path.join(self.config.get('pungi', 'workdirbase'),
0633eb2
                                    self.config.get('pungi', 'variant'),
5451453
                                    self.tree_arch)
1f85a11
1f85a11
1f85a11
1f85a11
    def doLoggerSetup(self):
1f85a11
        """Setup our logger"""
1f85a11
Jesse Keating 3ca7a26
        logdir = os.path.join(self.config.get('pungi', 'destdir'), 'logs')
1f85a11
f5c6d44
        pungi.util._ensuredir(logdir, None, force=True) # Always allow logs to be written out
1f85a11
0633eb2
        if self.config.get('pungi', 'variant'):
0633eb2
            logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('pungi', 'variant'),
5451453
                                                          self.tree_arch))
1f85a11
        else:
5451453
            logfile = os.path.join(logdir, '%s.log' % (self.tree_arch))
1f85a11
1f85a11
        # Create the root logger, that will log to our file
1f85a11
        logging.basicConfig(level=logging.DEBUG,
1f85a11
                            format='%(name)s.%(levelname)s: %(message)s',
1f85a11
                            filename=logfile)
1f85a11
Jesse Keating 6feda08
Jesse Keating 3b89e45
class CallBack(urlgrabber.progress.TextMeter):
Jesse Keating 3b89e45
    """A call back function used with yum."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
    def progressbar(self, current, total, name=None):
Jesse Keating 3b89e45
        return
Jesse Keating 3b89e45
Jesse Keating a81ead6
Jesse Keating 3b89e45
class PungiYum(yum.YumBase):
Jesse Keating 3b89e45
    """Subclass of Yum"""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
    def __init__(self, config):
Jesse Keating 3b89e45
        self.pungiconfig = config
Jesse Keating 3b89e45
        yum.YumBase.__init__(self)
Jesse Keating 3b89e45
Jesse Keating 9ca05af
    def doLoggingSetup(self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None):
Jesse Keating 3b89e45
        """Setup the logging facility."""
Jesse Keating 3b89e45
Jesse Keating 3ca7a26
        logdir = os.path.join(self.pungiconfig.get('pungi', 'destdir'), 'logs')
Jesse Keating 3b89e45
        if not os.path.exists(logdir):
Jesse Keating 3b89e45
            os.makedirs(logdir)
0633eb2
        if self.pungiconfig.get('pungi', 'variant'):
0633eb2
            logfile = os.path.join(logdir, '%s.%s.log' % (self.pungiconfig.get('pungi', 'variant'),
Jesse Keating 3ca7a26
                                                          self.pungiconfig.get('pungi', 'arch')))
Jesse Keating 3b89e45
        else:
Jesse Keating 3ca7a26
            logfile = os.path.join(logdir, '%s.log' % (self.pungiconfig.get('pungi', 'arch')))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
    def doFileLogSetup(self, uid, logfile):
Jesse Keating 3b89e45
        # This function overrides a yum function, allowing pungi to control
Jesse Keating 3b89e45
        # the logging.
Jesse Keating 3b89e45
        pass
Jesse Keating 3b89e45
17221a3
    def _compare_providers(self, *args, **kwargs):
17221a3
        # HACK: always prefer 64bit over 32bit packages
17221a3
        result = yum.YumBase._compare_providers(self, *args, **kwargs)
17221a3
        if len(result) >= 2:
17221a3
            pkg1 = result[0][0]
17221a3
            pkg2 = result[1][0]
17221a3
            if pkg1.name == pkg2.name:
17221a3
                best_arch = self.arch.get_best_arch_from_list([pkg1.arch, pkg2.arch], self.arch.canonarch)
17221a3
                if best_arch != "noarch" and best_arch != pkg1.arch:
17221a3
                    result[0:1] = result[0:1:-1]
17221a3
        return result
Jesse Keating a81ead6
f5c6d44
class Pungi(PungiBase):
Jesse Keating 3b89e45
    def __init__(self, config, ksparser):
f5c6d44
        PungiBase.__init__(self, config)
2244630
Jesse Keating 3b89e45
        # Set our own logging name space
Jesse Keating 3b89e45
        self.logger = logging.getLogger('Pungi')
Jesse Keating 3b89e45
2244630
        # Create a lock object for later use.
2244630
        filename = self.config.get('pungi', 'cachedir') + "/yumlock"
2244630
        lock = lockfile.LockFile(filename)
2244630
        self.yumlock = ReentrantYumLock(lock, self.logger)
2244630
Jesse Keating 3b89e45
        # Create the stdout/err streams and only send INFO+ stuff there
Jesse Keating 3b89e45
        formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
Jesse Keating 3b89e45
        console = logging.StreamHandler()
Jesse Keating 3b89e45
        console.setFormatter(formatter)
Jesse Keating 3b89e45
        console.setLevel(logging.INFO)
Jesse Keating 3b89e45
        self.logger.addHandler(console)
Jesse Keating 3b89e45
Jesse Keating 3ca7a26
        self.destdir = self.config.get('pungi', 'destdir')
Jesse Keating 3b89e45
        self.archdir = os.path.join(self.destdir,
Jesse Keating 3ca7a26
                                   self.config.get('pungi', 'version'),
0633eb2
                                   self.config.get('pungi', 'variant'),
5451453
                                   self.tree_arch)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        self.topdir = os.path.join(self.archdir, 'os')
Jesse Keating 3ca7a26
        self.isodir = os.path.join(self.archdir, self.config.get('pungi','isodir'))
Jesse Keating 3b89e45
f5c6d44
        pungi.util._ensuredir(self.workdir, self.logger, force=True)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        self.common_files = []
Jesse Keating 3ca7a26
        self.infofile = os.path.join(self.config.get('pungi', 'destdir'),
Jesse Keating 3ca7a26
                                    self.config.get('pungi', 'version'),
Jesse Keating 3b89e45
                                    '.composeinfo')
Jesse Keating 3b89e45
5451453
Jesse Keating 3b89e45
        self.ksparser = ksparser
17221a3
Jesse Keating 3b89e45
        self.resolved_deps = {} # list the deps we've already resolved, short circuit.
9d7f82d
        self.excluded_pkgs = {} # list the packages we've already excluded.
5069eb8
        self.seen_pkgs = {}     # list the packages we've already seen so we can check all deps only once
30c0f35
        self.multilib_methods = self.config.get('pungi', 'multilib').split(" ")
754a1bb
754a1bb
        # greedy methods:
754a1bb
        #  * none: only best match package
754a1bb
        #  * all: all packages matching a provide
754a1bb
        #  * build: best match package + all other packages from the same SRPM having the same provide
754a1bb
        self.greedy_method = self.config.get('pungi', 'greedy')
754a1bb
c076de2
        self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ")
c047fe5
        self.sourcerpm_arch_map = {}    # {sourcerpm: set[arches]} - used for gathering debuginfo
Jesse Keating e9e0a18
17221a3
        # package object lists
17221a3
        self.po_list = set()
17221a3
        self.srpm_po_list = set()
17221a3
        self.debuginfo_po_list = set()
17221a3
17221a3
        # get_srpm_po() cache
17221a3
        self.sourcerpm_srpmpo_map = {}
17221a3
9a09cf9
        # flags
8b1c243
        self.input_packages = set()         # packages specified in %packages kickstart section including those defined via comps groups
8b1c243
        self.comps_packages = set()         # packages specified in %packages kickstart section *indirectly* via comps groups
8b1c243
        self.prepopulate_packages = set()   # packages specified in %prepopulate kickstart section
9a09cf9
        self.fulltree_packages = set()
9a09cf9
        self.langpack_packages = set()
9a09cf9
        self.multilib_packages = set()
9a09cf9
17221a3
        # already processed packages
17221a3
        self.completed_add_srpms = set()        # srpms
17221a3
        self.completed_debuginfo = set()        # rpms
17221a3
        self.completed_depsolve = set()         # rpms
17221a3
        self.completed_langpacks = set()        # rpms
17221a3
        self.completed_multilib = set()         # rpms
17221a3
        self.completed_fulltree = set()         # srpms
17221a3
        self.completed_selfhosting = set()      # srpms
754a1bb
        self.completed_greedy_build = set()     # po.sourcerpm
17221a3
17221a3
        self.is_fulltree = self.config.getboolean("pungi", "fulltree")
17221a3
        self.is_selfhosting = self.config.getboolean("pungi", "selfhosting")
17221a3
        self.is_sources = not self.config.getboolean("pungi", "nosource")
17221a3
        self.is_debuginfo = not self.config.getboolean("pungi", "nodebuginfo")
14ffefd
        self.is_resolve_deps = self.config.getboolean("pungi", "resolve_deps")
17221a3
8548993
        self.fulltree_excludes = set(self.ksparser.handler.fulltree_excludes)
8548993
Jesse Keating 7a8ab88
    def _add_yum_repo(self, name, url, mirrorlist=False, groups=True,
17221a3
                      cost=1000, includepkgs=None, excludepkgs=None,
Jesse Keating 7a8ab88
                      proxy=None):
Jesse Keating 7a8ab88
        """This function adds a repo to the yum object.
Jesse Keating 7a8ab88
        name: Name of the repo
Jesse Keating 7a8ab88
        url: Full url to the repo
Jesse Keating 7a8ab88
        mirrorlist: Bool for whether or not url is a mirrorlist
Jesse Keating 7a8ab88
        groups: Bool for whether or not to use groupdata from this repo
Jesse Keating 7a8ab88
        cost: an optional int representing the cost of a repo
Jesse Keating 7a8ab88
        includepkgs: An optional list of includes to use
Jesse Keating 7a8ab88
        excludepkgs: An optional list of excludes to use
Jesse Keating 7a8ab88
        proxy: An optional proxy to use
Jesse Keating 7a8ab88
        """
17221a3
        includepkgs = includepkgs or []
17221a3
        excludepkgs = excludepkgs or []
Jesse Keating 7a8ab88
Jesse Keating 7a8ab88
        self.logger.info('Adding repo %s' % name)
Jesse Keating 7a8ab88
        thisrepo = yum.yumRepo.YumRepository(name)
Jesse Keating 7a8ab88
        thisrepo.name = name
Jesse Keating 7a8ab88
        # add excludes and such here when pykickstart gets them
Jesse Keating 7a8ab88
        if mirrorlist:
Jesse Keating 7a8ab88
            thisrepo.mirrorlist = yum.parser.varReplace(url,
Jesse Keating 7a8ab88
                                                        self.ayum.conf.yumvar)
Jesse Keating 7a8ab88
            self.mirrorlists.append(thisrepo.mirrorlist)
Jesse Keating 7a8ab88
            self.logger.info('Mirrorlist for repo %s is %s' %
Jesse Keating 7a8ab88
                             (thisrepo.name, thisrepo.mirrorlist))
Jesse Keating 7a8ab88
        else:
Jesse Keating 7a8ab88
            thisrepo.baseurl = yum.parser.varReplace(url,
Jesse Keating 7a8ab88
                                                     self.ayum.conf.yumvar)
Jesse Keating 7a8ab88
            self.repos.extend(thisrepo.baseurl)
Jesse Keating 7a8ab88
            self.logger.info('URL for repo %s is %s' %
Jesse Keating 7a8ab88
                             (thisrepo.name, thisrepo.baseurl))
Jesse Keating 7a8ab88
        thisrepo.basecachedir = self.ayum.conf.cachedir
Jesse Keating 7a8ab88
        thisrepo.enablegroups = groups
Jesse Keating 7a8ab88
        # This is until yum uses this failover by default
Jesse Keating 7a8ab88
        thisrepo.failovermethod = 'priority'
Jesse Keating 7a8ab88
        thisrepo.exclude = excludepkgs
Jesse Keating 7a8ab88
        thisrepo.includepkgs = includepkgs
Jesse Keating 7a8ab88
        thisrepo.cost = cost
Jesse Keating 7a8ab88
        # Yum doesn't like proxy being None
Jesse Keating 7a8ab88
        if proxy:
Jesse Keating 7a8ab88
            thisrepo.proxy = proxy
Jesse Keating 7a8ab88
        self.ayum.repos.add(thisrepo)
Jesse Keating 7a8ab88
        self.ayum.repos.enableRepo(thisrepo.id)
Jesse Keating 7a8ab88
        self.ayum._getRepos(thisrepo=thisrepo.id, doSetup=True)
Jesse Keating 7a8ab88
        # Set the repo callback.
Jesse Keating 7a8ab88
        self.ayum.repos.setProgressBar(CallBack())
Jesse Keating 7a8ab88
        self.ayum.repos.callback = CallBack()
Jesse Keating 7a8ab88
        thisrepo.metadata_expire = 0
Jesse Keating 7a8ab88
        thisrepo.mirrorlist_expire = 0
Jesse Keating 7a8ab88
        if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')):
Jesse Keating 7a8ab88
            os.remove(os.path.join(thisrepo.cachedir, 'repomd.xml'))
Jesse Keating 7a8ab88
2244630
    @yumlocked
Jesse Keating e9e0a18
    def _inityum(self):
Jesse Keating e9e0a18
        """Initialize the yum object.  Only needed for certain actions."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Create a yum object to use
Jesse Keating e9e0a18
        self.repos = []
Jesse Keating e9e0a18
        self.mirrorlists = []
Jesse Keating 7326b38
        self.ayum = PungiYum(self.config)
Jesse Keating 3b89e45
        self.ayum.doLoggingSetup(6, 6)
Jesse Keating 3b89e45
        yumconf = yum.config.YumConf()
Jesse Keating 3b89e45
        yumconf.debuglevel = 6
Jesse Keating 3b89e45
        yumconf.errorlevel = 6
Jesse Keating 3ca7a26
        yumconf.cachedir = self.config.get('pungi', 'cachedir')
Martin Gracik 19993ef
        yumconf.persistdir = "/var/lib/yum" # keep at default, gets appended to installroot
Jesse Keating 3b89e45
        yumconf.installroot = os.path.join(self.workdir, 'yumroot')
Jesse Keating 3b89e45
        yumconf.uid = os.geteuid()
Jesse Keating 3b89e45
        yumconf.cache = 0
Jesse Keating 3b89e45
        yumconf.failovermethod = 'priority'
49b530b
        yumconf.deltarpm = 0
Jesse Keating 3b89e45
        yumvars = yum.config._getEnvVar()
Jesse Keating 3ca7a26
        yumvars['releasever'] = self.config.get('pungi', 'version')
5451453
        yumvars['basearch'] = yum.rpmUtils.arch.getBaseArch(myarch=self.tree_arch)
Jesse Keating 3b89e45
        yumconf.yumvar = yumvars
Jesse Keating 3b89e45
        self.ayum._conf = yumconf
Jesse Keating a7a43e4
        # I have no idea why this fixes a traceback, but James says it does.
Jesse Keating a7a43e4
        del self.ayum.prerepoconf
Jesse Keating 3b89e45
        self.ayum.repos.setCacheDir(self.ayum.conf.cachedir)
Jesse Keating 3b89e45
17221a3
        self.ayum.arch.setup_arch(self.yum_arch)
17221a3
Jesse Keating 3b89e45
        # deal with our repos
Jesse Keating 3b89e45
        try:
Jesse Keating 7326b38
            self.ksparser.handler.repo.methodToRepo()
Jesse Keating 3b89e45
        except:
Jesse Keating 3b89e45
            pass
Jesse Keating 3b89e45
Jesse Keating 7326b38
        for repo in self.ksparser.handler.repo.repoList:
Jesse Keating 3b89e45
            if repo.mirrorlist:
Jesse Keating 7a8ab88
                # The not bool() thing is because pykickstart is yes/no on
Jesse Keating 7a8ab88
                # whether to ignore groups, but yum is a yes/no on whether to
Jesse Keating 7a8ab88
                # include groups.  Awkward.
Jesse Keating 7a8ab88
                self._add_yum_repo(repo.name, repo.mirrorlist,
Jesse Keating 7a8ab88
                                   mirrorlist=True,
Jesse Keating 7a8ab88
                                   groups=not bool(repo.ignoregroups),
Jesse Keating 7a8ab88
                                   cost=repo.cost,
Jesse Keating 7a8ab88
                                   includepkgs=repo.includepkgs,
Jesse Keating 7a8ab88
                                   excludepkgs=repo.excludepkgs,
Jesse Keating 7a8ab88
                                   proxy=repo.proxy)
Jesse Keating 3b89e45
            else:
Jesse Keating 7a8ab88
                self._add_yum_repo(repo.name, repo.baseurl,
Jesse Keating 7a8ab88
                                   mirrorlist=False,
Jesse Keating 7a8ab88
                                   groups=not bool(repo.ignoregroups),
Jesse Keating 7a8ab88
                                   cost=repo.cost,
Jesse Keating 7a8ab88
                                   includepkgs=repo.includepkgs,
Jesse Keating 7a8ab88
                                   excludepkgs=repo.excludepkgs,
Jesse Keating 7a8ab88
                                   proxy=repo.proxy)
5451453
5451453
        self.logger.info('Getting sacks for arches %s' % self.valid_arches)
5451453
        self.ayum._getSacks(archlist=self.valid_arches)
Jesse Keating 3b89e45
Jesse Keating cbe4777
    def _filtersrcdebug(self, po):
Jesse Keating 3b89e45
        """Filter out package objects that are of 'src' arch."""
Jesse Keating 3b89e45
Jesse Keating cbe4777
        if po.arch == 'src' or 'debuginfo' in po.name:
Jesse Keating 3b89e45
            return False
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        return True
Jesse Keating 3b89e45
c047fe5
    def add_package(self, po, msg=None):
c047fe5
        if not is_package(po):
c047fe5
            raise ValueError("Not a binary package: %s" % po)
c047fe5
        if msg:
c047fe5
            self.logger.info(msg)
17221a3
        if po not in self.po_list:
17221a3
            self.po_list.add(po)
c047fe5
        self.ayum.install(po)
c047fe5
        self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch)
c047fe5
c047fe5
    def add_debuginfo(self, po, msg=None):
c047fe5
        if not is_debug(po):
c047fe5
            raise ValueError("Not a debuginfog package: %s" % po)
c047fe5
        if msg:
c047fe5
            self.logger.info(msg)
17221a3
        if po not in self.debuginfo_po_list:
17221a3
            self.debuginfo_po_list.add(po)
c047fe5
c047fe5
    def add_source(self, po, msg=None):
c047fe5
        if not is_source(po):
c047fe5
            raise ValueError("Not a source package: %s" % po)
c047fe5
        if msg:
c047fe5
            self.logger.info(msg)
17221a3
        if po not in self.srpm_po_list:
17221a3
            self.srpm_po_list.add(po)
c047fe5
Jesse Keating 3b89e45
    def verifyCachePkg(self, po, path): # Stolen from yum
Jesse Keating 3b89e45
        """check the package checksum vs the cache
Jesse Keating 3b89e45
           return True if pkg is good, False if not"""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        (csum_type, csum) = po.returnIdSum()
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        try:
Jesse Keating 3b89e45
            filesum = yum.misc.checksum(csum_type, path)
Jesse Keating 3b89e45
        except yum.Errors.MiscError:
Jesse Keating 3b89e45
            return False
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        if filesum != csum:
Jesse Keating 3b89e45
            return False
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        return True
Jesse Keating 3b89e45
9d7f82d
    def excludePackages(self, pkg_sack):
9d7f82d
        """exclude packages according to config file"""
9d7f82d
        if not pkg_sack:
9d7f82d
            return pkg_sack
9d7f82d
9d7f82d
        excludes = [] # list of (name, arch, pattern)
9d7f82d
        for i in self.ksparser.handler.packages.excludedList:
bd35d6b
            pattern = i
bd35d6b
            multilib = False
bd35d6b
            if i.endswith(".+"):
bd35d6b
                multilib = True
bd35d6b
                i = i[:-2]
5451453
            name, arch = arch_module.split_name_arch(i)
bd35d6b
            excludes.append((name, arch, pattern, multilib))
9d7f82d
cb6f1db
        for name in self.ksparser.handler.multilib_blacklist:
cb6f1db
            excludes.append((name, None, "multilib-blacklist: %s" % name, True))
cb6f1db
9d7f82d
        for pkg in pkg_sack[:]:
bd35d6b
            for name, arch, exclude_pattern, multilib in excludes:
9d7f82d
                if fnmatch(pkg.name, name):
9d7f82d
                    if not arch or fnmatch(pkg.arch, arch):
bd35d6b
                        if multilib and pkg.arch not in self.valid_multilib_arches:
bd35d6b
                            continue
9d7f82d
                        if pkg.nvra not in self.excluded_pkgs:
9d7f82d
                            self.logger.info("Excluding %s.%s (pattern: %s)" % (pkg.name, pkg.arch, exclude_pattern))
9d7f82d
                            self.excluded_pkgs[pkg.nvra] = pkg
9d7f82d
                        pkg_sack.remove(pkg)
9d7f82d
                        break
9d7f82d
9d7f82d
        return pkg_sack
9d7f82d
17221a3
    def get_package_deps(self, po):
Jesse Keating 3b89e45
        """Add the dependencies for a given package to the
Jesse Keating 3b89e45
           transaction info"""
17221a3
        added = set()
17221a3
        if po in self.completed_depsolve:
17221a3
            return added
17221a3
        self.completed_depsolve.add(po)
5069eb8
Jesse Keating 3b89e45
        self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        reqs = po.requires
Jesse Keating 3b89e45
        provs = po.provides
cd75bc4
Jesse Keating 3b89e45
        for req in reqs:
bd35d6b
            if req in self.resolved_deps:
Jesse Keating 3b89e45
                continue
bd35d6b
            r, f, v = req
Jesse Keating 3b89e45
            if r.startswith('rpmlib(') or r.startswith('config('):
Jesse Keating 3b89e45
                continue
Jesse Keating 3b89e45
            if req in provs:
Jesse Keating 3b89e45
                continue
Jesse Keating 3b89e45
bd35d6b
            try:
8ccc24e
                deps = self.ayum.whatProvides(r, f, v).returnPackages()
9d7f82d
                deps = self.excludePackages(deps)
8ccc24e
                if not deps:
8ccc24e
                    self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
8ccc24e
                    continue
8ccc24e
754a1bb
                if self.greedy_method == "all":
bd35d6b
                    deps = yum.packageSack.ListPackageSack(deps).returnNewestByNameArch()
bd35d6b
                else:
a9581e2
                    found = False
a9581e2
                    for dep in deps:
17221a3
                        if dep in self.po_list:
754a1bb
                            # HACK: there can be builds in the input list on which we want to apply the "build" greedy rules
754a1bb
                            if self.greedy_method == "build" and dep.sourcerpm not in self.completed_greedy_build:
754a1bb
                                break
a9581e2
                            found = True
a9581e2
                            break
a9581e2
                    if found:
a9581e2
                        deps = []
a9581e2
                    else:
754a1bb
                        all_deps = deps
754a1bb
                        deps = [self.ayum._bestPackageFromList(all_deps)]
754a1bb
                        if self.greedy_method == "build":
e1f1231
                            # handle "build" greedy method
754a1bb
                            if deps:
754a1bb
                                build_po = deps[0]
754a1bb
                                if is_package(build_po):
754a1bb
                                    if build_po.arch != "noarch" and build_po.arch not in self.valid_multilib_arches:
754a1bb
                                        all_deps = [ i for i in all_deps if i.arch not in self.valid_multilib_arches ]
754a1bb
                                    for dep in all_deps:
754a1bb
                                        if dep != build_po and dep.sourcerpm == build_po.sourcerpm:
754a1bb
                                            deps.append(dep)
754a1bb
                                            self.completed_greedy_build.add(dep.sourcerpm)
8ccc24e
bd35d6b
                for dep in deps:
77dea0e
                    if dep not in added:
5dc0913
                        msg = 'Added %s.%s (repo: %s) for %s.%s' % (dep.name, dep.arch, dep.repoid, po.name, po.arch)
c047fe5
                        self.add_package(dep, msg)
30c0f35
                        added.add(dep)
8ccc24e
bd35d6b
            except (yum.Errors.InstallError, yum.Errors.YumBaseError), ex:
5dc0913
                self.logger.warn("Unresolvable dependency %s in %s.%s (repo: %s)" % (r, po.name, po.arch, po.repoid))
bd35d6b
                continue
Jesse Keating 3b89e45
            self.resolved_deps[req] = None
bd35d6b
Bill Nottingham f8865c3
        for add in added:
17221a3
            self.get_package_deps(add)
17221a3
        return added
17221a3
17221a3
    def add_langpacks(self, po_list=None):
17221a3
        po_list = po_list or self.po_list
17221a3
        added = set()
Jesse Keating 3b89e45
17221a3
        for po in sorted(po_list):
17221a3
            if po in self.completed_langpacks:
17221a3
                continue
cd75bc4
30c0f35
            # get all langpacks matching the package name
30c0f35
            langpacks = [ i for i in self.langpacks if i["name"] == po.name ]
30c0f35
            if not langpacks:
30c0f35
                continue
30c0f35
17221a3
            self.completed_langpacks.add(po)
17221a3
30c0f35
            for langpack in langpacks:
30c0f35
                pattern = langpack["install"] % "*" # replace '%s' with '*'
17221a3
                exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy())
30c0f35
                matches = filter(self._filtersrcdebug, exactmatched + matched)
30c0f35
                matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ]
30c0f35
                matches = [ i for i in matches if fnmatch(i.name, pattern) ]
30c0f35
30c0f35
                packages_by_name = {}
30c0f35
                for i in matches:
30c0f35
                    packages_by_name.setdefault(i.name, []).append(i)
30c0f35
30c0f35
                for i, pkg_sack in packages_by_name.iteritems():
30c0f35
                    pkg_sack = self.excludePackages(pkg_sack)
30c0f35
                    match = self.ayum._bestPackageFromList(pkg_sack)
5dc0913
                    msg = 'Added langpack %s.%s (repo: %s) for package %s (pattern: %s)' % (match.name, match.arch, match.repoid, po.name, pattern)
30c0f35
                    self.add_package(match, msg)
17221a3
                    self.completed_langpacks.add(match) # assuming langpack doesn't have langpacks
17221a3
                    added.add(match)
30c0f35
30c0f35
        return added
cd75bc4
17221a3
    def add_multilib(self, po_list=None):
17221a3
        po_list = po_list or self.po_list
17221a3
        added = set()
30c0f35
30c0f35
        if not self.multilib_methods:
30c0f35
            return added
30c0f35
17221a3
        for po in sorted(po_list):
17221a3
            if po in self.completed_multilib:
17221a3
                continue
17221a3
30c0f35
            if po.arch in ("noarch", "src", "nosrc"):
30c0f35
                continue
30c0f35
30c0f35
            if po.arch in self.valid_multilib_arches:
30c0f35
                continue
30c0f35
17221a3
            self.completed_multilib.add(po)
17221a3
30c0f35
            matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release)
30c0f35
            matches = [i for i in matches if i.arch in self.valid_multilib_arches]
30c0f35
            if not matches:
30c0f35
                continue
30c0f35
            matches = self.excludePackages(matches)
30c0f35
            match = self.ayum._bestPackageFromList(matches)
30c0f35
            if not match:
30c0f35
                continue
cb6f1db
cb6f1db
            if po.name in self.ksparser.handler.multilib_whitelist:
cb6f1db
                msg = "Added multilib package %s.%s (repo: %s) for package %s.%s (method: %s)" % (match.name, match.arch, match.repoid, po.name, po.arch, "multilib-whitelist")
cb6f1db
                self.add_package(match, msg)
cb6f1db
                self.completed_multilib.add(match)
cb6f1db
                added.add(match)
cb6f1db
                continue
cb6f1db
30c0f35
            method = multilib.po_is_multilib(po, self.multilib_methods)
30c0f35
            if not method:
30c0f35
                continue
5dc0913
            msg = "Added multilib package %s.%s (repo: %s) for package %s.%s (method: %s)" % (match.name, match.arch, match.repoid, po.name, po.arch, method)
30c0f35
            self.add_package(match, msg)
17221a3
            self.completed_multilib.add(match)
17221a3
            added.add(match)
cd75bc4
        return added
cd75bc4
Jesse Keating 3b89e45
    def getPackagesFromGroup(self, group):
Jesse Keating 3b89e45
        """Get a list of package names from a ksparser group object
Jesse Keating 3b89e45
Jesse Keating 3b89e45
            Returns a list of package names"""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        packages = []
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Check if we have the group
Jesse Keating 3b89e45
        if not self.ayum.comps.has_group(group.name):
Jesse Keating 3b89e45
            self.logger.error("Group %s not found in comps!" % group)
Jesse Keating 3b89e45
            return packages
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Get the group object to work with
Jesse Keating 3b89e45
        groupobj = self.ayum.comps.return_group(group.name)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Add the mandatory packages
Jesse Keating 3b89e45
        packages.extend(groupobj.mandatory_packages.keys())
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Add the default packages unless we don't want them
Jesse Keating 3b89e45
        if group.include == 1:
Jesse Keating 3b89e45
            packages.extend(groupobj.default_packages.keys())
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Add the optional packages if we want them
Jesse Keating 3b89e45
        if group.include == 2:
Jesse Keating 3b89e45
            packages.extend(groupobj.default_packages.keys())
Jesse Keating 3b89e45
            packages.extend(groupobj.optional_packages.keys())
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Deal with conditional packages
Jesse Keating 3b89e45
        # Populate a dict with the name of the required package and value
Jesse Keating 3b89e45
        # of the package objects it would bring in.  To be used later if
Jesse Keating 3b89e45
        # we match the conditional.
Jesse Keating 3b89e45
        for condreq, cond in groupobj.conditional_packages.iteritems():
17221a3
            matches = self.ayum.pkgSack.searchNevra(name=condreq)
17221a3
            if matches:
754a1bb
                if self.greedy_method != "all":
754a1bb
                    # works for both "none" and "build" greedy methods
17221a3
                    matches = [self.ayum._bestPackageFromList(matches)]
17221a3
                self.ayum.tsInfo.conditionals.setdefault(cond, []).extend(matches)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        return packages
Jesse Keating 3b89e45
17221a3
    def _addDefaultGroups(self, excludeGroups=None):
Jesse Keating b929b89
        """Cycle through the groups and return at list of the ones that ara
Jesse Keating b929b89
           default."""
17221a3
        excludeGroups = excludeGroups or []
Jesse Keating b929b89
Jesse Keating b929b89
        # This is mostly stolen from anaconda.
Jesse Keating b929b89
        groups = map(lambda x: x.groupid,
Jesse Keating b929b89
            filter(lambda x: x.default, self.ayum.comps.groups))
Jesse Keating 39326ef
f3dafc4
        groups = [x for x in groups if x not in excludeGroups]
Jesse Keating 39326ef
Jesse Keating b929b89
        self.logger.debug('Add default groups %s' % groups)
Jesse Keating b929b89
        return groups
Jesse Keating b929b89
17221a3
    def get_langpacks(self):
cd75bc4
        try:
cd75bc4
            self.langpacks = list(self.ayum.comps.langpacks)
cd75bc4
        except AttributeError:
cd75bc4
            # old yum
cd75bc4
            self.logger.warning("Could not get langpacks via yum.comps. You may need to update yum.")
cd75bc4
            self.langpacks = []
cd75bc4
        except yum.Errors.GroupsError:
cd75bc4
            # no groups or no comps at all
cd75bc4
            self.logger.warning("Could not get langpacks due to missing comps in repodata or --ignoregroups=true option.")
cd75bc4
            self.langpacks = []
cd75bc4
17221a3
    def getPackageObjects(self):
17221a3
        """Cycle through the list of packages and get package object matches."""
17221a3
17221a3
        searchlist = [] # The list of package names/globs to search for
17221a3
        matchdict = {} # A dict of objects to names
17221a3
        excludeGroups = [] # A list of groups for removal defined in the ks file
17221a3
17221a3
        # precompute pkgs and pkg_refs to speed things up
663ea07
        self.all_pkgs = list(set(self.ayum.pkgSack.returnPackages()))
663ea07
        self.all_pkgs = self.excludePackages(self.all_pkgs)
663ea07
663ea07
663ea07
        lookaside_nvrs = set()
663ea07
        for po in self.all_pkgs:
663ea07
            if po.repoid in self.lookaside_repos:
663ea07
                lookaside_nvrs.add(po.nvra)
663ea07
        for po in self.all_pkgs[:]:
663ea07
            if po.repoid not in self.lookaside_repos and po.nvra in lookaside_nvrs:
663ea07
                self.logger.debug("Removed %s (repo: %s), because it's also in a lookaside repo" % (po, po.repoid))
663ea07
                self.all_pkgs.remove(po)
663ea07
17221a3
        self.pkg_refs = yum.packages.buildPkgRefDict(self.all_pkgs, casematch=True)
17221a3
17221a3
        self.get_langpacks()
17221a3
Jesse Keating 3b89e45
        # First remove the excludes
Jesse Keating 3b89e45
        self.ayum.excludePackages()
9776e3c
f3dafc4
        # Get the groups set for removal
f3dafc4
        for group in self.ksparser.handler.packages.excludedGroupList:
f3dafc4
            excludeGroups.append(str(group)[1:])
f3dafc4
9776e3c
        if "core" in [ i.groupid for i in self.ayum.comps.groups ]:
9776e3c
            if "core" not in [ i.name for i in self.ksparser.handler.packages.groupList ]:
9776e3c
                self.logger.warning("The @core group is no longer added by default; Please add @core to the kickstart if you want it in.")
9776e3c
9776e3c
        if "base" in [ i.groupid for i in self.ayum.comps.groups ]:
9776e3c
            if "base" not in [ i.name for i in self.ksparser.handler.packages.groupList ]:
9776e3c
                if self.ksparser.handler.packages.addBase:
9776e3c
                    self.logger.warning("The --nobase kickstart option is no longer supported; Please add @base to the kickstart if you want it in.")
Jesse Keating 3b89e45
Jesse Keating b929b89
        # Check to see if we want all the defaults
Jesse Keating b929b89
        if self.ksparser.handler.packages.default:
f3dafc4
            for group in self._addDefaultGroups(excludeGroups):
Jesse Keating b929b89
                self.ksparser.handler.packages.add(['@%s' % group])
Jesse Keating b929b89
Jesse Keating 3b89e45
        # Get a list of packages from groups
8b1c243
        comps_package_names = set()
Jesse Keating 3b89e45
        for group in self.ksparser.handler.packages.groupList:
8b1c243
            comps_package_names.update(self.getPackagesFromGroup(group))
8b1c243
        searchlist.extend(sorted(comps_package_names))
Jesse Keating 3b89e45
8b1c243
        # Add packages
Jesse Keating 3b89e45
        searchlist.extend(self.ksparser.handler.packages.packageList)
8b1c243
        input_packages = searchlist[:]
8b1c243
8b1c243
        # Add prepopulate packages
8b1c243
        prepopulate_packages = self.ksparser.handler.prepopulate
8b1c243
        searchlist.extend(prepopulate_packages)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Make the search list unique
Jesse Keating 3b89e45
        searchlist = yum.misc.unique(searchlist)
Jesse Keating 3b89e45
bd35d6b
        for name in searchlist:
bd35d6b
            pattern = name
bd35d6b
            multilib = False
bd35d6b
            if name.endswith(".+"):
bd35d6b
                name = name[:-2]
bd35d6b
                multilib = True
8ccc24e
754a1bb
            if self.greedy_method == "all" and name == "system-release":
91f70cb
                # HACK: handles a special case, when system-release virtual provide is specified in the greedy mode
91f70cb
                matches = self.ayum.whatProvides(name, None, None).returnPackages()
91f70cb
            else:
17221a3
                exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy())
91f70cb
                matches = exactmatched + matched
91f70cb
91f70cb
            matches = filter(self._filtersrcdebug, matches)
8ccc24e
754a1bb
            if multilib and self.greedy_method != "all":
bd35d6b
                matches = [ po for po in matches if po.arch in self.valid_multilib_arches ]
8ccc24e
bd35d6b
            if not matches:
bd35d6b
                self.logger.warn('Could not find a match for %s in any configured repo' % pattern)
bd35d6b
                continue
dd6a68e
bd35d6b
            packages_by_name = {}
bd35d6b
            for po in matches:
bd35d6b
                packages_by_name.setdefault(po.name, []).append(po)
bd35d6b
bd35d6b
            for name, packages in packages_by_name.iteritems():
8548993
                packages = self.excludePackages(packages or [])
5dc0913
                if not packages:
5dc0913
                    continue
754a1bb
                if self.greedy_method == "all":
bd35d6b
                    packages = yum.packageSack.ListPackageSack(packages).returnNewestByNameArch()
bd35d6b
                else:
754a1bb
                    # works for both "none" and "build" greedy methods
bd35d6b
                    packages = [self.ayum._bestPackageFromList(packages)]
bd35d6b
8b1c243
                if name in input_packages:
8b1c243
                    self.input_packages.update(packages)
8b1c243
                if name in comps_package_names:
8b1c243
                    self.comps_packages.update(packages)
9a09cf9
bd35d6b
                for po in packages:
bd35d6b
                    msg = 'Found %s.%s' % (po.name, po.arch)
bd35d6b
                    self.add_package(po, msg)
8b1c243
                    name_arch = "%s.%s" % (po.name, po.arch)
8b1c243
                    if name_arch in prepopulate_packages:
8b1c243
                        self.prepopulate_packages.add(po)
Jesse Keating 3b89e45
17221a3
        if not self.po_list:
17221a3
            raise RuntimeError("No packages found")
17221a3
17221a3
        self.logger.info('Finished gathering package objects.')
17221a3
17221a3
    def gather(self):
17221a3
17221a3
        # get package objects according to the input list
17221a3
        self.getPackageObjects()
6ceadeb
        if self.is_sources:
6ceadeb
            self.createSourceHashes()
17221a3
17221a3
        pass_num = 0
17221a3
        added = set()
17221a3
        while 1:
17221a3
            if pass_num > 0 and not added:
17221a3
                break
17221a3
            added = set()
17221a3
            pass_num += 1
17221a3
            self.logger.info("Pass #%s" % pass_num)
Jesse Keating 3b89e45
c363b72
            if self.is_resolve_deps:
c363b72
                # get conditional deps (defined in comps)
c363b72
                for txmbr in self.ayum.tsInfo:
c363b72
                    if not txmbr.po in self.po_list:
2f0b9fc
                        if not is_package(txmbr.po):
2f0b9fc
                            # we don't want sources which can be pulled in, because 'src' arch is part of self.valid_arches
2f0b9fc
                            continue
c363b72
                        self.add_package(txmbr.po)
17221a3
17221a3
            # resolve deps
17221a3
            if self.is_resolve_deps:
17221a3
                for po in sorted(self.po_list):
17221a3
                    added.update(self.get_package_deps(po))
17221a3
6ceadeb
            if self.is_sources:
6ceadeb
                added_srpms = self.add_srpms()
6ceadeb
                added.update(added_srpms)
6ceadeb
17221a3
            if self.is_selfhosting:
17221a3
                for srpm_po in sorted(added_srpms):
17221a3
                    added.update(self.get_package_deps(srpm_po))
17221a3
17221a3
            if self.is_fulltree:
9a09cf9
                new = self.add_fulltree()
9a09cf9
                self.fulltree_packages.update(new)
9a09cf9
                self.fulltree_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
9a09cf9
                added.update(new)
17221a3
            if added:
17221a3
                continue
Jesse Keating 3b89e45
17221a3
            # add langpacks
9a09cf9
            new = self.add_langpacks(self.po_list)
9a09cf9
            self.langpack_packages.update(new)
6ceadeb
            if self.is_sources:
6ceadeb
                self.langpack_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
9a09cf9
            added.update(new)
17221a3
            if added:
17221a3
                continue
17221a3
17221a3
            # add multilib packages
9a09cf9
            new = self.add_multilib(self.po_list)
9a09cf9
            self.multilib_packages.update(new)
9a09cf9
            self.multilib_packages.update([ self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new ])
9a09cf9
            added.update(new)
17221a3
            if added:
17221a3
                continue
17221a3
17221a3
    def get_srpm_po(self, po):
17221a3
        """Given a package object, get a package object for the corresponding source rpm."""
Jesse Keating 3b89e45
17221a3
        # return srpm_po from cache if available
17221a3
        srpm_po = self.sourcerpm_srpmpo_map.get(po.sourcerpm, None)
17221a3
        if srpm_po is not None:
17221a3
            return srpm_po
17221a3
17221a3
        # arch can be "src" or "nosrc"
60803f3
        nvr, arch, _ = po.sourcerpm.rsplit(".", 2)
60803f3
        name, ver, rel = nvr.rsplit('-', 2)
17221a3
17221a3
        # ... but even "nosrc" packages are stored as "src" in repodata
17221a3
        srpm_po_list = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch="src")
17221a3
        if not srpm_po_list:
17221a3
            raise RuntimeError("Cannot find a source rpm for %s" % po.sourcerpm)
17221a3
        srpm_po = srpm_po_list[0]
17221a3
        self.sourcerpm_srpmpo_map[po.sourcerpm] = srpm_po
17221a3
        return srpm_po
Bill Nottingham f46d84e
Bill Nottingham 63adcfc
    def createSourceHashes(self):
Bill Nottingham 63adcfc
        """Create two dicts - one that maps binary POs to source POs, and
Bill Nottingham 63adcfc
           one that maps a single source PO to all binary POs it produces.
Bill Nottingham 63adcfc
           Requires yum still configured."""
Bill Nottingham 63adcfc
        self.src_by_bin = {}
Bill Nottingham 63adcfc
        self.bin_by_src = {}
Bill Nottingham 63adcfc
        self.logger.info("Generating source <-> binary package mappings")
17221a3
        #(dummy1, everything, dummy2) = yum.packages.parsePackages(self.all_pkgs, ['*'], pkgdict=self.pkg_refs.copy())
94235b0
        failed = []
17221a3
        for po in self.all_pkgs:
17221a3
            if is_source(po):
Bill Nottingham 63adcfc
                continue
94235b0
            try:
94235b0
                srpmpo = self.get_srpm_po(po)
94235b0
            except RuntimeError:
94235b0
                failed.append(po.sourcerpm)
94235b0
                continue
94235b0
Bill Nottingham 63adcfc
            self.src_by_bin[po] = srpmpo
Bill Nottingham 63adcfc
            if self.bin_by_src.has_key(srpmpo):
Bill Nottingham 63adcfc
                self.bin_by_src[srpmpo].append(po)
Bill Nottingham 63adcfc
            else:
Bill Nottingham 63adcfc
                self.bin_by_src[srpmpo] = [po]
Bill Nottingham 63adcfc
94235b0
        if failed:
94235b0
            self.logger.info("The following srpms could not be found: %s" % (
94235b0
                pprint.pformat(list(sorted(failed)))))
94235b0
            self.logger.info("Couldn't find %i of %i srpms." % (
94235b0
                len(failed), len(self.src_by_bin)))
94235b0
17221a3
    def add_srpms(self, po_list=None):
Jesse Keating 3b89e45
        """Cycle through the list of package objects and
Jesse Keating 3b89e45
           find the sourcerpm for them.  Requires yum still
Jesse Keating 3b89e45
           configured and a list of package objects"""
Bill Nottingham 28412ff
17221a3
        srpms = set()
17221a3
        po_list = po_list or self.po_list
17221a3
        for po in sorted(po_list):
Adam Miller e8f4b73
            try:
Adam Miller e8f4b73
                srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm]
Adam Miller e8f4b73
            except KeyError:
Adam Miller e8f4b73
                self.logger.error("Cannot get source RPM '%s' for %s" % (po.sourcerpm, po.nvra))
Adam Miller e8f4b73
                srpm_po = None
Adam Miller e8f4b73
Adam Miller e8f4b73
            if srpm_po is None:
17221a3
                continue
9a09cf9
9a09cf9
            # flags
9a09cf9
            if po in self.input_packages:
9a09cf9
                self.input_packages.add(srpm_po)
9a09cf9
            if po in self.fulltree_packages:
9a09cf9
                self.fulltree_packages.add(srpm_po)
9a09cf9
            if po in self.langpack_packages:
9a09cf9
                self.langpack_packages.add(srpm_po)
9a09cf9
            if po in self.multilib_packages:
9a09cf9
                self.multilib_packages.add(srpm_po)
9a09cf9
Adam Miller e8f4b73
            if srpm_po in self.completed_add_srpms:
Adam Miller e8f4b73
                continue
Adam Miller e8f4b73
Adam Miller e8f4b73
            msg = "Added source package %s.%s (repo: %s)" % (srpm_po.name, srpm_po.arch, srpm_po.repoid)
Adam Miller e8f4b73
            self.add_source(srpm_po, msg)
Adam Miller e8f4b73
17221a3
            self.completed_add_srpms.add(srpm_po)
17221a3
            srpms.add(srpm_po)
17221a3
        return srpms
17221a3
17221a3
    def add_fulltree(self, srpm_po_list=None):
Bill Nottingham 0074f79
        """Cycle through all package objects, and add any
Bill Nottingham 0074f79
           that correspond to a source rpm that we are including.
Bill Nottingham 0074f79
           Requires yum still configured and a list of package
Bill Nottingham 0074f79
           objects."""
17221a3
17221a3
        self.logger.info("Completing package set")
17221a3
17221a3
        srpm_po_list = srpm_po_list or self.srpm_po_list
17221a3
        srpms = []
17221a3
        for srpm_po in srpm_po_list:
17221a3
            if srpm_po in self.completed_fulltree:
17221a3
                continue
8548993
            if srpm_po.name not in self.fulltree_excludes:
8548993
                srpms.append(srpm_po)
17221a3
            self.completed_fulltree.add(srpm_po)
17221a3
17221a3
        added = set()
17221a3
        for srpm_po in srpms:
17221a3
            include_native = False
17221a3
            include_multilib = False
17221a3
            has_native = False
17221a3
            has_multilib = False
17221a3
17221a3
            for po in self.excludePackages(self.bin_by_src[srpm_po]):
17221a3
                if not is_package(po):
17221a3
                    continue
17221a3
                if po.arch == "noarch":
17221a3
                    continue
17221a3
                if po not in self.po_list:
17221a3
                    # process only already included packages
a9581e2
                    if po.arch in self.valid_multilib_arches:
17221a3
                        has_multilib = True
a9581e2
                    elif po.arch in self.valid_native_arches:
17221a3
                        has_native = True
17221a3
                    continue
43c9185
                if po.arch in self.valid_multilib_arches and self.greedy_method == "all":
17221a3
                    include_multilib = True
17221a3
                elif po.arch in self.valid_native_arches:
17221a3
                    include_native = True
17221a3
17221a3
            # XXX: this is very fragile!
17221a3
            # Do not make any changes unless you really know what you're doing!
17221a3
            if not include_native:
17221a3
                # if there's no native package already pulled in...
17221a3
                if has_native and not include_multilib:
17221a3
                    # include all native packages, but only if we're not pulling multilib already
17221a3
                    # SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set
17221a3
                    include_native = True
17221a3
                elif has_multilib:
17221a3
                    # SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set
17221a3
                    include_multilib = True
17221a3
17221a3
            for po in self.excludePackages(self.bin_by_src[srpm_po]):
17221a3
                if not is_package(po):
17221a3
                    continue
17221a3
                if po in self.po_list:
17221a3
                    continue
17221a3
                if po.arch != "noarch":
17221a3
                    if po.arch in self.valid_multilib_arches:
17221a3
                        if not include_multilib:
17221a3
                            continue
17221a3
                    if po.arch in self.valid_native_arches:
17221a3
                        if not include_native:
17221a3
                            continue
5dc0913
                msg = "Added %s.%s (repo: %s) to complete package set" % (po.name, po.arch, po.repoid)
17221a3
                self.add_package(po, msg)
17221a3
        return added
c047fe5
Jesse Keating 2679566
    def getDebuginfoList(self):
Jesse Keating 2679566
        """Cycle through the list of package objects and find
Jesse Keating 2679566
           debuginfo rpms for them.  Requires yum still
Jesse Keating 2679566
           configured and a list of package objects"""
Jesse Keating 2679566
17221a3
        added = set()
17221a3
        for po in self.all_pkgs:
c047fe5
            if not is_debug(po):
c047fe5
                continue
c047fe5
c047fe5
            if po.sourcerpm not in self.sourcerpm_arch_map:
c047fe5
                # TODO: print a warning / throw an error
c047fe5
                continue
a9581e2
            if not (set(self.compatible_arches[po.arch]) & set(self.sourcerpm_arch_map[po.sourcerpm]) - set(["noarch"])):
c047fe5
                # skip all incompatible arches
c047fe5
                # this pulls i386 debuginfo for a i686 package for example
c047fe5
                continue
5dc0913
            msg = 'Added debuginfo %s.%s (repo: %s)' % (po.name, po.arch, po.repoid)
c047fe5
            self.add_debuginfo(po, msg)
9a09cf9
9a09cf9
            # flags
9a09cf9
            srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm]
9a09cf9
            if srpm_po in self.input_packages:
9a09cf9
                self.input_packages.add(po)
9a09cf9
            if srpm_po in self.fulltree_packages:
9a09cf9
                self.fulltree_packages.add(po)
9a09cf9
            if srpm_po in self.langpack_packages:
9a09cf9
                self.langpack_packages.add(po)
9a09cf9
            if srpm_po in self.multilib_packages:
9a09cf9
                self.multilib_packages.add(po)
9a09cf9
17221a3
            added.add(po)
17221a3
        return added
Jesse Keating 2679566
Jesse Keating 3b89e45
    def _downloadPackageList(self, polist, relpkgdir):
Jesse Keating 3b89e45
        """Cycle through the list of package objects and
Jesse Keating 3b89e45
           download them from their respective repos."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        downloads = []
Jesse Keating 3b89e45
        for pkg in polist:
Jesse Keating 3b89e45
            downloads.append('%s.%s' % (pkg.name, pkg.arch))
Jesse Keating 3b89e45
            downloads.sort()
Jesse Keating 3b89e45
        self.logger.info("Download list: %s" % downloads)
Jesse Keating 3b89e45
Jesse Keating 3ca7a26
        pkgdir = os.path.join(self.config.get('pungi', 'destdir'),
Jesse Keating 3ca7a26
                              self.config.get('pungi', 'version'),
0633eb2
                              self.config.get('pungi', 'variant'),
Jesse Keating 3b89e45
                              relpkgdir)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Ensure the pkgdir exists, force if requested, and make sure we clean it out
Jesse Keating 3b89e45
        if relpkgdir.endswith('SRPMS'):
Jesse Keating 3b89e45
            # Since we share source dirs with other arches don't clean, but do allow us to use it
f5c6d44
            pungi.util._ensuredir(pkgdir, self.logger, force=True, clean=False)
Jesse Keating 3b89e45
        else:
f5c6d44
            pungi.util._ensuredir(pkgdir, self.logger, force=self.config.getboolean('pungi', 'force'), clean=True)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        probs = self.ayum.downloadPkgs(polist)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        if len(probs.keys()) > 0:
Jesse Keating 3b89e45
            self.logger.error("Errors were encountered while downloading packages.")
Jesse Keating 3b89e45
            for key in probs.keys():
Jesse Keating 3b89e45
                errors = yum.misc.unique(probs[key])
Jesse Keating 3b89e45
                for error in errors:
Jesse Keating 3b89e45
                    self.logger.error("%s: %s" % (key, error))
Jesse Keating 3b89e45
            sys.exit(1)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        for po in polist:
Jesse Keating 3b89e45
            basename = os.path.basename(po.relativepath)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
            local = po.localPkg()
8ed9aaf
            if self.config.getboolean('pungi', 'nohash'):
4596428
                target = os.path.join(pkgdir, basename)
4596428
            else:
4596428
                target = os.path.join(pkgdir, po.name[0].lower(), basename)
8ed9aaf
                # Make sure we have the hashed dir available to link into we only want dirs there to corrospond to packages
8ed9aaf
                # that we are including so we can not just do A-Z 0-9
f5c6d44
                pungi.util._ensuredir(os.path.join(pkgdir, po.name[0].lower()), self.logger, force=True, clean=False)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
            # Link downloaded package in (or link package from file repo)
Jesse Keating 3b89e45
            try:
f5c6d44
                pungi.util._link(local, target, self.logger, force=True)
Jesse Keating 3b89e45
                continue
Jesse Keating 3b89e45
            except:
Jesse Keating 3b89e45
                self.logger.error("Unable to link %s from the yum cache." % po.name)
Jesse Keating 3b89e45
                sys.exit(1)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        self.logger.info('Finished downloading packages.')
Jesse Keating 3b89e45
2244630
    @yumlocked
Jesse Keating 3b89e45
    def downloadPackages(self):
Jesse Keating 3b89e45
        """Download the package objects obtained in getPackageObjects()."""
Jesse Keating 3b89e45
17221a3
        self._downloadPackageList(self.po_list,
5451453
                                  os.path.join(self.tree_arch,
Jesse Keating 3ca7a26
                                               self.config.get('pungi', 'osdir'),
Jesse Keating 3ca7a26
                                               self.config.get('pungi', 'product_path')))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
    def makeCompsFile(self):
Jesse Keating 3b89e45
        """Gather any comps files we can from repos and merge them into one."""
Jesse Keating 3b89e45
0633eb2
        ourcompspath = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version')))
Jesse Keating 3b89e45
6cbf093
        # Filter out things we don't include
6cbf093
        ourgroups = []
6cbf093
        for item in self.ksparser.handler.packages.groupList:
6cbf093
            g = self.ayum.comps.return_group(item.name)
6cbf093
            if g:
6cbf093
                ourgroups.append(g.groupid)
6cbf093
        allgroups = [g.groupid for g in self.ayum.comps.get_groups()]
6cbf093
        for group in allgroups:
6cbf093
            if group not in ourgroups and not self.ayum.comps.return_group(group).langonly:
6cbf093
                self.logger.info('Removing extra group %s from comps file' % (group,))
6cbf093
                del self.ayum.comps._groups[group]
6cbf093
6cbf093
        groups = [g.groupid for g in self.ayum.comps.get_groups()]
6cbf093
        envs = self.ayum.comps.get_environments()
6cbf093
        for env in envs:
6cbf093
            for group in env.groups:
6cbf093
                if group not in groups:
6cbf093
                    self.logger.info('Removing incomplete environment %s from comps file' % (env,))
6cbf093
                    del self.ayum.comps._environments[env.environmentid]
6cbf093
                    break
Jesse Keating 3b89e45
6cbf093
        ourcomps = open(ourcompspath, 'w')
Jesse Keating 3b89e45
        ourcomps.write(self.ayum.comps.xml())
Jesse Keating 3b89e45
        ourcomps.close()
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Disable this until https://bugzilla.redhat.com/show_bug.cgi?id=442097 is fixed.
Jesse Keating 3b89e45
        # Run the xslt filter over our comps file
Jesse Keating 3b89e45
        #compsfilter = ['/usr/bin/xsltproc', '--novalid']
Jesse Keating 3b89e45
        #compsfilter.append('-o')
Jesse Keating 3b89e45
        #compsfilter.append(ourcompspath)
Jesse Keating 3b89e45
        #compsfilter.append('/usr/share/pungi/comps-cleanup.xsl')
Jesse Keating 3b89e45
        #compsfilter.append(ourcompspath)
Jesse Keating 3b89e45
f5c6d44
        #pungi.util._doRunCommand(compsfilter, self.logger)
Jesse Keating 3b89e45
2244630
    @yumlocked
Jesse Keating 3b89e45
    def downloadSRPMs(self):
Jesse Keating 3b89e45
        """Cycle through the list of srpms and
Jesse Keating 3b89e45
           find the package objects for them, Then download them."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # do the downloads
17221a3
        self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS'))
Jesse Keating 3b89e45
2244630
    @yumlocked
Jesse Keating 7c2e859
    def downloadDebuginfo(self):
Jesse Keating 7c2e859
        """Cycle through the list of debuginfo rpms and
Jesse Keating 7c2e859
           download them."""
Jesse Keating 7c2e859
Jesse Keating 7c2e859
        # do the downloads
17221a3
        self._downloadPackageList(self.debuginfo_po_list, os.path.join(self.tree_arch, 'debug'))
Jesse Keating 7c2e859
17221a3
    def _list_packages(self, po_list):
249efe1
        """Cycle through the list of packages and return their paths."""
9a09cf9
        result = []
9a09cf9
        for po in po_list:
9a09cf9
            if po.repoid in self.lookaside_repos:
9a09cf9
                continue
9a09cf9
9a09cf9
            flags = []
9a09cf9
9a09cf9
            # input
9a09cf9
            if po in self.input_packages:
9a09cf9
                flags.append("input")
9a09cf9
8b1c243
            # comps
8b1c243
            if po in self.comps_packages:
8b1c243
                flags.append("comps")
8b1c243
8b1c243
            # prepopulate
8b1c243
            if po in self.prepopulate_packages:
8b1c243
                flags.append("prepopulate")
8b1c243
9a09cf9
            # langpack
9a09cf9
            if po in self.langpack_packages:
9a09cf9
                flags.append("langpack")
9a09cf9
9a09cf9
            # multilib
9a09cf9
            if po in self.multilib_packages:
9a09cf9
                flags.append("multilib")
9a09cf9
9a09cf9
            # fulltree
9a09cf9
            if po in self.fulltree_packages:
9a09cf9
                flags.append("fulltree")
9a09cf9
9a09cf9
            # fulltree-exclude
9a09cf9
            if is_source(po):
9a09cf9
                srpm_name = po.name
9a09cf9
            else:
9a09cf9
                srpm_name = po.sourcerpm.rsplit("-", 2)[0]
9a09cf9
            if srpm_name in self.fulltree_excludes:
9a09cf9
                flags.append("fulltree-exclude")
9a09cf9
9a09cf9
            result.append({
9a09cf9
                "path": os.path.join(po.basepath or "", po.relativepath),
9a09cf9
                "flags": sorted(flags),
9a09cf9
            })
9a09cf9
        result.sort(lambda x, y: cmp(x["path"], y["path"]))
17221a3
        return result
249efe1
17221a3
    def list_packages(self):
249efe1
        """Cycle through the list of RPMs and return their paths."""
17221a3
        return self._list_packages(self.po_list)
249efe1
17221a3
    def list_srpms(self):
249efe1
        """Cycle through the list of SRPMs and return their paths."""
17221a3
        return self._list_packages(self.srpm_po_list)
249efe1
17221a3
    def list_debuginfo(self):
249efe1
        """Cycle through the list of DEBUGINFO RPMs and return their paths."""
17221a3
        return self._list_packages(self.debuginfo_po_list)
17221a3
17221a3
    def _size_packages(self, po_list):
17221a3
        return sum([ po.size for po in po_list if po.repoid not in self.lookaside_repos ])
17221a3
17221a3
    def size_packages(self):
17221a3
        return self._size_packages(self.po_list)
17221a3
17221a3
    def size_srpms(self):
17221a3
        return self._size_packages(self.srpm_po_list)
17221a3
17221a3
    def size_debuginfo(self):
17221a3
        return self._size_packages(self.debuginfo_po_list)
249efe1
Jesse Keating 3b89e45
    def writeinfo(self, line):
Jesse Keating 3b89e45
        """Append a line to the infofile in self.infofile"""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        f=open(self.infofile, "a+")
Jesse Keating 3b89e45
        f.write(line.strip() + "\n")
Jesse Keating 3b89e45
        f.close()
Jesse Keating 3b89e45
Jesse Keating 3b89e45
    def mkrelative(self, subfile):
Jesse Keating 3b89e45
        """Return the relative path for 'subfile' underneath the version dir."""
Jesse Keating 3b89e45
Jesse Keating 3ca7a26
        basedir = os.path.join(self.destdir, self.config.get('pungi', 'version'))
Jesse Keating 3b89e45
        if subfile.startswith(basedir):
Jesse Keating 3b89e45
            return subfile.replace(basedir + os.path.sep, '')
Jesse Keating 3b89e45
        
Jesse Keating 3b89e45
    def _makeMetadata(self, path, cachedir, comps=False, repoview=False, repoviewtitle=False,
9acb9c5
                      baseurl=False, output=False, basedir=False, update=True,
9acb9c5
                      compress_type=None):
Jesse Keating 3b89e45
        """Create repodata and repoview."""
Jesse Keating 3b89e45
        
Jesse Keating 3b89e45
        conf = createrepo.MetaDataConfig()
Jesse Keating 3b89e45
        conf.cachedir = os.path.join(cachedir, 'createrepocache')
Jesse Keating 3b89e45
        conf.update = update
Jesse Keating 6843a12
        conf.unique_md_filenames = True
Jesse Keating 3b89e45
        if output:
Jesse Keating 3b89e45
            conf.outputdir = output
Jesse Keating 3b89e45
        else:
Jesse Keating 3b89e45
            conf.outputdir = path
Jesse Keating 3b89e45
        conf.directory = path
Jesse Keating 3b89e45
        conf.database = True
Jesse Keating 3b89e45
        if comps:
17221a3
            conf.groupfile = comps
Jesse Keating 3b89e45
        if basedir:
Jesse Keating 3b89e45
            conf.basedir = basedir
Jesse Keating 3b89e45
        if baseurl:
Jesse Keating 3b89e45
            conf.baseurl = baseurl
9acb9c5
        if compress_type:
9acb9c5
            conf.compress_type = compress_type
Jesse Keating d5ef2c2
        repomatic = createrepo.MetaDataGenerator(conf)
Jesse Keating 3b89e45
        self.logger.info('Making repodata')
Jesse Keating 3b89e45
        repomatic.doPkgMetadata()
Jesse Keating 3b89e45
        repomatic.doRepoMetadata()
Jesse Keating 3b89e45
        repomatic.doFinalMove()
Jesse Keating 3b89e45
        
Jesse Keating 3b89e45
        if repoview:
Jesse Keating 3b89e45
            # setup the repoview call
Jesse Keating 3b89e45
            repoview = ['/usr/bin/repoview']
Jesse Keating 3b89e45
            repoview.append('--quiet')
Jesse Keating 3b89e45
            
Jesse Keating 3b89e45
            repoview.append('--state-dir')
Jesse Keating 3b89e45
            repoview.append(os.path.join(cachedir, 'repoviewcache'))
Jesse Keating 3b89e45
            
Jesse Keating 3b89e45
            if repoviewtitle:
Jesse Keating 3b89e45
                repoview.append('--title')
Jesse Keating 3b89e45
                repoview.append(repoviewtitle)
Jesse Keating 3b89e45
    
Jesse Keating 3b89e45
            repoview.append(path)
Jesse Keating 3b89e45
    
Jesse Keating 3b89e45
            # run the command
f5c6d44
            pungi.util._doRunCommand(repoview, self.logger)
Jesse Keating 3b89e45
        
Jesse Keating 3b89e45
    def doCreaterepo(self, comps=True):
Jesse Keating 3b89e45
        """Run createrepo to generate repodata in the tree."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        compsfile = None
Jesse Keating 3b89e45
        if comps:
0633eb2
            compsfile = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version')))
Jesse Keating 3b89e45
        
Jesse Keating 3b89e45
        # setup the cache dirs
Jesse Keating 3b89e45
        for target in ['createrepocache', 'repoviewcache']:
f5c6d44
            pungi.util._ensuredir(os.path.join(self.config.get('pungi', 'cachedir'),
Jesse Keating 3b89e45
                                            target), 
Jesse Keating 3b89e45
                               self.logger, 
Jesse Keating 3b89e45
                               force=True)
Jesse Keating 3b89e45
            
0633eb2
        repoviewtitle = '%s %s - %s' % (self.config.get('pungi', 'family'),
Jesse Keating 3ca7a26
                                        self.config.get('pungi', 'version'),
5451453
                                        self.tree_arch)
Jesse Keating 3b89e45
Jesse Keating 3ca7a26
        cachedir = self.config.get('pungi', 'cachedir')
9acb9c5
        compress_type = self.config.get('pungi', 'compress_type')
Jesse Keating 3e62130
Jesse Keating 3b89e45
        # setup the createrepo call
9acb9c5
        self._makeMetadata(self.topdir, cachedir, compsfile,
9acb9c5
                           repoview=True, repoviewtitle=repoviewtitle,
9acb9c5
                           compress_type=compress_type)
Jesse Keating 3e62130
Jesse Keating 3e62130
        # create repodata for debuginfo
Jesse Keating 3ca7a26
        if self.config.getboolean('pungi', 'debuginfo'):
Jesse Keating 3e62130
            path = os.path.join(self.archdir, 'debug')
Jesse Keating 7d92748
            if not os.path.isdir(path):
5451453
                self.logger.debug("No debuginfo for %s" % self.tree_arch)
Jesse Keating 7d92748
                return
9acb9c5
            self._makeMetadata(path, cachedir, repoview=False,
9acb9c5
                               compress_type=compress_type)
Jesse Keating 3b89e45
a5aa03f
    def _shortenVolID(self):
a5aa03f
        """shorten the volume id to make sure its under 32 characters"""
a5aa03f
d865c94
        substitutions = {'Workstation': 'WS',
a5aa03f
                        'Server': 'S',
a5aa03f
                        'Cloud': 'C',
a5aa03f
                        'Alpha': 'A',
a5aa03f
                        'Beta': 'B',
a5aa03f
                        'TC': 'T'}
f5eaa73
        if self.config.get('pungi', 'variant'):
1d87fca
            name = '%s-%s' % (self.config.get('pungi', 'family'), self.config.get('pungi', 'variant'))
f5eaa73
        else:
f5eaa73
            name = self.config.get('pungi', 'family')
a5aa03f
        version = self.config.get('pungi', 'version')
a5aa03f
        arch = self.tree_arch
a5aa03f
d865c94
        for k, v in substitutions.iteritems():
6177cf6
            if k in name:
6177cf6
                name = name.replace(k, v)
6177cf6
            if k in version:
6177cf6
                version = version.replace(k, v)
a5aa03f
        volid = "%s-%s-%s" % (name, version, arch)
a5aa03f
        if len(volid) > 32:
6177cf6
            raise RuntimeError("Volume ID %s is longer than 32 characters" % volid)
a5aa03f
        else:
a5aa03f
            return volid
a5aa03f
Jesse Keating 3b89e45
    def doBuildinstall(self):
Martin Gracik 7deae58
        """Run lorax on the tree."""
Jesse Keating 3b89e45
13526d1
        cmd = ["lorax"]
13526d1
        cmd.extend(["--workdir", self.workdir])
ad18e21
        cmd.extend(["--logfile", os.path.join(self.config.get('pungi', 'destdir'), 'logs/lorax-%s.log' % (self.config.get('pungi', 'arch')))])
Jesse Keating 3b89e45
13526d1
        try:
13526d1
            # Convert url method to a repo
13526d1
            self.ksparser.handler.repo.methodToRepo()
13526d1
        except:
13526d1
            pass
Jesse Keating 3b89e45
13526d1
        for repo in self.ksparser.handler.repo.repoList:
13526d1
            if repo.mirrorlist:
13526d1
                # The not bool() thing is because pykickstart is yes/no on
13526d1
                # whether to ignore groups, but yum is a yes/no on whether to
13526d1
                # include groups.  Awkward.
13526d1
                cmd.extend(["--mirrorlist", repo.mirrorlist])
13526d1
            else:
13526d1
                cmd.extend(["--source", repo.baseurl])
Jesse Keating 3b89e45
13526d1
        # Add the repo in the destdir to our yum object
13526d1
        cmd.extend(["--source", "file://%s" % self.topdir])
13526d1
        cmd.extend(["--product", self.config.get('pungi', 'family')])
13526d1
        cmd.extend(["--version", self.config.get('pungi', 'version')])
13526d1
        cmd.extend(["--release", "%s %s" % (self.config.get('pungi', 'family'), self.config.get('pungi', 'version'))])
13526d1
        if self.config.get('pungi', 'variant'):
13526d1
            cmd.extend(["--variant", self.config.get('pungi', 'variant')])
13526d1
        cmd.extend(["--bugurl", self.config.get('pungi', 'bugurl')])
ca17987
        if self.config.getboolean('pungi', 'isfinal'):
13526d1
            cmd.append("--isfinal")
13526d1
        cmd.extend(["--volid", self._shortenVolID()])
Jesse Keating 379020e
b7c6c80
        # on ppc64 we need to tell lorax to only use ppc64 packages so that the media will run on all 64 bit ppc boxes
44cd5ff
        if self.tree_arch == 'ppc64':
13526d1
            cmd.extend(["--buildarch", "ppc64"])
76c8941
        elif self.tree_arch == 'ppc64le':
13526d1
            cmd.extend(["--buildarch", "ppc64le"])
b7c6c80
2d2a3e8
        # Only supported mac hardware is x86 make sure we only enable mac support on arches that need it
13526d1
        if self.tree_arch in ['x86_64'] and not self.config.getboolean('pungi','nomacboot'):
13526d1
            cmd.append("--macboot")
2d2a3e8
        else:
13526d1
            cmd.append("--nomacboot")
2d2a3e8
2221f66
        try:
13526d1
            cmd.extend(["--conf", self.config.get('lorax', 'conf_file')])
2221f66
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
13526d1
            pass
Martin Gracik 7deae58
c4dd0e7
        try:
13526d1
            cmd.extend(["--installpkgs", self.config.get('lorax', 'installpkgs')])
c4dd0e7
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
13526d1
            pass
13526d1
13526d1
        # Allow the output directory to exist.
13526d1
        cmd.append("--force")
13526d1
13526d1
        # MUST be last in the list
13526d1
        cmd.append(self.topdir)
c4dd0e7
13526d1
        self.logger.info(" ".join(cmd))
07e90f0
        pungi.util._doRunCommand(cmd, self.logger)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # write out the tree data for snake
Jesse Keating 3b89e45
        self.writeinfo('tree: %s' % self.mkrelative(self.topdir))
Jesse Keating 3b89e45
Jesse Keating 74e0147
        # Write out checksums for verifytree
Jesse Keating 74e0147
        # First open the treeinfo file so that we can config parse it
Jesse Keating 74e0147
        treeinfofile = os.path.join(self.topdir, '.treeinfo')
Jesse Keating 74e0147
Jesse Keating 74e0147
        try:
Jesse Keating 74e0147
            treefile = open(treeinfofile, 'r')
Jesse Keating 74e0147
        except IOError:
Jesse Keating 74e0147
            self.logger.error("Could not read .treeinfo file: %s" % treefile)
Jesse Keating 74e0147
            sys.exit(1)
Jesse Keating 74e0147
Jesse Keating 74e0147
        # Create a ConfigParser object out of the contents so that we can
Jesse Keating 74e0147
        # write it back out later and not worry about formatting
Jesse Keating dcf4d90
        treeinfo = MyConfigParser()
Jesse Keating 74e0147
        treeinfo.readfp(treefile)
Jesse Keating 74e0147
        treefile.close()
Jesse Keating 74e0147
        treeinfo.add_section('checksums')
Jesse Keating 74e0147
Jesse Keating 74e0147
        # Create a function to use with os.path.walk to sum the files
Jesse Keating 74e0147
        # basepath is used to make the sum output relative
Jesse Keating 74e0147
        sums = []
Jesse Keating 74e0147
        def getsum(basepath, dir, files):
Jesse Keating 74e0147
            for file in files:
Jesse Keating 74e0147
                path = os.path.join(dir, file)
Jesse Keating 74e0147
                # don't bother summing directories.  Won't work.
Jesse Keating 74e0147
                if os.path.isdir(path):
Jesse Keating 74e0147
                    continue
f5c6d44
                sum = pungi.util._doCheckSum(path, 'sha256', self.logger)
Jesse Keating 74e0147
                outpath = path.replace(basepath, '')
Jesse Keating 74e0147
                sums.append((outpath, sum))
Jesse Keating 74e0147
Jesse Keating 74e0147
        # Walk the os/images path to get sums of all the files
Jesse Keating be8b06d
        os.path.walk(os.path.join(self.topdir, 'images'), getsum, self.topdir + '/')
Jesse Keating bdaf3a6
        
Jesse Keating bdaf3a6
        # Capture PPC images
76c8941
        if self.tree_arch in ['ppc', 'ppc64', 'ppc64le']:
Jesse Keating bdaf3a6
            os.path.walk(os.path.join(self.topdir, 'ppc'), getsum, self.topdir + '/')
Jesse Keating 74e0147
Jesse Keating 74e0147
        # Get a checksum of repomd.xml since it has within it sums for other files
Jesse Keating 74e0147
        repomd = os.path.join(self.topdir, 'repodata', 'repomd.xml')
f5c6d44
        sum = pungi.util._doCheckSum(repomd, 'sha256', self.logger)
Jesse Keating 74e0147
        sums.append((os.path.join('repodata', 'repomd.xml'), sum))
Jesse Keating 74e0147
Jesse Keating 74e0147
        # Now add the sums, and write the config out
Jesse Keating 74e0147
        try:
Jesse Keating 74e0147
            treefile = open(treeinfofile, 'w')
Jesse Keating 74e0147
        except IOError:
Jesse Keating 74e0147
            self.logger.error("Could not open .treeinfo for writing: %s" % treefile)
Jesse Keating 74e0147
            sys.exit(1)
Jesse Keating 74e0147
Jesse Keating 74e0147
        for path, sum in sums:
Jesse Keating 74e0147
            treeinfo.set('checksums', path, sum)
Jesse Keating 74e0147
Jesse Keating 74e0147
        treeinfo.write(treefile)
Jesse Keating 74e0147
        treefile.close()
Jesse Keating 74e0147
Jesse Keating 3b89e45
    def doGetRelnotes(self):
Jesse Keating 3b89e45
        """Get extra files from packages in the tree to put in the topdir of
Jesse Keating 3b89e45
           the tree."""
Jesse Keating 3b89e45
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        docsdir = os.path.join(self.workdir, 'docs')
Jesse Keating 3ca7a26
        relnoterpms = self.config.get('pungi', 'relnotepkgs').split()
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        fileres = []
Jesse Keating 3ca7a26
        for pattern in self.config.get('pungi', 'relnotefilere').split():
Jesse Keating 3b89e45
            fileres.append(re.compile(pattern))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        dirres = []
Jesse Keating 3ca7a26
        for pattern in self.config.get('pungi', 'relnotedirre').split():
Jesse Keating 3b89e45
            dirres.append(re.compile(pattern))
Jesse Keating 3b89e45
f5c6d44
        pungi.util._ensuredir(docsdir, self.logger, force=self.config.getboolean('pungi', 'force'), clean=True)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Expload the packages we list as relnote packages
Jesse Keating 3ca7a26
        pkgs = os.listdir(os.path.join(self.topdir, self.config.get('pungi', 'product_path')))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        rpm2cpio = ['/usr/bin/rpm2cpio']
Jesse Keating 3b89e45
        cpio = ['cpio', '-imud']
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        for pkg in pkgs:
Jesse Keating 3b89e45
            pkgname = pkg.rsplit('-', 2)[0]
Jesse Keating 3b89e45
            for relnoterpm in relnoterpms:
Jesse Keating 3b89e45
                if pkgname == relnoterpm:
Jesse Keating 3ca7a26
                    extraargs = [os.path.join(self.topdir, self.config.get('pungi', 'product_path'), pkg)]
Jesse Keating 3b89e45
                    try:
Jesse Keating 3b89e45
                        p1 = subprocess.Popen(rpm2cpio + extraargs, cwd=docsdir, stdout=subprocess.PIPE)
Jesse Keating 3b89e45
                        (out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout, stdout=subprocess.PIPE, 
Jesse Keating 3b89e45
                            stderr=subprocess.PIPE, universal_newlines=True).communicate()
Jesse Keating 3b89e45
                    except:
Jesse Keating 3b89e45
                        self.logger.error("Got an error from rpm2cpio")
Jesse Keating 3b89e45
                        self.logger.error(err)
Jesse Keating 3b89e45
                        raise
Jesse Keating 3b89e45
Jesse Keating 3b89e45
                    if out:
Jesse Keating 3b89e45
                        self.logger.debug(out)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Walk the tree for our files
Jesse Keating 3b89e45
        for dirpath, dirname, filelist in os.walk(docsdir):
Jesse Keating 3b89e45
            for filename in filelist:
Jesse Keating 3b89e45
                for regex in fileres:
Jesse Keating 3b89e45
                    if regex.match(filename) and not os.path.exists(os.path.join(self.topdir, filename)):
Jesse Keating 3b89e45
                        self.logger.info("Linking release note file %s" % filename)
f5c6d44
                        pungi.util._link(os.path.join(dirpath, filename),
6703e70
                                           os.path.join(self.topdir, filename),
6703e70
                                           self.logger,
6703e70
                                           force=self.config.getboolean('pungi',
6703e70
                                                                        'force'))
Jesse Keating 3b89e45
                        self.common_files.append(filename)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Walk the tree for our dirs
Jesse Keating 3b89e45
        for dirpath, dirname, filelist in os.walk(docsdir):
Jesse Keating 3b89e45
            for directory in dirname:
Jesse Keating 3b89e45
                for regex in dirres:
Jesse Keating 3b89e45
                    if regex.match(directory) and not os.path.exists(os.path.join(self.topdir, directory)):
Jesse Keating 3b89e45
                        self.logger.info("Copying release note dir %s" % directory)
Jesse Keating 3b89e45
                        shutil.copytree(os.path.join(dirpath, directory), os.path.join(self.topdir, directory))
Jesse Keating 3b89e45
        
Jesse Keating e073e0a
    def _doIsoChecksum(self, path, csumfile):
Jesse Keating e073e0a
        """Simple function to wrap creating checksums of iso files."""
Jesse Keating ec8b64e
Jesse Keating ec8b64e
        try:
Jesse Keating e073e0a
            checkfile = open(csumfile, 'a')
Jesse Keating ec8b64e
        except IOError:
Jesse Keating e073e0a
            self.logger.error("Could not open checksum file: %s" % csumfile)
Jesse Keating ec8b64e
Jesse Keating e073e0a
        self.logger.info("Generating checksum of %s" % path)
f5c6d44
        checksum = pungi.util._doCheckSum(path, 'sha256', self.logger)
Jesse Keating e073e0a
        if checksum:
64b6c80
            checkfile.write("SHA256 (%s) = %s\n" % (os.path.basename(path), checksum.replace('sha256:', '')))
Jesse Keating ec8b64e
        else:
Jesse Keating e073e0a
            self.logger.error('Failed to generate checksum for %s' % checkfile)
Jesse Keating ec8b64e
            sys.exit(1)
Jesse Keating e073e0a
        checkfile.close()
Jesse Keating ec8b64e
Jesse Keating d5ef2c2
    def doCreateIsos(self):
Jesse Keating d5ef2c2
        """Create iso of the tree."""
Jesse Keating 3b89e45
44cd5ff
        if self.tree_arch.startswith('arm'):
8d54c4b
            self.logger.info("ARCH: arm, not doing doCreateIsos().")
8d54c4b
            return
Jesse Keating 3b89e45
17221a3
        isolist = []
4b7685d
        ppcbootinfo = '/usr/share/lorax/config_files/ppc'
Jesse Keating 3b89e45
f5c6d44
        pungi.util._ensuredir(self.isodir, self.logger,
Jesse Keating 3ca7a26
                           force=self.config.getboolean('pungi', 'force'),
Jesse Keating 3b89e45
                           clean=True) # This is risky...
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # setup the base command
Jesse Keating 3b89e45
        mkisofs = ['/usr/bin/mkisofs']
Jesse Keating 179d3cd
        mkisofs.extend(['-v', '-U', '-J', '-R', '-T', '-m', 'repoview', '-m', 'boot.iso']) # common mkisofs flags
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        x86bootargs = ['-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', 
Jesse Keating 3b89e45
            '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table']
Jesse Keating 3b89e45
Jesse Keating 5b0c971
        efibootargs = ['-eltorito-alt-boot', '-e', 'images/efiboot.img',
Jesse Keating 5b0c971
                       '-no-emul-boot']
Jesse Keating 5b0c971
9cf7418
        macbootargs = ['-eltorito-alt-boot', '-e', 'images/macboot.img',
9cf7418
                       '-no-emul-boot']
9cf7418
Jesse Keating 3b89e45
        ia64bootargs = ['-b', 'images/boot.img', '-no-emul-boot']
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        ppcbootargs = ['-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop', '-allow-multidot', '-chrp-boot']
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        ppcbootargs.append('-map')
4b7685d
        ppcbootargs.append(os.path.join(ppcbootinfo, 'mapping'))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        ppcbootargs.append('-hfs-bless') # must be last
Jesse Keating 3b89e45
9cf7418
        isohybrid = ['/usr/bin/isohybrid']
9cf7418
Jesse Keating 3b89e45
        # Check the size of the tree
Jesse Keating 3b89e45
        # This size checking method may be bunk, accepting patches...
5451453
        if not self.tree_arch == 'source':
Jesse Keating 3b89e45
            treesize = int(subprocess.Popen(mkisofs + ['-print-size', '-quiet', self.topdir], stdout=subprocess.PIPE).communicate()[0])
Jesse Keating 3b89e45
        else:
Jesse Keating 3ca7a26
            srcdir = os.path.join(self.config.get('pungi', 'destdir'), self.config.get('pungi', 'version'), 
0633eb2
                                  self.config.get('pungi', 'variant'), 'source', 'SRPMS')
Jesse Keating 3b89e45
Jesse Keating 3b89e45
            treesize = int(subprocess.Popen(mkisofs + ['-print-size', '-quiet', srcdir], stdout=subprocess.PIPE).communicate()[0])
Jesse Keating 3b89e45
        # Size returned is 2KiB clusters or some such.  This translates that to MiB.
Jesse Keating 3b89e45
        treesize = treesize * 2048 / 1024 / 1024
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        if treesize > 700: # we're larger than a 700meg CD
76ba16d
            isoname = '%s-DVD-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'), self.tree_arch,
76ba16d
                self.config.get('pungi', 'version'))
Jesse Keating 3b89e45
        else:
76ba16d
            isoname = '%s-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'), self.tree_arch,
76ba16d
                self.config.get('pungi', 'version'))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        isofile = os.path.join(self.isodir, isoname)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # setup the extra mkisofs args
Jesse Keating 3b89e45
        extraargs = []
Jesse Keating 3b89e45
5451453
        if self.tree_arch == 'i386' or self.tree_arch == 'x86_64':
Jesse Keating 3b89e45
            extraargs.extend(x86bootargs)
5451453
            if self.tree_arch == 'x86_64':
Jesse Keating 5b0c971
                extraargs.extend(efibootargs)
9cf7418
                isohybrid.append('-u')
9cf7418
                if os.path.exists(os.path.join(self.topdir, 'images', 'macboot.img')):
9cf7418
                    extraargs.extend(macbootargs)
9cf7418
                    isohybrid.append('-m')
5451453
        elif self.tree_arch == 'ia64':
Jesse Keating 3b89e45
            extraargs.extend(ia64bootargs)
5451453
        elif self.tree_arch.startswith('ppc'):
Jesse Keating 3b89e45
            extraargs.extend(ppcbootargs)
Jesse Keating 3b89e45
            extraargs.append(os.path.join(self.topdir, "ppc/mac"))
33ebc4e
        elif self.tree_arch.startswith('aarch64'):
33ebc4e
            extraargs.extend(efibootargs)
Jesse Keating 3b89e45
Will Woods 32eacf5
        # NOTE: if this doesn't match what's in the bootloader config, the
Will Woods 32eacf5
        # image won't be bootable!
Jesse Keating 3b89e45
        extraargs.append('-V')
6a26176
        extraargs.append(self._shortenVolID())
Jesse Keating 3b89e45
Jeroen van Meeuwen (Fedora Unity) c4004dd
        extraargs.extend(['-o', isofile])
9cf7418
9cf7418
        isohybrid.append(isofile)
9cf7418
5451453
        if not self.tree_arch == 'source':
Jesse Keating 3b89e45
            extraargs.append(self.topdir)
Jesse Keating 3b89e45
        else:
Jesse Keating 3b89e45
            extraargs.append(os.path.join(self.archdir, 'SRPMS'))
Jesse Keating 3b89e45
5c9d28d
        if self.config.get('pungi', 'no_dvd') == "False":
a313fa8
            # run the command
f5c6d44
            pungi.util._doRunCommand(mkisofs + extraargs, self.logger)
Jesse Keating 3b89e45
5c9d28d
            # Run isohybrid on the iso as long as its not the source iso
5c9d28d
            if os.path.exists("/usr/bin/isohybrid") and not self.tree_arch == 'source':
f5c6d44
                pungi.util._doRunCommand(isohybrid, self.logger)
Tom "spot" Callaway 7e65e60
5c9d28d
            # implant md5 for mediacheck on all but source arches
5c9d28d
            if not self.tree_arch == 'source':
f5c6d44
                pungi.util._doRunCommand(['/usr/bin/implantisomd5', isofile], self.logger)
Jesse Keating 3b89e45
Jesse Keating 7dc76be
        # shove the checksum into a file
Jesse Keating 7dc76be
        csumfile = os.path.join(self.isodir, '%s-%s-%s-CHECKSUM' % (
Jesse Keating 7dc76be
                                self.config.get('pungi', 'iso_basename'),
Jesse Keating 7dc76be
                                self.config.get('pungi', 'version'),
5451453
                                self.tree_arch))
Jesse Keating be078f9
        # Write a line about what checksums are used.
Jesse Keating be078f9
        # sha256sum is magic...
Jesse Keating be078f9
        file = open(csumfile, 'w')
Jesse Keating be078f9
        file.write('# The image checksum(s) are generated with sha256sum.\n')
Jesse Keating be078f9
        file.close()
5c9d28d
        if self.config.get('pungi', 'no_dvd') == "False":
a313fa8
            self._doIsoChecksum(isofile, csumfile)
Jesse Keating 3b89e45
a313fa8
            # Write out a line describing the media
a313fa8
            self.writeinfo('media: %s' % self.mkrelative(isofile))
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        # Now link the boot iso
5451453
        if not self.tree_arch == 'source' and \
Jesse Keating 3b89e45
        os.path.exists(os.path.join(self.topdir, 'images', 'boot.iso')):
76ba16d
            isoname = '%s-netinst-%s-%s.iso' % (self.config.get('pungi', 'iso_basename'),
76ba16d
                self.tree_arch, self.config.get('pungi', 'version'))
Jesse Keating 3b89e45
            isofile = os.path.join(self.isodir, isoname)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
            # link the boot iso to the iso dir
f5c6d44
            pungi.util._link(os.path.join(self.topdir, 'images', 'boot.iso'), isofile, self.logger)
Jesse Keating 3b89e45
Jesse Keating 7dc76be
            # shove the checksum into a file
Jesse Keating 7dc76be
            self._doIsoChecksum(isofile, csumfile)
Jesse Keating 3b89e45
Jesse Keating 3b89e45
        self.logger.info("CreateIsos is done.")