From 793e9f3d067d2fe3a3aca0fdafcc8aea3c8c915a Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Jun 10 2015 14:53:31 +0000 Subject: [PATCH 1/2] Add some basic testing, dummy rpm creation, and a testing README --- diff --git a/tests/README b/tests/README new file mode 100644 index 0000000..d615a05 --- /dev/null +++ b/tests/README @@ -0,0 +1,44 @@ +Running pungi4 tests + +Pungi4 is an utility to perform composes of rpms, as such we will need some +rpms to perform composes on. + +In this directory you will find a small utility called 'createtestdata.py' that +takes two arguments, first is the package manifest JSON file that will contain +the list of packages that are architecture specific, the list of architectures +to build them for, and finally the packages that are noarch. (Don't worry about +your dev machine being the wrong arch as needed to be produced, we're using an +utility library called rpmfluff[0] that's faking a lot of this for us) + +Before we run any tests we will need to create a repo to work with using the +'createtestdata.py' script. NOTE: This script requires both the 'python-click +and 'python-rpmfluff' packages. + + $ ./createtestdata.py --pkgfile pkgs.json --outdir . + +You will now find a directory called ./repo in the current directory, this is +setup exactly as the pungi tests need it and you are now ready to run tests. + +Next you will find a hand full of scripts named test_* and these are to run the +actual tests. There is also a small wrapper called 'run_all_tests.sh' that will +run all scripts prefixed with test_* in the current directory. + + $ ./run_all_tests.sh + +Or alternatively, select the tests you want to run and run them one by one: + + $ ./test_arch.py + ...... + ---------------------------------------------------------------------- + Ran 6 tests in 0.001s + + OK + + $ ./test_pathmatch.py + ... + ---------------------------------------------------------------------- + Ran 3 tests in 0.001s + + OK + +[0] - https://fedorahosted.org/rpmfluff/ diff --git a/tests/createtestdata.py b/tests/createtestdata.py new file mode 100755 index 0000000..6cd9d82 --- /dev/null +++ b/tests/createtestdata.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import tempfile +import shutil +import libcomps +from contextlib import contextmanager + +#import pungi.phases.pkgsets.pkgsets +from rpmfluff import SimpleRpmBuild + +# helpers for creating RPMs to test with +@contextmanager +def in_tempdir(outdir, prefix='_'): + """ + py:class:: in_tempdir(prefix='_') + + Context manager for the rpmbuild tempdir + """ + oldcwd = os.getcwd() + tmpdir = tempfile.mkdtemp(prefix=prefix) + os.chdir(tmpdir) + yield + os.chdir(oldcwd) + shutil.rmtree(tmpdir) + +@contextmanager +def in_dir(directory): + """ + py:class:: in_dir(dir) + + Context manager to handle things in a generic method + """ + oldcwd = os.getcwd() + tmpdir = tempfile.mkdtemp() + os.chdir(tmpdir) + yield + os.chdir(oldcwd) + +def make_rpm(outdir, archlist, name, version='1.0', release='1'): + """ + py:function:: make_rpm(outdir, name='test', version='1.0', release='1', archlist=None) + + Create the fake test rpms + """ + + if (archlist is None): + raise TypeError( "No defined architectures for make_rpm") + + abs_outdir = os.path.abspath(outdir) + + if not os.path.isdir(abs_outdir): + os.mkdir(abs_outdir) + + p = SimpleRpmBuild(name, version, release, archlist) + with in_tempdir(abs_outdir, prefix="tmppkgs"): + p.make() + + srpm_outdir = os.path.join( + abs_outdir, + "repo", + "src", + ) + + if not os.path.isdir(srpm_outdir): + os.makedirs(srpm_outdir) + + srpmfile = p.get_built_srpm() + src_outfile = os.path.join( + os.path.abspath(abs_outdir), + "repo", + 'src', + os.path.basename(srpmfile) + ) + shutil.move(srpmfile, src_outfile) + + for arch in archlist: + + arch_outdir = os.path.join( + abs_outdir, + "repo", + arch, + ) + if not os.path.isdir(arch_outdir): + os.makedirs(arch_outdir) + + + rpmfile = p.get_built_rpm(arch) + bin_outfile = os.path.join( + os.path.abspath(abs_outdir), + "repo", + arch, + os.path.basename(rpmfile) + ) + shutil.move(rpmfile, bin_outfile) + return p + +def get_rpm_list_from_comps(compspath): + """ + py:function:: get_rpm_list_from_comps(compspath) + + Return a list of rpms from a compsfile + """ + + pkg_list = [] + + comps = libcomps.Comps() + comps.fromxml_f(compspath) + + for group in comps.groups: + for pkg in comps.groups[group.id].packages: + pkg_list.append(pkg.name) + + return pkg_list + + +if __name__ == "__main__": + import click + import json + + @click.command() + @click.option('--pkgfile', default=None, required=True, + help="Path to json pkg file") + @click.option('--outdir', default=None, required=True, + help="Directory to create temp dummy repo") + def createtestdata(pkgfile, outdir): + pkgdata = json.loads(open(pkgfile,'r').read()) + for pkg in pkgdata['archpkgs']: + make_rpm(outdir, pkgdata['archs'], pkg) + for pkg in pkgdata['noarchpkgs']: + make_rpm(outdir, ['noarch'], pkg) + + os.popen('/usr/bin/createrepo %s' % os.path.join(outdir, "repo")) + + createtestdata() diff --git a/tests/dummy-comps.xml b/tests/dummy-comps.xml new file mode 100644 index 0000000..9bb6595 --- /dev/null +++ b/tests/dummy-comps.xml @@ -0,0 +1,163 @@ + + + + + + + + core + Core + Smallest possible installation + true + false + + dummy-bash + + + + + standard + Standard + Common set of utilities that extend the minimal installation. + false + true + + dummy-lvm2 + + + + + text-internet + Text-based Internet + This group includes text-based email, Web, and chat clients. These applications do not require the X Window System. + false + true + + dummy-elinks + dummy-tftp + + + + + firefox + Firefox Web Browser + The Firefox web browser + false + false + + dummy-firefox + dummy-icedtea-web + + + + + skype + Skype + Free internet telephony + false + true + + dummy-skype + + + + + resilient-storage + Resilient Storage + Clustered storage, including the GFS2 filesystem. + false + true + + dummy-gfs2-utils + dummy-lvm2-cluster + dummy-pacemaker + dummy-resource-agents + + + + + gluster + Gluster + GlusterFS support packages + false + true + + dummy-glusterfs-resource-agents + + + + + basic-desktop + Desktop + Basic Desktop packages + true + true + + dummy-imsettings-gnome + + + + + + + minimal + Minimal install + Basic functionality. + 99 + + core + + + + + + + desktop + Desktop + Desktop. + 10 + + core + standard + basic-desktop + + + + + + + empty + Empty + Should not appear in the repos. + 10 + + does-not-exist + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/dummy-pungi.conf b/tests/dummy-pungi.conf new file mode 100644 index 0000000..09d65f0 --- /dev/null +++ b/tests/dummy-pungi.conf @@ -0,0 +1,139 @@ +# PRODUCT (RELEASE) INFO +product_name = "Dummy Product" +product_short = "DP" +product_version = "1.0" +product_is_layered = False +product_type = "ga" + + +# GENERAL SETTINGS +bootable = False +comps_file = "dummy-comps.xml" +variants_file = "dummy-variants.xml" +sigkeys = [None] # None = unsigned + +# limit tree architectures +# if undefined, all architectures from variants.xml will be included +atree_arches = ["x86_64"] + +# limit tree variants +# if undefined, all variants from variants.xml will be included +#tree_variants = ["Server"] + +multilib_arches = ["ppc64", "x86_64", "s390x"] +multilib_methods = ["devel", "runtime"] # devel (recommended), all, base, file, kernel, none, runtime + + +# RUNROOT settings +runroot = False +#runroot_channel = "" +#runroot_tag = "" + + +# PKGSET +pkgset_source = "repos" # koji, repos + +# PKGSET - REPOS +# pkgset_repos format: {arch: [repo1_url, repo2_url, ...]} +pkgset_repos = { + "i386": [ + "repo", + ], + "x86_64": [ + "repo", + ], + "s390x": [ + "repo", + ], +} + +# PKGSET - KOJI +#pkgset_koji_path_prefix = "/mnt/koji" +#pkgset_koji_url = "" +#pkgset_koji_tag = "" + + +# GATHER +gather_source = "comps" +gather_method = "deps" +check_deps = False +greedy_method = "build" + +# fomat: [(variant_uid_regex, {arch|*: [repos]})] +# gather_lookaside_repos = [] + +# GATHER - JSON +# format: {variant_uid: {arch: package: [arch1, arch2, None (for any arch)]}} +#gather_source_mapping = "/path/to/mapping.json" + + +# CREATEREPO +# TODO: checksum type - mandatory +createrepo_c = True + + +# BUILDINSTALL + + +# PRODUCTIMG + + +# CREATEISO +create_optional_isos = False +symlink_isos_to = None + + +# fomat: [(variant_uid_regex, {arch|*: [packages]})] +additional_packages = [ + ('^Server$', { + '*': [ +# 'dummy-lvm2-devel', + 'dummy-libtool', + ], + }), + ('^Client-optional$', { + '*': [ + 'dummy-httpd', + ], + }), +] + +filter_packages = [ + ('^.*$', { + '*': [ + 'dummy-pacemaker', + ], + }), + ('^Client$', { + '*': [ + 'dummy-httpd', + ], + }), + ('^Server-optional$', { + '*': [ + 'dummy-httpd.i686', + ], + }), + ('^.*-ResilientStorage$', { + '*': [ + 'dummy-glusterfs-resource-agents', + ], + }), +] + + +# format: {arch|*: [packages]} +multilib_blacklist = { + "*": [ + "kernel-devel", + "httpd-devel", + "*", +# "dummy-glibc", + ], +} + +multilib_whitelist = { + "*": [ + "dummy-glibc", + ], +} diff --git a/tests/dummy-variants.xml b/tests/dummy-variants.xml new file mode 100644 index 0000000..4997403 --- /dev/null +++ b/tests/dummy-variants.xml @@ -0,0 +1,70 @@ + + + + + + + x86_64 + + + resilient-storage + + + + + + + x86_64 + + + gluster + + + + + + i386 + x86_64 + + + core + standard + text-internet + firefox + skype + + + minimal + desktop + + + + + + x86_64 + s390x + + + core + standard + text-internet + + + minimal + + + + + + + x86_64 + s390x + + + firefox + + + + + + diff --git a/tests/pkgs.json b/tests/pkgs.json new file mode 100644 index 0000000..f093055 --- /dev/null +++ b/tests/pkgs.json @@ -0,0 +1,116 @@ +{ + "archpkgs" : [ + "dummy-AdobeReader_enu", + "dummy-atlas", + "dummy-atlas-3dnow", + "dummy-atlas-3dnow-devel", + "dummy-atlas-devel", + "dummy-atlas-sse", + "dummy-atlas-sse2", + "dummy-atlas-sse2-devel", + "dummy-atlas-sse3", + "dummy-atlas-sse3-devel", + "dummy-atlas-sse-devel", + "dummy-atlas-z10", + "dummy-atlas-z10-devel", + "dummy-atlas-z196", + "dummy-atlas-z196-devel", + "dummy-basesystem", + "dummy-bash", + "dummy-bash-debuginfo", + "dummy-bash-doc", + "dummy-elinks", + "dummy-elinks-debuginfo", + "dummy-fcoe-target-utils", + "dummy-filesystem", + "dummy-firefox", + "dummy-firefox-debuginfo", + "dummy-foo32", + "dummy-foo32-doc", + "dummy-freeipa", + "dummy-freeipa-server", + "dummy-gfs2-utils", + "dummy-gfs2-utils-debuginfo", + "dummy-glibc", + "dummy-glibc-common", + "dummy-glibc-debuginfo", + "dummy-glibc-debuginfo-common", + "dummy-glusterfs-resource-agents", + "dummy-httpd", + "dummy-httpd-debuginfo", + "dummy-imsettings", + "dummy-imsettings-gnome", + "dummy-imsettings-qt", + "dummy-ipw3945-kmod", + "dummy-ipw3945-kmod-debuginfo", + "dummy-kernel", + "dummy-kernel-doc", + "dummy-kernel-headers", + "dummy-kmod-ipw3945", + "dummy-kmod-ipw3945-xen", + "dummy-krb5", + "dummy-krb5-debuginfo", + "dummy-krb5-devel", + "dummy-krb5-libs", + "dummy-krb5-workstation", + "dummy-lvm2", + "dummy-lvm2-cluster", + "dummy-lvm2-debuginfo", + "dummy-lvm2-devel", + "dummy-lvm2-libs", + "dummy-nscd", + "dummy-postfix", + "dummy-postfix-debuginfo", + "dummy-release-client", + "dummy-release-client-workstation", + "dummy-release-notes", + "dummy-release-notes-cs-CZ", + "dummy-release-notes-en-US", + "dummy-release-server", + "dummy-resource-agents", + "dummy-resource-agents-debuginfo", + "dummy-selinux-policy", + "dummy-selinux-policy-doc", + "dummy-selinux-policy-minimal", + "dummy-selinux-policy-mls", + "dummy-selinux-policy-targeted", + "dummy-sendmail", + "dummy-sendmail-debuginfo", + "dummy-skype", + "dummy-tftp", + "dummy-tftp-debuginfo", + "dummy-vacation", + "dummy-vacation-debuginfo", + "dummy-xulrunner", + "dummy-xulrunner-debuginfo" + ], + + "archs" : [ + "i486", + "i586", + "i686", + "ppc", + "ppc64", + "s390", + "s390x", + "x86_64", + "src" + ], + + "noarchpkgs" : [ + "dummy-basesystem", + "dummy-bash-doc", + "dummy-bash-doc", + "dummy-fcoe-target-utils", + "dummy-foo32-doc", + "dummy-kernel-doc", + "dummy-release-notes", + "dummy-release-notes-cs-CZ", + "dummy-release-notes-en-US", + "dummy-selinux-policy-doc", + "dummy-selinux-policy-minimal", + "dummy-selinux-policy-mls", + "dummy-selinux-policy-targeted" + ] + +} diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh new file mode 100755 index 0000000..2adc027 --- /dev/null +++ b/tests/run_all_tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Thin wrapper to run all tests +for t in $(dirname $0)/test_* +do + $t +done diff --git a/tests/test_compose.sh b/tests/test_compose.sh new file mode 100755 index 0000000..012b2d1 --- /dev/null +++ b/tests/test_compose.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +export PYTHONPATH=$(pwd)/pmd:$(pwd)/../ +export PATH=$(pwd)/../bin:$PATH + +mkdir -p _composes + +pungi-koji \ +--target-dir=_composes \ +--old-composes=_composes \ +--config=dummy-pungi.conf \ +--test From e8f4b7334ec0d68f2248ec98d27fabe188ee4165 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Jun 10 2015 15:03:57 +0000 Subject: [PATCH 2/2] pull in gather.py patches from dmach for test compose --- diff --git a/pungi/gather.py b/pungi/gather.py index 9c9f660..bcc2861 100644 --- a/pungi/gather.py +++ b/pungi/gather.py @@ -922,7 +922,6 @@ class Pungi(PungiBase): pprint.pformat(list(sorted(failed))))) self.logger.info("Couldn't find %i of %i srpms." % ( len(failed), len(self.src_by_bin))) - raise RuntimeError("Could not find all srpms.") def add_srpms(self, po_list=None): """Cycle through the list of package objects and @@ -932,11 +931,14 @@ class Pungi(PungiBase): srpms = set() po_list = po_list or self.po_list for po in sorted(po_list): - srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm] - if srpm_po in self.completed_add_srpms: + try: + srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm] + except KeyError: + self.logger.error("Cannot get source RPM '%s' for %s" % (po.sourcerpm, po.nvra)) + srpm_po = None + + if srpm_po is None: continue - msg = "Added source package %s.%s (repo: %s)" % (srpm_po.name, srpm_po.arch, srpm_po.repoid) - self.add_source(srpm_po, msg) # flags if po in self.input_packages: @@ -948,6 +950,12 @@ class Pungi(PungiBase): if po in self.multilib_packages: self.multilib_packages.add(srpm_po) + if srpm_po in self.completed_add_srpms: + continue + + msg = "Added source package %s.%s (repo: %s)" % (srpm_po.name, srpm_po.arch, srpm_po.repoid) + self.add_source(srpm_po, msg) + self.completed_add_srpms.add(srpm_po) srpms.add(srpm_po) return srpms