From 27e845c2adc51c9b7ef26bc3254b46730fa98199 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sep 15 2017 15:23:01 +0000 Subject: Merge #684 `Fix #634: Conflict with packages filtered out from the build-requires to ensure they won't appear in a buildroot.` --- diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 285516c..0c8edab 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -137,6 +137,23 @@ class KojiModuleBuilder(GenericBuilder): version = "0.1" release = "1" today = datetime.date.today().strftime('%a %b %d %Y') + mmd = module_build.mmd() + + # Generate "Conflicts: name = version-release". This is workaround for + # Koji build system, because it does not filter out RPMs from the + # build-requires based on their "mmd.filter.rpms". So we set the + # module-build-macros to conflict with these filtered RPMs to ensure + # they won't be installed to buildroot. + filter_conflicts = "" + for req_name, req_data in mmd.xmd["mbs"]["buildrequires"].items(): + if req_data["filtered_rpms"]: + filter_conflicts += "# Filtered rpms from %s module:\n" % ( + req_name) + for nvr in req_data["filtered_rpms"]: + parsed_nvr = kobo.rpmlib.parse_nvr(nvr) + filter_conflicts += "Conflicts: %s = %s:%s-%s\n" % ( + parsed_nvr["name"], parsed_nvr["epoch"], + parsed_nvr["version"], parsed_nvr["release"]) spec_content = """ %global dist {disttag} @@ -156,6 +173,8 @@ URL: http://fedoraproject.org Source1: macros.modules +{filter_conflicts} + %description This package is used for building modules with a different dist tag. It provides a file /usr/lib/rpm/macros.d/macro.modules and gets read @@ -184,9 +203,9 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules release=release, module_name=module_build.name, module_stream=module_build.stream, - module_version=module_build.version) + module_version=module_build.version, + filter_conflicts=filter_conflicts) - mmd = module_build.mmd() modulemd_macros = "" if mmd.buildopts and mmd.buildopts.rpms: modulemd_macros = mmd.buildopts.rpms.macros diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index 70a3c5e..4c9fc5d 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -35,6 +35,7 @@ import pprint import logging import six import copy +import kobo.rpmlib log = logging.getLogger() @@ -427,37 +428,6 @@ def get_module_build_dependencies(session, module_info, strict=False): return module_tags -def get_module_commit_hash_and_version(session, module_info): - """ - Gets the commit hash and version of a module stored in PDC - :param module_info: a dict containing filters for PDC - :param session: a PDC session instance - :return: a tuple containing the string of the commit hash and the version - of the module stored in PDC. If a value is not found, None is - returned for the values that aren't found. - """ - commit_hash = None - version = None - module = get_module(session, module_info) - if module: - if module.get('modulemd'): - mmd = modulemd.ModuleMetadata() - mmd.loads(module['modulemd']) - if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'): - commit_hash = mmd.xmd['mbs']['commit'] - if module.get('variant_release'): - version = module['variant_release'] - if not commit_hash: - # TODO: Should this eventually be an exception? - log.warn( - 'The commit hash for {0!r} was not part of the modulemd in PDC' - .format(module_info)) - if not version: - # TODO: Should this eventually be an exception? - log.warn( - 'The version for {0!r} was not in PDC'.format(module_info)) - return commit_hash, version - def resolve_requires(session, requires): """ Takes `requires` dict with module_name as key and module_stream as value. @@ -469,6 +439,7 @@ def resolve_requires(session, requires): "ref": module_commit_hash, "stream": original_module_stream, "version": module_version, + "filtered_rpms": ["nvr", ...] }, ... } @@ -490,7 +461,11 @@ def resolve_requires(session, requires): # The commit ID isn't currently saved in modules.yaml 'ref': None, 'stream': local_build.stream, - 'version': local_build.version + 'version': local_build.version, + # No need to set filtered_rpms for local builds, because MBS + # filters the RPMs automatically when the module build is + # done. + 'filtered_rpms': [] } continue @@ -499,13 +474,41 @@ def resolve_requires(session, requires): 'name': module_name, 'version': module_stream, 'active': True} - commit_hash, version = get_module_commit_hash_and_version( - session, module_info) + module = get_module(session, module_info) + + commit_hash = None + version = None + filtered_rpms = [] + module = get_module(session, module_info, strict=True) + if module.get('modulemd'): + mmd = modulemd.ModuleMetadata() + mmd.loads(module['modulemd']) + if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'): + commit_hash = mmd.xmd['mbs']['commit'] + + # Find out the particular NVR of filtered packages + if "rpms" in module and mmd.filter and mmd.filter.rpms: + for rpm in module["rpms"]: + nvr = kobo.rpmlib.parse_nvra(rpm) + # If the package is not filtered, continue + if not nvr["name"] in mmd.filter.rpms: + continue + + # If the nvr is already in filtered_rpms, continue + nvr = kobo.rpmlib.make_nvr(nvr, force_epoch=True) + if nvr in filtered_rpms: + continue + filtered_rpms.append(nvr) + + if module.get('variant_release'): + version = module['variant_release'] + if version and commit_hash: new_requires[module_name] = { 'ref': commit_hash, 'stream': module_stream, - 'version': str(version) + 'version': str(version), + 'filtered_rpms': filtered_rpms, } else: raise RuntimeError( diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index a5c23b6..761e9e9 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -77,7 +77,151 @@ class MockedSCM(object): class TestUtils(unittest.TestCase): def setUp(self): - pass + self.filtered_rpms = [ + u'glibc-utils-0:2.25-4.module_5ccf9229', + u'glibc-benchtests-0:2.25-4.module_5ccf9229', + u'systemd-journal-remote-0:233-3.module_5ccf9229', + u'openldap-servers-0:2.4.44-8.module_5ccf9229', + u'kernel-debug-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'python-perf-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-tools-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'perf-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-tools-libs-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-PAE-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-PAEdebug-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'kernel-lpae-devel-0:4.11.0-0.rc7.git0.1.module_5ccf9229', + u'libcroco-devel-0:0.6.11-3.module_5ccf9229', + u'dbus-x11-1:1.11.10-2.module_5ccf9229', + u'qgpgme-0:1.9.0-1.module_5ccf9229', + u'qgpgme-devel-0:1.9.0-1.module_5ccf9229', + u'python2-gpg-0:1.9.0-1.module_5ccf9229', + u'libssh2-devel-0:1.8.0-2.module_5ccf9229', + u'python3-test-0:3.6.0-21.module_5ccf9229', + u'python3-debug-0:3.6.0-21.module_5ccf9229', + u'python3-tkinter-0:3.6.0-21.module_5ccf9229', + u'python3-tools-0:3.6.0-21.module_5ccf9229', + u'grub2-starfield-theme-1:2.02-0.38.module_5ccf9229', + u'libbabeltrace-devel-0:1.5.2-2.module_5ccf9229', + u'libidn-javadoc-0:1.33-2.module_5ccf9229', + u'libidn-java-0:1.33-2.module_5ccf9229', + u'cyrus-sasl-sql-0:2.1.26-30.module_5ccf9229', + u'python2-solv-0:0.6.26-1.module_5ccf9229', + u'perl-solv-0:0.6.26-1.module_5ccf9229', + u'python2-packaging-0:16.8-4.module_5ccf9229', + u'freetype-demos-0:2.7.1-2.module_5ccf9229', + u'util-linux-user-0:2.29.1-2.module_5ccf9229', + u'syslinux-perl-0:6.04-0.2.module_5ccf9229', + u'gnutls-utils-0:3.5.10-1.module_5ccf9229', + u'gnutls-guile-0:3.5.10-1.module_5ccf9229', + u'gnutls-devel-0:3.5.10-1.module_5ccf9229', + u'gnutls-dane-0:3.5.10-1.module_5ccf9229', + u'python-pwquality-0:1.3.0-8.module_5ccf9229', + u'openssl-perl-1:1.1.0e-1.module_5ccf9229', + u'glib2-fam-0:2.52.0-1.module_5ccf9229', + u'glib2-static-0:2.52.0-1.module_5ccf9229', + u'glib2-devel-0:2.52.0-1.module_5ccf9229', + u'libselinux-ruby-0:2.6-2.module_5ccf9229', + u'libselinux-python-0:2.6-2.module_5ccf9229', + u'cmirror-0:2.02.168-4.module_5ccf9229', + u'lvm2-cluster-standalone-0:2.02.168-4.module_5ccf9229', + u'lvm2-dbusd-0:2.02.168-4.module_5ccf9229', + u'lvm2-python-libs-0:2.02.168-4.module_5ccf9229', + u'lvm2-cluster-0:2.02.168-4.module_5ccf9229', + u'cmirror-standalone-0:2.02.168-4.module_5ccf9229', + u'lvm2-lockd-0:2.02.168-4.module_5ccf9229', + u'python2-pip-0:9.0.1-7.module_5ccf9229', + u'python-magic-0:5.30-5.module_5ccf9229', + u'cracklib-python-0:2.9.6-5.module_5ccf9229', + u'hfsutils-x11-0:3.2.6-31.module_5ccf9229', + u'kernel-rpm-macros-0:63-1.module_5ccf9229', + u'cryptsetup-python-0:1.7.3-3.module_5ccf9229', + u'python2-rpm-0:4.13.0.1-3.module_5ccf9229', + u'rpm-cron-0:4.13.0.1-3.module_5ccf9229', + u'texinfo-0:6.3-2.module_5ccf9229', + u'texinfo-tex-0:6.3-2.module_5ccf9229', + u'qt5-0:5.8.0-2.module_5ccf9229', + u'qt5-devel-0:5.8.0-2.module_5ccf9229', + u'qt5-rpm-macros-0:5.8.0-2.module_5ccf9229', + u'gnupg2-smime-0:2.1.18-2.module_5ccf9229', + u'emacs-nox-1:25.2-0.1.rc2.module_5ccf9229', + u'emacs-terminal-1:25.2-0.1.rc2.module_5ccf9229', + u'emacs-common-1:25.2-0.1.rc2.module_5ccf9229', + u'emacs-1:25.2-0.1.rc2.module_5ccf9229', + u'libverto-libevent-devel-0:0.2.6-7.module_5ccf9229', + u'libverto-tevent-devel-0:0.2.6-7.module_5ccf9229', + u'libverto-libevent-0:0.2.6-7.module_5ccf9229', + u'libverto-tevent-0:0.2.6-7.module_5ccf9229', + u'libtool-0:2.4.6-17.module_5ccf9229', + u'libpeas-loader-python3-0:1.20.0-5.module_5ccf9229', + u'libpeas-devel-0:1.20.0-5.module_5ccf9229', + u'libpeas-gtk-0:1.20.0-5.module_5ccf9229', + u'libpeas-loader-python-0:1.20.0-5.module_5ccf9229', + u'python2-setuptools-0:34.3.0-1.module_5ccf9229', + u'libsemanage-python-0:2.6-2.module_5ccf9229', + u'python-libxml2-0:2.9.4-2.module_5ccf9229', + u'krb5-server-ldap-0:1.15-9.module_5ccf9229', + u'krb5-server-0:1.15-9.module_5ccf9229', + u'python2-six-0:1.10.0-8.module_5ccf9229', + u'python2-appdirs-0:1.4.0-10.module_5ccf9229', + u'pyparsing-0:2.1.10-3.module_5ccf9229', + u'python2-pyparsing-0:2.1.10-3.module_5ccf9229', + u'ntsysv-0:1.9-1.module_5ccf9229', + u'audit-0:2.7.3-1.module_5ccf9229', + u'audit-libs-python-0:2.7.3-1.module_5ccf9229', + u'audispd-plugins-0:2.7.3-1.module_5ccf9229', + u'audispd-plugins-zos-0:2.7.3-1.module_5ccf9229', + u'audit-libs-python3-0:2.7.3-1.module_5ccf9229', + u'modeline2fb-0:2.1-40.module_5ccf9229', + u'msghack-0:0.19.8.1-8.module_5ccf9229', + u'emacs-gettext-0:0.19.8.1-8.module_5ccf9229', + u'python2-hawkey-0:0.8.2-1.module_987f08f4', + u'libdnf-devel-0:0.8.2-1.module_987f08f4', + u'sqlite-tcl-0:3.17.0-2.module_5ccf9229', + u'sqlite-analyzer-0:3.17.0-2.module_5ccf9229', + u'sssd-0:1.15.2-4.module_47fecbcd', + u'sssd-nfs-idmap-0:1.15.2-4.module_47fecbcd', + u'sssd-winbind-idmap-0:1.15.2-4.module_47fecbcd', + u'sssd-krb5-0:1.15.2-4.module_47fecbcd', + u'libsss_sudo-0:1.15.2-4.module_47fecbcd', + u'sssd-libwbclient-devel-0:1.15.2-4.module_47fecbcd', + u'sssd-common-pac-0:1.15.2-4.module_47fecbcd', + u'sssd-ipa-0:1.15.2-4.module_47fecbcd', + u'python3-sss-murmur-0:1.15.2-4.module_47fecbcd', + u'libsss_simpleifp-devel-0:1.15.2-4.module_47fecbcd', + u'sssd-common-0:1.15.2-4.module_47fecbcd', + u'sssd-dbus-0:1.15.2-4.module_47fecbcd', + u'libipa_hbac-0:1.15.2-4.module_47fecbcd', + u'sssd-proxy-0:1.15.2-4.module_47fecbcd', + u'python2-sss-murmur-0:1.15.2-4.module_47fecbcd', + u'python3-sss-0:1.15.2-4.module_47fecbcd', + u'libsss_autofs-0:1.15.2-4.module_47fecbcd', + u'python2-libipa_hbac-0:1.15.2-4.module_47fecbcd', + u'libsss_simpleifp-0:1.15.2-4.module_47fecbcd', + u'libsss_idmap-devel-0:1.15.2-4.module_47fecbcd', + u'sssd-libwbclient-0:1.15.2-4.module_47fecbcd', + u'python2-libsss_nss_idmap-0:1.15.2-4.module_47fecbcd', + u'python3-libipa_hbac-0:1.15.2-4.module_47fecbcd', + u'python2-sssdconfig-0:1.15.2-4.module_47fecbcd', + u'libipa_hbac-devel-0:1.15.2-4.module_47fecbcd', + u'libsss_nss_idmap-devel-0:1.15.2-4.module_47fecbcd', + u'python2-sss-0:1.15.2-4.module_47fecbcd', + u'sssd-ldap-0:1.15.2-4.module_47fecbcd', + u'sssd-ad-0:1.15.2-4.module_47fecbcd', + u'sssd-tools-0:1.15.2-4.module_47fecbcd', + u'sssd-krb5-common-0:1.15.2-4.module_47fecbcd', + u'librepo-devel-0:1.7.20-3.module_5ccf9229', + u'python2-librepo-0:1.7.20-3.module_5ccf9229', + u'iproute-tc-0:4.11.0-1.module_d6de39f1', + u'dracut-live-0:044-182.module_bd7491c8', + u'dracut-fips-aesni-0:044-182.module_bd7491c8', + u'dracut-fips-0:044-182.module_bd7491c8', + u'dracut-network-0:044-182.module_bd7491c8', + u'gobject-introspection-devel-0:1.52.0-1.module_5ccf9229', + u'iptables-compat-0:1.6.1-2.module_5ccf9229', + u'libcap-ng-python-0:0.7.8-3.module_5ccf9229', + u'python3-sssdconfig-0:1.15.2-4.module_47fecbcd' + ] def tearDown(self): init_data() @@ -120,12 +264,14 @@ class TestUtils(unittest.TestCase): 'base-runtime': { 'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df', 'stream': 'master', - 'version': '20170616125652'}}, + 'version': '20170616125652', + 'filtered_rpms': self.filtered_rpms}}, 'requires': { 'base-runtime': { 'version': '20170616125652', 'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df', - 'stream': 'master'}}, + 'stream': 'master', + 'filtered_rpms': self.filtered_rpms}}, 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'}, 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, @@ -163,12 +309,14 @@ class TestUtils(unittest.TestCase): 'base-runtime': { 'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df', 'stream': 'master', - 'version': '20170616125652'}}, + 'version': '20170616125652', + 'filtered_rpms': self.filtered_rpms}}, 'requires': { 'base-runtime': { 'version': '20170616125652', 'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df', - 'stream': 'master'}}, + 'stream': 'master', + 'filtered_rpms': self.filtered_rpms}}, 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'}, 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, diff --git a/tests/vcr-request-data.tar.gz b/tests/vcr-request-data.tar.gz index 48fad5d..1acbeb8 100644 Binary files a/tests/vcr-request-data.tar.gz and b/tests/vcr-request-data.tar.gz differ