From a1dfeeb7e3de19e1b9dfe947fab7fbfd2eecbe8e Mon Sep 17 00:00:00 2001 From: mprahl Date: Mar 03 2020 19:48:47 +0000 Subject: Move utils/ursine.py to scheduler/ursine.py --- diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index d0d1aaa..572b7a4 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -23,7 +23,7 @@ from module_build_service.scheduler.default_modules import ( from module_build_service.scheduler.greenwave import greenwave from module_build_service.utils.submit import format_mmd from module_build_service.scheduler import events -from module_build_service.utils.ursine import handle_stream_collision_modules +from module_build_service.scheduler.ursine import handle_stream_collision_modules from requests.exceptions import ConnectionError diff --git a/module_build_service/scheduler/ursine.py b/module_build_service/scheduler/ursine.py new file mode 100644 index 0000000..4085fd6 --- /dev/null +++ b/module_build_service/scheduler/ursine.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import re + +from module_build_service import conf, log +from module_build_service.common.koji import get_session +from module_build_service.db_session import db_session +from module_build_service.resolver import GenericResolver + + +""" +This module handles module stream collision with ursine content before a module +is built. + +This kind of collision would happen when packages, which are managed by +Ursa-Major, are used to build a module. Let's see an example, when a module foo +buildrequires bar with stream A, however module bar with another stream B was +already added to ursine content by Ursa-Major when it has been built and moved +to ready state. Hence, MBS has to ensure any packages from module bar:B are not +present in the buildroot. + +A technical background: + +Generally, each module buildrequires a platform module, which is associated +with ursine content by adding an external repository to the platform module's +tag. That repository is generated from a build tag that inherits from a set +modules' koji tag through its tag inheritance hierarchy. Ursa-Major manages +the inheritance relationship. + +Stream collision modules are just those modules added to tag inheritance by +Ursa-Major. +""" + + +def find_build_tags_from_external_repos(koji_session, repo_infos): + """Find build tags from external repos + + An external repo added to a tag could be an arbitrary external repository. + Hence, this method tries best to guess the tags from each external repo's + URL by a regular expression. + + :param repo_infos: list of mappings represeting external repos information. + :type repo_infos: list[dict] + :return: a list of tag names. + :rtype: list[str] + """ + re_external_repo_url = r"^{}/repos/(.+-build)/latest/\$arch/?$".format( + conf.koji_external_repo_url_prefix.rstrip("/")) + tag_names = [] + for info in repo_infos: + match = re.match(re_external_repo_url, info["url"]) + if match: + name = match.groups()[0] + if koji_session.getTag(name) is None: + log.warning( + "Ignoring the found tag %s because no tag info was found with this name.", + name, + ) + else: + tag_names.append(name) + else: + log.warning( + "The build tag could not be parsed from external repo %s whose url is %s.", + info["external_repo_name"], info["url"], + ) + return tag_names + + +def find_module_koji_tags(koji_session, build_tag): + """ + Find module koji tags from parents of build tag through the tag inheritance + + MBS supports a few prefixes which are configured in + ``conf.koij_tag_prefixes``. Tags with a configured prefix will be + considered as a module's koji tag. + + :param koji_session: instance of Koji client session. + :type koji_session: ClientSession + :param str build_tag: tag name, which is the build tag inheriting from + parent tags where module koji tags are contained. + :return: list of module koji tags. + :rtype: list[str] + """ + return [ + data["name"] + for data in koji_session.getFullInheritance(build_tag) + if any(data["name"].startswith(prefix) for prefix in conf.koji_tag_prefixes) + ] + + +def get_modulemds_from_ursine_content(tag): + """Get all modules metadata which were added to ursine content + + Ursine content is the tag inheritance managed by Ursa-Major by adding + specific modules' koji_tag. + + Background of module build based on ursine content: + + Each module build buildrequires a platform module, which is a presudo-module + used to connect to an external repository whose packages will be present + in the buildroot. In practice, the external repo is generated from a build + tag which could inherit from a few module koji_tags so that those module's + RPMs could be build dependencies for some specific packages. + + So, this function is to find out all module koji_tags from the build tag + and return corresponding module metadata. + + :param str tag: a base module's koji_tag. + :return: list of module metadata. Empty list will be returned if no ursine + modules metadata is found. + :rtype: list[Modulemd.Module] + """ + resolver = GenericResolver.create(db_session, conf) + + koji_session = get_session(conf, login=False) + repos = koji_session.getExternalRepoList(tag) + build_tags = find_build_tags_from_external_repos(koji_session, repos) + if not build_tags: + log.debug("No external repo containing ursine content is found.") + return [] + modulemds = [] + for tag in build_tags: + koji_tags = find_module_koji_tags(koji_session, tag) + for koji_tag in koji_tags: + md = resolver.get_modulemd_by_koji_tag(koji_tag) + if md: + modulemds.append(md) + else: + log.warning("No module is found by koji_tag '%s'", koji_tag) + return modulemds + + +def find_stream_collision_modules(buildrequired_modules, koji_tag): + """ + Find buildrequired modules that are part of the ursine content represented + by the koji_tag but with a different stream. + + :param dict buildrequired_modules: a mapping of buildrequires, which is just + the ``xmd/mbs/buildrequires``. This mapping is used to determine if a module + found from ursine content is a buildrequire with different stream. + :param str koji_tag: a base module's koji_tag. Modules will be retrieved from + ursine content associated with this koji_tag and check if there are + modules that collide. + :return: a list of NSVC of collision modules. If no collision module is + found, an empty list is returned. + :rtype: list[str] + """ + ursine_modulemds = get_modulemds_from_ursine_content(koji_tag) + if not ursine_modulemds: + log.debug("No module metadata is found from ursine content.") + return [] + + collision_modules = [ + item.get_nsvc() + for item in ursine_modulemds + # If some module in the ursine content is one of the buildrequires but has + # different stream, that is what we want to record here, whose RPMs will be + # excluded from buildroot by adding them into SRPM module-build-macros as + # Conflicts. + if ( + item.get_module_name() in buildrequired_modules + and item.get_stream_name() != buildrequired_modules[item.get_module_name()]["stream"] + ) + ] + + for item in collision_modules: + name, stream, _ = item.split(":", 2) + log.info( + "Buildrequired module %s exists in ursine content with " + "different stream %s, whose RPMs will be excluded.", + name, stream, + ) + + return collision_modules + + +def handle_stream_collision_modules(mmd): + """ + Find out modules from ursine content and record those that are buildrequire + module but have different stream. And finally, record built RPMs of these + found modules for later use to exclude them from buildroot. + + Note that this depends on the result of module stream expansion. + + MBS supports multiple base modules via option conf.base_module_names. A base + module name could be platform in most cases, but there could be others for + particular cases in practice. So, each expanded base module stored in + ``xmd/mbs/buildrequires`` will be handled and will have a new + key/value pair ``stream_collision_modules: [N-S-V-C, ...]``. This key/value + will then be handled by the module event handler. + + As a result, a new item is added xmd/mbs/buildrequires/platform/stream_collision_modules, + which is a list of NSVC strings. Each of them is the module added to ursine + content by Ursa-Major. + + :param mmd: a module's metadata which will be built. + :type mmd: Modulemd.Module + """ + log.info("Start to find out stream collision modules.") + xmd = mmd.get_xmd() + buildrequires = xmd["mbs"]["buildrequires"] + + for module_name in conf.base_module_names: + base_module_info = buildrequires.get(module_name) + if base_module_info is None: + log.info( + "Base module %s is not a buildrequire of module %s. " + "Skip handling module stream collision for this base module.", + module_name, mmd.get_module_name(), + ) + continue + + # Module stream collision is handled only for newly created module + # build. However, if a build is resumed and restarted from init + # state, we have to ensure stream collision is not handled twice for a + # base module. + # Just check the existence, and following code ensures this key exists + # even if no stream collision module is found. + if "stream_collision_modules" in base_module_info and "ursine_rpms" in base_module_info: + log.debug( + "Base module %s has stream collision modules and ursine " + "rpms. Skip to handle stream collision again for it.", + module_name, + ) + continue + + modules_nsvc = find_stream_collision_modules( + buildrequires, base_module_info["koji_tag"]) + + if modules_nsvc: + # Save modules NSVC for later use in subsequent event handlers to + # log readable messages. + base_module_info["stream_collision_modules"] = modules_nsvc + base_module_info["ursine_rpms"] = find_module_built_rpms(modules_nsvc) + else: + log.info("No stream collision module is found against base module %s.", module_name) + # Always set in order to mark it as handled already. + base_module_info["stream_collision_modules"] = None + base_module_info["ursine_rpms"] = None + + mmd.set_xmd(xmd) + + +def find_module_built_rpms(modules_nsvc): + """Find out built RPMs of given modules + + :param modules_nsvc: a list of modules' NSVC to find out built RPMs for + each of them. + :type modules_nsvc: list[str] + :return: a sorted list of RPMs, each of them is represented as NEVR. + :rtype: list[str] + """ + import kobo.rpmlib + resolver = GenericResolver.create(db_session, conf) + + built_rpms = [] + koji_session = get_session(conf, login=False) + + for nsvc in modules_nsvc: + name, stream, version, context = nsvc.split(":") + module = resolver.get_module(name, stream, version, context, strict=True) + rpms = koji_session.listTaggedRPMS(module["koji_tag"], latest=True)[0] + built_rpms.extend(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms) + + # In case there is duplicate NEVRs, ensure every NEVR is unique in the final list. + # And, sometimes, sorted list of RPMs would be easier to read. + return sorted(set(built_rpms)) diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index 565b544..14de7bc 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -3,4 +3,3 @@ from module_build_service.utils.views import * # noqa from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa -from module_build_service.utils.ursine import * # noqa diff --git a/module_build_service/utils/ursine.py b/module_build_service/utils/ursine.py deleted file mode 100644 index 4085fd6..0000000 --- a/module_build_service/utils/ursine.py +++ /dev/null @@ -1,267 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import re - -from module_build_service import conf, log -from module_build_service.common.koji import get_session -from module_build_service.db_session import db_session -from module_build_service.resolver import GenericResolver - - -""" -This module handles module stream collision with ursine content before a module -is built. - -This kind of collision would happen when packages, which are managed by -Ursa-Major, are used to build a module. Let's see an example, when a module foo -buildrequires bar with stream A, however module bar with another stream B was -already added to ursine content by Ursa-Major when it has been built and moved -to ready state. Hence, MBS has to ensure any packages from module bar:B are not -present in the buildroot. - -A technical background: - -Generally, each module buildrequires a platform module, which is associated -with ursine content by adding an external repository to the platform module's -tag. That repository is generated from a build tag that inherits from a set -modules' koji tag through its tag inheritance hierarchy. Ursa-Major manages -the inheritance relationship. - -Stream collision modules are just those modules added to tag inheritance by -Ursa-Major. -""" - - -def find_build_tags_from_external_repos(koji_session, repo_infos): - """Find build tags from external repos - - An external repo added to a tag could be an arbitrary external repository. - Hence, this method tries best to guess the tags from each external repo's - URL by a regular expression. - - :param repo_infos: list of mappings represeting external repos information. - :type repo_infos: list[dict] - :return: a list of tag names. - :rtype: list[str] - """ - re_external_repo_url = r"^{}/repos/(.+-build)/latest/\$arch/?$".format( - conf.koji_external_repo_url_prefix.rstrip("/")) - tag_names = [] - for info in repo_infos: - match = re.match(re_external_repo_url, info["url"]) - if match: - name = match.groups()[0] - if koji_session.getTag(name) is None: - log.warning( - "Ignoring the found tag %s because no tag info was found with this name.", - name, - ) - else: - tag_names.append(name) - else: - log.warning( - "The build tag could not be parsed from external repo %s whose url is %s.", - info["external_repo_name"], info["url"], - ) - return tag_names - - -def find_module_koji_tags(koji_session, build_tag): - """ - Find module koji tags from parents of build tag through the tag inheritance - - MBS supports a few prefixes which are configured in - ``conf.koij_tag_prefixes``. Tags with a configured prefix will be - considered as a module's koji tag. - - :param koji_session: instance of Koji client session. - :type koji_session: ClientSession - :param str build_tag: tag name, which is the build tag inheriting from - parent tags where module koji tags are contained. - :return: list of module koji tags. - :rtype: list[str] - """ - return [ - data["name"] - for data in koji_session.getFullInheritance(build_tag) - if any(data["name"].startswith(prefix) for prefix in conf.koji_tag_prefixes) - ] - - -def get_modulemds_from_ursine_content(tag): - """Get all modules metadata which were added to ursine content - - Ursine content is the tag inheritance managed by Ursa-Major by adding - specific modules' koji_tag. - - Background of module build based on ursine content: - - Each module build buildrequires a platform module, which is a presudo-module - used to connect to an external repository whose packages will be present - in the buildroot. In practice, the external repo is generated from a build - tag which could inherit from a few module koji_tags so that those module's - RPMs could be build dependencies for some specific packages. - - So, this function is to find out all module koji_tags from the build tag - and return corresponding module metadata. - - :param str tag: a base module's koji_tag. - :return: list of module metadata. Empty list will be returned if no ursine - modules metadata is found. - :rtype: list[Modulemd.Module] - """ - resolver = GenericResolver.create(db_session, conf) - - koji_session = get_session(conf, login=False) - repos = koji_session.getExternalRepoList(tag) - build_tags = find_build_tags_from_external_repos(koji_session, repos) - if not build_tags: - log.debug("No external repo containing ursine content is found.") - return [] - modulemds = [] - for tag in build_tags: - koji_tags = find_module_koji_tags(koji_session, tag) - for koji_tag in koji_tags: - md = resolver.get_modulemd_by_koji_tag(koji_tag) - if md: - modulemds.append(md) - else: - log.warning("No module is found by koji_tag '%s'", koji_tag) - return modulemds - - -def find_stream_collision_modules(buildrequired_modules, koji_tag): - """ - Find buildrequired modules that are part of the ursine content represented - by the koji_tag but with a different stream. - - :param dict buildrequired_modules: a mapping of buildrequires, which is just - the ``xmd/mbs/buildrequires``. This mapping is used to determine if a module - found from ursine content is a buildrequire with different stream. - :param str koji_tag: a base module's koji_tag. Modules will be retrieved from - ursine content associated with this koji_tag and check if there are - modules that collide. - :return: a list of NSVC of collision modules. If no collision module is - found, an empty list is returned. - :rtype: list[str] - """ - ursine_modulemds = get_modulemds_from_ursine_content(koji_tag) - if not ursine_modulemds: - log.debug("No module metadata is found from ursine content.") - return [] - - collision_modules = [ - item.get_nsvc() - for item in ursine_modulemds - # If some module in the ursine content is one of the buildrequires but has - # different stream, that is what we want to record here, whose RPMs will be - # excluded from buildroot by adding them into SRPM module-build-macros as - # Conflicts. - if ( - item.get_module_name() in buildrequired_modules - and item.get_stream_name() != buildrequired_modules[item.get_module_name()]["stream"] - ) - ] - - for item in collision_modules: - name, stream, _ = item.split(":", 2) - log.info( - "Buildrequired module %s exists in ursine content with " - "different stream %s, whose RPMs will be excluded.", - name, stream, - ) - - return collision_modules - - -def handle_stream_collision_modules(mmd): - """ - Find out modules from ursine content and record those that are buildrequire - module but have different stream. And finally, record built RPMs of these - found modules for later use to exclude them from buildroot. - - Note that this depends on the result of module stream expansion. - - MBS supports multiple base modules via option conf.base_module_names. A base - module name could be platform in most cases, but there could be others for - particular cases in practice. So, each expanded base module stored in - ``xmd/mbs/buildrequires`` will be handled and will have a new - key/value pair ``stream_collision_modules: [N-S-V-C, ...]``. This key/value - will then be handled by the module event handler. - - As a result, a new item is added xmd/mbs/buildrequires/platform/stream_collision_modules, - which is a list of NSVC strings. Each of them is the module added to ursine - content by Ursa-Major. - - :param mmd: a module's metadata which will be built. - :type mmd: Modulemd.Module - """ - log.info("Start to find out stream collision modules.") - xmd = mmd.get_xmd() - buildrequires = xmd["mbs"]["buildrequires"] - - for module_name in conf.base_module_names: - base_module_info = buildrequires.get(module_name) - if base_module_info is None: - log.info( - "Base module %s is not a buildrequire of module %s. " - "Skip handling module stream collision for this base module.", - module_name, mmd.get_module_name(), - ) - continue - - # Module stream collision is handled only for newly created module - # build. However, if a build is resumed and restarted from init - # state, we have to ensure stream collision is not handled twice for a - # base module. - # Just check the existence, and following code ensures this key exists - # even if no stream collision module is found. - if "stream_collision_modules" in base_module_info and "ursine_rpms" in base_module_info: - log.debug( - "Base module %s has stream collision modules and ursine " - "rpms. Skip to handle stream collision again for it.", - module_name, - ) - continue - - modules_nsvc = find_stream_collision_modules( - buildrequires, base_module_info["koji_tag"]) - - if modules_nsvc: - # Save modules NSVC for later use in subsequent event handlers to - # log readable messages. - base_module_info["stream_collision_modules"] = modules_nsvc - base_module_info["ursine_rpms"] = find_module_built_rpms(modules_nsvc) - else: - log.info("No stream collision module is found against base module %s.", module_name) - # Always set in order to mark it as handled already. - base_module_info["stream_collision_modules"] = None - base_module_info["ursine_rpms"] = None - - mmd.set_xmd(xmd) - - -def find_module_built_rpms(modules_nsvc): - """Find out built RPMs of given modules - - :param modules_nsvc: a list of modules' NSVC to find out built RPMs for - each of them. - :type modules_nsvc: list[str] - :return: a sorted list of RPMs, each of them is represented as NEVR. - :rtype: list[str] - """ - import kobo.rpmlib - resolver = GenericResolver.create(db_session, conf) - - built_rpms = [] - koji_session = get_session(conf, login=False) - - for nsvc in modules_nsvc: - name, stream, version, context = nsvc.split(":") - module = resolver.get_module(name, stream, version, context, strict=True) - rpms = koji_session.listTaggedRPMS(module["koji_tag"], latest=True)[0] - built_rpms.extend(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms) - - # In case there is duplicate NEVRs, ensure every NEVR is unique in the final list. - # And, sometimes, sorted list of RPMs would be easier to read. - return sorted(set(built_rpms)) diff --git a/tests/test_scheduler/test_ursine.py b/tests/test_scheduler/test_ursine.py new file mode 100644 index 0000000..b4edbda --- /dev/null +++ b/tests/test_scheduler/test_ursine.py @@ -0,0 +1,357 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from mock import patch, Mock + +from module_build_service import conf +from module_build_service.scheduler import ursine +from tests import make_module, make_module_in_db, clean_database + + +class TestFindModuleKojiTags: + """Test ursine.find_module_koji_tags""" + + @patch.object(conf, "koji_tag_prefixes", new=["module"]) + def test_find_out_all_module_koji_tags(self): + session = Mock() + session.getFullInheritance.return_value = [ + {"name": "module-tag1-s-v-c"}, + {"name": "module-tag2-s-v-c"}, + {"name": "tag-1"}, + ] + + expected_tags = ["module-tag1-s-v-c", "module-tag2-s-v-c"] + + tags = ursine.find_module_koji_tags(session, "tag-a-build") + assert expected_tags == tags + + @patch.object(conf, "koji_tag_prefixes", new=["module"]) + def test_return_empty_if_no_module_koji_tags(self): + session = Mock() + session.getFullInheritance.return_value = [{"name": "tag-1"}, {"name": "tag-2"}] + + tags = ursine.find_module_koji_tags(session, "tag-a-build") + assert [] == tags + + +class TestFindUrsineRootTags: + """Test ursine.find_build_tags_from_external_repos""" + + def setup_method(self): + self.koji_session = Mock() + self.koji_session.getTag.side_effect = \ + lambda name: None if name == "X-build" else {"name": name} + + def test_find_build_tags(self): + with patch.object( + conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" + ): + tags = ursine.find_build_tags_from_external_repos( + self.koji_session, + [ + { + "external_repo_name": "tag-1-external-repo", + "url": "http://example.com/brewroot/repos/tag-1-build/latest/$arch/", + }, + { + "external_repo_name": "tag-2-external-repo", + "url": "http://example.com/brewroot/repos/tag-2-build/latest/$arch/", + }, + ], + ) + + assert ["tag-1-build", "tag-2-build"] == tags + + def test_return_emtpy_if_no_match_external_repo_url(self): + with patch.object( + conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" + ): + tags = ursine.find_build_tags_from_external_repos( + self.koji_session, + [ + { + "external_repo_name": "tag-1-external-repo", + "url": "https://another-site.org/repos/tag-1-build/latest/$arch/", + }, + { + "external_repo_name": "tag-2-external-repo", + "url": "https://another-site.org/repos/tag-2-build/latest/$arch/", + }, + ], + ) + + assert [] == tags + + def test_some_tag_is_not_koji_tag(self): + with patch.object( + conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" + ): + tags = ursine.find_build_tags_from_external_repos( + self.koji_session, + [ + { + "external_repo_name": "tag-1-external-repo", + "url": "http://example.com/brewroot/repos/tag-1-build/latest/$arch/", + }, + { + "external_repo_name": "tag-2-external-repo", + "url": "http://example.com/brewroot/repos/X-build/latest/$arch/", + }, + ], + ) + + assert ["tag-1-build"] == tags + + +class TestGetModulemdsFromUrsineContent: + """Test ursine.get_modulemds_from_ursine_content""" + + def setup_method(self): + clean_database(False) + + def teardown_method(self, test_method): + clean_database() + + @patch("koji.ClientSession") + def test_return_empty_if_no_ursine_build_tag_is_found(self, ClientSession): + koji_session = ClientSession.return_value + + # No module koji_tag in ursine content yet. This will result in empty + # ursine modulemds is returned. + koji_session.getFullInheritance.return_value = [{"name": "tag-1.0-build"}] + koji_session.getExternalRepoList.return_value = [{ + "external_repo_name": "tag-1.0-external-repo", + "url": "http://example.com/repos/tag-4-build/latest/$arch/", + }] + + modulemds = ursine.get_modulemds_from_ursine_content("tag") + assert [] == modulemds + + @patch.object(conf, "koji_tag_prefixes", new=["module"]) + @patch("koji.ClientSession") + def test_get_modulemds(self, ClientSession): + koji_session = ClientSession.return_value + + # Ensure to to get build tag for further query of ursine content. + # For this test, the build tag is tag-4-build + koji_session.getExternalRepoList.return_value = [{ + "external_repo_name": "tag-1.0-external-repo", + "url": "http://example.com/repos/tag-4-build/latest/$arch/", + }] + + # Ensure to return module tags from ursine content of fake build tag + # specified in above external repo's url. + def mock_getFullInheritance(tag): + if tag == "tag-4-build": + return [ + {"name": "tag-1.0-build"}, + # Below two modules should be returned and whose modulemd + # should be also queried from database. + {"name": "module-name1-s-2020-c"}, + {"name": "module-name2-s-2021-c"}, + ] + raise ValueError("{} is not handled by test.".format(tag)) + + koji_session.getFullInheritance.side_effect = mock_getFullInheritance + + # Defaults to DB resolver, so create fake module builds and store them + # into database to ensure they can be queried. + # + # Switched to call function make_session to create a + # SQLAlchemy database session. Calling db.session causes failure to + # read attributes from a ModuleBuild object at following line calling + # mmd(). The error is ModuleBuild object is not bound to a Session. + # From the behavior of following code, the reason of the error is + # mixing use of db.session and make_session, the latter one is called + # from function ``get_modulemds_from_ursine_content``. + mmd_name1s2020c = make_module_in_db( + "name1:s:2020:c", + xmd={"mbs": {"koji_tag": "module-name1-s-2020-c"}}, + ) + mmd_name2s2021c = make_module_in_db( + "name2:s:2021:c", + xmd={"mbs": {"koji_tag": "module-name2-s-2021-c"}}, + ) + + koji_tag = "tag" # It's ok to use arbitrary tag name. + with patch.object(conf, "koji_external_repo_url_prefix", new="http://example.com/"): + modulemds = ursine.get_modulemds_from_ursine_content(koji_tag) + + test_nsvcs = [item.get_nsvc() for item in modulemds] + test_nsvcs.sort() + + expected_nsvcs = [mmd_name1s2020c.mmd().get_nsvc(), mmd_name2s2021c.mmd().get_nsvc()] + expected_nsvcs.sort() + + koji_session.getExternalRepoList.assert_called_once_with(koji_tag) + assert expected_nsvcs == test_nsvcs + + +class TestRecordStreamCollisionModules: + """Test ursine.record_stream_collision_modules""" + + @patch.object(conf, "base_module_names", new=["platform"]) + @patch.object(ursine, "find_stream_collision_modules") + def test_nothing_changed_if_no_base_module_is_in_buildrequires( + self, find_stream_collision_modules + ): + xmd = {"mbs": {"buildrequires": {"modulea": {"stream": "master"}}}} + fake_mmd = make_module("name1:s:2020:c", xmd=xmd) + original_xmd = fake_mmd.get_xmd() + + with patch.object(ursine, "log") as log: + ursine.handle_stream_collision_modules(fake_mmd) + assert 2 == log.info.call_count + find_stream_collision_modules.assert_not_called() + + assert original_xmd == fake_mmd.get_xmd() + + @patch.object(conf, "base_module_names", new=["platform"]) + @patch("module_build_service.scheduler.ursine.get_modulemds_from_ursine_content") + def test_mark_handled_even_if_no_modules_in_ursine_content( + self, get_modulemds_from_ursine_content + ): + xmd = { + "mbs": { + "buildrequires": { + "modulea": {"stream": "master"}, + "platform": {"stream": "master", "koji_tag": "module-rhel-8.0-build"}, + } + } + } + fake_mmd = make_module("name1:s:2020:c", xmd=xmd) + expected_xmd = fake_mmd.get_xmd() + + get_modulemds_from_ursine_content.return_value = [] + + with patch.object(ursine, "log") as log: + ursine.handle_stream_collision_modules(fake_mmd) + assert 2 == log.info.call_count + + # Ensure stream_collision_modules is set. + expected_xmd["mbs"]["buildrequires"]["platform"]["stream_collision_modules"] = "" + expected_xmd["mbs"]["buildrequires"]["platform"]["ursine_rpms"] = "" + assert expected_xmd == fake_mmd.get_xmd() + + @patch.object(conf, "base_module_names", new=["platform", "project-platform"]) + @patch("module_build_service.scheduler.ursine.get_modulemds_from_ursine_content") + @patch("module_build_service.resolver.GenericResolver.create") + @patch("koji.ClientSession") + def test_add_collision_modules( + self, ClientSession, resolver_create, get_modulemds_from_ursine_content + ): + xmd = { + "mbs": { + "buildrequires": { + "modulea": {"stream": "master"}, + "foo": {"stream": "1"}, + "bar": {"stream": "2"}, + "platform": {"stream": "master", "koji_tag": "module-rhel-8.0-build"}, + "project-platform": { + "stream": "master", + "koji_tag": "module-project-1.0-build", + }, + } + } + } + fake_mmd = make_module("name1:s:2020:c", xmd=xmd) + + def mock_get_ursine_modulemds(koji_tag): + if koji_tag == "module-rhel-8.0-build": + return [ + # This is the one + make_module("modulea:10:20180813041838:5ea3b708"), + make_module("moduleb:1.0:20180113042038:6ea3b105"), + ] + if koji_tag == "module-project-1.0-build": + return [ + # Both of them are the collided modules + make_module("bar:6:20181013041838:817fa3a8"), + make_module("foo:2:20180113041838:95f078a1"), + ] + + get_modulemds_from_ursine_content.side_effect = mock_get_ursine_modulemds + + # Mock for finding out built rpms + def mock_get_module(name, stream, version, context, strict=True): + return { + "modulea:10:20180813041838:5ea3b708": { + "koji_tag": "module-modulea-10-20180813041838-5ea3b708" + }, + "bar:6:20181013041838:817fa3a8": { + "koji_tag": "module-bar-6-20181013041838-817fa3a8" + }, + "foo:2:20180113041838:95f078a1": { + "koji_tag": "module-foo-2-20180113041838-95f078a1" + }, + }["{}:{}:{}:{}".format(name, stream, version, context)] + + resolver = resolver_create.return_value + resolver.get_module.side_effect = mock_get_module + + def mock_listTaggedRPMS(tag, latest): + return { + "module-modulea-10-20180813041838-5ea3b708": [ + [{"name": "pkg1", "version": "1.0", "release": "1.fc28", "epoch": None}] + ], + "module-bar-6-20181013041838-817fa3a8": [ + [{"name": "pkg2", "version": "2.0", "release": "1.fc28", "epoch": None}] + ], + "module-foo-2-20180113041838-95f078a1": [ + [{"name": "pkg3", "version": "3.0", "release": "1.fc28", "epoch": None}] + ], + }[tag] + + koji_session = ClientSession.return_value + koji_session.listTaggedRPMS.side_effect = mock_listTaggedRPMS + + ursine.handle_stream_collision_modules(fake_mmd) + + xmd = fake_mmd.get_xmd() + buildrequires = xmd["mbs"]["buildrequires"] + + modules = buildrequires["platform"]["stream_collision_modules"] + assert ["modulea:10:20180813041838:5ea3b708"] == modules + assert ["pkg1-0:1.0-1.fc28"] == buildrequires["platform"]["ursine_rpms"] + + modules = sorted(buildrequires["project-platform"]["stream_collision_modules"]) + expected_modules = ["bar:6:20181013041838:817fa3a8", "foo:2:20180113041838:95f078a1"] + assert expected_modules == modules + + rpms = sorted(buildrequires["project-platform"]["ursine_rpms"]) + assert ["pkg2-0:2.0-1.fc28", "pkg3-0:3.0-1.fc28"] == rpms + + +class TestFindStreamCollisionModules: + """Test ursine.find_stream_collision_modules""" + + @patch("module_build_service.scheduler.ursine.get_modulemds_from_ursine_content") + def test_no_modulemds_found_from_ursine_content( + self, get_modulemds_from_ursine_content + ): + get_modulemds_from_ursine_content.return_value = [] + assert not ursine.find_stream_collision_modules({}, "koji_tag") + + @patch("module_build_service.scheduler.ursine.get_modulemds_from_ursine_content") + def test_no_collisions_found(self, get_modulemds_from_ursine_content): + xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}} + get_modulemds_from_ursine_content.return_value = [ + make_module("moduler:1:1:c1"), + make_module("modules:2:1:c2"), + make_module("modulet:3:1:c3"), + ] + assert [] == ursine.find_stream_collision_modules( + xmd_mbs_buildrequires, "koji_tag") + + @patch("module_build_service.scheduler.ursine.get_modulemds_from_ursine_content") + def test_collision_modules_are_found(self, get_modulemds_from_ursine_content): + xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}} + fake_modules = [ + make_module("moduler:1:1:c1"), + make_module("moduleb:6:1:c2"), + make_module("modulet:3:1:c3"), + ] + get_modulemds_from_ursine_content.return_value = fake_modules + + modules = ursine.find_stream_collision_modules( + xmd_mbs_buildrequires, "koji_tag") + assert [fake_modules[1].get_nsvc()] == modules diff --git a/tests/test_utils/test_ursine.py b/tests/test_utils/test_ursine.py deleted file mode 100644 index 6e477f5..0000000 --- a/tests/test_utils/test_ursine.py +++ /dev/null @@ -1,357 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from mock import patch, Mock - -from module_build_service import conf -from module_build_service.utils import ursine -from tests import make_module, make_module_in_db, clean_database - - -class TestFindModuleKojiTags: - """Test ursine.find_module_koji_tags""" - - @patch.object(conf, "koji_tag_prefixes", new=["module"]) - def test_find_out_all_module_koji_tags(self): - session = Mock() - session.getFullInheritance.return_value = [ - {"name": "module-tag1-s-v-c"}, - {"name": "module-tag2-s-v-c"}, - {"name": "tag-1"}, - ] - - expected_tags = ["module-tag1-s-v-c", "module-tag2-s-v-c"] - - tags = ursine.find_module_koji_tags(session, "tag-a-build") - assert expected_tags == tags - - @patch.object(conf, "koji_tag_prefixes", new=["module"]) - def test_return_empty_if_no_module_koji_tags(self): - session = Mock() - session.getFullInheritance.return_value = [{"name": "tag-1"}, {"name": "tag-2"}] - - tags = ursine.find_module_koji_tags(session, "tag-a-build") - assert [] == tags - - -class TestFindUrsineRootTags: - """Test ursine.find_build_tags_from_external_repos""" - - def setup_method(self): - self.koji_session = Mock() - self.koji_session.getTag.side_effect = \ - lambda name: None if name == "X-build" else {"name": name} - - def test_find_build_tags(self): - with patch.object( - conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" - ): - tags = ursine.find_build_tags_from_external_repos( - self.koji_session, - [ - { - "external_repo_name": "tag-1-external-repo", - "url": "http://example.com/brewroot/repos/tag-1-build/latest/$arch/", - }, - { - "external_repo_name": "tag-2-external-repo", - "url": "http://example.com/brewroot/repos/tag-2-build/latest/$arch/", - }, - ], - ) - - assert ["tag-1-build", "tag-2-build"] == tags - - def test_return_emtpy_if_no_match_external_repo_url(self): - with patch.object( - conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" - ): - tags = ursine.find_build_tags_from_external_repos( - self.koji_session, - [ - { - "external_repo_name": "tag-1-external-repo", - "url": "https://another-site.org/repos/tag-1-build/latest/$arch/", - }, - { - "external_repo_name": "tag-2-external-repo", - "url": "https://another-site.org/repos/tag-2-build/latest/$arch/", - }, - ], - ) - - assert [] == tags - - def test_some_tag_is_not_koji_tag(self): - with patch.object( - conf, "koji_external_repo_url_prefix", new="http://example.com/brewroot/" - ): - tags = ursine.find_build_tags_from_external_repos( - self.koji_session, - [ - { - "external_repo_name": "tag-1-external-repo", - "url": "http://example.com/brewroot/repos/tag-1-build/latest/$arch/", - }, - { - "external_repo_name": "tag-2-external-repo", - "url": "http://example.com/brewroot/repos/X-build/latest/$arch/", - }, - ], - ) - - assert ["tag-1-build"] == tags - - -class TestGetModulemdsFromUrsineContent: - """Test ursine.get_modulemds_from_ursine_content""" - - def setup_method(self): - clean_database(False) - - def teardown_method(self, test_method): - clean_database() - - @patch("koji.ClientSession") - def test_return_empty_if_no_ursine_build_tag_is_found(self, ClientSession): - koji_session = ClientSession.return_value - - # No module koji_tag in ursine content yet. This will result in empty - # ursine modulemds is returned. - koji_session.getFullInheritance.return_value = [{"name": "tag-1.0-build"}] - koji_session.getExternalRepoList.return_value = [{ - "external_repo_name": "tag-1.0-external-repo", - "url": "http://example.com/repos/tag-4-build/latest/$arch/", - }] - - modulemds = ursine.get_modulemds_from_ursine_content("tag") - assert [] == modulemds - - @patch.object(conf, "koji_tag_prefixes", new=["module"]) - @patch("koji.ClientSession") - def test_get_modulemds(self, ClientSession): - koji_session = ClientSession.return_value - - # Ensure to to get build tag for further query of ursine content. - # For this test, the build tag is tag-4-build - koji_session.getExternalRepoList.return_value = [{ - "external_repo_name": "tag-1.0-external-repo", - "url": "http://example.com/repos/tag-4-build/latest/$arch/", - }] - - # Ensure to return module tags from ursine content of fake build tag - # specified in above external repo's url. - def mock_getFullInheritance(tag): - if tag == "tag-4-build": - return [ - {"name": "tag-1.0-build"}, - # Below two modules should be returned and whose modulemd - # should be also queried from database. - {"name": "module-name1-s-2020-c"}, - {"name": "module-name2-s-2021-c"}, - ] - raise ValueError("{} is not handled by test.".format(tag)) - - koji_session.getFullInheritance.side_effect = mock_getFullInheritance - - # Defaults to DB resolver, so create fake module builds and store them - # into database to ensure they can be queried. - # - # Switched to call function make_session to create a - # SQLAlchemy database session. Calling db.session causes failure to - # read attributes from a ModuleBuild object at following line calling - # mmd(). The error is ModuleBuild object is not bound to a Session. - # From the behavior of following code, the reason of the error is - # mixing use of db.session and make_session, the latter one is called - # from function ``get_modulemds_from_ursine_content``. - mmd_name1s2020c = make_module_in_db( - "name1:s:2020:c", - xmd={"mbs": {"koji_tag": "module-name1-s-2020-c"}}, - ) - mmd_name2s2021c = make_module_in_db( - "name2:s:2021:c", - xmd={"mbs": {"koji_tag": "module-name2-s-2021-c"}}, - ) - - koji_tag = "tag" # It's ok to use arbitrary tag name. - with patch.object(conf, "koji_external_repo_url_prefix", new="http://example.com/"): - modulemds = ursine.get_modulemds_from_ursine_content(koji_tag) - - test_nsvcs = [item.get_nsvc() for item in modulemds] - test_nsvcs.sort() - - expected_nsvcs = [mmd_name1s2020c.mmd().get_nsvc(), mmd_name2s2021c.mmd().get_nsvc()] - expected_nsvcs.sort() - - koji_session.getExternalRepoList.assert_called_once_with(koji_tag) - assert expected_nsvcs == test_nsvcs - - -class TestRecordStreamCollisionModules: - """Test ursine.record_stream_collision_modules""" - - @patch.object(conf, "base_module_names", new=["platform"]) - @patch.object(ursine, "find_stream_collision_modules") - def test_nothing_changed_if_no_base_module_is_in_buildrequires( - self, find_stream_collision_modules - ): - xmd = {"mbs": {"buildrequires": {"modulea": {"stream": "master"}}}} - fake_mmd = make_module("name1:s:2020:c", xmd=xmd) - original_xmd = fake_mmd.get_xmd() - - with patch.object(ursine, "log") as log: - ursine.handle_stream_collision_modules(fake_mmd) - assert 2 == log.info.call_count - find_stream_collision_modules.assert_not_called() - - assert original_xmd == fake_mmd.get_xmd() - - @patch.object(conf, "base_module_names", new=["platform"]) - @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content") - def test_mark_handled_even_if_no_modules_in_ursine_content( - self, get_modulemds_from_ursine_content - ): - xmd = { - "mbs": { - "buildrequires": { - "modulea": {"stream": "master"}, - "platform": {"stream": "master", "koji_tag": "module-rhel-8.0-build"}, - } - } - } - fake_mmd = make_module("name1:s:2020:c", xmd=xmd) - expected_xmd = fake_mmd.get_xmd() - - get_modulemds_from_ursine_content.return_value = [] - - with patch.object(ursine, "log") as log: - ursine.handle_stream_collision_modules(fake_mmd) - assert 2 == log.info.call_count - - # Ensure stream_collision_modules is set. - expected_xmd["mbs"]["buildrequires"]["platform"]["stream_collision_modules"] = "" - expected_xmd["mbs"]["buildrequires"]["platform"]["ursine_rpms"] = "" - assert expected_xmd == fake_mmd.get_xmd() - - @patch.object(conf, "base_module_names", new=["platform", "project-platform"]) - @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content") - @patch("module_build_service.resolver.GenericResolver.create") - @patch("koji.ClientSession") - def test_add_collision_modules( - self, ClientSession, resolver_create, get_modulemds_from_ursine_content - ): - xmd = { - "mbs": { - "buildrequires": { - "modulea": {"stream": "master"}, - "foo": {"stream": "1"}, - "bar": {"stream": "2"}, - "platform": {"stream": "master", "koji_tag": "module-rhel-8.0-build"}, - "project-platform": { - "stream": "master", - "koji_tag": "module-project-1.0-build", - }, - } - } - } - fake_mmd = make_module("name1:s:2020:c", xmd=xmd) - - def mock_get_ursine_modulemds(koji_tag): - if koji_tag == "module-rhel-8.0-build": - return [ - # This is the one - make_module("modulea:10:20180813041838:5ea3b708"), - make_module("moduleb:1.0:20180113042038:6ea3b105"), - ] - if koji_tag == "module-project-1.0-build": - return [ - # Both of them are the collided modules - make_module("bar:6:20181013041838:817fa3a8"), - make_module("foo:2:20180113041838:95f078a1"), - ] - - get_modulemds_from_ursine_content.side_effect = mock_get_ursine_modulemds - - # Mock for finding out built rpms - def mock_get_module(name, stream, version, context, strict=True): - return { - "modulea:10:20180813041838:5ea3b708": { - "koji_tag": "module-modulea-10-20180813041838-5ea3b708" - }, - "bar:6:20181013041838:817fa3a8": { - "koji_tag": "module-bar-6-20181013041838-817fa3a8" - }, - "foo:2:20180113041838:95f078a1": { - "koji_tag": "module-foo-2-20180113041838-95f078a1" - }, - }["{}:{}:{}:{}".format(name, stream, version, context)] - - resolver = resolver_create.return_value - resolver.get_module.side_effect = mock_get_module - - def mock_listTaggedRPMS(tag, latest): - return { - "module-modulea-10-20180813041838-5ea3b708": [ - [{"name": "pkg1", "version": "1.0", "release": "1.fc28", "epoch": None}] - ], - "module-bar-6-20181013041838-817fa3a8": [ - [{"name": "pkg2", "version": "2.0", "release": "1.fc28", "epoch": None}] - ], - "module-foo-2-20180113041838-95f078a1": [ - [{"name": "pkg3", "version": "3.0", "release": "1.fc28", "epoch": None}] - ], - }[tag] - - koji_session = ClientSession.return_value - koji_session.listTaggedRPMS.side_effect = mock_listTaggedRPMS - - ursine.handle_stream_collision_modules(fake_mmd) - - xmd = fake_mmd.get_xmd() - buildrequires = xmd["mbs"]["buildrequires"] - - modules = buildrequires["platform"]["stream_collision_modules"] - assert ["modulea:10:20180813041838:5ea3b708"] == modules - assert ["pkg1-0:1.0-1.fc28"] == buildrequires["platform"]["ursine_rpms"] - - modules = sorted(buildrequires["project-platform"]["stream_collision_modules"]) - expected_modules = ["bar:6:20181013041838:817fa3a8", "foo:2:20180113041838:95f078a1"] - assert expected_modules == modules - - rpms = sorted(buildrequires["project-platform"]["ursine_rpms"]) - assert ["pkg2-0:2.0-1.fc28", "pkg3-0:3.0-1.fc28"] == rpms - - -class TestFindStreamCollisionModules: - """Test ursine.find_stream_collision_modules""" - - @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content") - def test_no_modulemds_found_from_ursine_content( - self, get_modulemds_from_ursine_content - ): - get_modulemds_from_ursine_content.return_value = [] - assert not ursine.find_stream_collision_modules({}, "koji_tag") - - @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content") - def test_no_collisions_found(self, get_modulemds_from_ursine_content): - xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}} - get_modulemds_from_ursine_content.return_value = [ - make_module("moduler:1:1:c1"), - make_module("modules:2:1:c2"), - make_module("modulet:3:1:c3"), - ] - assert [] == ursine.find_stream_collision_modules( - xmd_mbs_buildrequires, "koji_tag") - - @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content") - def test_collision_modules_are_found(self, get_modulemds_from_ursine_content): - xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}} - fake_modules = [ - make_module("moduler:1:1:c1"), - make_module("moduleb:6:1:c2"), - make_module("modulet:3:1:c3"), - ] - get_modulemds_from_ursine_content.return_value = fake_modules - - modules = ursine.find_stream_collision_modules( - xmd_mbs_buildrequires, "koji_tag") - assert [fake_modules[1].get_nsvc()] == modules