From 78d5a9e48213278e23743ff61cd35c72eefcb7c5 Mon Sep 17 00:00:00 2001 From: mprahl Date: Jan 16 2020 19:39:46 +0000 Subject: Split utils/mse.py This moves the code used by the backend and API to common/resolve.py and moves the code used just by the API to web/mse.py. --- diff --git a/module_build_service/common/resolve.py b/module_build_service/common/resolve.py new file mode 100644 index 0000000..2436c5f --- /dev/null +++ b/module_build_service/common/resolve.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from module_build_service import conf +from module_build_service.errors import StreamAmbigous +from module_build_service.resolver import GenericResolver + + +def expand_single_mse_streams( + db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False): + """ + Helper method for `expand_mse_stream()` expanding single name:[streams]. + Returns list of expanded streams. + + :param db_session: SQLAlchemy DB session. + :param str name: Name of the module which will be expanded. + :param streams: List of streams to expand. + :type streams: list[str] + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + """ + from module_build_service import models + + default_streams = default_streams or {} + # Stream can be prefixed with '-' sign to define that this stream should + # not appear in a resulting list of streams. There can be two situations: + # a) all streams have '-' prefix. In this case, we treat list of streams + # as blacklist and we find all the valid streams and just remove those with + # '-' prefix. + # b) there is at least one stream without '-' prefix. In this case, we can + # ignore all the streams with '-' prefix and just add those without + # '-' prefix to the list of valid streams. + streams_is_blacklist = all(stream.startswith("-") for stream in streams) + if streams_is_blacklist or len(streams) == 0: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous("There are multiple streams to choose from for module %s." % name) + else: + builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name) + expanded_streams = [build.stream for build in builds] + else: + expanded_streams = [] + for stream in streams: + if stream.startswith("-"): + if streams_is_blacklist and stream[1:] in expanded_streams: + expanded_streams.remove(stream[1:]) + else: + expanded_streams.append(stream) + + if len(expanded_streams) > 1: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous( + "There are multiple streams %r to choose from for module %s." + % (expanded_streams, name) + ) + + return expanded_streams + + +def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None): + """ + Returns dict containing the base modules compatible with `base_mmd` grouped by their state. + + :param GenericResolver resolver: GenericResolver instance. + :param Modulemd base_mmd: Modulemd instant to return compatible modules for. + :param set ignore_ns: When set, defines the name:stream of base modules which will be ignored + by this function and therefore not returned. + :return dict: Dictionary with module's state name as a key and list of Modulemd objects for + each compatible base module in that state. For example: + { + "ready": [base_mmd_1, base_mmd_2] + "garbage": [base_mmd_3] + } + The input `base_mmd` is always included in the result in "ready" state. + """ + from module_build_service import models + + # Add the module to `seen` and `ret`. + ret = {"ready": [], "garbage": []} + ret["ready"].append(base_mmd) + ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()]) + seen = set() if not ignore_ns else set(ignore_ns) + seen.add(ns) + + # Get the list of compatible virtual streams. + xmd = base_mmd.get_xmd() + virtual_streams = xmd.get("mbs", {}).get("virtual_streams") + + # In case there are no virtual_streams in the buildrequired name:stream, + # it is clear that there are no compatible streams, so return just this + # `base_mmd`. + if not virtual_streams: + return ret + + if conf.allow_only_compatible_base_modules: + stream_version_lte = True + states = ["ready"] + else: + stream_version_lte = False + states = ["ready", "garbage"] + + for state in states: + mmds = resolver.get_compatible_base_module_modulemds( + base_mmd, stream_version_lte, virtual_streams, + [models.BUILD_STATES[state]]) + ret_chunk = [] + # Add the returned mmds to the `seen` set to avoid querying those + # individually if they are part of the buildrequire streams for this + # base module. + for mmd_ in mmds: + mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) + # An extra precaution to ensure there are no duplicates. This can happen when there + # are two modules with the same name:stream - one in ready state and one in garbage + # state. + if mmd_ns not in seen: + ret_chunk.append(mmd_) + seen.add(mmd_ns) + ret[state] += ret_chunk + + return ret + + +def get_base_module_mmds(db_session, mmd): + """ + Returns list of MMDs of base modules buildrequired by `mmd` including the compatible + old versions of the base module based on the stream version. + + :param Modulemd mmd: Input modulemd metadata. + :rtype: dict with lists of Modulemd + :return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules + buildrequired by `mmd` as a value. + """ + from module_build_service import models + + seen = set() + ret = {"ready": [], "garbage": []} + + resolver = GenericResolver.create(db_session, conf) + for deps in mmd.get_dependencies(): + buildrequires = { + module: deps.get_buildtime_streams(module) + for module in deps.get_buildtime_modules() + } + for name in conf.base_module_names: + if name not in buildrequires.keys(): + continue + + # Since the query below uses stream_version_lte, we can sort the streams by stream + # version in descending order to not perform unnecessary queries. Otherwise, if the + # streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be + # returned twice. Only one query for that scenario is necessary. + sorted_desc_streams = sorted( + buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version) + for stream in sorted_desc_streams: + ns = ":".join([name, stream]) + if ns in seen: + continue + + # Get the MMD for particular buildrequired name:stream to find out the virtual + # streams according to which we can find the compatible streams later. + # The returned MMDs are all the module builds for name:stream in the highest + # version. Given the base module does not depend on other modules, it can appear + # only in single context and therefore the `mmds` should always contain just + # zero or one module build. + mmds = resolver.get_module_modulemds(name, stream) + if not mmds: + continue + base_mmd = mmds[0] + + new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen) + for state in new_ret.keys(): + for mmd_ in new_ret[state]: + mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) + seen.add(mmd_ns) + ret[state] += new_ret[state] + break + return ret diff --git a/module_build_service/scheduler/default_modules.py b/module_build_service/scheduler/default_modules.py index e81ac7d..074bfed 100644 --- a/module_build_service/scheduler/default_modules.py +++ b/module_build_service/scheduler/default_modules.py @@ -12,12 +12,13 @@ import six.moves.xmlrpc_client as xmlrpclib from module_build_service import conf, log, models, Modulemd, scm from module_build_service.common.koji import get_session, koji_retrying_multicall_map +from module_build_service.common.resolve import ( + expand_single_mse_streams, get_compatible_base_module_mmds +) from module_build_service.common.retry import retry from module_build_service.db_session import db_session from module_build_service.errors import UnprocessableEntity from module_build_service.resolver.base import GenericResolver -from module_build_service.utils.mse import ( - get_compatible_base_module_mmds, expand_single_mse_streams) def add_default_modules(mmd): diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index 4b880ac..565b544 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -from module_build_service.utils.mse import * # noqa from module_build_service.utils.views import * # noqa from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa diff --git a/module_build_service/utils/mse.py b/module_build_service/utils/mse.py deleted file mode 100644 index 420f9d3..0000000 --- a/module_build_service/utils/mse.py +++ /dev/null @@ -1,541 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from module_build_service import log, models, Modulemd, conf -from module_build_service.common.utils import mmd_to_str -from module_build_service.errors import StreamAmbigous -from module_build_service.errors import UnprocessableEntity -from module_build_service.mmd_resolver import MMDResolver -from module_build_service.web.utils import deps_to_dict -from module_build_service.resolver import GenericResolver - - -def expand_single_mse_streams( - db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False): - """ - Helper method for `expand_mse_stream()` expanding single name:[streams]. - Returns list of expanded streams. - - :param db_session: SQLAlchemy DB session. - :param str name: Name of the module which will be expanded. - :param streams: List of streams to expand. - :type streams: list[str] - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - """ - default_streams = default_streams or {} - # Stream can be prefixed with '-' sign to define that this stream should - # not appear in a resulting list of streams. There can be two situations: - # a) all streams have '-' prefix. In this case, we treat list of streams - # as blacklist and we find all the valid streams and just remove those with - # '-' prefix. - # b) there is at least one stream without '-' prefix. In this case, we can - # ignore all the streams with '-' prefix and just add those without - # '-' prefix to the list of valid streams. - streams_is_blacklist = all(stream.startswith("-") for stream in streams) - if streams_is_blacklist or len(streams) == 0: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous("There are multiple streams to choose from for module %s." % name) - else: - builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name) - expanded_streams = [build.stream for build in builds] - else: - expanded_streams = [] - for stream in streams: - if stream.startswith("-"): - if streams_is_blacklist and stream[1:] in expanded_streams: - expanded_streams.remove(stream[1:]) - else: - expanded_streams.append(stream) - - if len(expanded_streams) > 1: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous( - "There are multiple streams %r to choose from for module %s." - % (expanded_streams, name) - ) - - return expanded_streams - - -def expand_mse_streams(db_session, mmd, default_streams=None, raise_if_stream_ambigous=False): - """ - Expands streams in both buildrequires/requires sections of MMD. - - :param db_session: SQLAlchemy DB session. - :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - """ - for deps in mmd.get_dependencies(): - new_deps = Modulemd.Dependencies() - for name in deps.get_runtime_modules(): - streams = deps.get_runtime_streams(name) - new_streams = expand_single_mse_streams( - db_session, name, streams, default_streams, raise_if_stream_ambigous) - - if not new_streams: - new_deps.set_empty_runtime_dependencies_for_module(name) - else: - for stream in new_streams: - new_deps.add_runtime_stream(name, stream) - - for name in deps.get_buildtime_modules(): - streams = deps.get_buildtime_streams(name) - new_streams = expand_single_mse_streams( - db_session, name, streams, default_streams, raise_if_stream_ambigous) - - if not new_streams: - new_deps.set_empty_buildtime_dependencies_for_module(name) - else: - for stream in new_streams: - new_deps.add_buildtime_stream(name, stream) - - # Replace the Dependencies object - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - -def _get_mmds_from_requires( - db_session, - requires, - mmds, - recursive=False, - default_streams=None, - raise_if_stream_ambigous=False, - base_module_mmds=None, -): - """ - Helper method for get_mmds_required_by_module_recursively returning - the list of module metadata objects defined by `requires` dict. - - :param db_session: SQLAlchemy database session. - :param dict requires: requires or buildrequires in the form {module: [streams]} - :param mmds: Dictionary with already handled name:streams as a keys and lists - of resulting mmds as values. - :param recursive: If True, the requires are checked recursively. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :param list base_module_mmds: List of modulemd metadata instances. When set, the - returned list contains MMDs build against each base module defined in - `base_module_mmds` list. - :return: Dict with name:stream as a key and list with mmds as value. - """ - default_streams = default_streams or {} - # To be able to call itself recursively, we need to store list of mmds - # we have added to global mmds list in this particular call. - added_mmds = {} - resolver = GenericResolver.create(db_session, conf) - - for name, streams in requires.items(): - # Base modules are already added to `mmds`. - if name in conf.base_module_names: - continue - - streams_to_try = streams - if name in default_streams: - streams_to_try = [default_streams[name]] - elif len(streams_to_try) > 1 and raise_if_stream_ambigous: - raise StreamAmbigous( - "There are multiple streams %r to choose from for module %s." - % (streams_to_try, name) - ) - - # For each valid stream, find the last build in a stream and also all - # its contexts and add mmds of these builds to `mmds` and `added_mmds`. - # Of course only do that if we have not done that already in some - # previous call of this method. - for stream in streams: - ns = "%s:%s" % (name, stream) - if ns not in mmds: - mmds[ns] = [] - if ns not in added_mmds: - added_mmds[ns] = [] - - if base_module_mmds: - for base_module_mmd in base_module_mmds: - mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_mmd) - else: - mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) - added_mmds[ns] += mmds[ns] - - # Get the requires recursively. - if recursive: - for mmd_list in added_mmds.values(): - for mmd in mmd_list: - for deps in mmd.get_dependencies(): - deps_dict = deps_to_dict(deps, 'runtime') - mmds = _get_mmds_from_requires( - db_session, deps_dict, mmds, True, base_module_mmds=base_module_mmds) - - return mmds - - -def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None): - """ - Returns dict containing the base modules compatible with `base_mmd` grouped by their state. - - :param GenericResolver resolver: GenericResolver instance. - :param Modulemd base_mmd: Modulemd instant to return compatible modules for. - :param set ignore_ns: When set, defines the name:stream of base modules which will be ignored - by this function and therefore not returned. - :return dict: Dictionary with module's state name as a key and list of Modulemd objects for - each compatible base module in that state. For example: - { - "ready": [base_mmd_1, base_mmd_2] - "garbage": [base_mmd_3] - } - The input `base_mmd` is always included in the result in "ready" state. - """ - # Add the module to `seen` and `ret`. - ret = {"ready": [], "garbage": []} - ret["ready"].append(base_mmd) - ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()]) - seen = set() if not ignore_ns else set(ignore_ns) - seen.add(ns) - - # Get the list of compatible virtual streams. - xmd = base_mmd.get_xmd() - virtual_streams = xmd.get("mbs", {}).get("virtual_streams") - - # In case there are no virtual_streams in the buildrequired name:stream, - # it is clear that there are no compatible streams, so return just this - # `base_mmd`. - if not virtual_streams: - return ret - - if conf.allow_only_compatible_base_modules: - stream_version_lte = True - states = ["ready"] - else: - stream_version_lte = False - states = ["ready", "garbage"] - - for state in states: - mmds = resolver.get_compatible_base_module_modulemds( - base_mmd, stream_version_lte, virtual_streams, - [models.BUILD_STATES[state]]) - ret_chunk = [] - # Add the returned mmds to the `seen` set to avoid querying those - # individually if they are part of the buildrequire streams for this - # base module. - for mmd_ in mmds: - mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) - # An extra precaution to ensure there are no duplicates. This can happen when there - # are two modules with the same name:stream - one in ready state and one in garbage - # state. - if mmd_ns not in seen: - ret_chunk.append(mmd_) - seen.add(mmd_ns) - ret[state] += ret_chunk - - return ret - - -def get_base_module_mmds(db_session, mmd): - """ - Returns list of MMDs of base modules buildrequired by `mmd` including the compatible - old versions of the base module based on the stream version. - - :param Modulemd mmd: Input modulemd metadata. - :rtype: dict with lists of Modulemd - :return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules - buildrequired by `mmd` as a value. - """ - seen = set() - ret = {"ready": [], "garbage": []} - - resolver = GenericResolver.create(db_session, conf) - for deps in mmd.get_dependencies(): - buildrequires = { - module: deps.get_buildtime_streams(module) - for module in deps.get_buildtime_modules() - } - for name in conf.base_module_names: - if name not in buildrequires.keys(): - continue - - # Since the query below uses stream_version_lte, we can sort the streams by stream - # version in descending order to not perform unnecessary queries. Otherwise, if the - # streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be - # returned twice. Only one query for that scenario is necessary. - sorted_desc_streams = sorted( - buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version) - for stream in sorted_desc_streams: - ns = ":".join([name, stream]) - if ns in seen: - continue - - # Get the MMD for particular buildrequired name:stream to find out the virtual - # streams according to which we can find the compatible streams later. - # The returned MMDs are all the module builds for name:stream in the highest - # version. Given the base module does not depend on other modules, it can appear - # only in single context and therefore the `mmds` should always contain just - # zero or one module build. - mmds = resolver.get_module_modulemds(name, stream) - if not mmds: - continue - base_mmd = mmds[0] - - new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen) - for state in new_ret.keys(): - for mmd_ in new_ret[state]: - mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) - seen.add(mmd_ns) - ret[state] += new_ret[state] - break - return ret - - -def get_mmds_required_by_module_recursively( - db_session, mmd, default_streams=None, raise_if_stream_ambigous=False -): - """ - Returns the list of Module metadata objects of all modules required while - building the module defined by `mmd` module metadata. This presumes the - module metadata streams are expanded using `expand_mse_streams(...)` - method. - - This method finds out latest versions of all the build-requires of - the `mmd` module and then also all contexts of these latest versions. - - For each build-required name:stream:version:context module, it checks - recursively all the "requires" and finds the latest version of each - required module and also all contexts of these latest versions. - - :param db_session: SQLAlchemy database session. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :rtype: list of Modulemd metadata - :return: List of all modulemd metadata of all modules required to build - the module `mmd`. - """ - # We use dict with name:stream as a key and list with mmds as value. - # That way, we can ensure we won't have any duplicate mmds in a resulting - # list and we also don't waste resources on getting the modules we already - # handled from DB. - mmds = {} - - # Get the MMDs of all compatible base modules based on the buildrequires. - base_module_mmds = get_base_module_mmds(db_session, mmd) - if not base_module_mmds["ready"]: - base_module_choices = " or ".join(conf.base_module_names) - raise UnprocessableEntity( - "None of the base module ({}) streams in the buildrequires section could be found" - .format(base_module_choices) - ) - - # Add base modules to `mmds`. - for base_module in base_module_mmds["ready"]: - ns = ":".join([base_module.get_module_name(), base_module.get_stream_name()]) - mmds.setdefault(ns, []) - mmds[ns].append(base_module) - - # The currently submitted module build must be built only against "ready" base modules, - # but its dependencies might have been built against some old platform which is already - # EOL ("garbage" state). In order to find such old module builds, we need to include - # also EOL platform streams. - all_base_module_mmds = base_module_mmds["ready"] + base_module_mmds["garbage"] - - # Get all the buildrequires of the module of interest. - for deps in mmd.get_dependencies(): - deps_dict = deps_to_dict(deps, 'buildtime') - mmds = _get_mmds_from_requires( - db_session, deps_dict, mmds, False, default_streams, raise_if_stream_ambigous, - all_base_module_mmds) - - # Now get the requires of buildrequires recursively. - for mmd_key in list(mmds.keys()): - for mmd in mmds[mmd_key]: - for deps in mmd.get_dependencies(): - deps_dict = deps_to_dict(deps, 'runtime') - mmds = _get_mmds_from_requires( - db_session, deps_dict, mmds, True, default_streams, - raise_if_stream_ambigous, all_base_module_mmds) - - # Make single list from dict of lists. - res = [] - for ns, mmds_list in mmds.items(): - if len(mmds_list) == 0: - raise UnprocessableEntity("Cannot find any module builds for %s" % (ns)) - res += mmds_list - return res - - -def generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous=False, default_streams=None): - """ - Returns list with MMDs with buildrequires and requires set according - to module stream expansion rules. These module metadata can be directly - built using MBS. - - :param db_session: SQLAlchemy DB session. - :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - """ - if not default_streams: - default_streams = {} - - # Create local copy of mmd, because we will expand its dependencies, - # which would change the module. - current_mmd = mmd.copy() - - # MMDResolver expects the input MMD to have no context. - current_mmd.set_context(None) - - # Expands the MSE streams. This mainly handles '-' prefix in MSE streams. - expand_mse_streams(db_session, current_mmd, default_streams, raise_if_stream_ambigous) - - # Get the list of all MMDs which this module can be possibly built against - # and add them to MMDResolver. - mmd_resolver = MMDResolver() - mmds_for_resolving = get_mmds_required_by_module_recursively( - db_session, current_mmd, default_streams, raise_if_stream_ambigous) - for m in mmds_for_resolving: - mmd_resolver.add_modules(m) - - # Show log.info message with the NSVCs we have added to mmd_resolver. - nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving] - log.info("Starting resolving with following input modules: %r", nsvcs_to_solve) - - # Resolve the dependencies between modules and get the list of all valid - # combinations in which we can build this module. - requires_combinations = mmd_resolver.solve(current_mmd) - log.info("Resolving done, possible requires: %r", requires_combinations) - - # This is where we are going to store the generated MMDs. - mmds = [] - for requires in requires_combinations: - # Each generated MMD must be new Module object... - mmd_copy = mmd.copy() - xmd = mmd_copy.get_xmd() - - # Requires contain the NSVC representing the input mmd. - # The 'context' of this NSVC defines the id of buildrequires/requires - # pair in the mmd.get_dependencies(). - dependencies_id = None - - # We don't want to depend on ourselves, so store the NSVC of the current_mmd - # to be able to ignore it later. - self_nsvca = None - - # Dict to store name:stream pairs from nsvca, so we are able to access it - # easily later. - req_name_stream = {} - - # Get the values for dependencies_id, self_nsvca and req_name_stream variables. - for nsvca in requires: - req_name, req_stream, _, req_context, req_arch = nsvca.split(":") - if req_arch == "src": - assert req_name == current_mmd.get_module_name() - assert req_stream == current_mmd.get_stream_name() - assert dependencies_id is None - assert self_nsvca is None - dependencies_id = int(req_context) - self_nsvca = nsvca - continue - req_name_stream[req_name] = req_stream - if dependencies_id is None or self_nsvca is None: - raise RuntimeError( - "%s:%s not found in requires %r" - % (current_mmd.get_module_name(), current_mmd.get_stream_name(), requires) - ) - - # The name:[streams, ...] pairs do not have to be the same in both - # buildrequires/requires. In case they are the same, we replace the streams - # in requires section with a single stream against which we will build this MMD. - # In case they are not the same, we have to keep the streams as they are in requires - # section. We always replace stream(s) for build-requirement with the one we - # will build this MMD against. - new_deps = Modulemd.Dependencies() - deps = mmd_copy.get_dependencies()[dependencies_id] - deps_requires = deps_to_dict(deps, 'runtime') - deps_buildrequires = deps_to_dict(deps, 'buildtime') - for req_name, req_streams in deps_requires.items(): - if req_name not in deps_buildrequires: - # This require is not a buildrequire so just copy this runtime requirement to - # new_dep and don't touch buildrequires - if not req_streams: - new_deps.set_empty_runtime_dependencies_for_module(req_name) - else: - for req_stream in req_streams: - new_deps.add_runtime_stream(req_name, req_stream) - elif set(req_streams) != set(deps_buildrequires[req_name]): - # Streams in runtime section are not the same as in buildtime section, - # so just copy this runtime requirement to new_dep. - if not req_streams: - new_deps.set_empty_runtime_dependencies_for_module(req_name) - else: - for req_stream in req_streams: - new_deps.add_runtime_stream(req_name, req_stream) - - new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) - else: - # This runtime requirement has the same streams in both runtime/buildtime - # requires sections, so replace streams in both sections by the one we - # really used in this resolved variant. - new_deps.add_runtime_stream(req_name, req_name_stream[req_name]) - new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) - - # There might be buildrequires which are not in runtime requires list. - # Such buildrequires must be copied to expanded MMD. - for req_name, req_streams in deps_buildrequires.items(): - if req_name not in deps_requires: - new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) - - # Set the new dependencies. - mmd_copy.remove_dependencies(deps) - mmd_copy.add_dependencies(new_deps) - - # The Modulemd.Dependencies() stores only streams, but to really build this - # module, we need NSVC of buildrequires, so we have to store this data in XMD. - # We also need additional data like for example list of filtered_rpms. We will - # get them using module_build_service.resolver.GenericResolver.resolve_requires, - # so prepare list with NSVCs of buildrequires as an input for this method. - br_list = [] - for nsvca in requires: - if nsvca == self_nsvca: - continue - # Remove the arch from nsvca - nsvc = ":".join(nsvca.split(":")[:-1]) - br_list.append(nsvc) - - # Resolve the buildrequires and store the result in XMD. - if "mbs" not in xmd: - xmd["mbs"] = {} - resolver = GenericResolver.create(db_session, conf) - xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list) - xmd["mbs"]["mse"] = True - - mmd_copy.set_xmd(xmd) - - # Now we have all the info to actually compute context of this module. - context = models.ModuleBuild.contexts_from_mmd(mmd_to_str(mmd_copy)).context - mmd_copy.set_context(context) - - mmds.append(mmd_copy) - - return mmds diff --git a/module_build_service/utils/reuse.py b/module_build_service/utils/reuse.py index a9163b4..6e4672c 100644 --- a/module_build_service/utils/reuse.py +++ b/module_build_service/utils/reuse.py @@ -6,7 +6,7 @@ from module_build_service import log, models, conf from module_build_service.db_session import db_session from module_build_service.resolver import GenericResolver from module_build_service.scheduler import events -from module_build_service.utils.mse import get_base_module_mmds +from module_build_service.common.resolve import get_base_module_mmds def reuse_component(component, previous_component_build, change_state_now=False, diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 0c141a5..b331777 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -20,6 +20,7 @@ from module_build_service import conf, log, models, Modulemd from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str, to_text_type from module_build_service.db_session import db_session from module_build_service.errors import ValidationError, UnprocessableEntity, Forbidden, Conflict +from module_build_service.web.mse import generate_expanded_mmds from module_build_service.web.utils import deps_to_dict @@ -955,8 +956,6 @@ def submit_module_build(db_session, username, mmd, params): :rtype: list with ModuleBuild :return: List with submitted module builds. """ - from .mse import generate_expanded_mmds - log.debug( "Submitted %s module build for %s:%s:%s", ("scratch" if params.get("scratch", False) else "normal"), diff --git a/module_build_service/web/mse.py b/module_build_service/web/mse.py new file mode 100644 index 0000000..d5efdb9 --- /dev/null +++ b/module_build_service/web/mse.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from module_build_service import log, models, Modulemd, conf +from module_build_service.common.resolve import expand_single_mse_streams, get_base_module_mmds +from module_build_service.common.utils import mmd_to_str +from module_build_service.errors import StreamAmbigous +from module_build_service.errors import UnprocessableEntity +from module_build_service.mmd_resolver import MMDResolver +from module_build_service.web.utils import deps_to_dict +from module_build_service.resolver import GenericResolver + + +def expand_mse_streams(db_session, mmd, default_streams=None, raise_if_stream_ambigous=False): + """ + Expands streams in both buildrequires/requires sections of MMD. + + :param db_session: SQLAlchemy DB session. + :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + """ + for deps in mmd.get_dependencies(): + new_deps = Modulemd.Dependencies() + for name in deps.get_runtime_modules(): + streams = deps.get_runtime_streams(name) + new_streams = expand_single_mse_streams( + db_session, name, streams, default_streams, raise_if_stream_ambigous) + + if not new_streams: + new_deps.set_empty_runtime_dependencies_for_module(name) + else: + for stream in new_streams: + new_deps.add_runtime_stream(name, stream) + + for name in deps.get_buildtime_modules(): + streams = deps.get_buildtime_streams(name) + new_streams = expand_single_mse_streams( + db_session, name, streams, default_streams, raise_if_stream_ambigous) + + if not new_streams: + new_deps.set_empty_buildtime_dependencies_for_module(name) + else: + for stream in new_streams: + new_deps.add_buildtime_stream(name, stream) + + # Replace the Dependencies object + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + +def _get_mmds_from_requires( + db_session, + requires, + mmds, + recursive=False, + default_streams=None, + raise_if_stream_ambigous=False, + base_module_mmds=None, +): + """ + Helper method for get_mmds_required_by_module_recursively returning + the list of module metadata objects defined by `requires` dict. + + :param db_session: SQLAlchemy database session. + :param dict requires: requires or buildrequires in the form {module: [streams]} + :param mmds: Dictionary with already handled name:streams as a keys and lists + of resulting mmds as values. + :param recursive: If True, the requires are checked recursively. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :param list base_module_mmds: List of modulemd metadata instances. When set, the + returned list contains MMDs build against each base module defined in + `base_module_mmds` list. + :return: Dict with name:stream as a key and list with mmds as value. + """ + default_streams = default_streams or {} + # To be able to call itself recursively, we need to store list of mmds + # we have added to global mmds list in this particular call. + added_mmds = {} + resolver = GenericResolver.create(db_session, conf) + + for name, streams in requires.items(): + # Base modules are already added to `mmds`. + if name in conf.base_module_names: + continue + + streams_to_try = streams + if name in default_streams: + streams_to_try = [default_streams[name]] + elif len(streams_to_try) > 1 and raise_if_stream_ambigous: + raise StreamAmbigous( + "There are multiple streams %r to choose from for module %s." + % (streams_to_try, name) + ) + + # For each valid stream, find the last build in a stream and also all + # its contexts and add mmds of these builds to `mmds` and `added_mmds`. + # Of course only do that if we have not done that already in some + # previous call of this method. + for stream in streams: + ns = "%s:%s" % (name, stream) + if ns not in mmds: + mmds[ns] = [] + if ns not in added_mmds: + added_mmds[ns] = [] + + if base_module_mmds: + for base_module_mmd in base_module_mmds: + mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_mmd) + else: + mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) + added_mmds[ns] += mmds[ns] + + # Get the requires recursively. + if recursive: + for mmd_list in added_mmds.values(): + for mmd in mmd_list: + for deps in mmd.get_dependencies(): + deps_dict = deps_to_dict(deps, 'runtime') + mmds = _get_mmds_from_requires( + db_session, deps_dict, mmds, True, base_module_mmds=base_module_mmds) + + return mmds + + +def get_mmds_required_by_module_recursively( + db_session, mmd, default_streams=None, raise_if_stream_ambigous=False +): + """ + Returns the list of Module metadata objects of all modules required while + building the module defined by `mmd` module metadata. This presumes the + module metadata streams are expanded using `expand_mse_streams(...)` + method. + + This method finds out latest versions of all the build-requires of + the `mmd` module and then also all contexts of these latest versions. + + For each build-required name:stream:version:context module, it checks + recursively all the "requires" and finds the latest version of each + required module and also all contexts of these latest versions. + + :param db_session: SQLAlchemy database session. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :rtype: list of Modulemd metadata + :return: List of all modulemd metadata of all modules required to build + the module `mmd`. + """ + # We use dict with name:stream as a key and list with mmds as value. + # That way, we can ensure we won't have any duplicate mmds in a resulting + # list and we also don't waste resources on getting the modules we already + # handled from DB. + mmds = {} + + # Get the MMDs of all compatible base modules based on the buildrequires. + base_module_mmds = get_base_module_mmds(db_session, mmd) + if not base_module_mmds["ready"]: + base_module_choices = " or ".join(conf.base_module_names) + raise UnprocessableEntity( + "None of the base module ({}) streams in the buildrequires section could be found" + .format(base_module_choices) + ) + + # Add base modules to `mmds`. + for base_module in base_module_mmds["ready"]: + ns = ":".join([base_module.get_module_name(), base_module.get_stream_name()]) + mmds.setdefault(ns, []) + mmds[ns].append(base_module) + + # The currently submitted module build must be built only against "ready" base modules, + # but its dependencies might have been built against some old platform which is already + # EOL ("garbage" state). In order to find such old module builds, we need to include + # also EOL platform streams. + all_base_module_mmds = base_module_mmds["ready"] + base_module_mmds["garbage"] + + # Get all the buildrequires of the module of interest. + for deps in mmd.get_dependencies(): + deps_dict = deps_to_dict(deps, 'buildtime') + mmds = _get_mmds_from_requires( + db_session, deps_dict, mmds, False, default_streams, raise_if_stream_ambigous, + all_base_module_mmds) + + # Now get the requires of buildrequires recursively. + for mmd_key in list(mmds.keys()): + for mmd in mmds[mmd_key]: + for deps in mmd.get_dependencies(): + deps_dict = deps_to_dict(deps, 'runtime') + mmds = _get_mmds_from_requires( + db_session, deps_dict, mmds, True, default_streams, + raise_if_stream_ambigous, all_base_module_mmds) + + # Make single list from dict of lists. + res = [] + for ns, mmds_list in mmds.items(): + if len(mmds_list) == 0: + raise UnprocessableEntity("Cannot find any module builds for %s" % (ns)) + res += mmds_list + return res + + +def generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous=False, default_streams=None): + """ + Returns list with MMDs with buildrequires and requires set according + to module stream expansion rules. These module metadata can be directly + built using MBS. + + :param db_session: SQLAlchemy DB session. + :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + """ + if not default_streams: + default_streams = {} + + # Create local copy of mmd, because we will expand its dependencies, + # which would change the module. + current_mmd = mmd.copy() + + # MMDResolver expects the input MMD to have no context. + current_mmd.set_context(None) + + # Expands the MSE streams. This mainly handles '-' prefix in MSE streams. + expand_mse_streams(db_session, current_mmd, default_streams, raise_if_stream_ambigous) + + # Get the list of all MMDs which this module can be possibly built against + # and add them to MMDResolver. + mmd_resolver = MMDResolver() + mmds_for_resolving = get_mmds_required_by_module_recursively( + db_session, current_mmd, default_streams, raise_if_stream_ambigous) + for m in mmds_for_resolving: + mmd_resolver.add_modules(m) + + # Show log.info message with the NSVCs we have added to mmd_resolver. + nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving] + log.info("Starting resolving with following input modules: %r", nsvcs_to_solve) + + # Resolve the dependencies between modules and get the list of all valid + # combinations in which we can build this module. + requires_combinations = mmd_resolver.solve(current_mmd) + log.info("Resolving done, possible requires: %r", requires_combinations) + + # This is where we are going to store the generated MMDs. + mmds = [] + for requires in requires_combinations: + # Each generated MMD must be new Module object... + mmd_copy = mmd.copy() + xmd = mmd_copy.get_xmd() + + # Requires contain the NSVC representing the input mmd. + # The 'context' of this NSVC defines the id of buildrequires/requires + # pair in the mmd.get_dependencies(). + dependencies_id = None + + # We don't want to depend on ourselves, so store the NSVC of the current_mmd + # to be able to ignore it later. + self_nsvca = None + + # Dict to store name:stream pairs from nsvca, so we are able to access it + # easily later. + req_name_stream = {} + + # Get the values for dependencies_id, self_nsvca and req_name_stream variables. + for nsvca in requires: + req_name, req_stream, _, req_context, req_arch = nsvca.split(":") + if req_arch == "src": + assert req_name == current_mmd.get_module_name() + assert req_stream == current_mmd.get_stream_name() + assert dependencies_id is None + assert self_nsvca is None + dependencies_id = int(req_context) + self_nsvca = nsvca + continue + req_name_stream[req_name] = req_stream + if dependencies_id is None or self_nsvca is None: + raise RuntimeError( + "%s:%s not found in requires %r" + % (current_mmd.get_module_name(), current_mmd.get_stream_name(), requires) + ) + + # The name:[streams, ...] pairs do not have to be the same in both + # buildrequires/requires. In case they are the same, we replace the streams + # in requires section with a single stream against which we will build this MMD. + # In case they are not the same, we have to keep the streams as they are in requires + # section. We always replace stream(s) for build-requirement with the one we + # will build this MMD against. + new_deps = Modulemd.Dependencies() + deps = mmd_copy.get_dependencies()[dependencies_id] + deps_requires = deps_to_dict(deps, 'runtime') + deps_buildrequires = deps_to_dict(deps, 'buildtime') + for req_name, req_streams in deps_requires.items(): + if req_name not in deps_buildrequires: + # This require is not a buildrequire so just copy this runtime requirement to + # new_dep and don't touch buildrequires + if not req_streams: + new_deps.set_empty_runtime_dependencies_for_module(req_name) + else: + for req_stream in req_streams: + new_deps.add_runtime_stream(req_name, req_stream) + elif set(req_streams) != set(deps_buildrequires[req_name]): + # Streams in runtime section are not the same as in buildtime section, + # so just copy this runtime requirement to new_dep. + if not req_streams: + new_deps.set_empty_runtime_dependencies_for_module(req_name) + else: + for req_stream in req_streams: + new_deps.add_runtime_stream(req_name, req_stream) + + new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) + else: + # This runtime requirement has the same streams in both runtime/buildtime + # requires sections, so replace streams in both sections by the one we + # really used in this resolved variant. + new_deps.add_runtime_stream(req_name, req_name_stream[req_name]) + new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) + + # There might be buildrequires which are not in runtime requires list. + # Such buildrequires must be copied to expanded MMD. + for req_name, req_streams in deps_buildrequires.items(): + if req_name not in deps_requires: + new_deps.add_buildtime_stream(req_name, req_name_stream[req_name]) + + # Set the new dependencies. + mmd_copy.remove_dependencies(deps) + mmd_copy.add_dependencies(new_deps) + + # The Modulemd.Dependencies() stores only streams, but to really build this + # module, we need NSVC of buildrequires, so we have to store this data in XMD. + # We also need additional data like for example list of filtered_rpms. We will + # get them using module_build_service.resolver.GenericResolver.resolve_requires, + # so prepare list with NSVCs of buildrequires as an input for this method. + br_list = [] + for nsvca in requires: + if nsvca == self_nsvca: + continue + # Remove the arch from nsvca + nsvc = ":".join(nsvca.split(":")[:-1]) + br_list.append(nsvc) + + # Resolve the buildrequires and store the result in XMD. + if "mbs" not in xmd: + xmd["mbs"] = {} + resolver = GenericResolver.create(db_session, conf) + xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list) + xmd["mbs"]["mse"] = True + + mmd_copy.set_xmd(xmd) + + # Now we have all the info to actually compute context of this module. + context = models.ModuleBuild.contexts_from_mmd(mmd_to_str(mmd_copy)).context + mmd_copy.set_context(context) + + mmds.append(mmd_copy) + + return mmds diff --git a/tests/test_common/test_resolve.py b/tests/test_common/test_resolve.py new file mode 100644 index 0000000..efd6277 --- /dev/null +++ b/tests/test_common/test_resolve.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from mock import patch, PropertyMock +import pytest + +from module_build_service.common.utils import load_mmd +from module_build_service.common.resolve import get_base_module_mmds +from module_build_service import Modulemd, models +from module_build_service.db_session import db_session +from tests import clean_database, make_module_in_db, init_data, read_staged_data + + +class TestResolve: + def setup_method(self, test_method): + clean_database(False) + + def teardown_method(self, test_method): + clean_database() + + def test__get_base_module_mmds(self): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=True) + mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "f29.1.0") + new_deps.add_buildtime_stream("platform", "f29.2.0") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + mmds = get_base_module_mmds(db_session, mmd) + expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} + # Verify no duplicates were returned before doing set operations + assert len(mmds["ready"]) == len(expected) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds["ready"]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected + + @pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"])) + def test__get_base_module_mmds_virtual_streams(self, virtual_streams): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=True) + mmd = load_mmd(read_staged_data("testmodule_v2")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "f29.2.0") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams) + + mmds = get_base_module_mmds(db_session, mmd) + if virtual_streams == ["f29"]: + expected = { + "platform:f29.0.0", + "platform:f29.1.0", + "platform:f29.2.0", + "platform:lp29.1.1" + } + else: + expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} + # Verify no duplicates were returned before doing set operations + assert len(mmds["ready"]) == len(expected) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds["ready"]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected + + @patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=PropertyMock, return_value=False + ) + def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"]) + + # Mark platform:foo28 as garbage to test that it is still considered as compatible. + platform = db_session.query(models.ModuleBuild).filter_by( + name="platform", stream="foo28").first() + platform.state = "garbage" + db_session.add(platform) + db_session.commit() + + mmd = load_mmd(read_staged_data("testmodule_v2")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "foo29") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + mmds = get_base_module_mmds(db_session, mmd) + expected = {} + expected["ready"] = {"platform:foo29", "platform:foo30"} + expected["garbage"] = {"platform:foo28"} + + # Verify no duplicates were returned before doing set operations + assert len(mmds) == len(expected) + for k in expected.keys(): + assert len(mmds[k]) == len(expected[k]) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds[k]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected[k] diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index b623975..66c19d0 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -703,7 +703,7 @@ class TestUtils: v = module_build_service.utils.submit.get_prefixed_version(mmd) assert v == 7000120180205135154 - @patch("module_build_service.utils.mse.generate_expanded_mmds") + @patch("module_build_service.utils.submit.generate_expanded_mmds") def test_submit_build_new_mse_build(self, generate_expanded_mmds): """ Tests that finished build can be resubmitted in case the resubmitted @@ -729,7 +729,7 @@ class TestUtils: assert builds[0].siblings(db_session) == [builds[1].id] assert builds[1].siblings(db_session) == [builds[0].id] - @patch("module_build_service.utils.mse.generate_expanded_mmds") + @patch("module_build_service.utils.submit.generate_expanded_mmds") @patch( "module_build_service.config.Config.scratch_build_only_branches", new_callable=mock.PropertyMock, diff --git a/tests/test_utils/test_utils_mse.py b/tests/test_utils/test_utils_mse.py deleted file mode 100644 index 094b36c..0000000 --- a/tests/test_utils/test_utils_mse.py +++ /dev/null @@ -1,564 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from mock import patch, PropertyMock -import pytest - -import module_build_service.utils -from module_build_service.common.utils import load_mmd -from module_build_service import Modulemd, models -from module_build_service.db_session import db_session -from module_build_service.errors import StreamAmbigous -from tests import clean_database, make_module_in_db, init_data, read_staged_data - - -class TestUtilsModuleStreamExpansion: - def setup_method(self, test_method): - clean_database(False) - - def teardown_method(self, test_method): - clean_database() - - def _get_mmds_required_by_module_recursively(self, module_build, db_session): - """ - Convenience wrapper around get_mmds_required_by_module_recursively - returning the list with nsvc strings of modules returned by this the wrapped - method. - """ - mmd = module_build.mmd() - module_build_service.utils.expand_mse_streams(db_session, mmd) - modules = module_build_service.utils.get_mmds_required_by_module_recursively( - db_session, mmd) - nsvcs = [m.get_nsvc() for m in modules] - return nsvcs - - def _generate_default_modules(self): - """ - Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the - platform:f28 and platform:f29 modules. - """ - platform_f28 = make_module_in_db("platform:f28:0:c10") - platform_f29 = make_module_in_db("platform:f29:0:c11") - f28_deps = [{ - "requires": {"platform": ["f28"]}, - "buildrequires": {"platform": ["f28"]}, - }] - f29_deps = [{ - "requires": {"platform": ["f29"]}, - "buildrequires": {"platform": ["f29"]}, - }] - make_module_in_db("gtk:1:0:c2", f28_deps, base_module=platform_f28) - make_module_in_db("gtk:1:0:c3", f29_deps, base_module=platform_f29) - make_module_in_db("gtk:2:0:c4", f28_deps, base_module=platform_f28) - make_module_in_db("gtk:2:0:c5", f29_deps, base_module=platform_f29) - make_module_in_db("foo:1:0:c2", f28_deps, base_module=platform_f28) - make_module_in_db("foo:1:0:c3", f29_deps, base_module=platform_f29) - make_module_in_db("foo:2:0:c4", f28_deps, base_module=platform_f28) - make_module_in_db("foo:2:0:c5", f29_deps, base_module=platform_f29) - make_module_in_db("app:1:0:c6", f29_deps, base_module=platform_f29) - - def test_generate_expanded_mmds_context(self): - self._generate_default_modules() - module_build = make_module_in_db( - "app:1:0:c1", [{ - "requires": {"gtk": ["1", "2"]}, - "buildrequires": { - "platform": ["f28"], - "gtk": ["1", "2"] - }, - }], - ) - mmds = module_build_service.utils.generate_expanded_mmds( - db_session, module_build.mmd()) - contexts = {mmd.get_context() for mmd in mmds} - assert {"e1e005fb", "ce132a1e"} == contexts - - @pytest.mark.parametrize( - "module_deps,stream_ambigous,expected_xmd,expected_buildrequires", - [ - ( - [{ - "requires": {"gtk": ["1", "2"]}, - "buildrequires": {"platform": ["f28"], "gtk": ["1", "2"]}, - }], - True, - { - frozenset(["platform:f28:0:c10", "gtk:2:0:c4"]), - frozenset(["platform:f28:0:c10", "gtk:1:0:c2"]) - }, - {frozenset(["gtk:1", "platform:f28"]), frozenset(["gtk:2", "platform:f28"])}, - ), - ( - [{ - "requires": {"foo": ["1"]}, - "buildrequires": {"platform": ["f28"], "foo": ["1"], "gtk": ["1", "2"]}, - }], - True, - { - frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"]), - frozenset(["foo:1:0:c2", "gtk:2:0:c4", "platform:f28:0:c10"]), - }, - { - frozenset(["foo:1", "gtk:1", "platform:f28"]), - frozenset(["foo:1", "gtk:2", "platform:f28"]), - }, - ), - ( - [{ - "requires": {"gtk": ["1"], "foo": ["1"]}, - "buildrequires": {"platform": ["f28"], "gtk": ["1"], "foo": ["1"]}, - }], - False, - {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, - {frozenset(["foo:1", "gtk:1", "platform:f28"])}, - ), - ( - [{ - "requires": {"gtk": ["1"], "foo": ["1"]}, - "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}, - }], - False, - {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, - {frozenset(["foo:1", "gtk:1", "platform:f28"])}, - ), - ( - [{ - "requires": {"gtk": ["-2"], "foo": ["-2"]}, - "buildrequires": {"platform": ["f28"], "gtk": ["-2"], "foo": ["-2"]}, - }], - True, - {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, - {frozenset(["foo:1", "gtk:1", "platform:f28"])}, - ), - ( - [{ - "requires": {"gtk": ["1"], "foo": ["1"]}, - "buildrequires": {"platform": ["f28"], "gtk": ["1"]}, - }], - False, - {frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])}, - {frozenset(["gtk:1", "platform:f28"])}, - ), - ( - [{ - "requires": {"gtk": []}, - "buildrequires": {"platform": ["f28"], "gtk": ["1"]}, - }], - True, - {frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])}, - {frozenset(["gtk:1", "platform:f28"])}, - ), - ( - [{ - "requires": {}, - "buildrequires": {"platform": ["f29"], "app": ["1"]}, - }], - False, - {frozenset(["app:1:0:c6", "platform:f29:0:c11"])}, - {frozenset(["app:1", "platform:f29"])}, - ), - ], - ) - def test_generate_expanded_mmds_buildrequires( - self, module_deps, stream_ambigous, expected_xmd, expected_buildrequires - ): - self._generate_default_modules() - module_build = make_module_in_db("app:1:0:c1", module_deps) - - # Check that generate_expanded_mmds raises an exception if stream is ambigous - # and also that it does not raise an exception otherwise. - if stream_ambigous: - with pytest.raises(StreamAmbigous): - module_build_service.utils.generate_expanded_mmds( - db_session, module_build.mmd(), raise_if_stream_ambigous=True) - else: - module_build_service.utils.generate_expanded_mmds( - db_session, module_build.mmd(), raise_if_stream_ambigous=True) - - # Check that if stream is ambigous and we define the stream, it does not raise - # an exception. - if stream_ambigous: - default_streams = {} - for ns in list(expected_buildrequires)[0]: - name, stream = ns.split(":") - default_streams[name] = stream - module_build_service.utils.generate_expanded_mmds( - db_session, - module_build.mmd(), - raise_if_stream_ambigous=True, - default_streams=default_streams, - ) - - mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd()) - - buildrequires_per_mmd_xmd = set() - buildrequires_per_mmd_buildrequires = set() - for mmd in mmds: - xmd = mmd.get_xmd() - br_nsvcs = [] - for name, detail in xmd["mbs"]["buildrequires"].items(): - br_nsvcs.append( - ":".join([name, detail["stream"], detail["version"], detail["context"]])) - buildrequires_per_mmd_xmd.add(frozenset(br_nsvcs)) - - assert len(mmd.get_dependencies()) == 1 - - buildrequires = set() - dep = mmd.get_dependencies()[0] - for req_name in dep.get_buildtime_modules(): - for req_stream in dep.get_buildtime_streams(req_name): - buildrequires.add(":".join([req_name, req_stream])) - buildrequires_per_mmd_buildrequires.add(frozenset(buildrequires)) - - assert buildrequires_per_mmd_xmd == expected_xmd - assert buildrequires_per_mmd_buildrequires == expected_buildrequires - - @pytest.mark.parametrize( - "module_deps,expected", - [ - ( - [{ - "requires": {"gtk": ["1", "2"]}, - "buildrequires": {"platform": [], "gtk": ["1", "2"]}, - }], - {frozenset(["gtk:1"]), frozenset(["gtk:2"])}, - ), - ( - [{ - "requires": {"gtk": ["1", "2"]}, - "buildrequires": {"platform": [], "gtk": ["1"]}, - }], - {frozenset(["gtk:1", "gtk:2"])}, - ), - ( - [{ - "requires": {"gtk": ["1"], "foo": ["1"]}, - "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}, - }], - {frozenset(["foo:1", "gtk:1"])}, - ), - ( - [{ - "requires": {"gtk": ["-2"], "foo": ["-2"]}, - "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]}, - }], - {frozenset(["foo:1", "gtk:1"])}, - ), - ( - [{ - "requires": {"gtk": [], "foo": []}, - "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}, - }], - {frozenset([])}, - ), - ], - ) - def test_generate_expanded_mmds_requires(self, module_deps, expected): - self._generate_default_modules() - module_build = make_module_in_db("app:1:0:c1", module_deps) - mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd()) - - requires_per_mmd = set() - for mmd in mmds: - assert len(mmd.get_dependencies()) == 1 - mmd_requires = set() - dep = mmd.get_dependencies()[0] - for req_name in dep.get_runtime_modules(): - for req_stream in dep.get_runtime_streams(req_name): - mmd_requires.add(":".join([req_name, req_stream])) - requires_per_mmd.add(frozenset(mmd_requires)) - - assert requires_per_mmd == expected - - @pytest.mark.parametrize( - "module_deps,expected", - [ - ( - [{"requires": {}, "buildrequires": {"platform": [], "gtk": ["1", "2"]}}], - [ - "platform:f29:0:c11", - "gtk:2:0:c4", - "gtk:2:0:c5", - "platform:f28:0:c10", - "gtk:1:0:c2", - "gtk:1:0:c3", - ], - ), - ( - [{ - "requires": {}, - "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]} - }], - [ - "platform:f28:0:c10", - "gtk:1:0:c2", - "gtk:1:0:c3", - "foo:1:0:c2", - "foo:1:0:c3", - "platform:f29:0:c11", - ], - ), - ( - [{ - "requires": {}, - "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]} - }], - ["platform:f28:0:c10", "gtk:1:0:c2", "foo:1:0:c2"], - ), - ( - [ - { - "requires": {}, - "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]} - }, - { - "requires": {}, - "buildrequires": {"platform": [], "gtk": ["2"], "foo": ["2"]}, - } - ], - [ - "foo:1:0:c2", - "foo:1:0:c3", - "foo:2:0:c4", - "foo:2:0:c5", - "platform:f28:0:c10", - "platform:f29:0:c11", - "gtk:1:0:c2", - "gtk:1:0:c3", - "gtk:2:0:c4", - "gtk:2:0:c5", - ], - ), - ( - [{ - "requires": {}, - "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]}, - }], - [ - "foo:1:0:c2", - "foo:1:0:c3", - "platform:f29:0:c11", - "platform:f28:0:c10", - "gtk:1:0:c2", - "gtk:1:0:c3", - ], - ), - ], - ) - def test_get_required_modules_simple(self, module_deps, expected): - module_build = make_module_in_db("app:1:0:c1", module_deps) - self._generate_default_modules() - nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) - assert set(nsvcs) == set(expected) - - def _generate_default_modules_recursion(self): - """ - Generates the gtk:1 module requiring foo:1 module requiring bar:1 - and lorem:1 modules which require base:f29 module requiring - platform:f29 module :). - """ - base_module = make_module_in_db("platform:f29:0:c11") - make_module_in_db( - "gtk:1:0:c2", - [{"requires": {"foo": ["unknown"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "gtk:1:1:c2", - [{"requires": {"foo": ["1"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "foo:1:0:c2", - [{"requires": {"bar": ["unknown"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "foo:1:1:c2", - [{"requires": {"bar": ["1"], "lorem": ["1"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "bar:1:0:c2", - [{"requires": {"base": ["unknown"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "bar:1:1:c2", - [{"requires": {"base": ["f29"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "lorem:1:0:c2", - [{"requires": {"base": ["unknown"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "lorem:1:1:c2", - [{"requires": {"base": ["f29"]}, "buildrequires": {}}], - base_module=base_module) - make_module_in_db( - "base:f29:0:c3", - [{"requires": {"platform": ["f29"]}, "buildrequires": {}}], - base_module=base_module) - - @pytest.mark.parametrize( - "module_deps,expected", - [ - ( - [{ - "requires": {}, - "buildrequires": {"platform": [], "gtk": ["1"]}, - }], - [ - "foo:1:1:c2", - "base:f29:0:c3", - "platform:f29:0:c11", - "bar:1:1:c2", - "gtk:1:1:c2", - "lorem:1:1:c2", - ], - ), - ( - [{ - "requires": {}, - "buildrequires": {"platform": [], "foo": ["1"]}, - }], - ["foo:1:1:c2", "base:f29:0:c3", "platform:f29:0:c11", "bar:1:1:c2", "lorem:1:1:c2"], - ), - ], - ) - def test_get_required_modules_recursion(self, module_deps, expected): - module_build = make_module_in_db("app:1:0:c1", module_deps) - self._generate_default_modules_recursion() - nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) - assert set(nsvcs) == set(expected) - - def _generate_default_modules_modules_multiple_stream_versions(self): - """ - Generates the gtk:1 module requiring foo:1 module requiring bar:1 - and lorem:1 modules which require base:f29 module requiring - platform:f29 module :). - """ - f29_dep = [{ - "requires": {"platform": ["f29"]}, - "buildrequires": {"platform": ["f29"]} - }] - - f290000 = make_module_in_db( - "platform:f29.0.0:0:c11", virtual_streams=["f29"]) - make_module_in_db("gtk:1:0:c2", f29_dep, base_module=f290000) - - f290100 = make_module_in_db( - "platform:f29.1.0:0:c11", virtual_streams=["f29"]) - make_module_in_db("gtk:1:1:c2", f29_dep, base_module=f290100) - make_module_in_db("gtk:1:2:c2", f29_dep, base_module=f290100) - - f290200 = make_module_in_db( - "platform:f29.2.0:0:c11", virtual_streams=["f29"]) - make_module_in_db("gtk:1:3:c2", f29_dep, base_module=f290200) - - @pytest.mark.parametrize( - "module_deps,expected", - [ - ( - [{ - "requires": {}, - "buildrequires": {"platform": ["f29.1.0"], "gtk": ["1"]}, - }], - ["platform:f29.0.0:0:c11", "gtk:1:0:c2", "gtk:1:2:c2", "platform:f29.1.0:0:c11"], - ) - ], - ) - def test_get_required_modules_stream_versions(self, module_deps, expected): - module_build = make_module_in_db("app:1:0:c1", module_deps) - self._generate_default_modules_modules_multiple_stream_versions() - nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) - assert set(nsvcs) == set(expected) - - def test__get_base_module_mmds(self): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=True) - mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "f29.1.0") - new_deps.add_buildtime_stream("platform", "f29.2.0") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} - # Verify no duplicates were returned before doing set operations - assert len(mmds["ready"]) == len(expected) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds["ready"]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected - - @pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"])) - def test__get_base_module_mmds_virtual_streams(self, virtual_streams): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=True) - mmd = load_mmd(read_staged_data("testmodule_v2")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "f29.2.0") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - if virtual_streams == ["f29"]: - expected = { - "platform:f29.0.0", - "platform:f29.1.0", - "platform:f29.2.0", - "platform:lp29.1.1" - } - else: - expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} - # Verify no duplicates were returned before doing set operations - assert len(mmds["ready"]) == len(expected) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds["ready"]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected - - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=PropertyMock, return_value=False - ) - def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"]) - - # Mark platform:foo28 as garbage to test that it is still considered as compatible. - platform = db_session.query(models.ModuleBuild).filter_by( - name="platform", stream="foo28").first() - platform.state = "garbage" - db_session.add(platform) - db_session.commit() - - mmd = load_mmd(read_staged_data("testmodule_v2")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "foo29") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - expected = {} - expected["ready"] = {"platform:foo29", "platform:foo30"} - expected["garbage"] = {"platform:foo28"} - - # Verify no duplicates were returned before doing set operations - assert len(mmds) == len(expected) - for k in expected.keys(): - assert len(mmds[k]) == len(expected[k]) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds[k]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected[k] diff --git a/tests/test_web/__init__.py b/tests/test_web/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/test_web/__init__.py diff --git a/tests/test_web/test_mse.py b/tests/test_web/test_mse.py new file mode 100644 index 0000000..d2729fb --- /dev/null +++ b/tests/test_web/test_mse.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import pytest + +from module_build_service.db_session import db_session +from module_build_service.errors import StreamAmbigous +from module_build_service.web.mse import ( + expand_mse_streams, generate_expanded_mmds, get_mmds_required_by_module_recursively +) +from tests import clean_database, make_module_in_db + + +class TestModuleStreamExpansion: + def setup_method(self, test_method): + clean_database(False) + + def teardown_method(self, test_method): + clean_database() + + def _get_mmds_required_by_module_recursively(self, module_build, db_session): + """ + Convenience wrapper around get_mmds_required_by_module_recursively + returning the list with nsvc strings of modules returned by this the wrapped + method. + """ + mmd = module_build.mmd() + expand_mse_streams(db_session, mmd) + modules = get_mmds_required_by_module_recursively(db_session, mmd) + nsvcs = [m.get_nsvc() for m in modules] + return nsvcs + + def _generate_default_modules(self): + """ + Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the + platform:f28 and platform:f29 modules. + """ + platform_f28 = make_module_in_db("platform:f28:0:c10") + platform_f29 = make_module_in_db("platform:f29:0:c11") + f28_deps = [{ + "requires": {"platform": ["f28"]}, + "buildrequires": {"platform": ["f28"]}, + }] + f29_deps = [{ + "requires": {"platform": ["f29"]}, + "buildrequires": {"platform": ["f29"]}, + }] + make_module_in_db("gtk:1:0:c2", f28_deps, base_module=platform_f28) + make_module_in_db("gtk:1:0:c3", f29_deps, base_module=platform_f29) + make_module_in_db("gtk:2:0:c4", f28_deps, base_module=platform_f28) + make_module_in_db("gtk:2:0:c5", f29_deps, base_module=platform_f29) + make_module_in_db("foo:1:0:c2", f28_deps, base_module=platform_f28) + make_module_in_db("foo:1:0:c3", f29_deps, base_module=platform_f29) + make_module_in_db("foo:2:0:c4", f28_deps, base_module=platform_f28) + make_module_in_db("foo:2:0:c5", f29_deps, base_module=platform_f29) + make_module_in_db("app:1:0:c6", f29_deps, base_module=platform_f29) + + def test_generate_expanded_mmds_context(self): + self._generate_default_modules() + module_build = make_module_in_db( + "app:1:0:c1", [{ + "requires": {"gtk": ["1", "2"]}, + "buildrequires": { + "platform": ["f28"], + "gtk": ["1", "2"] + }, + }], + ) + mmds = generate_expanded_mmds(db_session, module_build.mmd()) + contexts = {mmd.get_context() for mmd in mmds} + assert {"e1e005fb", "ce132a1e"} == contexts + + @pytest.mark.parametrize( + "module_deps,stream_ambigous,expected_xmd,expected_buildrequires", + [ + ( + [{ + "requires": {"gtk": ["1", "2"]}, + "buildrequires": {"platform": ["f28"], "gtk": ["1", "2"]}, + }], + True, + { + frozenset(["platform:f28:0:c10", "gtk:2:0:c4"]), + frozenset(["platform:f28:0:c10", "gtk:1:0:c2"]) + }, + {frozenset(["gtk:1", "platform:f28"]), frozenset(["gtk:2", "platform:f28"])}, + ), + ( + [{ + "requires": {"foo": ["1"]}, + "buildrequires": {"platform": ["f28"], "foo": ["1"], "gtk": ["1", "2"]}, + }], + True, + { + frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"]), + frozenset(["foo:1:0:c2", "gtk:2:0:c4", "platform:f28:0:c10"]), + }, + { + frozenset(["foo:1", "gtk:1", "platform:f28"]), + frozenset(["foo:1", "gtk:2", "platform:f28"]), + }, + ), + ( + [{ + "requires": {"gtk": ["1"], "foo": ["1"]}, + "buildrequires": {"platform": ["f28"], "gtk": ["1"], "foo": ["1"]}, + }], + False, + {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, + {frozenset(["foo:1", "gtk:1", "platform:f28"])}, + ), + ( + [{ + "requires": {"gtk": ["1"], "foo": ["1"]}, + "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}, + }], + False, + {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, + {frozenset(["foo:1", "gtk:1", "platform:f28"])}, + ), + ( + [{ + "requires": {"gtk": ["-2"], "foo": ["-2"]}, + "buildrequires": {"platform": ["f28"], "gtk": ["-2"], "foo": ["-2"]}, + }], + True, + {frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])}, + {frozenset(["foo:1", "gtk:1", "platform:f28"])}, + ), + ( + [{ + "requires": {"gtk": ["1"], "foo": ["1"]}, + "buildrequires": {"platform": ["f28"], "gtk": ["1"]}, + }], + False, + {frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])}, + {frozenset(["gtk:1", "platform:f28"])}, + ), + ( + [{ + "requires": {"gtk": []}, + "buildrequires": {"platform": ["f28"], "gtk": ["1"]}, + }], + True, + {frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])}, + {frozenset(["gtk:1", "platform:f28"])}, + ), + ( + [{ + "requires": {}, + "buildrequires": {"platform": ["f29"], "app": ["1"]}, + }], + False, + {frozenset(["app:1:0:c6", "platform:f29:0:c11"])}, + {frozenset(["app:1", "platform:f29"])}, + ), + ], + ) + def test_generate_expanded_mmds_buildrequires( + self, module_deps, stream_ambigous, expected_xmd, expected_buildrequires + ): + self._generate_default_modules() + module_build = make_module_in_db("app:1:0:c1", module_deps) + + # Check that generate_expanded_mmds raises an exception if stream is ambigous + # and also that it does not raise an exception otherwise. + if stream_ambigous: + with pytest.raises(StreamAmbigous): + generate_expanded_mmds( + db_session, module_build.mmd(), raise_if_stream_ambigous=True) + else: + generate_expanded_mmds(db_session, module_build.mmd(), raise_if_stream_ambigous=True) + + # Check that if stream is ambigous and we define the stream, it does not raise + # an exception. + if stream_ambigous: + default_streams = {} + for ns in list(expected_buildrequires)[0]: + name, stream = ns.split(":") + default_streams[name] = stream + generate_expanded_mmds( + db_session, + module_build.mmd(), + raise_if_stream_ambigous=True, + default_streams=default_streams, + ) + + mmds = generate_expanded_mmds(db_session, module_build.mmd()) + + buildrequires_per_mmd_xmd = set() + buildrequires_per_mmd_buildrequires = set() + for mmd in mmds: + xmd = mmd.get_xmd() + br_nsvcs = [] + for name, detail in xmd["mbs"]["buildrequires"].items(): + br_nsvcs.append( + ":".join([name, detail["stream"], detail["version"], detail["context"]])) + buildrequires_per_mmd_xmd.add(frozenset(br_nsvcs)) + + assert len(mmd.get_dependencies()) == 1 + + buildrequires = set() + dep = mmd.get_dependencies()[0] + for req_name in dep.get_buildtime_modules(): + for req_stream in dep.get_buildtime_streams(req_name): + buildrequires.add(":".join([req_name, req_stream])) + buildrequires_per_mmd_buildrequires.add(frozenset(buildrequires)) + + assert buildrequires_per_mmd_xmd == expected_xmd + assert buildrequires_per_mmd_buildrequires == expected_buildrequires + + @pytest.mark.parametrize( + "module_deps,expected", + [ + ( + [{ + "requires": {"gtk": ["1", "2"]}, + "buildrequires": {"platform": [], "gtk": ["1", "2"]}, + }], + {frozenset(["gtk:1"]), frozenset(["gtk:2"])}, + ), + ( + [{ + "requires": {"gtk": ["1", "2"]}, + "buildrequires": {"platform": [], "gtk": ["1"]}, + }], + {frozenset(["gtk:1", "gtk:2"])}, + ), + ( + [{ + "requires": {"gtk": ["1"], "foo": ["1"]}, + "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}, + }], + {frozenset(["foo:1", "gtk:1"])}, + ), + ( + [{ + "requires": {"gtk": ["-2"], "foo": ["-2"]}, + "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]}, + }], + {frozenset(["foo:1", "gtk:1"])}, + ), + ( + [{ + "requires": {"gtk": [], "foo": []}, + "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}, + }], + {frozenset([])}, + ), + ], + ) + def test_generate_expanded_mmds_requires(self, module_deps, expected): + self._generate_default_modules() + module_build = make_module_in_db("app:1:0:c1", module_deps) + mmds = generate_expanded_mmds(db_session, module_build.mmd()) + + requires_per_mmd = set() + for mmd in mmds: + assert len(mmd.get_dependencies()) == 1 + mmd_requires = set() + dep = mmd.get_dependencies()[0] + for req_name in dep.get_runtime_modules(): + for req_stream in dep.get_runtime_streams(req_name): + mmd_requires.add(":".join([req_name, req_stream])) + requires_per_mmd.add(frozenset(mmd_requires)) + + assert requires_per_mmd == expected + + @pytest.mark.parametrize( + "module_deps,expected", + [ + ( + [{"requires": {}, "buildrequires": {"platform": [], "gtk": ["1", "2"]}}], + [ + "platform:f29:0:c11", + "gtk:2:0:c4", + "gtk:2:0:c5", + "platform:f28:0:c10", + "gtk:1:0:c2", + "gtk:1:0:c3", + ], + ), + ( + [{ + "requires": {}, + "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]} + }], + [ + "platform:f28:0:c10", + "gtk:1:0:c2", + "gtk:1:0:c3", + "foo:1:0:c2", + "foo:1:0:c3", + "platform:f29:0:c11", + ], + ), + ( + [{ + "requires": {}, + "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]} + }], + ["platform:f28:0:c10", "gtk:1:0:c2", "foo:1:0:c2"], + ), + ( + [ + { + "requires": {}, + "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]} + }, + { + "requires": {}, + "buildrequires": {"platform": [], "gtk": ["2"], "foo": ["2"]}, + } + ], + [ + "foo:1:0:c2", + "foo:1:0:c3", + "foo:2:0:c4", + "foo:2:0:c5", + "platform:f28:0:c10", + "platform:f29:0:c11", + "gtk:1:0:c2", + "gtk:1:0:c3", + "gtk:2:0:c4", + "gtk:2:0:c5", + ], + ), + ( + [{ + "requires": {}, + "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]}, + }], + [ + "foo:1:0:c2", + "foo:1:0:c3", + "platform:f29:0:c11", + "platform:f28:0:c10", + "gtk:1:0:c2", + "gtk:1:0:c3", + ], + ), + ], + ) + def test_get_required_modules_simple(self, module_deps, expected): + module_build = make_module_in_db("app:1:0:c1", module_deps) + self._generate_default_modules() + nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) + assert set(nsvcs) == set(expected) + + def _generate_default_modules_recursion(self): + """ + Generates the gtk:1 module requiring foo:1 module requiring bar:1 + and lorem:1 modules which require base:f29 module requiring + platform:f29 module :). + """ + base_module = make_module_in_db("platform:f29:0:c11") + make_module_in_db( + "gtk:1:0:c2", + [{"requires": {"foo": ["unknown"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "gtk:1:1:c2", + [{"requires": {"foo": ["1"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "foo:1:0:c2", + [{"requires": {"bar": ["unknown"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "foo:1:1:c2", + [{"requires": {"bar": ["1"], "lorem": ["1"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "bar:1:0:c2", + [{"requires": {"base": ["unknown"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "bar:1:1:c2", + [{"requires": {"base": ["f29"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "lorem:1:0:c2", + [{"requires": {"base": ["unknown"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "lorem:1:1:c2", + [{"requires": {"base": ["f29"]}, "buildrequires": {}}], + base_module=base_module) + make_module_in_db( + "base:f29:0:c3", + [{"requires": {"platform": ["f29"]}, "buildrequires": {}}], + base_module=base_module) + + @pytest.mark.parametrize( + "module_deps,expected", + [ + ( + [{ + "requires": {}, + "buildrequires": {"platform": [], "gtk": ["1"]}, + }], + [ + "foo:1:1:c2", + "base:f29:0:c3", + "platform:f29:0:c11", + "bar:1:1:c2", + "gtk:1:1:c2", + "lorem:1:1:c2", + ], + ), + ( + [{ + "requires": {}, + "buildrequires": {"platform": [], "foo": ["1"]}, + }], + ["foo:1:1:c2", "base:f29:0:c3", "platform:f29:0:c11", "bar:1:1:c2", "lorem:1:1:c2"], + ), + ], + ) + def test_get_required_modules_recursion(self, module_deps, expected): + module_build = make_module_in_db("app:1:0:c1", module_deps) + self._generate_default_modules_recursion() + nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) + assert set(nsvcs) == set(expected) + + def _generate_default_modules_modules_multiple_stream_versions(self): + """ + Generates the gtk:1 module requiring foo:1 module requiring bar:1 + and lorem:1 modules which require base:f29 module requiring + platform:f29 module :). + """ + f29_dep = [{ + "requires": {"platform": ["f29"]}, + "buildrequires": {"platform": ["f29"]} + }] + + f290000 = make_module_in_db( + "platform:f29.0.0:0:c11", virtual_streams=["f29"]) + make_module_in_db("gtk:1:0:c2", f29_dep, base_module=f290000) + + f290100 = make_module_in_db( + "platform:f29.1.0:0:c11", virtual_streams=["f29"]) + make_module_in_db("gtk:1:1:c2", f29_dep, base_module=f290100) + make_module_in_db("gtk:1:2:c2", f29_dep, base_module=f290100) + + f290200 = make_module_in_db( + "platform:f29.2.0:0:c11", virtual_streams=["f29"]) + make_module_in_db("gtk:1:3:c2", f29_dep, base_module=f290200) + + @pytest.mark.parametrize( + "module_deps,expected", + [ + ( + [{ + "requires": {}, + "buildrequires": {"platform": ["f29.1.0"], "gtk": ["1"]}, + }], + ["platform:f29.0.0:0:c11", "gtk:1:0:c2", "gtk:1:2:c2", "platform:f29.1.0:0:c11"], + ) + ], + ) + def test_get_required_modules_stream_versions(self, module_deps, expected): + module_build = make_module_in_db("app:1:0:c1", module_deps) + self._generate_default_modules_modules_multiple_stream_versions() + nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) + assert set(nsvcs) == set(expected)