From 85e3a2c4964e5d7abf3fe576d760d7da41af4ed6 Mon Sep 17 00:00:00 2001 From: mprahl Date: Mar 03 2020 19:48:47 +0000 Subject: Move utils/reuse.py to scheduler/reuse.py --- diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index e310dbd..974603a 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -31,7 +31,7 @@ from module_build_service.common.koji import ( get_session, koji_multicall_map, koji_retrying_multicall_map, ) from module_build_service.scheduler import events -from module_build_service.utils import get_reusable_components, get_reusable_module +from module_build_service.scheduler.reuse import get_reusable_components, get_reusable_module logging.basicConfig(level=logging.DEBUG) diff --git a/module_build_service/scheduler/batches.py b/module_build_service/scheduler/batches.py index 59179c2..4bcabbb 100644 --- a/module_build_service/scheduler/batches.py +++ b/module_build_service/scheduler/batches.py @@ -6,7 +6,7 @@ import concurrent.futures from module_build_service import conf, log, models from module_build_service.db_session import db_session from module_build_service.scheduler import events -from module_build_service.utils.reuse import get_reusable_components, reuse_component +from module_build_service.scheduler.reuse import get_reusable_components, reuse_component def at_concurrent_component_threshold(config): diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 572b7a4..3ba5588 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -10,7 +10,6 @@ from module_build_service.common.retry import retry import module_build_service.resolver import module_build_service.utils from module_build_service.utils import ( - attempt_to_reuse_all_components, record_component_builds, record_filtered_rpms, record_module_build_arches @@ -23,6 +22,7 @@ from module_build_service.scheduler.default_modules import ( from module_build_service.scheduler.greenwave import greenwave from module_build_service.utils.submit import format_mmd from module_build_service.scheduler import events +from module_build_service.scheduler.reuse import attempt_to_reuse_all_components from module_build_service.scheduler.ursine import handle_stream_collision_modules from requests.exceptions import ConnectionError diff --git a/module_build_service/scheduler/reuse.py b/module_build_service/scheduler/reuse.py new file mode 100644 index 0000000..6e4672c --- /dev/null +++ b/module_build_service/scheduler/reuse.py @@ -0,0 +1,447 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import kobo.rpmlib + +from module_build_service import log, models, conf +from module_build_service.db_session import db_session +from module_build_service.resolver import GenericResolver +from module_build_service.scheduler import events +from module_build_service.common.resolve import get_base_module_mmds + + +def reuse_component(component, previous_component_build, change_state_now=False, + schedule_fake_events=True): + """ + Reuses component build `previous_component_build` instead of building + component `component` + + Please remember to commit the changes where the function is called. + This allows callers to reuse multiple component builds and commit them all + at once. + + :param ComponentBuild component: Component whihch will reuse previous module build. + :param ComponentBuild previous_component_build: Previous component build to reuse. + :param bool change_state_now: When True, the component.state will be set to + previous_component_build.state. Otherwise, the component.state will be set to BUILDING. + :param bool schedule_fake_events: When True, the `events.scheduler.add` will be used to + schedule handlers.component.build_task_finalize handler call. + """ + + import koji + from module_build_service.scheduler.handlers.components import ( + build_task_finalize as build_task_finalize_handler) + + log.info( + 'Reusing component "{0}" from a previous module ' + 'build with the nvr "{1}"'.format(component.package, previous_component_build.nvr) + ) + component.reused_component_id = previous_component_build.id + component.task_id = previous_component_build.task_id + if change_state_now: + component.state = previous_component_build.state + else: + # Use BUILDING state here, because we want the state to change to + # COMPLETE by scheduling a internal buildsys.build.state.change message + # we are generating few lines below. + # If we would set it to the right state right here, we would miss the + # code path handling that event which works only when switching from + # BUILDING to COMPLETE. + component.state = koji.BUILD_STATES["BUILDING"] + component.state_reason = "Reused component from previous module build" + component.nvr = previous_component_build.nvr + nvr_dict = kobo.rpmlib.parse_nvr(component.nvr) + # Add this event to scheduler so that the reused component will be tagged properly. + if schedule_fake_events: + args = ( + "reuse_component: fake msg", component.task_id, previous_component_build.state, + nvr_dict["name"], nvr_dict["version"], nvr_dict["release"], component.module_id, + component.state_reason) + events.scheduler.add(build_task_finalize_handler, args) + + +def get_reusable_module(module): + """ + Returns previous module build of the module `module` in case it can be + used as a source module to get the components to reuse from. + + In case there is no such module, returns None. + + :param module: the ModuleBuild object of module being built. + :return: ModuleBuild object which can be used for component reuse. + """ + + if module.reused_module: + return module.reused_module + + mmd = module.mmd() + previous_module_build = None + + # The `base_mmds` will contain the list of base modules against which the possible modules + # to reuse are built. There are three options how these base modules are found: + # + # 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should + # not try to find any compatible base modules in its DB and simply use the buildrequired + # base module as it is. + # 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means + # that MBS should try to find the compatible modules using its database. + # The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of + # all of them. + # 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This + # means that MBS should *not* try to find any compatible base modules in its DB, but + # instead just query Koji using KojiResolver later to find out the module to + # reuse. The list of compatible base modules is defined by Koji tag inheritance directly + # in Koji. + # The `get_base_module_mmds` in this case returns just the buildrequired base module. + if conf.allow_only_compatible_base_modules: + log.debug("Checking for compatible base modules") + base_mmds = get_base_module_mmds(db_session, mmd)["ready"] + # Sort the base_mmds based on the stream version, higher version first. + base_mmds.sort( + key=lambda mmd: models.ModuleBuild.get_stream_version(mmd.get_stream_name(), False), + reverse=True) + else: + log.debug("Skipping the check for compatible base modules") + base_mmds = [] + for br in module.buildrequires: + if br.name in conf.base_module_names: + base_mmds.append(br.mmd()) + + for base_mmd in base_mmds: + previous_module_build = ( + db_session.query(models.ModuleBuild) + .filter_by(name=mmd.get_module_name()) + .filter_by(stream=mmd.get_stream_name()) + .filter_by(state=models.BUILD_STATES["ready"]) + .filter(models.ModuleBuild.scmurl.isnot(None)) + .order_by(models.ModuleBuild.time_completed.desc())) + + koji_resolver_enabled = base_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules") + if koji_resolver_enabled: + # Find ModuleBuilds tagged in the Koji tag using KojiResolver. + resolver = GenericResolver.create(db_session, conf, backend="koji") + possible_modules_to_reuse = resolver.get_buildrequired_modules( + module.name, module.stream, base_mmd) + + # Limit the query to these modules. + possible_module_ids = [m.id for m in possible_modules_to_reuse] + previous_module_build = previous_module_build.filter( + models.ModuleBuild.id.in_(possible_module_ids)) + + # Limit the query to modules sharing the same `build_context_no_bms`. That means they + # have the same buildrequirements. + previous_module_build = previous_module_build.filter_by( + build_context_no_bms=module.build_context_no_bms) + else: + # Recompute the build_context with compatible base module stream. + mbs_xmd = mmd.get_xmd()["mbs"] + if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]: + previous_module_build = None + continue + mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \ + = base_mmd.get_stream_name() + build_context = module.calculate_build_context(mbs_xmd["buildrequires"]) + + # Limit the query to find only modules sharing the same build_context. + previous_module_build = previous_module_build.filter_by(build_context=build_context) + + # If we are rebuilding with the "changed-and-after" option, then we can't reuse + # components from modules that were built more liberally + if module.rebuild_strategy == "changed-and-after": + previous_module_build = previous_module_build.filter( + models.ModuleBuild.rebuild_strategy.in_(["all", "changed-and-after"]) + ) + + previous_module_build = previous_module_build.first() + + if previous_module_build: + break + + # The component can't be reused if there isn't a previous build in the done + # or ready state + if not previous_module_build: + log.info("Cannot re-use. %r is the first module build." % module) + return None + + module.reused_module_id = previous_module_build.id + db_session.commit() + + return previous_module_build + + +def attempt_to_reuse_all_components(builder, module): + """ + Tries to reuse all the components in a build. The components are also + tagged to the tags using the `builder`. + + Returns True if all components could be reused, otherwise False. When + False is returned, no component has been reused. + """ + + previous_module_build = get_reusable_module(module) + if not previous_module_build: + return False + + mmd = module.mmd() + old_mmd = previous_module_build.mmd() + + # [(component, component_to_reuse), ...] + component_pairs = [] + + # Find out if we can reuse all components and cache component and + # component to reuse pairs. + for c in module.component_builds: + if c.package == "module-build-macros": + continue + component_to_reuse = get_reusable_component( + module, + c.package, + previous_module_build=previous_module_build, + mmd=mmd, + old_mmd=old_mmd, + ) + if not component_to_reuse: + return False + + component_pairs.append((c, component_to_reuse)) + + # Stores components we will tag to buildroot and final tag. + components_to_tag = [] + + # Reuse all components. + for c, component_to_reuse in component_pairs: + # Set the module.batch to the last batch we have. + if c.batch > module.batch: + module.batch = c.batch + + # Reuse the component + reuse_component(c, component_to_reuse, True, False) + components_to_tag.append(c.nvr) + + # Tag them + builder.buildroot_add_artifacts(components_to_tag, install=False) + builder.tag_artifacts(components_to_tag, dest_tag=True) + + return True + + +def get_reusable_components(module, component_names, previous_module_build=None): + """ + Returns the list of ComponentBuild instances belonging to previous module + build which can be reused in the build of module `module`. + + The ComponentBuild instances in returned list are in the same order as + their names in the component_names input list. + + In case some component cannot be reused, None is used instead of a + ComponentBuild instance in the returned list. + + :param module: the ModuleBuild object of module being built. + :param component_names: List of component names to be reused. + :kwarg previous_module_build: the ModuleBuild instance of a module build + which contains the components to reuse. If not passed, get_reusable_module + is called to get the ModuleBuild instance. + :return: List of ComponentBuild instances to reuse in the same + order as `component_names` + """ + # We support components reusing only for koji and test backend. + if conf.system not in ["koji", "test"]: + return [None] * len(component_names) + + if not previous_module_build: + previous_module_build = get_reusable_module(module) + if not previous_module_build: + return [None] * len(component_names) + + mmd = module.mmd() + old_mmd = previous_module_build.mmd() + + ret = [] + for component_name in component_names: + ret.append( + get_reusable_component( + module, component_name, previous_module_build, mmd, old_mmd) + ) + + return ret + + +def get_reusable_component( + module, component_name, previous_module_build=None, mmd=None, old_mmd=None +): + """ + Returns the component (RPM) build of a module that can be reused + instead of needing to rebuild it + + :param module: the ModuleBuild object of module being built with a formatted + mmd + :param component_name: the name of the component (RPM) that you'd like to + reuse a previous build of + :param previous_module_build: the ModuleBuild instances of a module build + which contains the components to reuse. If not passed, get_reusable_module + is called to get the ModuleBuild instance. Consider passing the ModuleBuild + instance in case you plan to call get_reusable_component repeatedly for the + same module to make this method faster. + :param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from + module.mmd(). Consider passing this arg in case you plan to call + get_reusable_component repeatedly for the same module to make this method faster. + :param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed, + it is taken from previous_module_build.mmd(). Consider passing this arg in + case you plan to call get_reusable_component repeatedly for the same + module to make this method faster. + :return: the component (RPM) build SQLAlchemy object, if one is not found, + None is returned + """ + + # We support component reusing only for koji and test backend. + if conf.system not in ["koji", "test"]: + return None + + # If the rebuild strategy is "all", that means that nothing can be reused + if module.rebuild_strategy == "all": + message = ("Cannot reuse the component {component_name} because the module " + "rebuild strategy is \"all\".").format( + component_name=component_name) + module.log_message(db_session, message) + return None + + if not previous_module_build: + previous_module_build = get_reusable_module(module) + if not previous_module_build: + message = ("Cannot reuse because no previous build of " + "module {module_name} found!").format( + module_name=module.name) + module.log_message(db_session, message) + return None + + if not mmd: + mmd = module.mmd() + if not old_mmd: + old_mmd = previous_module_build.mmd() + + # If the chosen component for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + new_module_build_component = models.ComponentBuild.from_component_name( + db_session, component_name, module.id) + if ( + not new_module_build_component + or not new_module_build_component.batch + or not new_module_build_component.ref + ): + message = ("Cannot reuse the component {} because it can't be found in the " + "database").format(component_name) + module.log_message(db_session, message) + return None + + prev_module_build_component = models.ComponentBuild.from_component_name( + db_session, component_name, previous_module_build.id + ) + # If the component to reuse for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + if ( + not prev_module_build_component + or not prev_module_build_component.batch + or not prev_module_build_component.ref + ): + message = ("Cannot reuse the component {} because a previous build of " + "it can't be found in the database").format(component_name) + new_module_build_component.log_message(db_session, message) + return None + + # Make sure the ref for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.ref != new_module_build_component.ref: + message = ("Cannot reuse the component because the commit hash changed" + " since the last build") + new_module_build_component.log_message(db_session, message) + return None + + # At this point we've determined that both module builds contain the component + # and the components share the same commit hash + if module.rebuild_strategy == "changed-and-after": + # Make sure the batch number for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.batch != new_module_build_component.batch: + message = ("Cannot reuse the component because it is being built in " + "a different batch than in the compatible module build") + new_module_build_component.log_message(db_session, message) + return None + + # If the mmd.buildopts.macros.rpms changed, we cannot reuse + buildopts = mmd.get_buildopts() + if buildopts: + modulemd_macros = buildopts.get_rpm_macros() + else: + modulemd_macros = None + + old_buildopts = old_mmd.get_buildopts() + if old_buildopts: + old_modulemd_macros = old_buildopts.get_rpm_macros() + else: + old_modulemd_macros = None + + if modulemd_macros != old_modulemd_macros: + message = ("Cannot reuse the component because the modulemd's macros are" + " different than those of the compatible module build") + new_module_build_component.log_message(db_session, message) + return None + + # At this point we've determined that both module builds contain the component + # with the same commit hash and they are in the same batch. We've also determined + # that both module builds depend(ed) on the same exact module builds. Now it's time + # to determine if the components before it have changed. + # + # Convert the component_builds to a list and sort them by batch + new_component_builds = list(module.component_builds) + new_component_builds.sort(key=lambda x: x.batch) + prev_component_builds = list(previous_module_build.component_builds) + prev_component_builds.sort(key=lambda x: x.batch) + + new_module_build_components = [] + previous_module_build_components = [] + # Create separate lists for the new and previous module build. These lists + # will have an entry for every build batch *before* the component's + # batch except for 1, which is reserved for the module-build-macros RPM. + # Each batch entry will contain a set of "(name, ref, arches)" with the name, + # ref (commit), and arches of the component. + for i in range(new_module_build_component.batch - 1): + # This is the first batch which we want to skip since it will always + # contain only the module-build-macros RPM and it gets built every time + if i == 0: + continue + + new_module_build_components.append({ + (value.package, value.ref, + tuple(sorted(mmd.get_rpm_component(value.package).get_arches()))) + for value in new_component_builds + if value.batch == i + 1 + }) + + previous_module_build_components.append({ + (value.package, value.ref, + tuple(sorted(old_mmd.get_rpm_component(value.package).get_arches()))) + for value in prev_component_builds + if value.batch == i + 1 + }) + + # If the previous batches don't have the same ordering, hashes, and arches, then the + # component can't be reused + if previous_module_build_components != new_module_build_components: + message = ("Cannot reuse the component because a component in a previous" + " batch has been added, removed, or rebuilt") + new_module_build_component.log_message(db_session, message) + return None + + # check that arches have not changed + pkg = mmd.get_rpm_component(component_name) + if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(component_name).get_arches()): + message = ("Cannot reuse the component because its architectures" + " have changed since the compatible module build").format(component_name) + new_module_build_component.log_message(db_session, message) + return None + + reusable_component = db_session.query(models.ComponentBuild).filter_by( + package=component_name, module_id=previous_module_build.id).one() + log.debug("Found reusable component!") + return reusable_component diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index 37d4849..88410f4 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa diff --git a/module_build_service/utils/reuse.py b/module_build_service/utils/reuse.py deleted file mode 100644 index 6e4672c..0000000 --- a/module_build_service/utils/reuse.py +++ /dev/null @@ -1,447 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import kobo.rpmlib - -from module_build_service import log, models, conf -from module_build_service.db_session import db_session -from module_build_service.resolver import GenericResolver -from module_build_service.scheduler import events -from module_build_service.common.resolve import get_base_module_mmds - - -def reuse_component(component, previous_component_build, change_state_now=False, - schedule_fake_events=True): - """ - Reuses component build `previous_component_build` instead of building - component `component` - - Please remember to commit the changes where the function is called. - This allows callers to reuse multiple component builds and commit them all - at once. - - :param ComponentBuild component: Component whihch will reuse previous module build. - :param ComponentBuild previous_component_build: Previous component build to reuse. - :param bool change_state_now: When True, the component.state will be set to - previous_component_build.state. Otherwise, the component.state will be set to BUILDING. - :param bool schedule_fake_events: When True, the `events.scheduler.add` will be used to - schedule handlers.component.build_task_finalize handler call. - """ - - import koji - from module_build_service.scheduler.handlers.components import ( - build_task_finalize as build_task_finalize_handler) - - log.info( - 'Reusing component "{0}" from a previous module ' - 'build with the nvr "{1}"'.format(component.package, previous_component_build.nvr) - ) - component.reused_component_id = previous_component_build.id - component.task_id = previous_component_build.task_id - if change_state_now: - component.state = previous_component_build.state - else: - # Use BUILDING state here, because we want the state to change to - # COMPLETE by scheduling a internal buildsys.build.state.change message - # we are generating few lines below. - # If we would set it to the right state right here, we would miss the - # code path handling that event which works only when switching from - # BUILDING to COMPLETE. - component.state = koji.BUILD_STATES["BUILDING"] - component.state_reason = "Reused component from previous module build" - component.nvr = previous_component_build.nvr - nvr_dict = kobo.rpmlib.parse_nvr(component.nvr) - # Add this event to scheduler so that the reused component will be tagged properly. - if schedule_fake_events: - args = ( - "reuse_component: fake msg", component.task_id, previous_component_build.state, - nvr_dict["name"], nvr_dict["version"], nvr_dict["release"], component.module_id, - component.state_reason) - events.scheduler.add(build_task_finalize_handler, args) - - -def get_reusable_module(module): - """ - Returns previous module build of the module `module` in case it can be - used as a source module to get the components to reuse from. - - In case there is no such module, returns None. - - :param module: the ModuleBuild object of module being built. - :return: ModuleBuild object which can be used for component reuse. - """ - - if module.reused_module: - return module.reused_module - - mmd = module.mmd() - previous_module_build = None - - # The `base_mmds` will contain the list of base modules against which the possible modules - # to reuse are built. There are three options how these base modules are found: - # - # 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should - # not try to find any compatible base modules in its DB and simply use the buildrequired - # base module as it is. - # 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means - # that MBS should try to find the compatible modules using its database. - # The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of - # all of them. - # 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This - # means that MBS should *not* try to find any compatible base modules in its DB, but - # instead just query Koji using KojiResolver later to find out the module to - # reuse. The list of compatible base modules is defined by Koji tag inheritance directly - # in Koji. - # The `get_base_module_mmds` in this case returns just the buildrequired base module. - if conf.allow_only_compatible_base_modules: - log.debug("Checking for compatible base modules") - base_mmds = get_base_module_mmds(db_session, mmd)["ready"] - # Sort the base_mmds based on the stream version, higher version first. - base_mmds.sort( - key=lambda mmd: models.ModuleBuild.get_stream_version(mmd.get_stream_name(), False), - reverse=True) - else: - log.debug("Skipping the check for compatible base modules") - base_mmds = [] - for br in module.buildrequires: - if br.name in conf.base_module_names: - base_mmds.append(br.mmd()) - - for base_mmd in base_mmds: - previous_module_build = ( - db_session.query(models.ModuleBuild) - .filter_by(name=mmd.get_module_name()) - .filter_by(stream=mmd.get_stream_name()) - .filter_by(state=models.BUILD_STATES["ready"]) - .filter(models.ModuleBuild.scmurl.isnot(None)) - .order_by(models.ModuleBuild.time_completed.desc())) - - koji_resolver_enabled = base_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules") - if koji_resolver_enabled: - # Find ModuleBuilds tagged in the Koji tag using KojiResolver. - resolver = GenericResolver.create(db_session, conf, backend="koji") - possible_modules_to_reuse = resolver.get_buildrequired_modules( - module.name, module.stream, base_mmd) - - # Limit the query to these modules. - possible_module_ids = [m.id for m in possible_modules_to_reuse] - previous_module_build = previous_module_build.filter( - models.ModuleBuild.id.in_(possible_module_ids)) - - # Limit the query to modules sharing the same `build_context_no_bms`. That means they - # have the same buildrequirements. - previous_module_build = previous_module_build.filter_by( - build_context_no_bms=module.build_context_no_bms) - else: - # Recompute the build_context with compatible base module stream. - mbs_xmd = mmd.get_xmd()["mbs"] - if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]: - previous_module_build = None - continue - mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \ - = base_mmd.get_stream_name() - build_context = module.calculate_build_context(mbs_xmd["buildrequires"]) - - # Limit the query to find only modules sharing the same build_context. - previous_module_build = previous_module_build.filter_by(build_context=build_context) - - # If we are rebuilding with the "changed-and-after" option, then we can't reuse - # components from modules that were built more liberally - if module.rebuild_strategy == "changed-and-after": - previous_module_build = previous_module_build.filter( - models.ModuleBuild.rebuild_strategy.in_(["all", "changed-and-after"]) - ) - - previous_module_build = previous_module_build.first() - - if previous_module_build: - break - - # The component can't be reused if there isn't a previous build in the done - # or ready state - if not previous_module_build: - log.info("Cannot re-use. %r is the first module build." % module) - return None - - module.reused_module_id = previous_module_build.id - db_session.commit() - - return previous_module_build - - -def attempt_to_reuse_all_components(builder, module): - """ - Tries to reuse all the components in a build. The components are also - tagged to the tags using the `builder`. - - Returns True if all components could be reused, otherwise False. When - False is returned, no component has been reused. - """ - - previous_module_build = get_reusable_module(module) - if not previous_module_build: - return False - - mmd = module.mmd() - old_mmd = previous_module_build.mmd() - - # [(component, component_to_reuse), ...] - component_pairs = [] - - # Find out if we can reuse all components and cache component and - # component to reuse pairs. - for c in module.component_builds: - if c.package == "module-build-macros": - continue - component_to_reuse = get_reusable_component( - module, - c.package, - previous_module_build=previous_module_build, - mmd=mmd, - old_mmd=old_mmd, - ) - if not component_to_reuse: - return False - - component_pairs.append((c, component_to_reuse)) - - # Stores components we will tag to buildroot and final tag. - components_to_tag = [] - - # Reuse all components. - for c, component_to_reuse in component_pairs: - # Set the module.batch to the last batch we have. - if c.batch > module.batch: - module.batch = c.batch - - # Reuse the component - reuse_component(c, component_to_reuse, True, False) - components_to_tag.append(c.nvr) - - # Tag them - builder.buildroot_add_artifacts(components_to_tag, install=False) - builder.tag_artifacts(components_to_tag, dest_tag=True) - - return True - - -def get_reusable_components(module, component_names, previous_module_build=None): - """ - Returns the list of ComponentBuild instances belonging to previous module - build which can be reused in the build of module `module`. - - The ComponentBuild instances in returned list are in the same order as - their names in the component_names input list. - - In case some component cannot be reused, None is used instead of a - ComponentBuild instance in the returned list. - - :param module: the ModuleBuild object of module being built. - :param component_names: List of component names to be reused. - :kwarg previous_module_build: the ModuleBuild instance of a module build - which contains the components to reuse. If not passed, get_reusable_module - is called to get the ModuleBuild instance. - :return: List of ComponentBuild instances to reuse in the same - order as `component_names` - """ - # We support components reusing only for koji and test backend. - if conf.system not in ["koji", "test"]: - return [None] * len(component_names) - - if not previous_module_build: - previous_module_build = get_reusable_module(module) - if not previous_module_build: - return [None] * len(component_names) - - mmd = module.mmd() - old_mmd = previous_module_build.mmd() - - ret = [] - for component_name in component_names: - ret.append( - get_reusable_component( - module, component_name, previous_module_build, mmd, old_mmd) - ) - - return ret - - -def get_reusable_component( - module, component_name, previous_module_build=None, mmd=None, old_mmd=None -): - """ - Returns the component (RPM) build of a module that can be reused - instead of needing to rebuild it - - :param module: the ModuleBuild object of module being built with a formatted - mmd - :param component_name: the name of the component (RPM) that you'd like to - reuse a previous build of - :param previous_module_build: the ModuleBuild instances of a module build - which contains the components to reuse. If not passed, get_reusable_module - is called to get the ModuleBuild instance. Consider passing the ModuleBuild - instance in case you plan to call get_reusable_component repeatedly for the - same module to make this method faster. - :param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from - module.mmd(). Consider passing this arg in case you plan to call - get_reusable_component repeatedly for the same module to make this method faster. - :param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed, - it is taken from previous_module_build.mmd(). Consider passing this arg in - case you plan to call get_reusable_component repeatedly for the same - module to make this method faster. - :return: the component (RPM) build SQLAlchemy object, if one is not found, - None is returned - """ - - # We support component reusing only for koji and test backend. - if conf.system not in ["koji", "test"]: - return None - - # If the rebuild strategy is "all", that means that nothing can be reused - if module.rebuild_strategy == "all": - message = ("Cannot reuse the component {component_name} because the module " - "rebuild strategy is \"all\".").format( - component_name=component_name) - module.log_message(db_session, message) - return None - - if not previous_module_build: - previous_module_build = get_reusable_module(module) - if not previous_module_build: - message = ("Cannot reuse because no previous build of " - "module {module_name} found!").format( - module_name=module.name) - module.log_message(db_session, message) - return None - - if not mmd: - mmd = module.mmd() - if not old_mmd: - old_mmd = previous_module_build.mmd() - - # If the chosen component for some reason was not found in the database, - # or the ref is missing, something has gone wrong and the component cannot - # be reused - new_module_build_component = models.ComponentBuild.from_component_name( - db_session, component_name, module.id) - if ( - not new_module_build_component - or not new_module_build_component.batch - or not new_module_build_component.ref - ): - message = ("Cannot reuse the component {} because it can't be found in the " - "database").format(component_name) - module.log_message(db_session, message) - return None - - prev_module_build_component = models.ComponentBuild.from_component_name( - db_session, component_name, previous_module_build.id - ) - # If the component to reuse for some reason was not found in the database, - # or the ref is missing, something has gone wrong and the component cannot - # be reused - if ( - not prev_module_build_component - or not prev_module_build_component.batch - or not prev_module_build_component.ref - ): - message = ("Cannot reuse the component {} because a previous build of " - "it can't be found in the database").format(component_name) - new_module_build_component.log_message(db_session, message) - return None - - # Make sure the ref for the component that is trying to be reused - # hasn't changed since the last build - if prev_module_build_component.ref != new_module_build_component.ref: - message = ("Cannot reuse the component because the commit hash changed" - " since the last build") - new_module_build_component.log_message(db_session, message) - return None - - # At this point we've determined that both module builds contain the component - # and the components share the same commit hash - if module.rebuild_strategy == "changed-and-after": - # Make sure the batch number for the component that is trying to be reused - # hasn't changed since the last build - if prev_module_build_component.batch != new_module_build_component.batch: - message = ("Cannot reuse the component because it is being built in " - "a different batch than in the compatible module build") - new_module_build_component.log_message(db_session, message) - return None - - # If the mmd.buildopts.macros.rpms changed, we cannot reuse - buildopts = mmd.get_buildopts() - if buildopts: - modulemd_macros = buildopts.get_rpm_macros() - else: - modulemd_macros = None - - old_buildopts = old_mmd.get_buildopts() - if old_buildopts: - old_modulemd_macros = old_buildopts.get_rpm_macros() - else: - old_modulemd_macros = None - - if modulemd_macros != old_modulemd_macros: - message = ("Cannot reuse the component because the modulemd's macros are" - " different than those of the compatible module build") - new_module_build_component.log_message(db_session, message) - return None - - # At this point we've determined that both module builds contain the component - # with the same commit hash and they are in the same batch. We've also determined - # that both module builds depend(ed) on the same exact module builds. Now it's time - # to determine if the components before it have changed. - # - # Convert the component_builds to a list and sort them by batch - new_component_builds = list(module.component_builds) - new_component_builds.sort(key=lambda x: x.batch) - prev_component_builds = list(previous_module_build.component_builds) - prev_component_builds.sort(key=lambda x: x.batch) - - new_module_build_components = [] - previous_module_build_components = [] - # Create separate lists for the new and previous module build. These lists - # will have an entry for every build batch *before* the component's - # batch except for 1, which is reserved for the module-build-macros RPM. - # Each batch entry will contain a set of "(name, ref, arches)" with the name, - # ref (commit), and arches of the component. - for i in range(new_module_build_component.batch - 1): - # This is the first batch which we want to skip since it will always - # contain only the module-build-macros RPM and it gets built every time - if i == 0: - continue - - new_module_build_components.append({ - (value.package, value.ref, - tuple(sorted(mmd.get_rpm_component(value.package).get_arches()))) - for value in new_component_builds - if value.batch == i + 1 - }) - - previous_module_build_components.append({ - (value.package, value.ref, - tuple(sorted(old_mmd.get_rpm_component(value.package).get_arches()))) - for value in prev_component_builds - if value.batch == i + 1 - }) - - # If the previous batches don't have the same ordering, hashes, and arches, then the - # component can't be reused - if previous_module_build_components != new_module_build_components: - message = ("Cannot reuse the component because a component in a previous" - " batch has been added, removed, or rebuilt") - new_module_build_component.log_message(db_session, message) - return None - - # check that arches have not changed - pkg = mmd.get_rpm_component(component_name) - if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(component_name).get_arches()): - message = ("Cannot reuse the component because its architectures" - " have changed since the compatible module build").format(component_name) - new_module_build_component.log_message(db_session, message) - return None - - reusable_component = db_session.query(models.ComponentBuild).filter_by( - package=component_name, module_id=previous_module_build.id).one() - log.debug("Found reusable component!") - return reusable_component diff --git a/tests/test_scheduler/test_reuse.py b/tests/test_scheduler/test_reuse.py new file mode 100644 index 0000000..1d13ce6 --- /dev/null +++ b/tests/test_scheduler/test_reuse.py @@ -0,0 +1,443 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import mock +import pytest +from sqlalchemy.orm.session import make_transient + +from module_build_service import models, Modulemd +from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str +from module_build_service.db_session import db_session +from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module +from tests import clean_database, read_staged_data + + +@pytest.mark.usefixtures("reuse_component_init_data") +class TestUtilsComponentReuse: + @pytest.mark.parametrize( + "changed_component", ["perl-List-Compare", "perl-Tangerine", "tangerine", None] + ) + def test_get_reusable_component_different_component(self, changed_component): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + if changed_component: + mmd = second_module_build.mmd() + mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab") + second_module_build.modulemd = mmd_to_str(mmd) + + second_module_changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab" + db_session.add(second_module_changed_component) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + + if changed_component == "perl-List-Compare": + # perl-Tangerine can be reused even though a component in its batch has changed + assert plc_rv is None + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv is None + elif changed_component == "perl-Tangerine": + # perl-List-Compare can be reused even though a component in its batch has changed + assert plc_rv.package == "perl-List-Compare" + assert pt_rv is None + assert tangerine_rv is None + elif changed_component == "tangerine": + # perl-List-Compare and perl-Tangerine can be reused since they are in an earlier + # buildorder than tangerine + assert plc_rv.package == "perl-List-Compare" + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv is None + elif changed_component is None: + # Nothing has changed so everthing can be used + assert plc_rv.package == "perl-List-Compare" + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv.package == "tangerine" + + def test_get_reusable_component_different_rpm_macros(self): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + buildopts = Modulemd.Buildopts() + buildopts.set_rpm_macros("%my_macro 1") + mmd.set_buildopts(buildopts) + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + assert plc_rv is None + + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + assert pt_rv is None + + @pytest.mark.parametrize("set_current_arch", [True, False]) + @pytest.mark.parametrize("set_database_arch", [True, False]) + def test_get_reusable_component_different_arches( + self, set_database_arch, set_current_arch + ): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + if set_current_arch: # set architecture for current build + mmd = second_module_build.mmd() + component = mmd.get_rpm_component("tangerine") + component.reset_arches() + component.add_restricted_arch("i686") + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + if set_database_arch: # set architecture for build in database + second_module_changed_component = models.ComponentBuild.from_component_name( + db_session, "tangerine", 2) + mmd = second_module_changed_component.module_build.mmd() + component = mmd.get_rpm_component("tangerine") + component.reset_arches() + component.add_restricted_arch("i686") + second_module_changed_component.module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + tangerine = get_reusable_component(second_module_build, "tangerine") + assert bool(tangerine is None) != bool(set_current_arch == set_database_arch) + + @pytest.mark.parametrize( + "reuse_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + @pytest.mark.parametrize( + "changed_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + def test_get_reusable_component_different_batch( + self, changed_component, reuse_component + ): + """ + Test that we get the correct reuse behavior for the changed-and-after strategy. Changes + to earlier batches should prevent reuse, but changes to later batches should not. + For context, see https://pagure.io/fm-orchestrator/issue/1298 + """ + + if changed_component == reuse_component: + # we're only testing the cases where these are different + # this case is already covered by test_get_reusable_component_different_component + return + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + # update batch for changed component + changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + orig_batch = changed_component.batch + changed_component.batch = orig_batch + 1 + db_session.commit() + + reuse_component = models.ComponentBuild.from_component_name( + db_session, reuse_component, second_module_build.id) + + reuse_result = get_reusable_component(second_module_build, reuse_component.package) + # Component reuse should only be blocked when an earlier batch has been changed. + # In this case, orig_batch is the earliest batch that has been changed (the changed + # component has been removed from it and added to the following one). + assert bool(reuse_result is None) == bool(reuse_component.batch > orig_batch) + + @pytest.mark.parametrize( + "reuse_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + @pytest.mark.parametrize( + "changed_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + def test_get_reusable_component_different_arch_in_batch( + self, changed_component, reuse_component + ): + """ + Test that we get the correct reuse behavior for the changed-and-after strategy. Changes + to the architectures in earlier batches should prevent reuse, but such changes to later + batches should not. + For context, see https://pagure.io/fm-orchestrator/issue/1298 + """ + if changed_component == reuse_component: + # we're only testing the cases where these are different + # this case is already covered by test_get_reusable_component_different_arches + return + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + # update arch for changed component + mmd = second_module_build.mmd() + component = mmd.get_rpm_component(changed_component) + component.reset_arches() + component.add_restricted_arch("i686") + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + reuse_component = models.ComponentBuild.from_component_name( + db_session, reuse_component, second_module_build.id) + + reuse_result = get_reusable_component(second_module_build, reuse_component.package) + # Changing the arch of a component should prevent reuse only when the changed component + # is in a batch earlier than the component being considered for reuse. + assert bool(reuse_result is None) == bool(reuse_component.batch > changed_component.batch) + + @pytest.mark.parametrize("rebuild_strategy", models.ModuleBuild.rebuild_strategies.keys()) + def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy): + first_module_build = models.ModuleBuild.get_by_id(db_session, 2) + first_module_build.rebuild_strategy = rebuild_strategy + db_session.commit() + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different" + deps = Modulemd.Dependencies() + deps.add_buildtime_stream("platform", "different") + deps.add_runtime_stream("platform", "different") + mmd.clear_dependencies() + mmd.add_dependencies(deps) + + mmd.set_xmd(xmd) + second_module_build.modulemd = mmd_to_str(mmd) + second_module_build.build_context = \ + models.ModuleBuild.contexts_from_mmd(second_module_build.modulemd).build_context + second_module_build.rebuild_strategy = rebuild_strategy + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + + assert plc_rv is None + assert pt_rv is None + assert tangerine_rv is None + + def test_get_reusable_component_different_buildrequires(self): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master") + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"] = { + "some_module": { + "ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "stream": "master", + "version": "20170123140147", + } + } + mmd.set_xmd(xmd) + second_module_build.modulemd = mmd_to_str(mmd) + second_module_build.build_context = models.ModuleBuild.calculate_build_context( + xmd["mbs"]["buildrequires"]) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + assert plc_rv is None + + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + assert pt_rv is None + + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + assert tangerine_rv is None + + +class TestReuseSharedUserSpace: + def setup_method(self, test_method): + clean_database() + + def teardown_method(self, test_method): + clean_database() + + @pytest.mark.usefixtures("reuse_shared_userspace_init_data") + def test_get_reusable_component_shared_userspace_ordering(self): + """ + For modules with lot of components per batch, there is big chance that + the database will return them in different order than what we have for + current `new_module`. In this case, reuse code should still be able to + reuse the components. + """ + old_module = models.ModuleBuild.get_by_id(db_session, 2) + new_module = models.ModuleBuild.get_by_id(db_session, 3) + rv = get_reusable_component(new_module, "llvm", previous_module_build=old_module) + assert rv.package == "llvm" + + +@pytest.mark.usefixtures("reuse_component_init_data") +class TestUtilsModuleReuse: + + def test_get_reusable_module_when_reused_module_not_set(self): + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .order_by(models.ModuleBuild.id.desc())\ + .first() + module.state = models.BUILD_STATES["build"] + db_session.commit() + + assert not module.reused_module + + reusable_module = get_reusable_module(module) + + assert module.reused_module + assert reusable_module.id == module.reused_module_id + + def test_get_reusable_module_when_reused_module_already_set(self): + modules = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .order_by(models.ModuleBuild.id.desc())\ + .limit(2).all() + build_module = modules[0] + reused_module = modules[1] + build_module.state = models.BUILD_STATES["build"] + build_module.reused_module_id = reused_module.id + db_session.commit() + + assert build_module.reused_module + assert reused_module == build_module.reused_module + + reusable_module = get_reusable_module(build_module) + + assert build_module.reused_module + assert reusable_module.id == build_module.reused_module_id + assert reusable_module.id == reused_module.id + + @pytest.mark.parametrize("allow_ocbm", (True, False)) + @mock.patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=mock.PropertyMock, + ) + def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): + """ + Test that the `get_reusable_module` tries to reuse the latest module in case when + multiple modules can be reused allow_only_compatible_base_modules is True. + """ + cfg.return_value = allow_ocbm + # Set "fedora" virtual stream to platform:f28. + platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() + mmd = platform_f28.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f28.modulemd = mmd_to_str(mmd) + platform_f28.update_virtual_streams(db_session, ["fedora"]) + + # Create platform:f29 with "fedora" virtual stream. + mmd = load_mmd(read_staged_data("platform")) + mmd = mmd.copy("platform", "f29") + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f29 = import_mmd(db_session, mmd)[0] + + # Create another copy of `testmodule:master` which should be reused, because its + # stream version will be higher than the previous one. Also set its buildrequires + # to platform:f29. + latest_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).one() + # This is used to clone the ModuleBuild SQLAlchemy object without recreating it from + # scratch. + db_session.expunge(latest_module) + make_transient(latest_module) + + # Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context. + mmd = latest_module.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29" + mmd.set_xmd(xmd) + latest_module.modulemd = mmd_to_str(mmd) + latest_module.build_context = models.ModuleBuild.contexts_from_mmd( + latest_module.modulemd + ).build_context + latest_module.buildrequires = [platform_f29] + + # Set the `id` to None, so new one is generated by SQLAlchemy. + latest_module.id = None + db_session.add(latest_module) + db_session.commit() + + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .filter_by(state=models.BUILD_STATES["build"])\ + .one() + db_session.commit() + + reusable_module = get_reusable_module(module) + + if allow_ocbm: + assert reusable_module.id == latest_module.id + else: + first_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).first() + assert reusable_module.id == first_module.id + + @pytest.mark.parametrize("allow_ocbm", (True, False)) + @mock.patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=mock.PropertyMock, + ) + @mock.patch("koji.ClientSession") + @mock.patch( + "module_build_service.config.Config.resolver", + new_callable=mock.PropertyMock, return_value="koji" + ) + def test_get_reusable_module_koji_resolver( + self, resolver, ClientSession, cfg, allow_ocbm): + """ + Test that get_reusable_module works with KojiResolver. + """ + cfg.return_value = allow_ocbm + + # Mock the listTagged so the testmodule:master is listed as tagged in the + # module-fedora-27-build Koji tag. + koji_session = ClientSession.return_value + koji_session.listTagged.return_value = [ + { + "build_id": 123, "name": "testmodule", "version": "master", + "release": "20170109091357.78e4a6fd", "tag_name": "module-fedora-27-build" + }] + + koji_session.multiCall.return_value = [ + [build] for build in koji_session.listTagged.return_value] + + # Mark platform:f28 as KojiResolver ready by defining "koji_tag_with_modules". + # Also define the "virtual_streams" to possibly confuse the get_reusable_module. + platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() + mmd = platform_f28.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + xmd["mbs"]["koji_tag_with_modules"] = "module-fedora-27-build" + mmd.set_xmd(xmd) + platform_f28.modulemd = mmd_to_str(mmd) + platform_f28.update_virtual_streams(db_session, ["fedora"]) + + # Create platform:f27 without KojiResolver support. + mmd = load_mmd(read_staged_data("platform")) + mmd = mmd.copy("platform", "f27") + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f27 = import_mmd(db_session, mmd)[0] + + # Change the reusable testmodule:master to buildrequire platform:f27. + latest_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).one() + mmd = latest_module.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f27" + mmd.set_xmd(xmd) + latest_module.modulemd = mmd_to_str(mmd) + latest_module.buildrequires = [platform_f27] + + # Recompute the build_context and ensure that `build_context` changed while + # `build_context_no_bms` did not change. + contexts = models.ModuleBuild.contexts_from_mmd(latest_module.modulemd) + + assert latest_module.build_context_no_bms == contexts.build_context_no_bms + assert latest_module.build_context != contexts.build_context + + latest_module.build_context = contexts.build_context + latest_module.build_context_no_bms = contexts.build_context_no_bms + db_session.commit() + + # Get the module we want to build. + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .filter_by(state=models.BUILD_STATES["build"])\ + .one() + + reusable_module = get_reusable_module(module) + + assert reusable_module.id == latest_module.id diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 66c19d0..8e6489a 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -8,14 +8,12 @@ from shutil import copyfile, rmtree from datetime import datetime from werkzeug.datastructures import FileStorage from mock import patch -from sqlalchemy.orm.session import make_transient -from module_build_service.common.utils import import_mmd, load_mmd, load_mmd_file, mmd_to_str +from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str import module_build_service.utils import module_build_service.scm -from module_build_service import models, conf +from module_build_service import app, models, conf from module_build_service.errors import ValidationError, UnprocessableEntity -from module_build_service.utils.reuse import get_reusable_module, get_reusable_component from module_build_service.utils.submit import format_mmd from tests import ( clean_database, @@ -29,7 +27,6 @@ import pytest import module_build_service.scheduler.handlers.components from module_build_service.db_session import db_session from module_build_service.scheduler import events -from module_build_service import app, Modulemd BASE_DIR = path.abspath(path.dirname(__file__)) @@ -73,232 +70,12 @@ class FakeSCM(object): return commit_hash + sha1_hash[len(commit_hash):] -@pytest.mark.usefixtures("reuse_component_init_data") -class TestUtilsComponentReuse: - @pytest.mark.parametrize( - "changed_component", ["perl-List-Compare", "perl-Tangerine", "tangerine", None] - ) - def test_get_reusable_component_different_component(self, changed_component): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - if changed_component: - mmd = second_module_build.mmd() - mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab") - second_module_build.modulemd = mmd_to_str(mmd) - - second_module_changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab" - db_session.add(second_module_changed_component) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - - if changed_component == "perl-List-Compare": - # perl-Tangerine can be reused even though a component in its batch has changed - assert plc_rv is None - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv is None - elif changed_component == "perl-Tangerine": - # perl-List-Compare can be reused even though a component in its batch has changed - assert plc_rv.package == "perl-List-Compare" - assert pt_rv is None - assert tangerine_rv is None - elif changed_component == "tangerine": - # perl-List-Compare and perl-Tangerine can be reused since they are in an earlier - # buildorder than tangerine - assert plc_rv.package == "perl-List-Compare" - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv is None - elif changed_component is None: - # Nothing has changed so everthing can be used - assert plc_rv.package == "perl-List-Compare" - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv.package == "tangerine" - - def test_get_reusable_component_different_rpm_macros(self): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - buildopts = Modulemd.Buildopts() - buildopts.set_rpm_macros("%my_macro 1") - mmd.set_buildopts(buildopts) - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - assert plc_rv is None - - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - assert pt_rv is None - - @pytest.mark.parametrize("set_current_arch", [True, False]) - @pytest.mark.parametrize("set_database_arch", [True, False]) - def test_get_reusable_component_different_arches( - self, set_database_arch, set_current_arch - ): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - if set_current_arch: # set architecture for current build - mmd = second_module_build.mmd() - component = mmd.get_rpm_component("tangerine") - component.reset_arches() - component.add_restricted_arch("i686") - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - if set_database_arch: # set architecture for build in database - second_module_changed_component = models.ComponentBuild.from_component_name( - db_session, "tangerine", 2) - mmd = second_module_changed_component.module_build.mmd() - component = mmd.get_rpm_component("tangerine") - component.reset_arches() - component.add_restricted_arch("i686") - second_module_changed_component.module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - tangerine = get_reusable_component(second_module_build, "tangerine") - assert bool(tangerine is None) != bool(set_current_arch == set_database_arch) - - @pytest.mark.parametrize( - "reuse_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - @pytest.mark.parametrize( - "changed_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - def test_get_reusable_component_different_batch( - self, changed_component, reuse_component - ): - """ - Test that we get the correct reuse behavior for the changed-and-after strategy. Changes - to earlier batches should prevent reuse, but changes to later batches should not. - For context, see https://pagure.io/fm-orchestrator/issue/1298 - """ - - if changed_component == reuse_component: - # we're only testing the cases where these are different - # this case is already covered by test_get_reusable_component_different_component - return - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - # update batch for changed component - changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - orig_batch = changed_component.batch - changed_component.batch = orig_batch + 1 - db_session.commit() - - reuse_component = models.ComponentBuild.from_component_name( - db_session, reuse_component, second_module_build.id) - - reuse_result = module_build_service.utils.get_reusable_component( - second_module_build, reuse_component.package) - # Component reuse should only be blocked when an earlier batch has been changed. - # In this case, orig_batch is the earliest batch that has been changed (the changed - # component has been removed from it and added to the following one). - assert bool(reuse_result is None) == bool(reuse_component.batch > orig_batch) - - @pytest.mark.parametrize( - "reuse_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - @pytest.mark.parametrize( - "changed_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - def test_get_reusable_component_different_arch_in_batch( - self, changed_component, reuse_component - ): - """ - Test that we get the correct reuse behavior for the changed-and-after strategy. Changes - to the architectures in earlier batches should prevent reuse, but such changes to later - batches should not. - For context, see https://pagure.io/fm-orchestrator/issue/1298 - """ - if changed_component == reuse_component: - # we're only testing the cases where these are different - # this case is already covered by test_get_reusable_component_different_arches - return - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - # update arch for changed component - mmd = second_module_build.mmd() - component = mmd.get_rpm_component(changed_component) - component.reset_arches() - component.add_restricted_arch("i686") - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - reuse_component = models.ComponentBuild.from_component_name( - db_session, reuse_component, second_module_build.id) - - reuse_result = module_build_service.utils.get_reusable_component( - second_module_build, reuse_component.package) - # Changing the arch of a component should prevent reuse only when the changed component - # is in a batch earlier than the component being considered for reuse. - assert bool(reuse_result is None) == bool(reuse_component.batch > changed_component.batch) - - @pytest.mark.parametrize("rebuild_strategy", models.ModuleBuild.rebuild_strategies.keys()) - def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy): - first_module_build = models.ModuleBuild.get_by_id(db_session, 2) - first_module_build.rebuild_strategy = rebuild_strategy - db_session.commit() - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different" - deps = Modulemd.Dependencies() - deps.add_buildtime_stream("platform", "different") - deps.add_runtime_stream("platform", "different") - mmd.clear_dependencies() - mmd.add_dependencies(deps) - - mmd.set_xmd(xmd) - second_module_build.modulemd = mmd_to_str(mmd) - second_module_build.build_context = \ - module_build_service.models.ModuleBuild.contexts_from_mmd( - second_module_build.modulemd - ).build_context - second_module_build.rebuild_strategy = rebuild_strategy - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - - assert plc_rv is None - assert pt_rv is None - assert tangerine_rv is None - - def test_get_reusable_component_different_buildrequires(self): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master") - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"] = { - "some_module": { - "ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "stream": "master", - "version": "20170123140147", - } - } - mmd.set_xmd(xmd) - second_module_build.modulemd = mmd_to_str(mmd) - second_module_build.build_context = models.ModuleBuild.calculate_build_context( - xmd["mbs"]["buildrequires"]) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - assert plc_rv is None - - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - assert pt_rv is None +class TestUtils: + def setup_method(self, test_method): + clean_database() - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - assert tangerine_rv is None + def teardown_method(self, test_method): + clean_database() @patch("module_build_service.utils.submit.submit_module_build") def test_submit_module_build_from_yaml_with_skiptests(self, mock_submit): @@ -333,14 +110,6 @@ class TestUtilsComponentReuse: assert username_arg == username rmtree(module_dir) - -class TestUtils: - def setup_method(self, test_method): - clean_database() - - def teardown_method(self, test_method): - clean_database() - @patch("koji.ClientSession") def test_get_build_arches(self, ClientSession): session = ClientSession.return_value @@ -457,19 +226,6 @@ class TestUtils: mmd_xmd = mmd.get_xmd() assert mmd_xmd == xmd - @pytest.mark.usefixtures("reuse_shared_userspace_init_data") - def test_get_reusable_component_shared_userspace_ordering(self): - """ - For modules with lot of components per batch, there is big chance that - the database will return them in different order than what we have for - current `new_module`. In this case, reuse code should still be able to - reuse the components. - """ - old_module = models.ModuleBuild.get_by_id(db_session, 2) - new_module = models.ModuleBuild.get_by_id(db_session, 3) - rv = get_reusable_component(new_module, "llvm", previous_module_build=old_module) - assert rv.package == "llvm" - @patch("module_build_service.scm.SCM") def test_record_component_builds_duplicate_components(self, mocked_scm): # Mock for format_mmd to get components' latest ref @@ -822,191 +578,3 @@ class TestLocalBuilds: assert len(local_modules) == 1 assert local_modules[0].koji_tag.endswith("/module-platform-f28-3/results") - - -@pytest.mark.usefixtures("reuse_component_init_data") -class TestUtilsModuleReuse: - - def test_get_reusable_module_when_reused_module_not_set(self): - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .order_by(models.ModuleBuild.id.desc())\ - .first() - module.state = models.BUILD_STATES["build"] - db_session.commit() - - assert not module.reused_module - - reusable_module = get_reusable_module(module) - - assert module.reused_module - assert reusable_module.id == module.reused_module_id - - def test_get_reusable_module_when_reused_module_already_set(self): - modules = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .order_by(models.ModuleBuild.id.desc())\ - .limit(2).all() - build_module = modules[0] - reused_module = modules[1] - build_module.state = models.BUILD_STATES["build"] - build_module.reused_module_id = reused_module.id - db_session.commit() - - assert build_module.reused_module - assert reused_module == build_module.reused_module - - reusable_module = get_reusable_module(build_module) - - assert build_module.reused_module - assert reusable_module.id == build_module.reused_module_id - assert reusable_module.id == reused_module.id - - @pytest.mark.parametrize("allow_ocbm", (True, False)) - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=mock.PropertyMock, - ) - def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): - """ - Test that the `get_reusable_module` tries to reuse the latest module in case when - multiple modules can be reused allow_only_compatible_base_modules is True. - """ - cfg.return_value = allow_ocbm - # Set "fedora" virtual stream to platform:f28. - platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() - mmd = platform_f28.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f28.modulemd = mmd_to_str(mmd) - platform_f28.update_virtual_streams(db_session, ["fedora"]) - - # Create platform:f29 with "fedora" virtual stream. - mmd = load_mmd(read_staged_data("platform")) - mmd = mmd.copy("platform", "f29") - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f29 = import_mmd(db_session, mmd)[0] - - # Create another copy of `testmodule:master` which should be reused, because its - # stream version will be higher than the previous one. Also set its buildrequires - # to platform:f29. - latest_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).one() - # This is used to clone the ModuleBuild SQLAlchemy object without recreating it from - # scratch. - db_session.expunge(latest_module) - make_transient(latest_module) - - # Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context. - mmd = latest_module.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29" - mmd.set_xmd(xmd) - latest_module.modulemd = mmd_to_str(mmd) - latest_module.build_context = module_build_service.models.ModuleBuild.contexts_from_mmd( - latest_module.modulemd - ).build_context - latest_module.buildrequires = [platform_f29] - - # Set the `id` to None, so new one is generated by SQLAlchemy. - latest_module.id = None - db_session.add(latest_module) - db_session.commit() - - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .filter_by(state=models.BUILD_STATES["build"])\ - .one() - db_session.commit() - - reusable_module = get_reusable_module(module) - - if allow_ocbm: - assert reusable_module.id == latest_module.id - else: - first_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).first() - assert reusable_module.id == first_module.id - - @pytest.mark.parametrize("allow_ocbm", (True, False)) - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=mock.PropertyMock, - ) - @patch("koji.ClientSession") - @patch( - "module_build_service.config.Config.resolver", - new_callable=mock.PropertyMock, return_value="koji" - ) - def test_get_reusable_module_koji_resolver( - self, resolver, ClientSession, cfg, allow_ocbm): - """ - Test that get_reusable_module works with KojiResolver. - """ - cfg.return_value = allow_ocbm - - # Mock the listTagged so the testmodule:master is listed as tagged in the - # module-fedora-27-build Koji tag. - koji_session = ClientSession.return_value - koji_session.listTagged.return_value = [ - { - "build_id": 123, "name": "testmodule", "version": "master", - "release": "20170109091357.78e4a6fd", "tag_name": "module-fedora-27-build" - }] - - koji_session.multiCall.return_value = [ - [build] for build in koji_session.listTagged.return_value] - - # Mark platform:f28 as KojiResolver ready by defining "koji_tag_with_modules". - # Also define the "virtual_streams" to possibly confuse the get_reusable_module. - platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() - mmd = platform_f28.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - xmd["mbs"]["koji_tag_with_modules"] = "module-fedora-27-build" - mmd.set_xmd(xmd) - platform_f28.modulemd = mmd_to_str(mmd) - platform_f28.update_virtual_streams(db_session, ["fedora"]) - - # Create platform:f27 without KojiResolver support. - mmd = load_mmd(read_staged_data("platform")) - mmd = mmd.copy("platform", "f27") - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f27 = import_mmd(db_session, mmd)[0] - - # Change the reusable testmodule:master to buildrequire platform:f27. - latest_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).one() - mmd = latest_module.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f27" - mmd.set_xmd(xmd) - latest_module.modulemd = mmd_to_str(mmd) - latest_module.buildrequires = [platform_f27] - - # Recompute the build_context and ensure that `build_context` changed while - # `build_context_no_bms` did not change. - contexts = module_build_service.models.ModuleBuild.contexts_from_mmd( - latest_module.modulemd) - - assert latest_module.build_context_no_bms == contexts.build_context_no_bms - assert latest_module.build_context != contexts.build_context - - latest_module.build_context = contexts.build_context - latest_module.build_context_no_bms = contexts.build_context_no_bms - db_session.commit() - - # Get the module we want to build. - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .filter_by(state=models.BUILD_STATES["build"])\ - .one() - - reusable_module = get_reusable_module(module) - - assert reusable_module.id == latest_module.id