From 42429cf0433ab6316063c05f6f092da376e519db Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Nov 20 2018 11:00:48 +0000 Subject: [PATCH 1/2] Refactor _get_deps_and_tag Signed-off-by: Chenxiong Qi --- diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index d32abe0..2a0bc14 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -171,6 +171,76 @@ def init(config, session, msg): session.commit() +def generate_module_build_koji_tag(build): + """Used by wait handler to get module build koji tag + + :param build: a module build. + :type build: :class:`ModuleBuild` + :return: generated koji tag. + :rtype: str + """ + log.info('Getting tag for %s:%s:%s', build.name, build.stream, build.version) + if conf.system in ['koji', 'test']: + return generate_koji_tag(build.name, build.stream, build.version, build.context) + else: + return '-'.join(['module', build.name, build.stream, build.version]) + + +@module_build_service.utils.retry( + interval=10, timeout=120, + wait_on=(ValueError, RuntimeError, ConnectionError)) +def get_module_build_dependencies(build): + """Used by wait handler to get module's build dependencies + + :param build: a module build. + :type build: :class:`ModuleBuild` + :return: the value returned from :meth:`get_module_build_dependencies` + according to the configured resolver. + :rtype: dict[str, Modulemd.Module] + """ + resolver = module_build_service.resolver.system_resolver + if conf.system in ['koji', 'test']: + # For Koji backend, query for the module we are going to + # build to get the koji_tag and deps from it. + log.info('Getting tag for %s:%s:%s', build.name, build.stream, build.version) + return resolver.get_module_build_dependencies( + build.name, build.stream, build.version, build.context, strict=True) + else: + # In case of non-koji backend, we want to get the dependencies + # of the local module build based on Modulemd.Module, because the + # local build is not stored in the external MBS and therefore we + # cannot query it using the `query` as for Koji below. + return resolver.get_module_build_dependencies(mmd=build.mmd(), strict=True) + + +def get_content_generator_build_koji_tag(module_deps): + """Used by wait handler to get koji tag for import by content generator + + :param module_deps: a mapping from module's koji tag to its module + metadata. + :type: dict[str, Modulemd.Module] + :return: the koji tag. + :rtype: str + """ + if conf.system in ['koji', 'test']: + # Find out the name of Koji tag to which the module's Content + # Generator build should be tagged once the build finishes. + module_names_streams = { + mmd.get_name(): mmd.get_stream() for mmd in module_deps.values() + } + for base_module_name in conf.base_module_names: + if base_module_name in module_names_streams: + return conf.koji_cg_build_tag_template.format( + module_names_streams[base_module_name]) + + log.debug('No configured base module is a buildrequire. Hence use' + ' default content generator build koji tag %s', + conf.koji_cg_default_build_tag) + return conf.koji_cg_default_build_tag + else: + return conf.koji_cg_default_build_tag + + def wait(config, session, msg): """ Called whenever a module enters the 'wait' state. @@ -206,55 +276,8 @@ def wait(config, session, msg): # This is ok.. it's a race condition we can ignore. pass - resolver = module_build_service.resolver.system_resolver - - @module_build_service.utils.retry( - interval=10, timeout=120, - wait_on=(ValueError, RuntimeError, ConnectionError)) - def _get_deps_and_tag(): - """ - Private method to get the dependencies and koji tag of a module we - are going to build. We use private method here to allow "retry" - on failure. - """ - cg_build_koji_tag = conf.koji_cg_default_build_tag - if conf.system not in ['koji', 'test']: - # In case of non-koji backend, we want to get the dependencies - # of the local module build based on Modulemd.Module, because the - # local build is not stored in the external MBS and therefore we - # cannot query it using the `query` as for Koji below. - dep_koji_tags = resolver.get_module_build_dependencies( - mmd=build.mmd(), strict=True).keys() - - # We also don't want to get the tag name from the MBS, but just - # generate it locally instead. - tag = '-'.join(['module', build.name, build.stream, build.version]) - else: - # For Koji backend, query for the module we are going to - # build to get the koji_tag and deps from it. - nsvc = ':'.join([build.name, build.stream, build.version]) - log.info("Getting deps for %s" % (nsvc)) - deps_dict = resolver.get_module_build_dependencies( - build.name, build.stream, build.version, build.context, strict=True) - dep_koji_tags = set(deps_dict.keys()) - - # Find out the name of Koji tag to which the module's Content - # Generator build should be tagged once the build finishes. - module_names_streams = {mmd.get_name(): mmd.get_stream() - for mmd in deps_dict.values()} - for base_module_name in conf.base_module_names: - if base_module_name in module_names_streams: - cg_build_koji_tag = conf.koji_cg_build_tag_template.format( - module_names_streams[base_module_name]) - break - - log.info('Getting tag for {0}'.format(nsvc)) - tag = generate_koji_tag(build.name, build.stream, build.version, build.context) - - return dep_koji_tags, tag, cg_build_koji_tag - try: - dep_koji_tags, tag, cg_build_koji_tag = _get_deps_and_tag() + build_deps = get_module_build_dependencies(build) except ValueError: reason = "Failed to get module info from MBS. Max retries reached." log.exception(reason) @@ -262,6 +285,7 @@ def wait(config, session, msg): session.commit() raise + tag = generate_module_build_koji_tag(build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated @@ -270,6 +294,7 @@ def wait(config, session, msg): log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag + cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) log.debug("Assigning Content Generator build koji tag=%s to module " "build", cg_build_koji_tag) build.cg_build_koji_tag = cg_build_koji_tag @@ -277,8 +302,9 @@ def wait(config, session, msg): builder = module_build_service.builder.GenericBuilder.create_from_module( session, build, config) - log.debug("Adding dependencies %s into buildroot for module %s" % (dep_koji_tags, ':'.join( - [build.name, build.stream, build.version]))) + dep_koji_tags = build_deps.keys() + log.debug("Adding dependencies %s into buildroot for module %s:%s:%s", + dep_koji_tags, build.name, build.stream, build.version) builder.buildroot_add_repos(dep_koji_tags) if not build.component_builds: From b5860fa0394218b7bb41e96850d8988aa59088d3 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Nov 21 2018 01:28:03 +0000 Subject: [PATCH 2/2] Make it optional to tag build for CG import Fixes FACTORY-3467 Signed-off-by: Chenxiong Qi --- diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 887f016..a430578 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -41,7 +41,7 @@ from six import text_type import koji import pungi.arch -from module_build_service import log, build_logs, Modulemd, glib +from module_build_service import conf, log, build_logs, Modulemd, glib logging.basicConfig(level=logging.DEBUG) @@ -734,7 +734,8 @@ class KojiContentGenerator(object): try: serverdir = self._upload_outputs(session, metadata, file_dir) build_info = session.CGImport(metadata, serverdir) - self._tag_cg_build() + if conf.koji_cg_tag_build: + self._tag_cg_build() log.info("Content generator import done.") log.debug(json.dumps(build_info, sort_keys=True, indent=4)) diff --git a/module_build_service/config.py b/module_build_service/config.py index 34c6c09..1b94eb5 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -423,6 +423,11 @@ class Config(object): 'type': dict, 'default': {}, 'desc': 'Per base-module name:stream Koji arches list.'}, + 'koji_cg_tag_build': { + 'type': bool, + 'default': True, + 'desc': 'Indicate whether tagging build is enabled during importing ' + 'module to Koji.'}, 'koji_cg_build_tag_template': { 'type': str, 'default': "{}-modular-updates-candidate", diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 2a0bc14..28254db 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -294,10 +294,15 @@ def wait(config, session, msg): log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag - cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) - log.debug("Assigning Content Generator build koji tag=%s to module " - "build", cg_build_koji_tag) - build.cg_build_koji_tag = cg_build_koji_tag + if conf.koji_cg_tag_build: + cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) + log.debug("Assigning Content Generator build koji tag=%s to module build", + cg_build_koji_tag) + build.cg_build_koji_tag = cg_build_koji_tag + else: + log.debug('It is disabled to tag module build during importing into' + ' Koji by Content Generator.') + log.debug('Skip to assign Content Generator build koji tag to module build.') builder = module_build_service.builder.GenericBuilder.create_from_module( session, build, config) diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 7e6c068..a1c2635 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -26,6 +26,7 @@ import module_build_service.messaging import module_build_service.scheduler.handlers.modules import os import koji +import pytest from tests import conf, db, app, scheduler_init_data import module_build_service.resolver from module_build_service import build_logs, Modulemd @@ -206,6 +207,10 @@ class TestModuleWait: module_build = ModuleBuild.query.filter_by(id=2).one() assert module_build.cg_build_koji_tag == "modular-updates-candidate" + @pytest.mark.parametrize('koji_cg_tag_build,expected_cg_koji_build_tag', [ + [True, 'f27-modular-updates-candidate'], + [False, None], + ]) @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder.get_session") @@ -215,7 +220,8 @@ class TestModuleWait: @patch("module_build_service.config.Config.base_module_names", new_callable=mock.PropertyMock, return_value=set(["base-runtime", "platform"])) def test_set_cg_build_koji_tag( - self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg): + self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg, + koji_cg_tag_build, expected_cg_koji_build_tag): """ Test that build.cg_build_koji_tag is set. """ @@ -243,10 +249,12 @@ class TestModuleWait: resolver.get_module_build_dependencies.return_value = { "module-bootstrap-tag": base_mmd} - with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): - msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, - module_build_state='some state') - module_build_service.scheduler.handlers.modules.wait( - config=conf, session=db.session, msg=msg) - module_build = ModuleBuild.query.filter_by(id=2).one() - assert module_build.cg_build_koji_tag == "f27-modular-updates-candidate" + with patch.object(module_build_service.scheduler.handlers.modules.conf, + 'koji_cg_tag_build', new=koji_cg_tag_build): + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + module_build = ModuleBuild.query.filter_by(id=2).one() + assert module_build.cg_build_koji_tag == expected_cg_koji_build_tag