From ad6874b0e6c8aae40245f08858461bf5bc3bdaf8 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 25 2017 06:36:29 +0000 Subject: Fix #670 - Tag Content Generator Koji build to special tag based on the base module stream --- diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 10ff68a..aed1273 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -359,6 +359,42 @@ class KojiContentGenerator(object): return serverdir + def _tag_cg_build(self): + """ + Tags the Content Generator build to module.cg_build_koji_tag. + """ + session = KojiModuleBuilder.get_session(self.config, self.owner) + + tag_name = self.module.cg_build_koji_tag + if not tag_name: + log.info("%r: Not tagging Content Generator build, no " + "cg_build_koji_tag set", self.module) + return + + tag_names_to_try = [tag_name, self.config.koji_cg_default_build_tag] + for tag in tag_names_to_try: + log.info("Trying %s", tag) + tag_info = session.getTag(tag) + if tag_info: + break + + log.info("%r: Tag %s not found in Koji, trying next one.", + self.module, tag) + + if not tag_info: + log.warn("%r:, Not tagging Content Generator build, no " + "available tag found, tried %r", self.module, + tag_names_to_try) + return + + build = self._get_build() + nvr = "%s-%s-%s" % (build["name"], build["version"], build["release"]) + + log.info("Content generator build %s will be tagged as %s in " + "Koji", nvr, tag) + session.tagBuild(tag_info["id"], nvr) + + def koji_import(self): """This method imports given module into the configured koji instance as a content generator based build @@ -371,6 +407,7 @@ class KojiContentGenerator(object): try: serverdir = self._upload_outputs(session, metadata, file_dir) build_info = session.CGImport(metadata, serverdir) + self._tag_cg_build() log.info("Content generator import done.") log.debug(json.dumps(build_info, sort_keys=True, indent=4)) diff --git a/module_build_service/config.py b/module_build_service/config.py index 5c72aa0..74e2cc4 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -389,7 +389,25 @@ class Config(object): 'type': str, 'default': '', 'desc': ('The distinguished name of the container or organizational unit containing ' - 'the groups in LDAP')} + 'the groups in LDAP')}, + 'base_module_names': { + 'type': set, + 'default': set(['platform', 'bootstrap']), + 'desc': "Set of module names which defines the product version " + "(by their stream) of modules depending on them."}, + 'koji_cg_build_tag_template': { + 'type': str, + 'default': "{}-modular-updates-candidate", + 'desc': "Name of a Koji tag where the top-level Content Generator " + "build is tagged to. The '{}' string is replaced by a " + "stream name of a base module on top of which the " + "module is built."}, + 'koji_cg_default_build_tag': { + 'type': str, + 'default': "modular-updates-candidate", + 'desc': "The name of Koji tag which should be used as fallback " + "when koji_cg_build_tag_template tag is not found in " + "Koji."}, } def __init__(self, conf_section_obj): diff --git a/module_build_service/migrations/versions/edb537dd1e8c_.py b/module_build_service/migrations/versions/edb537dd1e8c_.py new file mode 100644 index 0000000..a14aa1f --- /dev/null +++ b/module_build_service/migrations/versions/edb537dd1e8c_.py @@ -0,0 +1,22 @@ +"""Add cg_build_koji_tag + +Revision ID: edb537dd1e8c +Revises: c11a3cfec2a9 +Create Date: 2017-09-22 13:50:41.433144 + +""" + +# revision identifiers, used by Alembic. +revision = 'edb537dd1e8c' +down_revision = 'c11a3cfec2a9' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('module_builds', sa.Column('cg_build_koji_tag', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('module_builds', 'cg_build_koji_tag') diff --git a/module_build_service/models.py b/module_build_service/models.py index 843cd52..ccaee83 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -148,6 +148,8 @@ class ModuleBuild(MBSBase): state_reason = db.Column(db.String) modulemd = db.Column(db.String, nullable=False) koji_tag = db.Column(db.String) # This gets set after 'wait' + # Koji tag to which tag the Content Generator Koji build. + cg_build_koji_tag = db.Column(db.String) # This gets set after wait copr_owner = db.Column(db.String) copr_project = db.Column(db.String) scmurl = db.Column(db.String) diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index 45c4360..277cbe7 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -389,7 +389,7 @@ def get_module_build_dependencies(session, module_info, strict=False): instance. :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. - :return final list of koji tags + :return dict with koji_tag as a key and ModuleMetadata object as value. Example minimal module_info: { @@ -402,7 +402,7 @@ def get_module_build_dependencies(session, module_info, strict=False): # XXX get definitive list of modules # This is the set we're going to build up and return. - module_tags = set() + module_tags = {} if not isinstance(module_info, modulemd.ModuleMetadata): queried_module = get_module(session, module_info, strict=strict) @@ -430,7 +430,10 @@ def get_module_build_dependencies(session, module_info, strict=False): modules = _get_recursively_required_modules( session, modified_dep, strict=strict) tags = [m["koji_tag"] for m in modules] - module_tags = module_tags.union(set(tags)) + for m in modules: + if m["koji_tag"] in module_tags: + continue + module_tags[m["koji_tag"]] = _extract_modulemd(m["modulemd"]) return module_tags diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index dc6e184..bb1eb07 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -172,13 +172,15 @@ def wait(config, session, msg): are going to build. We use private method here to allow "retry" on failure. """ + cg_build_koji_tag = conf.koji_cg_default_build_tag + if conf.system != "koji": # In case of non-koji backend, we want to get the dependencies # of the local module build based on ModuleMetadata, because the # local build is not stored in PDC and therefore we cannot query # it using the `pdc_query` as for Koji below. dependencies = module_build_service.pdc.get_module_build_dependencies( - pdc_session, build.mmd(), strict=True) + pdc_session, build.mmd(), strict=True).keys() # We also don't want to get the tag name from the PDC, but just # generate it locally instead. @@ -194,16 +196,28 @@ def wait(config, session, msg): 'release': module_info['version'], } log.info("Getting %s deps from pdc (query %r)" % (module_info['name'], pdc_query)) - dependencies = module_build_service.pdc.get_module_build_dependencies( + deps_dict = module_build_service.pdc.get_module_build_dependencies( pdc_session, pdc_query, strict=True) + dependencies = set(deps_dict.keys()) + + # Find out the name of Koji tag to which the module's Content + # Generator build should be tagged once the build finishes. + module_names_streams = {mmd.name:mmd.stream + for mmd in deps_dict.values()} + for base_module_name in conf.base_module_names: + if base_module_name in module_names_streams: + cg_build_koji_tag = conf.koji_cg_build_tag_template.format( + module_names_streams[base_module_name]) + break + log.info("Getting %s tag from pdc (query %r)" % (module_info['name'], pdc_query)) tag = module_build_service.pdc.get_module_tag( pdc_session, pdc_query, strict=True) - return dependencies, tag + return dependencies, tag, cg_build_koji_tag try: - dependencies, tag = _get_deps_and_tag() + dependencies, tag, cg_build_koji_tag = _get_deps_and_tag() except ValueError: reason = "Failed to get module info from PDC. Max retries reached." log.exception(reason) @@ -219,6 +233,10 @@ def wait(config, session, msg): log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag + log.debug("Assigning Content Generator build koji tag=%s to module " + "build", cg_build_koji_tag) + build.cg_build_koji_tag = cg_build_koji_tag + builder = module_build_service.builder.GenericBuilder.create_from_module( session, build, config) diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index db8b7f7..e572ed7 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -32,7 +32,7 @@ import module_build_service.messaging import module_build_service.scheduler.handlers.repos from module_build_service import models, conf, build_logs -from mock import patch, Mock +from mock import patch, Mock, MagicMock, call from tests import init_data @@ -52,6 +52,7 @@ class TestBuild(unittest.TestCase): def setUp(self): init_data() module = models.ModuleBuild.query.filter_by(id=1).one() + module.cg_build_koji_tag = "f27-module-candidate" self.cg = KojiContentGenerator(module, conf) filename = cassette_dir + self.id() @@ -158,3 +159,52 @@ class TestBuild(unittest.TestCase): dir_path = self.cg._prepare_file_directory() with open(path.join(dir_path, "modulemd.txt")) as mmd: self.assertEqual(len(mmd.read()), 1134) + + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + def test_tag_cg_build(self, get_session): + """ Test that the CG build is tagged. """ + koji_session = MagicMock() + koji_session.getTag.return_value = {'id': 123} + get_session.return_value = koji_session + + self.cg._tag_cg_build() + + koji_session.getTag.assert_called_once_with(self.cg.module.cg_build_koji_tag) + koji_session.tagBuild.assert_called_once_with(123, "nginx-1-2") + + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + def test_tag_cg_build_fallback_to_default_tag(self, get_session): + """ Test that the CG build is tagged to default tag. """ + koji_session = MagicMock() + koji_session.getTag.side_effect = [{}, {'id': 123}] + get_session.return_value = koji_session + + self.cg._tag_cg_build() + + self.assertEqual(koji_session.getTag.mock_calls, + [call(self.cg.module.cg_build_koji_tag), + call(conf.koji_cg_default_build_tag)]) + koji_session.tagBuild.assert_called_once_with(123, "nginx-1-2") + + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + def test_tag_cg_build_no_tag_set(self, get_session): + """ Test that the CG build is not tagged when no tag set. """ + koji_session = MagicMock() + koji_session.getTag.side_effect = [{}, {'id': 123}] + get_session.return_value = koji_session + + self.cg.module.cg_build_koji_tag = None + self.cg._tag_cg_build() + + koji_session.tagBuild.assert_not_called() + + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + def test_tag_cg_build_no_tag_available(self, get_session): + """ Test that the CG build is not tagged when no tag available. """ + koji_session = MagicMock() + koji_session.getTag.side_effect = [{}, {}] + get_session.return_value = koji_session + + self.cg._tag_cg_build() + + koji_session.tagBuild.assert_not_called() diff --git a/tests/test_pdc.py b/tests/test_pdc.py index 8a779df..efed3fd 100644 --- a/tests/test_pdc.py +++ b/tests/test_pdc.py @@ -86,7 +86,7 @@ class TestPDCModule(unittest.TestCase): 'version': 'master', 'release': '20170315134803', } - result = mbs_pdc.get_module_build_dependencies(self.pdc, query) + result = mbs_pdc.get_module_build_dependencies(self.pdc, query).keys() expected = [ u'module-bootstrap-rawhide', ] @@ -103,7 +103,7 @@ class TestPDCModule(unittest.TestCase): 'version': 'master', 'release': '20170322155247' } - result = mbs_pdc.get_module_build_dependencies(self.pdc, query) + result = mbs_pdc.get_module_build_dependencies(self.pdc, query).keys() expected = [ u'module-base-runtime-master-20170315134803', ] @@ -127,7 +127,7 @@ class TestPDCModule(unittest.TestCase): build = module_build_service.models.ModuleBuild.local_modules( db.session, "child", "master") - result = mbs_pdc.get_module_build_dependencies(self.pdc, build[0].mmd()) + result = mbs_pdc.get_module_build_dependencies(self.pdc, build[0].mmd()).keys() local_path = os.path.join(base_dir, 'staged_data', "local_builds") diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index b7be974..c98a420 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -31,7 +31,7 @@ import vcr import koji from tests import conf, db, app, scheduler_init_data from module_build_service import build_logs -from module_build_service.models import ComponentBuild +from module_build_service.models import ComponentBuild, ModuleBuild base_dir = os.path.dirname(os.path.dirname(__file__)) cassette_dir = base_dir + '/vcr-request-data/' @@ -154,3 +154,81 @@ class TestModuleWait(unittest.TestCase): module_build_service.scheduler.handlers.modules.wait( config=conf, session=db.session, msg=msg) self.assertTrue(koji_session.newRepo.called) + + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value={'build': [], 'srpm-build': []}) + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + @patch("module_build_service.builder.GenericBuilder.create_from_module") + @patch('module_build_service.pdc') + def test_set_cg_build_koji_tag_fallback_to_default( + self, pdc, create_builder, koji_get_session, dbg): + """ + Test that build.cg_build_koji_tag fallbacks to default tag. + """ + with app.app_context(): + pdc.get_module_tag.return_value = "module-testmodule-master-20170109091357" + base_mmd = _modulemd.ModuleMetadata() + base_mmd.name = "base-runtime" + base_mmd.stream = "f27" + pdc.get_module_build_dependencies.return_value = { + "module-bootstrap-tag": base_mmd} + + scheduler_init_data() + koji_session = mock.MagicMock() + koji_session.newRepo.return_value = 123456 + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + builder.koji_session = koji_session + builder.module_build_tag = {"name": "module-123-build"} + builder.get_disttag_srpm.return_value = 'some srpm disttag' + builder.build.return_value = 1234, koji.BUILD_STATES['BUILDING'], "", "module-build-macros-1-1" + create_builder.return_value = builder + + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + module_build = ModuleBuild.query.filter_by(id=1).one() + self.assertEqual(module_build.cg_build_koji_tag, "modular-updates-candidate") + + + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value={'build': [], 'srpm-build': []}) + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + @patch("module_build_service.builder.GenericBuilder.create_from_module") + @patch('module_build_service.pdc') + @patch("module_build_service.config.Config.base_module_names", + new_callable=mock.PropertyMock, return_value=set(["base-runtime"])) + def test_set_cg_build_koji_tag( + self, cfg, pdc, create_builder, koji_get_session, dbg): + """ + Test that build.cg_build_koji_tag is set. + """ + with app.app_context(): + pdc.get_module_tag.return_value = "module-testmodule-master-20170109091357" + base_mmd = _modulemd.ModuleMetadata() + base_mmd.name = "base-runtime" + base_mmd.stream = "f27" + pdc.get_module_build_dependencies.return_value = { + "module-bootstrap-tag": base_mmd} + + scheduler_init_data() + koji_session = mock.MagicMock() + koji_session.newRepo.return_value = 123456 + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + builder.koji_session = koji_session + builder.module_build_tag = {"name": "module-123-build"} + builder.get_disttag_srpm.return_value = 'some srpm disttag' + builder.build.return_value = 1234, koji.BUILD_STATES['BUILDING'], "", "module-build-macros-1-1" + create_builder.return_value = builder + + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + module_build = ModuleBuild.query.filter_by(id=1).one() + self.assertEqual(module_build.cg_build_koji_tag, + "f27-modular-updates-candidate")