From b147d02bcb3a3168baed45b0d52fca52df5e9041 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 20 2017 09:01:14 +0000 Subject: Submit container image builds depending on finished container build. --- diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 8cca2c7..815a615 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -30,7 +30,8 @@ from freshmaker import conf, log, db, models from freshmaker.kojiservice import koji_service from freshmaker.mbs import MBS from freshmaker.models import ArtifactBuildState -from freshmaker.models import Event +from freshmaker.types import ArtifactType +from freshmaker.models import ArtifactBuild, Event from krbcontext import krbContext from freshmaker.odcsclient import ODCS @@ -203,7 +204,9 @@ class ContainerBuildHandler(BaseHandler): log.error('Could not login server %s', service.server) return None - log.debug('Building container from source: %s', scm_url) + log.info('Building container from source: %s, ' + 'release=%r, parent=%r, target=%r', + scm_url, release, koji_parent_build, target) return service.build_container(scm_url, branch, @@ -214,12 +217,50 @@ class ContainerBuildHandler(BaseHandler): koji_parent_build=koji_parent_build, scratch=conf.koji_container_scratch_build) - def _build_first_batch(self, db_event): - """ - Rebuilds all the parents images - images in the first batch which don't - depend on other images. + def build_image_artifact_build(self, build, repo_urls=[]): """ + Submits ArtifactBuild of 'image' type to Koji. + :param build: ArtifactBuild of 'image' type. + :rtype: number + :return: Koji build id. + """ + if build.state != ArtifactBuildState.PLANNED.value: + log.error("Trying to build %r container image, " + "but build it is not in PLANNED state", build) + return + + if not build.build_args: + log.error("Cannot rebuild container image %r, build_args not " + "defined", build) + return + + args = json.loads(build.build_args) + if not args["parent"]: + # TODO: Rebuild base image. + log.error("Base image %r should be rebuild, but this is not " + "supported yet", build) + return + + parent = args["parent"] + scm_url = "%s/%s#%s" % (conf.git_base_url, args["repository"], + args["commit"]) + release = build.name.split("-")[-1] + "." + str(int(time.time())) + # According to Luiz from OSBS team, it is OK to use "unknown" if + # we don't know the branch name. TODO: Get the branch name from + # Koji in lightblue.py. + branch = "unknown" + target = args["target"] + + return self.build_container( + scm_url, branch, target, repo_urls=repo_urls, + isolated=True, release=release, koji_parent_build=parent) + + def get_repo_urls(self, db_event): + """ + Returns list of URLs to ODCS repositories which should be used + to rebuild the container image for this event. + """ rebuild_event = Event.get(db.session, db_event.message_id) # Get compose ids of ODCS composes of all event dependencies. @@ -234,41 +275,22 @@ class ContainerBuildHandler(BaseHandler): verify_ssl=conf.odcs_verify_ssl) compose = odcs.get_compose(compose_id) repo_urls.append(compose["result_repofile"]) + return repo_urls + + def _build_first_batch(self, db_event): + """ + Rebuilds all the parents images - images in the first batch which don't + depend on other images. + """ + + repo_urls = self.get_repo_urls(db_event) + + builds = db.session.query(ArtifactBuild).filter_by( + type=ArtifactType.IMAGE.value, event_id=db_event.id, + dep_on=None).all() - for build in rebuild_event.builds: - if build.dep_on: - continue - - if build.state != ArtifactBuildState.PLANNED.value: - log.error("Trying to build first batch of container images, " - "but build %r is not in PLANNED state", build) - continue - - if not build.build_args: - log.error("Cannot rebuild container image %r, build_args not " - "defined", build) - continue - - args = json.loads(build.build_args) - - if not args["parent"]: - log.error("Base image %r should be rebuild, but this is not " - "supported yet", build) - continue - - parent = args["parent"] - scm_url = "%s/%s#%s" % (conf.git_base_url, args["repository"], - args["commit"]) - release = build.name.split("-")[-1] + "." + str(int(time.time())) - # According to Luiz from OSBS team, it is OK to use "unknown" if - # we don't know the branch name. TODO: Get the branch name from - # Koji in lightblue.py. - branch = "unknown" - target = args["target"] - - build.build_id = self.build_container( - scm_url, branch, target, repo_urls=repo_urls, - isolated=True, release=release, koji_parent_build=parent) + for build in builds: + build.build_id = self.build_image_artifact_build(build, repo_urls) build.state = ArtifactBuildState.BUILD.value db.session.add(build) db.session.commit() diff --git a/freshmaker/handlers/brew/container_task_state_change.py b/freshmaker/handlers/brew/container_task_state_change.py index 5fd26f2..676faee 100644 --- a/freshmaker/handlers/brew/container_task_state_change.py +++ b/freshmaker/handlers/brew/container_task_state_change.py @@ -23,11 +23,11 @@ from freshmaker import log from freshmaker import db from freshmaker.events import BrewContainerTaskStateChangeEvent from freshmaker.models import ArtifactBuild -from freshmaker.handlers import BaseHandler +from freshmaker.handlers import ContainerBuildHandler from freshmaker.types import ArtifactType, ArtifactBuildState -class BrewContainerTaskStateChangeHandler(BaseHandler): +class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): """Rebuild container when a dependecy container is built in Brew""" name = 'BrewContainerTaskStateChangeHandler' @@ -59,6 +59,7 @@ class BrewContainerTaskStateChangeHandler(BaseHandler): planned_builds = db.session.query(ArtifactBuild).filter_by(type=ArtifactType.IMAGE.value, state=ArtifactBuildState.PLANNED.value, dep_on=found_build).all() + repo_urls = self.get_repo_urls(found_build.event) for build in planned_builds: - # TODO: enable rebuild for these containers - log.info("Build %s depends on build %s" % (str(build), str(found_build))) + log.info("Build %r depends on build %r" % (build, found_build)) + self.build_image_artifact_build(build, repo_urls) diff --git a/tests/test_brew_container_task_state_change_handler.py b/tests/test_brew_container_task_state_change_handler.py index f75b45d..2a60890 100644 --- a/tests/test_brew_container_task_state_change_handler.py +++ b/tests/test_brew_container_task_state_change_handler.py @@ -62,11 +62,13 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): event = self.get_event_from_msg(get_fedmsg('brew_container_task_failed')) self.assertTrue(self.handler.can_handle(event)) - @mock.patch('freshmaker.handlers.brew.container_task_state_change.log') - def test_build_containers_when_dependency_container_is_built(self, log): + @mock.patch('freshmaker.handlers.ContainerBuildHandler.build_image_artifact_build') + @mock.patch('freshmaker.handlers.ContainerBuildHandler.get_repo_urls') + def test_build_containers_when_dependency_container_is_built(self, repo_urls, build_image): """ Tests when dependency container is built, rebuild containers depend on it. """ + repo_urls.return_value = ["url"] e1 = models.Event.create(db.session, "test_msg_id", "RHSA-2018-001", events.TestingEvent) event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) @@ -82,17 +84,18 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): self.handler.handle(event) self.assertEqual(base_build.state, ArtifactBuildState.DONE.value) # we only log the builds at this moment - log.info.assert_has_calls([ - mock.call('Build %s depends on build %s' % (str(build_0), str(base_build))), - mock.call('Build %s depends on build %s' % (str(build_1), str(base_build))), - mock.call('Build %s depends on build %s' % (str(build_2), str(base_build))), + build_image.assert_has_calls([ + mock.call(build_0, ['url']), mock.call(build_1, ['url']), + mock.call(build_2, ['url']), ]) - @mock.patch('freshmaker.handlers.brew.container_task_state_change.log') - def test_not_build_containers_when_dependency_container_build_task_failed(self, log): + @mock.patch('freshmaker.handlers.ContainerBuildHandler.build_image_artifact_build') + @mock.patch('freshmaker.handlers.ContainerBuildHandler.get_repo_urls') + def test_not_build_containers_when_dependency_container_build_task_failed(self, repo_urls, build_image): """ Tests when dependency container build task failed in brew, only update build state in db. """ + repo_urls.return_value = ["url"] e1 = models.Event.create(db.session, "test_msg_id", "RHSA-2018-001", events.TestingEvent) event = self.get_event_from_msg(get_fedmsg('brew_container_task_failed')) @@ -102,7 +105,7 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): dep_on=base_build, state=ArtifactBuildState.PLANNED.value) self.handler.handle(event) self.assertEqual(base_build.state, ArtifactBuildState.FAILED.value) - log.info.assert_not_called() + build_image.assert_not_called() if __name__ == '__main__':