From 5196e4e93960d50af74107a4ea0fa4b25b47873a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mar 15 2018 07:18:56 +0000 Subject: Generate Pulp repositories only if OSBS/Koji does not support pulp_repos and content-sets. --- diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index 7b8e37b..193c35a 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -25,6 +25,8 @@ import json import koji import requests +import yaml +import os from six.moves import cStringIO from six.moves import configparser @@ -40,7 +42,8 @@ from freshmaker.errata import Errata from freshmaker.types import ArtifactType, ArtifactBuildState, EventState from freshmaker.models import Event, Compose from freshmaker.consumer import work_queue_put -from freshmaker.utils import krb_context, retry, get_rebuilt_nvr +from freshmaker.utils import ( + krb_context, retry, get_rebuilt_nvr, temp_dir, clone_distgit_repo) from freshmaker.odcsclient import create_odcs_client from odcs.common.types import COMPOSE_STATES @@ -151,6 +154,42 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): return [] + def _should_generate_yum_repourls(self, repository, branch, commit): + """ + Returns False if Koji/OSBS can build container without Freshmaker + generating yum_repourls for content sets. + + This returns False if both content_sets.yml and container.yaml exists + and the "pulp_repos" in container.yaml is set to True. + """ + if "/" in repository: + namespace, name = repository.split("/") + else: + namespace = "rpms" + name = repository + + prefix = "freshmaker-%s-%s-%s" % (namespace, name, commit) + with temp_dir(prefix=prefix) as repodir: + clone_distgit_repo(namespace, name, repodir, commit=commit, + ssh=False, logger=log) + + content_sets_path = os.path.join(repodir, "content_sets.yml") + if not os.path.exists(content_sets_path): + return True + + container_path = os.path.join(repodir, "container.yaml") + if not os.path.exists(container_path): + return True + + with open(container_path, 'r') as f: + container_yaml = yaml.load(f) + + if ("pulp_repos" not in container_yaml or + not container_yaml["pulp_repos"]): + return True + + return False + def _fake_odcs_new_compose( self, compose_source, tag, packages=None, results=[]): """ @@ -546,12 +585,15 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): if state != ArtifactBuildState.FAILED.value: # Store odcs pulp compose to build - compose = self._prepare_pulp_repo( - build.event, image["content_sets"]) - db_compose = Compose(odcs_compose_id=compose['id']) - db.session.add(db_compose) - db.session.commit() - build.add_composes(db.session, [db_compose]) + build_pulp_compose = self._should_generate_yum_repourls( + image["repository"], image["git_branch"], image["commit"]) + if build_pulp_compose: + compose = self._prepare_pulp_repo( + build.event, image["content_sets"]) + db_compose = Compose(odcs_compose_id=compose['id']) + db.session.add(db_compose) + db.session.commit() + build.add_composes(db.session, [db_compose]) # TODO: uncomment following code after boot.iso compose is # deployed in ODCS server. diff --git a/freshmaker/utils.py b/freshmaker/utils.py index 1ba9982..f3191b7 100644 --- a/freshmaker/utils.py +++ b/freshmaker/utils.py @@ -164,13 +164,19 @@ def temp_dir(logger=None, *args, **kwargs): logger.warn('Error removing %s: %s', dir, exc.strerror) -def clone_repo(url, dest, branch='master', logger=None): +def clone_repo(url, dest, branch='master', logger=None, commit=None): cmd = ['git', 'clone', '-b', branch, url, dest] _run_command(cmd, logger=logger) + + if commit: + cmd = ['git', 'checkout', commit] + _run_command(cmd, logger=logger, rundir=dest) + return dest -def clone_distgit_repo(namespace, name, dest, branch='master', ssh=True, user=None, logger=None): +def clone_distgit_repo(namespace, name, dest, branch='master', ssh=True, + user=None, logger=None, commit=None): """clone a git repo""" if ssh: if user is None: @@ -183,7 +189,8 @@ def clone_distgit_repo(namespace, name, dest, branch='master', ssh=True, user=No repo_url = conf.git_base_url repo_url = os.path.join(repo_url, namespace, name) - return clone_repo(repo_url, dest, branch=branch, logger=logger) + return clone_repo(repo_url, dest, branch=branch, logger=logger, + commit=commit) def add_empty_commit(repo, msg="bump", author=None, logger=None): diff --git a/tests/helpers.py b/tests/helpers.py index f5639d9..1d0e1df 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -44,6 +44,31 @@ BUILD_STATES = { } +class AnyStringWith(str): + def __eq__(self, other): + return self in other + + +class Patcher(object): + def __init__(self, prefix=None): + self.prefix = prefix or "" + self.patchers = [] + + def patch(self, name, *args, **kwargs): + if name.startswith("freshmaker."): + prefix = "" + else: + prefix = self.prefix + patcher = patch("%s%s" % (prefix, name), *args, **kwargs) + self.patchers.append(patcher) + patched_object = patcher.start() + return patched_object + + def unpatch_all(self): + for patcher in self.patchers: + patcher.stop() + + class FreshmakerTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/test_errata_advisory_rpms_signed_handler.py b/tests/test_errata_advisory_rpms_signed_handler.py index 6798298..fce83b2 100644 --- a/tests/test_errata_advisory_rpms_signed_handler.py +++ b/tests/test_errata_advisory_rpms_signed_handler.py @@ -21,11 +21,11 @@ import requests -from mock import patch +from mock import patch, mock_open import freshmaker -from freshmaker import db +from freshmaker import db, log from freshmaker.events import ErrataAdvisoryRPMsSignedEvent from freshmaker.handlers.errata import ErrataAdvisoryRPMsSignedHandler from freshmaker.lightblue import ContainerImage @@ -69,6 +69,13 @@ class TestErrataAdvisoryRPMsSignedHandler(helpers.ModelsTestCase): self.mock_request_boot_iso_compose = \ self.request_boot_iso_compose_patcher.start() + self.should_generate_yum_repourls_patcher = patch( + 'freshmaker.handlers.errata.' + 'ErrataAdvisoryRPMsSignedHandler._should_generate_yum_repourls', + return_value=True) + self.should_generate_yum_repourls = \ + self.should_generate_yum_repourls_patcher.start() + # Fake images found to rebuild has these relationships # # Batch 1 | Batch 2 | Batch 3 @@ -199,6 +206,7 @@ class TestErrataAdvisoryRPMsSignedHandler(helpers.ModelsTestCase): self.find_images_patcher.stop() self.prepare_pulp_repo_patcher.stop() self.messaging_publish_patcher.stop() + self.should_generate_yum_repourls_patcher.stop() @patch.object(freshmaker.conf, 'handler_build_whitelist', new={ 'ErrataAdvisoryRPMsSignedHandler': { @@ -611,3 +619,85 @@ class TestFindImagesToRebuild(helpers.FreshmakerTestCase): 'httpd', ['content-set-1'], filter_fnc=self.handler._filter_out_not_allowed_builds, published=True, release_category='Generally Available') + + +class TestShouldGenerateYumRepourls(helpers.FreshmakerTestCase): + + def setUp(self): + super(TestShouldGenerateYumRepourls, self).setUp() + + self.patcher = helpers.Patcher( + "freshmaker.handlers.errata.errata_advisory_rpms_signed.") + self.clone_distgit_repo = self.patcher.patch("clone_distgit_repo") + self.path_exists = self.patcher.patch("os.path.exists") + self.patched_open = self.patcher.patch("open", create=True) + + self.handler = ErrataAdvisoryRPMsSignedHandler() + + def tearDown(self): + super(TestShouldGenerateYumRepourls, self).tearDown() + self.patcher.unpatch_all() + + def test_generate(self): + self.path_exists.return_value = True + self.patched_open.return_value = mock_open( + read_data="pulp_repos: True").return_value + + ret = self.handler._should_generate_yum_repourls( + "rpms/foo-docker", "branch", "commit") + self.assertEqual(ret, False) + + self.clone_distgit_repo.assert_called_once_with( + 'rpms', 'foo-docker', + helpers.AnyStringWith('freshmaker-rpms-foo-docker'), + commit='commit', logger=log, ssh=False) + + def test_generate_no_namespace(self): + self.path_exists.return_value = True + self.patched_open.return_value = mock_open( + read_data="pulp_repos: True").return_value + + ret = self.handler._should_generate_yum_repourls( + "foo-docker", "branch", "commit") + self.assertEqual(ret, False) + + self.clone_distgit_repo.assert_called_once_with( + 'rpms', 'foo-docker', + helpers.AnyStringWith('freshmaker-rpms-foo-docker'), + commit='commit', logger=log, ssh=False) + + def test_generate_no_pulp_repos(self): + self.path_exists.return_value = True + self.patched_open.return_value = mock_open( + read_data="pulp_repos_x: True").return_value + + ret = self.handler._should_generate_yum_repourls( + "rpms/foo-docker", "branch", "commit") + self.assertEqual(ret, True) + + def test_generate_pulp_repos_false(self): + self.path_exists.return_value = True + self.patched_open.return_value = mock_open( + read_data="pulp_repos: False").return_value + + ret = self.handler._should_generate_yum_repourls( + "rpms/foo-docker", "branch", "commit") + self.assertEqual(ret, True) + + def test_generate_no_content_sets_yml(self): + def mocked_path_exists(path): + return not path.endswith("content_sets.yml") + self.path_exists.side_effect = mocked_path_exists + + ret = self.handler._should_generate_yum_repourls( + "rpms/foo-docker", "branch", "commit") + self.assertEqual(ret, True) + + def test_generate_no_container_yaml(self): + def mocked_path_exists(path): + return not path.endswith("container.yaml") + self.path_exists.side_effect = mocked_path_exists + + ret = self.handler._should_generate_yum_repourls( + "rpms/foo-docker", "branch", "commit") + self.assertEqual(ret, True) diff --git a/tests/test_errata_advisory_state_changed.py b/tests/test_errata_advisory_state_changed.py index 639418e..f15ab03 100644 --- a/tests/test_errata_advisory_state_changed.py +++ b/tests/test_errata_advisory_state_changed.py @@ -225,6 +225,18 @@ class TestAllowBuild(helpers.ModelsTestCase): class TestBatches(helpers.ModelsTestCase): """Test handling of batches""" + def setUp(self): + super(TestBatches, self).setUp() + self.should_generate_yum_repourls_patcher = patch( + 'freshmaker.handlers.errata.' + 'ErrataAdvisoryRPMsSignedHandler._should_generate_yum_repourls', + return_value=True) + self.should_generate_yum_repourls = \ + self.should_generate_yum_repourls_patcher.start() + + def tearDown(self): + self.should_generate_yum_repourls_patcher.stop() + def _mock_build(self, build, parent=None, error=None): if parent: parent = {"brew": {"build": parent + "-1-1.25"}} @@ -759,12 +771,20 @@ class TestRecordBatchesImages(helpers.ModelsTestCase): self.mock_request_boot_iso_compose = \ self.request_boot_iso_compose_patcher.start() + self.should_generate_yum_repourls_patcher = patch( + 'freshmaker.handlers.errata.' + 'ErrataAdvisoryRPMsSignedHandler._should_generate_yum_repourls', + return_value=True) + self.should_generate_yum_repourls = \ + self.should_generate_yum_repourls_patcher.start() + def tearDown(self): super(TestRecordBatchesImages, self).tearDown() self.request_boot_iso_compose_patcher.stop() self.prepare_pulp_repo_patcher.stop() self.event_types_patcher.stop() + self.should_generate_yum_repourls_patcher.stop() def test_record_batches(self): batches = [ @@ -849,6 +869,43 @@ class TestRecordBatchesImages(helpers.ModelsTestCase): self.assertEqual(parent_image, child_image.dep_on) self.assertEqual(ArtifactBuildState.PLANNED.value, child_image.state) + def test_record_batches_should_not_generate_pulp_repos(self): + self.should_generate_yum_repourls.return_value = False + batches = [ + [ContainerImage({ + "brew": { + "completion_date": "20170420T17:05:37.000-0400", + "build": "rhel-server-docker-7.3-82", + "package": "rhel-server-docker" + }, + 'parsed_data': { + 'layers': [ + 'sha512:12345678980', + 'sha512:10987654321' + ] + }, + "parent": None, + "content_sets": ["content-set-1"], + "repository": "repo-1", + "commit": "123456789", + "target": "target-candidate", + "git_branch": "rhel-7", + "error": None + })] + ] + + handler = ErrataAdvisoryRPMsSignedHandler() + handler._record_batches(batches, self.mock_event) + + # Check parent image + query = db.session.query(ArtifactBuild) + parent_image = query.filter( + ArtifactBuild.original_nvr == 'rhel-server-docker-7.3-82' + ).first() + self.assertNotEqual(None, parent_image) + self.assertEqual(ArtifactBuildState.PLANNED.value, parent_image.state) + self.mock_prepare_pulp_repo.assert_not_called() + @unittest.skip('Enable again when enable to request boot.iso compose') def test_pulp_compose_is_stored_for_each_build(self): batches = [