From 1a6a12144b9b1c580b940324e2f90171e1412a65 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jul 12 2018 12:03:43 +0000 Subject: Add 'include_unpublished_pulp_repos' compose flag. --- diff --git a/README.md b/README.md index 40d2436..1405a48 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ There are also additional optional attributes you can pass to `new_compose(...)` - `packages` - List of packages which should be included in a compose. This is used only when `source_type` is set to `tag` to further limit the compose repository. - `flags` - List of flags to further modify the compose output: - `no_deps` - For `tag` `source_type`, do not resolve dependencies between packages and include only packages listed in the `packages` in the compose. For `module` `source_type`, do not resolve dependencies between modules and include only the requested module in the compose. + - `include_unpublished_pulp_repos` - For `pulp` `source_type`, include also unpublished repositories for input content-sets. - `sigkeys` - List of signature keys IDs. Only packages signed by one of these keys will be included in a compose. If there is no signed version of a package, compose will fail. It is also possible to pass an empty-string in a list meaning unsigned packages are allowed. For example if you want to prefer packages signed by key with ID `123` and also allow unsigned packages to appear in a compose, you can do it by setting sigkeys to `["123", ""]`. - `results` - List of additional results which will be generated as part of a compose. Valid keys are: - `iso` - Generates non-installable ISO files with RPMs from a compose. diff --git a/common/odcs/common/types.py b/common/odcs/common/types.py index 29a2118..68ab6b5 100644 --- a/common/odcs/common/types.py +++ b/common/odcs/common/types.py @@ -75,6 +75,8 @@ COMPOSE_FLAGS = { "no_deps": 1, # Compose without pulling-in packages from the parent Koji tags. "no_inheritance": 2, + # For "pulp" source_type, include unpublished Pulp repos. + "include_unpublished_pulp_repos": 4, } INVERSE_COMPOSE_FLAGS = {v: k for k, v in COMPOSE_FLAGS.items()} diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 35953c9..28c4e24 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -458,7 +458,9 @@ def generate_pulp_compose(compose): compose=compose) repofile = "" - repos = pulp.get_repos_from_content_sets(content_sets) + repos = pulp.get_repos_from_content_sets( + content_sets, + compose.flags & COMPOSE_FLAGS["include_unpublished_pulp_repos"]) if len(repos) != len(content_sets): err = "Failed to find all the content_sets %r in the Pulp, " \ "found only %r" % (content_sets, repos.keys()) diff --git a/server/odcs/server/pulp.py b/server/odcs/server/pulp.py index 14c4122..2ff5d8d 100644 --- a/server/odcs/server/pulp.py +++ b/server/odcs/server/pulp.py @@ -128,7 +128,8 @@ class Pulp(object): "sigkeys": content_set_repos[0]["sigkeys"], } - def get_repos_from_content_sets(self, content_sets): + def get_repos_from_content_sets(self, content_sets, + include_unpublished_repos=False): """ Returns dictionary with URLs of all shipped repositories defined by the content_sets. @@ -151,12 +152,14 @@ class Pulp(object): 'criteria': { 'filters': { 'notes.content_set': {'$in': content_sets}, - 'notes.include_in_download_service': "True", }, 'fields': ['notes.relative_url', 'notes.content_set', 'notes.arch', 'notes.signatures'], } } + + if not include_unpublished_repos: + query_data['criteria']['filters']['notes.include_in_download_service'] = 'True' repos = self._rest_post('repositories/search/', query_data) per_content_set_repos = {} diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 761cc87..8d2dff7 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -334,6 +334,44 @@ gpgcheck=0 @patch("odcs.server.pulp.Pulp._rest_post") @patch("odcs.server.backend._write_repo_file") + def test_generate_pulp_compose_include_inpublished_pulp_repos_passed( + self, _write_repo_file, pulp_rest_post): + pulp_rest_post.return_value = [ + { + "notes": { + "relative_url": "content/1/x86_64/os", + "content_set": "foo-1", + "arch": "ppc64", + "signatures": "SIG1,SIG2" + }, + }, + ] + + c = Compose.create( + db.session, "me", PungiSourceType.PULP, "foo-1 foo-2", + COMPOSE_RESULTS["repository"], 3600, + flags=COMPOSE_FLAGS["include_unpublished_pulp_repos"]) + db.session.add(c) + db.session.commit() + + with patch.object(odcs.server.backend.conf, 'pulp_server_url', + "https://localhost/"): + generate_compose(1) + + expected_query = { + "criteria": { + "fields": ["notes.relative_url", "notes.content_set", + "notes.arch", "notes.signatures"], + "filters": { + "notes.content_set": {"$in": ["foo-1", "foo-2"]}, + } + } + } + pulp_rest_post.assert_called_once_with('repositories/search/', + expected_query) + + @patch("odcs.server.pulp.Pulp._rest_post") + @patch("odcs.server.backend._write_repo_file") def test_generate_pulp_compose_content_set_not_found( self, _write_repo_file, pulp_rest_post): pulp_rest_post.return_value = [ diff --git a/server/tests/test_pulp.py b/server/tests/test_pulp.py index 1a721dd..f3f0895 100644 --- a/server/tests/test_pulp.py +++ b/server/tests/test_pulp.py @@ -33,6 +33,45 @@ from .utils import ModelsBaseTest @patch("odcs.server.pulp.Pulp._rest_post") class TestPulp(ModelsBaseTest): + def test_pulp_request(self, pulp_rest_post): + c = Compose.create( + db.session, "me", PungiSourceType.PULP, "foo-1", 0, 3600) + db.session.commit() + + pulp_rest_post.return_value = [] + + pulp = Pulp("http://localhost/", "user", "pass", c) + pulp.get_repos_from_content_sets(["foo-1", "foo-2"]) + pulp_rest_post.assert_called_once_with( + 'repositories/search/', + {'criteria': { + 'fields': ['notes.relative_url', 'notes.content_set', + 'notes.arch', 'notes.signatures'], + 'filters': { + 'notes.include_in_download_service': 'True', + 'notes.content_set': {'$in': ['foo-1', 'foo-2']} + } + }}) + + def test_pulp_request_include_inpublished(self, pulp_rest_post): + c = Compose.create( + db.session, "me", PungiSourceType.PULP, "foo-1", 0, 3600) + db.session.commit() + + pulp_rest_post.return_value = [] + + pulp = Pulp("http://localhost/", "user", "pass", c) + pulp.get_repos_from_content_sets(["foo-1", "foo-2"], True) + pulp_rest_post.assert_called_once_with( + 'repositories/search/', + {'criteria': { + 'fields': ['notes.relative_url', 'notes.content_set', + 'notes.arch', 'notes.signatures'], + 'filters': { + 'notes.content_set': {'$in': ['foo-1', 'foo-2']} + } + }}) + def test_generate_pulp_compose_arch_merge(self, pulp_rest_post): """ Tests that multiple repos in single content_set are merged into