From aec2bde43d7dfbdf37fa90f658c2640bf9ef8ed4 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jan 27 2020 14:02:59 +0000 Subject: Add ignore_absent_pulp_repos flag For `pulp` source, if a content set is specified but does not exist in Pulp instance, an error is returned to the caller and no repofile will be created. When this flag is set, any content set absent in Pulp is ignored. This is logged on the server, but the user will not see this. If all of them are missing, an empty repo file will be produced. JIRA: COMPOSE-4008 --- diff --git a/README.md b/README.md index eb20637..3c5be10 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ If the `packages` is not set, all packages in Koji tag or all packages in a `bui - `runtime` - Packages whose name ends with "-devel" or "-static" suffix will be considered as multilib. - `devel` - Packages that install some shared object file "*.so.*" will be considered as multilib. - `all` - All pakages will be considered as multilib. + - `ignore_absent_pulp_repos` - For `pulp` `source_type`, ignore any content set that does not exist in Pulp - `builds` - List of NVRs defining the Koji builds to include in a compose. Only valid for `tag` and `build` source types. For `tag` source type, the NVRs will be considered for inclusion in a compose on top of Koji tag defined by `source`. For `build` source type, only the Koji builds defined by the NVRs will be considered for inclusion. The `packages` still need to be set to include particular packages from the Koji builds in a compose. - `lookaside_repos` - List of base URLs of RPM repositories which should be considered when choosing packages for a compose. diff --git a/common/odcs/common/types.py b/common/odcs/common/types.py index 4f78765..692ca90 100644 --- a/common/odcs/common/types.py +++ b/common/odcs/common/types.py @@ -103,6 +103,9 @@ COMPOSE_FLAGS = { "check_deps": 8, # Include also module builds in the "done" state. "include_done_modules": 16, + # For "pulp" source_type, ignore any repos do not exist in the remote Pulp + # instance. + "ignore_absent_pulp_repos": 32, } INVERSE_COMPOSE_FLAGS = {v: k for k, v in COMPOSE_FLAGS.items()} diff --git a/docs/api.rst b/docs/api.rst index 6776b56..d8da2d3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -69,6 +69,7 @@ The fields used in the ODCS compose JSON have following meaning: - *no_deps* - Compose will contain only the requested packages/modules without pulling-in their RPM-level or Module-level dependencies. - *no_inheritance* - Only packages/modules directly tagged in the requested Koji tag will be added to the module. Inherited tags will be ignored. - *include_unpublished_pulp_repos* - Even unpublished Pulp repositories will be included in the resulting compose. + - *ignore_absent_pulp_repos* - Ignore non-existing content sets in the source of Pulp compose. The source field on the compose will be updated to match what was actually used in the compose. - *check_deps* - Compose will fail if the RPM-level dependencies between packages in the compose are not satisfied. - *include_done_modules* - Compose can include also modules which are in the ``done`` state. By default, only modules in ``ready`` state are allowed to be included in a composes. diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index e6e48f7..58eb36c 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -633,11 +633,20 @@ def generate_pulp_compose(compose): repos = pulp.get_repos_from_content_sets( content_sets, compose.flags & COMPOSE_FLAGS["include_unpublished_pulp_repos"]) + ignore_absent_pulp_repos = compose.flags & COMPOSE_FLAGS["ignore_absent_pulp_repos"] if len(repos) != len(content_sets): + found_content_sets = repos.keys() err = "Failed to find all the content_sets %r in the Pulp, " \ - "found only %r" % (content_sets, repos.keys()) - log.error(err) - raise ValueError(err) + "found only %r" % (content_sets, found_content_sets) + if ignore_absent_pulp_repos: + log.info(err) + # Update the source in the compose. This ensures the source matches + # what is actually in the compose. However it makes it invisible + # that user actually requested something else. + compose.source = " ".join(found_content_sets) + else: + log.error(err) + raise ValueError(err) arches = set() sigkeys = set() diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 13d4cae..894faf7 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -633,6 +633,58 @@ gpgcheck=0 self.assertEqual(c1.state, COMPOSE_STATES["failed"]) six.assertRegex(self, c1.state_reason, r'Error while generating compose: Failed to find all the content_sets.*') + @patch("odcs.server.pulp.Pulp._rest_post") + @patch("odcs.server.backend._write_repo_file") + def test_generate_pulp_compose_content_set_not_found_allow_absent( + self, _write_repo_file, pulp_rest_post + ): + pulp_rest_post.return_value = [ + { + "notes": { + "relative_url": "content/1/x86_64/os", + "content_set": "foo-1", + "arch": "ppc64", + "signatures": "SIG1,SIG2" + }, + }, + ] + + c = Compose.create( + db.session, "me", PungiSourceType.PULP, "foo-1 foo-2", + COMPOSE_RESULTS["repository"], 3600, + flags=COMPOSE_FLAGS["ignore_absent_pulp_repos"], + ) + db.session.add(c) + db.session.commit() + + with patch.object(odcs.server.backend.conf, 'pulp_server_url', + "https://localhost/"): + generate_compose(1) + + expected_query = { + "criteria": { + "fields": ["notes"], + "filters": { + "notes.content_set": {"$in": ["foo-1", "foo-2"]}, + "notes.include_in_download_service": "True" + } + } + } + pulp_rest_post.assert_called_once_with('repositories/search/', + expected_query) + expected_repofile = """ +[foo-1] +name=foo-1 +baseurl=http://localhost/content/1/x86_64/os +enabled=1 +gpgcheck=0 +""" + _write_repo_file.assert_called_once_with(c, expected_repofile) + + c1 = Compose.query.filter(Compose.id == 1).one() + self.assertEqual(c1.state, COMPOSE_STATES["done"]) + self.assertEqual(c1.source, "foo-1") + @patch("odcs.server.backend.resolve_compose") @patch("odcs.server.backend.generate_pungi_compose") @patch("odcs.server.pungi.PungiLogs.get_error_string")