#211 Add 'include_unpublished_pulp_repos' compose flag.
Merged a year ago by jkaluza. Opened a year ago by jkaluza.

file modified
+1

@@ -132,6 +132,7 @@ 

  - `packages` - List of packages which should be included in a compose. This is used only when `source_type` is set to `tag` to further limit the compose repository.

  - `flags` - List of flags to further modify the compose output:

      - `no_deps` - For `tag` `source_type`, do not resolve dependencies between packages and include only packages listed in the `packages` in the compose. For `module` `source_type`, do not resolve dependencies between modules and include only the requested module in the compose.

+     - `include_unpublished_pulp_repos` - For `pulp` `source_type`, include also unpublished repositories for input content-sets.

  - `sigkeys` - List of signature keys IDs. Only packages signed by one of these keys will be included in a compose. If there is no signed version of a package, compose will fail. It is also possible to pass an empty-string in a list meaning unsigned packages are allowed. For example if you want to prefer packages signed by key with ID `123` and also allow unsigned packages to appear in a compose, you can do it by setting sigkeys to `["123", ""]`.

  - `results` - List of additional results which will be generated as part of a compose. Valid keys are:

      - `iso` - Generates non-installable ISO files with RPMs from a compose.

@@ -75,6 +75,8 @@ 

      "no_deps": 1,

      # Compose without pulling-in packages from the parent Koji tags.

      "no_inheritance": 2,

+     # For "pulp" source_type, include unpublished Pulp repos.

+     "include_unpublished_pulp_repos": 4,

  }

  

  INVERSE_COMPOSE_FLAGS = {v: k for k, v in COMPOSE_FLAGS.items()}

@@ -458,7 +458,9 @@ 

                  compose=compose)

  

      repofile = ""

-     repos = pulp.get_repos_from_content_sets(content_sets)

+     repos = pulp.get_repos_from_content_sets(

+         content_sets,

+         compose.flags & COMPOSE_FLAGS["include_unpublished_pulp_repos"])

      if len(repos) != len(content_sets):

          err = "Failed to find all the content_sets %r in the Pulp, " \

              "found only %r" % (content_sets, repos.keys())

file modified
+5 -2

@@ -128,7 +128,8 @@ 

              "sigkeys": content_set_repos[0]["sigkeys"],

          }

  

-     def get_repos_from_content_sets(self, content_sets):

+     def get_repos_from_content_sets(self, content_sets,

+                                     include_unpublished_repos=False):

          """

          Returns dictionary with URLs of all shipped repositories defined by

          the content_sets.

@@ -151,12 +152,14 @@ 

              'criteria': {

                  'filters': {

                      'notes.content_set': {'$in': content_sets},

-                     'notes.include_in_download_service': "True",

                  },

                  'fields': ['notes.relative_url', 'notes.content_set',

                             'notes.arch', 'notes.signatures'],

              }

          }

+ 

+         if not include_unpublished_repos:

+             query_data['criteria']['filters']['notes.include_in_download_service'] = 'True'

          repos = self._rest_post('repositories/search/', query_data)

  

          per_content_set_repos = {}

@@ -334,6 +334,44 @@ 

  

      @patch("odcs.server.pulp.Pulp._rest_post")

      @patch("odcs.server.backend._write_repo_file")

+     def test_generate_pulp_compose_include_inpublished_pulp_repos_passed(

+             self, _write_repo_file, pulp_rest_post):

+         pulp_rest_post.return_value = [

+             {

+                 "notes": {

+                     "relative_url": "content/1/x86_64/os",

+                     "content_set": "foo-1",

+                     "arch": "ppc64",

+                     "signatures": "SIG1,SIG2"

+                 },

+             },

+         ]

+ 

+         c = Compose.create(

+             db.session, "me", PungiSourceType.PULP, "foo-1 foo-2",

+             COMPOSE_RESULTS["repository"], 3600,

+             flags=COMPOSE_FLAGS["include_unpublished_pulp_repos"])

+         db.session.add(c)

+         db.session.commit()

+ 

+         with patch.object(odcs.server.backend.conf, 'pulp_server_url',

+                           "https://localhost/"):

+             generate_compose(1)

+ 

+         expected_query = {

+             "criteria": {

+                 "fields": ["notes.relative_url", "notes.content_set",

+                            "notes.arch", "notes.signatures"],

+                 "filters": {

+                     "notes.content_set": {"$in": ["foo-1", "foo-2"]},

+                 }

+             }

+         }

+         pulp_rest_post.assert_called_once_with('repositories/search/',

+                                                expected_query)

+ 

+     @patch("odcs.server.pulp.Pulp._rest_post")

+     @patch("odcs.server.backend._write_repo_file")

      def test_generate_pulp_compose_content_set_not_found(

              self, _write_repo_file, pulp_rest_post):

          pulp_rest_post.return_value = [

file modified
+39

@@ -33,6 +33,45 @@ 

  @patch("odcs.server.pulp.Pulp._rest_post")

  class TestPulp(ModelsBaseTest):

  

+     def test_pulp_request(self, pulp_rest_post):

+         c = Compose.create(

+             db.session, "me", PungiSourceType.PULP, "foo-1", 0, 3600)

+         db.session.commit()

+ 

+         pulp_rest_post.return_value = []

+ 

+         pulp = Pulp("http://localhost/", "user", "pass", c)

+         pulp.get_repos_from_content_sets(["foo-1", "foo-2"])

+         pulp_rest_post.assert_called_once_with(

+             'repositories/search/',

+             {'criteria': {

+                 'fields': ['notes.relative_url', 'notes.content_set',

+                            'notes.arch', 'notes.signatures'],

+                 'filters': {

+                     'notes.include_in_download_service': 'True',

+                     'notes.content_set': {'$in': ['foo-1', 'foo-2']}

+                 }

+             }})

+ 

+     def test_pulp_request_include_inpublished(self, pulp_rest_post):

+         c = Compose.create(

+             db.session, "me", PungiSourceType.PULP, "foo-1", 0, 3600)

+         db.session.commit()

+ 

+         pulp_rest_post.return_value = []

+ 

+         pulp = Pulp("http://localhost/", "user", "pass", c)

+         pulp.get_repos_from_content_sets(["foo-1", "foo-2"], True)

+         pulp_rest_post.assert_called_once_with(

+             'repositories/search/',

+             {'criteria': {

+                 'fields': ['notes.relative_url', 'notes.content_set',

+                            'notes.arch', 'notes.signatures'],

+                 'filters': {

+                     'notes.content_set': {'$in': ['foo-1', 'foo-2']}

+                 }

+             }})

+ 

      def test_generate_pulp_compose_arch_merge(self, pulp_rest_post):

          """

          Tests that multiple repos in single content_set are merged into

We currently filter repositories in content set with "notes.include_in_download_service": "True", but some projects want to create .repo file with unpublished Pulp repositories (for example in case they want to build container images for/with on unreleased content).

This PR allows to do that by setting new compose flag include_unpublished_pulp_repos. If you know about better/shorter name, tell me.

Can we make sure this flag is documented, at least in the README.md?

LGTM, but better to document this flag as Luiz suggested.

rebased onto 1a6a121

a year ago

Pull-Request has been merged by jkaluza

a year ago