#241 When no are set in input, include all the packages in a compose.
Merged 5 months ago by jkaluza. Opened 5 months ago by jkaluza.
jkaluza/odcs all-packages  into  master

file modified
+2 -1

@@ -130,7 +130,8 @@ 

  There are also additional optional attributes you can pass to `new_compose(...)` method:

  

  - `seconds_to_live` - Number of seconds after which the generated compose should expire and will be removed.

- - `packages` - List of packages which should be included in a compose. This is used only when `source_type` is set to `tag` to further limit the compose repository.

+ - `packages` - List of packages which should be included in a compose. This is used only when `source_type` is set to `tag` or `build` to further limit the compose repository.

+ If the `packages` is not set, all packages in Koji tag or all packages in a `builds` list will be included in a final compose.

  - `flags` - List of flags to further modify the compose output:

      - `no_deps` - For `tag` `source_type`, do not resolve dependencies between packages and include only packages listed in the `packages` in the compose. For `module` `source_type`, do not resolve dependencies between modules and include only the requested module in the compose.

      - `include_unpublished_pulp_repos` - For `pulp` `source_type`, include also unpublished repositories for input content-sets.

file modified
+12

@@ -48,6 +48,18 @@ 

  ]

  {%- endif %}

  

+ 

+ {%- if config.source_type_str in ["tag", "build"] and not config.packages %}

+ # In case no package is requested, include all of them.

+ additional_packages = [

+     ('^Temporary$', {

+         '*': [

+             '*',

+         ],

+     }),

+ ]

+ {%- endif %}

+ 

  filter_system_release_packages = False

  

  multilib = [

file modified
+7 -1

@@ -34,7 +34,9 @@ 

  import odcs.server.utils

  from odcs.server import conf, log, db

  from odcs.server import comps

- from odcs.common.types import PungiSourceType, COMPOSE_RESULTS, MULTILIB_METHODS

+ from odcs.common.types import (

+     PungiSourceType, COMPOSE_RESULTS, MULTILIB_METHODS,

+     INVERSE_PUNGI_SOURCE_TYPE_NAMES)

  from odcs.server.utils import makedirs, clone_repo, copytree

  

  

@@ -167,6 +169,10 @@ 

              raise ValueError("Unknown source_type %r" % source_type)

  

      @property

+     def source_type_str(self):

+         return INVERSE_PUNGI_SOURCE_TYPE_NAMES[self.source_type]

+ 

+     @property

      def release_short(self):

          return self.release_name[:16]

  

@@ -266,10 +266,6 @@ 

          if "builds" in source_data:

              builds = ' '.join(source_data["builds"])

  

-         if not packages and source_type == PungiSourceType.KOJI_TAG:

-             raise ValueError(

-                 '"packages" must be defined for "tag" source_type.')

- 

          sigkeys = ""

          if "sigkeys" in source_data:

              sigkeys = ' '.join(source_data["sigkeys"])

@@ -188,6 +188,8 @@ 

              cfg = self._load_pungi_cfg(template)

              self.assertEqual(set(cfg["pkgset_koji_builds"]),

                               set(["foo-1-1", "bar-1-1"]))

+             self.assertEqual(cfg["additional_packages"],

+                              [(u'^Temporary$', {u'*': [u'*']})])

  

      def test_get_pungi_conf_source_type_build(self):

          _, mock_path = tempfile.mkstemp()

@@ -204,6 +206,40 @@ 

              self.assertEqual(cfg["pkgset_koji_tag"], '')

              self.assertEqual(set(cfg["pkgset_koji_builds"]),

                               set(["foo-1-1", "bar-1-1"]))

+             self.assertEqual(cfg["additional_packages"],

+                              [(u'^Temporary$', {u'*': [u'*']})])

+ 

+     def test_get_pungi_conf_source_type_koji_tag_all_packages(self):

+         _, mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(os.path.join(test_dir,

+                                                      "../conf/pungi.conf"))

+         shutil.copy2(template_path, mock_path)

+ 

+         with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):

+             pungi_cfg = PungiConfig(

+                 "MBS-512", "1", PungiSourceType.KOJI_TAG, "f26")

+ 

+             template = pungi_cfg.get_pungi_config()

+             cfg = self._load_pungi_cfg(template)

+             self.assertEqual(cfg["pkgset_koji_tag"], 'f26')

+             self.assertEqual(cfg["additional_packages"],

+                              [('^Temporary$', {'*': ['*']})])

+ 

+     def test_get_pungi_conf_source_type_koji_tag_some_packages(self):

+         _, mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(os.path.join(test_dir,

+                                                      "../conf/pungi.conf"))

+         shutil.copy2(template_path, mock_path)

+ 

+         with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):

+             pungi_cfg = PungiConfig(

+                 "MBS-512", "1", PungiSourceType.KOJI_TAG, "f26",

+                 packages=["file"])

+ 

+             template = pungi_cfg.get_pungi_config()

+             cfg = self._load_pungi_cfg(template)

+             self.assertEqual(cfg["pkgset_koji_tag"], 'f26')

+             self.assertTrue("additional_packages" not in cfg)

  

  

  class TestPungi(unittest.TestCase):

file modified
+6 -3

@@ -321,9 +321,12 @@ 

                   'flags': ['no_deps']}))

              data = json.loads(rv.get_data(as_text=True))

  

-         self.assertEqual(

-             data['message'],

-             '"packages" must be defined for "tag" source_type.')

+         self.assertEqual(data["state_name"], "wait")

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+         self.assertEqual(c.packages, None)

  

      def test_submit_build_nodeps(self):

          with self.test_request_context(user='dev'):

This will need to be tested on staging on some big Koji tag, ideally rhel-7 one, but the amount work it does is quite similar to what we do already for such big tags and based on the tests on ODCS QE, it does not slow down the compose dramatically.

Pull-Request has been merged by jkaluza

5 months ago

A bit too late, but still +1.