#1269 pkgset: Allow filtering modules from Koji tags
Merged 4 years ago by lsedlar. Opened 4 years ago by lsedlar.
lsedlar/pungi filter-modules  into  master

file modified
+10
@@ -700,6 +700,16 @@ 

      See :ref:`additional_packages <additional_packages>` for details about

      package specification.

  

+ **filter_modules**

+     (*list*) -- modules to be excluded from a variant and architecture;

+     format: ``[(variant_uid_regex, {arch|*: [name:stream]})]``

+ 

+     Both name and stream can use shell-style globs. If stream is omitted, all

+     streams are removed.

+ 

+     This option only applies to modules taken from Koji tags, not modules

+     explicitly listed in variants XML without any tags.

+ 

  **filter_system_release_packages**

      (*bool*) -- for each variant, figure out the best system release package

      and filter out all others. This will not work if a variant needs more than

file modified
+4
@@ -733,6 +733,10 @@ 

                  "$ref": "#/definitions/package_mapping",

                  "default": [],

              },

+             "filter_modules": {

+                 "$ref": "#/definitions/package_mapping",

+                 "default": [],

+             },

              "sigkeys": {

                  "type": "array",

                  "items": {"$ref": "#/definitions/optional_string"},

@@ -17,6 +17,7 @@ 

  import os

  import json

  import re

+ from fnmatch import fnmatch

  from itertools import groupby

  

  from kobo.rpmlib import parse_nvra
@@ -26,7 +27,7 @@ 

  from pungi.wrappers.comps import CompsWrapper

  import pungi.phases.pkgset.pkgsets

  from pungi.arch import getBaseArch

- from pungi.util import retry, find_old_compose

+ from pungi.util import retry, find_old_compose, get_arch_variant_data

  from pungi import Modulemd

  

  from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -190,7 +191,9 @@ 

      return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)

  

  

- def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False):

+ def _add_module_to_variant(

+     koji_wrapper, variant, build, add_to_variant_modules=False, compose=None

+ ):

      """

      Adds module defined by Koji build info to variant.

  
@@ -198,6 +201,7 @@ 

      :param int: build id

      :param bool add_to_variant_modules: Adds the modules also to

          variant.modules.

+     :param compose: Compose object to get filters from

      """

      mmds = {}

      archives = koji_wrapper.koji_proxy.listArchives(build["id"])
@@ -225,26 +229,57 @@ 

          # longer supported and should be rebuilt. Let's skip it.

          return

  

-     nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % build["extra"]["typeinfo"]["module"]

+     info = build["extra"]["typeinfo"]["module"]

+     nsvc = "%(name)s:%(stream)s:%(version)s:%(context)s" % info

+ 

+     added = False

  

      for arch in variant.arches:

+         if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):

+             compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)

+             continue

+ 

          try:

              mmd = Modulemd.ModuleStream.read_file(

                  mmds["modulemd.%s.txt" % arch], strict=True

              )

              variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd

+             added = True

          except KeyError:

              # There is no modulemd for this arch. This could mean an arch was

              # added to the compose after the module was built. We don't want to

              # process this, let's skip this module.

              pass

  

+     if not added:

+         # The module is filtered on all arches of this variant.

+         return None

+ 

      if add_to_variant_modules:

          variant.modules.append({"name": nsvc, "glob": False})

  

      return nsvc

  

  

+ def _is_filtered_out(compose, variant, arch, module_name, module_stream):

+     """Check if module with given name and stream is filter out from this stream.

+     """

+     if not compose:

+         return False

+ 

+     for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):

+         if ":" not in filter:

+             name_filter = filter

+             stream_filter = "*"

+         else:

+             name_filter, stream_filter = filter.split(":", 1)

+ 

+         if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):

+             return True

+ 

+     return False

+ 

+ 

  def _get_modules_from_koji(

      compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd

  ):
@@ -264,26 +299,34 @@ 

      for module in variant.get_modules():

          koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])

          for koji_module in koji_modules:

-             nsvc = _add_module_to_variant(koji_wrapper, variant, koji_module)

+             nsvc = _add_module_to_variant(

+                 koji_wrapper, variant, koji_module, compose=compose

+             )

              if not nsvc:

                  continue

  

              tag = koji_module["tag"]

              variant_tags[variant].append(tag)

  

-             # Store mapping NSVC --> koji_tag into variant.

-             # This is needed in createrepo phase where metadata is exposed by producmd

-             variant.module_uid_to_koji_tag[nsvc] = tag

- 

              tag_to_mmd.setdefault(tag, {})

              for arch in variant.arch_mmds:

-                 tag_to_mmd[tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])

+                 try:

+                     mmd = variant.arch_mmds[arch][nsvc]

+                 except KeyError:

+                     # Module was filtered from here

+                     continue

+                 tag_to_mmd[tag].setdefault(arch, set()).add(mmd)

+ 

+             if tag_to_mmd[tag]:

+                 compose.log_info(

+                     "Module '%s' in variant '%s' will use Koji tag '%s' "

+                     "(as a result of querying module '%s')",

+                     nsvc, variant, tag, module["name"]

+                 )

  

-             module_msg = (

-                 "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' "

-                 "(as a result of querying module '{module}')"

-             ).format(uid=nsvc, variant=variant, tag=tag, module=module["name"])

-             compose.log_info("%s" % module_msg)

+                 # Store mapping NSVC --> koji_tag into variant. This is needed

+                 # in createrepo phase where metadata is exposed by producmd

+                 variant.module_uid_to_koji_tag[nsvc] = tag

  

  

  def filter_inherited(koji_proxy, event, module_builds, top_tag):
@@ -449,21 +492,31 @@ 

  

              variant_tags[variant].append(module_tag)

  

-             nsvc = _add_module_to_variant(koji_wrapper, variant, build, True)

+             nsvc = _add_module_to_variant(

+                 koji_wrapper, variant, build, True, compose=compose

+             )

              if not nsvc:

                  continue

  

-             # Store mapping module-uid --> koji_tag into variant.

-             # This is needed in createrepo phase where metadata is exposed by producmd

-             variant.module_uid_to_koji_tag[nsvc] = module_tag

- 

              tag_to_mmd.setdefault(module_tag, {})

              for arch in variant.arch_mmds:

-                 tag_to_mmd[module_tag].setdefault(arch, set()).add(variant.arch_mmds[arch][nsvc])

+                 try:

+                     mmd = variant.arch_mmds[arch][nsvc]

+                 except KeyError:

+                     # Module was filtered from here

+                     continue

+                 tag_to_mmd[module_tag].setdefault(arch, set()).add(mmd)

+ 

+             if tag_to_mmd[module_tag]:

+                 compose.log_info(

+                     "Module %s in variant %s will use Koji tag %s.",

+                     nsvc, variant, module_tag

+                 )

  

-             module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(

-                 variant=variant, tag=module_tag, module=build["nvr"])

-             compose.log_info("%s" % module_msg)

+                 # Store mapping module-uid --> koji_tag into variant. This is

+                 # needed in createrepo phase where metadata is exposed by

+                 # productmd

+                 variant.module_uid_to_koji_tag[nsvc] = module_tag

  

      if expected_modules:

          # There are some module names that were listed in configuration and not
@@ -624,7 +677,12 @@ 

                      # Not current tag, skip it

                      continue

                  for arch_modules in variant.arch_mmds.values():

-                     for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts():

+                     try:

+                         module = arch_modules[nsvc]

+                     except KeyError:

+                         # The module was filtered out

+                         continue

+                     for rpm_nevra in module.get_rpm_artifacts():

                          nevra = parse_nvra(rpm_nevra)

                          modular_packages.add((nevra["name"], nevra["arch"]))

  

@@ -615,9 +615,10 @@ 

  

  @mock.patch("pungi.Modulemd.ModuleStream.read_file", new=MockModule)

  @unittest.skipIf(Modulemd is None, "Skipping tests, no module support")

- class TestAddModuleToVariant(unittest.TestCase):

+ class TestAddModuleToVariant(helpers.PungiTestCase):

  

      def setUp(self):

+         super(TestAddModuleToVariant, self).setUp()

          self.koji = mock.Mock()

          self.koji.koji_module.pathinfo.typedir.return_value = "/koji"

          files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
@@ -741,3 +742,61 @@ 

                  {"name": "module:master:20190318:abcdef", "glob": False},

              ],

          )

+ 

+     def test_adding_module_but_filtered(self):

+         compose = helpers.DummyCompose(

+             self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}

+         )

+         variant = mock.Mock(

+             arches=["armhfp", "x86_64"], arch_mmds={}, modules=[], uid="Variant"

+         )

+ 

+         nsvc = source_koji._add_module_to_variant(

+             self.koji,

+             variant,

+             self.buildinfo,

+             add_to_variant_modules=True,

+             compose=compose,

+         )

+ 

+         self.assertIsNone(nsvc)

+         self.assertEqual(variant.arch_mmds, {})

+         self.assertEqual(variant.modules, [])

+ 

+ 

+ class TestIsModuleFiltered(helpers.PungiTestCase):

+     def assertIsFiltered(self, name, stream):

+         self.assertTrue(

+             source_koji._is_filtered_out(

+                 self.compose, self.compose.variants["Server"], "x86_64", name, stream

+             )

+         )

+ 

+     def assertIsNotFiltered(self, name, stream):

+         self.assertFalse(

+             source_koji._is_filtered_out(

+                 self.compose, self.compose.variants["Server"], "x86_64", name, stream

+             )

+         )

+ 

+     def test_no_filters(self):

+         self.compose = helpers.DummyCompose(self.topdir, {})

+ 

+         self.assertIsNotFiltered("foo", "master")

+ 

+     def test_filter_by_name(self):

+         self.compose = helpers.DummyCompose(

+             self.topdir, {"filter_modules": [(".*", {"*": ["foo"]})]}

+         )

+ 

+         self.assertIsFiltered("foo", "master")

+         self.assertIsNotFiltered("bar", "master")

+ 

+     def test_filter_by_stream(self):

+         self.compose = helpers.DummyCompose(

+             self.topdir, {"filter_modules": [(".*", {"*": ["foo:master"]})]}

+         )

+ 

+         self.assertIsFiltered("foo", "master")

+         self.assertIsNotFiltered("bar", "master")

+         self.assertIsNotFiltered("foo", "stable")

Add a configuration option to enable skipping some modules found in the configured tag.

Fixes: https://pagure.io/pungi/issue/1260

rebased onto af4fd77edb1c37b9569e3705f720a52a0a819118

4 years ago

rebased onto 21d45eb

4 years ago

Pull-Request has been merged by lsedlar

4 years ago