From 2b112d53f7ea443dd11be584269f559d7d911227 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Sep 23 2019 07:23:39 +0000
Subject: Allow loading overrides for module defaults


This patch adds a new config option. This is expected to be a name of
subdirectory in the repo with module defaults. If supplied, overrides
from that location are loaded every time defaults are loaded.

This raises the minimal required version of libmodulemd to 2.8.0

JIRA: COMPOSE-3828
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

---

diff --git a/pungi/checks.py b/pungi/checks.py
index 17ff8b5..7e9f65a 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -746,6 +746,7 @@ def make_schema():
                 "default": True
             },
             "module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},
+            "module_defaults_override_dir": {"type": "string"},
 
             "pkgset_repos": {
                 "type": "object",
diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
index 6455359..b2c67ad 100644
--- a/pungi/phases/createrepo.py
+++ b/pungi/phases/createrepo.py
@@ -211,7 +211,10 @@ def create_variant_repo(compose, arch, variant, pkg_type, pkgset, modules_metada
 
         module_names = set(mod_index.get_module_names())
         defaults_dir = compose.paths.work.module_defaults_dir()
-        collect_module_defaults(defaults_dir, module_names, mod_index)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
+        collect_module_defaults(
+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+        )
 
         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
         add_modular_metadata(repo, repo_dir, mod_index, log_file)
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index afba72f..736c10e 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -382,7 +382,10 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
 
         module_names = set(mod_index.get_module_names())
         defaults_dir = compose.paths.work.module_defaults_dir()
-        collect_module_defaults(defaults_dir, module_names, mod_index)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
+        collect_module_defaults(
+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+        )
 
         log_file = compose.paths.log.log_file(
             arch, "lookaside_repo_modules_%s" % (variant.uid)
diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
index 6ab86e5..d7e06cf 100644
--- a/pungi/phases/pkgset/common.py
+++ b/pungi/phases/pkgset/common.py
@@ -155,8 +155,9 @@ def _create_arch_repo(worker_thread, args, task_num):
     # Add modulemd to the repo for all modules in all variants on this architecture.
     if Modulemd and mmd:
         names = set(x.get_module_name() for x in mmd)
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
         mod_index = collect_module_defaults(
-            compose.paths.work.module_defaults_dir(), names
+            compose.paths.work.module_defaults_dir(), names, overrides_dir=overrides_dir
         )
         for x in mmd:
             mod_index.add_module_stream(x)
diff --git a/pungi/util.py b/pungi/util.py
index c2c9a94..bd3e022 100644
--- a/pungi/util.py
+++ b/pungi/util.py
@@ -934,9 +934,10 @@ def iter_module_defaults(path):
     # and work with it. However that does not allow for detecting conflicting
     # defaults. That should not happen in practice, but better safe than sorry.
     # Once libmodulemd can report the error, this code can be simplifed by a
-    # lot. It's implemented in
+    # lot. It was implemented in
     # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
-    # and released in 2.6.0
+    # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
+    # that breaks this use case again.
     for file in glob.glob(os.path.join(path, "*.yaml")):
         index = Modulemd.ModuleIndex()
         index.update_from_file(file, strict=False)
@@ -944,7 +945,9 @@ def iter_module_defaults(path):
             yield module_name, index.get_module(module_name).get_defaults()
 
 
-def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
+def collect_module_defaults(
+    defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
+):
     """Load module defaults into index.
 
     If `modules_to_load` is passed in, it should be a set of module names. Only
@@ -954,7 +957,15 @@ def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
     not, a new ModuleIndex will be created and returned
     """
     mod_index = mod_index or Modulemd.ModuleIndex()
-    for module_name, defaults in iter_module_defaults(defaults_dir):
+
+    temp_index = Modulemd.ModuleIndex.new()
+    temp_index.update_from_defaults_directory(
+        defaults_dir, overrides_path=overrides_dir, strict=False
+    )
+
+    for module_name in temp_index.get_module_names():
+        defaults = temp_index.get_module(module_name).get_defaults()
+
         if not modules_to_load or module_name in modules_to_load:
             mod_index.add_defaults(defaults)
 
diff --git a/tests/test_pkgset_common.py b/tests/test_pkgset_common.py
index 64366a4..d1e147f 100755
--- a/tests/test_pkgset_common.py
+++ b/tests/test_pkgset_common.py
@@ -105,6 +105,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
         cmd.assert_called_once_with(
             os.path.join(self.topdir, "work/global/module_defaults"),
             set(x.get_module_name.return_value for x in mmd["x86_64"]),
+            overrides_dir=None,
         )
         amm.assert_called_once_with(
             mock.ANY,