PR#1408: createrepo: Add extra modulemd files to the repo - pungi

pungi

#1408 createrepo: Add extra modulemd files to the repo

Merged 3 years ago by lsedlar. Opened 3 years ago by lsedlar.

lsedlar/pungi extra-modulemd into master

createrepo: Add extra modulemd files to the repo

Lubomír Sedlář • 3 years ago

3bb1e3d

doc/configuration.rst

file modified

+21

		`@@ -439,6 +439,13 @@`
		``createrepo_c`` executable. This could be useful for enabling zchunk
		`generation and pointing it to correct dictionaries.`

		`+ createrepo_extra_modulemd`
		+ (dict) -- a mapping of variant UID to :ref:`an scm dict <scm_support>`.
		`+ If specified, it should point to a directory with extra module metadata`
		`+ YAML files that will be added to the repository for this variant. The`
		`+ cloned files should be split into subdirectories for each architecture of`
		`+ the variant.`
		`+`
		`product_id = None`
		(:ref:`scm_dict <scm_support>`) -- If specified, it should point to a
		directory with certificates ``<variant_uid>-<arch>-*.pem``. Pungi will
		`@@ -467,6 +474,20 @@`
		`# Also Server.x86_64 should have them (but not on other arches).`
		`('^Server$', {'x86_64': True}),`
		`]`
		`+ createrepo_extra_modulemd = {`
		`+ "Server": {`
		`+ "scm": "git",`
		`+ "repo": "https://example.com/extra-server-modulemd.git",`
		`+ "dir": ".",`
		`+ # The directory should have this layout. Each architecture for the`
		`+ # variant should be included (even if the directory is empty.`
		`+ # .`
		`+ # ├── aarch64`
		`+ # │ ├── some-file.yaml`
		`+ # │ └ ...`
		`+ # └── x86_64`
		`+ }`
		`+ }`


		`Package Set Settings`

pungi/checks.py

file modified

		`@@ -659,6 +659,13 @@`
		`"default": [],`
		`"items": {"type": "string"},`
		`},`
		`+ "createrepo_extra_modulemd": {`
		`+ "type": "object",`
		`+ "patternProperties": {`
		`+ ".+": {"$ref": "#/definitions/scm_dict"},`
		`+ "additionalProperties": False,`
		`+ },`
		`+ },`
		`"repoclosure_strictness": _variant_arch_mapping(`
		`{`
		`"type": "string",`

pungi/phases/createrepo.py

file modified

+24

		`@@ -73,6 +73,14 @@`
		`for variant in self.compose.get_variants():`
		`if variant.is_empty:`
		`continue`
		`+`
		`+ if variant.uid in self.compose.conf.get("createrepo_extra_modulemd", {}):`
		`+ # Clone extra modulemd repository if it's configured.`
		`+ get_dir_from_scm(`
		`+ self.compose.conf["createrepo_extra_modulemd"][variant.uid],`
		`+ self.compose.paths.work.tmp_dir(variant=variant, create_dir=False),`
		`+ )`
		`+`
		`self.pool.queue_put((self.compose, None, variant, "srpm"))`
		`for arch in variant.arches:`
		`self.pool.queue_put((self.compose, arch, variant, "rpm"))`
		`@@ -237,6 +245,22 @@`
		`defaults_dir, module_names, mod_index, overrides_dir=overrides_dir`
		`)`

		`+ # Add extra modulemd files`
		`+ if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):`
		`+ compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)`
		`+ dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)`
		`+ for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):`
		`+ module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)`
		`+ if not mod_index.add_module_stream(module_stream):`
		`+ raise RuntimeError(`
		`+ "Failed parsing modulemd data from %s" % filepath`
		`+ )`
		`+ # Add the module to metadata with dummy tag. We can't leave the`
		`+ # value empty, but we don't know what the correct tag is.`
		`+ nsvc = module_stream.get_nsvc()`
		`+ variant.module_uid_to_koji_tag[nsvc] = "DUMMY"`
		`+ metadata.append((nsvc, []))`
		`+`
		`log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)`
		`add_modular_metadata(repo, repo_dir, mod_index, log_file)`

tests/fixtures/fake-modulemd.yaml

file added

+20

		`@@ -0,0 +1,20 @@`
		`+ ---`
		`+ document: modulemd`
		`+ version: 2`
		`+ data:`
		`+ name: mymodule`
		`+ stream: master`
		`+ version: 1`
		`+ context: cafe`
		`+ arch: x86_64`
		`+ summary: Dummy module`
		`+ description: Dummy module`
		`+ license:`
		`+ module:`
		`+ - Beerware`
		`+ content:`
		`+ - Beerware`
		`+ artifacts:`
		`+ rpms:`
		`+ - foobar-0:1.0-1.noarch`
		`+ ...`

tests/test_createrepophase.py

file modified

+66

		`@@ -128,6 +128,22 @@`
		`],`
		`)`

		`+ @mock.patch("pungi.phases.createrepo.get_dir_from_scm")`
		`+ @mock.patch("pungi.phases.createrepo.ThreadPool")`
		`+ def test_clones_extra_modulemd(self, ThreadPoolCls, get_dir_from_scm):`
		`+ scm = mock.Mock()`
		`+ compose = DummyCompose(`
		`+ self.topdir, {"createrepo_extra_modulemd": {"Server": scm}}`
		`+ )`
		`+`
		`+ phase = CreaterepoPhase(compose)`
		`+ phase.run()`
		`+`
		`+ self.assertEqual(`
		`+ get_dir_from_scm.call_args_list,`
		`+ [mock.call(scm, os.path.join(compose.topdir, "work/global/tmp-Server"))],`
		`+ )`
		`+`

		`def make_mocked_modifyrepo_cmd(tc, module_artifacts):`
		`def mocked_modifyrepo_cmd(repodir, mmd_path, **kwargs):`
		`@@ -1190,6 +1206,56 @@`
		`[mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")],`
		`)`

		`+ @unittest.skipUnless(Modulemd is not None, "Skipped test, no module support.")`
		`+ @mock.patch("pungi.phases.createrepo.find_file_in_repodata")`
		`+ @mock.patch("pungi.phases.createrepo.run")`
		`+ @mock.patch("pungi.phases.createrepo.CreaterepoWrapper")`
		`+ def test_variant_repo_extra_modulemd(`
		`+ self, CreaterepoWrapperCls, run, modulemd_filename`
		`+ ):`
		`+ compose = DummyCompose(`
		`+ self.topdir, {"createrepo_extra_modulemd": {"Server": mock.Mock()}}`
		`+ )`
		`+ compose.has_comps = False`
		`+`
		`+ variant = compose.variants["Server"]`
		`+ variant.arch_mmds["x86_64"] = {}`
		`+ variant.module_uid_to_koji_tag = {}`
		`+`
		`+ repo = CreaterepoWrapperCls.return_value`
		`+`
		`+ copy_fixture("server-rpms.json", compose.paths.compose.metadata("rpms.json"))`
		`+ copy_fixture(`
		`+ "fake-modulemd.yaml",`
		`+ os.path.join(compose.topdir, "work/global/tmp-Server/x86_64/*.yaml"),`
		`+ )`
		`+`
		`+ repodata_dir = os.path.join(`
		`+ compose.paths.compose.os_tree("x86_64", compose.variants["Server"]),`
		`+ "repodata",`
		`+ )`
		`+`
		`+ modules_metadata = ModulesMetadata(compose)`
		`+`
		`+ modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz" # noqa: E501`
		`+ create_variant_repo(`
		`+ compose,`
		`+ "x86_64",`
		`+ compose.variants["Server"],`
		`+ "rpm",`
		`+ self.pkgset,`
		`+ modules_metadata,`
		`+ )`
		`+`
		`+ self.assertEqual(`
		`+ repo.get_modifyrepo_cmd.mock_calls,`
		`+ [mock.call(repodata_dir, mock.ANY, compress_type="gz", mdtype="modules")],`
		`+ )`
		`+ self.assertEqual(`
		`+ list(modules_metadata.productmd_modules_metadata["Server"]["x86_64"]),`
		`+ ["mymodule:master:1:cafe"],`
		`+ )`
		`+`

		`class TestGetProductIds(PungiTestCase):`
		`def mock_get(self, filenames):`

lsedlar commented 3 years ago

This is a workaround for modularity design issues and DNF bugs. If there were gaps in contexts, DNF has trouble handling the upgrades. Thus we may need to add module metadata for older versions of previously released module streams and add the missing contexts.

rebased onto 86584c1e84aa365914403cef49a30de2f73e6e7a

3 years ago