PR#1500: Include module metadata in repo directly - pungi

pungi

#1500 Include module metadata in repo directly

Opened 3 years ago by fdiprete. Modified 2 years ago

fdiprete/pungi module_metadata into master

Include module metadata in repo directly

fdiprete • 2 years ago

6e52003

pkgset: Check tag inheritance change before reuse

Haibo Lin • 3 years ago

e8f11b6

pungi.spec

file modified

+1 -1

		`@@ -32,7 +32,7 @@`
		`Requires: python-productmd >= 1.11`
		`Requires: python-kickstart`
		`Requires: libselinux-python`
		`- Requires: createrepo_c`
		`+ Requires: createrepo_c >= 0.16`
		`Requires: python3-createrepo_c`
		`Requires: python-lxml`
		`Requires: koji >= 1.10.1-13`

pungi/phases/createrepo.py

file modified

+59 -64

		`@@ -185,6 +185,43 @@`
		`# This seems to only affect createrepo_c though.`
		`repo_dir_arch = None`

		`+ # inject modulemd if needed`
		`+ metadata = []`
		`+ if pkg_type == "rpm" and arch in variant.arch_mmds and Modulemd is not None:`
		`+ mod_index = Modulemd.ModuleIndex()`
		`+`
		`+ for module_id, mmd in variant.arch_mmds.get(arch, {}).items():`
		`+ if modules_metadata:`
		`+ module_rpms = mmd.get_rpm_artifacts()`
		`+ metadata.append((module_id, module_rpms))`
		`+ mod_index.add_module_stream(mmd)`
		`+`
		`+ module_names = set(mod_index.get_module_names())`
		`+ defaults_dir = compose.paths.work.module_defaults_dir()`
		`+ overrides_dir = compose.conf.get("module_defaults_override_dir")`
		`+ collect_module_defaults(`
		`+ defaults_dir, module_names, mod_index, overrides_dir=overrides_dir`
		`+ )`
		`+`
		`+ # Add extra modulemd files`
		`+ if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):`
		`+ compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)`
		`+ dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)`
		`+ for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):`
		`+ module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)`
		`+ if not mod_index.add_module_stream(module_stream):`
		`+ raise RuntimeError(`
		`+ "Failed parsing modulemd data from %s" % filepath`
		`+ )`
		`+ # Add the module to metadata with dummy tag. We can't leave the`
		`+ # value empty, but we don't know what the correct tag is.`
		`+ nsvc = module_stream.get_nsvc()`
		`+ variant.module_uid_to_koji_tag[nsvc] = "DUMMY"`
		`+ metadata.append((nsvc, []))`
		`+`
		`+ log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)`
		`+ add_modular_metadata(repo_dir, mod_index)`
		`+`
		`comps_path = None`
		`if compose.has_comps and pkg_type == "rpm":`
		`comps_path = compose.paths.work.comps(arch=arch, variant=variant)`
		`@@ -204,11 +241,29 @@`
		`use_xz=compose.conf["createrepo_use_xz"],`
		`extra_args=compose.conf["createrepo_extra_args"],`
		`)`
		`+`
		`+ print cmd`
		`+`
		`log_file = compose.paths.log.log_file(`
		`arch, "createrepo-%s.%s" % (variant, pkg_type)`
		`)`
		`run(cmd, logfile=log_file, show_cmd=True)`

		`+ for module_id, module_rpms in metadata:`
		`+ modulemd_path = os.path.join(`
		`+ types[pkg_type][1](relative=True),`
		`+ find_file_in_repodata(repo_dir, "modules"),`
		`+ )`
		`+ modules_metadata.prepare_module_metadata(`
		`+ variant,`
		`+ arch,`
		`+ module_id,`
		`+ modulemd_path,`
		`+ types[pkg_type][0],`
		`+ list(module_rpms),`
		`+ )`
		`+`
		`+`
		`# call modifyrepo to inject productid`
		`product_id = compose.conf.get("product_id")`
		`if product_id and pkg_type == "rpm":`
		`@@ -227,78 +282,18 @@`
		`product_id_path, os.path.join(repo_dir, "repodata", "productid")`
		`)`

		`- # call modifyrepo to inject modulemd if needed`
		`- if pkg_type == "rpm" and arch in variant.arch_mmds and Modulemd is not None:`
		`- mod_index = Modulemd.ModuleIndex()`
		`- metadata = []`
		`-`
		`- for module_id, mmd in variant.arch_mmds.get(arch, {}).items():`
		`- if modules_metadata:`
		`- module_rpms = mmd.get_rpm_artifacts()`
		`- metadata.append((module_id, module_rpms))`
		`- mod_index.add_module_stream(mmd)`
		`-`
		`- module_names = set(mod_index.get_module_names())`
		`- defaults_dir = compose.paths.work.module_defaults_dir()`
		`- overrides_dir = compose.conf.get("module_defaults_override_dir")`
		`- collect_module_defaults(`
		`- defaults_dir, module_names, mod_index, overrides_dir=overrides_dir`
		`- )`
		`-`
		`- # Add extra modulemd files`
		`- if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):`
		`- compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)`
		`- dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)`
		`- for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):`
		`- module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)`
		`- if not mod_index.add_module_stream(module_stream):`
		`- raise RuntimeError(`
		`- "Failed parsing modulemd data from %s" % filepath`
		`- )`
		`- # Add the module to metadata with dummy tag. We can't leave the`
		`- # value empty, but we don't know what the correct tag is.`
		`- nsvc = module_stream.get_nsvc()`
		`- variant.module_uid_to_koji_tag[nsvc] = "DUMMY"`
		`- metadata.append((nsvc, []))`
		`-`
		`- log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)`
		`- add_modular_metadata(repo, repo_dir, mod_index, log_file)`
		`-`
		`- for module_id, module_rpms in metadata:`
		`- modulemd_path = os.path.join(`
		`- types[pkg_type][1](relative=True),`
		`- find_file_in_repodata(repo_dir, "modules"),`
		`- )`
		`- modules_metadata.prepare_module_metadata(`
		`- variant,`
		`- arch,`
		`- module_id,`
		`- modulemd_path,`
		`- types[pkg_type][0],`
		`- list(module_rpms),`
		`- )`
		`-`
		`compose.log_info("[DONE ] %s" % msg)`


		`- def add_modular_metadata(repo, repo_path, mod_index, log_file):`
		`+ def add_modular_metadata(repo_path, mod_index):`
		`"""Add modular metadata into a repository."""`
		`# Dumping empty index fails, we need to check for that.`
		`if not mod_index.get_module_names():`
		`return`

		`- with temp_dir() as tmp_dir:`
		`- modules_path = os.path.join(tmp_dir, "modules.yaml")`
		`- with open(modules_path, "w") as f:`
		`- f.write(mod_index.dump_to_string())`
		`-`
		`- cmd = repo.get_modifyrepo_cmd(`
		`- os.path.join(repo_path, "repodata"),`
		`- modules_path,`
		`- mdtype="modules",`
		`- compress_type="gz",`
		`- )`
		`- run(cmd, logfile=log_file, show_cmd=True)`
		`+ modules_path = os.path.join(repo_path, "modules.yaml")`
		`+ with open(modules_path, "w") as f:`
		`+ f.write(mod_index.dump_to_string())`


		`def find_file_in_repodata(repo_path, type_):`

pungi/phases/gather/__init__.py

file modified

+1 -1

		`@@ -659,7 +659,7 @@`
		`log_file = compose.paths.log.log_file(`
		`arch, "lookaside_repo_modules_%s" % (variant.uid)`
		`)`
		`- add_modular_metadata(cr, repo, mod_index, log_file)`
		`+ add_modular_metadata(repo, mod_index)`

		`compose.log_info("[DONE ] %s", msg)`

pungi/phases/pkgset/common.py

file modified

-2

		`@@ -162,10 +162,8 @@`
		`for x in mmd:`
		`mod_index.add_module_stream(x)`
		`add_modular_metadata(`
		`- repo,`
		`repo_dir,`
		`mod_index,`
		`- compose.paths.log.log_file(arch, "arch_repo_modulemd.%s" % pkgset.name),`
		`)`

		`compose.log_info("[DONE ] %s", msg)`

pungi/phases/pkgset/pkgsets.py

file modified

+10 -2

		`@@ -731,17 +731,22 @@`
		`% (old_koji_event, koji_event)`
		`)`
		`changed = self.koji_proxy.queryHistory(`
		`- tables=["tag_listing"], tag=tag, afterEvent=old_koji_event`
		`+ tables=["tag_listing", "tag_inheritance"],`
		`+ tag=tag,`
		`+ afterEvent=old_koji_event,`
		`)`
		`if changed["tag_listing"]:`
		`self.log_debug("Builds under tag %s changed. Can't reuse." % tag)`
		`return False`
		`+ if changed["tag_inheritance"]:`
		`+ self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)`
		`+ return False`

		`if inherit:`
		`inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)`
		`for t in inherit_tags:`
		`changed = self.koji_proxy.queryHistory(`
		`- tables=["tag_listing"],`
		`+ tables=["tag_listing", "tag_inheritance"],`
		`tag=t["name"],`
		`afterEvent=old_koji_event,`
		`beforeEvent=koji_event + 1,`
		`@@ -752,6 +757,9 @@`
		`% t["name"]`
		`)`
		`return False`
		`+ if changed["tag_inheritance"]:`
		`+ self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)`
		`+ return False`

		`repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)`
		`old_repo_dir = compose.paths.old_compose_path(repo_dir)`

tests/test_pkgset_pkgsets.py

file modified

+18 -6

		`@@ -632,7 +632,10 @@`
		`def test_reuse_build_under_tag_changed(self, mock_old_topdir):`
		`mock_old_topdir.return_value = self.old_compose_dir`
		`self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])`
		`- self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": [{}]}`
		`+ self.koji_wrapper.koji_proxy.queryHistory.return_value = {`
		`+ "tag_listing": [{}],`
		`+ "tag_inheritance": [],`
		`+ }`

		`self.pkgset.try_to_reuse(self.compose, self.tag)`

		`@@ -652,8 +655,8 @@`
		`mock_old_topdir.return_value = self.old_compose_dir`
		`self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])`
		`self.koji_wrapper.koji_proxy.queryHistory.side_effect = [`
		`- {"tag_listing": []},`
		`- {"tag_listing": [{}]},`
		`+ {"tag_listing": [], "tag_inheritance": []},`
		`+ {"tag_listing": [{}], "tag_inheritance": []},`
		`]`
		`self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [`
		`{"name": self.inherited_tag}`
		`@@ -680,7 +683,10 @@`
		`def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists):`
		`mock_old_topdir.return_value = self.old_compose_dir`
		`self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])`
		`- self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}`
		`+ self.koji_wrapper.koji_proxy.queryHistory.return_value = {`
		`+ "tag_listing": [],`
		`+ "tag_inheritance": [],`
		`+ }`
		`self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []`
		`self.pkgset.load_old_file_cache = mock.Mock(`
		`side_effect=Exception("unknown error")`
		`@@ -712,7 +718,10 @@`
		`def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists):`
		`mock_old_topdir.return_value = self.old_compose_dir`
		`self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])`
		`- self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}`
		`+ self.koji_wrapper.koji_proxy.queryHistory.return_value = {`
		`+ "tag_listing": [],`
		`+ "tag_inheritance": [],`
		`+ }`
		`self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []`
		`self.pkgset.load_old_file_cache = mock.Mock(`
		`return_value={"allow_invalid_sigkeys": True}`
		`@@ -751,7 +760,10 @@`
		`def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all):`
		`mock_old_topdir.return_value = self.old_compose_dir`
		`self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])`
		`- self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}`
		`+ self.koji_wrapper.koji_proxy.queryHistory.return_value = {`
		`+ "tag_listing": [],`
		`+ "tag_inheritance": [],`
		`+ }`
		`self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []`
		`self.pkgset.load_old_file_cache = mock.Mock(`
		`return_value={`

fdiprete commented 3 years ago

Createrepo_c now has support for libmodulemd. It can create a repo with modular metadata in one invocation. Modify createrepo phase in Pungi so that we don't need a separate modifyrepo_c call to include the metadata.