PR#1035: [DO_NOT_MERGE] Include module translations in repodata - pungi

pungi

#1035 [DO_NOT_MERGE] Include module translations in repodata

Closed 5 years ago by sgallagh. Opened 5 years ago by sgallagh.

sgallagh/pungi module-i18n into master

Include module translations in repodata

Stephen Gallagher • 5 years ago

856286c

pungi/checks.py

file modified

+14

		`@@ -698,6 +698,20 @@`
		`"default": True`
		`},`
		`"module_defaults_dir": {"$ref": "#/definitions/str_or_scm_dict"},`
		`+ "module_translations": {`
		`+ "type": "object",`
		`+ "properties": {`
		`+ "branch": {"type": "string"},`
		`+ "zanata_rest_url": {"type": "string"},`
		`+ "zanata_project": {"type": "string"},`
		`+ "zanata_file": {"type": "string"},`
		`+ },`
		`+ "additionalProperties": False,`
		`+ "required": ["branch",`
		`+ "zanata_rest_url",`
		`+ "zanata_project",`
		`+ "zanata_file"],`
		`+ },`

		`"pkgset_repos": {`
		`"type": "object",`

pungi/compose.py

file modified

		`@@ -209,6 +209,10 @@`
		`return bool(self.conf.get("module_defaults_dir", False))`

		`@property`
		`+ def has_module_translations(self):`
		`+ return bool(self.conf.get("module_translations", False))`
		`+`
		`+ @property`
		`def config_dir(self):`
		`return os.path.dirname(self.conf._open_file or "")`

pungi/paths.py

file modified

		`@@ -406,6 +406,15 @@`
		`makedirs(path)`
		`return path`

		`+ def module_translations_dir(self, create_dir=True):`
		`+ """`
		`+ """`
		`+ path = os.path.join(self.topdir(create_dir=create_dir),`
		`+ 'module_translations')`
		`+ if create_dir:`
		`+ makedirs(path)`
		`+ return path`
		`+`
		`def pkgset_file_cache(self):`
		`"""`
		`Returns the path to file in which the cached version of`

pungi/phases/createrepo.py

file modified

		`@@ -37,6 +37,7 @@`
		`find_old_compose,`
		`get_arch_variant_data,`
		`iter_module_defaults,`
		`+ iter_module_translations,`
		`temp_dir,`
		`)`
		`from pungi import Modulemd`
		`@@ -254,6 +255,12 @@`
		`if mmddef.peek_module_name() in module_names:`
		`modules.append(mmddef)`

		`+ # Include translation data in the modulemd YAML`
		`+ for mmdtrans in iter_module_translations(`
		`+ compose.paths.work.module_translations_dir()):`
		`+ if mmdtrans.get_module_name() in module_names:`
		`+ modules.append(mmdtrans)`
		`+`
		`with temp_dir() as tmp_dir:`
		`modules_path = os.path.join(tmp_dir, "modules.yaml")`
		`Modulemd.dump(modules, modules_path)`

pungi/phases/init.py

file modified

+23

		`@@ -27,6 +27,8 @@`
		`from pungi.wrappers.createrepo import CreaterepoWrapper`
		`from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm`

		`+ from pungi.util import Modulemd`
		`+ from mmdzanata import get_modulemd_translations`

		`class InitPhase(PhaseBase):`
		`"""INIT is a mandatory phase"""`
		`@@ -61,6 +63,10 @@`
		`self.compose.paths.work.module_defaults_dir(create_dir=False)`
		`)`

		`+ # Prepare module translations`
		`+ if self.compose.has_module_translations:`
		`+ write_module_translations(self.compose)`
		`+`
		`# write prepopulate file`
		`write_prepopulate_file(self.compose)`

		`@@ -179,6 +185,23 @@`
		`shutil.copytree(tmp_dir, compose.paths.work.module_defaults_dir(create_dir=False))`


		`+ def write_module_translations(compose):`
		`+ translation_dict = compose.conf["module_translations_branch"]`
		`+ translation_mmds = get_modulemd_translations(`
		`+ zanata_rest_url=translation_dict['zanata_rest_url'],`
		`+ zanata_project=translation_dict['zanata_project'],`
		`+ os_branch=translation_dict['branch'],`
		`+ zanata_translation_file=translation_dict['zanata_file']`
		`+ )`
		`+`
		`+ with temp_dir(prefix="moduletranslations_") as tmp_dir:`
		`+ Modulemd.dump(translation_mmds, "%s/module_translations.yaml")`
		`+ compose.log_debug("Writing module translations")`
		`+ shutil.copytree(temp_dir,`
		`+ compose.paths.work.module_translations_dir(`
		`+ create_dir=False))`
		`+`
		`+`
		`def validate_module_defaults(path):`
		`"""Make sure there are no conflicting defaults. Each module name can only`
		`have one default stream.`

pungi/util.py

file modified

		`@@ -874,3 +874,11 @@`
		`for mmddef in Modulemd.objects_from_file(file):`
		`if isinstance(mmddef, Modulemd.Defaults):`
		`yield mmddef`
		`+`
		`+ def iter_module_translations(path):`
		`+ """Given a path to a directory with yaml files, yield each module default in there.`
		`+ """`
		`+ for file in glob.glob(os.path.join(path, "*.yaml")):`
		`+ for mmdtrans in Modulemd.objects_from_file(file):`
		`+ if isinstance(mmdtrans, Modulemd.Translation):`
		`+ yield mmdtrans`

setup.py

file modified

		`@@ -60,6 +60,7 @@`
		`"kobo",`
		`"lockfile",`
		`"lxml",`
		`+ "mmdzanata",`
		`"productmd",`
		`"six",`
		`'dogpile.cache',`

tests/test_initphase.py

file modified

+34

		`@@ -42,6 +42,7 @@`
		`compose = DummyCompose(self.topdir, {})`
		`compose.has_comps = True`
		`compose.has_module_defaults = False`
		`+ compose.has_module_translations = False`
		`compose.setup_optional()`
		`phase = init.InitPhase(compose)`
		`phase.run()`
		`@@ -82,6 +83,7 @@`
		`compose = DummyCompose(self.topdir, {})`
		`compose.has_comps = True`
		`compose.has_module_defaults = False`
		`+ compose.has_module_translations = False`
		`compose.variants['Everything'].groups = []`
		`compose.variants['Everything'].modules = []`
		`phase = init.InitPhase(compose)`
		`@@ -124,6 +126,7 @@`
		`compose = DummyCompose(self.topdir, {})`
		`compose.has_comps = False`
		`compose.has_module_defaults = False`
		`+ compose.has_module_translations = False`
		`phase = init.InitPhase(compose)`
		`phase.run()`

		`@@ -150,6 +153,37 @@`
		`compose = DummyCompose(self.topdir, {})`
		`compose.has_comps = False`
		`compose.has_module_defaults = True`
		`+ compose.has_module_translations = False`
		`+ phase = init.InitPhase(compose)`
		`+ phase.run()`
		`+`
		`+ self.assertItemsEqual(write_global.mock_calls, [])`
		`+ self.assertEqual(validate_comps.call_args_list, [])`
		`+ self.assertItemsEqual(write_prepopulate.mock_calls, [mock.call(compose)])`
		`+ self.assertItemsEqual(write_arch.mock_calls, [])`
		`+ self.assertItemsEqual(create_comps.mock_calls, [])`
		`+ self.assertItemsEqual(write_variant.mock_calls, [])`
		`+ self.assertItemsEqual(write_defaults.call_args_list, [mock.call(compose)])`
		`+ self.assertItemsEqual(`
		`+ validate_defaults.call_args_list,`
		`+ [mock.call(compose.paths.work.module_defaults_dir())],`
		`+ )`
		`+`
		`+ def test_with_module_translations(`
		`+ self,`
		`+ write_prepopulate,`
		`+ write_variant,`
		`+ create_comps,`
		`+ write_arch,`
		`+ write_global,`
		`+ write_defaults,`
		`+ validate_defaults,`
		`+ validate_comps,`
		`+ ):`
		`+ compose = DummyCompose(self.topdir, {})`
		`+ compose.has_comps = False`
		`+ compose.has_module_defaults = False`
		`+ compose.has_module_translations = True`
		`phase = init.InitPhase(compose)`
		`phase.run()`

sgallagh commented 5 years ago

This patch will generate modulemd-translations YAML documents from the appropriate Zanata translation project and add them to the modulemd repodata.

Signed-off-by: Stephen Gallagher sgallagh@redhat.com

sgallagh commented 5 years ago

These patches are a work-in-progress for the moment. I'm providing a PR for ongoing feedback as well as to get the tests to run.

lsedlar commented 5 years ago

I haven't tested it yet, but I don't see anything suspicious about the code.

A couple comments:

can you explain how the translations work in general? or point me to some documentation?
the mmdzanata package is Python3 only; and we need to work even with Python 2.6 in general (though it's fine not to provide this feature there). However if you want this used internally, Python 2.7 is required (and it will fail tests once the dependencies are installed).

rebased onto 92968fe

5 years ago

sgallagh commented 5 years ago

I haven't tested it yet, but I don't see anything suspicious about the code.
A couple comments:

can you explain how the translations work in general? or point me to some documentation?

The mmdzanata package provides two tools, one for interrogating Koji for the modulemd of all modules that are tagged for a particular Fedora release and then uploading the translatable strings to the Zanata translation service.

The second tool (the one relevant here), retrieves the translated strings from the Zanata translation service and converts it into modulemd-translations documents which will be used by libmodulemd (via DNF) to return translated summaries, descriptions and profile descriptions to users.

the mmdzanata package is Python3 only; and we need to work even with Python 2.6 in general (though it's fine not to provide this feature there). However if you want this used internally, Python 2.7 is required (and it will fail tests once the dependencies are installed).

OK, I don't think I used any real python3-isms (nothing unsupported by future at least), so I can probably make it available for Python 2 as well. I'll look at that today.

rebased onto 1088550434de49f4df80443295333549e07513b5

5 years ago

lsedlar commented 5 years ago

The mmdzanata package is now installed in the jenkins job. It turns out there is another dependency issue, importing gi on Python 2 does not work (but it does work on Python 3). The other tests that need it are just skipped if the dependency is missing.

sgallagh commented 5 years ago

The mmdzanata package is now installed in the jenkins job. It turns out there is another dependency issue, importing gi on Python 2 does not work (but it does work on Python 3). The other tests that need it are just skipped if the dependency is missing.

OK, thanks. I saw https://pagure.io/fedora-infrastructure/issue/7210 requesting that it be added.

rebased onto 856286c

5 years ago

sgallagh commented 5 years ago

@lsedlar Importing gi is still not working.

lsedlar commented 5 years ago

The package seems to be missing, there is only python3 version. I updated the infra ticket.

sgallagh commented 5 years ago

I'm canceling this pull request. I am going to redesign it to not be reliant on Zanata as a translation source.

Pull-Request has been closed by sgallagh

5 years ago