PR#1205: pkgset: Create arch repos in parallel - pungi

pungi

#1205 pkgset: Create arch repos in parallel

Merged 4 years ago by lsedlar. Opened 4 years ago by lsedlar.

lsedlar/pungi parallel-pkgset into master

pkgset: Create arch repos in parallel

Lubomír Sedlář • 4 years ago

82580ed

pungi/phases/pkgset/common.py

file modified

+12 -1

		`@@ -17,6 +17,7 @@`
		`import os`

		`from kobo.shortcuts import run, relative_path`
		`+ from kobo.threads import run_in_threads`

		`import pungi.phases.pkgset.pkgsets`
		`from pungi.arch import get_valid_arches`
		`@@ -87,7 +88,17 @@`
		`compose.log_info("[DONE ] %s" % msg)`


		`- def create_arch_repos(compose, arch, path_prefix):`
		`+ def create_arch_repos(compose, path_prefix):`
		`+ run_in_threads(`
		`+ _create_arch_repo,`
		`+ [(compose, arch, path_prefix) for arch in compose.get_arches()],`
		`+ threads=compose.conf['createrepo_num_threads'],`
		`+ )`
		`+`
		`+`
		`+ def _create_arch_repo(worker_thread, args, task_num):`
		`+ """Create a single pkgset repo for given arch."""`
		`+ compose, arch, path_prefix = args`
		`createrepo_c = compose.conf["createrepo_c"]`
		`createrepo_checksum = compose.conf["createrepo_checksum"]`
		`repo = CreaterepoWrapper(createrepo_c=createrepo_c)`

pungi/phases/pkgset/sources/source_koji.py

file modified

+2 -3

		`@@ -23,6 +23,7 @@`

		`from kobo.rpmlib import parse_nvra`
		`from kobo.shortcuts import force_list, relative_path`
		`+ from kobo.threads import run_in_threads`

		`import pungi.wrappers.kojiwrapper`
		`from pungi.wrappers.comps import CompsWrapper`
		`@@ -203,9 +204,7 @@`

		`t.join()`

		`- for arch in compose.get_arches():`
		`- # TODO: threads? runroot?`
		`- create_arch_repos(compose, arch, path_prefix)`
		`+ create_arch_repos(compose, path_prefix)`

		`return package_sets`

pungi/phases/pkgset/sources/source_repos.py

file modified

+1 -3

		`@@ -126,9 +126,7 @@`

		`t.join()`

		`- for arch in compose.get_arches():`
		`- # TODO: threads? runroot?`
		`- create_arch_repos(compose, arch, path_prefix)`
		`+ create_arch_repos(compose, path_prefix)`

		`package_sets["global"] = pkgset_global`
		`return package_sets, path_prefix`

tests/test_pkgset_source_koji.py

file modified

+1 -3

		`@@ -258,9 +258,7 @@`
		`[mock.call(self.compose, '/prefix')])`
		`self.assertEqual(rcgr.call_args_list,`
		`[mock.call(self.compose, gcgrc.return_value)])`
		`- self.assertItemsEqual(car.call_args_list,`
		`- [mock.call(self.compose, 'x86_64', '/prefix'),`
		`- mock.call(self.compose, 'amd64', '/prefix')])`
		`+ self.assertItemsEqual(car.call_args_list, [mock.call(self.compose, '/prefix')])`

		`self.assertEqual(pkgsets, expected)`

lsedlar commented 4 years ago

This patch reuses the existing createrepo_num_threads options to limit maximum number of parallel createrepo processes.

It seems to be working fine on dummy compose, more tests on bigger data are needed.

Fixes: https://pagure.io/pungi/issue/955

onosek commented 4 years ago

It looks very promising.

lsedlar commented 4 years ago

The test on Fedora stage composer showed speedup from 90 minutes to 77 minutes. Something is wrong with my internal measuring environment, so I might not get any usable results from there.

rebased onto 82580ed

4 years ago

Pull-Request has been merged by lsedlar

4 years ago