PR#727: Allow extracting profiling information from pungi-gather. - pungi

pungi

#727 Allow extracting profiling information from pungi-gather.

Merged 6 years ago by ralph. Opened 6 years ago by ralph.

ralph/pungi gather-profiler into master

Allow extracting profiling information from pungi-gather.

Ralph Bean • 6 years ago

98e41b8

doc/configuration.rst

file modified

		`@@ -642,6 +642,11 @@`
		path to JSON file with following mapping: ``{variant: {arch: {rpm_name:
		[rpm_arch\|None]}}}``.

		`+ gather_profiler = False`
		+ (bool) -- When set to ``True`` the gather tool will produce additional
		`+ performance profiling information at the end of its logs. Only takes`
		+ effect when ``gather_backend = "dnf"``.
		`+`

		`Example`
		`-------`

doc/gathering.rst

file modified

		`@@ -67,3 +67,10 @@`
		`+-----------+--------------+--------------+`
		`\| optional \| enabled \| enabled \|`
		`+-----------+--------------+--------------+`
		`+`
		`+`
		`+ Profiling`
		`+ =========`
		`+`
		+ Profiling data on the ``pungi-gather`` tool can be enabled by setting the
		+ ``gather_profiler`` configuration option to ``True``.

pungi/phases/gather/methods/method_deps.py

file modified

+5 -1

		`@@ -117,6 +117,9 @@`
		`fulltree = compose.conf["gather_fulltree"]`
		`selfhosting = compose.conf["gather_selfhosting"]`

		`+ # profiling`
		`+ profiler = compose.conf["gather_profiler"]`
		`+`
		`# optional`
		`if variant.type == "optional":`
		`fulltree = True`
		`@@ -144,7 +147,8 @@`
		`cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid,`
		`selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch,`
		`full_archlist=True, greedy=greedy_method, cache_dir=cache_dir,`
		`- lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)`
		`+ lookaside_repos=lookaside_repos, multilib_methods=multilib_methods,`
		`+ profiler=profiler)`
		`# Use temp working directory directory as workaround for`
		`# https://bugzilla.redhat.com/show_bug.cgi?id=795137`
		`tmp_dir = compose.mkdtemp(prefix="pungi_")`

pungi/phases/pkgset/sources/source_repos.py

file modified

+4 -1

		`@@ -45,6 +45,8 @@`
		`# TODO: noarch hack - secondary arches, use x86_64 noarch where possible`
		`flist = []`

		`+ profiler = compose.conf["gather_profiler"]`
		`+`
		`link_type = compose.conf["link_type"]`
		`pool = LinkerPool(link_type, logger=compose._logger)`
		`for i in range(10):`
		`@@ -79,7 +81,8 @@`
		`cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO",`
		`selfhosting=True, fulltree=True, multilib_methods=["all"],`
		`nodownload=False, full_archlist=True, arch=arch,`
		`- cache_dir=compose.paths.work.pungi_cache_dir(arch=arch))`
		`+ cache_dir=compose.paths.work.pungi_cache_dir(arch=arch),`
		`+ profiler=profiler)`
		`if compose.conf['gather_backend'] == 'yum':`
		`cmd.append("--force")`

pungi/wrappers/pungi.py

file modified

+5 -2

		`@@ -109,7 +109,7 @@`

		`kickstart.close()`

		`- def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):`
		`+ def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):`
		`cmd = ["pungi"]`

		`# Gather stage`
		`@@ -169,7 +169,7 @@`

		`return cmd`

		`- def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):`
		`+ def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):`
		`cmd = ["pungi-gather"]`

		`# path to a kickstart file`
		`@@ -203,6 +203,9 @@`
		`for i in lookaside_repos:`
		`cmd.append("--lookaside=%s" % i)`

		`+ if profiler:`
		`+ cmd.append("--profiler")`
		`+`
		`return cmd`

		`def parse_log(self, f):`

ralph commented 6 years ago

pungi-gather (the tool that underlies both the pkgset and gather
phases) contains profiling code that will log statistics about how long
different function calls take. However, pungi-koji did not contain a
way to pass the --profiler argument to enable this.

This change adds a new configuration option gather_profiler which,
when set to true, simply passes the argument to pungi-koji. Hopefully
this can help shed some light on what is happening in some of our
longer-running composes.

Signed-off-by: Ralph Bean rbean@redhat.com

ralph commented 6 years ago

/cc @psabata @ausil @kevin