PR#727 Merged Allow extracting profiling information from pungi-gather.

Proposed 3 months ago by ralph
Modified 3 months ago
From forks/ralph/pungi gather-profiler  into pungi master

pungi-gather (the tool that underlies both the pkgset and gather
phases) contains profiling code that will log statistics about how long
different function calls take. However, pungi-koji did not contain a
way to pass the --profiler argument to enable this.

This change adds a new configuration option gather_profiler which,
when set to true, simply passes the argument to pungi-koji. Hopefully
this can help shed some light on what is happening in some of our
longer-running composes.

Signed-off-by: Ralph Bean rbean@redhat.com

file changed

@@ -642,6 +642,11 @@ 

      path to JSON file with following mapping: ``{variant: {arch: {rpm_name:

      [rpm_arch|None]}}}``.

  

+ **gather_profiler** = False

+     (*bool*) -- When set to ``True`` the gather tool will produce additional

+     performance profiling information at the end of its logs.  Only takes

+     effect when ``gather_backend = "dnf"``.

+ 

  

  Example

  -------
file changed

@@ -67,3 +67,10 @@ 

     +-----------+--------------+--------------+

     | optional  | enabled      | enabled      |

     +-----------+--------------+--------------+

+ 

+ 

+ Profiling

+ =========

+ 

+ Profiling data on the ``pungi-gather`` tool can be enabled by setting the

+ ``gather_profiler`` configuration option to ``True``.

@@ -117,6 +117,9 @@ 

      fulltree = compose.conf["gather_fulltree"]

      selfhosting = compose.conf["gather_selfhosting"]

  

+     # profiling

+     profiler = compose.conf["gather_profiler"]

+ 

      # optional

      if variant.type == "optional":

          fulltree = True

@@ -144,7 +147,8 @@

      cmd = get_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid,

                    selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch,

                    full_archlist=True, greedy=greedy_method, cache_dir=cache_dir,

-                   lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)

+                   lookaside_repos=lookaside_repos, multilib_methods=multilib_methods,

+                   profiler=profiler)

      # Use temp working directory directory as workaround for

      # https://bugzilla.redhat.com/show_bug.cgi?id=795137

      tmp_dir = compose.mkdtemp(prefix="pungi_")

@@ -45,6 +45,8 @@ 

      # TODO: noarch hack - secondary arches, use x86_64 noarch where possible

      flist = []

  

+     profiler = compose.conf["gather_profiler"]

+ 

      link_type = compose.conf["link_type"]

      pool = LinkerPool(link_type, logger=compose._logger)

      for i in range(10):

@@ -79,7 +81,8 @@

          cmd = get_cmd(pungi_conf, destdir=pungi_dir, name="FOO",

                        selfhosting=True, fulltree=True, multilib_methods=["all"],

                        nodownload=False, full_archlist=True, arch=arch,

-                       cache_dir=compose.paths.work.pungi_cache_dir(arch=arch))

+                       cache_dir=compose.paths.work.pungi_cache_dir(arch=arch),

+                       profiler=profiler)

          if compose.conf['gather_backend'] == 'yum':

              cmd.append("--force")

  
file changed

@@ -109,7 +109,7 @@ 

  

          kickstart.close()

  

-     def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):

+     def get_pungi_cmd(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):

          cmd = ["pungi"]

  

          # Gather stage

@@ -169,7 +169,7 @@

  

          return cmd

  

-     def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None):

+     def get_pungi_cmd_dnf(self, config, destdir, name, version=None, flavor=None, selfhosting=False, fulltree=False, greedy=None, nodeps=False, nodownload=True, full_archlist=False, arch=None, cache_dir=None, lookaside_repos=None, multilib_methods=None, profiler=False):

          cmd = ["pungi-gather"]

  

          # path to a kickstart file

@@ -203,6 +203,9 @@

              for i in lookaside_repos:

                  cmd.append("--lookaside=%s" % i)

  

+         if profiler:

+             cmd.append("--profiler")

+ 

          return cmd

  

      def parse_log(self, f):

In Fedora space this will only give insight into the gather phase: pkgset is using Koji as a source, and pungi-gather is not involved in that.

The option also needs to be listed in schema in pungi/checks.py. I'll amend that, rebase on master and merge outside of pagure.

Commit 5b6e468 fixes this pull-request

3 months ago

Pull-Request has been merged by rbean @redhat.com

  • Jenkins
    success (100%)
    Build successful
    3 months ago
Changes summary
+5 -0
file changed
+7 -0
file changed
+5 -2
file changed