#6 Cache dnf filter results
Merged 6 years ago by ncoghlan. Opened 6 years ago by ncoghlan.
modularity/ ncoghlan/fedmod switch-to-depchase  into  master

file modified
+25 -19
@@ -3,6 +3,8 @@ 

  import requests

  from six.moves import configparser

  

+ from functools import lru_cache, partial

+ 

  import dnf

  

  FALLBACK_STREAM = 'master'
@@ -34,6 +36,14 @@ 

      """

      return pkg.source_name

  

+ @lru_cache()

+ def get_package_from_base(base, *pkg, **kwds):

+     if pkg:

+         kwds["name"] = pkg[0]

+     result = base.sack.query().filter(latest=True, **kwds)

+     if not result:

+         return None

+     return result[0]

  

  class ModuleDepsDiffer(object):

  
@@ -59,6 +69,7 @@ 

          pass

  

      @staticmethod

+     @lru_cache()

      def get_base_from_repo(reponame, repourl):

          logging.info('Loading repo: %s', reponame)

          base = dnf.Base()
@@ -80,9 +91,10 @@ 

          pkgs = []

          base = ModuleDepsDiffer.get_base_from_repo('f27', REPO_F27)

          for pkg in self.packages:

-             filter_result = base.sack.query().filter(name=pkg, arch=['noarch', ARCH], latest=True)

-             ModuleDepsDiffer.package_unique(filter_result)

-             pkgs.append(filter_result[0])

+             filter_result = get_package_from_base(base, pkg, arch=('noarch', ARCH))

+             if filter_result is None:

+                 raise ValueError('RPM not found in repo {!}'.format(pkg))

+             pkgs.append(filter_result)

          for pkg in pkgs:

              requires = getattr(pkg, 'requires')

              for q in requires:
@@ -93,9 +105,10 @@ 

          base = ModuleDepsDiffer.get_base_from_repo('f27-source', REPO_F27_SOURCE)

          for pkg in pkgs:

              source_rpm_name = get_pkgs_source_rpm_name(pkg)

-             filter_result = base.sack.query().filter(name=source_rpm_name)

-             ModuleDepsDiffer.package_unique(filter_result)

-             source_pkgs.append(filter_result[0])

+             filter_result = get_package_from_base(base, source_rpm_name)

+             if filter_result is None:

+                 raise ValueError('SRPM not found in repo {!}'.format(source_rpm_name))

+             source_pkgs.append(filter_result)

          for pkg in source_pkgs:

              requires = getattr(pkg, 'requires')

              for q in requires:
@@ -121,25 +134,18 @@ 

                  caps_classified['result'].add(cap)

  

      @staticmethod

-     def package_unique(result):

-         if len(result) > 1:

-             raise ValueError('Name of package is not unique: ' + str(result.result))

-         if len(result) == 0:

-             raise ValueError('No package found in repo')

- 

-     @staticmethod

      def whatprovides(caps):

          result = set()

          base = ModuleDepsDiffer.get_base_from_repo('f27', REPO_F27)

-         q = base.sack.query()

+         query = partial(get_package_from_base, base)

          for cap in caps:

              logging.info('Getting source package for %s:', cap)

-             pkg = q.filter(provides__glob=[cap], arch=['noarch', ARCH], latest=True)

-             if not pkg:

-                 pkg = q.filter(file__glob=cap)

-                 if len(pkg) == 0:

+             pkg = query(provides__glob=(cap,), arch=('noarch', ARCH))

+             if pkg is None:

+                 pkg = query(file__glob=cap)

+                 if pkg is None:

                      raise ValueError('No package provides capability \'' + cap + '\'')

-             source_rpm_name = get_pkgs_source_rpm_name(pkg.result[0])

+             source_rpm_name = get_pkgs_source_rpm_name(pkg)

              result.add(source_rpm_name)

          return result

  

This reduces the test runtime to around
2 1/2 minutes, and is as far as I'm going
to take the incremental improvements.

Instead, the next step will be to switch
to the depchase algorithms, which move
most of the work down into libsolv.

Pull-Request has been merged by ncoghlan

6 years ago
Metadata