#12 WIP: Port to use DNF, createrepo_c, and python-multilib
Merged 7 years ago by ausil. Opened 7 years ago by ngompa.
ngompa/mash dnfport  into  master

file modified
+7 -1
@@ -9,7 +9,13 @@ 

  URL:            https://pagure.io/mash

  Source0:        http://fedorahosted.org/releases/m/a/mash/%{name}-%{version}.tar.gz

  BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)

- Requires:       yum, createrepo, koji

+ Requires:       yum, koji

+ Requires:       python2-multilib

+ %if 0%{?rhel} >= 6 || 0%{?fedora}

+ Requires:       %{_bindir}/createrepo_c

+ %else

+ Requires:       createrepo

+ %endif

  Conflicts:	pungi < 1.0.0

  BuildRequires:  python-devel

  BuildArch:      noarch

file modified
+1 -1
@@ -26,7 +26,7 @@ 

  import time

  

  import arch as masharch

- import multilib

+ import multilib.multilib as multilib

  import metadata

  import yum

  import yum.config

file modified
+28 -18
@@ -9,11 +9,12 @@ 

  import rpmUtils

  import rpmUtils.miscutils

  

- usenew = True

- try:

+ uselegacy = True

+ if os.path.isfile("/usr/bin/createrepo_c"):

+     uselegacy = False

+ else:

      import createrepo

- except:

-     usenew = False

+     uselegacy = True

  

  

  def _make_ancient(path, excludes, previous, logger):
@@ -34,10 +35,10 @@ 

      return status

  

  

- class MetadataOld:

+ class MetadataCurrent:

  

      def __init__(self, logger):

-         self.args = ['createrepo', '--update', '-q']

+         self.args = ['createrepo_c', '--update', '-q']

          self.previous = None

          self.logger = logger

          self.excludes = []
@@ -67,27 +68,36 @@ 

          self.args.append(compress_type)

  

      def set_hash(self, hashtype):

-         # Sorry, can't do that here.

-         pass

+         self.args.append('--checksum')

+         self.args.append(hashtype)

  

      def set_skipstat(self, skip):

          if skip:

              self.args.append('--skip-stat')

  

      def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers):

-         # Sorry, can't do that here.

+         #self.args.append('--deltas')

+         #self.args.append('--max-delta-rpm-size')

+         #self.args.append(max_delta_rpm_size)

+         # FIXME: Following is not yet implemented

+         #self.args.append('--max-delta-rpm-age')

+         #self.args.append(max_delta_rpm_age)

+         # FIXME: Following is not yet implemented

+         #self.args.append('--delta-workers')

+         #self.args.append(delta_workers)

+         ## Sorry, can't do that here.

          pass

  

      def set_previous(self, previous):

          self.previous = previous

  

      def set_distro_tags(self, distro_tags):

-         # Sorry, can't do that here.

-         pass

+         self.args.append('--distro')

+         self.args.append(distro_tags)

  

      def set_content_tags(self, content_tags):

-         # Sorry, can't do that here.

-         pass

+         self.args.append('--content')

+         self.args.append(content_tags)

  

      def run(self, path):

          self.args.append(path)
@@ -100,14 +110,14 @@ 

                      "Couldn't copy repodata from %s" % (self.previous,))

          pid = os.fork()

          if not pid:

-             os.execv("/usr/bin/createrepo", self.args)

+             os.execv("/usr/bin/createrepo_c", self.args)

          else:

              (p, status) = os.waitpid(pid, 0)

          if self.ancient:

              _make_ancient(path, self.excludes, self.previous, self.logger)

  

  

- class MetadataNew:

+ class MetadataLegacy:

  

      def __init__(self, logger):

          self.conf = createrepo.MetaDataConfig()
@@ -238,10 +248,10 @@ 

  class Metadata:

  

      def __init__(self, logger):

-         if usenew:

-             self.obj = MetadataNew(logger)

+         if uselegacy:

+             self.obj = MetadataLegacy(logger)

          else:

-             self.obj = MetadataOld(logger)

+             self.obj = MetadataCurrent(logger)

  

      def set_ancient(self, ancient):

          self.obj.set_ancient(ancient)

file removed
-238
@@ -1,238 +0,0 @@ 

- # This program is free software; you can redistribute it and/or modify

- # it under the terms of the GNU General Public License as published by

- # the Free Software Foundation; version 2 of the License.

- #

- # This program is distributed in the hope that it will be useful,

- # but WITHOUT ANY WARRANTY; without even the implied warranty of

- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

- # GNU Library General Public License for more details.

- #

- # You should have received a copy of the GNU General Public License along

- # with this program; if not, write to the Free Software Foundation, Inc.,

- # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

- 

- import os

- from fnmatch import fnmatch

- 

- 

- class MultilibMethod(object):

-     PREFER_64 = frozenset(

-         ('gdb', 'frysk', 'systemtap', 'systemtap-runtime', 'ltrace', 'strace'))

- 

-     def __init__(self, config):

-         self.name = 'base'

- 

-     def select(self, po):

-         if po.arch.find('64') != -1:

-             if po.name in self.PREFER_64:

-                 return True

-             if po.name.startswith('kernel'):

-                 for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:

-                     if p_name == 'kernel' or p_name == 'kernel-devel':

-                         return True

-         return False

- 

- 

- class NoMultilibMethod(object):

- 

-     def __init__(self, config):

-         self.name = 'none'

- 

-     def select(self, po):

-         return False

- 

- 

- class AllMultilibMethod(MultilibMethod):

- 

-     def __init__(self, config):

-         self.name = 'all'

- 

-     def select(self, po):

-         return True

- 

- 

- class FileMultilibMethod(MultilibMethod):

- 

-     def __init__(self, config):

-         file = config.multilib_file

-         self.name = 'file'

-         self.list = []

-         if file:

-             f = open(file, 'r')

-             lines = f.readlines()

-             f.close()

-             for line in lines:

-                 line = line.strip()

-                 if not line.startswith('#'):

-                     self.list.append(line)

- 

-     def select(self, po):

-         for item in self.list:

-             if fnmatch(po.name, item):

-                 return True

-         return False

- 

- 

- class KernelMultilibMethod(object):

- 

-     def __init__(self, config):

-         self.name = 'base'

- 

-     def select(self, po):

-         if po.arch.find('64') != -1:

-             if po.name.startswith('kernel'):

-                 for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:

-                     if p_name == 'kernel' or p_name == 'kernel-devel':

-                         return True

-         return False

- 

- 

- class YabootMultilibMethod(object):

- 

-     def __init__(self, config):

-         self.name = 'base'

- 

-     def select(self, po):

-         if po.arch in ['ppc']:

-             if po.name.startswith('yaboot'):

-                 return True

-         return False

- 

- 

- class RuntimeMultilibMethod(MultilibMethod):

-     ROOTLIBDIRS = frozenset(('/lib', '/lib64'))

-     USRLIBDIRS = frozenset(('/usr/lib', '/usr/lib64'))

-     LIBDIRS = ROOTLIBDIRS.union(USRLIBDIRS)

-     OPROFILEDIRS = frozenset(('/usr/lib/oprofile', '/usr/lib64/oprofile'))

-     WINEDIRS = frozenset(('/usr/lib/wine', '/usr/lib64/wine'))

-     SANEDIRS = frozenset(('/usr/lib/sane', '/usr/lib64/sane'))

- 

-     by_dir = set()

- 

-     # alsa, dri, gtk-accessibility, scim-bridge-gtk, krb5, sasl, vdpau

-     by_dir.update(frozenset(os.path.join('/usr/lib', p) for p in ('alsa-lib',

-                                                                   'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',

-                                                                   'sasl2', 'vdpau')))

-     by_dir.update(frozenset(os.path.join('/usr/lib64', p) for p in ('alsa-lib',

-                                                                     'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',

-                                                                     'sasl2', 'vdpau')))

- 

-     # pam

-     by_dir.update(frozenset(os.path.join(p, 'security') for p in ROOTLIBDIRS))

- 

-     # lsb

-     by_dir.add('/etc/lsb-release.d')

- 

-     def __init__(self, config):

-         self.name = 'runtime'

-         self.config = config

- 

-     def select(self, po):

-         if po.name in self.config.multilib_runtime_blacklist:

-             return False

-         if po.name in self.config.multilib_runtime_whitelist:

-             return True

-         if MultilibMethod.select(self, po):

-             return True

-         if po.name.startswith('kernel'):

-             for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:

-                 if p_name == 'kernel':

-                     return False

-         for file in po.returnFileEntries():

-             (dirname, filename) = file.rsplit('/', 1)

- 

-             # libraries in standard dirs

-             if dirname in self.LIBDIRS and fnmatch(filename, '*.so.*'):

-                 return True

-             if dirname in self.by_dir:

-                 return True

-             # mysql, qt, etc.

-             if dirname == '/etc/ld.so.conf.d' and filename.endswith('.conf'):

-                 return True

-             # nss (Some nss modules end in .so instead of .so.X)

-             # db (db modules end in .so instead of .so.X)

-             if dirname in self.ROOTLIBDIRS and (filename.startswith('libnss_') or filename.startswith('libdb-')):

-                 return True

-             # Optimization:

-             # All tests beyond here are for things in USRLIBDIRS

-             if not dirname.startswith(tuple(self.USRLIBDIRS)):

-                 # The dirname does not start with a USRLIBDIR so we can move

-                 # on to the next file

-                 continue

- 

-             if dirname.startswith(('/usr/lib/gtk-2.0', '/usr/lib64/gtk-2.0')):

-                 # gtk2-engines

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/engines'):

-                     return True

-                 # accessibility

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/modules'):

-                     return True

-                 # scim-bridge-gtk

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/immodules'):

-                     return True

-                 # images

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/loaders'):

-                     return True

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/printbackends'):

-                     return True

-                 if fnmatch(dirname, '/usr/lib*/gtk-2.0/*/filesystems'):

-                     return True

-                 # Optimization:

-                 # No tests beyond here for things in /usr/lib*/gtk-2.0

-                 continue

- 

-             # gstreamer

-             if dirname.startswith(('/usr/lib/gstreamer-', '/usr/lib64/gstreamer-')):

-                 return True

-             # qt/kde fun

-             if fnmatch(dirname, '/usr/lib*/qt*/plugins/*'):

-                 return True

-             if fnmatch(dirname, '/usr/lib*/kde*/plugins/*'):

-                 return True

-             # qml

-             if fnmatch(dirname, '/usr/lib*/qt5/qml/*'):

-                 return True

-             # images

-             if fnmatch(dirname, '/usr/lib*/gdk-pixbuf-2.0/*/loaders'):

-                 return True

-             # xine-lib

-             if fnmatch(dirname, '/usr/lib*/xine/plugins/*'):

-                 return True

-             # oprofile

-             if dirname in self.OPROFILEDIRS and fnmatch(filename, '*.so.*'):

-                 return True

-             # wine

-             if dirname in self.WINEDIRS and filename.endswith('.so'):

-                 return True

-             # sane drivers

-             if dirname in self.SANEDIRS and filename.startswith('libsane-'):

-                 return True

-         return False

- 

- 

- class DevelMultilibMethod(RuntimeMultilibMethod):

- 

-     def __init__(self, config):

-         self.name = 'devel'

-         self.config = config

- 

-     def select(self, po):

-         if po.name in self.config.multilib_devel_blacklist:

-             return False

-         if po.name in self.config.multilib_devel_whitelist:

-             return True

-         if RuntimeMultilibMethod.select(self, po):

-             return True

-         if po.name.startswith('ghc-'):

-             return False

-         if po.name.startswith('kernel'):

-             for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:

-                 if p_name == 'kernel-devel':

-                     return False

-                 if p_name.endswith('-devel') or p_name.endswith('-static'):

-                     return True

-         if po.name.endswith('-devel'):

-             return True

-         if po.name.endswith('-static'):

-             return True

-         return False

This PR ports mash to use python-multilib, as well as using createrepo_c and DNF when available.

rebased

7 years ago

%{_bindir}/createrepo_c

At this time, there's one big problem. @tmlcoch's createrepo_c is missing the necessary flags to support mash's use of createrepo_c to generate the requisite DeltaRPMs. The expected flags are commented out right now with FIXME statements above them in set_delta() in mash/metadata.py.

2 new commits added

  • Use createrepo_c for "current", declare old createrepo as "legacy"
  • Use python-multilib instead of internal implementation
7 years ago

any port to use createrepo_c needs to introduce a new class, Ideally using the createrepo_c api and not execing createrepo_c command. we stull support using the MetadataOld class, it is needed to make repodata that is compatible with RHEL 4 and older, admittedly it is not often we use that code path anymore. it is a config option.

Pull-Request has been merged by ausil

7 years ago

I did not mean to merge that. I have backed out the merge