PR#13: WIP: Port to use DNF, createrepo_c, and python-multilib - mash

mash

#13 WIP: Port to use DNF, createrepo_c, and python-multilib

Opened 7 years ago by ngompa. Modified 7 years ago

ngompa/mash dnfport into master

Use createrepo_c for "current", declare old createrepo as "legacy"

Neal Gompa • 7 years ago

b8d105c

Use python-multilib instead of internal implementation

Neal Gompa • 7 years ago

aedaac9

mash.spec

file modified

+7 -1

		`@@ -9,7 +9,13 @@`
		`URL: https://pagure.io/mash`
		`Source0: http://fedorahosted.org/releases/m/a/mash/%{name}-%{version}.tar.gz`
		`BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)`
		`- Requires: yum, createrepo, koji`
		`+ Requires: yum, koji`
		`+ Requires: python2-multilib`
		`+ %if 0%{?rhel} >= 6 \|\| 0%{?fedora}`
		`+ Requires: %{_bindir}/createrepo_c`
		`+ %else`
		`+ Requires: createrepo`
		`+ %endif`
		`Conflicts: pungi < 1.0.0`
		`BuildRequires: python-devel`
		`BuildArch: noarch`

mash/__init__.py

file modified

+1 -1

		`@@ -26,7 +26,7 @@`
		`import time`

		`import arch as masharch`
		`- import multilib`
		`+ import multilib.multilib as multilib`
		`import metadata`
		`import yum`
		`import yum.config`

mash/metadata.py

file modified

+28 -18

		`@@ -9,11 +9,12 @@`
		`import rpmUtils`
		`import rpmUtils.miscutils`

		`- usenew = True`
		`- try:`
		`+ uselegacy = True`
		`+ if os.path.isfile("/usr/bin/createrepo_c"):`
		`+ uselegacy = False`
		`+ else:`
		`import createrepo`
		`- except:`
		`- usenew = False`
		`+ uselegacy = True`


		`def _make_ancient(path, excludes, previous, logger):`
		`@@ -34,10 +35,10 @@`
		`return status`


		`- class MetadataOld:`
ausil commented 7 years ago lets keep the MetadataOld class it should be renamed to something like MetadataPythonCLI
		`+ class MetadataCurrent:`

		`def __init__(self, logger):`
		`- self.args = ['createrepo', '--update', '-q']`
		`+ self.args = ['createrepo_c', '--update', '-q']`
		`self.previous = None`
		`self.logger = logger`
		`self.excludes = []`
		`@@ -67,27 +68,36 @@`
		`self.args.append(compress_type)`

		`def set_hash(self, hashtype):`
		`- # Sorry, can't do that here.`
		`- pass`
		`+ self.args.append('--checksum')`
		`+ self.args.append(hashtype)`

		`def set_skipstat(self, skip):`
		`if skip:`
		`self.args.append('--skip-stat')`

		`def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers):`
		`- # Sorry, can't do that here.`
		`+ #self.args.append('--deltas')`
		`+ #self.args.append('--max-delta-rpm-size')`
		`+ #self.args.append(max_delta_rpm_size)`
		`+ # FIXME: Following is not yet implemented`
		`+ #self.args.append('--max-delta-rpm-age')`
		`+ #self.args.append(max_delta_rpm_age)`
		`+ # FIXME: Following is not yet implemented`
		`+ #self.args.append('--delta-workers')`
		`+ #self.args.append(delta_workers)`
		`+ ## Sorry, can't do that here.`
ausil commented 7 years ago without delta support we can not move.
		`pass`

		`def set_previous(self, previous):`
		`self.previous = previous`

		`def set_distro_tags(self, distro_tags):`
		`- # Sorry, can't do that here.`
		`- pass`
		`+ self.args.append('--distro')`
		`+ self.args.append(distro_tags)`

		`def set_content_tags(self, content_tags):`
		`- # Sorry, can't do that here.`
		`- pass`
		`+ self.args.append('--content')`
		`+ self.args.append(content_tags)`

		`def run(self, path):`
		`self.args.append(path)`
		`@@ -100,14 +110,14 @@`
		`"Couldn't copy repodata from %s" % (self.previous,))`
		`pid = os.fork()`
		`if not pid:`
		`- os.execv("/usr/bin/createrepo", self.args)`
		`+ os.execv("/usr/bin/createrepo_c", self.args)`
		`else:`
		`(p, status) = os.waitpid(pid, 0)`
		`if self.ancient:`
		`_make_ancient(path, self.excludes, self.previous, self.logger)`


		`- class MetadataNew:`
		`+ class MetadataLegacy:`
ausil commented 7 years ago I would rather call this MatadatPythonAPI

		`def __init__(self, logger):`
		`self.conf = createrepo.MetaDataConfig()`
		`@@ -238,10 +248,10 @@`
		`class Metadata:`

		`def __init__(self, logger):`
		`- if usenew:`
		`- self.obj = MetadataNew(logger)`
		`+ if uselegacy:`
		`+ self.obj = MetadataLegacy(logger)`
		`else:`
		`- self.obj = MetadataOld(logger)`
		`+ self.obj = MetadataCurrent(logger)`

		`def set_ancient(self, ancient):`
		`self.obj.set_ancient(ancient)`

mash/multilib.py

file removed

-238

		`@@ -1,238 +0,0 @@`
		`- # This program is free software; you can redistribute it and/or modify`
		`- # it under the terms of the GNU General Public License as published by`
		`- # the Free Software Foundation; version 2 of the License.`
		`- #`
		`- # This program is distributed in the hope that it will be useful,`
		`- # but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
		`- # GNU Library General Public License for more details.`
		`- #`
		`- # You should have received a copy of the GNU General Public License along`
		`- # with this program; if not, write to the Free Software Foundation, Inc.,`
		`- # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.`
		`-`
		`- import os`
		`- from fnmatch import fnmatch`
		`-`
		`-`
		`- class MultilibMethod(object):`
		`- PREFER_64 = frozenset(`
		`- ('gdb', 'frysk', 'systemtap', 'systemtap-runtime', 'ltrace', 'strace'))`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'base'`
		`-`
		`- def select(self, po):`
		`- if po.arch.find('64') != -1:`
		`- if po.name in self.PREFER_64:`
		`- return True`
		`- if po.name.startswith('kernel'):`
		`- for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:`
		`- if p_name == 'kernel' or p_name == 'kernel-devel':`
		`- return True`
		`- return False`
		`-`
		`-`
		`- class NoMultilibMethod(object):`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'none'`
		`-`
		`- def select(self, po):`
		`- return False`
		`-`
		`-`
		`- class AllMultilibMethod(MultilibMethod):`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'all'`
		`-`
		`- def select(self, po):`
		`- return True`
		`-`
		`-`
		`- class FileMultilibMethod(MultilibMethod):`
		`-`
		`- def __init__(self, config):`
		`- file = config.multilib_file`
		`- self.name = 'file'`
		`- self.list = []`
		`- if file:`
		`- f = open(file, 'r')`
		`- lines = f.readlines()`
		`- f.close()`
		`- for line in lines:`
		`- line = line.strip()`
		`- if not line.startswith('#'):`
		`- self.list.append(line)`
		`-`
		`- def select(self, po):`
		`- for item in self.list:`
		`- if fnmatch(po.name, item):`
		`- return True`
		`- return False`
		`-`
		`-`
		`- class KernelMultilibMethod(object):`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'base'`
		`-`
		`- def select(self, po):`
		`- if po.arch.find('64') != -1:`
		`- if po.name.startswith('kernel'):`
		`- for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:`
		`- if p_name == 'kernel' or p_name == 'kernel-devel':`
		`- return True`
		`- return False`
		`-`
		`-`
		`- class YabootMultilibMethod(object):`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'base'`
		`-`
		`- def select(self, po):`
		`- if po.arch in ['ppc']:`
		`- if po.name.startswith('yaboot'):`
		`- return True`
		`- return False`
		`-`
		`-`
		`- class RuntimeMultilibMethod(MultilibMethod):`
		`- ROOTLIBDIRS = frozenset(('/lib', '/lib64'))`
		`- USRLIBDIRS = frozenset(('/usr/lib', '/usr/lib64'))`
		`- LIBDIRS = ROOTLIBDIRS.union(USRLIBDIRS)`
		`- OPROFILEDIRS = frozenset(('/usr/lib/oprofile', '/usr/lib64/oprofile'))`
		`- WINEDIRS = frozenset(('/usr/lib/wine', '/usr/lib64/wine'))`
		`- SANEDIRS = frozenset(('/usr/lib/sane', '/usr/lib64/sane'))`
		`-`
		`- by_dir = set()`
		`-`
		`- # alsa, dri, gtk-accessibility, scim-bridge-gtk, krb5, sasl, vdpau`
		`- by_dir.update(frozenset(os.path.join('/usr/lib', p) for p in ('alsa-lib',`
		`- 'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',`
		`- 'sasl2', 'vdpau')))`
		`- by_dir.update(frozenset(os.path.join('/usr/lib64', p) for p in ('alsa-lib',`
		`- 'dri', 'gtk-2.0/modules', 'gtk-2.0/immodules', 'krb5/plugins',`
		`- 'sasl2', 'vdpau')))`
		`-`
		`- # pam`
		`- by_dir.update(frozenset(os.path.join(p, 'security') for p in ROOTLIBDIRS))`
		`-`
		`- # lsb`
		`- by_dir.add('/etc/lsb-release.d')`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'runtime'`
		`- self.config = config`
		`-`
		`- def select(self, po):`
		`- if po.name in self.config.multilib_runtime_blacklist:`
		`- return False`
		`- if po.name in self.config.multilib_runtime_whitelist:`
		`- return True`
		`- if MultilibMethod.select(self, po):`
		`- return True`
		`- if po.name.startswith('kernel'):`
		`- for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:`
		`- if p_name == 'kernel':`
		`- return False`
		`- for file in po.returnFileEntries():`
		`- (dirname, filename) = file.rsplit('/', 1)`
		`-`
		`- # libraries in standard dirs`
		`- if dirname in self.LIBDIRS and fnmatch(filename, '.so.'):`
		`- return True`
		`- if dirname in self.by_dir:`
		`- return True`
		`- # mysql, qt, etc.`
		`- if dirname == '/etc/ld.so.conf.d' and filename.endswith('.conf'):`
		`- return True`
		`- # nss (Some nss modules end in .so instead of .so.X)`
		`- # db (db modules end in .so instead of .so.X)`
		`- if dirname in self.ROOTLIBDIRS and (filename.startswith('libnss_') or filename.startswith('libdb-')):`
		`- return True`
		`- # Optimization:`
		`- # All tests beyond here are for things in USRLIBDIRS`
		`- if not dirname.startswith(tuple(self.USRLIBDIRS)):`
		`- # The dirname does not start with a USRLIBDIR so we can move`
		`- # on to the next file`
		`- continue`
		`-`
		`- if dirname.startswith(('/usr/lib/gtk-2.0', '/usr/lib64/gtk-2.0')):`
		`- # gtk2-engines`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//engines'):`
		`- return True`
		`- # accessibility`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//modules'):`
		`- return True`
		`- # scim-bridge-gtk`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//immodules'):`
		`- return True`
		`- # images`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//loaders'):`
		`- return True`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//printbackends'):`
		`- return True`
		`- if fnmatch(dirname, '/usr/lib/gtk-2.0//filesystems'):`
		`- return True`
		`- # Optimization:`
		`- # No tests beyond here for things in /usr/lib*/gtk-2.0`
		`- continue`
		`-`
		`- # gstreamer`
		`- if dirname.startswith(('/usr/lib/gstreamer-', '/usr/lib64/gstreamer-')):`
		`- return True`
		`- # qt/kde fun`
		`- if fnmatch(dirname, '/usr/lib/qt/plugins/*'):`
		`- return True`
		`- if fnmatch(dirname, '/usr/lib/kde/plugins/*'):`
		`- return True`
		`- # qml`
		`- if fnmatch(dirname, '/usr/lib/qt5/qml/'):`
		`- return True`
		`- # images`
		`- if fnmatch(dirname, '/usr/lib/gdk-pixbuf-2.0//loaders'):`
		`- return True`
		`- # xine-lib`
		`- if fnmatch(dirname, '/usr/lib/xine/plugins/'):`
		`- return True`
		`- # oprofile`
		`- if dirname in self.OPROFILEDIRS and fnmatch(filename, '.so.'):`
		`- return True`
		`- # wine`
		`- if dirname in self.WINEDIRS and filename.endswith('.so'):`
		`- return True`
		`- # sane drivers`
		`- if dirname in self.SANEDIRS and filename.startswith('libsane-'):`
		`- return True`
		`- return False`
		`-`
		`-`
		`- class DevelMultilibMethod(RuntimeMultilibMethod):`
		`-`
		`- def __init__(self, config):`
		`- self.name = 'devel'`
		`- self.config = config`
		`-`
		`- def select(self, po):`
		`- if po.name in self.config.multilib_devel_blacklist:`
		`- return False`
		`- if po.name in self.config.multilib_devel_whitelist:`
		`- return True`
		`- if RuntimeMultilibMethod.select(self, po):`
		`- return True`
		`- if po.name.startswith('ghc-'):`
		`- return False`
		`- if po.name.startswith('kernel'):`
		`- for (p_name, p_flag, (p_e, p_v, p_r)) in po.provides:`
		`- if p_name == 'kernel-devel':`
		`- return False`
		`- if p_name.endswith('-devel') or p_name.endswith('-static'):`
		`- return True`
		`- if po.name.endswith('-devel'):`
		`- return True`
		`- if po.name.endswith('-static'):`
		`- return True`
		`- return False`

ngompa commented 7 years ago

This PR ports mash to use python-multilib, as well as using createrepo_c and DNF when available.

(Opened as new PR because pagure doesn't let you reopen "merged" PRs that were backed out)

ngompa commented 7 years ago

At this time, there's one big problem. @tmlcoch's createrepo_c is missing the necessary flags to support mash's use of createrepo_c to generate the requisite DeltaRPMs. The expected flags are commented out right now with FIXME statements above them in set_delta() in mash/metadata.py.

ausil commented on line 55 of mash/metadata.py 7 years ago

without delta support we can not move.

ausil commented on line 22 of mash/metadata.py 7 years ago

lets keep the MetadataOld class it should be renamed to something like MetadataPythonCLI

ausil commented on line 88 of mash/metadata.py 7 years ago

I would rather call this MatadatPythonAPI

ausil commented 7 years ago

support should be added as a new Class so we keep the old options working. though they will not work with python3. We should use the API and not call out to commands

deltarpm support is a hard requirement

ngompa commented 7 years ago

@ausil It's not that createrepo_c can't generate DeltaRPMs, it's that it doesn't offer the granularity that you expect in mash currently.