#576 remove the dependency of rpmUtils
Closed 7 years ago by qwan. Opened 7 years ago by qwan.

file modified
+8 -9
@@ -13,9 +13,8 @@ 

  # You should have received a copy of the GNU General Public License

  # along with this program; if not, see <https://gnu.org/licenses/>.

  

- 

- import rpmUtils.arch

- 

+ from .arch_utils import arches as ALL_ARCHES

+ from .arch_utils import getBaseArch, getMultiArchInfo, getArchList

  

  TREE_ARCH_YUM_ARCH_MAP = {

      "i386": "athlon",
@@ -27,13 +26,13 @@ 

  

  

  def tree_arch_to_yum_arch(tree_arch):

-     # this is basically an opposite to rpmUtils.arch.getBaseArch()

+     # this is basically an opposite to pungi.arch_utils.getBaseArch()

      yum_arch = TREE_ARCH_YUM_ARCH_MAP.get(tree_arch, tree_arch)

      return yum_arch

  

  

  def get_multilib_arch(yum_arch):

-     arch_info = rpmUtils.arch.getMultiArchInfo(yum_arch)

+     arch_info = getMultiArchInfo(yum_arch)

      if arch_info is None:

          return None

      return arch_info[0]
@@ -44,14 +43,14 @@ 

      multilib_arch = get_multilib_arch(yum_arch)

      if not multilib_arch:

          return []

-     return [i for i in rpmUtils.arch.getArchList(multilib_arch) if i not in ("noarch", "src")]

+     return [i for i in getArchList(multilib_arch) if i not in ("noarch", "src")]

  

  

  def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False):

      result = []

  

      yum_arch = tree_arch_to_yum_arch(tree_arch)

-     for arch in rpmUtils.arch.getArchList(yum_arch):

+     for arch in getArchList(yum_arch):

          if arch not in result:

              result.append(arch)

  
@@ -70,7 +69,7 @@ 

  

  

  def get_compatible_arches(arch, multilib=False):

-     tree_arch = rpmUtils.arch.getBaseArch(arch)

+     tree_arch = getBaseArch(arch)

      compatible_arches = get_valid_arches(tree_arch, multilib=multilib)

      return compatible_arches

  
@@ -78,7 +77,7 @@ 

  def is_valid_arch(arch):

      if arch in ("noarch", "src", "nosrc"):

          return True

-     if arch in rpmUtils.arch.arches:

+     if arch in ALL_ARCHES:

          return True

      return False

  

file added
+372
@@ -0,0 +1,372 @@ 

+ #!/usr/bin/python

+ 

+ # A copy of some necessary parts from yum.rpmUtils.arch, with slightly changes:

+ # 1. _ppc64_native_is_best changed to True

+ # 2. code style fixes for flake8 reported errors

+ 

+ import os

+ import rpm

+ import ctypes

+ import struct

+ 

+ # _ppc64_native_is_best is False in yum's source code, but patched with a

+ # seperate patch when built from source rpm, so we set it to True here.

+ _ppc64_native_is_best = True

+ 

+ # dict mapping arch -> ( multicompat, best personality, biarch personality )

+ multilibArches = {"x86_64": ("athlon", "x86_64", "athlon"),

+                   "sparc64v": ("sparcv9v", "sparcv9v", "sparc64v"),

+                   "sparc64": ("sparcv9", "sparcv9", "sparc64"),

+                   "ppc64": ("ppc", "ppc", "ppc64"),

+                   "s390x": ("s390", "s390x", "s390"),

+                   }

+ if _ppc64_native_is_best:

+     multilibArches["ppc64"] = ("ppc", "ppc64", "ppc64")

+ 

+ arches = {

+     # ia32

+     "athlon": "i686",

+     "i686": "i586",

+     "geode": "i586",

+     "i586": "i486",

+     "i486": "i386",

+     "i386": "noarch",

+ 

+     # amd64

+     "x86_64": "athlon",

+     "amd64": "x86_64",

+     "ia32e": "x86_64",

+ 

+     # ppc64le

+     "ppc64le": "noarch",

+ 

+     # ppc

+     "ppc64p7": "ppc64",

+     "ppc64pseries": "ppc64",

+     "ppc64iseries": "ppc64",

+     "ppc64": "ppc",

+     "ppc": "noarch",

+ 

+     # s390{,x}

+     "s390x": "s390",

+     "s390": "noarch",

+ 

+     # sparc

+     "sparc64v": "sparcv9v",

+     "sparc64": "sparcv9",

+     "sparcv9v": "sparcv9",

+     "sparcv9": "sparcv8",

+     "sparcv8": "sparc",

+     "sparc": "noarch",

+ 

+     # alpha

+     "alphaev7": "alphaev68",

+     "alphaev68": "alphaev67",

+     "alphaev67": "alphaev6",

+     "alphaev6": "alphapca56",

+     "alphapca56": "alphaev56",

+     "alphaev56": "alphaev5",

+     "alphaev5": "alphaev45",

+     "alphaev45": "alphaev4",

+     "alphaev4": "alpha",

+     "alpha": "noarch",

+ 

+     # arm

+     "armv7l": "armv6l",

+     "armv6l": "armv5tejl",

+     "armv5tejl": "armv5tel",

+     "armv5tel": "noarch",

+ 

+     # arm hardware floating point

+     "armv7hnl": "armv7hl",

+     "armv7hl": "armv6hl",

+     "armv6hl": "noarch",

+ 

+     # arm64

+     "arm64": "noarch",

+ 

+     # aarch64

+     "aarch64": "noarch",

+ 

+     # super-h

+     "sh4a": "sh4",

+     "sh4": "noarch",

+     "sh3": "noarch",

+ 

+     # itanium

+     "ia64": "noarch",

+ }

+ 

+ #  Will contain information parsed from /proc/self/auxv via _parse_auxv().

+ # Should move into rpm really.

+ _aux_vector = {

+     "platform": "",

+     "hwcap": 0,

+ }

+ 

+ 

+ def isMultiLibArch(arch=None):

+     """returns true if arch is a multilib arch, false if not"""

+     if arch is None:

+         arch = canonArch

+ 

+     if arch not in arches:  # or we could check if it is noarch

+         return 0

+ 

+     if arch in multilibArches:

+         return 1

+ 

+     if arches[arch] in multilibArches:

+         return 1

+ 

+     return 0

+ 

+ 

+ def getArchList(thisarch=None):

+     # this returns a list of archs that are compatible with arch given

+     if not thisarch:

+         thisarch = canonArch

+ 

+     archlist = [thisarch]

+     while thisarch in arches:

+         thisarch = arches[thisarch]

+         archlist.append(thisarch)

+ 

+     # hack hack hack

+     # sparc64v is also sparc64 compat

+     if archlist[0] == "sparc64v":

+         archlist.insert(1, "sparc64")

+ 

+     # if we're a weirdo arch - add noarch on there.

+     if len(archlist) == 1 and archlist[0] == thisarch:

+         archlist.append('noarch')

+     return archlist

+ 

+ 

+ def _try_read_cpuinfo():

+     """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not

+         mounted). """

+     try:

+         return open("/proc/cpuinfo", "r")

+     except:

+         return []

+ 

+ 

+ def _parse_auxv():

+     """ Read /proc/self/auxv and parse it into global dict for easier access

+         later on, very similar to what rpm does. """

+     # In case we can't open and read /proc/self/auxv, just return

+     try:

+         data = open("/proc/self/auxv", "rb").read()

+     except:

+         return

+ 

+     # Define values from /usr/include/elf.h

+     AT_PLATFORM = 15

+     AT_HWCAP = 16

+     fmtlen = struct.calcsize("LL")

+     offset = 0

+     platform = ctypes.c_char_p()

+ 

+     # Parse the data and fill in _aux_vector dict

+     while offset <= len(data) - fmtlen:

+         at_type, at_val = struct.unpack_from("LL", data, offset)

+         if at_type == AT_PLATFORM:

+             platform.value = at_val

+             _aux_vector["platform"] = platform.value

+         if at_type == AT_HWCAP:

+             _aux_vector["hwcap"] = at_val

+         offset = offset + fmtlen

+ 

+ 

+ def getCanonX86Arch(arch):

+     if arch == "i586":

+         for line in _try_read_cpuinfo():

+             if line.startswith("model name"):

+                 if line.find("Geode(TM)") != -1:

+                     return "geode"

+                 break

+         return arch

+     # only athlon vs i686 isn't handled with uname currently

+     if arch != "i686":

+         return arch

+ 

+     # if we're i686 and AuthenticAMD, then we should be an athlon

+     for line in _try_read_cpuinfo():

+         if line.startswith("vendor") and line.find("AuthenticAMD") != -1:

+             return "athlon"

+         # i686 doesn't guarantee cmov, but we depend on it

+         elif line.startswith("flags"):

+             if line.find("cmov") == -1:

+                 return "i586"

+             break

+ 

+     return arch

+ 

+ 

+ def getCanonARMArch(arch):

+     # the %{_target_arch} macro in rpm will let us know the abi we are using

+     target = rpm.expandMacro('%{_target_cpu}')

+     if target.startswith('armv6h'):

+         return target

+     if target.startswith('armv7h'):

+         return target

+     return arch

+ 

+ 

+ def getCanonPPCArch(arch):

+     # FIXME: should I do better handling for mac, etc?

+     if arch != "ppc64":

+         return arch

+ 

+     machine = None

+     for line in _try_read_cpuinfo():

+         if line.find("machine") != -1:

+             machine = line.split(':')[1]

+             break

+ 

+     platform = _aux_vector["platform"]

+     if machine is None and not platform:

+         return arch

+ 

+     try:

+         if platform.startswith("power") and int(platform[5:].rstrip('+')) >= 7:

+             return "ppc64p7"

+     except:

+         pass

+ 

+     if machine is None:

+         return arch

+ 

+     if machine.find("CHRP IBM") != -1:

+         return "ppc64pseries"

+     if machine.find("iSeries") != -1:

+         return "ppc64iseries"

+     return arch

+ 

+ 

+ def getCanonSPARCArch(arch):

+     # Deal with sun4v, sun4u, sun4m cases

+     SPARCtype = None

+     for line in _try_read_cpuinfo():

+         if line.startswith("type"):

+             SPARCtype = line.split(':')[1]

+             break

+     if SPARCtype is None:

+         return arch

+ 

+     if SPARCtype.find("sun4v") != -1:

+         if arch.startswith("sparc64"):

+             return "sparc64v"

+         else:

+             return "sparcv9v"

+     if SPARCtype.find("sun4u") != -1:

+         if arch.startswith("sparc64"):

+             return "sparc64"

+         else:

+             return "sparcv9"

+     if SPARCtype.find("sun4m") != -1:

+         return "sparcv8"

+     return arch

+ 

+ 

+ def getCanonX86_64Arch(arch):

+     if arch != "x86_64":

+         return arch

+ 

+     vendor = None

+     for line in _try_read_cpuinfo():

+         if line.startswith("vendor_id"):

+             vendor = line.split(':')[1]

+             break

+     if vendor is None:

+         return arch

+ 

+     if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1:

+         return "amd64"

+     if vendor.find("GenuineIntel") != -1:

+         return "ia32e"

+     return arch

+ 

+ 

+ def getCanonArch(skipRpmPlatform=0):

+     if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):

+         try:

+             f = open("/etc/rpm/platform", "r")

+             line = f.readline()

+             f.close()

+             (arch, vendor, opersys) = line.split("-", 2)

+             return arch

+         except:

+             pass

+ 

+     arch = os.uname()[4]

+ 

+     _parse_auxv()

+ 

+     if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"):

+         return getCanonX86Arch(arch)

+ 

+     if arch.startswith("arm"):

+         return getCanonARMArch(arch)

+     if arch.startswith("ppc"):

+         return getCanonPPCArch(arch)

+     if arch.startswith("sparc"):

+         return getCanonSPARCArch(arch)

+     if arch == "x86_64":

+         return getCanonX86_64Arch(arch)

+ 

+     return arch

+ 

+ canonArch = getCanonArch()

+ 

+ 

+ # this gets you the "compat" arch of a biarch pair

+ def getMultiArchInfo(arch=canonArch):

+     if arch in multilibArches:

+         return multilibArches[arch]

+     if arch in arches and arches[arch] != "noarch":

+         return getMultiArchInfo(arch=arches[arch])

+     return None

+ 

+ 

+ def getBaseArch(myarch=None):

+     """returns 'base' arch for myarch, if specified, or canonArch if not.

+        base arch is the arch before noarch in the arches dict if myarch is not

+        a key in the multilibArches."""

+ 

+     if not myarch:

+         myarch = canonArch

+ 

+     if myarch not in arches:  # this is dumb, but <shrug>

+         return myarch

+ 

+     if myarch.startswith("sparc64"):

+         return "sparc"

+     elif myarch == "ppc64le":

+         return "ppc64le"

+     elif myarch.startswith("ppc64") and not _ppc64_native_is_best:

+         return "ppc"

+     elif myarch.startswith("arm64"):

+         return "arm64"

+     elif myarch.startswith("armv6h"):

+         return "armhfp"

+     elif myarch.startswith("armv7h"):

+         return "armhfp"

+     elif myarch.startswith("arm"):

+         return "arm"

+ 

+     if isMultiLibArch(arch=myarch):

+         if myarch in multilibArches:

+             return myarch

+         else:

+             return arches[myarch]

+ 

+     if myarch in arches:

+         basearch = myarch

+         value = arches[basearch]

+         while value != 'noarch':

+             basearch = value

+             value = arches[basearch]

+ 

+         return basearch

file modified
+2 -3
@@ -17,11 +17,10 @@ 

  import os

  import sys

  import time

- import yum

  

  from ConfigParser import SafeConfigParser

  

- 

+ from .arch_utils import getBaseArch

  # In development, `here` will point to the bin/ directory with scripts.

  here = sys.path[0]

  MULTILIBCONF = (os.path.join(os.path.dirname(__file__), '..', 'share', 'multilib')
@@ -47,7 +46,7 @@ 

          self.set('pungi', 'product_path', 'Packages')

          self.set('pungi', 'cachedir', '/var/cache/pungi')

          self.set('pungi', 'compress_type', 'xz')

-         self.set('pungi', 'arch', yum.rpmUtils.arch.getBaseArch())

+         self.set('pungi', 'arch', getBaseArch())

          self.set('pungi', 'family', 'Fedora')

          self.set('pungi', 'iso_basename', 'Fedora')

          self.set('pungi', 'version', time.strftime('%Y%m%d', time.localtime()))

file modified
+3 -3
@@ -18,8 +18,8 @@ 

  import json

  import logging

  import os

- import rpmUtils.arch

  

+ from pungi.arch_utils import getBaseArch

  from pungi.util import makedirs

  

  
@@ -43,9 +43,9 @@ 

              try:

                  parsed = json.loads(f.read())

                  if arch is None:

-                     basearch = rpmUtils.arch.getBaseArch()

+                     basearch = getBaseArch()

                  else:

-                     basearch = rpmUtils.arch.getBaseArch(arch)

+                     basearch = getBaseArch(arch)

                  ref = parsed['ref'].replace('${basearch}', basearch)

              except Exception as e:

                  logger.error('Unable to get ref from treefile: %s' % e)

@@ -21,10 +21,11 @@ 

  import threading

  

  import koji

- import rpmUtils.arch

  from kobo.shortcuts import run

  from ConfigParser import ConfigParser

  

+ from ..arch_utils import getBaseArch

+ 

  

  class KojiWrapper(object):

      lock = threading.Lock()
@@ -97,7 +98,7 @@ 

          cmd.append(target)

  

          # i686 -> i386 etc.

-         arch = rpmUtils.arch.getBaseArch(arch)

+         arch = getBaseArch(arch)

          cmd.append(arch)

  

          if isinstance(command, list):
@@ -261,7 +262,7 @@ 

          cmd.append(target)

  

          # i686 -> i386 etc.

-         arch = rpmUtils.arch.getBaseArch(arch)

+         arch = getBaseArch(arch)

          cmd.append(arch)

  

          cmd.append(ks_file)

file modified
+1 -1
@@ -56,7 +56,7 @@ 

          return ARCHES[yum_arch]

  

  

- @mock.patch('rpmUtils.arch', MockArchModule)

+ @mock.patch('pungi.arch_utils', MockArchModule)

  class TestArch(unittest.TestCase):

  

      def test_i386(self):

In case of the absenence of rpmUtils in new version of systems, we need to
drop the dependency of rpmUtils. DNF doesn't provide the mapping APIs in
rpmUtils, the only one available in dnf is getBaseArch and doesn't
support getting current system's arch, dnf detects the os with hawkey.
Since the most tricky part in rpmUtils.arch is detecting the current
system's arch and we can't easy test this to ensure no regression will be
introduced by using hawkey, it would be much safer to just copy the necessary
parts from rpmUtils.arch to pungi to remove the dependency.

FIXES: #533

Signed-off-by: Qixiang Wan qwan@redhat.com

rebased

7 years ago

We don't need to care about gather.py and multilib.py (renamed to multilib_yum.py in #526) files. They use yum for other things, so this is not an issue there.

I really don't like copying the code from rpmUtils. The magic to determine current architecture is really scary. We don't really need it for pungi/config.py which again is using Yum and can stay on rpmUtils. Is there a case where pungi/ostree/utils.py needs to determine current arch?

We don't need to care about gather.py and multilib.py (renamed to multilib_yum.py in #526) files. They use yum for other things, so this is not an issue there.
I really don't like copying the code from rpmUtils. The magic to determine current architecture is really scary.

TBH, I don't like this either :(

We don't really need it for pungi/config.py which again is using Yum and can stay on rpmUtils. Is there a case where pungi/ostree/utils.py needs to determine current arch?

ok, so now we have only one place that is using getBaseArch() to determine current arch, the case is https://pagure.io/pungi/issue/513, since we only support OSTree on x86_64 and some other new arches like aarch64, not any ancient arches, I think the original proposal from you getBaseArch(platform.machine()) or hawkey.detect_arch() would just be safe enough, what do you think?

So we can remove most of the code we copied from rpmUtils, for some code about multilib arch checking, I think we still need to copy them, do you have any suggestion on this?

Unfortunately I don't see a better way (other than asking dnf guys to support all of this). If I recall correctly, @ignatenkobrain said hawkey API should not be used directly, so that's not a solution.

please, stop reimplementing stuff, go to https://bugzilla.redhat.com/show_bug.cgi?id=1072972 and comment what you need RPM to do.

Drop this, will use APIs provided by rpm once they're available.

Pull-Request has been closed by qwan

7 years ago