#522 Release related fixes and changes vol.3 (modules)
Merged 9 months ago by msuchy. Opened 10 months ago by frostyx.
copr/ frostyx/copr DockerTestEnv-F28-2-modules  into  master

file modified
+9 -6

@@ -17,10 +17,11 @@ 

  gi.require_version('Modulemd', '1.0')

  from gi.repository import Modulemd

  

+ from copr_common.rpm import splitFilename

  from .sign import create_user_keys, CoprKeygenRequestError

  from .createrepo import createrepo

  from .exceptions import CreateRepoError, CoprSignError

- from .helpers import get_redis_logger, silent_remove, ensure_dir_exists, get_chroot_arch, cmd_debug

+ from .helpers import get_redis_logger, silent_remove, ensure_dir_exists, get_chroot_arch, cmd_debug, format_filename

  from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey

  

  from .vm_manage.manager import VmManager

@@ -416,12 +417,13 @@ 

              project_path = os.path.join(self.opts.destdir, ownername, projectname)

              self.log.info(modulemd_data)

  

-             mmd = Modulemd.ModuleStream()

-             mmd.import_from_string(modulemd_data)

-             artifacts = Modulemd.SimpleSet()

- 

              for chroot in chroots:

                  arch = get_chroot_arch(chroot)

+                 mmd = Modulemd.ModuleStream()

+                 mmd.import_from_string(modulemd_data)

+                 mmd.set_arch(arch)

+                 artifacts = Modulemd.SimpleSet()

+ 

                  srcdir = os.path.join(project_path, chroot)

                  module_tag = "{}+{}-{}-{}".format(chroot, mmd.get_name(), (mmd.get_stream() or ''),

                                                    (str(mmd.get_version()) or '1'))

@@ -444,7 +446,8 @@ 

                          for f in os.listdir(os.path.join(destdir, folder)):

                              if not f.endswith(".rpm") or f.endswith(".src.rpm"):

                                  continue

-                             artifacts.add(str(f.rstrip(".rpm")))

+                             artifact = format_filename(zero_epoch=True, *splitFilename(f))

+                             artifacts.add(artifact)

  

                      mmd.set_rpm_artifacts(artifacts)

                      self.log.info("Module artifacts: %s", mmd.get_rpm_artifacts())

@@ -108,6 +108,7 @@ 

  INCLUDE_MODULES = \

      """/usr/bin/modifyrepo_c \

  --mdtype modules \

+ --compress-type gz \

  {packages_dir}/modules.yaml \

  {packages_dir}/repodata

  """

@@ -531,3 +531,11 @@ 

          evr = "{}-{}".format(version, release)

  

      return name, evr

+ 

+ 

+ def format_filename(name, version, release, epoch, arch, zero_epoch=False):

+     if not epoch.isdigit() and zero_epoch:

+         epoch = "0"

+     if epoch.isdigit():

+         return "{}-{}:{}-{}.{}".format(name, epoch, version, release, arch)

+     return "{}-{}-{}.{}".format(name, version, release, arch)

@@ -60,6 +60,7 @@ 

  Requires:   python3-dateutil

  Requires:   python3-pytz

  Requires:   python3-netaddr

+ Requires:   python3-copr-common > 0.4

  Requires:   python3-gobject

  Requires:   libmodulemd

  Requires:   python3-configparser

@@ -3,7 +3,7 @@ 

  from munch import Munch

  

  from backend.exceptions import BuilderError

- from backend.helpers import get_redis_logger, get_chroot_arch

+ from backend.helpers import get_redis_logger, get_chroot_arch, format_filename

  

  """

  SOME TESTS REQUIRES RUNNING REDIS

@@ -34,3 +34,11 @@ 

          assert get_chroot_arch("fedora-26-x86_64") == "x86_64"

          assert get_chroot_arch("epel-7-ppc64le") == "ppc64le"

          assert get_chroot_arch("epel-7-ppc64") == "ppc64"

+ 

+     def test_format_filename(self):

+         assert format_filename("ed", "1.14.2", "5.fc30", "", "x86_64") == "ed-1.14.2-5.fc30.x86_64"

+         assert format_filename("ed", "1.14.2", "5.fc30", "", "x86_64", zero_epoch=True) == "ed-0:1.14.2-5.fc30.x86_64"

+         assert format_filename("ed", "1.14.2", "5.fc30", "2", "x86_64") == "ed-2:1.14.2-5.fc30.x86_64"

+ 

+         split = ("ed", "1.14.2", "5.fc30", "2", "x86_64")

+         assert format_filename(zero_epoch=True, *split) == "ed-2:1.14.2-5.fc30.x86_64"

@@ -75,7 +75,7 @@ 

      local tmp=$2

      rlAssertEquals "All packages should succeed" `cat $tmp |grep "state" | grep "succeeded" |wc -l` `echo $packages |wc -w`

      for pkg in $packages; do

-         rlAssertEquals "Package $pkg is missing" `cat $tmp | grep "name" |grep "$pkg" |wc -l` 1

+         rlAssertEquals "Package $pkg is missing" `cat $tmp |jq '.[] | .name' |grep "$pkg" |wc -l` 1

      done

  }

  

@@ -128,7 +128,7 @@ 

          # Test URL submit

          PROJECT=module-testmoduleurl-beakertest-$DATE

          copr-cli create $PROJECT --chroot fedora-rawhide-x86_64 --chroot fedora-rawhide-i386

-         rlRun "copr-cli build-module --url https://src.fedoraproject.org/modules/testmodule/raw/master/f/testmodule.yaml $PROJECT"

+         rlRun "copr-cli build-module --url https://src.fedoraproject.org/modules/testmodule/raw/f28/f/testmodule.yaml $PROJECT"

          PACKAGES=`mktemp`

          wait_for_finished_module "module-testmoduleurl-beakertest-$DATE" 3 600 $PACKAGES

          test_successful_packages "perl-List-Compare perl-Tangerine tangerine" $PACKAGES

@@ -191,8 +191,6 @@ 

  

  

          # Test that module can be enabled with dnf

-         rlRun "dnf -y upgrade dnf --enablerepo mhatina-dnf-modularity-nightly"

- 

          # Module repository should be allowed via DNF, but the code isn't merged yet

          # https://github.com/rpm-software-management/dnf-plugins-core/pull/214

          rlRun "curl $FRONTEND_URL/coprs/$USER/module-testmodule-beakertest-$DATE/module_repo/fedora-rawhide/testmodule-beakertest-$DATE.repo > /etc/yum.repos.d/testmodule.repo"

@@ -203,7 +201,6 @@ 

          rlRun "rpm -q mksh"

          rlRun "dnf -y module remove testmodule"

          rlRun "dnf -y module disable testmodule"

-         rlRun "dnf -y downgrade dnf"

  

  

          # @TODO Test that enabled module info is correct

file modified
+5 -9

@@ -670,15 +670,11 @@ 

          """

          ownername, projectname = self.parse_name(args.copr)

  

-         try:

-             if args.yaml:

-                 module = self.client.module_proxy.build_from_file(ownername, projectname, args.yaml)

-             else:

-                 module = self.client.module_proxy.build_from_url(ownername, projectname, args.url)

-             print("Created module {0}".format(module.nsv))

- 

-         except CoprRequestException as ex:

-             print(ex)

+         if args.yaml:

+             module = self.client.module_proxy.build_from_file(ownername, projectname, args.yaml)

+         else:

+             module = self.client.module_proxy.build_from_url(ownername, projectname, args.url)

+         print("Created module {0}".format(module.nsv))

  

  

  def setup_parser():

@@ -0,0 +1,30 @@ 

+ # TODO: is there something like python-rpm-utils or python-dnf-utils for this?

+ def splitFilename(filename):

+     """

+     Pass in a standard style rpm fullname

+ 

+     Return a name, version, release, epoch, arch, e.g.::

+         foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386

+         1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64

+     """

+ 

+     if filename[-4:] == '.rpm':

+         filename = filename[:-4]

+ 

+     archIndex = filename.rfind('.')

+     arch = filename[archIndex+1:]

+ 

+     relIndex = filename[:archIndex].rfind('-')

+     rel = filename[relIndex+1:archIndex]

+ 

+     verIndex = filename[:relIndex].rfind('-')

+     ver = filename[verIndex+1:relIndex]

+ 

+     epochIndex = filename.find(':')

+     if epochIndex == -1:

+         epoch = ''

+     else:

+         epoch = filename[:epochIndex]

+ 

+     name = filename[epochIndex + 1:verIndex]

+     return name, ver, rel, epoch, arch

@@ -16,6 +16,7 @@ 

  import json

  

  from copr_common.enums import EnumType

+ from copr_common.rpm import splitFilename

  from coprs import constants

  from coprs import app

  

@@ -156,38 +157,6 @@ 

      return "{}{}".format(os, version)

  

  

- # TODO: is there something like python-rpm-utils or python-dnf-utils for this?

- def splitFilename(filename):

-     """

-     Pass in a standard style rpm fullname

- 

-     Return a name, version, release, epoch, arch, e.g.::

-         foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386

-         1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64

-     """

- 

-     if filename[-4:] == '.rpm':

-         filename = filename[:-4]

- 

-     archIndex = filename.rfind('.')

-     arch = filename[archIndex+1:]

- 

-     relIndex = filename[:archIndex].rfind('-')

-     rel = filename[relIndex+1:archIndex]

- 

-     verIndex = filename[:relIndex].rfind('-')

-     ver = filename[verIndex+1:relIndex]

- 

-     epochIndex = filename.find(':')

-     if epochIndex == -1:

-         epoch = ''

-     else:

-         epoch = filename[:epochIndex]

- 

-     name = filename[epochIndex + 1:verIndex]

-     return name, ver, rel, epoch, arch

- 

- 

  def parse_package_name(pkg):

      """

      Parse package name from possibly incomplete nvra string.

@@ -209,7 +209,7 @@ 

  

      @classmethod

      def from_file(cls, ref):

-         return cls(ref.filename, ref.read())

+         return cls(ref.filename, ref.read().decode("utf-8"))

  

      @classmethod

      def from_url(cls, url):

@@ -37,5 +37,7 @@ 

          raise BadRequest(str(ex))

  

      except sqlalchemy.exc.IntegrityError:

-         raise DuplicateException("Module {}-{}-{} already exists".format(

-                                  facade.modulemd.name, facade.modulemd.stream, facade.modulemd.version))

+         raise DuplicateException("Module {}-{}-{} already exists"

+                                  .format(facade.modulemd.get_name(),

+                                          facade.modulemd.get_stream(),

+                                          facade.modulemd.get_version()))

@@ -774,7 +774,7 @@ 

  def render_generate_module_repo_file(copr, name_release, module_nsv):

      module = ModulesLogic.get_by_nsv_str(copr, module_nsv).one()

      mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(name_release, noarch=True).first()

-     url = os.path.join(copr_dir.repo_url, '') # adds trailing slash

+     url = os.path.join(copr.main_dir.repo_url, '') # adds trailing slash

      repo_url = generate_repo_url(mock_chroot, copr.modules_url)

      baseurl = "{}+{}/latest/$basearch".format(repo_url.rstrip("/"), module_nsv)

      pubkey_url = urljoin(url, "pubkey.gpg")

This is a next PR with release related changes.
The previous ones are PR#508 and PR#514.

This PR makes beaker-tests/Sanity/copr-cli-basic-operations/runtest-modules.sh succeed again. By extend it means, that Copr is again able to build modules, that are visible and installable through DNF.

  • typo in 4553e9c -> [frontend][commong]
  • c7a8f50 I'd prefer not to use * operator here to make the behavior more obvious, but I don't insist .. -> artifact = format_filename(zero_epoch=True, *splitFilename(f))
  • I'd squash c7a8f50 and 4553e9c since both are related to splitFilename() movement

Otherwise lgtm.

@msuchy, thanks for the reference .. the TODO is kept, so we can change the implementation later once it's generally available

rebased onto 5c0d5be

9 months ago

Pull-Request has been merged by msuchy

9 months ago