copr / copr

Clone

#522 Release related fixes and changes vol.3 (modules)
Merged 5 years ago by msuchy. Opened 5 years ago by frostyx.
copr/ frostyx/copr DockerTestEnv-F28-2-modules  into  master

file modified
+9 -6 
@@ -17,10 +17,11 @@ 
 

  gi.require_version('Modulemd', '1.0')
 

  from gi.repository import Modulemd
 

  

+ from copr_common.rpm import splitFilename
 

  from .sign import create_user_keys, CoprKeygenRequestError
 

  from .createrepo import createrepo
 

  from .exceptions import CreateRepoError, CoprSignError
 

- from .helpers import get_redis_logger, silent_remove, ensure_dir_exists, get_chroot_arch, cmd_debug
 

+ from .helpers import get_redis_logger, silent_remove, ensure_dir_exists, get_chroot_arch, cmd_debug, format_filename
 

  from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey
 

  

  from .vm_manage.manager import VmManager
 
@@ -416,12 +417,13 @@ 
 

              project_path = os.path.join(self.opts.destdir, ownername, projectname)
 

              self.log.info(modulemd_data)
 

  

-             mmd = Modulemd.ModuleStream()
 

-             mmd.import_from_string(modulemd_data)
 

-             artifacts = Modulemd.SimpleSet()
 

- 

              for chroot in chroots:
 

                  arch = get_chroot_arch(chroot)
 

+                 mmd = Modulemd.ModuleStream()
 

+                 mmd.import_from_string(modulemd_data)
 

+                 mmd.set_arch(arch)
 

+                 artifacts = Modulemd.SimpleSet()
 

+ 

                  srcdir = os.path.join(project_path, chroot)
 

                  module_tag = "{}+{}-{}-{}".format(chroot, mmd.get_name(), (mmd.get_stream() or ''),
 

                                                    (str(mmd.get_version()) or '1'))
 
@@ -444,7 +446,8 @@ 
 

                          for f in os.listdir(os.path.join(destdir, folder)):
 

                              if not f.endswith(".rpm") or f.endswith(".src.rpm"):
 

                                  continue
 

-                             artifacts.add(str(f.rstrip(".rpm")))
 

+                             artifact = format_filename(zero_epoch=True, *splitFilename(f))
 

+                             artifacts.add(artifact)
 

  

                      mmd.set_rpm_artifacts(artifacts)
 

                      self.log.info("Module artifacts: %s", mmd.get_rpm_artifacts())

file modified
+1 
@@ -108,6 +108,7 @@ 
 

  INCLUDE_MODULES = \
 

      """/usr/bin/modifyrepo_c \
 

  --mdtype modules \
 

+ --compress-type gz \
 

  {packages_dir}/modules.yaml \
 

  {packages_dir}/repodata
 

  """

file modified
+8 
@@ -531,3 +531,11 @@ 
 

          evr = "{}-{}".format(version, release)
 

  

      return name, evr
 

+ 

+ 

+ def format_filename(name, version, release, epoch, arch, zero_epoch=False):
 

+     if not epoch.isdigit() and zero_epoch:
 

+         epoch = "0"
 

+     if epoch.isdigit():
 

+         return "{}-{}:{}-{}.{}".format(name, epoch, version, release, arch)
 

+     return "{}-{}-{}.{}".format(name, version, release, arch)

file modified
+1 
@@ -60,6 +60,7 @@ 
 

  Requires:   python3-dateutil
 

  Requires:   python3-pytz
 

  Requires:   python3-netaddr
 

+ Requires:   python3-copr-common > 0.4
 

  Requires:   python3-gobject
 

  Requires:   libmodulemd
 

  Requires:   python3-configparser

file modified
+9 -1 
@@ -3,7 +3,7 @@ 
 

  from munch import Munch
 

  

  from backend.exceptions import BuilderError
 

- from backend.helpers import get_redis_logger, get_chroot_arch
 

+ from backend.helpers import get_redis_logger, get_chroot_arch, format_filename
 

  

  """
 

  SOME TESTS REQUIRES RUNNING REDIS
 
@@ -34,3 +34,11 @@ 
 

          assert get_chroot_arch("fedora-26-x86_64") == "x86_64"
 

          assert get_chroot_arch("epel-7-ppc64le") == "ppc64le"
 

          assert get_chroot_arch("epel-7-ppc64") == "ppc64"
 

+ 

+     def test_format_filename(self):
 

+         assert format_filename("ed", "1.14.2", "5.fc30", "", "x86_64") == "ed-1.14.2-5.fc30.x86_64"
 

+         assert format_filename("ed", "1.14.2", "5.fc30", "", "x86_64", zero_epoch=True) == "ed-0:1.14.2-5.fc30.x86_64"
 

+         assert format_filename("ed", "1.14.2", "5.fc30", "2", "x86_64") == "ed-2:1.14.2-5.fc30.x86_64"
 

+ 

+         split = ("ed", "1.14.2", "5.fc30", "2", "x86_64")
 

+         assert format_filename(zero_epoch=True, *split) == "ed-2:1.14.2-5.fc30.x86_64"

file modified
+2 -5 
@@ -75,7 +75,7 @@ 
 

      local tmp=$2
 

      rlAssertEquals "All packages should succeed" `cat $tmp |grep "state" | grep "succeeded" |wc -l` `echo $packages |wc -w`
 

      for pkg in $packages; do
 

-         rlAssertEquals "Package $pkg is missing" `cat $tmp | grep "name" |grep "$pkg" |wc -l` 1
 

+         rlAssertEquals "Package $pkg is missing" `cat $tmp |jq '.[] | .name' |grep "$pkg" |wc -l` 1
 

      done
 

  }
 

  
@@ -128,7 +128,7 @@ 
 

          # Test URL submit
 

          PROJECT=module-testmoduleurl-beakertest-$DATE
 

          copr-cli create $PROJECT --chroot fedora-rawhide-x86_64 --chroot fedora-rawhide-i386
 

-         rlRun "copr-cli build-module --url https://src.fedoraproject.org/modules/testmodule/raw/master/f/testmodule.yaml $PROJECT"
 

+         rlRun "copr-cli build-module --url https://src.fedoraproject.org/modules/testmodule/raw/f28/f/testmodule.yaml $PROJECT"
 

          PACKAGES=`mktemp`
 

          wait_for_finished_module "module-testmoduleurl-beakertest-$DATE" 3 600 $PACKAGES
 

          test_successful_packages "perl-List-Compare perl-Tangerine tangerine" $PACKAGES
 
@@ -191,8 +191,6 @@ 
 

  

  

          # Test that module can be enabled with dnf
 

-         rlRun "dnf -y upgrade dnf --enablerepo mhatina-dnf-modularity-nightly"
 

- 

          # Module repository should be allowed via DNF, but the code isn't merged yet
 

          # https://github.com/rpm-software-management/dnf-plugins-core/pull/214
 

          rlRun "curl $FRONTEND_URL/coprs/$USER/module-testmodule-beakertest-$DATE/module_repo/fedora-rawhide/testmodule-beakertest-$DATE.repo > /etc/yum.repos.d/testmodule.repo"
 
@@ -203,7 +201,6 @@ 
 

          rlRun "rpm -q mksh"
 

          rlRun "dnf -y module remove testmodule"
 

          rlRun "dnf -y module disable testmodule"
 

-         rlRun "dnf -y downgrade dnf"
 

  

  

          # @TODO Test that enabled module info is correct

file modified
+5 -9 
@@ -670,15 +670,11 @@ 
 

          """
 

          ownername, projectname = self.parse_name(args.copr)
 

  

-         try:
 

-             if args.yaml:
 

-                 module = self.client.module_proxy.build_from_file(ownername, projectname, args.yaml)
 

-             else:
 

-                 module = self.client.module_proxy.build_from_url(ownername, projectname, args.url)
 

-             print("Created module {0}".format(module.nsv))
 

- 

-         except CoprRequestException as ex:
 

-             print(ex)
 

+         if args.yaml:
 

+             module = self.client.module_proxy.build_from_file(ownername, projectname, args.yaml)
 

+         else:
 

+             module = self.client.module_proxy.build_from_url(ownername, projectname, args.url)
 

+         print("Created module {0}".format(module.nsv))
 

  

  

  def setup_parser():

file added
+30 
@@ -0,0 +1,30 @@ 
 

+ # TODO: is there something like python-rpm-utils or python-dnf-utils for this?

https://github.com/rpm-software-management/rpm/issues/561

 
+ def splitFilename(filename):
 

+     """
 

+     Pass in a standard style rpm fullname
 

+ 

+     Return a name, version, release, epoch, arch, e.g.::
 

+         foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
 

+         1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
 

+     """
 

+ 

+     if filename[-4:] == '.rpm':
 

+         filename = filename[:-4]
 

+ 

+     archIndex = filename.rfind('.')
 

+     arch = filename[archIndex+1:]
 

+ 

+     relIndex = filename[:archIndex].rfind('-')
 

+     rel = filename[relIndex+1:archIndex]
 

+ 

+     verIndex = filename[:relIndex].rfind('-')
 

+     ver = filename[verIndex+1:relIndex]
 

+ 

+     epochIndex = filename.find(':')
 

+     if epochIndex == -1:
 

+         epoch = ''
 

+     else:
 

+         epoch = filename[:epochIndex]
 

+ 

+     name = filename[epochIndex + 1:verIndex]
 

+     return name, ver, rel, epoch, arch

file modified
+1 -32 
@@ -16,6 +16,7 @@ 
 

  import json
 

  

  from copr_common.enums import EnumType
 

+ from copr_common.rpm import splitFilename
 

  from coprs import constants
 

  from coprs import app
 

  
@@ -156,38 +157,6 @@ 
 

      return "{}{}".format(os, version)
 

  

  

- # TODO: is there something like python-rpm-utils or python-dnf-utils for this?
 

- def splitFilename(filename):
 

-     """
 

-     Pass in a standard style rpm fullname
 

- 

-     Return a name, version, release, epoch, arch, e.g.::
 

-         foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
 

-         1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
 

-     """
 

- 

-     if filename[-4:] == '.rpm':
 

-         filename = filename[:-4]
 

- 

-     archIndex = filename.rfind('.')
 

-     arch = filename[archIndex+1:]
 

- 

-     relIndex = filename[:archIndex].rfind('-')
 

-     rel = filename[relIndex+1:archIndex]
 

- 

-     verIndex = filename[:relIndex].rfind('-')
 

-     ver = filename[verIndex+1:relIndex]
 

- 

-     epochIndex = filename.find(':')
 

-     if epochIndex == -1:
 

-         epoch = ''
 

-     else:
 

-         epoch = filename[:epochIndex]
 

- 

-     name = filename[epochIndex + 1:verIndex]
 

-     return name, ver, rel, epoch, arch
 

- 

- 

  def parse_package_name(pkg):
 

      """
 

      Parse package name from possibly incomplete nvra string.

file modified
+1 -1 
@@ -209,7 +209,7 @@ 
 

  

      @classmethod
 

      def from_file(cls, ref):
 

-         return cls(ref.filename, ref.read())
 

+         return cls(ref.filename, ref.read().decode("utf-8"))
 

  

      @classmethod
 

      def from_url(cls, url):

file modified
+4 -2 
@@ -37,5 +37,7 @@ 
 

          raise BadRequest(str(ex))
 

  

      except sqlalchemy.exc.IntegrityError:
 

-         raise DuplicateException("Module {}-{}-{} already exists".format(
 

-                                  facade.modulemd.name, facade.modulemd.stream, facade.modulemd.version))
 

+         raise DuplicateException("Module {}-{}-{} already exists"
 

+                                  .format(facade.modulemd.get_name(),
 

+                                          facade.modulemd.get_stream(),
 

+                                          facade.modulemd.get_version()))

file modified
+1 -1 
@@ -774,7 +774,7 @@ 
 

  def render_generate_module_repo_file(copr, name_release, module_nsv):
 

      module = ModulesLogic.get_by_nsv_str(copr, module_nsv).one()
 

      mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(name_release, noarch=True).first()
 

-     url = os.path.join(copr_dir.repo_url, '') # adds trailing slash
 

+     url = os.path.join(copr.main_dir.repo_url, '') # adds trailing slash
 

      repo_url = generate_repo_url(mock_chroot, copr.modules_url)
 

      baseurl = "{}+{}/latest/$basearch".format(repo_url.rstrip("/"), module_nsv)
 

      pubkey_url = urljoin(url, "pubkey.gpg")

This is a next PR with release related changes.
The previous ones are PR#508 and PR#514.

This PR makes beaker-tests/Sanity/copr-cli-basic-operations/runtest-modules.sh succeed again. By extend it means, that Copr is again able to build modules, that are visible and installable through DNF.
  • typo in 4553e9c -> [frontend][commong]
  • c7a8f50 I'd prefer not to use * operator here to make the behavior more obvious, but I don't insist .. -> artifact = format_filename(zero_epoch=True, *splitFilename(f))
  • I'd squash c7a8f50 and 4553e9c since both are related to splitFilename() movement

Otherwise lgtm.

@msuchy, thanks for the reference .. the TODO is kept, so we can change the implementation later once it's generally available

+1

rebased onto 5c0d5be
5 years ago

Pull-Request has been merged by msuchy
5 years ago
Metadata
None
No Tags
Changes Summary 12
+9 -6
file changed
backend/backend/actions.py
+1 -0
file changed
backend/backend/createrepo.py
+8 -0
file changed
backend/backend/helpers.py
+1 -0
file changed
backend/copr-backend.spec
+9 -1
file changed
backend/tests/test_helpers.py
+2 -5
file changed
beaker-tests/Sanity/copr-cli-basic-operations/runtest-modules.sh
+5 -9
file changed
cli/copr_cli/main.py
+30
file added
common/copr_common/rpm.py
+1 -32
file changed
frontend/coprs_frontend/coprs/helpers.py
+1 -1
file changed
frontend/coprs_frontend/coprs/logic/modules_logic.py
+4 -2
file changed
frontend/coprs_frontend/coprs/views/apiv3_ns/apiv3_modules.py
+1 -1
file changed
frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.