#557 Preprocess spec files using rpmautospec features and use %autorelease when parsing spec files
Merged 2 years ago by onosek. Opened 2 years ago by nphilipp.

file modified
+6 -1
@@ -15,6 +15,7 @@ 

  import os

  import sys

  

+ import six

  from six.moves import configparser

  

  import pyrpkg
@@ -34,7 +35,11 @@ 

      sys.exit(1)

  

  # Setup a configuration object and read config file data

- config = configparser.SafeConfigParser()

+ if six.PY2:

+     config = configparser.SafeConfigParser()

+ else:

+     # The SafeConfigParser class has been renamed to ConfigParser in Python 3.2.

+     config = configparser.ConfigParser()

  config.read(args.config)

  

  client = pyrpkg.cli.cliClient(config)

file modified
+59 -16
@@ -72,6 +72,13 @@ 

  except ImportError:

      specfile_uses_rpmautospec = None

  

+ try:

+     from rpmautospec import process_distgit as rpmautospec_process_distgit

+     from rpmautospec import calculate_release_number as rpmautospec_calculate_release_number

+ except ImportError:

+     rpmautospec_process_distgit = None

+     rpmautospec_calculate_release_number = None

+ 

  

  class NullHandler(logging.Handler):

      """Null logger to avoid spurious messages, add a handler in app code"""
@@ -161,6 +168,8 @@ 

          self._rel = None

          # Whether the spec file uses %autorelease

          self._uses_autorelease = None

+         # Whether the spec file uses rpmautospec features (at all)

+         self._uses_rpmautospec = None

          # The cloned repo object

          self._repo = None

          # The rpm defines used when calling rpm
@@ -646,21 +655,33 @@ 

              self.load_nameverrel()

          return self._uses_autorelease

  

+     @property

+     def uses_rpmautospec(self):

+         if self._uses_rpmautospec is None:

+             self.load_nameverrel()

+         return self._uses_rpmautospec

+ 

      def load_nameverrel(self):

          """Set the release of a package."""

  

+         cmd = ['rpm']

+         cmd.extend(self.rpmdefines)

+ 

          specfile_path = os.path.join(self.path, self.spec)

  

          if specfile_uses_rpmautospec:

              self._uses_autorelease = specfile_uses_rpmautospec(

                  specfile_path, check_autorelease=True, check_autochangelog=False

              )

+             self._uses_rpmautospec = specfile_uses_rpmautospec(specfile_path)

+             if self._uses_rpmautospec and rpmautospec_calculate_release_number:

+                 release_number = rpmautospec_calculate_release_number(specfile_path)

+                 cmd.append("--define '_rpmautospec_release_number %d'" % release_number)

          else:

              # Set to 0 so it evaluates false-ish but differs from (unset) None.

              self._uses_autorelease = 0

+             self._uses_rpmautospec = 0

  

-         cmd = ['rpm']

-         cmd.extend(self.rpmdefines)

          # We make sure there is a space at the end of our query so that

          # we can split it later.  When there are subpackages, we get a

          # listing for each subpackage.  We only care about the first.
@@ -2575,19 +2596,30 @@ 

                          % hashtype,

                          "--define '_binary_filedigest_algorithm %s'"

                          % hashtype])

-         cmd.extend(['-ba', os.path.join(self.path, self.spec)])

-         logfile = '.build-%s-%s.log' % (self.ver, self.rel)

- 

-         cmd = '%s 2>&1 | tee %s' % (' '.join(cmd), logfile)

+         specpath = os.path.join(self.path, self.spec)

+         tmpdir = None

          try:

-             # Since zsh is a widely used, which is supported by fedpkg

-             # actually, pipestatus is for checking the first command when zsh

-             # is used.

-             subprocess.check_call(

-                 '%s; exit "${PIPESTATUS[0]} ${pipestatus[1]}"' % cmd,

-                 shell=True)

-         except subprocess.CalledProcessError:

-             raise rpkgError(cmd)

+             if not self.uses_rpmautospec or not rpmautospec_process_distgit:

+                 cmd.extend(['-ba', specpath])

+             else:

+                 tmpdir = tempfile.mkdtemp(prefix="rpkg-rpmautospec")

+                 tmpspecpath = os.path.join(tmpdir, self.spec)

+                 rpmautospec_process_distgit(specpath, tmpspecpath)

+                 cmd.extend(['-ba', tmpspecpath])

+             logfile = '.build-%s-%s.log' % (self.ver, self.rel)

+ 

+             cmd = '%s 2>&1 | tee %s' % (' '.join(cmd), logfile)

+             try:

+                 # Since zsh is a widely used, which is supported by fedpkg

+                 # actually, pipestatus is for checking the first command when zsh

+                 # is used.

+                 subprocess.check_call(

+                     '%s; exit "${PIPESTATUS[0]} ${pipestatus[1]}"' % cmd,

+                     shell=True)

+             except subprocess.CalledProcessError:

+                 raise rpkgError(cmd)

+         finally:

+             self._cleanup_tmp_dir(tmpdir)

  

      # Not to be confused with mockconfig the property

      def mock_config(self, target=None, arch=None):
@@ -2950,8 +2982,19 @@ 

                          % hashtype,

                          "--define '_binary_filedigest_algorithm %s'"

                          % hashtype])

-         cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)])

-         self._run_command(cmd, shell=True)

+         specpath = os.path.join(self.path, self.spec)

+         tmpdir = None

+         try:

+             if not self.uses_rpmautospec or not rpmautospec_process_distgit:

+                 cmd.extend(['--nodeps', '-bs', specpath])

+             else:

+                 tmpdir = tempfile.mkdtemp(prefix="rpkg-rpmautospec")

+                 tmpspecpath = os.path.join(tmpdir, self.spec)

+                 rpmautospec_process_distgit(specpath, tmpspecpath)

+                 cmd.extend(['--nodeps', '-bs', tmpspecpath])

+             self._run_command(cmd, shell=True)

+         finally:

+             self._cleanup_tmp_dir(tmpdir)

  

      def unused_patches(self):

          """Discover patches checked into source control that are not used

file modified
+1 -1
@@ -58,7 +58,7 @@ 

          # Create a bare Git repository

          moduledir = os.path.join(self.gitroot, module)

          os.makedirs(moduledir)

-         subprocess.check_call(['git', 'init', '--bare'], cwd=moduledir,

+         subprocess.check_call(['git', 'init', '--bare', '-b', 'master'], cwd=moduledir,

                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  

          # Clone it, and do the minimal Dist Git setup

file modified
+1 -1
@@ -1444,7 +1444,7 @@ 

  

          self.chaos_repo = tempfile.mkdtemp(prefix='rpkg-tests-chaos-repo-')

          cmds = (

-             ['git', 'init'],

+             ['git', 'init', '-b', 'master'],

              ['touch', 'README.md'],

              ['git', 'add', 'README.md'],

              ['git', 'config', 'user.name', 'tester'],

file modified
+37
@@ -74,9 +74,20 @@ 

          self.cmd = self.make_commands()

          self.checkout_branch(self.cmd.repo, 'eng-rhel-6')

          self.tempdir = tempfile.mkdtemp(prefix='rpkg_test_')

+         self._patchers = {

+             name: patch("pyrpkg.%s" % name) for name in (

+                 "specfile_uses_rpmautospec",

+                 "rpmautospec_process_distgit",

+                 "rpmautospec_calculate_release_number",

+             )

+         }

+         self.mocks = {name: patcher.start() for name, patcher in self._patchers.items()}

+         self.mocks["specfile_uses_rpmautospec"].return_value = False

  

      def tearDown(self):

          super(LoadNameVerRelTest, self).tearDown()

+         for patcher in self._patchers.values():

+             patcher.stop()

          shutil.rmtree(self.tempdir)

  

      def test_load_from_spec(self):
@@ -118,6 +129,8 @@ 

          self.assertEqual('0', cmd._epoch)

          self.assertEqual('1.2', cmd._ver)

          self.assertEqual('2.el6', cmd._rel)

+         self.assertIs(False, cmd._uses_autorelease)

+         self.assertIs(False, cmd._uses_rpmautospec)

  

      @patch('pyrpkg.Commands.load_rpmdefines', new=mock_load_rpmdefines)

      @patch('pyrpkg.Commands.load_spec',
@@ -145,6 +158,30 @@ 

          self.assertEqual('1.2', self.cmd._ver)

          self.assertEqual('2.el6', self.cmd._rel)

  

+     @patch("pyrpkg.specfile_uses_rpmautospec", new=None)

+     @patch("pyrpkg.rpmautospec_process_distgit", new=None)

+     @patch("pyrpkg.rpmautospec_calculate_release_number", new=None)

+     def test_load_with_rpmautospec_pkg_missing(self):

+         self.cmd.load_nameverrel()

+         self.assertIs(0, self.cmd._uses_autorelease)

+         self.assertIs(0, self.cmd._uses_rpmautospec)

+ 

+     @patch("subprocess.Popen", wraps=subprocess.Popen)

+     def test_load_with_rpmautospec(self, wrapped_popen):

+         test_release_number = 123

+ 

+         self.mocks["specfile_uses_rpmautospec"].return_value = True

+         self.mocks["rpmautospec_process_distgit"].return_value = True

+         self.mocks["rpmautospec_calculate_release_number"].return_value = test_release_number

+ 

+         self.cmd.load_nameverrel()

+ 

+         self.assertIs(True, self.cmd._uses_autorelease)

+         self.assertIs(True, self.cmd._uses_rpmautospec)

+         self.assertEqual(1, wrapped_popen.call_count)

+         args, kwargs = wrapped_popen.call_args

+         self.assertIn("--define '_rpmautospec_release_number %d'" % test_release_number, args[0])

+ 

  

  class LoadBranchMergeTest(CommandTestCase):

      """Test case for testing Commands.load_branch_merge"""

file modified
+7 -2
@@ -6,6 +6,7 @@ 

  import tempfile

  

  import mock

+ import six

  from six.moves import configparser

  

  import pyrpkg.cli
@@ -31,7 +32,7 @@ 

  

      def _setup_repo(self, origin):

          cmds = (

-             ['git', 'init'],

+             ['git', 'init', '-b', 'master'],

              ['git', 'config', 'user.name', 'John Doe'],

              ['git', 'config', 'user.email', 'jdoe@example.com'],

              ['git', 'remote', 'add', 'origin', origin],
@@ -52,7 +53,11 @@ 

          return out.strip()

  

      def _fake_client(self, args):

-         config = configparser.SafeConfigParser()

+         if six.PY2:

+             config = configparser.SafeConfigParser()

+         else:

+             # The SafeConfigParser class has been renamed to ConfigParser in Python 3.2.

+             config = configparser.ConfigParser()

          config.read(TEST_CONFIG)

          with mock.patch('sys.argv', new=args):

              client = pyrpkg.cli.cliClient(config, name='rpkg')

file modified
+1 -1
@@ -157,7 +157,7 @@ 

              f.write(spec_file)

  

          git_cmds = [

-             ['git', 'init'],

+             ['git', 'init', '-b', 'master'],

              ['touch', 'sources', 'CHANGELOG.rst'],

              ['git', 'add', spec_file_path, 'sources', 'CHANGELOG.rst'],

              ['git', 'config', 'user.email', 'tester@example.com'],

If spec files use rpmautospec features, preprocess them into a temporary directory and point rpmbuild at the pre-processed spec file.

Depends-On: https://pagure.io/fedora-infra/rpmautospec/pull-request/183

If the necessary rpmautospec features aren't present (i.e. before fedora-infra/rpmautospec#183 is merged and release), all this will be skipped.

Also, calculate the release number and pass it to the macro when parsing the spec file which should make fedpkg/rpkg commands display/expect the right NVR when working on rpmautospec-enabled packages.

rebased onto 391813f72cc976a97f3e53967ea09712801b61af

2 years ago

pretty please pagure-ci rebuild

2 years ago

3 new commits added

  • Reflect %autorelease when parsing spec files
  • Preprocess spec files using rpmautospec features
  • Detect generic use of rpmautospec features
2 years ago

@onosek This blocks wider adaptation of rpmautospec and I would really like to start using that. Can I do anything to make this land faster? I can do code review or testing.

rebased onto 4dd9be94443fcec843cf13460fa8ff564388effa

2 years ago

I checked the code (without visible issues) and I can merge it. It would be nice to have one or two simple unittests added.
How urgent releasing of the functionality is?
1) There might be a regular release of rpkg+fedpkg in ~3 weeks (Bodhi waiting is not included).
2) Make a patch of the rpkg package. I suppose it should go out together with the previous PR https://pagure.io/rpkg/pull-request/548# I can possibly work on it on this Thu/Fri.

If we patch the rpkh package, it will be easier to test before the release and we can get some fixes in before the 3 weeks ETA.

I can add unit tests. In order not to require rpmautospec for testing I'll mock its functions.

rebased onto d5e895d

2 years ago

@onosek the latest push adds and augments tests, and fixes some unrelated small things (explicitly sets git repo default branch name in tests, doesn't use deprecated SafeConfigParser with Python 3.x).

Thanks for the tests and extra ConfigParser's fix.

Pull-Request has been merged by onosek

2 years ago

Hello @nphilipp, @churchyard I built a rpkg-1.62-6 packages for testing purposes (for F33, F34, rawhide).
Just a rpkg patch with two PRs:
https://pagure.io/rpkg/pull-request/557
https://pagure.io/rpkg/pull-request/548
Updates are waiting in Bodhi.

I had to remove one of your commits from the build - it caused unit tests to fail in epel8 and epel8-playground:
https://kojipkgs.fedoraproject.org//work/tasks/4827/71484827/build.log
Maybe, I will have to revert the commit later
https://pagure.io/fork/nphilipp/rpkg/c/d5e895d13e95bfc1c3ca2e5d9c61ff8d368aebb8

Hi @onosek, there's an easy fix for old git versions where git init doesn't accept the -b/--initial-branch parameter. I'll submit a PR.

Hi @onosek, here's the PR which should fix tests on EL8: #563