#1133 Get MBS running under Python 3
Merged 2 months ago by mprahl. Opened 2 months ago by mikeb.
mikeb/fm-orchestrator py3-fixes  into  master

file modified
+4 -2

@@ -56,13 +56,15 @@ 

              onmyduffynode "cd fm-orchestrator && git log -2"

          }

  

-         stage('Build Docker Image') {

+         stage('Build Docker Images') {

              onmyduffynode 'cd fm-orchestrator && docker build -t mbs/test -f docker/Dockerfile-tests .'

+             onmyduffynode 'cd fm-orchestrator && docker build -t mbs/test-py3 -f docker/Dockerfile-tests-py3 .'

          }

  

-         stage('Run Test Suite') {

+         stage('Run Test Suites') {

              timeout(20) {

                  onmyduffynode 'docker run -v ~/fm-orchestrator:/src:Z mbs/test'

+                 onmyduffynode 'docker run -v ~/fm-orchestrator:/src:Z mbs/test-py3'

              }

          }

  

file modified
+1 -1

@@ -44,7 +44,7 @@ 

      rpm-build \

      && yum clean all

  # We currently require a newer versions of these Python packages for the tests

- RUN pip install --upgrade flask-sqlalchemy pytest flake8 tox

+ RUN pip install --upgrade flask-sqlalchemy pytest flake8 tox pip

  VOLUME /src

  WORKDIR /src

  CMD ["bash", "docker/test.sh"]

@@ -0,0 +1,38 @@ 

+ FROM fedora:29

+ 

+ WORKDIR /build

+ RUN dnf -y install \

+     --nogpgcheck \

+     --setopt=deltarpm=0 \

+     --setopt=install_weak_deps=false \

+     --setopt=tsflags=nodocs \

+     git-core \

+     createrepo_c \

+     python3-fedmsg \

+     python3-kobo-rpmlib \

+     python3-rpm \

+     libmodulemd \

+     python3-gobject \

+     python3-dogpile-cache \

+     python3-flask \

+     python3-flask-migrate \

+     python3-flask-sqlalchemy \

+     python3-koji \

+     python3-ldap3 \

+     python3-munch \

+     python3-pip \

+     python3-requests \

+     python3-six \

+     python3-solv \

+     python3-sqlalchemy \

+     python3-pungi \

+     # Test-only dependencies

+     python3-pytest \

+     python3-flake8 \

+     python3-mock \

+     python3-tox \

+     rpm-build \

+     && dnf clean all

+ VOLUME /src

+ WORKDIR /src

+ CMD ["bash", "docker/test-py3.sh"]

file added
+34

@@ -0,0 +1,34 @@ 

+ #!/bin/bash

+ 

+ # Remove requirements not necessary for Python 3.7.

+ # Also, prevent koji from being re-installed from PyPi.

+ cp requirements.txt requirements.txt.orig.py3

+ sed -i \

+     -e '/enum34/d' \

+     -e '/funcsigs/d' \

+     -e '/futures/d' \

+     -e '/koji/d' \

+     requirements.txt

+ 

+ # Run everything with Python 3

+ cp tox.ini tox.ini.orig.py3

+ sed -i \

+     -e 's/py.test/py.test-3/g' \

+     -e '/basepython/d' \

+     tox.ini

+ 

+ # Delete any leftover compiled Python files

+ for dir in module_build_service tests; do

+     find ${dir} -type f \( -name '*.pyc' -or -name '*.pyc' \) -exec rm -f {} \;

+ done

+ 

+ # Since tox seems to ignore `usedevelop` when we have `sitepackages` on, we have to run it manually

+ python3 setup.py develop --no-deps

+ /usr/bin/tox -e flake8,py3

+ rv=$?

+ 

+ # After running tox, we can revert back to the original files

+ rm -f requirements.txt tox.ini

+ mv requirements.txt.orig.py3 requirements.txt

+ mv tox.ini.orig.py3 tox.ini

+ exit $rv

file modified
+3

@@ -8,6 +8,9 @@ 

  for dir in module_build_service tests; do

      find ${dir} -type f \( -name '*.pyc' -or -name '*.pyc' \) -exec rm -f {} \;

  done

+ # The python-virtualenv package bundles a very old version of pip,

+ # which is incompatible with modern virtualenv.

+ rm -f /usr/lib/python2.7/site-packages/virtualenv_support/pip-9*

  # Since tox seems to ignore `usedevelop` when we have `sitepackages` on, we have to run it manually

  python setup.py develop --no-deps

  /usr/bin/tox -e flake8,py27

@@ -36,6 +36,8 @@ 

  import time

  from io import open

  from collections import defaultdict

+ from itertools import chain

+ 

  import kobo.rpmlib

  

  from six import text_type

@@ -200,7 +202,7 @@ 

  

              (stdout, stderr) = p.communicate()

              status = p.wait()

-             output = stdout

+             output = stdout.decode("utf-8")

  

          if status != 0:

              log.debug("%s: stderr output: %s", cmd, stderr)

@@ -276,7 +278,7 @@ 

          # Handle the multicall result. For each build associated with the source RPM,

          # store the exclusivearch and excludearch lists. For each RPM, store the 'license' and

          # also other useful data from the Build associated with the RPM.

-         for rpm, headers in zip(src_rpms.values() + binary_rpms.values(), rpms_headers):

+         for rpm, headers in zip(chain(src_rpms.values(), binary_rpms.values()), rpms_headers):

              if not headers:

                  raise RuntimeError(

                      "No RPM headers received from Koji for RPM %s" % rpm["name"])

@@ -708,7 +710,7 @@ 

          # Fill in the list of built RPMs.

          mmd = self._fill_in_rpms_list(mmd, arch)

  

-         return unicode(mmd.dumps())

+         return text_type(mmd.dumps())

  

      def _download_source_modulemd(self, mmd, output_path):

          """

@@ -295,7 +295,7 @@ 

          work_queue_put(wrapped_msg)

      except ValueError as e:

          log.warning("No MBSConsumer found.  Shutting down?  %r" % e)

-     except AttributeError as e:

+     except AttributeError:

          # In the event that `moksha.hub._hub` hasn't yet been initialized, we

          # need to store messages on the side until it becomes available.

          # As a last-ditch effort, try to hang initial messages in the config.

@@ -115,7 +115,7 @@ 

      if len(nsvc_tag) + len('-build') > max_length:

          # Fallback to the old format of 'module-<hash>' if the generated koji tag

          # name is longer than max_length

-         nsvc_hash = hashlib.sha1('.'.join(nsvc_list)).hexdigest()[:16]

+         nsvc_hash = hashlib.sha1('.'.join(nsvc_list).encode('utf-8')).hexdigest()[:16]

          return 'module-' + nsvc_hash

      return nsvc_tag

  

@@ -213,47 +213,47 @@ 

  

  

  def get_prefixed_version(mmd):

-         """

-         Return the prefixed version of the module based on the buildrequired base module stream.

- 

-         :param mmd: the Modulemd.Module object to format

-         :return: the prefixed version

-         :rtype: int

-         """

-         xmd = mmd.get_xmd()

-         version = mmd.get_version()

- 

-         base_module_stream = None

-         for base_module in conf.base_module_names:

-             # xmd is a GLib Variant and doesn't support .get() syntax

-             try:

-                 base_module_stream = xmd['mbs']['buildrequires'].get(

-                     base_module, {}).get('stream')

-                 if base_module_stream:

-                     # Break after finding the first base module that is buildrequired

-                     break

-             except KeyError:

-                 log.warning('The module\'s mmd is missing information in the xmd section')

-                 return version

-         else:

-             log.warning('This module does not buildrequire a base module ({0})'

-                         .format(' or '.join(conf.base_module_names)))

-             return version

+     """

+     Return the prefixed version of the module based on the buildrequired base module stream.

  

-         # The platform version (e.g. prefix1.2.0 => 010200)

-         version_prefix = models.ModuleBuild.get_stream_version(base_module_stream, right_pad=False)

+     :param mmd: the Modulemd.Module object to format

+     :return: the prefixed version

+     :rtype: int

+     """

+     xmd = mmd.get_xmd()

+     version = mmd.get_version()

  

-         if not version_prefix:

-             log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s '

-                         'version'.format(base_module, base_module_stream))

+     base_module_stream = None

+     for base_module in conf.base_module_names:

+         # xmd is a GLib Variant and doesn't support .get() syntax

+         try:

+             base_module_stream = xmd['mbs']['buildrequires'].get(

+                 base_module, {}).get('stream')

+             if base_module_stream:

+                 # Break after finding the first base module that is buildrequired

+                 break

+         except KeyError:

+             log.warning('The module\'s mmd is missing information in the xmd section')

              return version

+     else:

+         log.warning('This module does not buildrequire a base module ({0})'

+                     .format(' or '.join(conf.base_module_names)))

+         return version

  

-         new_version = int(str(version_prefix) + str(version))

-         if new_version > GLib.MAXUINT64:

-             log.warning('The "{0}" stream "{1}" caused the module\'s version prefix to be '

-                         'too long'.format(base_module, base_module_stream))

-             return version

-         return new_version

+     # The platform version (e.g. prefix1.2.0 => 010200)

+     version_prefix = models.ModuleBuild.get_stream_version(base_module_stream, right_pad=False)

+ 

+     if not version_prefix:

+         log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s '

+                     'version'.format(base_module, base_module_stream))

+         return version

+ 

+     new_version = int(str(version_prefix) + str(version))

+     if new_version > GLib.MAXUINT64:

+         log.warning('The "{0}" stream "{1}" caused the module\'s version prefix to be '

+                     'too long'.format(base_module, base_module_stream))

+         return version

+     return new_version

  

  

  def validate_mmd(mmd):

@@ -371,7 +371,7 @@ 

  

  def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False,

                                    optional_params=None):

-     yaml_file = handle.read()

+     yaml_file = handle.read().decode("utf-8")

      mmd = load_mmd(yaml_file)

      dt = datetime.utcfromtimestamp(int(time.time()))

      def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])

file modified
+6 -5

@@ -145,13 +145,14 @@ 

              build_one.state = BUILD_STATES['ready']

              if contexts:

                  build_one.stream = str(index)

-                 unique_hash = hashlib.sha1("%s:%s:%d:%d" % (

-                     build_one.name, build_one.stream, build_one.version, context)).hexdigest()

+                 unique_hash = hashlib.sha1(("%s:%s:%d:%d" % (

+                     build_one.name, build_one.stream, build_one.version,

+                     context)).encode("utf-8")).hexdigest()

                  build_one.build_context = unique_hash

                  build_one.runtime_context = unique_hash

                  build_one.ref_build_context = unique_hash

                  combined_hashes = '{0}:{1}'.format(unique_hash, unique_hash)

-                 build_one.context = hashlib.sha1(combined_hashes).hexdigest()[:8]

+                 build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8]

              with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:

                  build_one.modulemd = mmd.read()

              build_one.koji_tag = 'module-nginx-1.2'

@@ -632,7 +633,7 @@ 

  

          session.add(build_one)

  

-         components = mmd.get_rpm_components().values()

+         components = list(mmd.get_rpm_components().values())

          components.sort(key=lambda x: x.get_buildorder())

          previous_buildorder = None

          batch = 1

@@ -683,7 +684,7 @@ 

  

          session.add(build_one)

  

-         components2 = mmd2.get_rpm_components().values()

+         components2 = list(mmd2.get_rpm_components().values())

          # Store components to database in different order than for 570 to

          # reproduce the reusing issue.

          components2.sort(key=lambda x: len(x.get_name()))

@@ -324,6 +324,7 @@ 

          expected_calls = [mock.call(1, 'foo'), mock.call(2, 'foo'), mock.call(1, 'bar')]

          assert mock_session.untagBuild.mock_calls == expected_calls

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights(self, ClientSession):

          session = ClientSession.return_value

@@ -347,6 +348,7 @@ 

          # getLoggedInUser requires to a logged-in session

          session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights_no_task_id(self, ClientSession):

          session = ClientSession.return_value

@@ -368,6 +370,7 @@ 

          assert session.getTaskDescendents.mock_calls == expected_calls

          session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights_no_build(self, ClientSession):

          session = ClientSession.return_value

@@ -389,6 +392,7 @@ 

          assert session.getTaskDescendents.mock_calls == expected_calls

          session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights_listBuilds_failed(self, ClientSession):

          session = ClientSession.return_value

@@ -406,6 +410,7 @@ 

          assert session.listBuilds.mock_calls == expected_calls

          session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights_getPackageID_failed(self, ClientSession):

          session = ClientSession.return_value

@@ -421,6 +426,7 @@ 

  

          session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_get_build_weights_getLoggedInUser_failed(self, ClientSession):

          session = ClientSession.return_value

@@ -686,6 +692,7 @@ 

          assert ClientSession.return_value == session

          assert ClientSession.return_value.krb_login.assert_not_called

  

+     @patch.dict('sys.modules', krbV=MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_ensure_builder_use_a_logged_in_koji_session(self, ClientSession):

          builder = KojiModuleBuilder('owner', MagicMock(), conf, 'module-tag', [])

@@ -107,8 +107,8 @@ 

          pkg_res.return_value = Mock()

          pkg_res.return_value.version = "current-tested-version"

          rpm_mock = Mock()

-         rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \

-                   "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"

+         rpm_out = b"rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \

+                   b"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"

          attrs = {'communicate.return_value': (rpm_out, 'error'),

                   'wait.return_value': 0}

          rpm_mock.configure_mock(**attrs)

@@ -163,8 +163,8 @@ 

          pkg_res.return_value = Mock()

          pkg_res.return_value.version = "current-tested-version"

          rpm_mock = Mock()

-         rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \

-                   "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"

+         rpm_out = b"rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \

+                   b"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"

          attrs = {'communicate.return_value': (rpm_out, 'error'),

                   'wait.return_value': 0}

          rpm_mock.configure_mock(**attrs)

@@ -208,6 +208,7 @@ 

          with open(path.join(dir_path, "modulemd.i686.txt")) as mmd:

              assert len(mmd.read()) == 255

  

+     @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      def test_tag_cg_build(self, ClientSession):

          """ Test that the CG build is tagged. """

@@ -223,6 +224,7 @@ 

          # tagBuild requires logging into a session in advance.

          koji_session.krb_login.assert_called_once()

  

+     @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      def test_tag_cg_build_fallback_to_default_tag(self, ClientSession):

          """ Test that the CG build is tagged to default tag. """

@@ -240,6 +242,7 @@ 

          # tagBuild requires logging into a session in advance.

          koji_session.krb_login.assert_called_once()

  

+     @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      def test_tag_cg_build_no_tag_set(self, ClientSession):

          """ Test that the CG build is not tagged when no tag set. """

@@ -254,6 +257,7 @@ 

          # tagBuild requires logging into a session in advance.

          koji_session.krb_login.assert_called_once()

  

+     @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      def test_tag_cg_build_no_tag_available(self, ClientSession):

          """ Test that the CG build is not tagged when no tag available. """

@@ -270,7 +274,7 @@ 

      @patch("module_build_service.builder.KojiContentGenerator.open", create=True)

      def test_get_arch_mmd_output(self, patched_open):

          patched_open.return_value = mock_open(

-             read_data=self.cg.mmd).return_value

+             read_data=self.cg.mmd.encode("utf-8")).return_value

          ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64")

          assert ret == {

              'arch': 'x86_64',

@@ -290,7 +294,7 @@ 

          rpm_artifacts = mmd.get_rpm_artifacts()

          rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")

          mmd.set_rpm_artifacts(rpm_artifacts)

-         mmd_data = mmd.dumps()

+         mmd_data = mmd.dumps().encode("utf-8")

  

          patched_open.return_value = mock_open(

              read_data=mmd_data).return_value

@@ -872,6 +876,7 @@ 

                      requires.append("%s:%s" % (name, stream))

              assert "%s:%s" % (mmd.get_name(), mmd.get_stream()) in requires

  

+     @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._tag_cg_build")

      @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._load_koji_tag")

@@ -99,6 +99,7 @@ 

  

          assert len(start_build_component.mock_calls) == expected_build_calls

  

+     @patch.dict("sys.modules", krbV=mock.MagicMock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      def test_trigger_new_repo_when_failed(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -132,6 +133,7 @@ 

          koji_session.newRepo.assert_called_once_with(

              "module-testmodule-master-20170219191323-c40c156c-build")

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_trigger_new_repo_when_succeeded(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -204,6 +206,7 @@ 

          for component in components:

              assert component.state is None

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_old_build_targets_are_not_associated_with_any_module_builds(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -224,6 +227,7 @@ 

  

          koji_session.deleteBuildTarget.assert_not_called()

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_dont_delete_base_module_build_target(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -249,6 +253,7 @@ 

  

              koji_session.deleteBuildTarget.assert_not_called()

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_dont_delete_build_target_for_unfinished_module_builds(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -276,6 +281,7 @@ 

  

              koji_session.deleteBuildTarget.assert_not_called()

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_only_delete_build_target_with_allowed_koji_tag_prefix(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -321,6 +327,7 @@ 

              koji_session.deleteBuildTarget.assert_called_once_with(1)

              koji_session.krb_login.assert_called_once()

  

+     @patch.dict('sys.modules', krbV=mock.MagicMock())

      @patch('module_build_service.builder.KojiModuleBuilder.KojiClientSession')

      def test_cant_delete_build_target_if_not_reach_delete_time(

              self, ClientSession, create_builder, global_consumer, dbg):

@@ -277,7 +277,7 @@ 

          with open(modulemd_file_path, "w") as fd:

              fd.write(modulemd_yaml)

  

-         with open(modulemd_file_path, "r") as fd:

+         with open(modulemd_file_path, "rb") as fd:

              handle = FileStorage(fd)

              module_build_service.utils.submit_module_build_from_yaml(username, handle,

                                                                       stream=stream, skiptests=True)

@@ -368,8 +368,8 @@ 

          mmd_pkg_refs = [pkg.get_ref() for pkg in mmd.get_rpm_components().values()]

          assert set(mmd_pkg_refs) == set(hashes_returned.keys())

          br = mmd.get_dependencies()[0].get_buildrequires()

-         assert br.keys() == ['platform']

-         assert br.values()[0].get() == ['f28']

+         assert list(br.keys()) == ['platform']

+         assert list(br.values())[0].get() == ['f28']

          xmd = {

              'mbs': {

                  'commit': '',

@@ -1447,7 +1447,7 @@ 

          assert data['error'] == 'Forbidden'

          assert data['message'] == (

              'Homer J. Simpson is not in any of '

-             'set([\'mbs-import-module\']), only set([\'packager\'])')

+             '{0}, only {1}'.format(set(['mbs-import-module']), set(['packager'])))

  

      @pytest.mark.parametrize('api_version', [1, 2])

      @patch('module_build_service.auth.get_user', return_value=import_module_user)

A few small changes to get MBS running under Python 3, and a lot of fixes to get the tests running.

Weird, CentOS CI doesn't seem to use the version of .cico-pr.pipeline included in the pull request, so Python 3 tests were not run. Anyone know how to change that?

@mikeb I'm not sure, but perhaps it only uses the .cico-pr.pipeline from the master branch.

@mikeb since krbV doesn't work on Python 3, what's the long-term plan to support Kerberos authentication under Python 3?

@mprahl MBS should be ported to use GSSAPI auth, which works under Python 3.

rebased onto a44e1fe

2 months ago

Pull-Request has been merged by mprahl

2 months ago