#1144 Fix the handling of modulemd files with unicode characters.
Merged 2 months ago by mprahl. Opened 2 months ago by jkaluza.
jkaluza/fm-orchestrator ls-remote  into  master

@@ -22,7 +22,6 @@ 

  # Written by Stanislav Ochotnicky <sochotnicky@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

  

- 

  import calendar

  import hashlib

  import logging

@@ -46,6 +45,7 @@ 

  

  from module_build_service import conf, log, build_logs, Modulemd, glib

  from module_build_service.scm import SCM

+ from module_build_service.utils import to_text_type

  

  logging.basicConfig(level=logging.DEBUG)

  

@@ -95,7 +95,7 @@ 

          self.owner = module.owner

          self.module = module

          self.module_name = module.name

-         self.mmd = module.modulemd

+         self.mmd = to_text_type(module.modulemd)

          self.config = config

          self.devel = False

          # List of architectures the module is built for.

@@ -407,7 +407,7 @@ 

          mmd_path = os.path.join(output_path, mmd_filename)

          try:

              with open(mmd_path, 'rb') as mmd_f:

-                 data = mmd_f.read().decode('utf-8')

+                 data = to_text_type(mmd_f.read())

                  mmd = Modulemd.Module().new_from_string(data)

                  ret['filename'] = mmd_filename

                  ret['filesize'] = len(data)

@@ -710,7 +710,7 @@ 

          # Fill in the list of built RPMs.

          mmd = self._fill_in_rpms_list(mmd, arch)

  

-         return text_type(mmd.dumps())

+         return to_text_type(mmd.dumps())

  

      def _download_source_modulemd(self, mmd, output_path):

          """

@@ -759,7 +759,7 @@ 

          prepdir = tempfile.mkdtemp(prefix="koji-cg-import")

          mmd_path = os.path.join(prepdir, "modulemd.txt")

          log.info("Writing generic modulemd.yaml to %r" % mmd_path)

-         with open(mmd_path, "w") as mmd_f:

+         with open(mmd_path, "w", encoding="utf-8") as mmd_f:

              mmd_f.write(self.mmd)

  

          mmd_path = os.path.join(prepdir, "modulemd.src.txt")

@@ -769,7 +769,7 @@ 

              mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch)

              log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path))

              mmd = self._finalize_mmd(arch)

-             with open(mmd_path, "w") as mmd_f:

+             with open(mmd_path, "w", encoding="utf-8") as mmd_f:

                  mmd_f.write(mmd)

  

          log_path = os.path.join(prepdir, "build.log")

@@ -38,6 +38,7 @@ 

  from module_build_service.utils.ursine import handle_stream_collision_modules

  

  from requests.exceptions import ConnectionError

+ from module_build_service.utils import to_text_type

  

  import koji

  

@@ -155,7 +156,7 @@ 

          record_component_builds(mmd, build, session=session)

          handle_stream_collision_modules(mmd)

          mmd = record_filtered_rpms(mmd)

-         build.modulemd = mmd.dumps()

+         build.modulemd = to_text_type(mmd.dumps())

          build.transition(conf, models.BUILD_STATES["wait"])

      # Catch custom exceptions that we can expose to the user

      except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e:

@@ -27,12 +27,23 @@ 

  import hashlib

  import time

  from datetime import datetime

+ from six import text_type

  

  from module_build_service import conf, log, models

  from module_build_service.errors import (

      ValidationError, ProgrammingError, UnprocessableEntity)

  

  

+ def to_text_type(s):

+     """

+     Converts `s` to `text_type`. In case it fails, returns `s`.

+     """

+     try:

+         return text_type(s, "utf-8")

+     except TypeError:

+         return s

+ 

+ 

  def scm_url_schemes(terse=False):

      """

      Definition of URL schemes supported by both frontend and scheduler.

@@ -323,7 +334,7 @@ 

      build.version = version

      build.koji_tag = koji_tag

      build.state = models.BUILD_STATES['ready']

-     build.modulemd = mmd.dumps()

+     build.modulemd = to_text_type(mmd.dumps())

      build.context = context

      build.owner = "mbs_import"

      build.rebuild_strategy = 'all'

@@ -29,6 +29,7 @@ 

  import os

  from multiprocessing.dummy import Pool as ThreadPool

  from datetime import datetime

+ from module_build_service.utils import to_text_type

  

  import kobo.rpmlib

  import requests

@@ -562,7 +563,7 @@ 

                  name=mmd.get_name(),

                  stream=mmd.get_stream(),

                  version=version_str,

-                 modulemd=mmd.dumps(),

+                 modulemd=to_text_type(mmd.dumps()),

                  scmurl=url,

                  username=username,

                  **(optional_params or {})

@@ -767,7 +768,7 @@ 

              stream=mmd.get_stream(),

              version=str(mmd.get_version()),

              context=mmd.get_context(),

-             modulemd=mmd.dumps(),

+             modulemd=to_text_type(mmd.dumps()),

              scmurl="",

              username="mbs",

              publish_msg=False)

file modified
+11 -10

@@ -26,6 +26,7 @@ 

  import time

  import hashlib

  from traceback import extract_stack

+ from module_build_service.utils import to_text_type

  

  import koji

  import module_build_service

@@ -54,7 +55,7 @@ 

      if not os.path.exists(filename):

          raise ValueError('Staged data {}.yaml does not exist.'.format(yaml_name))

      with open(filename, 'r') as mmd:

-         return mmd.read()

+         return to_text_type(mmd.read())

  

  

  def patch_config():

@@ -153,8 +154,7 @@ 

                  build_one.ref_build_context = unique_hash

                  combined_hashes = '{0}:{1}'.format(unique_hash, unique_hash)

                  build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8]

-             with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:

-                 build_one.modulemd = mmd.read()

+             build_one.modulemd = read_staged_data('nginx_mmd')

              build_one.koji_tag = 'module-nginx-1.2'

              build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?'

                                  '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')

@@ -343,7 +343,7 @@ 

      build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18)

      build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35)

      build_one.rebuild_strategy = 'changed-and-after'

-     build_one.modulemd = mmd.dumps()

+     build_one.modulemd = to_text_type(mmd.dumps())

      build_one_component_release = get_rpm_release(build_one)

      build_one.buildrequires.append(platform_br)

  

@@ -456,7 +456,7 @@ 

      xmd['mbs']['scmurl'] = build_one.scmurl

      xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'

      mmd.set_xmd(glib.dict_values(xmd))

-     build_one.modulemd = mmd.dumps()

+     build_one.modulemd = to_text_type(mmd.dumps())

      build_one.buildrequires.append(platform_br)

  

      component_one_build_one = module_build_service.models.ComponentBuild()

@@ -541,7 +541,7 @@ 

      xmd['mbs']['scmurl'] = build_one.scmurl

      xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'

      mmd.set_xmd(glib.dict_values(xmd))

-     build_two.modulemd = mmd.dumps()

+     build_two.modulemd = to_text_type(mmd.dumps())

      build_two.buildrequires.append(platform_br)

  

      component_one_build_two = module_build_service.models.ComponentBuild()

@@ -620,7 +620,7 @@ 

          build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'

          build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'

          build_one.state = BUILD_STATES['ready']

-         build_one.modulemd = mmd.dumps()

+         build_one.modulemd = to_text_type(mmd.dumps())

          build_one.koji_tag = 'module-shared-userspace-f26-20170601141014-75f92abb'

          build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.'

                              'git?#7fea453')

@@ -671,7 +671,7 @@ 

          build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'

          build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'

          build_one.state = BUILD_STATES['done']

-         build_one.modulemd = mmd2.dumps()

+         build_one.modulemd = to_text_type(mmd2.dumps())

          build_one.koji_tag = 'module-shared-userspace-f26-20170605091544-75f92abb'

          build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.'

                              'git?#7fea453')

@@ -741,7 +741,8 @@ 

      mmd.set_version(int(version))

      mmd.set_context(context)

      mmd.set_summary("foo")

-     mmd.set_description("foo")

+     # Test unicode in mmd.

+     mmd.set_description(u"foo \u2019s")

      licenses = Modulemd.SimpleSet()

      licenses.add("GPL")

      mmd.set_module_licenses(licenses)

@@ -802,7 +803,7 @@ 

      module_build.build_context = context

      module_build.stream_build_context = context

      module_build.runtime_context = context

-     module_build.modulemd = mmd.dumps()

+     module_build.modulemd = to_text_type(mmd.dumps())

      if base_module:

          module_build.buildrequires.append(base_module)

      if 'koji_tag' in xmd['mbs']:

file modified
+1 -1

@@ -6,7 +6,7 @@ 

      version: 5

      summary: A fake module containing the bash shell

      description: >

-         A fake module used for testing

+         A fake module used for testing ’

      license:

          module:

              - MIT

@@ -3,7 +3,7 @@ 

  data:

      summary: A fake module containing the bash shell

      description: >

-         A fake module used for testing

+         A fake module used for testing ’

      license:

          module:

              - MIT

@@ -19,7 +19,7 @@ 

  

          Programmers can write new built-in modules for Python in C or C++.

          Python can be used as an extension language for applications that

-         need a programmable interface.

+         need a programmable interface. ’

      license:

          module:

              - MIT

@@ -7,7 +7,7 @@ 

    context: 9c690d0e

    summary: A test module in all its beautiful beauty

    description: This module demonstrates how to write simple modulemd files And can

-     be used for testing the build and release pipeline.

+     be used for testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -7,7 +7,7 @@ 

    context: 9c690d0e

    summary: A test module in all its beautiful beauty

    description: This module demonstrates how to write simple modulemd files And can

-     be used for testing the build and release pipeline.

+     be used for testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -3,7 +3,7 @@ 

  data:

      summary: A fake module containing the bash shell

      description: >

-         A fake module used for testing

+         A fake module used for testing ’

      license:

          module:

              - MIT

@@ -20,7 +20,7 @@ 

      summary: An example nginx module

      # A verbose description of the module, required

      description: >

-         A module for the tests of module build service

+         A module for the tests of module build service ’

      # Module and content licenses in the Fedora license identifier

      # format, required

      license:

@@ -1,7 +1,7 @@ 

  document: modulemd

  version: 1

  data:

-   description: Fedora 28 traditional base

+   description: Fedora 28 traditional base ’

    name: platform

    license:

      module: [MIT]

@@ -19,7 +19,7 @@ 

  

          Programmers can write new built-in modules for Python in C or C++.

          Python can be used as an extension language for applications that

-         need a programmable interface.

+         need a programmable interface. ’

      license:

          module:

              - MIT

@@ -4,7 +4,7 @@ 

      summary: A test module in all its beautiful beauty

      description: >-

          This module demonstrates how to write simple modulemd files And

-         can be used for testing the build and release pipeline.

+         can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -4,7 +4,7 @@ 

    summary: A test module in all its beautiful beauty

    description: >-

      This module demonstrates how to write simple modulemd files And can be used for

-     testing the build and release pipeline.

+     testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -4,7 +4,7 @@ 

      summary: A test module in all its beautiful beauty

      description: >-

          This module demonstrates how to write simple modulemd files And

-         can be used for testing the build and release pipeline.

+         can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -2,7 +2,7 @@ 

  version: 1

  data:

      summary: A test module in all its beauty

-     description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline.

+     description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’

      version: 0123456789

      license:

          module: [ MIT ]

@@ -4,7 +4,7 @@ 

      summary: A test module in all its beautiful beauty

      description: >-

          This module demonstrates how to write simple modulemd files And

-         can be used for testing the build and release pipeline.

+         can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -3,7 +3,7 @@ 

  data:

      stream: wrong_stream

      summary: A test module in all its beauty

-     description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline.

+     description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -4,7 +4,7 @@ 

      summary: A test module in all its beautiful beauty

      description: >-

          This module demonstrates how to write simple modulemd files And

-         can be used for testing the build and release pipeline.

+         can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -23,7 +23,7 @@ 

        silly: puddy

        glass: water

    description: This module demonstrates how to write simple modulemd files And can

-     be used for testing the build and release pipeline.

+     be used for testing the build and release pipeline. 😄

    filter: {}

    license:

      module: [MIT]

@@ -4,7 +4,7 @@ 

      summary: A test module in all its beautiful beauty

      description: >-

          This module demonstrates how to write simple modulemd files And

-         can be used for testing the build and release pipeline.

+         can be used for testing the build and release pipeline. ’

      license:

          module: [ MIT ]

      dependencies:

@@ -4,7 +4,7 @@ 

    summary: A test module in all its beautiful beauty

    description: >-

      This module demonstrates how to write simple modulemd files And can be used for

-     testing the build and release pipeline.

+     testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -4,7 +4,7 @@ 

    summary: A test module in all its beautiful beauty

    description: >-

      This module demonstrates how to write simple modulemd files And can be used for

-     testing the build and release pipeline.

+     testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -4,7 +4,7 @@ 

    summary: A test module in all its beautiful beauty

    description: >-

      This module demonstrates how to write simple modulemd files And can be used for

-     testing the build and release pipeline.

+     testing the build and release pipeline. ’

    license:

      module:

      - MIT

@@ -28,6 +28,7 @@ 

  from shutil import copyfile

  from datetime import datetime, timedelta

  from random import randint

+ from module_build_service.utils import to_text_type

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos

@@ -443,7 +444,7 @@ 

  

          testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml')

          with open(testmodule) as f:

-             yaml = f.read()

+             yaml = to_text_type(f.read())

  

          with patch.object(module_build_service.config.Config, 'yaml_submit_allowed',

                            new_callable=PropertyMock, return_value=False):

@@ -885,7 +886,7 @@ 

          formatted_testmodule_yml_path = os.path.join(

              current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')

          with open(formatted_testmodule_yml_path, 'r') as f:

-             build_one.modulemd = f.read()

+             build_one.modulemd = to_text_type(f.read())

          build_one.koji_tag = 'module-testmodule-master-20180205135154-9c690d0e'

          build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'

          build_one.batch = 2

@@ -1012,7 +1013,7 @@ 

          formatted_testmodule_yml_path = os.path.join(

              current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')

          with open(formatted_testmodule_yml_path, 'r') as f:

-             build_one.modulemd = f.read()

+             build_one.modulemd = to_text_type(f.read())

          build_one.koji_tag = 'module-testmodule-master-20180205135154-6ef9a711'

          build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'

          build_one.batch = 2

@@ -31,6 +31,7 @@ 

  except ImportError:

      import xmlrpc.client as xmlrpclib

  from collections import OrderedDict

+ from module_build_service.utils import to_text_type

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos

@@ -266,7 +267,7 @@ 

              xmd = glib.from_variant_dict(mmd.get_xmd())

              xmd["mbs_options"] = {"blocked_packages": ["foo", "bar", "new"]}

              mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          builder = FakeKojiModuleBuilder(owner=self.module.owner,

                                          module=self.module,

@@ -448,14 +449,14 @@ 

              xmd = glib.from_variant_dict(mmd.get_xmd())

              xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}

              mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          if custom_whitelist:

              mmd = self.module.mmd()

              opts = mmd.get_buildopts()

              opts.set_rpm_whitelist(['custom1', 'custom2'])

              mmd.set_buildopts(opts)

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          if repo_include_all is False:

              mmd = self.module.mmd()

@@ -464,7 +465,7 @@ 

              mbs_options["repo_include_all"] = False

              xmd["mbs_options"] = mbs_options

              mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          if override_arches:

              mmd = self.module.mmd()

@@ -473,7 +474,7 @@ 

              mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}

              xmd["mbs"] = mbs_options

              mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          builder = FakeKojiModuleBuilder(

              owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo',

@@ -532,7 +533,7 @@ 

              xmd = glib.from_variant_dict(mmd.get_xmd())

              xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}

              mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = mmd.dumps()

+             self.module.modulemd = to_text_type(mmd.dumps())

  

          builder = FakeKojiModuleBuilder(

              owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo',

@@ -4,6 +4,7 @@ 

  import tempfile

  import shutil

  from textwrap import dedent

+ from module_build_service.utils import to_text_type

  

  import kobo.rpmlib

  

@@ -92,7 +93,7 @@ 

              name="mbs-testmodule",

              stream="test",

              version="20171027111452",

-             modulemd=mmd.dumps(),

+             modulemd=to_text_type(mmd.dumps()),

              scmurl="file:///testdir",

              username="test",

          )

file modified
+21 -19

@@ -21,11 +21,13 @@ 

  # Written by Stanislav Ochotnicky <sochotnicky@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

  

+ import io

  import pytest

  import json

  

  import os

  from os import path

+ from module_build_service.utils import to_text_type

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos # noqa

@@ -192,21 +194,21 @@ 

      def test_prepare_file_directory(self):

          """ Test preparation of directory with output files """

          dir_path = self.cg._prepare_file_directory()

-         with open(path.join(dir_path, "modulemd.txt")) as mmd:

-             assert len(mmd.read()) == 1134

+         with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd:

+             assert len(mmd.read()) == 1136

  

      def test_prepare_file_directory_per_arch_mmds(self):

          """ Test preparation of directory with output files """

          self.cg.arches = ["x86_64", "i686"]

          dir_path = self.cg._prepare_file_directory()

-         with open(path.join(dir_path, "modulemd.txt")) as mmd:

-             assert len(mmd.read()) == 1134

+         with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd:

+             assert len(mmd.read()) == 1136

  

-         with open(path.join(dir_path, "modulemd.x86_64.txt")) as mmd:

-             assert len(mmd.read()) == 257

+         with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd:

+             assert len(mmd.read()) == 259

  

-         with open(path.join(dir_path, "modulemd.i686.txt")) as mmd:

-             assert len(mmd.read()) == 255

+         with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd:

+             assert len(mmd.read()) == 257

  

      @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

@@ -274,17 +276,17 @@ 

      @patch("module_build_service.builder.KojiContentGenerator.open", create=True)

      def test_get_arch_mmd_output(self, patched_open):

          patched_open.return_value = mock_open(

-             read_data=self.cg.mmd.encode("utf-8")).return_value

+             read_data=self.cg.mmd).return_value

          ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64")

          assert ret == {

              'arch': 'x86_64',

              'buildroot_id': 1,

-             'checksum': 'bf1615b15f6a0fee485abe94af6b56b6',

+             'checksum': '96b7739ffa3918e6ac3e3bd422b064ea',

              'checksum_type': 'md5',

              'components': [],

              'extra': {'typeinfo': {'module': {}}},

              'filename': 'modulemd.x86_64.txt',

-             'filesize': 1134,

+             'filesize': 1136,

              'type': 'file'

          }

  

@@ -294,7 +296,7 @@ 

          rpm_artifacts = mmd.get_rpm_artifacts()

          rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")

          mmd.set_rpm_artifacts(rpm_artifacts)

-         mmd_data = mmd.dumps().encode("utf-8")

+         mmd_data = to_text_type(mmd.dumps())

  

          patched_open.return_value = mock_open(

              read_data=mmd_data).return_value

@@ -323,7 +325,7 @@ 

          assert ret == {

              'arch': 'x86_64',

              'buildroot_id': 1,

-             'checksum': '1bcc38b6f19285b3656b84a0443f46d2',

+             'checksum': '502e46889affec24d98a281289104d4d',

              'checksum_type': 'md5',

              'components': [{u'arch': 'x86_64',

                              u'epoch': '12',

@@ -334,7 +336,7 @@ 

                              u'version': '4.3.5'}],

              'extra': {'typeinfo': {'module': {}}},

              'filename': 'modulemd.x86_64.txt',

-             'filesize': 315,

+             'filesize': 317,

              'type': 'file'

          }

  

@@ -546,8 +548,8 @@ 

                  component.set_multilib(multilib_set)

  

              mmd.add_rpm_component(component)

-             self.cg.module.modulemd = mmd.dumps()

-             self.cg.modulemd = mmd.dumps()

+             self.cg.module.modulemd = to_text_type(mmd.dumps())

+             self.cg.modulemd = to_text_type(mmd.dumps())

  

      @pytest.mark.parametrize("devel", (False, True))

      def test_fill_in_rpms_list(self, devel):

@@ -855,10 +857,10 @@ 

          mmd.set_xmd(glib.dict_values({"mbs": {

              "commit": "foo",

              "scmurl": "git://localhost/modules/foo.git#master"}}))

-         self.cg.module.modulemd = mmd.dumps()

+         self.cg.module.modulemd = to_text_type(mmd.dumps())

          file_dir = self.cg._prepare_file_directory()

-         with open(path.join(file_dir, "modulemd.src.txt")) as mmd:

-             assert len(mmd.read()) == 1337

+         with io.open(path.join(file_dir, "modulemd.src.txt"), encoding="utf-8") as mmd:

+             assert len(mmd.read()) == 1339

  

      def test_finalize_mmd_devel(self):

          self.cg.devel = True

@@ -625,8 +625,8 @@ 

                  }

              ],

              "arch": "noarch",

-             "filesize": 1134,

-             "checksum": "bf1615b15f6a0fee485abe94af6b56b6",

+             "filesize": 1136,

+             "checksum": "96b7739ffa3918e6ac3e3bd422b064ea",

              "checksum_type": "md5",

              "type": "file",

              "extra": {

@@ -652,7 +652,7 @@ 

                      "version": "2",

                      "module_build_service_id": 2,

                      "content_koji_tag": "module-nginx-1.2",

-                     "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"

+                     "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service ’\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"

                  }

              }

          },

@@ -625,8 +625,8 @@ 

                  }

              ],

              "arch": "noarch",

-             "filesize": 1134,

-             "checksum": "bf1615b15f6a0fee485abe94af6b56b6",

+             "filesize": 1136,

+             "checksum": "96b7739ffa3918e6ac3e3bd422b064ea",

              "checksum_type": "md5",

              "type": "file",

              "extra": {

@@ -661,7 +661,7 @@ 

                      "version": "2",

                      "module_build_service_id": 2,

                      "content_koji_tag": "module-nginx-1.2",

-                     "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"

+                     "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service ’\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"

                  }

              }

          },

@@ -21,6 +21,7 @@ 

  # Written by Ralph Bean <rbean@redhat.com>

  

  import os

+ from module_build_service.utils import to_text_type

  

  from tests.test_models import init_data, module_build_from_modulemd

  from tests import (init_data as init_data_contexts, clean_database)

@@ -66,7 +67,7 @@ 

              os.path.dirname(__file__), '..', 'staged_data', 'testmodule_dependencies.yaml')

          mmd = Modulemd.Module.new_from_file(yaml_path)

          mmd.upgrade()

-         build.modulemd = mmd.dumps()

+         build.modulemd = to_text_type(mmd.dumps())

          (build.ref_build_context, build.build_context, build.runtime_context,

           build.context) = ModuleBuild.contexts_from_mmd(build.modulemd)

          assert build.ref_build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'

@@ -85,7 +86,7 @@ 

          mmd.upgrade()

          with make_session(conf) as session:

              for i in range(3):

-                 build = module_build_from_modulemd(mmd.dumps())

+                 build = module_build_from_modulemd(to_text_type(mmd.dumps()))

                  build.build_context = 'f6e2aeec7576196241b9afa0b6b22acf2b6873d' + str(i)

                  build.runtime_context = 'bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c' + str(i)

                  session.add(build)

@@ -25,6 +25,7 @@ 

  from datetime import datetime

  from mock import patch, PropertyMock

  import pytest

+ from module_build_service.utils import to_text_type

  

  import module_build_service.resolver as mbs_resolver

  from module_build_service import app, db, models, glib, utils, Modulemd

@@ -65,7 +66,7 @@ 

              time_submitted=datetime(2018, 11, 15, 16, 8, 18),

              time_modified=datetime(2018, 11, 15, 16, 19, 35),

              rebuild_strategy='changed-and-after',

-             modulemd=mmd.dumps()

+             modulemd=to_text_type(mmd.dumps())

          )

          build.buildrequires.append(platform_f300103)

          db.session.add(build)

@@ -105,7 +106,7 @@ 

              xmd = glib.from_variant_dict(mmd.get_xmd())

              xmd['mbs']['buildrequires'] = {}

              mmd.set_xmd(glib.dict_values(xmd))

-             module.modulemd = mmd.dumps()

+             module.modulemd = to_text_type(mmd.dumps())

              db.session.add(module)

              db.session.commit()

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')

@@ -134,7 +135,7 @@ 

              'version': '20180205135154'

          }

          mmd.set_xmd(glib.dict_values(xmd))

-         module.modulemd = mmd.dumps()

+         module.modulemd = to_text_type(mmd.dumps())

          module.name = 'testmodule2'

          module.version = str(mmd.get_version())

          module.koji_tag = 'module-ae2adf69caf0e1b6'

@@ -24,6 +24,7 @@ 

  

  from mock import patch, PropertyMock

  from gi.repository import GLib

+ from module_build_service.utils import to_text_type

  

  from tests import conf, clean_database

  from tests.test_views.test_views import FakeSCM

@@ -42,7 +43,7 @@ 

          testmodule_yml_path = os.path.join(

              self.staged_data_dir, 'testmodule_init.yaml')

          with open(testmodule_yml_path, 'r') as f:

-             yaml = f.read()

+             yaml = to_text_type(f.read())

          scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77'

          clean_database()

          with make_session(conf) as session:

@@ -78,7 +79,7 @@ 

          filter_list.add("foo")

          filter_list.add("bar")

          mmd.set_rpm_filter(filter_list)

-         platform_build.modulemd = mmd.dumps()

+         platform_build.modulemd = to_text_type(mmd.dumps())

          db.session.commit()

  

          msg = module_build_service.messaging.MBSModule(

@@ -137,7 +138,7 @@ 

          includedmodules_yml_path = os.path.join(

              self.staged_data_dir, 'includedmodules.yaml')

          with open(includedmodules_yml_path, 'r') as f:

-             yaml = f.read()

+             yaml = to_text_type(f.read())

          scmurl = 'git://pkgs.domain.local/modules/includedmodule?#da95886'

          with make_session(conf) as session:

              ModuleBuild.create(

file modified
+13 -11

@@ -18,12 +18,14 @@ 

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

  

+ import io

  import tempfile

  from os import path, mkdir

  from shutil import copyfile, rmtree

  from datetime import datetime

  from werkzeug.datastructures import FileStorage

  from mock import patch

+ from module_build_service.utils import to_text_type

  import module_build_service.utils

  import module_build_service.scm

  from module_build_service import models, conf

@@ -93,7 +95,7 @@ 

              mmd = second_module_build.mmd()

              mmd.get_rpm_components()['tangerine'].set_ref(

                  '00ea1da4192a2030f9ae023de3b3143ed647bbab')

-             second_module_build.modulemd = mmd.dumps()

+             second_module_build.modulemd = to_text_type(mmd.dumps())

              second_module_changed_component = models.ComponentBuild.query.filter_by(

                  package=changed_component, module_id=3).one()

              second_module_changed_component.ref = '00ea1da4192a2030f9ae023de3b3143ed647bbab'

@@ -133,7 +135,7 @@ 

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = second_module_build.mmd()

          mmd.set_rpm_buildopts({'macros': '%my_macro 1'})

-         second_module_build.modulemd = mmd.dumps()

+         second_module_build.modulemd = to_text_type(mmd.dumps())

          db.session.commit()

  

          plc_rv = module_build_service.utils.get_reusable_component(

@@ -153,7 +155,7 @@ 

              arches = Modulemd.SimpleSet()

              arches.set(['i686'])

              mmd.get_rpm_components()['tangerine'].set_arches(arches)

-             second_module_build.modulemd = mmd.dumps()

+             second_module_build.modulemd = to_text_type(mmd.dumps())

          if set_database_arch:       # set architecture for build in database

              second_module_changed_component = models.ComponentBuild.query.filter_by(

                  package='tangerine', module_id=2).one()

@@ -161,7 +163,7 @@ 

              arches = Modulemd.SimpleSet()

              arches.set(['i686'])

              mmd.get_rpm_components()['tangerine'].set_arches(arches)

-             second_module_changed_component.module_build.modulemd = mmd.dumps()

+             second_module_changed_component.module_build.modulemd = to_text_type(mmd.dumps())

              db.session.add(second_module_changed_component)

              db.session.commit()

  

@@ -179,7 +181,7 @@ 

          xmd['mbs']['buildrequires']['platform']['ref'] = \

              'da39a3ee5e6b4b0d3255bfef95601890afd80709'

          mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = mmd.dumps()

+         second_module_build.modulemd = to_text_type(mmd.dumps())

          second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'

          second_module_build.rebuild_strategy = rebuild_strategy

          db.session.commit()

@@ -209,7 +211,7 @@ 

          xmd = glib.from_variant_dict(mmd.get_xmd())

          xmd['mbs']['buildrequires']['platform']['stream'] = 'different'

          mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = mmd.dumps()

+         second_module_build.modulemd = to_text_type(mmd.dumps())

          second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'

          second_module_build.rebuild_strategy = rebuild_strategy

          db.session.commit()

@@ -240,7 +242,7 @@ 

              }

          }

          mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = mmd.dumps()

+         second_module_build.modulemd = to_text_type(mmd.dumps())

          second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'

          db.session.commit()

  

@@ -268,13 +270,13 @@ 

          module_dir = tempfile.mkdtemp()

          module = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = module.mmd()

-         modulemd_yaml = mmd.dumps()

+         modulemd_yaml = to_text_type(mmd.dumps())

          modulemd_file_path = path.join(module_dir, "testmodule.yaml")

  

          username = "test"

          stream = "dev"

  

-         with open(modulemd_file_path, "w") as fd:

+         with io.open(modulemd_file_path, "w", encoding='utf-8') as fd:

              fd.write(modulemd_yaml)

  

          with open(modulemd_file_path, "rb") as fd:

@@ -530,7 +532,7 @@ 

              module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)

              module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)

              module_build.rebuild_strategy = 'changed-and-after'

-             module_build.modulemd = mmd.dumps()

+             module_build.modulemd = to_text_type(mmd.dumps())

              db.session.add(module_build)

              db.session.commit()

              # Rename the the modulemd to include

@@ -578,7 +580,7 @@ 

              module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)

              module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)

              module_build.rebuild_strategy = 'changed-and-after'

-             module_build.modulemd = mmd.dumps()

+             module_build.modulemd = to_text_type(mmd.dumps())

              db.session.add(module_build)

              db.session.commit()

  

@@ -32,6 +32,7 @@ 

  from requests.utils import quote

  import hashlib

  import pytest

+ from module_build_service.utils import to_text_type

  

  from tests import app, init_data, clean_database, reuse_component_init_data

  from tests import read_staged_data

@@ -183,7 +184,7 @@ 

          assert data['runtime_context'] is None

          assert data['id'] == 2

          with open(path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:

-             assert data['modulemd'] == mmd.read()

+             assert data['modulemd'] == to_text_type(mmd.read())

          assert data['name'] == 'nginx'

          assert data['owner'] == 'Moe Szyslak'

          assert data['scmurl'] == ('git://pkgs.domain.local/modules/nginx'

@@ -1614,7 +1615,7 @@ 

              buildrequires['modulea'] = br_modulea

              buildrequires['moduleb'] = br_moduleb

              mmd.set_xmd(dict_values(xmd))

-             build.modulemd = mmd.dumps()

+             build.modulemd = to_text_type(mmd.dumps())

              session.commit()

  

          rv = self.client.get('/module-build-service/1/module-builds/{}'.format(build.id))

This commit introduces new to_unicode helper method and calls it
for return value of all mmd.dumps() calls. That way, we always
end up with proper unicode string represntation on both python
major versions.

This commit also adds unicode character to description of all
the yaml files we use in the tests so we can be sure MBS can
handle unicode characters properly.

This might be temporary fix, depending on the result of discussion
at https://github.com/fedora-modularity/libmodulemd/issues/184.

Can we call it to_text_type ? Unicode is python2 lingo.

rebased onto 6e6b08adb86fb681b0bda78c3d37f6baf2592755

2 months ago

If you want to check the python version, use six.PY2. But it might just be simpler to wrap the whole thing in a try/except clause to catch other cases as well. (What if runtime is python3 and s is of type bytes? We want to convert to text type in that case too.

rebased onto 73015201d00b91feab53252fca56b7e4fd23137d

2 months ago

pretty please pagure-ci rebuild

2 months ago

@mprahl, hm, looks like for some reason the "tox -e py3" runs with py2 on my laptop. I've workarounded that by using "py-test-3" in the tox.ini, but not sure what's going on...

Anyway, I will fix the tests.

rebased onto fcc8a83c34a8485342e033802d58737217a6e15e

2 months ago

Fixed the tests, it also uses try/except now in to_text_type.

rebased onto 8dbe06a9065689df2c03125347c7ec2f7f41325a

2 months ago

pretty please pagure-ci rebuild

2 months ago

rb is preferred if possible because io.open and open in Python 3 will automatically convert all line endings to Unix-style line endings by default when using r (there is a way to disable that).

See the following for context:
https://pagure.io/fm-orchestrator/pull-request/1119#

rebased onto cca15637d17d82102c9f84c4d1bb74110850df83

2 months ago

rebased onto ec9faf4f6c37a0d2cb00a999ec91bd1fa2177d43

2 months ago

rebased onto 90e6842b1c7b2a36b7f881941594e28415a33099

2 months ago

rebased onto 5ff271093fe1fd14b8f6a7397f1740468573ad56

2 months ago

It now passes the tests:

https://ci.centos.org/job/fm-orchestrator-pr/383/

But it seems the results are not sent to Pagure.
@mprahl , can you check this pr again?

rebased onto 40f23b6

2 months ago

Pull-Request has been merged by mprahl

2 months ago
Changes Summary 37
+6 -6
file changed
module_build_service/builder/KojiContentGenerator.py
+2 -1
file changed
module_build_service/scheduler/handlers/modules.py
+12 -1
file changed
module_build_service/utils/general.py
+3 -2
file changed
module_build_service/utils/submit.py
+11 -10
file changed
tests/__init__.py
+1 -1
file changed
tests/staged_data/bad.yaml
+1 -1
file changed
tests/staged_data/fakemodule.yaml
+1 -1
file changed
tests/staged_data/formatted_python3-no-components.yaml
+1 -1
file changed
tests/staged_data/formatted_testmodule-more-components.yaml
+1 -1
file changed
tests/staged_data/formatted_testmodule.yaml
+1 -1
file changed
tests/staged_data/includedmodules.yaml
+1 -1
file changed
tests/staged_data/nginx_mmd.yaml
+1 -1
file changed
tests/staged_data/platform.yaml
+1 -1
file changed
tests/staged_data/python3-no-components.yaml
+1 -1
file changed
tests/staged_data/testmodule-more-components.yaml
+1 -1
file changed
tests/staged_data/testmodule-no-base-module.yaml
+1 -1
file changed
tests/staged_data/testmodule-no-deps.yaml
+1 -1
file changed
tests/staged_data/testmodule-version-set.yaml
+1 -1
file changed
tests/staged_data/testmodule-with-filters.yaml
+1 -1
file changed
tests/staged_data/testmodule-wrong-stream.yaml
+1 -1
file changed
tests/staged_data/testmodule.yaml
+1 -1
file changed
tests/staged_data/testmodule_dependencies.yaml
+1 -1
file changed
tests/staged_data/testmodule_init.yaml
+1 -1
file changed
tests/staged_data/testmodule_mse.yaml
+1 -1
file changed
tests/staged_data/testmodule_platform_f290000.yaml
+1 -1
file changed
tests/staged_data/testmodule_v2.yaml
+4 -3
file changed
tests/test_build/test_build.py
+7 -6
file changed
tests/test_builder/test_koji.py
+2 -1
file changed
tests/test_builder/test_mock.py
+21 -19
file changed
tests/test_content_generator.py
+3 -3
file changed
tests/test_get_generator_json_expected_output.json
+3 -3
file changed
tests/test_get_generator_json_expected_output_with_log.json
+3 -2
file changed
tests/test_models/test_models.py
+4 -3
file changed
tests/test_resolver/test_db.py
+4 -3
file changed
tests/test_scheduler/test_module_init.py
+13 -11
file changed
tests/test_utils/test_utils.py
+3 -2
file changed
tests/test_views/test_views.py