#1252 Migrate to libmodulemd v2
Merged 4 years ago by mprahl. Opened 4 years ago by mprahl.

file modified
+7 -1
@@ -2,9 +2,11 @@ 

  # vi: set ft=ruby ts=2 sw=2 ai et:

  

  $script = <<SCRIPT

+     set -e

      grep -q '^127\.0\.0\.1 fedmsg-relay$' /etc/hosts || echo "127.0.0.1 fedmsg-relay" >> /etc/hosts

      echo "export MODULE_BUILD_SERVICE_DEVELOPER_ENV=1" > /etc/profile.d/module_build_service_developer_env.sh

      source /etc/profile.d/module_build_service_developer_env.sh

+     # Unpin the libmodulemd RPMs and replace with `python2-libmodulemd` after FEDORA-2019-b1cdf17e35

      dnf install -y \

          fedmsg-hub \

          fedmsg-relay \
@@ -13,9 +15,9 @@ 

          gcc-c++ \

          git \

          koji \

+         krb5-devel \

          krb5-workstation \

          libffi-devel \

-         libmodulemd \

          mock-scm \

          openssl-devel \

          python \
@@ -24,6 +26,9 @@ 

          python-docutils \

          python-flask \

          python-gobject-base \

+         https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/python2-libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+         https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+         https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd1-1.8.10-1.fc29.x86_64.rpm \

          python-m2ext \

          python-mock \

          python-qpid \
@@ -55,6 +60,7 @@ 

  SCRIPT

  

  $make_devenv = <<DEVENV

+   set -e

    env_dir=~/devenv

    pip=${env_dir}/bin/pip

    py=${env_dir}/bin/python

file modified
+5 -1
@@ -4,6 +4,8 @@ 

  RUN yum -y update

  RUN yum -y install epel-release yum-utils

  RUN yum-config-manager --add-repo https://kojipkgs.fedoraproject.org/repos-dist/epel7Server-infra/latest/x86_64/

+ # Unpin the libmodulemd RPMs and replace with `python2-libmodulemd` after 2.4.0 is in

+ # epel7Server-infra

  RUN yum -y install \

      --nogpgcheck \

      --setopt=deltarpm=0 \
@@ -17,7 +19,6 @@ 

      git \

      kobo \

      kobo-rpmlib \

-     libmodulemd \

      python-backports-ssl_match_hostname \

      python-dnf \

      python-dogpile-cache \
@@ -29,6 +30,9 @@ 

      python-futures \

      python-koji \

      python-ldap3 \

+     https://kojipkgs.fedoraproject.org/work/tasks/3269/34833269/libmodulemd-2.4.0-1.el7.infra.x86_64.rpm \

+     https://kojipkgs.fedoraproject.org/work/tasks/3269/34833269/python2-libmodulemd-2.4.0-1.el7.infra.x86_64.rpm \

+     https://kojipkgs.fedoraproject.org/work/tasks/5596/34835596/libmodulemd1-1.8.10-1.el7.infra.x86_64.rpm \

      python-mock \

      python-munch \

      python-pip \

file modified
+4 -1
@@ -1,6 +1,7 @@ 

  FROM fedora:29

  

  WORKDIR /build

+ # Unpin the libmodulemd RPMs and replace with `python3-libmodulemd` after FEDORA-2019-b1cdf17e35

  RUN dnf -y install \

      --nogpgcheck \

      --setopt=deltarpm=0 \
@@ -11,7 +12,6 @@ 

      python3-fedmsg \

      python3-kobo-rpmlib \

      python3-rpm \

-     libmodulemd \

      python3-gobject \

      python3-dnf \

      python3-dogpile-cache \
@@ -20,6 +20,9 @@ 

      python3-flask-sqlalchemy \

      python3-koji \

      python3-ldap3 \

+     https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/python3-libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+     https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+     https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd1-1.8.10-1.fc29.x86_64.rpm \

      python3-munch \

      python3-pip \

      python3-prometheus_client \

@@ -46,7 +46,7 @@ 

  from sqlalchemy.pool import StaticPool

  from logging import getLogger

  import gi  # noqa

- gi.require_version("Modulemd", "1.0")  # noqa

+ gi.require_version("Modulemd", "2.0")  # noqa

  from gi.repository import Modulemd  # noqa

  

  from module_build_service.logger import init_logging, ModuleBuildLogs, level_flags, MBSLogger

@@ -42,9 +42,9 @@ 

  import koji

  import pungi.arch

  

- from module_build_service import conf, log, build_logs, Modulemd, glib

+ from module_build_service import conf, log, build_logs, Modulemd

  from module_build_service.scm import SCM

- from module_build_service.utils import to_text_type, load_mmd

+ from module_build_service.utils import to_text_type, load_mmd, mmd_to_str

  

  logging.basicConfig(level=logging.DEBUG)

  
@@ -211,15 +211,7 @@ 

      def __get_tools(self):

          """Return list of tools which are important for reproducing mbs outputs"""

  

-         # TODO: In libmodulemd v1.5, there'll be a property we can check instead

-         # of using RPM

-         try:

-             cmd = ["rpm", "--queryformat", "%{VERSION}", "-q", "libmodulemd"]

-             libmodulemd_version = subprocess.check_output(cmd, universal_newlines=True).strip()

-         except subprocess.CalledProcessError:

-             libmodulemd_version = "unknown"

- 

-         return [{"name": "libmodulemd", "version": libmodulemd_version}]

+         return [{"name": "libmodulemd", "version": Modulemd.get_version()}]

  

      def _koji_rpms_in_tag(self, tag):

          """ Return the list of koji rpms in a tag. """
@@ -417,7 +409,7 @@ 

              # to generate list of components for this architecture.

              # We cannot simply use the data from MMD here without `rpms_dict`, because

              # RPM sigmd5 signature is not stored in MMD.

-             for rpm in mmd.get_rpm_artifacts().get():

+             for rpm in mmd.get_rpm_artifacts():

                  if rpm not in self.rpms_dict:

                      raise RuntimeError(

                          "RPM %s found in the final modulemd but not in Koji tag." % rpm)
@@ -471,11 +463,11 @@ 

          should be included in a final MMD file for given arch.

          """

          # Check the "whitelist" buildopts section of MMD.

-         # When "whitelist" is defined, it overrides component names in

-         # `mmd.get_rpm_components()`. The whitelist is used when module needs to build

+         # When "whitelist" is defined, it overrides component names from

+         # `mmd.get_rpm_component(component)`. The whitelist is used when module needs to build

          # package with different SRPM name than the package name. This is case for example

          # for software collections where SRPM name can be "httpd24-httpd", but package name

-         # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the

+         # is still "httpd". In this case, the component would contain "httpd", but the

          # rpm["srpm_name"] would be "httpd24-httpd".

          srpm = rpm["srpm_name"]

          whitelist = None
@@ -490,21 +482,20 @@ 

          # If there is no whitelist, just check that the SRPM name we have here

          # exists in the list of components.

          # In theory, there should never be situation where modular tag contains

-         # some RPM built from SRPM not included in get_rpm_components() or in whitelist,

+         # some RPM built from SRPM not included in get_rpm_component_names() or in whitelist,

          # but the original Pungi code checked for this case.

-         if not whitelist and srpm not in mmd.get_rpm_components().keys():

+         if not whitelist and srpm not in mmd.get_rpm_component_names():

              return False

  

          # Do not include this RPM if it is filtered.

-         if rpm["name"] in mmd.get_rpm_filter().get():

+         if rpm["name"] in mmd.get_rpm_filters():

              return False

  

          # Skip the rpm if it's built for multilib arch, but

          # multilib is not enabled for this srpm in MMD.

          try:

-             mmd_component = mmd.get_rpm_components()[srpm]

-             multilib = mmd_component.get_multilib()

-             multilib = multilib.get() if multilib else set()

+             mmd_component = mmd.get_rpm_component(srpm)

+             multilib = set(mmd_component.get_multilib_arches())

              # The `multilib` set defines the list of architectures for which

              # the multilib is enabled.

              #
@@ -513,7 +504,7 @@ 

              # architectures.

              if arch not in multilib and rpm["arch"] in multilib_arches:

                  return False

-         except KeyError:

+         except AttributeError:

              # TODO: This exception is raised only when "whitelist" is used.

              # Since components in whitelist have different names than ones in

              # components list, we won't find them there.
@@ -532,7 +523,7 @@ 

          Fills in the list of built RPMs in architecture specific `mmd` for `arch`

          using the data from `self.rpms_dict` as well as the content licenses field.

  

-         :param Modulemd.Module mmd: MMD to add built RPMs to.

+         :param Modulemd.ModuleStream mmd: MMD to add built RPMs to.

          :param str arch: Architecture for which to add RPMs.

          :rtype: Modulemd.Module

          :return: MMD with built RPMs filled in.
@@ -552,11 +543,11 @@ 

          # from ExcludeArch tag. Multilib should not be enabled here.

          exclusive_arches = pungi.arch.get_valid_arches(arch, multilib=False, add_noarch=False)

  

-         # Modulemd.SimpleSet into which we will add the RPMs.

-         rpm_artifacts = Modulemd.SimpleSet()

+         # A set into which we will add the RPMs.

+         rpm_artifacts = set()

  

-         # Modulemd.SimpleSet into which we will add licenses of all RPMs.

-         rpm_licenses = Modulemd.SimpleSet()

+         # A set into which we will add licenses of all RPMs.

+         rpm_licenses = set()

  

          # RPM name suffixes for debug RPMs.

          debug_suffixes = ("-debuginfo", "-debugsource")
@@ -669,8 +660,17 @@ 

              for source_nevra in non_devel_source_rpms.values():

                  rpm_artifacts.add(source_nevra)

  

-         mmd.set_content_licenses(rpm_licenses)

-         mmd.set_rpm_artifacts(rpm_artifacts)

+         # There is no way to replace the licenses, so remove any extra licenses

If preferred, I can pretty easily add a .clear_content_license() and .clear_rpm_artifacts() method to libmodulemd v2.

@sgallagh that would be helpful. Thank you!

+         for license_to_remove in (set(mmd.get_content_licenses()) - rpm_licenses):

+             mmd.remove_content_license(license_to_remove)

+         for license_to_add in (rpm_licenses - set(mmd.get_content_licenses())):

+             mmd.add_content_license(license_to_add)

+ 

+         # There is no way to replace the RPM artifacts, so remove any extra RPM artifacts

+         for artifact_to_remove in (set(mmd.get_rpm_artifacts()) - rpm_artifacts):

+             mmd.remove_rpm_artifact(artifact_to_remove)

+         for artifact_to_add in (rpm_artifacts - set(mmd.get_rpm_artifacts())):

+             mmd.add_rpm_artifact(artifact_to_add)

          return mmd

  

      def _sanitize_mmd(self, mmd):
@@ -685,17 +685,18 @@ 

          :return: Sanitized Modulemd instance.

          """

          # Remove components.repository and components.cache.

-         for pkg in mmd.get_rpm_components().values():

+         for pkg_name in mmd.get_rpm_component_names():

+             pkg = mmd.get_rpm_component(pkg_name)

              if pkg.get_repository():

                  pkg.set_repository(None)

              if pkg.get_cache():

                  pkg.set_cache(None)

  

          # Remove 'mbs' XMD section.

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          if "mbs" in xmd:

              del xmd["mbs"]

-             mmd.set_xmd(glib.dict_values(xmd))

+             mmd.set_xmd(xmd)

  

          return mmd

  
@@ -710,15 +711,16 @@ 

          """

          mmd = self._sanitize_mmd(self.module.mmd())

          if self.devel:

+             # Set the new name

+             mmd = mmd.copy(mmd.get_module_name() + "-devel")

+ 

              # Depend on the actual module

              for dep in mmd.get_dependencies():

-                 dep.add_requires_single(mmd.get_name(), mmd.get_stream())

- 

-             # Set the new name

-             mmd.set_name(mmd.get_name() + "-devel")

+                 dep.add_runtime_stream(mmd.get_module_name(), mmd.get_stream_name())

  

              # Delete API and profiles

-             mmd.set_rpm_api(Modulemd.SimpleSet())

+             for rpm in mmd.get_rpm_api():

+                 mmd.remove_rpm_api(rpm)

              mmd.clear_profiles()

  

          # Set the "Arch" field in mmd.
@@ -726,7 +728,7 @@ 

          # Fill in the list of built RPMs.

          mmd = self._fill_in_rpms_list(mmd, arch)

  

-         return to_text_type(mmd.dumps())

+         return mmd_to_str(mmd)

  

      def _download_source_modulemd(self, mmd, output_path):

          """
@@ -739,7 +741,7 @@ 

          :param str output_path: Full path to file into which the original modulemd

              file will be stored.

          """

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          commit = xmd.get("mbs", {}).get("commit")

          scmurl = xmd.get("mbs", {}).get("scmurl")

          if not commit or not scmurl:

@@ -404,9 +404,9 @@ 

          )

  

          modulemd_macros = ""

-         rpm_buildopts = mmd.get_rpm_buildopts()

-         if rpm_buildopts:

-             modulemd_macros = rpm_buildopts.get("macros")

+         buildopts = mmd.get_buildopts()

+         if buildopts:

+             modulemd_macros = buildopts.get_rpm_macros() or ""

  

          macros_content = textwrap.dedent("""

              # General macros set by MBS
@@ -531,15 +531,19 @@ 

          self.module_build_tag = self._koji_create_tag(

              self.tag_name + "-build", self.arches, perm="admin")

  

-         self._koji_whitelist_packages(

-             self.mmd.props.buildopts.props.rpm_whitelist or self.components)

+         buildopts = self.mmd.get_buildopts()

+         if buildopts:

+             rpm_whitelist = buildopts.get_rpm_whitelist()

+         else:

+             rpm_whitelist = self.components

+         self._koji_whitelist_packages(rpm_whitelist)

  

          # If we have just created the build tag in this buildroot_connect call, block all

          # the components in `blocked_packages` list. We want to do that just once, because

          # there might be some unblocked packages later and we would block them again...

          if not build_tag_exists:

              xmd = self.mmd.get_xmd()

-             if "mbs_options" in xmd.keys() and "blocked_packages" in xmd["mbs_options"].keys():

+             if "blocked_packages" in xmd.get("mbs_options", {}):

                  self._koji_block_packages(xmd["mbs_options"]["blocked_packages"])

  

          @module_build_service.utils.retry(wait_on=SysCallError, interval=5)
@@ -591,7 +595,7 @@ 

          build_tag = self._get_tag(self.module_build_tag)["id"]

  

          xmd = self.mmd.get_xmd()

-         if "mbs_options" in xmd.keys() and "blocked_packages" in xmd["mbs_options"].keys():

+         if "blocked_packages" in xmd.get("mbs_options", {}):

              packages = [kobo.rpmlib.parse_nvr(nvr)["name"] for nvr in artifacts]

              packages = [

                  package for package in packages
@@ -842,8 +846,7 @@ 

  

              # disabled by default, wouldn't work until Koji issue #1158 is done

              if conf.allow_arch_override:

-                 build_opts["arch_override"] = (

-                     self.mmd.get_rpm_components()[artifact_name].get_arches().get())

+                 build_opts["arch_override"] = self.mmd.get_rpm_component(artifact_name).get_arches()

  

              task_id = self.koji_session.build(

                  source, module_target, build_opts, priority=self.build_priority)
@@ -1006,7 +1009,7 @@ 

          opts["extra"] = copy.deepcopy(conf.koji_tag_extra_opts)

  

          xmd = self.mmd.get_xmd()

-         if "mbs_options" in xmd.keys() and "repo_include_all" in xmd["mbs_options"].keys():

+         if "repo_include_all" in xmd.get("mbs_options", {}):

              opts["extra"]["repo_include_all"] = xmd["mbs_options"]["repo_include_all"]

  

          # edit tag with opts
@@ -1256,7 +1259,12 @@ 

          """

          with models.make_session(conf) as db_session:

              build = models.ModuleBuild.get_build_from_nsvc(

-                 db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context())

+                 db_session,

+                 mmd.get_module_name(),

+                 mmd.get_stream_name(),

+                 mmd.get_version(),

+                 mmd.get_context()

+             )

              koji_session = KojiModuleBuilder.get_session(conf, login=False)

              rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]

              nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)

@@ -32,7 +32,7 @@ 

  import subprocess

  import threading

  

- from module_build_service import conf, log, Modulemd

+ from module_build_service import conf, log

  import module_build_service.scm

  import module_build_service.utils

  import module_build_service.scheduler
@@ -170,7 +170,7 @@ 

  

          # Generate the mmd the same way as pungi does.

          m1_mmd = self.module.mmd()

-         artifacts = Modulemd.SimpleSet()

+         artifacts = set()

  

          rpm_files = [f for f in os.listdir(self.resultsdir) if f.endswith(".rpm")]

  
@@ -195,14 +195,18 @@ 

  

                  if self.module.last_batch_id() == self.module.batch:

                      # If RPM is filtered-out, do not add it to artifacts list.

-                     if name in m1_mmd.get_rpm_filter().get():

+                     if name in m1_mmd.get_rpm_filters():

                          continue

  

                  pkglist_f.write(rpm_file + "\n")

                  artifacts.add("{}-{}:{}-{}.{}".format(name, epoch, version, release, arch))

  

          pkglist_f.close()

-         m1_mmd.set_rpm_artifacts(artifacts)

+         # There is no way to replace the RPM artifacts, so remove any extra RPM artifacts

+         for artifact_to_remove in (set(m1_mmd.get_rpm_artifacts()) - artifacts):

+             m1_mmd.remove_rpm_artifact(artifact_to_remove)

+         for artifact_to_add in (artifacts - set(m1_mmd.get_rpm_artifacts())):

+             m1_mmd.add_rpm_artifact(artifact_to_add)

  

          # Generate repo.

          execute_cmd(["/usr/bin/createrepo_c", "--pkglist", pkglist, path])
@@ -340,20 +344,23 @@ 

              # repo file.

              elif source.startswith("repofile://"):

                  # For the base module, we want to include all the `conf.base_module_repofiles`.

-                 if len(mmds) == 1 and mmds[0].get_name() in conf.base_module_names:

+                 if len(mmds) == 1 and mmds[0].get_module_name() in conf.base_module_names:

                      for repofile in conf.base_module_repofiles:

                          self._add_repo_from_path(repofile)

                      # Also set the platform_id.

                      mmd = mmds[0]

                      self.yum_conf = self.yum_conf.replace(

-                         "$module_platform_id", "%s:%s" % (mmd.get_name(), mmd.get_stream()))

+                         "$module_platform_id",

+                         "%s:%s" % (mmd.get_module_name(), mmd.get_stream_name())

+                     )

                  else:

                      # Add repositories defined in repofile to mock config.

                      repofile = source[len("repofile://"):]

                      self._add_repo_from_path(repofile)

                      # Enabled all the modular dependencies by default in Mock.

                      for mmd in mmds:

-                         self.enabled_modules.append("%s:%s" % (mmd.get_name(), mmd.get_stream()))

+                         self.enabled_modules.append(

+                             "%s:%s" % (mmd.get_module_name(), mmd.get_stream_name()))

                  continue

              else:

                  repo_name = tag = source
@@ -558,10 +565,15 @@ 

          """

          with models.make_session(conf) as db_session:

              build = models.ModuleBuild.get_build_from_nsvc(

-                 db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context())

+                 db_session,

+                 mmd.get_module_name(),

+                 mmd.get_stream_name(),

+                 mmd.get_version(),

+                 mmd.get_context()

+             )

              if build.koji_tag.startswith("repofile://"):

                  # Modules from local repository have already the RPMs filled in mmd.

-                 return list(mmd.get_rpm_artifacts().get())

+                 return mmd.get_rpm_artifacts()

              else:

                  koji_session = KojiModuleBuilder.get_session(conf, login=False)

                  rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]

@@ -1,110 +0,0 @@ 

- # Copyright (c) 2016  Red Hat, Inc.

- #

- # Permission is hereby granted, free of charge, to any person obtaining a copy

- # of this software and associated documentation files (the "Software"), to deal

- # in the Software without restriction, including without limitation the rights

- # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

- # copies of the Software, and to permit persons to whom the Software is

- # furnished to do so, subject to the following conditions:

- #

- # The above copyright notice and this permission notice shall be included in all

- # copies or substantial portions of the Software.

- #

- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

- # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

- # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

- # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

- # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

- # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

- # SOFTWARE.

- #

- """

- GLib Related helper functions

- Originally written by Nils Philippsen in https://pagure.io/modulemd/pull-request/63

- """

- from six import text_type

- from gi.repository import GLib

- 

- 

- def from_variant_dict(d):

-     """ Converts a variant dictionary to a Python dictionary

-     """

-     rv = {}

-     for key, value in d.items():

-         rv[key] = value.unpack()

-     return rv

- 

- 

- def variant_str(s):

-     """ Converts a string to a GLib.Variant

-     """

-     if not isinstance(s, str):

-         raise TypeError("Only strings are supported for scalars")

- 

-     return GLib.Variant("s", s)

- 

- 

- def variant_list(l):

-     """ Converts a list to a GLib.Variant

-     """

-     l_variant = list()

-     for item in l:

-         if item is None:

-             item = ""

-         if type(item) == str:

-             l_variant.append(variant_str(item))

-         elif type(item) == text_type:

-             l_variant.append(variant_str(item.encode("utf-8")))

-         elif type(item) == list:

-             l_variant.append(variant_list(item))

-         elif type(item) == dict:

-             l_variant.append(variant_dict(item))

-         elif type(item) == bool:

-             l_variant.append(variant_bool(item))

-         else:

-             raise TypeError("Cannot convert unknown type")

-     return GLib.Variant("av", l_variant)

- 

- 

- def variant_bool(b):

-     """ Converts a boolean to a GLib.Varant

-     """

-     if not isinstance(b, bool):

-         raise TypeError("Only booleans are supported")

- 

-     return GLib.Variant("b", b)

- 

- 

- def dict_values(d):

-     """ Converts each dictionary value to a GLib.Variant

-     """

-     if not isinstance(d, dict):

-         raise TypeError("Only dictionaries are supported for mappings")

- 

-     d_variant = dict()

-     for k, v in d.items():

-         if v is None:

-             v = ""

-         if type(v) == str:

-             d_variant[k] = variant_str(v)

-         elif type(v) == text_type:

-             d_variant[k] = variant_str(v.encode("utf-8"))

-         elif type(v) == list:

-             d_variant[k] = variant_list(v)

-         elif type(v) == dict:

-             d_variant[k] = variant_dict(v)

-         elif type(v) == bool:

-             d_variant[k] = variant_bool(v)

-         else:

-             raise TypeError("Cannot convert unknown type")

-     return d_variant

- 

- 

- def variant_dict(d):

-     """ Converts a dictionary to a dictionary of GLib.Variant

-     """

-     if not isinstance(d, dict):

-         raise TypeError("Only dictionaries are supported for mappings")

- 

-     d_variant = dict_values(d)

-     return GLib.Variant("a{sv}", d_variant)

@@ -63,8 +63,8 @@ 

  

          brs = None

          try:

-             mmd = Modulemd.Module().new_from_string(build.modulemd)

-             mmd.upgrade()

+             mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)

+             mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)

              brs = mmd.get_xmd()['mbs']['buildrequires']

          except Exception:

              # If the modulemd isn't parseable then skip this build

@@ -76,8 +76,8 @@ 

              continue

  

          try:

-             mmd = Modulemd.Module().new_from_string(build.modulemd)

-             mmd.upgrade()

+             mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)

+             mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)

          except Exception:

              # If the modulemd isn't parseable, then skip this build

              continue

@@ -36,8 +36,8 @@ 

          if not build.modulemd:

              continue

          try:

-             mmd = Modulemd.Module().new_from_string(build.modulemd)

-             mmd.upgrade()

+             mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)

+             mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)

          except Exception:

              # If the modulemd isn't parseable then skip this build

              continue

@@ -41,27 +41,26 @@ 

          if not build.modulemd:

              continue

          try:

-             mmd = Modulemd.Module().new_from_string(build.modulemd)

-             mmd.upgrade()

+             mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)

+             mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)

          except Exception:

              # If the modulemd isn't parseable then skip this build

              continue

  

          mbs_xmd = mmd.get_xmd().get('mbs', {})

          contexts = {}

-         for property_name in ['buildrequires', 'requires']:

+         for mmd_name, xmd_name in (('buildtime', 'buildrequires'), ('runtime', 'requires')):

              # It's possible this module build was built before MBS filled out xmd or before MBS

-             # filled out the requires in xmd. We also have to use keys because GLib.Variant

-             # doesn't support `in` directly.

-             if property_name not in mbs_xmd.keys():

+             # filled out the requires in xmd

+             if xmd_name not in mbs_xmd:

                  break

-             mmd_property = getattr(mmd.get_dependencies()[0], 'get_{0}'.format(property_name))()

-             if set(mbs_xmd[property_name].keys()) != set(mmd_property.keys()):

+             mmd_property = getattr(mmd.get_dependencies()[0], 'get_{0}_modules'.format(mmd_name))()

+             if set(mbs_xmd[xmd_name].keys()) != set(mmd_property):

                  break

              mmd_formatted_property = {

-                 dep: info['ref'] for dep, info in mbs_xmd[property_name].items()}

+                 dep: info['ref'] for dep, info in mbs_xmd[xmd_name].items()}

              property_json = json.dumps(OrderedDict(sorted(mmd_formatted_property.items())))

-             contexts[property_name] = hashlib.sha1(property_json).hexdigest()

+             contexts[xmd_name] = hashlib.sha1(property_json).hexdigest()

  

          # Update the database now

          if len(contexts) == 2:

@@ -42,8 +42,8 @@ 

          if not build.modulemd:

              continue

          try:

-             mmd = Modulemd.Module().new_from_string(build.modulemd)

-             mmd.upgrade()

+             mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)

+             mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)

          except Exception:

              # If the modulemd isn't parseable then skip this build

              continue
@@ -51,13 +51,12 @@ 

          mbs_xmd = mmd.get_xmd().get('mbs', {})

          # Skip the non-MSE builds, so the "context" will be set default one

          # in models.ModuleBuild.

-         if "mse" not in mbs_xmd.keys() or not mbs_xmd["mse"]:

+         if not mbs_xmd.get("mse"):

              continue

  

          # It's possible this module build was built before MBS filled out xmd or before MBS

-         # filled out the requires in xmd. We also have to use keys because GLib.Variant

-         # doesn't support `in` directly.

-         if 'buildrequires' not in mbs_xmd.keys():

+         # filled out the requires in xmd

+         if 'buildrequires' not in mbs_xmd:

              continue

          # Get the streams of buildrequires and hash it.

          mmd_formatted_buildrequires = {

@@ -84,10 +84,7 @@ 

          See the inline comments for more information.

  

          :param list deps: List of dicts with dependency name as key and list of

-             streams as value. Generally, it is just the return value from

-             ``Modulemd.Dependencies.get_requires`` or

-             ``Modulemd.Dependencies.get_buildrequires`` whose value is

-             converted from ``Modulemd.SimpleSet`` to list.

+             streams as value.

          :param dict base_module_stream_overrides: The key is base module name, value

              is the stream string which will be used to compute `version` part of the

              base module solv.Dep expression.
@@ -229,7 +226,7 @@ 

          - module(platform:el8.1.0) = 80100 - Modules can require specific platform stream.

          - module(platform:el8) = 80100 - Module can also require just platform:el8.

          """

-         if mmd.get_name() not in conf.base_module_names:

+         if mmd.get_module_name() not in conf.base_module_names:

              return

  

          # When depsolving, we will need to follow specific rules to choose the right base
@@ -238,21 +235,24 @@ 

          # and so on. We therefore need to convert the stream and version of base module to

          # integer representation and add "module($name:$stream) = $stream_based_version"

          # to Provides.

-         version = ModuleBuild.get_stream_version(mmd.get_stream(), right_pad=False)

+         version = ModuleBuild.get_stream_version(mmd.get_stream_name(), right_pad=False)

          if version:

-             dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), mmd.get_stream())).Rel(

-                 solv.REL_EQ, self.pool.Dep(str(version)))

+             dep = self.pool.Dep(

+                 "module(%s:%s)" % (mmd.get_module_name(), mmd.get_stream_name())

+             ).Rel(

+                 solv.REL_EQ, self.pool.Dep(str(version))

+             )

              solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep)

  

          xmd = mmd.get_xmd()

          # Return in case virtual_streams are not set for this mmd.

-         if "mbs" not in xmd.keys() or "virtual_streams" not in xmd["mbs"].keys():

+         if not xmd.get("mbs", {}).get("virtual_streams"):

              return

  

          # For each virtual stream, add

          # "module($name:$stream) = $virtual_stream_based_version" provide.

          for stream in xmd["mbs"]["virtual_streams"]:

-             dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), stream)).Rel(

+             dep = self.pool.Dep("module(%s:%s)" % (mmd.get_module_name(), stream)).Rel(

                  solv.REL_EQ, self.pool.Dep(str(version)))

              solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep)

  
@@ -269,11 +269,11 @@ 

          """

          overrides = {}

          xmd = mmd.get_xmd()

-         if "mbs" in xmd.keys() and "buildrequires" in xmd["mbs"].keys():

+         if "buildrequires" in xmd.get("mbs", {}):

              for base_module_name in conf.base_module_names:

-                 if base_module_name not in xmd["mbs"]["buildrequires"].keys():

+                 if base_module_name not in xmd["mbs"]["buildrequires"]:

                      continue

-                 if "stream" not in xmd["mbs"]["buildrequires"][base_module_name].keys():

+                 if "stream" not in xmd["mbs"]["buildrequires"][base_module_name]:

                      continue

                  stream = xmd["mbs"]["buildrequires"][base_module_name]["stream"]

  
@@ -291,14 +291,18 @@ 

          :rtype: list

          :return: list of solv.Solvable instances representing the module in libsolv world.

          """

-         n, s, v, c = mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context()

+         n, s, v, c = \

+             mmd.get_module_name(), mmd.get_stream_name(), mmd.get_version(), mmd.get_context()

          pool = self.pool

  

          # Helper method to return the dependencies of `mmd` in the {name: [streams], ... form}.

-         # The `fn` is either "get_requires" or "get_buildrequires" str depending on whether

+         # The `dep_type` is either "runtime" or "buildtime" str depending on whether

          # the return deps should be runtime requires or buildrequires.

-         normdeps = lambda mmd, fn: [

-             {name: streams.get() for name, streams in getattr(dep, fn)().items()}

+         normdeps = lambda mmd, dep_type: [

+             {

+                 name: getattr(dep, "get_{}_streams".format(dep_type))(name)

+                 for name in getattr(dep, "get_{}_modules".format(dep_type))()

+             }

              for dep in mmd.get_dependencies()

          ]

  
@@ -339,7 +343,7 @@ 

              # Fill in the "Requires" of this module, so we can track its dependencies

              # on other modules.

              requires = self._deps2reqs(

-                 normdeps(mmd, "get_requires"), base_module_stream_overrides, False

+                 normdeps(mmd, "runtime"), base_module_stream_overrides, False

              )

              log.debug("Adding module %s with requires: %r", solvable.name, requires)

              solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires)
@@ -391,7 +395,7 @@ 

              # Using this trick, libsolv will try to solve all the buildrequires/requires pairs,

              # because they are expressed as separate Solvables and we are able to distinguish

              # between them thanks to context value.

-             normalized_deps = normdeps(mmd, "get_buildrequires")

+             normalized_deps = normdeps(mmd, "buildtime")

              for c, deps in enumerate(mmd.get_dependencies()):

                  # $n:$s:$c-$v.src

                  solvable = self.build_repo.add_solvable()

@@ -41,7 +41,6 @@ 

  from sqlalchemy.orm import validates, scoped_session, sessionmaker, load_only

  

  import module_build_service.messaging

- from module_build_service.glib import from_variant_dict

  from module_build_service import db, log, get_url_for, app, conf

  from module_build_service.errors import UnprocessableEntity

  
@@ -544,7 +543,7 @@ 

          :return: Tuple with build_context, strem_build_context, runtime_context and

                   context hashes.

          """

-         from module_build_service.utils import load_mmd

+         from module_build_service.utils.general import load_mmd

  

          try:

              mmd = load_mmd(mmd_str)
@@ -555,8 +554,7 @@ 

  

          # Get the buildrequires from the XMD section, because it contains

          # all the buildrequires as we resolved them using dependency resolver.

-         # We have to use keys because GLib.Variant doesn't support `in` directly.

-         if "buildrequires" not in mbs_xmd.keys():

+         if "buildrequires" not in mbs_xmd:

              raise ValueError("The module's modulemd hasn't been formatted by MBS")

          mmd_formatted_buildrequires = {

              dep: info["ref"] for dep, info in mbs_xmd["buildrequires"].items()
@@ -575,10 +573,11 @@ 

          # Get the requires from the real "dependencies" section in MMD.

          mmd_requires = {}

          for deps in mmd.get_dependencies():

-             for name, streams in deps.get_requires().items():

+             for name in deps.get_runtime_modules():

+                 streams = deps.get_runtime_streams(name)

                  if name not in mmd_requires:

                      mmd_requires[name] = set()

-                 mmd_requires[name] = mmd_requires[name].union(streams.get())

+                 mmd_requires[name] = mmd_requires[name].union(streams)

  

          # Sort the streams for each module name and also sort the module names.

          mmd_requires = {dep: sorted(list(streams)) for dep, streams in mmd_requires.items()}
@@ -800,11 +799,8 @@ 

  

      def json(self, show_tasks=True):

          mmd = self.mmd()

-         xmd = from_variant_dict(mmd.get_xmd())

-         try:

-             buildrequires = xmd["mbs"]["buildrequires"]

-         except KeyError:

-             buildrequires = {}

+         xmd = mmd.get_xmd()

+         buildrequires = xmd.get("mbs", {}).get("buildrequires", {})

          rv = self.short_json()

          rv.update({

              "component_builds": [build.id for build in self.component_builds],
@@ -950,7 +946,6 @@ 

          xmd = self.mmd().get_xmd()

          with make_session(conf) as db_session:

              for bm in conf.base_module_names:

-                 # xmd is a GLib Variant and doesn't support .get() syntax

                  try:

                      bm_dict = xmd["mbs"]["buildrequires"].get(bm)

                  except KeyError:

@@ -28,7 +28,7 @@ 

  from module_build_service.resolver.base import GenericResolver

  from module_build_service import models

  from module_build_service.errors import UnprocessableEntity

- from module_build_service.utils.submit import load_mmd

+ from module_build_service.utils.general import load_mmd

  import sqlalchemy

  

  
@@ -72,7 +72,7 @@ 

          :param str name: the module name to search for

          :param str virtual_stream: the module virtual stream to search for

          :return: the module's modulemd or None

-         :rtype: Modulemd.Module or None

+         :rtype: Modulemd.ModuleStream or None

          """

          with models.make_session(self.config) as session:

              query = session.query(models.ModuleBuild).filter_by(name=name)
@@ -191,9 +191,7 @@ 

  

              mmds = [build.mmd() for build in builds]

              nsvcs = [

-                 ":".join(

-                     [mmd.get_name(), mmd.get_stream(), str(mmd.get_version()), mmd.get_context()]

-                 )

+                 mmd.get_nsvc()

                  for mmd in mmds

              ]

              log.debug("Found: %r", nsvcs)
@@ -206,7 +204,7 @@ 

          the key in all buildrequires. If there are some modules loaded by

          utils.load_local_builds(...), these local modules will be considered when returning

          the profiles.

-         :param mmd: Modulemd.Module instance representing the module

+         :param mmd: Modulemd.ModuleStream instance representing the module

          :param keys: list of modulemd installation profiles to include in the result

          :return: a dictionary

          """
@@ -222,8 +220,9 @@ 

                      log.info("Using local module {0!r} to resolve profiles.".format(local_module))

                      dep_mmd = local_module.mmd()

                      for key in keys:

-                         if key in dep_mmd.get_profiles().keys():

-                             results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())

+                         profile = dep_mmd.get_profile(key)

+                         if profile:

+                             results[key] |= set(profile.get_rpms())

                      continue

  

                  build = models.ModuleBuild.get_build_from_nsvc(
@@ -247,8 +246,9 @@ 

  

                  # Take note of what rpms are in this dep's profile

                  for key in keys:

-                     if key in dep_mmd.get_profiles().keys():

-                         results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())

+                     profile = dep_mmd.get_profile(key)

+                     if profile:

+                         results[key] |= set(profile.get_rpms())

  

          # Return the union of all rpms in all profiles of the given keys

          return results
@@ -267,7 +267,7 @@ 

          :kwarg stream: a string of a module's stream (required if mmd is not set)

          :kwarg version: a string of a module's version (required if mmd is not set)

          :kwarg context: a string of a module's context (required if mmd is not set)

-         :kwarg mmd: Modulemd.Module object. If this is set, the mmd will be used instead of

+         :kwarg mmd: Modulemd.ModuleStream object. If this is set, the mmd will be used instead of

              querying the DB with the name, stream, version, and context.

          :kwarg strict: Normally this function returns None if no module can be

              found.  If strict=True, then an UnprocessableEntity is raised.
@@ -289,8 +289,8 @@ 

              if mmd:

                  queried_mmd = mmd

                  nsvc = ":".join([

-                     mmd.get_name(),

-                     mmd.get_stream(),

+                     mmd.get_module_name(),

+                     mmd.get_stream_name(),

                      str(mmd.get_version()),

                      mmd.get_context() or models.DEFAULT_MODULE_CONTEXT,

                  ])
@@ -305,8 +305,8 @@ 

                  queried_mmd = build.mmd()

                  nsvc = ":".join([name, stream, version, context])

  

-             xmd_mbs = queried_mmd.get_xmd().get("mbs")

-             if not xmd_mbs or "buildrequires" not in xmd_mbs.keys():

+             xmd_mbs = queried_mmd.get_xmd().get("mbs", {})

+             if "buildrequires" not in xmd_mbs:

                  raise RuntimeError(

                      "The module {} did not contain its modulemd or did not have "

                      "its xmd attribute filled out in MBS".format(nsvc)
@@ -385,8 +385,8 @@ 

  

                  commit_hash = None

                  mmd = build.mmd()

-                 mbs_xmd = mmd.get_xmd().get("mbs")

-                 if mbs_xmd and "commit" in mbs_xmd.keys():

+                 mbs_xmd = mmd.get_xmd().get("mbs", {})

+                 if mbs_xmd.get("commit"):

                      commit_hash = mbs_xmd["commit"]

                  else:

                      raise RuntimeError(
@@ -394,7 +394,7 @@ 

                              module_name)

                      )

  

-                 if "mse" not in mbs_xmd.keys() or not mbs_xmd["mse"]:

+                 if not mbs_xmd.get("mse"):

                      raise RuntimeError(

                          'The module "{}" is not built using Module Stream Expansion. '

                          "Please rebuild this module first".format(nsvc)

@@ -33,8 +33,7 @@ 

  from module_build_service import models

  from module_build_service.errors import UnprocessableEntity

  from module_build_service.resolver.base import GenericResolver

- from module_build_service.utils.submit import load_mmd

- from module_build_service.utils.general import import_mmd

+ from module_build_service.utils.general import import_mmd, load_mmd

  

  log = logging.getLogger()

  
@@ -150,7 +149,7 @@ 

          :param str name: the module name to search for

          :param str virtual_stream: the module virtual stream to search for

          :return: the module's modulemd or None

-         :rtype: Modulemd.Module or None

+         :rtype: Modulemd.ModuleStream or None

          """

          query = {

              "name": name,
@@ -247,7 +246,7 @@ 

  

      def resolve_profiles(self, mmd, keys):

          """

-         :param mmd: Modulemd.Module instance of module

+         :param mmd: Modulemd.ModuleStream instance of module

          :param keys: list of modulemd installation profiles to include in

                       the result.

          :return: Dictionary with keys set according to `keys` param and values
@@ -271,8 +270,9 @@ 

                  log.info("Using local module %r to resolve profiles.", local_module)

                  dep_mmd = local_module.mmd()

                  for key in keys:

-                     if key in dep_mmd.get_profiles().keys():

-                         results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())

+                     profile = dep_mmd.get_profile(key)

+                     if profile:

+                         results[key] |= set(profile.get_rpms())

                  continue

  

              # Find the dep in the built modules in MBS
@@ -289,8 +289,9 @@ 

                  dep_mmd = load_mmd(yaml)

                  # Take note of what rpms are in this dep's profile.

                  for key in keys:

-                     if key in dep_mmd.get_profiles().keys():

-                         results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())

+                     profile = dep_mmd.get_profile(key)

+                     if profile:

+                         results[key] |= set(profile.get_rpms())

  

          # Return the union of all rpms in all profiles of the given keys.

          return results
@@ -313,7 +314,7 @@ 

          :param strict: Normally this function returns None if no module can be

              found.  If strict=True, then an UnprocessableEntity is raised.

          :return: a mapping containing buildrequire modules info in key/value pairs,

-             where key is koji_tag and value is list of Modulemd.Module objects.

+             where key is koji_tag and value is list of Modulemd.ModuleStream objects.

          :rtype: dict(str, :class:`Modulemd.Module`)

          """

  
@@ -338,11 +339,7 @@ 

              yaml = queried_module["modulemd"]

              queried_mmd = load_mmd(yaml)

  

-         if (

-             not queried_mmd

-             or not queried_mmd.get_xmd().get("mbs")

-             or "buildrequires" not in queried_mmd.get_xmd()["mbs"].keys()

-         ):

+         if not queried_mmd or "buildrequires" not in queried_mmd.get_xmd().get("mbs", {}):

              raise RuntimeError(

                  'The module "{0!r}" did not contain its modulemd or did not have '

                  "its xmd attribute filled out in MBS".format(queried_mmd)
@@ -427,15 +424,15 @@ 

              )

              if module.get("modulemd"):

                  mmd = load_mmd(module["modulemd"])

-                 if mmd.get_xmd().get("mbs") and "commit" in mmd.get_xmd()["mbs"].keys():

+                 if mmd.get_xmd().get("mbs", {}).get("commit"):

                      commit_hash = mmd.get_xmd()["mbs"]["commit"]

  

                  # Find out the particular NVR of filtered packages

-                 if "rpms" in module and mmd.get_rpm_filter().get():

+                 if "rpms" in module and mmd.get_rpm_filters():

                      for rpm in module["rpms"]:

                          nvr = kobo.rpmlib.parse_nvra(rpm)

                          # If the package is not filtered, continue

-                         if not nvr["name"] in mmd.get_rpm_filter().get():

+                         if not nvr["name"] in mmd.get_rpm_filters():

                              continue

  

                          # If the nvr is already in filtered_rpms, continue

@@ -39,7 +39,7 @@ 

  from module_build_service.utils.ursine import handle_stream_collision_modules

  

  from requests.exceptions import ConnectionError

- from module_build_service.utils import to_text_type

+ from module_build_service.utils import mmd_to_str

  

  import koji

  import six.moves.xmlrpc_client as xmlrpclib
@@ -159,7 +159,7 @@ 

          if conf.system in ["koji", "test"]:

              handle_stream_collision_modules(mmd)

          mmd = record_filtered_rpms(mmd)

-         build.modulemd = to_text_type(mmd.dumps())

+         build.modulemd = mmd_to_str(mmd)

          build.transition(conf, models.BUILD_STATES["wait"])

      # Catch custom exceptions that we can expose to the user

      except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e:
@@ -251,7 +251,9 @@ 

          # Find out the name of Koji tag to which the module's Content

          # Generator build should be tagged once the build finishes.

          module_names_streams = {

-             mmd.get_name(): mmd.get_stream() for mmds in module_deps.values() for mmd in mmds

+             mmd.get_module_name(): mmd.get_stream_name()

+             for mmds in module_deps.values()

+             for mmd in mmds

          }

          for base_module_name in conf.base_module_names:

              if base_module_name in module_names_streams:

@@ -22,14 +22,18 @@ 

  # Written by Ralph Bean <rbean@redhat.com>

  #            Matt Prahl <mprahl@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

+ import os

  import functools

  import inspect

  import hashlib

  import time

  from datetime import datetime

+ from functools import partial

+ 

  from six import text_type, string_types

+ from gi.repository.GLib import Error as ModuleMDError

  

- from module_build_service import conf, log, models, Modulemd, glib

+ from module_build_service import conf, log, models, Modulemd

  from module_build_service.errors import ValidationError, ProgrammingError, UnprocessableEntity

  

  
@@ -43,6 +47,58 @@ 

          return s

  

  

+ def load_mmd(yaml, is_file=False):

+     if not yaml:

+         raise UnprocessableEntity('The input modulemd was empty')

+ 

+     target_mmd_version = Modulemd.ModuleStreamVersionEnum.TWO

+     try:

+         if is_file:

+             mmd = Modulemd.ModuleStream.read_file(yaml, True)

+         else:

+             mmd = Modulemd.ModuleStream.read_string(to_text_type(yaml), True)

+         mmd.validate()

+         if mmd.get_mdversion() < target_mmd_version:

+             mmd = mmd.upgrade(target_mmd_version)

+         elif mmd.get_mdversion() > target_mmd_version:

+             log.error("Encountered a modulemd file with the version %d", mmd.get_mdversion())

+             raise UnprocessableEntity(

+                 "The modulemd version cannot be greater than {}".format(target_mmd_version))

+     except ModuleMDError as e:

+         not_found = False

+         if is_file:

+             error = "The modulemd {} is invalid.".format(os.path.basename(yaml))

+             if os.path.exists(yaml):

+                 with open(yaml, "rt") as yaml_hdl:

+                     log.debug("Modulemd that failed to load:\n%s", yaml_hdl.read())

+             else:

+                 not_found = True

+                 error = "The modulemd file {} was not found.".format(os.path.basename(yaml))

+                 log.error("The modulemd file %s was not found.", yaml)

+         else:

+             error = "The modulemd is invalid."

+             log.debug("Modulemd that failed to load:\n%s", yaml)

+ 

+         if "modulemd-error-quark: " in str(e):

+             error = "{} The error was '{}'.".format(

+                 error, str(e).split("modulemd-error-quark: ")[-1])

+         elif "Unknown ModuleStream version" in str(e):

+             error = (

+                 "{}. The modulemd version can't be greater than {}."

+                 .format(error, target_mmd_version)

+             )

+         elif not_found is False:

+             error = "{} Please verify the syntax is correct.".format(error)

+ 

+         log.exception(error)

+         raise UnprocessableEntity(error)

+ 

+     return mmd

+ 

+ 

+ load_mmd_file = partial(load_mmd, is_file=True)

+ 

+ 

  def scm_url_schemes(terse=False):

      """

      Definition of URL schemes supported by both frontend and scheduler.
@@ -363,8 +419,8 @@ 

      """

      if not mmd.get_context():

          mmd.set_context(models.DEFAULT_MODULE_CONTEXT)

-     name = mmd.get_name()

-     stream = mmd.get_stream()

+     name = mmd.get_module_name()

+     stream = mmd.get_stream_name()

      version = str(mmd.get_version())

      context = mmd.get_context()

  
@@ -414,13 +470,13 @@ 

          raise UnprocessableEntity(

              "The imported module's dependencies list should contain just one element")

  

-     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd = mmd.get_xmd()

      # Set some defaults in xmd["mbs"] if they're not provided by the user

      if "mbs" not in xmd:

          xmd["mbs"] = {"mse": True}

  

      if check_buildrequires and mmd.get_dependencies():

-         brs = set(mmd.get_dependencies()[0].get_buildrequires().keys())

+         brs = set(mmd.get_dependencies()[0].get_buildtime_modules())

          xmd_brs = set(xmd["mbs"].get("buildrequires", {}).keys())

          if brs - xmd_brs:

              raise UnprocessableEntity(
@@ -429,7 +485,7 @@ 

              )

      elif "buildrequires" not in xmd["mbs"]:

          xmd["mbs"]["buildrequires"] = {}

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

      koji_tag = xmd["mbs"].get("koji_tag")

      if koji_tag is None:
@@ -449,7 +505,7 @@ 

      build.version = version

      build.koji_tag = koji_tag

      build.state = models.BUILD_STATES["ready"]

-     build.modulemd = to_text_type(mmd.dumps())

+     build.modulemd = mmd_to_str(mmd)

      build.context = context

      build.owner = "mbs_import"

      build.rebuild_strategy = "all"
@@ -494,26 +550,19 @@ 

      :param str nsvc: name:stream:version:context of a module.

      """

      name, stream, version, context = nsvc.split(":")

-     mmd = Modulemd.Module()

-     mmd.set_mdversion(2)

-     mmd.set_name(name)

-     mmd.set_stream(stream)

+     mmd = Modulemd.ModuleStreamV2.new(name, stream)

      mmd.set_version(int(version))

      mmd.set_context(context)

      mmd.set_summary("fake base module")

      mmd.set_description("fake base module")

-     licenses = Modulemd.SimpleSet()

-     licenses.add("GPL")

-     mmd.set_module_licenses(licenses)

+     mmd.add_module_license("GPL")

  

-     buildroot = Modulemd.Profile()

-     buildroot.set_name("buildroot")

+     buildroot = Modulemd.Profile.new("buildroot")

      for rpm in conf.default_buildroot_packages:

          buildroot.add_rpm(rpm)

      mmd.add_profile(buildroot)

  

-     srpm_buildroot = Modulemd.Profile()

-     srpm_buildroot.set_name("srpm-buildroot")

+     srpm_buildroot = Modulemd.Profile.new("srpm-buildroot")

      for rpm in conf.default_srpm_buildroot_packages:

          srpm_buildroot.add_rpm(rpm)

      mmd.add_profile(srpm_buildroot)
@@ -527,7 +576,7 @@ 

      # Use empty "repofile://" URI for base module. The base module will use the

      # `conf.base_module_names` list as list of default repositories.

      xmd_mbs["koji_tag"] = "repofile://"

-     mmd.set_xmd(glib.dict_values(xmd))

+     mmd.set_xmd(xmd)

  

      with models.make_session(conf) as session:

          import_mmd(session, mmd, False)
@@ -570,16 +619,22 @@ 

                  log.warning(str(e))

                  continue

              mmd_data = repo.get_metadata_content("modules")

-             mmds = Modulemd.Module.new_all_from_string(mmd_data)

-             for mmd in mmds:

-                 xmd = glib.from_variant_dict(mmd.get_xmd())

-                 xmd["mbs"] = {}

-                 xmd["mbs"]["koji_tag"] = "repofile://" + repo.repofile

-                 xmd["mbs"]["mse"] = True

-                 xmd["mbs"]["commit"] = "unknown"

-                 mmd.set_xmd(glib.dict_values(xmd))

+             mmd_index = Modulemd.ModuleIndex.new()

+             ret, _ = mmd_index.update_from_string(mmd_data, True)

+             if not ret:

+                 log.warning("Loading the repo '%s' failed", repo.name)

+                 continue

+ 

+             for module_name in mmd_index.get_module_names():

+                 for mmd in mmd_index.get_module(module_name).get_all_streams():

+                     xmd = mmd.get_xmd()

+                     xmd["mbs"] = {}

+                     xmd["mbs"]["koji_tag"] = "repofile://" + repo.repofile

+                     xmd["mbs"]["mse"] = True

+                     xmd["mbs"]["commit"] = "unknown"

+                     mmd.set_xmd(xmd)

  

-                 import_mmd(session, mmd, False)

+                     import_mmd(session, mmd, False)

  

      if not platform_id:

          # Parse the /etc/os-release to find out the local platform:stream.
@@ -623,10 +678,40 @@ 

  

      # Handle BASE_MODULE_ARCHES. Find out the base modules in buildrequires

      # section of XMD and set the Koji tag arches according to it.

-     if "mbs" in mmd.get_xmd().keys():

+     if "mbs" in mmd.get_xmd():

          for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():

              ns = ":".join([req_name, req_data["stream"]])

              if ns in config.base_module_arches:

                  arches = config.base_module_arches[ns]

                  break

      return arches

+ 

+ 

+ def deps_to_dict(deps, deps_type):

+     """

+     Helper method to convert a Modulemd.Dependencies object to a dictionary.

+ 

+     :param Modulemd.Dependencies deps: the Modulemd.Dependencies object to convert

+     :param str deps_type: the type of dependency (buildtime or runtime)

+     :return: a dictionary with the keys as module names and values as a list of strings

+     :rtype dict

+     """

+     names_func = getattr(deps, 'get_{}_modules'.format(deps_type))

+     streams_func = getattr(deps, 'get_{}_streams'.format(deps_type))

+     return {

+         module: streams_func(module)

+         for module in names_func()

+     }

+ 

+ 

+ def mmd_to_str(mmd):

+     """

+     Helper method to convert a Modulemd.ModuleStream object to a YAML string.

+ 

+     :param Modulemd.ModuleStream mmd: the modulemd to convert

+     :return: the YAML string of the modulemd

+     :rtype: str

+     """

+     index = Modulemd.ModuleIndex()

+     index.add_module_stream(mmd)

+     return to_text_type(index.dump_to_string())

@@ -26,7 +26,7 @@ 

  from module_build_service.errors import StreamAmbigous

  from module_build_service.errors import UnprocessableEntity

  from module_build_service.mmd_resolver import MMDResolver

- from module_build_service import glib

+ from module_build_service.utils.general import deps_to_dict, mmd_to_str

  import module_build_service.resolver

  

  
@@ -90,7 +90,7 @@ 

      Expands streams in both buildrequires/requires sections of MMD.

  

      :param session: SQLAlchemy DB session.

-     :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module.

+     :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.

      :param dict default_streams: Dict in {module_name: module_stream, ...} format defining

          the default stream to choose for module in case when there are multiple streams to

          choose from.
@@ -99,25 +99,32 @@ 

          defined in `default_streams`, so it is not clear which stream should be used.

      """

      for deps in mmd.get_dependencies():

-         expanded = {}

-         for name, streams in deps.get_requires().items():

-             streams_set = Modulemd.SimpleSet()

-             streams_set.set(

-                 _expand_mse_streams(

-                     session, name, streams.get(), default_streams, raise_if_stream_ambigous)

-             )

-             expanded[name] = streams_set

-         deps.set_requires(expanded)

- 

-         expanded = {}

-         for name, streams in deps.get_buildrequires().items():

-             streams_set = Modulemd.SimpleSet()

-             streams_set.set(

-                 _expand_mse_streams(

-                     session, name, streams.get(), default_streams, raise_if_stream_ambigous)

-             )

-             expanded[name] = streams_set

-         deps.set_buildrequires(expanded)

+         new_deps = Modulemd.Dependencies()

+         for name in deps.get_runtime_modules():

+             streams = deps.get_runtime_streams(name)

+             new_streams = _expand_mse_streams(

+                 session, name, streams, default_streams, raise_if_stream_ambigous)

+ 

+             if new_streams == []:

+                 new_deps.set_empty_runtime_dependencies_for_module(name)

+             else:

+                 for stream in new_streams:

+                     new_deps.add_runtime_stream(name, stream)

+ 

+         for name in deps.get_buildtime_modules():

+             streams = deps.get_buildtime_streams(name)

+             new_streams = _expand_mse_streams(

+                 session, name, streams, default_streams, raise_if_stream_ambigous)

+ 

+             if new_streams == []:

+                 new_deps.set_empty_buildtime_dependencies_for_module(name)

+             else:

+                 for stream in new_streams:

+                     new_deps.add_buildtime_stream(name, stream)

+ 

+         # Replace the Dependencies object

+         mmd.remove_dependencies(deps)

+         mmd.add_dependencies(new_deps)

  

  

  def _get_mmds_from_requires(
@@ -132,7 +139,7 @@ 

      Helper method for get_mmds_required_by_module_recursively returning

      the list of module metadata objects defined by `requires` dict.

  

-     :param requires: Modulemd.Module requires or buildrequires.

+     :param dict requires: requires or buildrequires in the form {module: [streams]}

      :param mmds: Dictionary with already handled name:streams as a keys and lists

          of resulting mmds as values.

      :param recursive: If True, the requires are checked recursively.
@@ -158,7 +165,7 @@ 

          if name in conf.base_module_names:

              continue

  

-         streams_to_try = streams.get()

+         streams_to_try = streams

          if name in default_streams:

              streams_to_try = [default_streams[name]]

          elif len(streams_to_try) > 1 and raise_if_stream_ambigous:
@@ -171,7 +178,7 @@ 

          # its contexts and add mmds of these builds to `mmds` and `added_mmds`.

          # Of course only do that if we have not done that already in some

          # previous call of this method.

-         for stream in streams.get():

+         for stream in streams:

              ns = "%s:%s" % (name, stream)

              if ns not in mmds:

                  mmds[ns] = []
@@ -180,12 +187,7 @@ 

  

              if base_module_mmds:

                  for base_module_mmd in base_module_mmds:

-                     base_module_nsvc = ":".join([

-                         base_module_mmd.get_name(),

-                         base_module_mmd.get_stream(),

-                         str(base_module_mmd.get_version()),

-                         base_module_mmd.get_context(),

-                     ])

+                     base_module_nsvc = base_module_mmd.get_nsvc()

                      mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_nsvc)

              else:

                  mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True)
@@ -196,8 +198,9 @@ 

          for mmd_list in added_mmds.values():

              for mmd in mmd_list:

                  for deps in mmd.get_dependencies():

+                     deps_dict = deps_to_dict(deps, 'runtime')

                      mmds = _get_mmds_from_requires(

-                         deps.get_requires(), mmds, True, base_module_mmds=base_module_mmds)

+                         deps_dict, mmds, True, base_module_mmds=base_module_mmds)

  

      return mmds

  
@@ -216,7 +219,10 @@ 

  

      resolver = module_build_service.resolver.system_resolver

      for deps in mmd.get_dependencies():

-         buildrequires = deps.get_buildrequires()

+         buildrequires = {

+             module: deps.get_buildtime_streams(module)

+             for module in deps.get_buildtime_modules()

+         }

          for name in conf.base_module_names:

              if name not in buildrequires.keys():

                  continue
@@ -226,7 +232,7 @@ 

              # streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be

              # returned twice. Only one query for that scenario is necessary.

              sorted_desc_streams = sorted(

-                 buildrequires[name].get(), reverse=True, key=models.ModuleBuild.get_stream_version)

+                 buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version)

              for stream in sorted_desc_streams:

                  ns = ":".join([name, stream])

                  if ns in seen:
@@ -243,16 +249,18 @@ 

                      continue

                  stream_mmd = mmds[0]

  

+                 # Get the list of compatible virtual streams.

+                 xmd = stream_mmd.get_xmd()

+                 virtual_streams = xmd.get("mbs", {}).get("virtual_streams")

+ 

                  # In case there are no virtual_streams in the buildrequired name:stream,

                  # it is clear that there are no compatible streams, so return just this

                  # `stream_mmd`.

-                 xmd = stream_mmd.get_xmd()

-                 if "mbs" not in xmd.keys() or "virtual_streams" not in xmd["mbs"].keys():

+                 if not virtual_streams:

                      seen.add(ns)

                      ret.append(stream_mmd)

                      continue

  

-                 # Get the list of compatible virtual streams.

                  virtual_streams = xmd["mbs"]["virtual_streams"]

  

                  mmds = resolver.get_module_modulemds(
@@ -261,7 +269,7 @@ 

                  # Add the returned mmds to the `seen` set to avoid querying those individually if

                  # they are part of the buildrequire streams for this base module

                  for mmd_ in mmds:

-                     mmd_ns = ":".join([mmd_.get_name(), mmd_.get_stream()])

+                     mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()])

                      # An extra precaution to ensure there are no duplicates in the event the sorting

                      # above wasn't flawless

                      if mmd_ns not in seen:
@@ -317,27 +325,23 @@ 

  

      # Add base modules to `mmds`.

      for base_module in base_module_mmds:

-         ns = ":".join([base_module.get_name(), base_module.get_stream()])

+         ns = ":".join([base_module.get_module_name(), base_module.get_stream_name()])

          mmds.setdefault(ns, [])

          mmds[ns].append(base_module)

  

      # Get all the buildrequires of the module of interest.

      for deps in mmd.get_dependencies():

+         deps_dict = deps_to_dict(deps, 'buildtime')

          mmds = _get_mmds_from_requires(

-             deps.get_buildrequires(),

-             mmds,

-             False,

-             default_streams,

-             raise_if_stream_ambigous,

-             base_module_mmds,

-         )

+             deps_dict, mmds, False, default_streams, raise_if_stream_ambigous, base_module_mmds)

  

      # Now get the requires of buildrequires recursively.

      for mmd_key in list(mmds.keys()):

          for mmd in mmds[mmd_key]:

              for deps in mmd.get_dependencies():

+                 deps_dict = deps_to_dict(deps, 'runtime')

                  mmds = _get_mmds_from_requires(

-                     deps.get_requires(),

+                     deps_dict,

                      mmds,

                      True,

                      default_streams,
@@ -361,7 +365,7 @@ 

      built using MBS.

  

      :param session: SQLAlchemy DB session.

-     :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module.

+     :param Modulemd.ModuleStream mmd: Modulemd metadata with original unexpanded module.

      :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case

          there are multiple streams for some dependency of module and the module name is not

          defined in `default_streams`, so it is not clear which stream should be used.
@@ -377,9 +381,7 @@ 

  

      # Create local copy of mmd, because we will expand its dependencies,

      # which would change the module.

-     # TODO: Use copy method once its in released libmodulemd:

-     # https://github.com/fedora-modularity/libmodulemd/pull/20

-     current_mmd = Modulemd.Module.new_from_string(mmd.dumps())

+     current_mmd = mmd.copy()

  

      # MMDResolver expects the input MMD to have no context.

      current_mmd.set_context(None)
@@ -396,10 +398,7 @@ 

          mmd_resolver.add_modules(m)

  

      # Show log.info message with the NSVCs we have added to mmd_resolver.

-     nsvcs_to_solve = [

-         ":".join([m.get_name(), m.get_stream(), str(m.get_version()), str(m.get_context())])

-         for m in mmds_for_resolving

-     ]

+     nsvcs_to_solve = [m.get_nsvc() for m in mmds_for_resolving]

      log.info("Starting resolving with following input modules: %r", nsvcs_to_solve)

  

      # Resolve the dependencies between modules and get the list of all valid
@@ -411,10 +410,8 @@ 

      mmds = []

      for requires in requires_combinations:

          # Each generated MMD must be new Module object...

-         # TODO: Use copy method once its in released libmodulemd:

-         # https://github.com/fedora-modularity/libmodulemd/pull/20

-         mmd_copy = Modulemd.Module.new_from_string(mmd.dumps())

-         xmd = glib.from_variant_dict(mmd_copy.get_xmd())

+         mmd_copy = mmd.copy()

+         xmd = mmd_copy.get_xmd()

  

          # Requires contain the NSVC representing the input mmd.

          # The 'context' of this NSVC defines the id of buildrequires/requires
@@ -433,8 +430,8 @@ 

          for nsvca in requires:

              req_name, req_stream, _, req_context, req_arch = nsvca.split(":")

              if req_arch == "src":

-                 assert req_name == current_mmd.get_name()

-                 assert req_stream == current_mmd.get_stream()

+                 assert req_name == current_mmd.get_module_name()

+                 assert req_stream == current_mmd.get_stream_name()

                  assert dependencies_id is None

                  assert self_nsvca is None

                  dependencies_id = int(req_context)
@@ -444,7 +441,7 @@ 

          if dependencies_id is None or self_nsvca is None:

              raise RuntimeError(

                  "%s:%s not found in requires %r"

-                 % (current_mmd.get_name(), current_mmd.get_stream(), requires)

+                 % (current_mmd.get_module_name(), current_mmd.get_stream_name(), requires)

              )

  

          # The name:[streams, ...] pairs do not have to be the same in both
@@ -453,35 +450,45 @@ 

          # In case they are not the same, we have to keep the streams as they are in requires

          # section.  We always replace stream(s) for build-requirement with the one we

          # will build this MMD against.

-         new_dep = Modulemd.Dependencies()

-         dep = mmd_copy.get_dependencies()[dependencies_id]

-         dep_requires = dep.get_requires()

-         dep_buildrequires = dep.get_buildrequires()

-         for req_name, req_streams in dep_requires.items():

-             if req_name not in dep_buildrequires:

+         new_deps = Modulemd.Dependencies()

+         deps = mmd_copy.get_dependencies()[dependencies_id]

+         deps_requires = deps_to_dict(deps, 'runtime')

+         deps_buildrequires = deps_to_dict(deps, 'buildtime')

+         for req_name, req_streams in deps_requires.items():

+             if req_name not in deps_buildrequires:

                  # This require is not a buildrequire so just copy this runtime requirement to

                  # new_dep and don't touch buildrequires

-                 new_dep.add_requires(req_name, req_streams.get())

-             elif set(req_streams.get()) != set(dep_buildrequires[req_name].get()):

+                 if req_streams == []:

+                     new_deps.set_empty_runtime_dependencies_for_module(req_name)

+                 else:

+                     for req_stream in req_streams:

+                         new_deps.add_runtime_stream(req_name, req_stream)

+             elif set(req_streams) != set(deps_buildrequires[req_name]):

                  # Streams in runtime section are not the same as in buildtime section,

                  # so just copy this runtime requirement to new_dep.

-                 new_dep.add_requires(req_name, req_streams.get())

-                 new_dep.add_buildrequires(req_name, [req_name_stream[req_name]])

+                 if req_streams == []:

+                     new_deps.set_empty_runtime_dependencies_for_module(req_name)

+                 else:

+                     for req_stream in req_streams:

+                         new_deps.add_runtime_stream(req_name, req_stream)

+ 

+                 new_deps.add_buildtime_stream(req_name, req_name_stream[req_name])

              else:

                  # This runtime requirement has the same streams in both runtime/buildtime

                  # requires sections, so replace streams in both sections by the one we

                  # really used in this resolved variant.

-                 new_dep.add_requires(req_name, [req_name_stream[req_name]])

-                 new_dep.add_buildrequires(req_name, [req_name_stream[req_name]])

+                 new_deps.add_runtime_stream(req_name, req_name_stream[req_name])

+                 new_deps.add_buildtime_stream(req_name, req_name_stream[req_name])

  

          # There might be buildrequires which are not in runtime requires list.

          # Such buildrequires must be copied to expanded MMD.

-         for req_name, req_streams in dep_buildrequires.items():

-             if req_name not in dep_requires:

-                 new_dep.add_buildrequires(req_name, [req_name_stream[req_name]])

+         for req_name, req_streams in deps_buildrequires.items():

+             if req_name not in deps_requires:

+                 new_deps.add_buildtime_stream(req_name, req_name_stream[req_name])

  

          # Set the new dependencies.

-         mmd_copy.set_dependencies((new_dep,))

+         mmd_copy.remove_dependencies(deps)

+         mmd_copy.add_dependencies(new_deps)

  

          # The Modulemd.Dependencies() stores only streams, but to really build this

          # module, we need NSVC of buildrequires, so we have to store this data in XMD.
@@ -503,11 +510,10 @@ 

          xmd["mbs"]["buildrequires"] = resolver.resolve_requires(br_list)

          xmd["mbs"]["mse"] = True

  

-         mmd_copy.set_xmd(glib.dict_values(xmd))

+         mmd_copy.set_xmd(xmd)

  

          # Now we have all the info to actually compute context of this module.

-         ref_build_context, build_context, runtime_context, context = \

-             models.ModuleBuild.contexts_from_mmd(mmd_copy.dumps())

+         _, _, _, context = models.ModuleBuild.contexts_from_mmd(mmd_to_str(mmd_copy))

          mmd_copy.set_context(context)

  

          mmds.append(mmd_copy)

@@ -90,8 +90,8 @@ 

      # Find the latest module that is in the done or ready state

      previous_module_build = (

          session.query(models.ModuleBuild)

-         .filter_by(name=mmd.get_name())

-         .filter_by(stream=mmd.get_stream())

+         .filter_by(name=mmd.get_module_name())

+         .filter_by(stream=mmd.get_stream_name())

          .filter_by(state=models.BUILD_STATES["ready"])

          .filter(models.ModuleBuild.scmurl.isnot(None))

          .filter_by(build_context=module.build_context)
@@ -229,10 +229,10 @@ 

          is called to get the ModuleBuild instance. Consider passing the ModuleBuild

          instance in case you plan to call get_reusable_component repeatedly for the

          same module to make this method faster.

-     :param mmd: ModuleMd.Module of `module`. If not passed, it is taken from

+     :param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from

          module.mmd(). Consider passing this arg in case you plan to call

          get_reusable_component repeatedly for the same module to make this method faster.

-     :param old_mmd: ModuleMd.Module of `previous_module_build`. If not passed,

+     :param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed,

          it is taken from previous_module_build.mmd(). Consider passing this arg in

          case you plan to call get_reusable_component repeatedly for the same

          module to make this method faster.
@@ -302,7 +302,19 @@ 

              return None

  

          # If the mmd.buildopts.macros.rpms changed, we cannot reuse

-         if mmd.get_rpm_buildopts().get("macros") != old_mmd.get_rpm_buildopts().get("macros"):

+         buildopts = mmd.get_buildopts()

+         if buildopts:

+             modulemd_macros = buildopts.get_rpm_macros()

+         else:

+             modulemd_macros = None

+ 

+         old_buildopts = old_mmd.get_buildopts()

+         if old_buildopts:

+             old_modulemd_macros = old_buildopts.get_rpm_macros()

+         else:

+             old_modulemd_macros = None

+ 

+         if modulemd_macros != old_modulemd_macros:

              log.info("Cannot re-use.  Old modulemd macros do not match the new.")

              return None

  
@@ -348,13 +360,12 @@ 

              log.info("Cannot re-use.  Ordering or commit hashes of previous batches differ.")

              return None

  

-     for pkg_name, pkg in mmd.get_rpm_components().items():

-         if pkg_name not in old_mmd.get_rpm_components():

+     for pkg_name in mmd.get_rpm_component_names():

+         pkg = mmd.get_rpm_component(pkg_name)

+         if pkg_name not in old_mmd.get_rpm_component_names():

              log.info("Cannot re-use. Package lists are different.")

              return None

-         if set(pkg.get_arches().get()) != set(

-             old_mmd.get_rpm_components()[pkg_name].get_arches().get()

-         ):

+         if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(pkg_name).get_arches()):

              log.info("Cannot re-use. Architectures are different for package: %s." % pkg_name)

              return None

  

@@ -29,7 +29,6 @@ 

  import shutil

  import tempfile

  import os

- from functools import partial

  from multiprocessing.dummy import Pool as ThreadPool

  from datetime import datetime

  import copy
@@ -41,8 +40,9 @@ 

  import module_build_service.scm

  from module_build_service import conf, db, log, models, Modulemd

  from module_build_service.errors import ValidationError, UnprocessableEntity, Forbidden, Conflict

- from module_build_service import glib

- from module_build_service.utils import to_text_type

+ from module_build_service.utils import (

+     to_text_type, deps_to_dict, mmd_to_str, load_mmd, load_mmd_file

+ )

  

  

  def record_filtered_rpms(mmd):
@@ -80,9 +80,8 @@ 

  

          # Find out the particular NVR of filtered packages

          filtered_rpms = []

-         rpm_filter = req_mmd.get_rpm_filter()

-         if rpm_filter and rpm_filter.get():

-             rpm_filter = rpm_filter.get()

+         rpm_filter = req_mmd.get_rpm_filters()

+         if rpm_filter:

              built_nvrs = builder.get_built_rpms_in_module_build(req_mmd)

              for nvr in built_nvrs:

                  parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
@@ -93,9 +92,9 @@ 

          new_buildrequires[req_name] = req_data

  

      # Replace the old buildrequires with new ones.

-     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd = mmd.get_xmd()

      xmd["mbs"]["buildrequires"] = new_buildrequires

-     mmd.set_xmd(glib.dict_values(xmd))

+     mmd.set_xmd(xmd)

      return mmd

  

  
@@ -124,7 +123,7 @@ 

      Prepares the modulemd for the MBS. This does things such as replacing the

      branches of components with commit hashes and adding metadata in the xmd

      dictionary.

-     :param mmd: the Modulemd.Module object to format

+     :param mmd: the Modulemd.ModuleStream object to format

      :param scmurl: the url to the modulemd

      :param module: When specified together with `session`, the time_modified

          of a module is updated regularly in case this method takes lot of time.
@@ -134,7 +133,7 @@ 

      # them because of dep-chain.

      from module_build_service.scm import SCM

  

-     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd = mmd.get_xmd()

      if "mbs" not in xmd:

          xmd["mbs"] = {}

      if "scmurl" not in xmd["mbs"]:
@@ -153,11 +152,12 @@ 

  

          xmd["mbs"]["commit"] = full_scm_hash

  

-     if mmd.get_rpm_components() or mmd.get_module_components():

+     if mmd.get_rpm_component_names() or mmd.get_module_component_names():

          if "rpms" not in xmd["mbs"]:

              xmd["mbs"]["rpms"] = {}

          # Add missing data in RPM components

-         for pkgname, pkg in mmd.get_rpm_components().items():

+         for pkgname in mmd.get_rpm_component_names():

+             pkg = mmd.get_rpm_component(pkgname)

              # In case of resubmit of existing module which have been

              # cancelled/failed during the init state, the package

              # was maybe already handled by MBS, so skip it in this case.
@@ -171,21 +171,25 @@ 

              if pkg.get_cache() and not conf.rpms_allow_cache:

                  raise Forbidden(

                      "Custom component caches aren't allowed.  "

-                     "%r bears cache %r" % (pkgname, pkg.cache)

+                     "%r bears cache %r" % (pkgname, pkg.get_cache())

                  )

+             if pkg.get_buildonly() is True:

+                 raise ValidationError('The usage of "buildonly" is not yet supported')

+             if pkg.get_buildafter():

+                 raise ValidationError('The usage of "buildafter" is not yet supported')

              if not pkg.get_repository():

                  pkg.set_repository(conf.rpms_default_repository + pkgname)

              if not pkg.get_cache():

                  pkg.set_cache(conf.rpms_default_cache + pkgname)

              if not pkg.get_ref():

                  pkg.set_ref("master")

-             if pkg.get_arches().size() == 0:

-                 arches = Modulemd.SimpleSet()

-                 arches.set(conf.arches)

-                 pkg.set_arches(arches)

+             if not pkg.get_arches():

+                 for arch in conf.arches:

+                     pkg.add_restricted_arch(arch)

  

          # Add missing data in included modules components

-         for modname, mod in mmd.get_module_components().items():

+         for modname in mmd.get_module_component_names():

+             mod = mmd.get_module_component(modname)

              if mod.get_repository() and not conf.modules_allow_repository:

                  raise Forbidden(

                      "Custom module repositories aren't allowed.  "
@@ -203,8 +207,9 @@ 

              # Filter out the packages which we have already resolved in possible

              # previous runs of this method (can be caused by module build resubmition).

              pkgs_to_resolve = [

-                 pkg for pkg in mmd.get_rpm_components().values()

-                 if pkg.get_name() not in xmd["mbs"]["rpms"]

+                 mmd.get_rpm_component(name)

+                 for name in mmd.get_rpm_component_names()

+                 if name not in xmd["mbs"]["rpms"]

              ]

              async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)

  
@@ -232,14 +237,14 @@ 

              raise UnprocessableEntity(err_msg)

  

      # Set the modified xmd back to the modulemd

-     mmd.set_xmd(glib.dict_values(xmd))

+     mmd.set_xmd(xmd)

  

  

  def get_prefixed_version(mmd):

      """

      Return the prefixed version of the module based on the buildrequired base module stream.

  

-     :param mmd: the Modulemd.Module object to format

+     :param mmd: the Modulemd.ModuleStream object to format

      :return: the prefixed version

      :rtype: int

      """
@@ -248,7 +253,6 @@ 

  

      base_module_stream = None

      for base_module in conf.base_module_names:

-         # xmd is a GLib Variant and doesn't support .get() syntax

          try:

              base_module_stream = xmd["mbs"]["buildrequires"].get(base_module, {}).get("stream")

              if base_module_stream:
@@ -298,18 +302,19 @@ 

          allowed.

      :raise ValidationError: if the xmd has the "mbs" key set.

      """

-     for modname, mod in mmd.get_module_components().items():

+     for modname in mmd.get_module_component_names():

+         mod = mmd.get_module_component(modname)

          if mod.get_repository() and not conf.modules_allow_repository:

              raise Forbidden(

                  "Custom module repositories aren't allowed.  "

                  "%r bears repository %r" % (modname, mod.get_repository())

              )

  

-     name = mmd.get_name()

+     name = mmd.get_module_name()

      xmd = mmd.get_xmd()

      if "mbs" in xmd:

          allowed_to_mark_disttag = name in conf.allowed_disttag_marking_module_names

-         if not (xmd["mbs"].keys() == ["disttag_marking"] and allowed_to_mark_disttag):

+         if not (set(xmd["mbs"].keys()) == {"disttag_marking"} and allowed_to_mark_disttag):

              raise ValidationError('The "mbs" xmd field is reserved for MBS')

  

      if name in conf.base_module_names:
@@ -322,15 +327,15 @@ 

      Merges two modulemds. This merges only metadata which are needed in

      the `main` when it includes another module defined by `included_mmd`

      """

-     included_xmd = glib.from_variant_dict(included_mmd.get_xmd())

+     included_xmd = included_mmd.get_xmd()

      if "rpms" in included_xmd["mbs"]:

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          if "rpms" not in xmd["mbs"]:

              xmd["mbs"]["rpms"] = included_xmd["mbs"]["rpms"]

          else:

              xmd["mbs"]["rpms"].update(included_xmd["mbs"]["rpms"])

      # Set the modified xmd back to the modulemd

-     mmd.set_xmd(glib.dict_values(xmd))

+     mmd.set_xmd(xmd)

  

  

  def get_module_srpm_overrides(module):
@@ -403,14 +408,19 @@ 

      if main_mmd:

          # Check for components that are in both MMDs before merging since MBS

          # currently can't handle that situation.

+         main_mmd_rpms = main_mmd.get_rpm_component_names()

+         mmd_rpms = mmd.get_rpm_component_names()

          duplicate_components = [

-             rpm for rpm in main_mmd.get_rpm_components().keys() if rpm in mmd.get_rpm_components()

+             rpm for rpm in main_mmd_rpms

+             if rpm in mmd_rpms

          ]

          if duplicate_components:

              error_msg = (

                  'The included module "{0}" in "{1}" have the following '

                  "conflicting components: {2}".format(

-                     mmd.get_name(), main_mmd.get_name(), ", ".join(duplicate_components))

+                     mmd.get_module_name(), main_mmd.get_module_name(),

+                     ", ".join(duplicate_components)

+                 )

              )

              raise UnprocessableEntity(error_msg)

          merge_included_mmd(main_mmd, mmd)
@@ -418,8 +428,14 @@ 

          main_mmd = mmd

  

      # If the modulemd yaml specifies components, then submit them for build

-     rpm_components = mmd.get_rpm_components().values()

-     module_components = mmd.get_module_components().values()

+     rpm_components = [

+         mmd.get_rpm_component(name)

+         for name in mmd.get_rpm_component_names()

+     ]

+     module_components = [

+         mmd.get_module_component(name)

+         for name in mmd.get_module_component_names()

+     ]

      all_components = list(rpm_components) + list(module_components)

      if not all_components:

          return
@@ -478,7 +494,7 @@ 

              ):

                  raise ValidationError(

                      "Module build %s already exists in database, but its attributes "

-                     " are different from resubmitted one." % component.get_name()

+                     " are different from resubmitted one." % component.get_module_name()

                  )

              continue

  
@@ -502,19 +518,23 @@ 

      dt = datetime.utcfromtimestamp(int(time.time()))

      if hasattr(handle, "filename"):

          def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])

-     elif not mmd.get_name():

+     elif not mmd.get_module_name():

          raise ValidationError(

              "The module's name was not present in the modulemd file. Please use the "

              '"module_name" parameter'

          )

      def_version = int(dt.strftime("%Y%m%d%H%M%S"))

-     mmd.set_name(mmd.get_name() or def_name)

-     mmd.set_stream(stream or mmd.get_stream() or "master")

+     module_name = mmd.get_module_name() or def_name

+     module_stream = stream or mmd.get_stream_name() or "master"

+     if module_name != mmd.get_module_name() or module_stream != mmd.get_stream_name():

+         # This is how you set the name and stream in the modulemd

+         mmd = mmd.copy(module_name, module_stream)

      mmd.set_version(mmd.get_version() or def_version)

      if skiptests:

-         buildopts = mmd.get_rpm_buildopts()

-         buildopts["macros"] = buildopts.get("macros", "") + "\n\n%__spec_check_pre exit 0\n"

-         mmd.set_rpm_buildopts(buildopts)

+         buildopts = mmd.get_buildopts() or Modulemd.Buildopts()

+         macros = buildopts.get_rpm_macros() or ""

+         buildopts.set_rpm_macros(macros + "\n\n%__spec_check_pre exit 0\n")

+         mmd.set_buildopts(buildopts)

      return submit_module_build(username, mmd, params)

  

  
@@ -538,7 +558,7 @@ 

      """

      Apply the dependency override parameters (if specified) on the input modulemd.

  

-     :param Modulemd.Module mmd: the modulemd to apply the overrides on

+     :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on

      :param dict params: the API parameters passed in by the user

      :raises ValidationError: if one of the overrides doesn't apply

      """
@@ -594,18 +614,37 @@ 

  

      deps = mmd.get_dependencies()

      for dep in deps:

+         overridden = False

+         new_dep = Modulemd.Dependencies()

          for dep_type, overrides in dep_overrides.items():

-             overridden = False

-             # Get the existing streams (e.g. dep.get_buildrequires())

-             reqs = getattr(dep, "get_" + dep_type)()

-             for name, streams in dep_overrides[dep_type].items():

-                 if name in reqs:

-                     reqs[name].set(streams)

+             if dep_type == "buildrequires":

+                 mmd_dep_type = "buildtime"

+             else:

+                 mmd_dep_type = "runtime"

+             # Get the existing streams

+             reqs = deps_to_dict(dep, mmd_dep_type)

+             # Get the method to add a new stream for this dependency type

+             # (e.g. add_buildtime_stream)

+             add_func = getattr(new_dep, "add_{}_stream".format(mmd_dep_type))

+             add_empty_func = getattr(

+                 new_dep, "set_empty_{}_dependencies_for_module".format(mmd_dep_type))

+             for name, streams in reqs.items():

+                 if name in dep_overrides[dep_type]:

+                     streams_to_add = dep_overrides[dep_type][name]

                      unused_dep_overrides[dep_type].remove(name)

                      overridden = True

-             if overridden:

-                 # Set the overridden streams (e.g. dep.set_buildrequires(reqs))

-                 getattr(dep, "set_" + dep_type)(reqs)

+                 else:

+                     streams_to_add = reqs[name]

+ 

+                 if streams_to_add == []:

+                     add_empty_func(name)

+                 else:

+                     for stream in streams_to_add:

+                         add_func(name, stream)

+         if overridden:

+             # Set the overridden streams

+             mmd.remove_dependencies(dep)

+             mmd.add_dependencies(new_dep)

  

      for dep_type in unused_dep_overrides.keys():

          # If a stream override was applied from parsing the branch and it wasn't applicable,
@@ -618,28 +657,33 @@ 

                      dep_type[:-1], ", ".join(sorted(unused_dep_overrides[dep_type])))

              )

  

-     mmd.set_dependencies(deps)

- 

  

  def _handle_base_module_virtual_stream_br(mmd):

      """

      Translate a base module virtual stream buildrequire to an actual stream on the input modulemd.

  

-     :param Modulemd.Module mmd: the modulemd to apply the overrides on

+     :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on

      """

      from module_build_service.resolver import system_resolver

  

-     overridden = False

      deps = mmd.get_dependencies()

      for dep in deps:

-         brs = dep.get_buildrequires()

- 

-         for base_module in conf.base_module_names:

-             if base_module not in brs:

+         overridden = False

+         brs = deps_to_dict(dep, "buildtime")

+         # There is no way to replace streams, so create a new Dependencies object that will end up

+         # being a copy, but with the streams replaced if a virtual stream is detected

+         new_dep = Modulemd.Dependencies()

+ 

+         for name, streams in brs.items():

+             if name not in conf.base_module_names:

+                 if streams == []:

+                     new_dep.set_empty_buildtime_dependencies_for_module(name)

+                 else:

+                     for stream in streams:

+                         new_dep.add_buildtime_stream(name, stream)

                  continue

  

-             streams = list(brs[base_module].get())

-             new_streams = copy.copy(streams)

+             new_streams = copy.deepcopy(streams)

              for i, stream in enumerate(streams):

                  # Ignore streams that start with a minus sign, since those are handled in the

                  # MSE code
@@ -647,45 +691,54 @@ 

                      continue

  

                  # Check if the base module stream is available

-                 log.debug(

-                     'Checking to see if the base module "%s:%s" is available', base_module, stream)

-                 if system_resolver.get_module_count(name=base_module, stream=stream) > 0:

+                 log.debug('Checking to see if the base module "%s:%s" is available', name, stream)

+                 if system_resolver.get_module_count(name=name, stream=stream) > 0:

                      continue

  

                  # If the base module stream is not available, check if there's a virtual stream

                  log.debug(

                      'Checking to see if there is a base module "%s" with the virtual stream "%s"',

-                     base_module, stream,

+                     name, stream,

                  )

                  base_module_mmd = system_resolver.get_latest_with_virtual_stream(

-                     name=base_module, virtual_stream=stream

+                     name=name, virtual_stream=stream

                  )

                  if not base_module_mmd:

                      # If there isn't this base module stream or virtual stream available, skip it,

                      # and let the dep solving code deal with it like it normally would

                      log.warning(

                          'There is no base module "%s" with stream/virtual stream "%s"',

-                         base_module, stream,

+                         name, stream,

                      )

                      continue

  

-                 latest_stream = base_module_mmd.get_stream()

+                 latest_stream = base_module_mmd.get_stream_name()

                  log.info(

                      'Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual '

                      "stream",

-                     base_module, stream, base_module, latest_stream, stream

+                     name, stream, name, latest_stream, stream

                  )

                  new_streams[i] = latest_stream

                  overridden = True

  

-             if streams != new_streams:

-                 brs[base_module].set(new_streams)

+             if new_streams == []:

+                 new_dep.set_empty_buildtime_dependencies_for_module(name)

+             else:

+                 for stream in new_streams:

+                     new_dep.add_buildtime_stream(name, stream)

  

          if overridden:

-             dep.set_buildrequires(brs)

- 

-     if overridden:

-         mmd.set_dependencies(deps)

+             # Copy the runtime streams as is

+             reqs = deps_to_dict(dep, "runtime")

+             for name, streams in reqs.items():

+                 if streams == []:

+                     new_dep.set_empty_runtime_dependencies_for_module(name)

+                 else:

+                     for stream in streams:

+                         new_dep.add_runtime_stream(name, stream)

+             # Replace the old Dependencies object with the new one with the overrides

+             mmd.remove_dependencies(dep)

+             mmd.add_dependencies(new_dep)

  

  

  def submit_module_build(username, mmd, params):
@@ -693,7 +746,7 @@ 

      Submits new module build.

  

      :param str username: Username of the build's owner.

-     :param Modulemd.Module mmd: Modulemd defining the build.

+     :param Modulemd.ModuleStream mmd: Modulemd defining the build.

      :param dict params: the API parameters passed in by the user

      :rtype: list with ModuleBuild

      :return: List with submitted module builds.
@@ -704,8 +757,8 @@ 

      log.debug(

          "Submitted %s module build for %s:%s:%s",

          ("scratch" if params.get("scratch", False) else "normal"),

-         mmd.get_name(),

-         mmd.get_stream(),

+         mmd.get_module_name(),

+         mmd.get_stream_name(),

          mmd.get_version(),

      )

      validate_mmd(mmd)
@@ -739,12 +792,10 @@ 

          # Prefix the version of the modulemd based on the base module it buildrequires

          version = get_prefixed_version(mmd)

          mmd.set_version(version)

-         version_str = str(version)

-         nsvc = ":".join([mmd.get_name(), mmd.get_stream(), version_str, mmd.get_context()])

+         nsvc = mmd.get_nsvc()

  

          log.debug("Checking whether module build already exists: %s.", nsvc)

-         module = models.ModuleBuild.get_build_from_nsvc(

-             db.session, mmd.get_name(), mmd.get_stream(), version_str, mmd.get_context())

+         module = models.ModuleBuild.get_build_from_nsvc(db.session, *nsvc.split(":"))

          if module and not params.get("scratch", False):

              if module.state != models.BUILD_STATES["failed"]:

                  log.info(
@@ -783,7 +834,7 @@ 

              if params.get("scratch", False):

                  log.debug("Checking for existing scratch module builds by NSVC")

                  scrmods = models.ModuleBuild.get_scratch_builds_from_nsvc(

-                     db.session, mmd.get_name(), mmd.get_stream(), version_str, mmd.get_context())

+                     db.session, *nsvc.split(":"))

                  scrmod_contexts = [scrmod.context for scrmod in scrmods]

                  log.debug(

                      "Found %d previous scratch module build context(s): %s",
@@ -792,14 +843,15 @@ 

                  # append incrementing counter to context

                  context_suffix = "_" + str(len(scrmods) + 1)

                  mmd.set_context(mmd.get_context() + context_suffix)

+ 

              log.debug("Creating new module build")

              module = models.ModuleBuild.create(

                  db.session,

                  conf,

-                 name=mmd.get_name(),

-                 stream=mmd.get_stream(),

-                 version=version_str,

-                 modulemd=to_text_type(mmd.dumps()),

+                 name=mmd.get_module_name(),

+                 stream=mmd.get_stream_name(),

+                 version=str(mmd.get_version()),

+                 modulemd=mmd_to_str(mmd),

                  scmurl=params.get("scmurl"),

                  username=username,

                  rebuild_strategy=params.get("rebuild_strategy"),
@@ -818,10 +870,7 @@ 

          db.session.add(module)

          db.session.commit()

          modules.append(module)

-         log.info(

-             "%s submitted build of %s, stream=%s, version=%s, context=%s",

-             username, mmd.get_name(), mmd.get_stream(), version_str, mmd.get_context()

-         )

+         log.info('The user "%s" submitted the build "%s"', username, nsvc)

  

      if all_modules_skipped:

          err_msg = (
@@ -891,23 +940,27 @@ 

  

      # If the name was set in the modulemd, make sure it matches what the scmurl

      # says it should be

-     if mmd.get_name() and mmd.get_name() != scm.name:

+     if mmd.get_module_name() and mmd.get_module_name() != scm.name:

          if not conf.allow_name_override_from_scm:

              raise ValidationError(

-                 'The name "{0}" that is stored in the modulemd is not valid'.format(mmd.get_name()))

+                 'The name "{0}" that is stored in the modulemd is not valid'

+                 .format(mmd.get_module_name())

+             )

      else:

-         mmd.set_name(scm.name)

+         # Set the module name

+         mmd = mmd.copy(scm.name)

  

      # If the stream was set in the modulemd, make sure it matches what the repo

      # branch is

-     if mmd.get_stream() and mmd.get_stream() != scm.branch:

+     if mmd.get_stream_name() and mmd.get_stream_name() != scm.branch:

          if not conf.allow_stream_override_from_scm:

              raise ValidationError(

                  'The stream "{0}" that is stored in the modulemd does not match the branch "{1}"'

-                 .format(mmd.get_stream(), scm.branch)

+                 .format(mmd.get_stream_name(), scm.branch)

              )

      else:

-         mmd.set_stream(scm.branch)

+         # Set the module stream

+         mmd = mmd.copy(mmd.get_module_name(), scm.branch)

  

      # If the version is in the modulemd, throw an exception since the version

      # since the version is generated by MBS
@@ -922,36 +975,6 @@ 

      return mmd, scm

  

  

- def load_mmd(yaml, is_file=False):

-     try:

-         if is_file:

-             mmd = Modulemd.Module().new_from_file(yaml)

-         else:

-             mmd = Modulemd.Module().new_from_string(yaml)

-         # If the modulemd was v1, it will be upgraded to v2

-         mmd.upgrade()

-     except Exception:

-         if is_file:

-             error = "The modulemd {} is invalid. Please verify the syntax is correct".format(

-                 os.path.basename(yaml))

-             if os.path.exists(yaml):

-                 with open(yaml, "rt") as yaml_hdl:

-                     log.debug("Modulemd content:\n%s", yaml_hdl.read())

-             else:

-                 error = "The modulemd file {} not found!".format(os.path.basename(yaml))

-                 log.error("The modulemd file %s not found!", yaml)

-         else:

-             error = "The modulemd is invalid. Please verify the syntax is correct."

-             log.debug("Modulemd content:\n%s", yaml)

-         log.exception(error)

-         raise UnprocessableEntity(error)

- 

-     return mmd

- 

- 

- load_mmd_file = partial(load_mmd, is_file=True)

- 

- 

  def load_local_builds(local_build_nsvs, session=None):

      """

      Loads previously finished local module builds from conf.mock_resultsdir
@@ -1025,11 +1048,11 @@ 

          module = models.ModuleBuild.create(

              session,

              conf,

-             name=mmd.get_name(),

-             stream=mmd.get_stream(),

+             name=mmd.get_module_name(),

+             stream=mmd.get_stream_name(),

              version=str(mmd.get_version()),

              context=mmd.get_context(),

-             modulemd=to_text_type(mmd.dumps()),

+             modulemd=mmd_to_str(mmd),

              scmurl="",

              username="mbs",

              publish_msg=False,

@@ -23,7 +23,7 @@ 

  

  import re

  

- from module_build_service import conf, log, glib

+ from module_build_service import conf, log

  from module_build_service.resolver import system_resolver

  

  
@@ -170,15 +170,15 @@ 

          return []

  

      collision_modules = [

-         item.dup_nsvc()

+         item.get_nsvc()

          for item in ursine_modulemds

          # If some module in the ursine content is one of the buildrequires but has

          # different stream, that is what we want to record here, whose RPMs will be

          # excluded from buildroot by adding them into SRPM module-build-macros as

          # Conflicts.

          if (

-             item.get_name() in buildrequired_modules

-             and item.get_stream() != buildrequired_modules[item.get_name()]["stream"]

+             item.get_module_name() in buildrequired_modules

+             and item.get_stream_name() != buildrequired_modules[item.get_module_name()]["stream"]

          )

      ]

  
@@ -216,8 +216,8 @@ 

      :type mmd: Modulemd.Module

      """

      log.info("Start to find out stream collision modules.")

-     unpacked_xmd = glib.from_variant_dict(mmd.get_xmd())

-     buildrequires = unpacked_xmd["mbs"]["buildrequires"]

+     xmd = mmd.get_xmd()

+     buildrequires = xmd["mbs"]["buildrequires"]

  

      for module_name in conf.base_module_names:

          base_module_info = buildrequires.get(module_name)
@@ -225,7 +225,7 @@ 

              log.info(

                  "Base module %s is not a buildrequire of module %s. "

                  "Skip handling module stream collision for this base module.",

-                 module_name, mmd.get_name(),

+                 module_name, mmd.get_module_name(),

              )

              continue

  
@@ -256,7 +256,7 @@ 

              base_module_info["stream_collision_modules"] = None

              base_module_info["ursine_rpms"] = None

  

-     mmd.set_xmd(glib.dict_values(unpacked_xmd))

+     mmd.set_xmd(xmd)

  

  

  def find_module_built_rpms(modules_nsvc):

file modified
+9 -1
@@ -10,9 +10,13 @@ 

  COPY . /src

  WORKDIR /src

  

+ # Unpin the libmodulemd RPMs once MBS is released with libmodulemd v2

  RUN ${DNF_CMD} install \

        'dnf-command(builddep)' rpm-build rpmdevtools rpmlint \

-       python3-tox python3-pytest python3-mock python3-flake8 bandit && \

+       python3-tox python3-pytest python3-mock python3-flake8 bandit \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/python3-libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd1-1.8.10-1.fc29.x86_64.rpm && \

      ${DNF_CMD} builddep *.spec && \

      ${DNF_CMD} clean all

  RUN rpmdev-setuptree && \
@@ -42,11 +46,15 @@ 

  COPY --from=builder /srv/RPMS /srv/RPMS

  COPY repos/ /etc/yum.repos.d/

  

+ # Unpin the libmodulemd RPMs once MBS is released with libmodulemd v2

  RUN $DNF_CMD install \

        python3-psycopg2 \

        python3-docopt \

        python3-service-identity \

        /srv/*/*/*.rpm \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/python3-libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd-2.4.0-1.fc29.x86_64.rpm \

+       https://kojipkgs.fedoraproject.org//packages/libmodulemd/2.4.0/1.fc29/x86_64/libmodulemd1-1.8.10-1.fc29.x86_64.rpm \

        $EXTRA_RPMS && \

      $DNF_CMD clean all && \

      rm -rf /srv/RPMS

file modified
-1
@@ -4,4 +4,3 @@ 

  pytest

  flake8

  tox

- pyyaml

file modified
+47 -41
@@ -31,12 +31,12 @@ 

  import koji

  import module_build_service

  from module_build_service import db

- from module_build_service.utils import get_rpm_release, import_mmd

+ from module_build_service.utils import get_rpm_release, import_mmd, mmd_to_str

  from module_build_service.config import init_config

  from module_build_service.models import (

      ModuleBuild, ComponentBuild, VirtualStream, make_session, BUILD_STATES,

  )

- from module_build_service import glib, Modulemd

+ from module_build_service import Modulemd

  

  

  base_dir = os.path.dirname(__file__)
@@ -126,19 +126,17 @@ 

      if multiple_stream_versions:

          mmd = load_mmd_file(os.path.join(base_dir, "staged_data", "platform.yaml"))

          for stream in ["f28.0.0", "f29.0.0", "f29.1.0", "f29.2.0"]:

-             mmd.set_name("platform")

-             mmd.set_stream(stream)

+             mmd = mmd.copy("platform", stream)

  

              # Set the virtual_streams based on "fXY" to mark the platform streams

              # with the same major stream_version compatible.

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              xmd["mbs"]["virtual_streams"] = [stream[:3]]

-             mmd.set_xmd(glib.dict_values(xmd))

+             mmd.set_xmd(xmd)

              import_mmd(db.session, mmd)

  

              # Just to possibly confuse tests by adding another base module.

-             mmd.set_name("bootstrap")

-             mmd.set_stream(stream)

+             mmd = mmd.copy("bootstrap", stream)

              import_mmd(db.session, mmd)

      with make_session(conf) as session:

          _populate_data(session, data_size, contexts=contexts, scratch=scratch)
@@ -327,7 +325,7 @@ 

      formatted_testmodule_yml_path = os.path.join(

          current_dir, "staged_data", "formatted_testmodule.yaml")

      mmd = load_mmd_file(formatted_testmodule_yml_path)

-     mmd.get_rpm_components()["tangerine"].set_buildorder(0)

+     mmd.get_rpm_component("tangerine").set_buildorder(0)

  

      platform_br = module_build_service.models.ModuleBuild.query.get(1)

  
@@ -350,7 +348,7 @@ 

          time_submitted=datetime(2017, 2, 15, 16, 8, 18),

          time_modified=datetime(2017, 2, 15, 16, 19, 35),

          rebuild_strategy="changed-and-after",

-         modulemd=to_text_type(mmd.dumps()),

+         modulemd=mmd_to_str(mmd),

      )

  

      module_build.buildrequires.append(platform_br)
@@ -450,11 +448,11 @@ 

      build_one_component_release = get_rpm_release(build_one)

  

      mmd.set_version(int(build_one.version))

-     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd = mmd.get_xmd()

      xmd["mbs"]["scmurl"] = build_one.scmurl

      xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba"

-     mmd.set_xmd(glib.dict_values(xmd))

-     build_one.modulemd = to_text_type(mmd.dumps())

+     mmd.set_xmd(xmd)

+     build_one.modulemd = mmd_to_str(mmd)

      build_one.buildrequires.append(platform_br)

  

      build_one.component_builds.extend([
@@ -532,11 +530,11 @@ 

      build_two_component_release = get_rpm_release(build_two)

  

      mmd.set_version(int(build_one.version))

-     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd = mmd.get_xmd()

      xmd["mbs"]["scmurl"] = build_one.scmurl

      xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"

-     mmd.set_xmd(glib.dict_values(xmd))

-     build_two.modulemd = to_text_type(mmd.dumps())

+     mmd.set_xmd(xmd)

+     build_two.modulemd = mmd_to_str(mmd)

      build_two.buildrequires.append(platform_br)

  

      build_two.component_builds.extend([
@@ -597,13 +595,13 @@ 

          mmd = load_mmd_file(formatted_testmodule_yml_path)

  

          module_build = module_build_service.models.ModuleBuild(

-             name=mmd.get_name(),

-             stream=mmd.get_stream(),

+             name=mmd.get_module_name(),

+             stream=mmd.get_stream_name(),

              version=mmd.get_version(),

              build_context="e046b867a400a06a3571f3c71142d497895fefbe",

              runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a",

              state=BUILD_STATES["ready"],

-             modulemd=to_text_type(mmd.dumps()),

+             modulemd=mmd_to_str(mmd),

              koji_tag="module-shared-userspace-f26-20170601141014-75f92abb",

              scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453",

              batch=16,
@@ -614,7 +612,10 @@ 

              rebuild_strategy="changed-and-after",

          )

  

-         components = list(mmd.get_rpm_components().values())

+         components = [

+             mmd.get_rpm_component(rpm)

+             for rpm in mmd.get_rpm_component_names()

+         ]

          components.sort(key=lambda x: x.get_buildorder())

          previous_buildorder = None

          batch = 1
@@ -650,13 +651,13 @@ 

          mmd2 = load_mmd_file(formatted_testmodule_yml_path)

  

          module_build = module_build_service.models.ModuleBuild(

-             name=mmd2.get_name(),

-             stream=mmd2.get_stream(),

+             name=mmd2.get_module_name(),

+             stream=mmd2.get_stream_name(),

              version=mmd2.get_version(),

              build_context="e046b867a400a06a3571f3c71142d497895fefbe",

              runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a",

              state=BUILD_STATES["done"],

-             modulemd=to_text_type(mmd2.dumps()),

+             modulemd=mmd_to_str(mmd2),

              koji_tag="module-shared-userspace-f26-20170605091544-75f92abb",

              scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453",

              batch=0,
@@ -667,7 +668,10 @@ 

              rebuild_strategy="changed-and-after",

          )

  

-         components2 = list(mmd2.get_rpm_components().values())

+         components2 = [

+             mmd2.get_rpm_component(rpm)

+             for rpm in mmd2.get_rpm_component_names()

+         ]

          # Store components to database in different order than for 570 to

          # reproduce the reusing issue.

          components2.sort(key=lambda x: len(x.get_name()))
@@ -725,23 +729,17 @@ 

      :rtype: ModuleBuild or Modulemd.Module

      """

      name, stream, version, context = nsvc.split(":")

-     mmd = Modulemd.Module()

-     mmd.set_mdversion(2)

-     mmd.set_name(name)

-     mmd.set_stream(stream)

+     mmd = Modulemd.ModuleStreamV2.new(name, stream)

      mmd.set_version(int(version))

      mmd.set_context(context)

      mmd.set_summary("foo")

      # Test unicode in mmd.

      mmd.set_description(u"foo \u2019s")

-     licenses = Modulemd.SimpleSet()

-     licenses.add("GPL")

-     mmd.set_module_licenses(licenses)

+     mmd.add_module_license("GPL")

  

      if filtered_rpms:

-         rpm_filter_set = Modulemd.SimpleSet()

-         rpm_filter_set.set(filtered_rpms)

-         mmd.set_rpm_filter(rpm_filter_set)

+         for rpm in filtered_rpms:

+             mmd.add_rpm_filter(rpm)

  

      if requires_list is not None and build_requires_list is not None:

          if not isinstance(requires_list, list):
@@ -749,15 +747,23 @@ 

          if not isinstance(build_requires_list, list):

              build_requires_list = [build_requires_list]

  

-         deps_list = []

          for requires, build_requires in zip(requires_list, build_requires_list):

              deps = Modulemd.Dependencies()

              for req_name, req_streams in requires.items():

-                 deps.add_requires(req_name, req_streams)

+                 if req_streams == []:

+                     deps.set_empty_runtime_dependencies_for_module(req_name)

+                 else:

+                     for req_stream in req_streams:

+                         deps.add_runtime_stream(req_name, req_stream)

+ 

              for req_name, req_streams in build_requires.items():

-                 deps.add_buildrequires(req_name, req_streams)

-             deps_list.append(deps)

-         mmd.set_dependencies(deps_list)

+                 if req_streams == []:

+                     deps.set_empty_buildtime_dependencies_for_module(req_name)

+                 else:

+                     for req_stream in req_streams:

+                         deps.add_buildtime_stream(req_name, req_stream)

+ 

+             mmd.add_dependencies(deps)

  

      # Caller could pass whole xmd including mbs, but if something is missing,

      # default values are given here.
@@ -775,7 +781,7 @@ 

      if virtual_streams:

          xmd_mbs["virtual_streams"] = virtual_streams

  

-     mmd.set_xmd(glib.dict_values(xmd))

+     mmd.set_xmd(xmd)

  

      if not store_to_db:

          return mmd
@@ -795,7 +801,7 @@ 

          rebuild_strategy="changed-and-after",

          build_context=context,

          runtime_context=context,

-         modulemd=to_text_type(mmd.dumps()),

+         modulemd=mmd_to_str(mmd),

          koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None,

      )

      if base_module:

file modified
+7 -11
@@ -18,29 +18,25 @@ 

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

  #

- # Written by Matt Prahl <mprahl@redhat.com>

  import os

  

  import pytest

  

- from module_build_service import Modulemd

+ from module_build_service.utils.general import load_mmd_file, mmd_to_str

  

  

  BASE_DIR = os.path.dirname(__file__)

  STAGED_DATA_DIR = os.path.join(BASE_DIR, "staged_data")

  

- _mmd = Modulemd.Module().new_from_file(os.path.join(STAGED_DATA_DIR, "platform.yaml"))

- _mmd.upgrade()

- PLATFORM_MODULEMD = _mmd.dumps()

+ _mmd = load_mmd_file(os.path.join(STAGED_DATA_DIR, "platform.yaml"))

+ PLATFORM_MODULEMD = mmd_to_str(_mmd)

  

- _mmd2 = Modulemd.Module().new_from_file(os.path.join(STAGED_DATA_DIR, "formatted_testmodule.yaml"))

- _mmd2.upgrade()

- TESTMODULE_MODULEMD = _mmd2.dumps()

+ _mmd2 = load_mmd_file(os.path.join(STAGED_DATA_DIR, "formatted_testmodule.yaml"))

+ TESTMODULE_MODULEMD = mmd_to_str(_mmd2)

  

- _mmd3 = Modulemd.Module().new_from_file(os.path.join(STAGED_DATA_DIR, "formatted_testmodule.yaml"))

- _mmd3.upgrade()

+ _mmd3 = load_mmd_file(os.path.join(STAGED_DATA_DIR, "formatted_testmodule.yaml"))

  _mmd3.set_context("c2c572ed")

- TESTMODULE_MODULEMD_SECOND_CONTEXT = _mmd3.dumps()

+ TESTMODULE_MODULEMD_SECOND_CONTEXT = mmd_to_str(_mmd3)

  

  

  @pytest.fixture()

file modified
+23 -22
@@ -28,13 +28,13 @@ 

  

  import six.moves.xmlrpc_client as xmlrpclib

  from collections import OrderedDict

- from module_build_service.utils import to_text_type

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos

  import module_build_service.models

  import module_build_service.builder

- from module_build_service import glib, db

+ from module_build_service import Modulemd, db

+ from module_build_service.utils.general import mmd_to_str

  

  import pytest

  from mock import patch, MagicMock
@@ -276,10 +276,10 @@ 

          """

          if blocklist:

              mmd = self.module.mmd()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              xmd["mbs_options"] = {"blocked_packages": ["foo", "bar", "new"]}

-             mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             self.module.modulemd = mmd_to_str(mmd)

  

          builder = FakeKojiModuleBuilder(

              owner=self.module.owner,
@@ -473,35 +473,36 @@ 

      ):

          if blocklist:

              mmd = self.module.mmd()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}

-             mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             self.module.modulemd = mmd_to_str(mmd)

  

          if custom_whitelist:

              mmd = self.module.mmd()

-             opts = mmd.get_buildopts()

-             opts.set_rpm_whitelist(["custom1", "custom2"])

+             opts = Modulemd.Buildopts()

+             opts.add_rpm_to_whitelist("custom1")

+             opts.add_rpm_to_whitelist("custom2")

              mmd.set_buildopts(opts)

-             self.module.modulemd = to_text_type(mmd.dumps())

+             self.module.modulemd = mmd_to_str(mmd)

  

          if repo_include_all is False:

              mmd = self.module.mmd()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              mbs_options = xmd["mbs_options"] if "mbs_options" in xmd.keys() else {}

              mbs_options["repo_include_all"] = False

              xmd["mbs_options"] = mbs_options

-             mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             self.module.modulemd = mmd_to_str(mmd)

  

          if override_arches:

              mmd = self.module.mmd()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              mbs_options = xmd["mbs"] if "mbs" in xmd.keys() else {}

              mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}

              xmd["mbs"] = mbs_options

-             mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             self.module.modulemd = mmd_to_str(mmd)

  

          builder = FakeKojiModuleBuilder(

              owner=self.module.owner,
@@ -579,10 +580,10 @@ 

      def test_buildroot_connect_create_tag(self, blocklist):

          if blocklist:

              mmd = self.module.mmd()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}

-             mmd.set_xmd(glib.dict_values(xmd))

-             self.module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             self.module.modulemd = mmd_to_str(mmd)

  

          builder = FakeKojiModuleBuilder(

              owner=self.module.owner,
@@ -682,8 +683,8 @@ 

          # the module's name/stream/version/context does not have to match it.

          # But for this test, we need it to match.

          mmd = self.module.mmd()

-         self.module.name = mmd.get_name()

-         self.module.stream = mmd.get_stream()

+         self.module.name = mmd.get_module_name()

+         self.module.stream = mmd.get_stream_name()

          self.module.version = mmd.get_version()

          self.module.context = mmd.get_context()

          db.session.commit()

@@ -4,15 +4,13 @@ 

  import tempfile

  import shutil

  from textwrap import dedent

- from module_build_service.utils import to_text_type

  

  import kobo.rpmlib

  

  from module_build_service import conf, db

  from module_build_service.models import ModuleBuild, ComponentBuild, make_session

  from module_build_service.builder.MockModuleBuilder import MockModuleBuilder

- from module_build_service import glib, Modulemd

- from module_build_service.utils import import_fake_base_module

+ from module_build_service.utils import import_fake_base_module, load_mmd_file, mmd_to_str

  from tests import clean_database, make_module

  

  
@@ -52,10 +50,11 @@ 

          ]

  

          base_dir = os.path.abspath(os.path.dirname(__file__))

-         mmd = Modulemd.Module().new_from_file(

+         mmd = load_mmd_file(

              os.path.join(base_dir, "..", "staged_data", "testmodule-with-filters.yaml"))

-         mmd.upgrade()

-         mmd.set_xmd(glib.dict_values({

+         # Set the name and stream

+         mmd = mmd.copy("mbs-testmodule", "test")

+         mmd.set_xmd({

              "mbs": {

                  "rpms": {

                      "ed": {"ref": "01bf8330812fea798671925cc537f2f29b0bd216"},
@@ -89,14 +88,14 @@ 

                      }

                  },

              }

-         }))

+         })

          module = ModuleBuild.create(

              session,

              conf,

              name="mbs-testmodule",

              stream="test",

              version="20171027111452",

-             modulemd=to_text_type(mmd.dumps()),

+             modulemd=mmd_to_str(mmd),

              scmurl="file:///testdir",

              username="test",

          )

file modified
+59 -63
@@ -27,11 +27,11 @@ 

  

  import os

  from os import path

- from module_build_service.utils import to_text_type

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos  # noqa

- from module_build_service import models, conf, build_logs, Modulemd, glib

+ from module_build_service import models, conf, build_logs, Modulemd

+ from module_build_service.utils.general import mmd_to_str

  

  from mock import patch, Mock, call, mock_open

  import kobo.rpmlib
@@ -95,7 +95,7 @@ 

  

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      @patch("subprocess.Popen")

-     @patch("subprocess.check_output", return_value="1.4")

+     @patch("module_build_service.builder.KojiContentGenerator.Modulemd")

      @patch("pkg_resources.get_distribution")

      @patch("platform.linux_distribution")

      @patch("platform.machine")
@@ -104,9 +104,10 @@ 

      )

      @pytest.mark.parametrize("devel", (False, True))

      def test_get_generator_json(

-         self, rpms_in_tag, machine, distro, pkg_res, coutput, popen, ClientSession, devel

+         self, rpms_in_tag, machine, distro, pkg_res, mock_Modulemd, popen, ClientSession, devel

      ):

          """ Test generation of content generator json """

+         mock_Modulemd.get_version.return_value = "2.3.1"

          koji_session = ClientSession.return_value

          koji_session.getUser.return_value = GET_USER_RV

          koji_session.getTag.return_value = {"arches": ""}
@@ -154,7 +155,7 @@ 

  

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")

      @patch("subprocess.Popen")

-     @patch("subprocess.check_output", return_value="1.4")

+     @patch("module_build_service.builder.KojiContentGenerator.Modulemd")

      @patch("pkg_resources.get_distribution")

      @patch("platform.linux_distribution")

      @patch("platform.machine")
@@ -162,9 +163,10 @@ 

          "module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag"

      )

      def test_get_generator_json_no_log(

-         self, rpms_in_tag, machine, distro, pkg_res, coutput, popen, ClientSession

+         self, rpms_in_tag, machine, distro, pkg_res, mock_Modulemd, popen, ClientSession

      ):

          """ Test generation of content generator json """

+         mock_Modulemd.get_version.return_value = "2.3.1"

          koji_session = ClientSession.return_value

          koji_session.getUser.return_value = GET_USER_RV

          koji_session.getTag.return_value = {"arches": ""}
@@ -212,10 +214,10 @@ 

              assert len(mmd.read()) == 1136

  

          with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd:

-             assert len(mmd.read()) == 259

+             assert len(mmd.read()) == 236

  

          with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd:

-             assert len(mmd.read()) == 257

+             assert len(mmd.read()) == 234

  

      @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
@@ -300,10 +302,8 @@ 

      @patch("module_build_service.builder.KojiContentGenerator.open", create=True)

      def test_get_arch_mmd_output_components(self, patched_open):

          mmd = self.cg.module.mmd()

-         rpm_artifacts = mmd.get_rpm_artifacts()

-         rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")

-         mmd.set_rpm_artifacts(rpm_artifacts)

-         mmd_data = to_text_type(mmd.dumps()).encode("utf-8")

+         mmd.add_rpm_artifact("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")

+         mmd_data = mmd_to_str(mmd).encode("utf-8")

  

          patched_open.return_value = mock_open(read_data=mmd_data).return_value

  
@@ -333,7 +333,7 @@ 

          assert ret == {

              "arch": "x86_64",

              "buildroot_id": 1,

-             "checksum": "502e46889affec24d98a281289104d4d",

+             "checksum": "bb6ba47519b68aee215d095fd2c57c70",

              "checksum_type": "md5",

              "components": [

                  {
@@ -348,7 +348,7 @@ 

              ],

              "extra": {"typeinfo": {"module": {}}},

              "filename": "modulemd.x86_64.txt",

-             "filesize": 319,

+             "filesize": 296,

              "type": "file",

          }

  
@@ -557,20 +557,17 @@ 

          self.cg.rpms_dict[nevra] = parsed_nevra

  

          mmd = self.cg.module.mmd()

-         if srpm_name not in mmd.get_rpm_components().keys():

-             component = Modulemd.ComponentRpm()

-             component.set_name(srpm_name)

+         if srpm_name not in mmd.get_rpm_component_names():

+             component = Modulemd.ComponentRpm.new(srpm_name)

              component.set_rationale("foo")

  

              if multilib:

-                 multilib_set = Modulemd.SimpleSet()

                  for arch in multilib:

-                     multilib_set.add(arch)

-                 component.set_multilib(multilib_set)

+                     component.add_multilib_arch(arch)

  

-             mmd.add_rpm_component(component)

-             self.cg.module.modulemd = to_text_type(mmd.dumps())

-             self.cg.modulemd = to_text_type(mmd.dumps())

+             mmd.add_component(component)

+             self.cg.module.modulemd = mmd_to_str(mmd)

+             self.cg.modulemd = mmd_to_str(mmd)

  

      @pytest.mark.parametrize("devel", (False, True))

      def test_fill_in_rpms_list(self, devel):
@@ -609,7 +606,7 @@ 

  

          if not devel:

              # Only x86_64 packages should be filled in, because we requested x86_64 arch.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "dhcp-12:4.3.5-5.module_2118aef6.src",

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

                  "perl-Tangerine-12:4.3.5-5.module_2118aef6.src",
@@ -618,7 +615,7 @@ 

          else:

              # The i686 packages are filtered out in normal packages, because multilib

              # is not enabled for them - therefore we want to include them in -devel.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

                  "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686",

              ])
@@ -646,7 +643,7 @@ 

  

          # Only dhcp-libs should be filled in, because perl-Tangerine has different

          # exclusivearch.

-         assert set(mmd.get_rpm_artifacts().get()) == set(

+         assert set(mmd.get_rpm_artifacts()) == set(

              ["dhcp-12:4.3.5-5.module_2118aef6.src", "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])

  

      def test_fill_in_rpms_excludearch(self):
@@ -671,7 +668,7 @@ 

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

          # Only perl-Tangerine should be filled in, because dhcp-libs is excluded from x86_64.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              "perl-Tangerine-12:4.3.5-5.module_2118aef6.src",

              "perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch",

          ])
@@ -712,7 +709,9 @@ 

          self.cg.devel = devel

          mmd = self.cg.module.mmd()

          opts = mmd.get_buildopts()

-         opts.set_rpm_whitelist(["python27-dhcp"])

+         if not opts:

+             opts = Modulemd.Buildopts()

+         opts.add_rpm_to_whitelist("python27-dhcp")

          mmd.set_buildopts(opts)

  

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
@@ -720,12 +719,12 @@ 

          if not devel:

              # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted

              # srpm name.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "python27-dhcp-12:4.3.5-5.module_2118aef6.src",

                  "python27-dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

              ])

          else:

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "python27-dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

                  "foo-perl-Tangerine-12:4.3.5-5.module_2118aef6.src",

                  "foo-perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",
@@ -787,22 +786,22 @@ 

  

          self.cg.devel = devel

          mmd = self.cg.module.mmd()

-         filter_list = Modulemd.SimpleSet()

-         filter_list.add("dhcp-libs")

-         mmd.set_rpm_filter(filter_list)

+         for rpm in mmd.get_rpm_filters():

+             mmd.remove_rpm_filter(rpm)

+         mmd.add_rpm_filter("dhcp-libs")

  

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

          if not devel:

              # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "perl-Tangerine-12:4.3.5-5.module_2118aef6.src",

                  "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",

                  "perl-Tangerine-debuginfo-12:4.3.5-5.module_2118aef6.x86_64",

                  "perl-Tangerine-debugsource-12:4.3.5-5.module_2118aef6.x86_64",

              ])

          else:

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "dhcp-12:4.3.5-5.module_2118aef6.src",

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

                  "dhcp-libs-debuginfo-12:4.3.5-5.module_2118aef6.x86_64",
@@ -855,7 +854,7 @@ 

          if not devel:

              # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

              # multilib set.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.src",

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
@@ -863,7 +862,7 @@ 

                  "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",

              ])

          else:

-             assert set(mmd.get_rpm_artifacts().get()) == set(

+             assert set(mmd.get_rpm_artifacts()) == set(

                  ["perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])

  

      @pytest.mark.parametrize(
@@ -898,7 +897,7 @@ 

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

          # Only x86_64 packages should be filled in, because we requested x86_64 arch.

-         assert set(mmd.get_content_licenses().get()) == set(expected)

+         assert set(mmd.get_content_licenses()) == set(expected)

  

      @pytest.mark.parametrize("devel", (False, True))

      def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self, devel):
@@ -923,25 +922,25 @@ 

          if not devel:

              # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

              # multilib set. The "dhcp" SRPM should be also included.

-             assert set(mmd.get_rpm_artifacts().get()) == set([

+             assert set(mmd.get_rpm_artifacts()) == set([

                  "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch",

                  "dhcp-12:4.3.5-5.module_2118aef6.src",

              ])

          else:

-             assert set(mmd.get_rpm_artifacts().get()) == set([])

+             assert set(mmd.get_rpm_artifacts()) == set([])

  

      def test_sanitize_mmd(self):

          mmd = self.cg.module.mmd()

-         component = Modulemd.ComponentRpm()

-         component.set_name("foo")

+         component = Modulemd.ComponentRpm.new("foo")

          component.set_rationale("foo")

          component.set_repository("http://private.tld/foo.git")

          component.set_cache("http://private.tld/cache")

-         mmd.add_rpm_component(component)

-         mmd.set_xmd(glib.dict_values({"mbs": {"buildrequires": []}}))

+         mmd.add_component(component)

+         mmd.set_xmd({"mbs": {"buildrequires": []}})

          mmd = self.cg._sanitize_mmd(mmd)

  

-         for pkg in mmd.get_rpm_components().values():

+         for pkg_name in mmd.get_rpm_component_names():

+             pkg = mmd.get_rpm_component(pkg_name)

              assert pkg.get_repository() is None

              assert pkg.get_cache() is None

  
@@ -956,11 +955,8 @@ 

              "620ec77321b2ea7b0d67d82992dda3e1d67055b4",

          )

          mmd = self.cg.module.mmd()

-         mmd.set_xmd(

-             glib.dict_values(

-                 {"mbs": {"commit": "foo", "scmurl": "git://localhost/modules/foo.git#master"}})

-         )

-         self.cg.module.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd({"mbs": {"commit": "foo", "scmurl": "git://localhost/modules/foo.git#master"}})

+         self.cg.module.modulemd = mmd_to_str(mmd)

          file_dir = self.cg._prepare_file_directory()

          with io.open(path.join(file_dir, "modulemd.src.txt"), encoding="utf-8") as mmd:

              assert len(mmd.read()) == 1339
@@ -971,15 +967,15 @@ 

          new_mmd = module_build_service.utils.load_mmd(self.cg._finalize_mmd("x86_64"))

  

          # Check that -devel suffix is set.

-         assert new_mmd.get_name().endswith("-devel")

+         assert new_mmd.get_module_name().endswith("-devel")

  

          # Check that -devel requires non-devel.

          for dep in new_mmd.get_dependencies():

              requires = []

-             for name, streams in dep.get_requires().items():

-                 for stream in streams.get():

+             for name in dep.get_runtime_modules():

+                 for stream in dep.get_runtime_streams(name):

                      requires.append("%s:%s" % (name, stream))

-             assert "%s:%s" % (mmd.get_name(), mmd.get_stream()) in requires

+             assert "%s:%s" % (mmd.get_module_name(), mmd.get_stream_name()) in requires

  

      @patch.dict("sys.modules", krbV=Mock())

      @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
@@ -1018,7 +1014,7 @@ 

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              'python-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64',

              'python3-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64',

              'python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src',
@@ -1065,7 +1061,7 @@ 

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              "python2-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",

              "python2-psycopg2-debug-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",

              "python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
@@ -1091,7 +1087,7 @@ 

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              "python2-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",

              "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",

              "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src"])
@@ -1120,7 +1116,7 @@ 

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              "glibc-common-2.29.9000-16.fc31.x86_64",

              "glibc-2.29.9000-16.fc31.src",

              "glibc-2.29.9000-16.fc31.x86_64",
@@ -1147,7 +1143,7 @@ 

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "aarch64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([

+         assert set(mmd.get_rpm_artifacts()) == set([

              "kernel-debuginfo-common-aarch64-5.0.9-301.fc30.aarch64",

              "kernel-5.0.9-301.fc30.src",

              "kernel-debuginfo-5.0.9-301.fc30.aarch64",
@@ -1167,9 +1163,9 @@ 

              "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")

  

          mmd = self.cg.module.mmd()

-         filter_list = Modulemd.SimpleSet()

-         filter_list.add("python2-psycopg2")

-         mmd.set_rpm_filter(filter_list)

+         for rpm in mmd.get_rpm_filters():

+             mmd.remove_rpm_filter(rpm)

+         mmd.add_rpm_filter("python2-psycopg2")

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         assert set(mmd.get_rpm_artifacts().get()) == set([])

+         assert set(mmd.get_rpm_artifacts()) == set([])

@@ -12,7 +12,7 @@ 

          "tools": [

              {

                  "name": "libmodulemd",

-                 "version": "1.4"

+                 "version": "2.3.1"

              }

          ],

          "components": [

@@ -12,7 +12,7 @@ 

          "tools": [

              {

                  "name": "libmodulemd",

-                 "version": "1.4"

+                 "version": "2.3.1"

              }

          ],

          "components": [

file modified
+16 -16
@@ -28,7 +28,6 @@ 

  

  from module_build_service.mmd_resolver import MMDResolver

  from module_build_service import Modulemd

- from module_build_service import glib

  

  

  class TestMMDResolver:
@@ -41,17 +40,14 @@ 

      @staticmethod

      def _make_mmd(nsvc, requires, xmd_buildrequires=None, virtual_streams=None):

          name, stream, version = nsvc.split(":", 2)

-         mmd = Modulemd.Module()

-         mmd.set_mdversion(2)

-         mmd.set_name(name)

-         mmd.set_stream(stream)

+         mmd = Modulemd.ModuleStreamV2.new(name, stream)

          mmd.set_summary("foo")

          mmd.set_description("foo")

-         licenses = Modulemd.SimpleSet()

-         licenses.add("GPL")

-         mmd.set_module_licenses(licenses)

+         for license_ in mmd.get_module_licenses():

+             mmd.remove_module_license(license_)

+         mmd.add_module_license("GPL")

  

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          xmd["mbs"] = {}

          xmd["mbs"]["buildrequires"] = {}

          if xmd_buildrequires:
@@ -60,14 +56,16 @@ 

                  xmd["mbs"]["buildrequires"][n] = {"stream": s}

          if virtual_streams:

              xmd["mbs"]["virtual_streams"] = virtual_streams

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

          if ":" in version:

              version, context = version.split(":")

              mmd.set_context(context)

-             add_requires = Modulemd.Dependencies.add_requires

+             add_requires = Modulemd.Dependencies.add_runtime_stream

+             add_empty_requires = Modulemd.Dependencies.set_empty_runtime_dependencies_for_module

          else:

-             add_requires = Modulemd.Dependencies.add_buildrequires

+             add_requires = Modulemd.Dependencies.add_buildtime_stream

+             add_empty_requires = Modulemd.Dependencies.set_empty_buildtime_dependencies_for_module

          mmd.set_version(int(version))

  

          if not isinstance(requires, list):
@@ -75,13 +73,15 @@ 

          else:

              requires = requires

  

-         deps_list = []

          for reqs in requires:

              deps = Modulemd.Dependencies()

              for req_name, req_streams in reqs.items():

-                 add_requires(deps, req_name, req_streams)

-             deps_list.append(deps)

-         mmd.set_dependencies(deps_list)

+                 if req_streams == []:

+                     add_empty_requires(deps, req_name)

+                 else:

+                     for req_stream in req_streams:

+                         add_requires(deps, req_name, req_stream)

+             mmd.add_dependencies(deps)

  

          return mmd

  

@@ -40,8 +40,8 @@ 

  def module_build_from_modulemd(yaml):

      mmd = load_mmd(yaml)

      build = ModuleBuild()

-     build.name = mmd.get_name()

-     build.stream = mmd.get_stream()

+     build.name = mmd.get_module_name()

+     build.stream = mmd.get_stream_name()

      build.version = mmd.get_version()

      build.state = BUILD_STATES["ready"]

      build.modulemd = yaml

@@ -24,9 +24,9 @@ 

  import pytest

  

  from mock import patch

- from module_build_service import conf, Modulemd

+ from module_build_service import conf

  from module_build_service.models import ComponentBuild, ModuleBuild, make_session

- from module_build_service.utils import to_text_type

+ from module_build_service.utils.general import load_mmd_file, mmd_to_str

  from tests import init_data as init_data_contexts, clean_database, make_module

  from tests.test_models import init_data, module_build_from_modulemd

  
@@ -68,9 +68,8 @@ 

          build = ModuleBuild.query.filter_by(id=1).one()

          yaml_path = os.path.join(

              os.path.dirname(__file__), "..", "staged_data", "testmodule_dependencies.yaml")

-         mmd = Modulemd.Module.new_from_file(yaml_path)

-         mmd.upgrade()

-         build.modulemd = to_text_type(mmd.dumps())

+         mmd = load_mmd_file(yaml_path)

+         build.modulemd = mmd_to_str(mmd)

          (

              build.ref_build_context,

              build.build_context,
@@ -89,11 +88,10 @@ 

          clean_database()

          yaml_path = os.path.join(

              os.path.dirname(__file__), "..", "staged_data", "formatted_testmodule.yaml")

-         mmd = Modulemd.Module.new_from_file(yaml_path)

-         mmd.upgrade()

+         mmd = load_mmd_file(yaml_path)

          with make_session(conf) as session:

              for i in range(3):

-                 build = module_build_from_modulemd(to_text_type(mmd.dumps()))

+                 build = module_build_from_modulemd(mmd_to_str(mmd))

                  build.build_context = "f6e2aeec7576196241b9afa0b6b22acf2b6873d" + str(i)

                  build.runtime_context = "bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c" + str(i)

                  session.add(build)

file modified
+21 -25
@@ -25,11 +25,10 @@ 

  from datetime import datetime

  from mock import patch, PropertyMock

  import pytest

- from module_build_service.utils import to_text_type

  

  import module_build_service.resolver as mbs_resolver

- from module_build_service import app, db, models, glib, utils, Modulemd

- from module_build_service.utils import import_mmd, load_mmd_file

+ from module_build_service import app, db, models, utils, Modulemd

+ from module_build_service.utils import import_mmd, load_mmd_file, mmd_to_str

  from module_build_service.models import ModuleBuild

  import tests

  
@@ -43,13 +42,10 @@ 

  

      def test_get_buildrequired_modulemds(self):

          mmd = load_mmd_file(os.path.join(base_dir, "staged_data", "platform.yaml"))

-         mmd.set_stream("f30.1.3")

+         mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")

          import_mmd(db.session, mmd)

          platform_f300103 = ModuleBuild.query.filter_by(stream="f30.1.3").one()

-         mmd.set_name("testmodule")

-         mmd.set_stream("master")

-         mmd.set_version(20170109091357)

-         mmd.set_context("123")

+         mmd = tests.make_module("testmodule:master:20170109091357:123", store_to_db=False)

          build = ModuleBuild(

              name="testmodule",

              stream="master",
@@ -65,7 +61,7 @@ 

              time_submitted=datetime(2018, 11, 15, 16, 8, 18),

              time_modified=datetime(2018, 11, 15, 16, 19, 35),

              rebuild_strategy="changed-and-after",

-             modulemd=to_text_type(mmd.dumps()),

+             modulemd=mmd_to_str(mmd),

          )

          build.buildrequires.append(platform_f300103)

          db.session.add(build)
@@ -73,8 +69,8 @@ 

  

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="db")

          result = resolver.get_buildrequired_modulemds(

-             "testmodule", "master", platform_f300103.mmd().dup_nsvc())

-         nsvcs = set([m.dup_nsvc() for m in result])

+             "testmodule", "master", platform_f300103.mmd().get_nsvc())

+         nsvcs = set([m.get_nsvc() for m in result])

          assert nsvcs == set(["testmodule:master:20170109091357:123"])

  

      @pytest.mark.parametrize("stream_versions", [False, True])
@@ -83,7 +79,7 @@ 

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="db")

          result = resolver.get_module_modulemds(

              "platform", "f29.1.0", stream_version_lte=stream_versions)

-         nsvcs = set([mmd.dup_nsvc() for mmd in result])

+         nsvcs = set([mmd.get_nsvc() for mmd in result])

          if stream_versions:

              assert nsvcs == set(["platform:f29.1.0:3:00000000", "platform:f29.0.0:3:00000000"])

          else:
@@ -101,11 +97,11 @@ 

              module = models.ModuleBuild.query.get(2)

              mmd = module.mmd()

              # Wipe out the dependencies

-             mmd.set_dependencies()

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             mmd.clear_dependencies()

+             xmd = mmd.get_xmd()

              xmd["mbs"]["buildrequires"] = {}

-             mmd.set_xmd(glib.dict_values(xmd))

-             module.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             module.modulemd = mmd_to_str(mmd)

              db.session.add(module)

              db.session.commit()

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="db")
@@ -120,21 +116,21 @@ 

          # Add testmodule2 that requires testmodule

          module = models.ModuleBuild.query.get(3)

          mmd = module.mmd()

-         mmd.set_name("testmodule2")

+         # Rename the module

+         mmd = mmd.copy("testmodule2")

          mmd.set_version(20180123171545)

-         requires = mmd.get_dependencies()[0].get_requires()

-         requires["testmodule"] = Modulemd.SimpleSet()

-         requires["testmodule"].add("master")

-         mmd.get_dependencies()[0].set_requires(requires)

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         deps = Modulemd.Dependencies()

+         deps.add_runtime_stream("testmodule", "master")

+         mmd.add_dependencies(deps)

+         xmd = mmd.get_xmd()

          xmd["mbs"]["requires"]["testmodule"] = {

              "filtered_rpms": [],

              "ref": "620ec77321b2ea7b0d67d82992dda3e1d67055b4",

              "stream": "master",

              "version": "20180205135154",

          }

-         mmd.set_xmd(glib.dict_values(xmd))

-         module.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd(xmd)

+         module.modulemd = mmd_to_str(mmd)

          module.name = "testmodule2"

          module.version = str(mmd.get_version())

          module.koji_tag = "module-ae2adf69caf0e1b6"
@@ -256,7 +252,7 @@ 

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="db")

          mmd = resolver.get_latest_with_virtual_stream("platform", "f29")

          assert mmd

-         assert mmd.get_stream() == "f29.2.0"

+         assert mmd.get_stream_name() == "f29.2.0"

  

      def test_get_latest_with_virtual_stream_none(self):

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="db")

@@ -23,10 +23,9 @@ 

  import os

  from datetime import datetime

  

- from module_build_service.utils import to_text_type

  import module_build_service.resolver as mbs_resolver

  from module_build_service import db

- from module_build_service.utils import import_mmd, load_mmd_file

+ from module_build_service.utils.general import import_mmd, load_mmd_file, mmd_to_str

  from module_build_service.models import ModuleBuild

  import tests

  
@@ -40,11 +39,10 @@ 

  

      def test_get_buildrequired_modulemds(self):

          mmd = load_mmd_file(os.path.join(base_dir, "staged_data", "platform.yaml"))

-         mmd.set_stream("f8")

+         mmd = mmd.copy(mmd.get_module_name(), "f8")

          import_mmd(db.session, mmd)

          platform_f8 = ModuleBuild.query.filter_by(stream="f8").one()

-         mmd.set_name("testmodule")

-         mmd.set_stream("master")

+         mmd = mmd.copy("testmodule", "master")

          mmd.set_version(20170109091357)

          mmd.set_context("123")

          build = ModuleBuild(
@@ -62,14 +60,14 @@ 

              time_submitted=datetime(2018, 11, 15, 16, 8, 18),

              time_modified=datetime(2018, 11, 15, 16, 19, 35),

              rebuild_strategy="changed-and-after",

-             modulemd=to_text_type(mmd.dumps()),

+             modulemd=mmd_to_str(mmd),

          )

          db.session.add(build)

          db.session.commit()

  

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="local")

          result = resolver.get_buildrequired_modulemds(

-             "testmodule", "master", platform_f8.mmd().dup_nsvc())

-         nsvcs = set([m.dup_nsvc() for m in result])

+             "testmodule", "master", platform_f8.mmd().get_nsvc())

+         nsvcs = set([m.get_nsvc() for m in result])

          assert nsvcs == set(

              ["testmodule:master:20170109091357:9c690d0e", "testmodule:master:20170109091357:123"])

file modified
+18 -13
@@ -24,8 +24,8 @@ 

  

  import module_build_service.resolver as mbs_resolver

  import module_build_service.utils

+ from module_build_service.utils.general import mmd_to_str

  import module_build_service.models

- from module_build_service import glib, app

  import tests

  

  
@@ -58,7 +58,7 @@ 

              "testmodule", "master", "20180205135154", "9c690d0e", virtual_streams=["f28"]

          )

          nsvcs = set(

-             "{}:{}:{}:{}".format(m.peek_name(), m.peek_stream(), m.peek_version(), m.peek_context())

+             m.get_nsvc()

              for m in module_mmds

          )

          expected = set(["testmodule:master:20180205135154:9c690d0e"])
@@ -112,7 +112,7 @@ 

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          ret = resolver.get_module_modulemds("testmodule", "master", version)

          nsvcs = set(

-             "{}:{}:{}:{}".format(m.peek_name(), m.peek_stream(), m.peek_version(), m.peek_context())

+             m.get_nsvc()

              for m in ret

          )

          expected = set([
@@ -217,10 +217,11 @@ 

  

          mmd = module_build_service.utils.load_mmd(testmodule_mmd_9c690d0e)

          # Wipe out the dependencies

-         mmd.set_dependencies()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         for deps in mmd.get_dependencies():

+             mmd.remove_dependencies(deps)

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"] = {}

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

          mock_res = Mock()

          mock_res.ok.return_value = True
@@ -232,7 +233,7 @@ 

                          "stream": "master",

                          "version": "20180205135154",

                          "context": "9c690d0e",

-                         "modulemd": mmd.dumps(),

+                         "modulemd": mmd_to_str(mmd),

                          "build_deps": [],

                      }

                  ],
@@ -352,7 +353,7 @@ 

          self, local_builds, conf_system, formatted_testmodule_mmd

      ):

          tests.clean_database()

-         with app.app_context():

+         with tests.app.app_context():

              module_build_service.utils.load_local_builds(["platform"])

  

              resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")
@@ -383,14 +384,18 @@ 

                          "stream": "10",

                          "version": 1,

                          "context": "c1",

-                         "modulemd": tests.make_module("nodejs:10:1:c1", store_to_db=False).dumps(),

+                         "modulemd": mmd_to_str(

+                             tests.make_module("nodejs:10:1:c1", store_to_db=False),

+                         ),

                      },

                      {

                          "name": "nodejs",

                          "stream": "10",

                          "version": 2,

                          "context": "c1",

-                         "modulemd": tests.make_module("nodejs:10:2:c1", store_to_db=False).dumps(),

+                         "modulemd": mmd_to_str(

+                             tests.make_module("nodejs:10:2:c1", store_to_db=False),

+                         ),

                      },

                  ],

                  "meta": {"next": None},
@@ -400,8 +405,8 @@ 

  

              assert 1 == len(result)

              mmd = result[0]

-             assert "nodejs" == mmd.get_name()

-             assert "10" == mmd.get_stream()

+             assert "nodejs" == mmd.get_module_name()

+             assert "10" == mmd.get_stream_name()

              assert 1 == mmd.get_version()

              assert "c1" == mmd.get_context()

  
@@ -445,7 +450,7 @@ 

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          mmd = resolver.get_latest_with_virtual_stream("platform", "virtualf28")

  

-         assert mmd.get_name() == "platform"

+         assert mmd.get_module_name() == "platform"

          mock_session.return_value.get.assert_called_once_with(

              "https://mbs.fedoraproject.org/module-build-service/1/module-builds/",

              params={

@@ -20,18 +20,16 @@ 

  #

  

  import os

- import yaml

  

  from mock import patch, PropertyMock

- from gi.repository import GLib

- from module_build_service.utils import to_text_type

  

  from tests import conf, clean_database

  from tests.test_views.test_views import FakeSCM

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.modules

- from module_build_service import build_logs, Modulemd, db

+ from module_build_service import build_logs, db

  from module_build_service.models import make_session, ModuleBuild, ComponentBuild

+ from module_build_service.utils.general import mmd_to_str, load_mmd, load_mmd_file

  

  

  class TestModuleInit:
@@ -39,12 +37,14 @@ 

          self.fn = module_build_service.scheduler.handlers.modules.init

          self.staged_data_dir = os.path.join(os.path.dirname(__file__), "../", "staged_data")

          testmodule_yml_path = os.path.join(self.staged_data_dir, "testmodule_init.yaml")

-         with open(testmodule_yml_path, "r") as f:

-             yaml = to_text_type(f.read())

+         mmd = load_mmd_file(testmodule_yml_path)

+         # Set the name and stream

+         mmd = mmd.copy("testmodule", "1")

          scmurl = "git://pkgs.domain.local/modules/testmodule?#620ec77"

          clean_database()

          with make_session(conf) as session:

-             ModuleBuild.create(session, conf, "testmodule", "1", 3, yaml, scmurl, "mprahl")

+             ModuleBuild.create(

+                 session, conf, "testmodule", "1", 3, mmd_to_str(mmd), scmurl, "mprahl")

  

      def teardown_method(self, test_method):

          try:
@@ -78,11 +78,11 @@ 

  

          platform_build = ModuleBuild.query.get(1)

          mmd = platform_build.mmd()

-         filter_list = Modulemd.SimpleSet()

-         filter_list.add("foo")

-         filter_list.add("bar")

-         mmd.set_rpm_filter(filter_list)

-         platform_build.modulemd = to_text_type(mmd.dumps())

+         for rpm in mmd.get_rpm_filters():

+             mmd.remove_rpm_filter(rpm)

+         mmd.add_rpm_filter("foo")

+         mmd.add_rpm_filter("bar")

+         platform_build.modulemd = mmd_to_str(mmd)

          db.session.commit()

  

          msg = module_build_service.messaging.MBSModule(
@@ -96,7 +96,6 @@ 

          assert build.state == 1, build.state

          # Make sure format_mmd was run properly

          xmd_mbs = build.mmd().get_xmd()["mbs"]

-         assert type(xmd_mbs) is GLib.Variant

          assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [

              "foo-0:2.4.48-3.el8+1308+551bfa71",

              "bar-0:2.5.48-3.el8+1308+551bfa71",
@@ -106,7 +105,7 @@ 

      def test_init_called_twice(self):

          build = self.test_init_basic()

          old_component_builds = len(build.component_builds)

-         old_mmd = yaml.safe_load(build.modulemd)

+         old_mmd = load_mmd(build.modulemd)

  

          build.state = 4

          db.session.commit()
@@ -116,8 +115,8 @@ 

          assert build.state == 1

          assert old_component_builds == len(build.component_builds)

  

-         new_mmd = yaml.safe_load(build.modulemd)

-         assert old_mmd == new_mmd

+         new_mmd = load_mmd(build.modulemd)

+         assert mmd_to_str(old_mmd) == mmd_to_str(new_mmd)

  

      @patch("module_build_service.scm.SCM")

      def test_init_scm_not_available(self, mocked_scm):
@@ -145,11 +144,13 @@ 

      def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo):

          FakeSCM(mocked_scm, "includedmodules", ["testmodule_init.yaml"])

          includedmodules_yml_path = os.path.join(self.staged_data_dir, "includedmodules.yaml")

-         with open(includedmodules_yml_path, "r") as f:

-             yaml = to_text_type(f.read())

+         mmd = load_mmd_file(includedmodules_yml_path)

+         # Set the name and stream

+         mmd = mmd.copy("includedmodules", "1")

          scmurl = "git://pkgs.domain.local/modules/includedmodule?#da95886"

          with make_session(conf) as session:

-             ModuleBuild.create(session, conf, "includemodule", "1", 3, yaml, scmurl, "mprahl")

+             ModuleBuild.create(

+                 session, conf, "includemodule", "1", 3, mmd_to_str(mmd), scmurl, "mprahl")

              msg = module_build_service.messaging.MBSModule(

                  msg_id=None, module_build_id=3, module_build_state="init")

              self.fn(config=conf, session=session, msg=msg)

@@ -30,6 +30,7 @@ 

  from tests import conf, db, app, scheduler_init_data

  import module_build_service.resolver

  from module_build_service import build_logs, Modulemd

+ from module_build_service.utils.general import load_mmd_file

  from module_build_service.models import ComponentBuild, ModuleBuild

  

  base_dir = os.path.dirname(os.path.dirname(__file__))
@@ -73,8 +74,7 @@ 

  

          formatted_testmodule_yml_path = os.path.join(

              base_dir, "staged_data", "formatted_testmodule.yaml")

-         mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)

-         mmd.upgrade()

+         mmd = load_mmd_file(formatted_testmodule_yml_path)

          mocked_module_build.id = 1

          mocked_module_build.mmd.return_value = mmd

          mocked_module_build.component_builds = []
@@ -191,9 +191,7 @@ 

          Test that build.cg_build_koji_tag fallbacks to default tag.

          """

          with app.app_context():

-             base_mmd = Modulemd.Module()

-             base_mmd.set_name("base-runtime")

-             base_mmd.set_stream("f27")

+             base_mmd = Modulemd.ModuleStreamV2.new("base-runtime", "f27")

  

              scheduler_init_data()

              koji_session = mock.MagicMock()
@@ -259,9 +257,7 @@ 

          Test that build.cg_build_koji_tag is set.

          """

          with app.app_context():

-             base_mmd = Modulemd.Module()

-             base_mmd.set_name("base-runtime")

-             base_mmd.set_stream("f27")

+             base_mmd = Modulemd.ModuleStreamV2.new("base-runtime", "f27")

  

              scheduler_init_data()

              koji_session = mock.MagicMock()

@@ -17,12 +17,9 @@ 

  # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

- 

- import copy

- 

  from mock import patch, Mock

  

- from module_build_service import conf, glib

+ from module_build_service import conf

  from module_build_service.utils import ursine

  from tests import make_module, clean_database

  
@@ -184,10 +181,10 @@ 

          with patch.object(conf, "koji_external_repo_url_prefix", new="http://example.com/"):

              modulemds = ursine.get_modulemds_from_ursine_content(koji_tag)

  

-         test_nsvcs = [item.dup_nsvc() for item in modulemds]

+         test_nsvcs = [item.get_nsvc() for item in modulemds]

          test_nsvcs.sort()

  

-         expected_nsvcs = [mmd_name1s2020c.mmd().dup_nsvc(), mmd_name2s2021c.mmd().dup_nsvc()]

+         expected_nsvcs = [mmd_name1s2020c.mmd().get_nsvc(), mmd_name2s2021c.mmd().get_nsvc()]

          expected_nsvcs.sort()

  

          session.getExternalRepoList.assert_called_once_with(koji_tag)
@@ -204,14 +201,14 @@ 

      ):

          xmd = {"mbs": {"buildrequires": {"modulea": {"stream": "master"}}}}

          fake_mmd = make_module("name1:s:2020:c", xmd=xmd, store_to_db=False)

-         original_xmd = glib.from_variant_dict(fake_mmd.get_xmd())

+         original_xmd = fake_mmd.get_xmd()

  

          with patch.object(ursine, "log") as log:

              ursine.handle_stream_collision_modules(fake_mmd)

              assert 2 == log.info.call_count

              find_stream_collision_modules.assert_not_called()

  

-         assert original_xmd == glib.from_variant_dict(fake_mmd.get_xmd())

+         assert original_xmd == fake_mmd.get_xmd()

  

      @patch.object(conf, "base_module_names", new=["platform"])

      @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content")
@@ -227,7 +224,7 @@ 

              }

          }

          fake_mmd = make_module("name1:s:2020:c", xmd=xmd, store_to_db=False)

-         original_xmd = glib.from_variant_dict(fake_mmd.get_xmd())

+         expected_xmd = fake_mmd.get_xmd()

  

          get_modulemds_from_ursine_content.return_value = []

  
@@ -235,11 +232,10 @@ 

              ursine.handle_stream_collision_modules(fake_mmd)

              assert 2 == log.info.call_count

  

-         expected_xmd = copy.deepcopy(original_xmd)

          # Ensure stream_collision_modules is set.

          expected_xmd["mbs"]["buildrequires"]["platform"]["stream_collision_modules"] = ""

          expected_xmd["mbs"]["buildrequires"]["platform"]["ursine_rpms"] = ""

-         assert expected_xmd == glib.from_variant_dict(fake_mmd.get_xmd())

+         assert expected_xmd == fake_mmd.get_xmd()

  

      @patch.object(conf, "base_module_names", new=["platform", "project-platform"])

      @patch("module_build_service.utils.ursine.get_modulemds_from_ursine_content")
@@ -315,7 +311,7 @@ 

  

          ursine.handle_stream_collision_modules(fake_mmd)

  

-         xmd = glib.from_variant_dict(fake_mmd.get_xmd())

+         xmd = fake_mmd.get_xmd()

          buildrequires = xmd["mbs"]["buildrequires"]

  

          modules = buildrequires["platform"]["stream_collision_modules"]
@@ -359,4 +355,4 @@ 

          get_modulemds_from_ursine_content.return_value = fake_modules

  

          modules = ursine.find_stream_collision_modules(xmd_mbs_buildrequires, "koji_tag")

-         assert [fake_modules[1].dup_nsvc()] == modules

+         assert [fake_modules[1].get_nsvc()] == modules

file modified
+95 -108
@@ -26,7 +26,7 @@ 

  from datetime import datetime

  from werkzeug.datastructures import FileStorage

  from mock import patch

- from module_build_service.utils import to_text_type, load_mmd_file

+ from module_build_service.utils.general import load_mmd_file, mmd_to_str

  import module_build_service.utils

  import module_build_service.scm

  from module_build_service import models, conf
@@ -46,7 +46,7 @@ 

  import module_build_service.scheduler.handlers.components

  from module_build_service.builder.base import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

- from module_build_service import glib, Modulemd

+ from module_build_service import Modulemd

  from tests import app

  

  BASE_DIR = path.abspath(path.dirname(__file__))
@@ -107,10 +107,8 @@ 

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          if changed_component:

              mmd = second_module_build.mmd()

-             mmd.get_rpm_components()["tangerine"].set_ref(

-                 "00ea1da4192a2030f9ae023de3b3143ed647bbab"

-             )

-             second_module_build.modulemd = to_text_type(mmd.dumps())

+             mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab")

+             second_module_build.modulemd = mmd_to_str(mmd)

              second_module_changed_component = models.ComponentBuild.query.filter_by(

                  package=changed_component, module_id=3).one()

              second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab"
@@ -149,8 +147,10 @@ 

      def test_get_reusable_component_different_rpm_macros(self):

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = second_module_build.mmd()

-         mmd.set_rpm_buildopts({"macros": "%my_macro 1"})

-         second_module_build.modulemd = to_text_type(mmd.dumps())

+         buildopts = Modulemd.Buildopts()

+         buildopts.set_rpm_macros("%my_macro 1")

+         mmd.set_buildopts(buildopts)

+         second_module_build.modulemd = mmd_to_str(mmd)

          db.session.commit()

  

          plc_rv = module_build_service.utils.get_reusable_component(
@@ -167,18 +167,18 @@ 

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          if set_current_arch:  # set architecture for current build

              mmd = second_module_build.mmd()

-             arches = Modulemd.SimpleSet()

-             arches.set(["i686"])

-             mmd.get_rpm_components()["tangerine"].set_arches(arches)

-             second_module_build.modulemd = to_text_type(mmd.dumps())

+             component = mmd.get_rpm_component("tangerine")

+             component.reset_arches()

+             component.add_restricted_arch("i686")

+             second_module_build.modulemd = mmd_to_str(mmd)

          if set_database_arch:  # set architecture for build in database

              second_module_changed_component = models.ComponentBuild.query.filter_by(

                  package="tangerine", module_id=2).one()

              mmd = second_module_changed_component.module_build.mmd()

-             arches = Modulemd.SimpleSet()

-             arches.set(["i686"])

-             mmd.get_rpm_components()["tangerine"].set_arches(arches)

-             second_module_changed_component.module_build.modulemd = to_text_type(mmd.dumps())

+             component = mmd.get_rpm_component("tangerine")

+             component.reset_arches()

+             component.add_restricted_arch("i686")

+             second_module_changed_component.module_build.modulemd = mmd_to_str(mmd)

              db.session.add(second_module_changed_component)

              db.session.commit()

  
@@ -192,10 +192,10 @@ 

          first_module_build.rebuild_strategy = rebuild_strategy

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = second_module_build.mmd()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"]["platform"]["ref"] = "da39a3ee5e6b4b0d3255bfef95601890afd80709"

-         mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd(xmd)

+         second_module_build.modulemd = mmd_to_str(mmd)

          second_module_build.ref_build_context = "37c6c57bedf4305ef41249c1794760b5cb8fad17"

          second_module_build.rebuild_strategy = rebuild_strategy

          db.session.commit()
@@ -222,10 +222,10 @@ 

          first_module_build.rebuild_strategy = rebuild_strategy

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = second_module_build.mmd()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different"

-         mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd(xmd)

+         second_module_build.modulemd = mmd_to_str(mmd)

          second_module_build.build_context = "37c6c57bedf4305ef41249c1794760b5cb8fad17"

          second_module_build.rebuild_strategy = rebuild_strategy

          db.session.commit()
@@ -244,10 +244,8 @@ 

      def test_get_reusable_component_different_buildrequires(self):

          second_module_build = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = second_module_build.mmd()

-         br_list = Modulemd.SimpleSet()

-         br_list.add("master")

-         mmd.get_dependencies()[0].set_buildrequires({"some_module": br_list})

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master")

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"] = {

              "some_module": {

                  "ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
@@ -255,8 +253,8 @@ 

                  "version": "20170123140147",

              }

          }

-         mmd.set_xmd(glib.dict_values(xmd))

-         second_module_build.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd(xmd)

+         second_module_build.modulemd = mmd_to_str(mmd)

          second_module_build.ref_build_context = "37c6c57bedf4305ef41249c1794760b5cb8fad17"

          db.session.commit()

  
@@ -284,7 +282,7 @@ 

          module_dir = tempfile.mkdtemp()

          module = models.ModuleBuild.query.filter_by(id=3).one()

          mmd = module.mmd()

-         modulemd_yaml = to_text_type(mmd.dumps())

+         modulemd_yaml = mmd_to_str(mmd)

          modulemd_file_path = path.join(module_dir, "testmodule.yaml")

  

          username = "test"
@@ -300,8 +298,8 @@ 

              mock_submit_args = mock_submit.call_args[0]

              username_arg = mock_submit_args[0]

              mmd_arg = mock_submit_args[1]

-             assert mmd_arg.get_stream() == stream

-             assert "\n\n%__spec_check_pre exit 0\n" in mmd_arg.get_rpm_buildopts()["macros"]

+             assert mmd_arg.get_stream_name() == stream

+             assert "\n\n%__spec_check_pre exit 0\n" in mmd_arg.get_buildopts().get_rpm_macros()

              assert username_arg == username

          rmtree(module_dir)

  
@@ -318,9 +316,9 @@ 

          mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

          mmd.set_context(context)

  

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          xmd["mbs"]["koji_tag"] = "foo"

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

          build, msgs = module_build_service.utils.import_mmd(db.session, mmd)

  
@@ -333,10 +331,8 @@ 

              assert build.context == models.DEFAULT_MODULE_CONTEXT

  

      def test_import_mmd_multiple_dependencies(self):

-         mmd = Modulemd.Module().new_from_file(

-             path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

-         mmd.upgrade()

-         mmd.add_dependencies(mmd.get_dependencies()[0])

+         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

+         mmd.add_dependencies(mmd.get_dependencies()[0].copy())

  

          expected_error = "The imported module's dependencies list should contain just one element"

          with pytest.raises(UnprocessableEntity) as e:
@@ -344,12 +340,10 @@ 

              assert str(e.value) == expected_error

  

      def test_import_mmd_no_xmd_buildrequires(self):

-         mmd = Modulemd.Module().new_from_file(

-             path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

-         mmd.upgrade()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

+         xmd = mmd.get_xmd()

          del xmd["mbs"]["buildrequires"]

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

          expected_error = (

              "The imported module buildrequires other modules, but the metadata in the "
@@ -360,18 +354,16 @@ 

              assert str(e.value) == expected_error

  

      def test_import_mmd_minimal_xmd_from_local_repository(self):

-         mmd = Modulemd.Module().new_from_file(

-             path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

-         mmd.upgrade()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))

+         xmd = mmd.get_xmd()

          xmd["mbs"] = {}

          xmd["mbs"]["koji_tag"] = "repofile:///etc/yum.repos.d/fedora-modular.repo"

          xmd["mbs"]["mse"] = True

          xmd["mbs"]["commit"] = "unknown"

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

  

          build, msgs = module_build_service.utils.import_mmd(db.session, mmd, False)

-         assert build.name == mmd.get_name()

+         assert build.name == mmd.get_module_name()

  

      @pytest.mark.parametrize(

          "stream, disttag_marking, error_msg",
@@ -386,15 +378,13 @@ 

      )

      def test_import_mmd_base_module(self, stream, disttag_marking, error_msg):

          clean_database(add_platform_module=False)

-         mmd = Modulemd.Module().new_from_file(

-             path.join(BASE_DIR, "..", "staged_data", "platform.yaml"))

-         mmd.upgrade()

-         mmd.set_stream(stream)

+         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "platform.yaml"))

+         mmd = mmd.copy(mmd.get_module_name(), stream)

  

          if disttag_marking:

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              xmd["mbs"]["disttag_marking"] = disttag_marking

-             mmd.set_xmd(glib.dict_values(xmd))

+             mmd.set_xmd(xmd)

  

          if error_msg:

              with pytest.raises(UnprocessableEntity, match=error_msg):
@@ -423,10 +413,10 @@ 

          # Set the disttag_marking override on the platform

          platform = models.ModuleBuild.query.filter_by(name="platform", stream="f28").first()

          platform_mmd = platform.mmd()

-         platform_xmd = glib.from_variant_dict(platform_mmd.get_xmd())

+         platform_xmd = platform_mmd.get_xmd()

          platform_xmd["mbs"]["disttag_marking"] = "fedora28"

-         platform_mmd.set_xmd(glib.dict_values(platform_xmd))

-         platform.modulemd = to_text_type(platform_mmd.dumps())

+         platform_mmd.set_xmd(platform_xmd)

+         platform.modulemd = mmd_to_str(platform_mmd)

          db.session.add(platform)

          db.session.commit()

  
@@ -454,10 +444,9 @@ 

  

          build_one = models.ModuleBuild.query.get(2)

          mmd = build_one.mmd()

-         dep = mmd.get_dependencies()[0]

-         dep.add_buildrequires("build", ["product1.2"])

-         mmd.set_dependencies((dep,))

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         deps = mmd.get_dependencies()[0]

+         deps.add_buildtime_stream("build", "product1.2")

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"]["build"] = {

              "filtered_rpms": [],

              "ref": "virtual",
@@ -465,8 +454,8 @@ 

              "version": "1",

              "context": "00000000",

          }

-         mmd.set_xmd(glib.dict_values(xmd))

-         build_one.modulemd = to_text_type(mmd.dumps())

+         mmd.set_xmd(xmd)

+         build_one.modulemd = mmd_to_str(mmd)

          db.session.add(build_one)

          db.session.commit()

  
@@ -516,20 +505,21 @@ 

              return hashes_returned[ref]

  

          mocked_scm.return_value.get_latest = mocked_get_latest

-         mmd = Modulemd.Module().new_from_file(

-             path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml"))

-         mmd.upgrade()

+         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml"))

          # Modify the component branches so we can identify them later on

-         mmd.get_rpm_components()["perl-Tangerine"].set_ref("f28")

-         mmd.get_rpm_components()["tangerine"].set_ref("f27")

+         mmd.get_rpm_component("perl-Tangerine").set_ref("f28")

+         mmd.get_rpm_component("tangerine").set_ref("f27")

          module_build_service.utils.format_mmd(mmd, scmurl)

  

          # Make sure that original refs are not changed.

-         mmd_pkg_refs = [pkg.get_ref() for pkg in mmd.get_rpm_components().values()]

+         mmd_pkg_refs = [

+             mmd.get_rpm_component(pkg_name).get_ref()

+             for pkg_name in mmd.get_rpm_component_names()

+         ]

          assert set(mmd_pkg_refs) == set(hashes_returned.keys())

-         br = mmd.get_dependencies()[0].get_buildrequires()

-         assert list(br.keys()) == ["platform"]

-         assert list(br.values())[0].get() == ["f28"]

+         deps = mmd.get_dependencies()[0]

+         assert deps.get_buildtime_modules() == ["platform"]

+         assert deps.get_buildtime_streams("platform") == ["f28"]

          xmd = {

              "mbs": {

                  "commit": "",
@@ -544,7 +534,7 @@ 

          if scmurl:

              xmd["mbs"]["commit"] = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"

              xmd["mbs"]["scmurl"] = scmurl

-         mmd_xmd = glib.from_variant_dict(mmd.get_xmd())

+         mmd_xmd = mmd.get_xmd()

          assert mmd_xmd == xmd

  

      def test_get_reusable_component_shared_userspace_ordering(self):
@@ -675,9 +665,8 @@ 

              ]

  

              testmodule_mmd_path = path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml")

-             mmd = Modulemd.Module().new_from_file(testmodule_mmd_path)

-             mmd.upgrade()

-             mmd.set_name("testmodule-variant")

+             mmd = load_mmd_file(testmodule_mmd_path)

+             mmd = mmd.copy("testmodule-variant", "master")

              module_build = module_build_service.models.ModuleBuild()

              module_build.name = "testmodule-variant"

              module_build.stream = "master"
@@ -690,17 +679,15 @@ 

              module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)

              module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)

              module_build.rebuild_strategy = "changed-and-after"

-             module_build.modulemd = to_text_type(mmd.dumps())

+             module_build.modulemd = mmd_to_str(mmd)

              db.session.add(module_build)

              db.session.commit()

              # Rename the the modulemd to include

-             mmd.set_name("testmodule")

+             mmd = mmd.copy("testmodule")

              # Remove perl-Tangerine and tangerine from the modulemd to include so only one

              # component conflicts

-             comps = mmd.get_rpm_components()

-             del comps["perl-Tangerine"]

-             del comps["tangerine"]

-             mmd.set_rpm_components(comps)

+             mmd.remove_rpm_component("perl-Tangerine")

+             mmd.remove_rpm_component("tangerine")

  

              error_msg = (

                  'The included module "testmodule" in "testmodule-variant" have '
@@ -724,8 +711,9 @@ 

              ]

  

              testmodule_mmd_path = path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml")

-             mmd = Modulemd.Module().new_from_file(testmodule_mmd_path)

-             mmd.upgrade()

+             mmd = load_mmd_file(testmodule_mmd_path)

+             # Set the module name and stream

+             mmd = mmd.copy("testmodule", "master")

              module_build = module_build_service.models.ModuleBuild()

              module_build.name = "testmodule"

              module_build.stream = "master"
@@ -738,7 +726,7 @@ 

              module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)

              module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)

              module_build.rebuild_strategy = "changed-and-after"

-             module_build.modulemd = to_text_type(mmd.dumps())

+             module_build.modulemd = mmd_to_str(mmd)

              db.session.add(module_build)

              db.session.commit()

  
@@ -766,26 +754,26 @@ 

              testmodule_mmd_path = path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml")

              test_archs = ["powerpc", "i486"]

  

-             mmd1 = Modulemd.Module().new_from_file(testmodule_mmd_path)

-             mmd1.upgrade()

- 

+             mmd1 = load_mmd_file(testmodule_mmd_path)

              module_build_service.utils.format_mmd(mmd1, None)

  

-             for pkg in mmd1.get_rpm_components().values():

-                 assert set(pkg.get_arches().get()) == set(conf.arches)

+             for pkg_name in mmd1.get_rpm_component_names():

+                 pkg = mmd1.get_rpm_component(pkg_name)

+                 assert set(pkg.get_arches()) == set(conf.arches)

  

-             mmd2 = Modulemd.Module().new_from_file(testmodule_mmd_path)

-             mmd2.upgrade()

+             mmd2 = load_mmd_file(testmodule_mmd_path)

  

-             new_arches = Modulemd.SimpleSet()

-             new_arches.set(test_archs)

-             for pkg in mmd2.get_rpm_components().values():

-                 pkg.set_arches(new_arches)

+             for pkg_name in mmd2.get_rpm_component_names():

+                 pkg = mmd2.get_rpm_component(pkg_name)

+                 pkg.reset_arches()

+                 for arch in test_archs:

+                     pkg.add_restricted_arch(arch)

  

              module_build_service.utils.format_mmd(mmd2, None)

  

-             for pkg in mmd2.get_rpm_components().values():

-                 assert set(pkg.get_arches().get()) == set(test_archs)

+             for pkg_name in mmd2.get_rpm_component_names():

+                 pkg = mmd2.get_rpm_component(pkg_name)

+                 assert set(pkg.get_arches()) == set(test_archs)

  

      @patch("module_build_service.scm.SCM")

      @patch("module_build_service.utils.submit.ThreadPool")
@@ -802,8 +790,7 @@ 

  

              testmodule_mmd_path = path.join(BASE_DIR, "..", "staged_data", "testmodule.yaml")

  

-             mmd1 = Modulemd.Module().new_from_file(testmodule_mmd_path)

-             mmd1.upgrade()

+             mmd1 = load_mmd_file(testmodule_mmd_path)

  

              with patch("module_build_service.utils.submit.datetime") as dt:

                  dt.utcnow.return_value = test_datetime
@@ -863,9 +850,9 @@ 

          scheduler_init_data(1)

          build_one = models.ModuleBuild.query.get(2)

          mmd = build_one.mmd()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          xmd["mbs"]["buildrequires"]["platform"]["stream"] = "fl7.0.1-beta"

-         mmd.set_xmd(glib.dict_values(xmd))

+         mmd.set_xmd(xmd)

          v = module_build_service.utils.submit.get_prefixed_version(mmd)

          assert v == 7000120180205135154

  
@@ -886,7 +873,7 @@ 

          generate_expanded_mmds.return_value = [mmd1, mmd2]

  

          # Create a copy of mmd1 without xmd.mbs, since that will cause validate_mmd to fail

-         mmd1_copy = module_build_service.utils.load_mmd(mmd1.dumps())

+         mmd1_copy = mmd1.copy()

          mmd1_copy.set_xmd({})

          builds = module_build_service.utils.submit_module_build("foo", mmd1_copy, {})

          ret = {b.mmd().get_context(): b.state for b in builds}
@@ -1339,7 +1326,7 @@ 

          assert module_build

  

          mmd = module_build.mmd()

-         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd = mmd.get_xmd()

          assert xmd == {

              "mbs": {

                  "buildrequires": {},
@@ -1350,16 +1337,16 @@ 

              }

          }

  

-         profiles = mmd.get_profiles()

-         assert set(profiles.keys()) == set(["buildroot", "srpm-buildroot"])

+         assert set(mmd.get_profile_names()) == set(["buildroot", "srpm-buildroot"])

  

      @patch("module_build_service.utils.general.open", create=True, new_callable=mock.mock_open)

      def test_import_builds_from_local_dnf_repos(self, patched_open):

          with patch("dnf.Base") as dnf_base:

              repo = mock.MagicMock()

              repo.repofile = "/etc/yum.repos.d/foo.repo"

-             with open(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml")) as f:

-                 repo.get_metadata_content.return_value = f.read()

+             tm_path = path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml")

+             mmd = load_mmd_file(tm_path)

+             repo.get_metadata_content.return_value = mmd_to_str(mmd)

              base = dnf_base.return_value

              base.repos = {"reponame": repo}

              patched_open.return_value.readlines.return_value = ("FOO=bar", "PLATFORM_ID=platform:x")

@@ -23,7 +23,7 @@ 

  import pytest

  

  import module_build_service.utils

- from module_build_service import glib

+ from module_build_service import Modulemd

  from module_build_service.errors import StreamAmbigous

  from tests import db, clean_database, make_module, init_data, base_dir

  
@@ -45,7 +45,7 @@ 

          module_build_service.utils.expand_mse_streams(db.session, mmd)

          modules = module_build_service.utils.get_mmds_required_by_module_recursively(mmd)

          nsvcs = [

-             ":".join([m.get_name(), m.get_stream(), str(m.get_version()), m.get_context()])

+             m.get_nsvc()

              for m in modules

          ]

          return nsvcs
@@ -130,13 +130,6 @@ 

              ),

              (

                  {"gtk": ["1"], "foo": ["1"]},

-                 {"platform": ["f28"], "gtk": ["-1", "1"], "foo": ["-2", "1"]},

-                 False,

-                 set([frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])]),

-                 set([frozenset(["foo:1", "gtk:1", "platform:f28"])]),

-             ),

-             (

-                 {"gtk": ["1"], "foo": ["1"]},

                  {"platform": ["f28"], "gtk": ["1"]},

                  False,

                  set([frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])]),
@@ -193,7 +186,7 @@ 

          buildrequires_per_mmd_xmd = set()

          buildrequires_per_mmd_buildrequires = set()

          for mmd in mmds:

-             xmd = glib.from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              br_nsvcs = []

              for name, detail in xmd["mbs"]["buildrequires"].items():

                  br_nsvcs.append(
@@ -204,8 +197,8 @@ 

  

              buildrequires = set()

              dep = mmd.get_dependencies()[0]

-             for req_name, req_streams in dep.get_buildrequires().items():

-                 for req_stream in req_streams.get():

+             for req_name in dep.get_buildtime_modules():

+                 for req_stream in dep.get_buildtime_streams(req_name):

                      buildrequires.add(":".join([req_name, req_stream]))

              buildrequires_per_mmd_buildrequires.add(frozenset(buildrequires))

  
@@ -236,11 +229,6 @@ 

                  set([frozenset(["foo:1", "gtk:1"])]),

              ),

              (

-                 {"gtk": ["-1", "1"], "foo": ["-2", "1"]},

-                 {"platform": [], "gtk": ["-1", "1"], "foo": ["-2", "1"]},

-                 set([frozenset(["foo:1", "gtk:1"])]),

-             ),

-             (

                  {"gtk": [], "foo": []},

                  {"platform": [], "gtk": ["1"], "foo": ["1"]},

                  set([frozenset([])]),
@@ -257,8 +245,8 @@ 

              assert len(mmd.get_dependencies()) == 1

              mmd_requires = set()

              dep = mmd.get_dependencies()[0]

-             for req_name, req_streams in dep.get_requires().items():

-                 for req_stream in req_streams.get():

+             for req_name in dep.get_runtime_modules():

+                 for req_stream in dep.get_runtime_streams(req_name):

                      mmd_requires.add(":".join([req_name, req_stream]))

              requires_per_mmd.add(frozenset(mmd_requires))

  
@@ -327,18 +315,6 @@ 

                      "gtk:1:0:c3",

                  ],

              ),

-             (

-                 {},

-                 {"platform": [], "gtk": ["-1", "1"], "foo": ["-2", "1"]},

-                 [

-                     "foo:1:0:c2",

-                     "foo:1:0:c3",

-                     "platform:f29:0:c11",

-                     "platform:f28:0:c10",

-                     "gtk:1:0:c2",

-                     "gtk:1:0:c3",

-                 ],

-             ),

          ],

      )

      def test_get_required_modules_simple(self, requires, build_requires, expected):
@@ -427,11 +403,14 @@ 

          init_data(data_size=1, multiple_stream_versions=True)

          mmd = module_build_service.utils.load_mmd_file(

              os.path.join(base_dir, "staged_data", "testmodule_v2.yaml"))

-         deps = mmd.get_dependencies()

-         brs = deps[0].get_buildrequires()

-         brs["platform"].set(["f29.1.0", "f29.2.0"])

-         deps[0].set_buildrequires(brs)

-         mmd.set_dependencies(deps)

+         deps = mmd.get_dependencies()[0]

+         new_deps = Modulemd.Dependencies()

+         for stream in deps.get_runtime_streams("platform"):

+             new_deps.add_runtime_stream("platform", stream)

+         new_deps.add_buildtime_stream("platform", "f29.1.0")

+         new_deps.add_buildtime_stream("platform", "f29.2.0")

+         mmd.remove_dependencies(deps)

+         mmd.add_dependencies(new_deps)

  

          mmds = module_build_service.utils.mse._get_base_module_mmds(mmd)

          expected = set(["platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"])
@@ -440,7 +419,7 @@ 

          # Verify the expected ones were returned

          actual = set()

          for mmd_ in mmds:

-             actual.add("{}:{}".format(mmd_.get_name(), mmd_.get_stream()))

+             actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))

          assert actual == expected

  

      @pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"]))
@@ -449,11 +428,13 @@ 

          init_data(data_size=1, multiple_stream_versions=True)

          mmd = module_build_service.utils.load_mmd_file(

              os.path.join(base_dir, "staged_data", "testmodule_v2.yaml"))

-         deps = mmd.get_dependencies()

-         brs = deps[0].get_buildrequires()

-         brs["platform"].set(["f29.2.0"])

-         deps[0].set_buildrequires(brs)

-         mmd.set_dependencies(deps)

+         deps = mmd.get_dependencies()[0]

+         new_deps = Modulemd.Dependencies()

+         for stream in deps.get_runtime_streams("platform"):

+             new_deps.add_runtime_stream("platform", stream)

+         new_deps.add_buildtime_stream("platform", "f29.2.0")

+         mmd.remove_dependencies(deps)

+         mmd.add_dependencies(new_deps)

  

          make_module("platform:lp29.1.1:12:c11", {}, {}, virtual_streams=virtual_streams)

  
@@ -468,5 +449,5 @@ 

          # Verify the expected ones were returned

          actual = set()

          for mmd_ in mmds:

-             actual.add("{}:{}".format(mmd_.get_name(), mmd_.get_stream()))

+             actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))

          assert actual == expected

file modified
+23 -23
@@ -32,7 +32,6 @@ 

  from requests.utils import quote

  import hashlib

  import pytest

- from module_build_service.utils import to_text_type, load_mmd_file, load_mmd

  import re

  

  from tests import app, init_data, clean_database, reuse_component_init_data
@@ -43,8 +42,9 @@ 

  from module_build_service import db, version

  import module_build_service.config as mbs_config

  import module_build_service.scheduler.handlers.modules

- from module_build_service.utils import import_mmd

- from module_build_service.glib import dict_values, from_variant_dict

+ from module_build_service.utils.general import (

+     import_mmd, mmd_to_str, to_text_type, load_mmd_file, load_mmd

+ )

  

  

  user = ("Homer J. Simpson", set(["packager"]))
@@ -806,7 +806,7 @@ 

      def test_query_base_module_br_filters(self):

          reuse_component_init_data()

          mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))

-         mmd.set_stream("f30.1.3")

+         mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")

          import_mmd(db.session, mmd)

          platform_f300103 = ModuleBuild.query.filter_by(stream="f30.1.3").one()

          build = ModuleBuild(
@@ -1522,8 +1522,8 @@ 

          mmd = module_build_service.utils.load_mmd(data[0]["modulemd"])

          assert len(mmd.get_dependencies()) == 1

          dep = mmd.get_dependencies()[0]

-         assert set(dep.get_buildrequires()["platform"].get()) == expected_br

-         assert set(dep.get_requires()["platform"].get()) == expected_req

+         assert set(dep.get_buildtime_streams("platform")) == expected_br

+         assert set(dep.get_runtime_streams("platform")) == expected_req

  

      @patch("module_build_service.auth.get_user", return_value=user)

      @patch("module_build_service.scm.SCM")
@@ -1994,13 +1994,13 @@ 

          with make_session(conf) as session:

              build = ModuleBuild.query.first()

              mmd = build.mmd()

-             xmd = from_variant_dict(mmd.get_xmd())

+             xmd = mmd.get_xmd()

              mbs = xmd.setdefault("mbs", {})

              buildrequires = mbs.setdefault("buildrequires", {})

              buildrequires["modulea"] = br_modulea

              buildrequires["moduleb"] = br_moduleb

-             mmd.set_xmd(dict_values(xmd))

-             build.modulemd = to_text_type(mmd.dumps())

+             mmd.set_xmd(xmd)

+             build.modulemd = mmd_to_str(mmd)

              session.commit()

  

          rv = self.client.get("/module-build-service/1/module-builds/{}".format(build.id))
@@ -2264,11 +2264,11 @@ 

          # Create a platform for whatever the override is so the build submission succeeds

          if platform_override:

              platform_mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))

-             platform_mmd.set_stream(platform_override)

+             platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), platform_override)

              if platform_override == "el8.0.0":

-                 xmd = from_variant_dict(platform_mmd.get_xmd())

+                 xmd = platform_mmd.get_xmd()

                  xmd["mbs"]["virtual_streams"] = ["el8"]

-                 platform_mmd.set_xmd(dict_values(xmd))

+                 platform_mmd.set_xmd(xmd)

              import_mmd(db.session, platform_mmd)

  

          FakeSCM(
@@ -2291,9 +2291,9 @@ 

              expected_br = {platform_override}

          else:

              expected_br = {"f28"}

-         assert set(dep.get_buildrequires()["platform"].get()) == expected_br

+         assert set(dep.get_buildtime_streams("platform")) == expected_br

          # The requires should not change

-         assert set(dep.get_requires()["platform"].get()) == {"f28"}

+         assert dep.get_runtime_streams("platform") == ["f28"]

  

      @patch("module_build_service.auth.get_user", return_value=user)

      @patch("module_build_service.scm.SCM")
@@ -2311,7 +2311,7 @@ 

          init_data(data_size=1, multiple_stream_versions=True)

          # Create a platform for the override so the build submission succeeds

          platform_mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))

-         platform_mmd.set_stream("product1.3")

+         platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "product1.3")

          import_mmd(db.session, platform_mmd)

  

          FakeSCM(
@@ -2337,9 +2337,9 @@ 

          dep = mmd.get_dependencies()[0]

          # The buildrequire_override value should take precedence over the stream override from

          # parsing the branch

-         assert set(dep.get_buildrequires()["platform"].get()) == {"product1.3"}

+         assert dep.get_buildtime_streams("platform") == ["product1.3"]

          # The requires should not change

-         assert set(dep.get_requires()["platform"].get()) == {"f28"}

+         assert dep.get_runtime_streams("platform") == ["f28"]

  

      @patch("module_build_service.auth.get_user", return_value=user)

      @patch("module_build_service.scm.SCM")
@@ -2350,7 +2350,7 @@ 

          """

          init_data(data_size=1, multiple_stream_versions=True)

          platform_mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))

-         platform_mmd.set_stream("el8.0.0")

+         platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "el8.0.0")

          import_mmd(db.session, platform_mmd)

  

          FakeSCM(
@@ -2408,10 +2408,10 @@ 

      def test_submit_build_request_platform_virtual_stream(self, mocked_scm, mocked_get_user):

          # Create a platform with el8.25.0 but with the virtual stream el8

          mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))

-         mmd.set_stream("el8.25.0")

-         xmd = from_variant_dict(mmd.get_xmd())

+         mmd = mmd.copy(mmd.get_module_name(), "el8.25.0")

+         xmd = mmd.get_xmd()

          xmd["mbs"]["virtual_streams"] = ["el8"]

-         mmd.set_xmd(dict_values(xmd))

+         mmd.set_xmd(xmd)

          import_mmd(db.session, mmd)

  

          # Use a testmodule that buildrequires platform:el8
@@ -2434,5 +2434,5 @@ 

          mmd = load_mmd(data[0]["modulemd"])

          assert len(mmd.get_dependencies()) == 1

          dep = mmd.get_dependencies()[0]

-         assert set(dep.get_buildrequires()["platform"].get()) == set(["el8.25.0"])

-         assert set(dep.get_requires()["platform"].get()) == set(["el8"])

+         assert dep.get_buildtime_streams("platform") == ["el8.25.0"]

+         assert dep.get_runtime_streams("platform") == ["el8"]

This also moves the methods load_mmd and load_mmd_file to module_build_service.utils.general.

This also removes some MSE unit tests with a mix of positive and negative streams since this is not supported in libmodulemd v2. The user will be presented with a syntax error if they try to submit such a modulemd file.

rebased onto 34c41111ff8cebc37744b50c377bfe96bbab5526

4 years ago

Build 5ea7a1db5e83ecaea7516146d1e1a655372cd193 FAILED!
Rebase or make new commits to rebuild.

Build 5ea7a1db5e83ecaea7516146d1e1a655372cd193 FAILED!
Rebase or make new commits to rebuild.

rebased onto 14098ce

4 years ago

1 new commit added

  • Pin the RPMs for libmodulemd v2 in the Jenkins job
4 years ago

This also removes some MSE unit tests with a mix of positive and negative streams since this is not supported in libmodulemd v2. The user will be presented with a syntax error if they try to submit such a modulemd file.

For the record, this was not supposed to be supported in v1 either; if it was accepting that, it was an oversight and not a feature. Mixed positive and negative was always disallowed by the specification.

@jkaluza @sgallagh could you please review?

Taking a look.

If preferred, I can pretty easily add a .clear_content_license() and .clear_rpm_artifacts() method to libmodulemd v2.

Build 7fe3dc34fca88a3139cbf42573ae930d9f7e8bdc FAILED!
Rebase or make new commits to rebuild.

It'll get caught below, but note that if (somehow) you read a modulemd document from a version higher than Modulemd.ModuleStreamVersionEnum.TWO, this will also throw an exception. We might want to handle that case explicitly, but maybe it's fine to just treat that as "the modulemd is invalid" for now.

OK, I've at least scanned through all of the non-test code and it looks fine to me. I had one slight nitpick, but it's not worth holding up this PR for. :thumbsup:

6 new commits added

  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

6 new commits added

  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

Build 31b1da6280389b9efff61cd00b862535f589abb9 FAILED!
Rebase or make new commits to rebuild.

Build 31b1da6280389b9efff61cd00b862535f589abb9 FAILED!
Rebase or make new commits to rebuild.

1 new commit added

  • Provide a better error message when the modulemd version is unsupported
4 years ago

Build 6f905be47386e03e079ba09f06f968faa74070b2 FAILED!
Rebase or make new commits to rebuild.

7 new commits added

  • Provide a better error message when the modulemd version is unsupported
  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

It'll get caught below, but note that if (somehow) you read a modulemd document from a version higher than Modulemd.ModuleStreamVersionEnum.TWO, this will also throw an exception. We might want to handle that case explicitly, but maybe it's fine to just treat that as "the modulemd is invalid" for now.

Good point. I just added a commit that should address this. Could you please take a look?

7 new commits added

  • Provide a better error message when the modulemd version is unsupported
  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

It looks good to me. I hope there is no missing libmodulemd v1 call hidden somewhere in a code which is not tested...

It'll get caught below, but note that if (somehow) you read a modulemd document from a version higher than Modulemd.ModuleStreamVersionEnum.TWO, this will also throw an exception. We might want to handle that case explicitly, but maybe it's fine to just treat that as "the modulemd is invalid" for now.

Good point. I just added a commit that should address this. Could you please take a look?

As a general rule, I’d be consistent about the use of the ENUM rather than using a literal “2” some places, if only to make it easier to locate them all if and when we do support a V3 format.

7 new commits added

  • Provide a better error message when the modulemd version is unsupported
  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

7 new commits added

  • Provide a better error message when the modulemd version is unsupported
  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

It'll get caught below, but note that if (somehow) you read a modulemd document from a version higher than Modulemd.ModuleStreamVersionEnum.TWO, this will also throw an exception. We might want to handle that case explicitly, but maybe it's fine to just treat that as "the modulemd is invalid" for now.
Good point. I just added a commit that should address this. Could you please take a look?

As a general rule, I’d be consistent about the use of the ENUM rather than using a literal “2” some places, if only to make it easier to locate them all if and when we do support a V3 format.

Good point. I refactored the commit to use a variable for the target modulemd version.

Build c9b571596a21079d27ab8bbacb444053e293baa8 FAILED!
Rebase or make new commits to rebuild.

7 new commits added

  • Provide a better error message when the modulemd version is unsupported
  • Pin the RPMs for libmodulemd v2 in the Jenkins job
  • Fix the Python 3 unit tests
  • Add krb5-devel as a dependency in Vagrant to fix the installation of Python dependencies
  • Raise a ValidationError when buildonly or buildafter are set
  • Clean up test_format_mmd
  • Migrate to libmodulemd v2
4 years ago

Build 40e0e8f FAILED!
Rebase or make new commits to rebuild.

Okay, the C3I tests passed. Thanks for the reviews. I'll merge this.

Pull-Request has been merged by mprahl

4 years ago

Build 40e0e8f FAILED!
Rebase or make new commits to rebuild.

Metadata
Changes Summary 44
+7 -1
file changed
Vagrantfile
+5 -1
file changed
docker/Dockerfile-tests
+4 -1
file changed
docker/Dockerfile-tests-py3
+1 -1
file changed
module_build_service/__init__.py
+41 -39
file changed
module_build_service/builder/KojiContentGenerator.py
+19 -11
file changed
module_build_service/builder/KojiModuleBuilder.py
+21 -9
file changed
module_build_service/builder/MockModuleBuilder.py
-110
file removed
module_build_service/glib.py
+2 -2
file changed
module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py
+2 -2
file changed
module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py
+2 -2
file changed
module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py
+9 -10
file changed
module_build_service/migrations/versions/9ca1c166f426_contexts.py
+5 -6
file changed
module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py
+23 -19
file changed
module_build_service/mmd_resolver.py
+7 -12
file changed
module_build_service/models.py
+18 -18
file changed
module_build_service/resolver/DBResolver.py
+14 -17
file changed
module_build_service/resolver/MBSResolver.py
+5 -3
file changed
module_build_service/scheduler/handlers/modules.py
+114 -29
file changed
module_build_service/utils/general.py
+86 -80
file changed
module_build_service/utils/mse.py
+21 -10
file changed
module_build_service/utils/reuse.py
+152 -129
file changed
module_build_service/utils/submit.py
+8 -8
file changed
module_build_service/utils/ursine.py
+9 -1
file changed
openshift/backend/Dockerfile
+0 -1
file changed
test-requirements.txt
+47 -41
file changed
tests/__init__.py
+7 -11
file changed
tests/conftest.py
+23 -22
file changed
tests/test_builder/test_koji.py
+7 -8
file changed
tests/test_builder/test_mock.py
+59 -63
file changed
tests/test_content_generator.py
+1 -1
file changed
tests/test_get_generator_json_expected_output.json
+1 -1
file changed
tests/test_get_generator_json_expected_output_with_log.json
+16 -16
file changed
tests/test_mmd_resolver.py
+2 -2
file changed
tests/test_models/__init__.py
+6 -8
file changed
tests/test_models/test_models.py
+21 -25
file changed
tests/test_resolver/test_db.py
+6 -8
file changed
tests/test_resolver/test_local.py
+18 -13
file changed
tests/test_resolver/test_mbs.py
+20 -19
file changed
tests/test_scheduler/test_module_init.py
+4 -8
file changed
tests/test_scheduler/test_module_wait.py
+9 -13
file changed
tests/test_utils/test_ursine.py
+95 -108
file changed
tests/test_utils/test_utils.py
+24 -43
file changed
tests/test_utils/test_utils_mse.py
+23 -23
file changed
tests/test_views/test_views.py