#1193 Merge "offline-builds" to "master".
Merged 4 years ago by mprahl. Opened 4 years ago by jkaluza.

file modified
+6
@@ -139,6 +139,12 @@ 

  

      ALLOW_CUSTOM_SCMURLS = True

      RESOLVER = 'mbs'

+     RPMS_ALLOW_REPOSITORY = True

+     MODULES_ALLOW_REPOSITORY = True

+ 

+ 

+ class OfflineLocalBuildConfiguration(LocalBuildConfiguration):

+     RESOLVER = 'local'

  

  

  class DevConfiguration(LocalBuildConfiguration):

file modified
+1
@@ -8,6 +8,7 @@ 

  config_opts['package_manager'] = 'dnf'

  config_opts['nosync'] = True

  config_opts['use_bootstrap_container'] = False

+ config_opts['module_enable'] = $enabled_modules

  

  config_opts['yum.conf'] = """

  $yum_conf

file modified
+1
@@ -12,6 +12,7 @@ 

  install_weak_deps=0

  metadata_expire=3600

  mdpolicy=group:primary

+ module_platform_id=$module_platform_id

  

  # repos

  

@@ -534,8 +534,9 @@ 

          log.info("%r buildroot sucessfully connected." % self)

  

      def buildroot_add_repos(self, dependencies):

-         log.info("%r adding deps on %r" % (self, dependencies))

-         self._koji_add_many_tag_inheritance(self.module_build_tag, dependencies)

+         koji_tags = dependencies.keys()

+         log.info("%r adding deps on %r" % (self, koji_tags))

+         self._koji_add_many_tag_inheritance(self.module_build_tag, koji_tags)

  

      def _get_tagged_nvrs(self, tag):

          """

@@ -90,6 +90,7 @@ 

          self.tag_name = tag_name

          self.config = config

          self.groups = []

+         self.enabled_modules = []

          self.yum_conf = MockModuleBuilder.yum_config_template

          self.koji_session = None

  
@@ -170,8 +171,7 @@ 

          pkglist_f = open(pkglist, "w")

  

          # Generate the mmd the same way as pungi does.

-         m1 = models.ModuleBuild.query.filter(models.ModuleBuild.name == self.module_str).one()

-         m1_mmd = m1.mmd()

+         m1_mmd = self.module.mmd()

          artifacts = Modulemd.SimpleSet()

  

          rpm_files = [f
@@ -192,7 +192,7 @@ 

              for rpm_file, nevra in zip(rpm_files, nevras):

                  name, epoch, version, release, arch = nevra.split()

  

-                 if m1.last_batch_id() == m1.batch:

+                 if self.module.last_batch_id() == self.module.batch:

                      # If RPM is filtered-out, do not add it to artifacts list.

                      if name in m1_mmd.get_rpm_filter().get():

                          continue
@@ -223,6 +223,14 @@ 

          self.yum_conf += extra

          self.yum_conf += "enabled=1\n\n"

  

+     def _add_repo_from_path(self, path):

+         """

+         Adds repository stored in `path` to Mock config file. Call _write_mock_config() to

+         actually write the config file to filesystem.

+         """

+         with open(path) as fd:

+             self.yum_conf += fd.read()

+ 

      def _load_mock_config(self):

          """

          Loads the variables which are generated only during the first
@@ -248,6 +256,7 @@ 

  

                  self.groups = config_opts["chroot_setup_cmd"].split(" ")[1:]

                  self.yum_conf = config_opts['yum.conf']

+                 self.enabled_modules = config_opts['module_enable']

  

      def _write_mock_config(self):

          """
@@ -261,6 +270,7 @@ 

              config = config.replace("$arch", self.arch)

              config = config.replace("$group", " ".join(self.groups))

              config = config.replace("$yum_conf", self.yum_conf)

+             config = config.replace("$enabled_modules", str(self.enabled_modules))

  

              # We write the most recent config to "mock.cfg", so thread-related

              # configs can be later (re-)generated from it using _load_mock_config.
@@ -313,17 +323,36 @@ 

  

      def buildroot_add_repos(self, dependencies):

          self._load_mock_config()

-         for tag in dependencies:

-             # If tag starts with mock_resultdir, it means it is path to local

+         for source, mmds in dependencies.items():

+             # If source starts with mock_resultdir, it means it is path to local

              # module build repository.

-             if tag.startswith(conf.mock_resultsdir):

-                 repo_name = os.path.basename(tag)

+             if source.startswith(conf.mock_resultsdir):

+                 repo_name = os.path.basename(source)

                  if repo_name.startswith("module-"):

                      repo_name = repo_name[7:]

-                 repo_dir = tag

+                 repo_dir = source

                  baseurl = "file://" + repo_dir

+             # If source starts with "repofile://", it is path to local /etc/yum.repos.d

+             # repo file.

+             elif source.startswith("repofile://"):

+                 # For the base module, we want to include all the `conf.base_module_repofiles`.

+                 if len(mmds) == 1 and mmds[0].get_name() in conf.base_module_names:

+                     for repofile in conf.base_module_repofiles:

+                         self._add_repo_from_path(repofile)

+                     # Also set the platform_id.

+                     mmd = mmds[0]

+                     self.yum_conf = self.yum_conf.replace(

+                         "$module_platform_id", "%s:%s" % (mmd.get_name(), mmd.get_stream()))

+                 else:

+                     # Add repositories defined in repofile to mock config.

+                     repofile = source[len("repofile://"):]

+                     self._add_repo_from_path(repofile)

+                     # Enabled all the modular dependencies by default in Mock.

+                     for mmd in mmds:

+                         self.enabled_modules.append("%s:%s" % (mmd.get_name(), mmd.get_stream()))

+                 continue

              else:

-                 repo_name = tag

+                 repo_name = tag = source

                  koji_config = get_koji_config(self.config)

                  koji_session = koji.ClientSession(koji_config.server, opts=koji_config)

                  repo = koji_session.getRepo(repo_name)
@@ -517,6 +546,25 @@ 

          if self.module.state == models.BUILD_STATES["done"]:

              self._createrepo(include_module_yaml=True)

  

+     @classmethod

+     def get_built_rpms_in_module_build(cls, mmd):

+         """

+         :param Modulemd mmd: Modulemd to get the built RPMs from.

+         :return: list of NVRs

+         """

+         with models.make_session(conf) as db_session:

+             build = models.ModuleBuild.get_build_from_nsvc(

+                 db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(),

+                 mmd.get_context())

+             if build.koji_tag.startswith("repofile://"):

+                 # Modules from local repository have already the RPMs filled in mmd.

+                 return list(mmd.get_rpm_artifacts().get())

+             else:

+                 koji_session = KojiModuleBuilder.get_session(conf, login=False)

+                 rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]

+                 nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)

+                 return list(nvrs)

+ 

  

  class BaseBuilder(object):

      def __init__(self, config, resultsdir):
@@ -540,9 +588,14 @@ 

      def __init__(self, config, resultsdir, source, artifact_name):

          super(SCMBuilder, self).__init__(config, resultsdir)

          with open(config, "a") as f:

-             branch = source.split("?#")[1]

+             git_repo, branch = source.split("?#")

              distgit_cmds = self._get_distgit_commands(source)

-             distgit_get = distgit_cmds[0].format(artifact_name)

+ 

+             if source.startswith("file://"):

+                 # For local git repositories, pass the full path to repository to git command.

+                 distgit_get = distgit_cmds[0].format(git_repo)

+             else:

+                 distgit_get = distgit_cmds[0].format(artifact_name)

  

              # mock-scm cannot checkout particular commit hash, but only branch.

              # We therefore use a command that combines the distgit-command with
@@ -562,10 +615,23 @@ 

                      artifact_name),

                  "config_opts['scm_opts']['distgit_get'] = {!r}\n".format(

                      distgit_get_branch),

-                 "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(

-                     distgit_cmds[1]),

              ])

  

+             # Set distgit_src_get only if it's defined.

+             if distgit_cmds[1]:

+                 f.write("config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(

+                     distgit_cmds[1]))

+ 

+             # The local git repositories cloned by `fedpkg clone` typically do not have

+             # the tarballs with sources committed in a git repo. They normally live in lookaside

+             # cache on remote server, but we should not try getting them from there for true

+             # local builds.

+             # Instead, get them from local path with git repository by passing that path to Mock

+             # using the `ext_src_dir`.

+             if git_repo.startswith("file://"):

+                 src_dir = git_repo[len("file://"):]

+                 f.write("config_opts['scm_opts']['ext_src_dir'] = '{}'\n".format(src_dir))

+ 

      def _make_executable(self, path):

          mode = os.stat(path).st_mode

          mode |= (mode & 0o444) >> 2    # copy R bits to X

@@ -39,7 +39,8 @@ 

  

  SUPPORTED_RESOLVERS = {

      'mbs': {'builders': ['mock']},

-     'db': {'builders': ['koji', 'mock', 'copr']}

+     'db': {'builders': ['koji', 'mock', 'copr']},

+     'local': {'builders': ['mock']},

  }

  

  
@@ -66,7 +67,10 @@ 

      # Load LocalBuildConfiguration section in case we are building modules

      # locally.

      if "build_module_locally" in sys.argv:

-         config_section = "LocalBuildConfiguration"

+         if "--offline"in sys.argv:

+             config_section = "OfflineLocalBuildConfiguration"

+         else:

+             config_section = "LocalBuildConfiguration"

  

      # try getting config_file from os.environ

      if 'MBS_CONFIG_FILE' in os.environ:
@@ -332,6 +336,7 @@ 

              'default': {

                  'https://src.fedoraproject.org': ('fedpkg clone --anonymous {}',

                                                    'fedpkg --release module sources'),

+                 'file://': ('git clone {}', None),

              },

              'desc': 'Mapping between dist-git and command to '},

          'mock_config': {
@@ -516,6 +521,20 @@ 

                       'concatenated. Any null capture groups will be ignored. The first regex that '

                       'matches the branch will be used.')

          },

+         'default_buildroot_packages': {

+             'type': list,

+             'default': ["bash", "bzip2", "coreutils", "cpio", "diffutils", "findutils", "gawk",

+                         "gcc", "gcc-c++", "grep", "gzip", "info", "make",

+                         "patch", "fedora-release", "redhat-rpm-config", "rpm-build", "sed",

+                         "shadow-utils", "tar", "unzip", "util-linux", "which", "xz"],

+             'desc': ('The list packages for offline module build RPM buildroot.')

+         },

+         'default_srpm_buildroot_packages': {

+             'type': list,

+             'default': ["bash", "gnupg2", "fedora-release",

+                         "redhat-rpm-config", "fedpkg-minimal", "rpm-build", "shadow-utils"],

+             'desc': ('The list packages for offline module build RPM buildroot.')

+         },

      }

  

      def __init__(self, conf_section_obj):

@@ -34,7 +34,8 @@ 

  from module_build_service import models

  from module_build_service.utils import (

      submit_module_build_from_yaml,

-     load_local_builds, load_mmd, import_mmd

+     load_local_builds, load_mmd, import_mmd,

+     import_builds_from_local_dnf_repos

  )

  from module_build_service.errors import StreamAmbigous

  import module_build_service.messaging
@@ -104,10 +105,14 @@ 

  @manager.option('--file', action='store', dest="yaml_file")

  @manager.option('--srpm', action='append', default=[], dest="srpms", metavar='SRPM')

  @manager.option('--skiptests', action='store_true', dest="skiptests")

+ @manager.option('--offline', action='store_true', dest="offline")

  @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs')

  @manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams')

+ @manager.option('-r', '--platform-repo-file', action='append', default=[],

+                 dest='platform_repofiles')

  def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None,

-                          stream=None, skiptests=False, default_streams=None):

+                          stream=None, skiptests=False, default_streams=None,

+                          offline=False, platform_repofiles=None):

      """ Performs local module build using Mock

      """

      if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']:
@@ -119,6 +124,7 @@ 

  

      with app.app_context():

          conf.set_item("system", "mock")

+         conf.set_item("base_module_repofiles", platform_repofiles)

  

          # Use our own local SQLite3 database.

          confdir = os.path.abspath(os.getcwd())
@@ -132,6 +138,8 @@ 

              os.remove(dbpath)

  

          db.create_all()

+         if offline:

+             import_builds_from_local_dnf_repos()

          load_local_builds(local_build_nsvs)

  

          params = {}

@@ -190,7 +190,12 @@ 

      def get_module_build_dependencies(self, name=None, stream=None, version=None, context=None,

                                        mmd=None, strict=False):

          """

-         Returns a dictionary of koji_tag:mmd of all the dependencies

+         Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module.

+ 

+         Although it is expected that single Koji tag always contain just single module build,

+         it does not have to be a true for Offline local builds which use the local repository

+         identifier as `koji_tag`.

+ 

          :kwarg name: a string of a module's name (required if mmd is not set)

          :kwarg stream: a string of a module's stream (required if mmd is not set)

          :kwarg version: a string of a module's version (required if mmd is not set)
@@ -240,7 +245,8 @@ 

                  if not build:

                      raise RuntimeError(

                          'Buildrequired module %s %r does not exist in MBS db' % (br_name, details))

-                 module_tags[build.koji_tag] = build.mmd()

+                 module_tags.setdefault(build.koji_tag, [])

+                 module_tags[build.koji_tag].append(build.mmd())

  

          return module_tags

  

@@ -0,0 +1,56 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ from module_build_service.resolver.DBResolver import DBResolver

+ 

+ 

+ class LocalResolver(DBResolver):

+     """

+     Resolver using DNF and local repositories.

+ 

+     It is subclass of DBResolver with small changes to DBResolver logic to fit

+     the offline local module builds. See particular methods for more information.

+     """

+     backend = 'local'

+ 

+     def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):

+         """

+         Returns modulemd metadata of all module builds with `name` and `stream`.

+ 

+         For LocalResolver which is used only for Offline local builds,

+         the `base_module_nsvc` is ignored. Normally, the `base_module_nsvc is used

+         to filter out platform:streams which are not compatible with currently used

+         stream version. But during offline local builds, we always have just single

+         platform:stream derived from PLATFORM_ID in /etc/os-release.

+ 

+         Because we have just single platform stream, there is no reason to filter

+         incompatible streams. This platform stream is also expected to not follow

+         the "X.Y.Z" formatting which is needed for stream versions.

+ 

+         :param str name: Name of module to return.

+         :param str stream: Stream of module to return.

+         :param str base_module_nsvc: Ignored in LocalResolver.

+         :rtype: list

+         :return: List of modulemd metadata.

+         """

+         return self.get_module_modulemds(name, stream)

@@ -242,7 +242,12 @@ 

  

      def get_module_build_dependencies(self, name=None, stream=None, version=None, context=None,

                                        mmd=None, strict=False):

-         """Get module's build dependencies defined in xmd/mbs section

+         """

+         Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module.

+ 

+         Although it is expected that single Koji tag always contain just single module build,

+         it does not have to be a true for Offline local builds which use the local repository

+         identifier as `koji_tag`.

  

          :param name: a module's name (required if mmd is not set)

          :param stream: a module's stream (required if mmd is not set)
@@ -252,7 +257,7 @@ 

          :param strict: Normally this function returns None if no module can be

              found.  If strict=True, then an UnprocessableEntity is raised.

          :return: a mapping containing buildrequire modules info in key/value pairs,

-             where key is koji_tag and value is the Modulemd.Module object.

+             where key is koji_tag and value is list of Modulemd.Module objects.

          :rtype: dict(str, :class:`Modulemd.Module`)

          """

  
@@ -300,7 +305,8 @@ 

              for m in modules:

                  if m["koji_tag"] in module_tags:

                      continue

-                 module_tags[m["koji_tag"]] = self.extract_modulemd(m["modulemd"])

+                 module_tags.setdefault(m["koji_tag"], [])

+                 module_tags[m["koji_tag"]].append(self.extract_modulemd(m["modulemd"]))

  

          return module_tags

  

@@ -152,7 +152,9 @@ 

      try:

          mmd = build.mmd()

          record_component_builds(mmd, build, session=session)

-         handle_stream_collision_modules(mmd)

+         # The ursine.handle_stream_collision_modules is Koji specific.

+         if conf.system in ['koji', 'test']:

+             handle_stream_collision_modules(mmd)

          mmd = record_filtered_rpms(mmd)

          build.modulemd = to_text_type(mmd.dumps())

          build.transition(conf, models.BUILD_STATES["wait"])
@@ -232,7 +234,7 @@ 

          # Find out the name of Koji tag to which the module's Content

          # Generator build should be tagged once the build finishes.

          module_names_streams = {

-             mmd.get_name(): mmd.get_stream() for mmd in module_deps.values()

+             mmd.get_name(): mmd.get_stream() for mmds in module_deps.values() for mmd in mmds

          }

          for base_module_name in conf.base_module_names:

              if base_module_name in module_names_streams:
@@ -315,10 +317,9 @@ 

      builder = module_build_service.builder.GenericBuilder.create_from_module(

          session, build, config)

  

-     dep_koji_tags = build_deps.keys()

      log.debug("Adding dependencies %s into buildroot for module %s:%s:%s",

-               dep_koji_tags, build.name, build.stream, build.version)

-     builder.buildroot_add_repos(dep_koji_tags)

+               build_deps.keys(), build.name, build.stream, build.version)

+     builder.buildroot_add_repos(build_deps)

  

      if not build.component_builds:

          log.info("There are no components in module %r, skipping build" % build)

@@ -29,7 +29,7 @@ 

  from datetime import datetime

  from six import text_type

  

- from module_build_service import conf, log, models

+ from module_build_service import conf, log, models, Modulemd, glib

  from module_build_service.errors import (

      ValidationError, ProgrammingError, UnprocessableEntity)

  
@@ -406,6 +406,100 @@ 

      return build, msgs

  

  

+ def import_fake_base_module(nsvc):

+     """

+     Creates and imports new fake base module to be used with offline local builds.

+ 

+     :param str nsvc: name:stream:version:context of a module.

+     """

+     name, stream, version, context = nsvc.split(":")

+     mmd = Modulemd.Module()

+     mmd.set_mdversion(2)

+     mmd.set_name(name)

+     mmd.set_stream(stream)

+     mmd.set_version(int(version))

+     mmd.set_context(context)

+     mmd.set_summary("fake base module")

+     mmd.set_description("fake base module")

+     licenses = Modulemd.SimpleSet()

+     licenses.add("GPL")

+     mmd.set_module_licenses(licenses)

+ 

+     buildroot = Modulemd.Profile()

+     buildroot.set_name("buildroot")

+     for rpm in conf.default_buildroot_packages:

+         buildroot.add_rpm(rpm)

+     mmd.add_profile(buildroot)

+ 

+     srpm_buildroot = Modulemd.Profile()

+     srpm_buildroot.set_name("srpm-buildroot")

+     for rpm in conf.default_srpm_buildroot_packages:

+         srpm_buildroot.add_rpm(rpm)

+     mmd.add_profile(srpm_buildroot)

+ 

+     xmd = {'mbs': {}}

+     xmd_mbs = xmd['mbs']

+     xmd_mbs['buildrequires'] = {}

+     xmd_mbs['requires'] = {}

+     xmd_mbs['commit'] = 'ref_%s' % context

+     xmd_mbs['mse'] = 'true'

+     # Use empty "repofile://" URI for base module. The base module will use the

+     # `conf.base_module_names` list as list of default repositories.

+     xmd_mbs['koji_tag'] = 'repofile://'

+     mmd.set_xmd(glib.dict_values(xmd))

+ 

+     with models.make_session(conf) as session:

+         import_mmd(session, mmd)

+ 

+ 

+ def import_builds_from_local_dnf_repos():

+     """

+     Imports the module builds from all available local repositories to MBS DB.

+ 

+     This is used when building modules locally without any access to MBS infra.

+     This method also generates and imports the base module according to /etc/os-release.

+     """

+     # Import DNF here to not force it as a hard MBS dependency.

+     import dnf

+ 

+     log.info("Loading available RPM repositories.")

+     dnf_base = dnf.Base()

+     dnf_base.read_all_repos()

+ 

+     log.info("Importing available modules to MBS local database.")

+     with models.make_session(conf) as session:

+         for repo in dnf_base.repos.values():

+             try:

+                 repo.load()

+             except Exception as e:

+                 log.warning(str(e))

+                 continue

+             mmd_data = repo.get_metadata_content("modules")

+             mmds = Modulemd.Module.new_all_from_string(mmd_data)

+             for mmd in mmds:

+                 xmd = glib.from_variant_dict(mmd.get_xmd())

+                 xmd["mbs"]["koji_tag"] = "repofile://" + repo.repofile

+                 xmd["mbs"]["mse"] = True

+                 mmd.set_xmd(glib.dict_values(xmd))

+ 

+                 import_mmd(session, mmd)

+ 

+     # Parse the /etc/os-release to find out the local platform:stream.

+     platform_id = None

+     with open("/etc/os-release", "r") as fd:

+         for l in fd.readlines():

+             if not l.startswith("PLATFORM_ID"):

+                 continue

+             platform_id = l.split("=")[1].strip("\"' \n")

+     if not platform_id:

+         raise ValueError("Cannot get PLATFORM_ID from /etc/os-release.")

+ 

+     # Create the fake platform:stream:1:000000 module to fulfill the

+     # dependencies for local offline build and also to define the

+     # srpm-buildroot and buildroot.

+     import_fake_base_module("%s:1:000000" % platform_id)

+ 

+ 

  def get_mmd_from_scm(url):

      """

      Provided an SCM URL, fetch mmd from the corresponding module YAML

file modified
+1
@@ -57,6 +57,7 @@ 

            'mbs.resolver_backends': [

                'mbs = module_build_service.resolver.MBSResolver:MBSResolver',

                'db = module_build_service.resolver.DBResolver:DBResolver',

+               'local = module_build_service.resolver.LocalResolver:LocalResolver',

            ],

        },

        scripts=['client/mbs-cli'],

@@ -8,11 +8,12 @@ 

  

  import kobo.rpmlib

  

- from module_build_service import conf

+ from module_build_service import conf, db

  from module_build_service.models import ModuleBuild, ComponentBuild, make_session

  from module_build_service.builder.MockModuleBuilder import MockModuleBuilder

  from module_build_service import glib, Modulemd

- from tests import clean_database

+ from module_build_service.utils import import_fake_base_module

+ from tests import clean_database, make_module

  

  

  class TestMockModuleBuilder:
@@ -176,3 +177,46 @@ 

              with open(os.path.join(self.resultdir, "pkglist"), "r") as fd:

                  pkglist = fd.read().strip()

                  assert not pkglist

+ 

+ 

+ class TestMockModuleBuilderAddRepos:

+ 

+     def setup_method(self, test_method):

+         clean_database(add_platform_module=False)

+         import_fake_base_module("platform:f29:1:000000")

+         self.platform = ModuleBuild.get_last_build_in_stream(db.session, "platform", "f29")

+         self.foo = make_module("foo:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})

+         self.app = make_module("app:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})

+ 

+     @mock.patch("module_build_service.conf.system", new="mock")

+     @mock.patch(

+         'module_build_service.config.Config.base_module_repofiles',

+         new_callable=mock.PropertyMock,

+         return_value=["/etc/yum.repos.d/bar.repo", "/etc/yum.repos.d/bar-updates.repo"],

+         create=True)

+     @mock.patch("module_build_service.builder.MockModuleBuilder.open", create=True)

+     @mock.patch(

+         "module_build_service.builder.MockModuleBuilder.MockModuleBuilder._load_mock_config")

+     @mock.patch(

+         "module_build_service.builder.MockModuleBuilder.MockModuleBuilder._write_mock_config")

+     def test_buildroot_add_repos(self, write_config, load_config, patched_open,

+                                  base_module_repofiles):

+         patched_open.side_effect = [

+             mock.mock_open(read_data="[fake]\nrepofile 1\n").return_value,

+             mock.mock_open(read_data="[fake]\nrepofile 2\n").return_value,

+             mock.mock_open(read_data="[fake]\nrepofile 3\n").return_value]

+ 

+         builder = MockModuleBuilder("user", self.app, conf, "module-app", [])

+ 

+         dependencies = {

+             "repofile://": [self.platform.mmd()],

+             "repofile:///etc/yum.repos.d/foo.repo": [self.foo.mmd(), self.app.mmd()]

+         }

+ 

+         builder.buildroot_add_repos(dependencies)

+ 

+         assert "repofile 1" in builder.yum_conf

+         assert "repofile 2" in builder.yum_conf

+         assert "repofile 3" in builder.yum_conf

+ 

+         assert set(builder.enabled_modules) == set(["foo:1", "app:1"])

@@ -0,0 +1,76 @@ 

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import os

+ from datetime import datetime

+ 

+ from module_build_service.utils import to_text_type

+ import module_build_service.resolver as mbs_resolver

+ from module_build_service import db

+ from module_build_service.utils import import_mmd, load_mmd

+ from module_build_service.models import ModuleBuild

+ import tests

+ 

+ 

+ base_dir = os.path.join(os.path.dirname(__file__), "..")

+ 

+ 

+ class TestLocalResolverModule:

+ 

+     def setup_method(self):

+         tests.reuse_component_init_data()

+ 

+     def test_get_buildrequired_modulemds(self):

+         mmd = load_mmd(os.path.join(base_dir, 'staged_data', 'platform.yaml'), True)

+         mmd.set_stream('f8')

+         import_mmd(db.session, mmd)

+         platform_f8 = ModuleBuild.query.filter_by(stream='f8').one()

+         mmd.set_name("testmodule")

+         mmd.set_stream("master")

+         mmd.set_version(20170109091357)

+         mmd.set_context("123")

+         build = ModuleBuild(

+             name='testmodule',

+             stream='master',

+             version=20170109091357,

+             state=5,

+             build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3',

+             runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7',

+             context='7c29193d',

+             koji_tag='module-testmodule-master-20170109091357-7c29193d',

+             scmurl='https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79',

+             batch=3,

+             owner='Dr. Pepper',

+             time_submitted=datetime(2018, 11, 15, 16, 8, 18),

+             time_modified=datetime(2018, 11, 15, 16, 19, 35),

+             rebuild_strategy='changed-and-after',

+             modulemd=to_text_type(mmd.dumps())

+         )

+         db.session.add(build)

+         db.session.commit()

+ 

+         resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='local')

+         result = resolver.get_buildrequired_modulemds(

+             "testmodule", "master", platform_f8.mmd().dup_nsvc())

+         nsvcs = set([m.dup_nsvc() for m in result])

+         assert nsvcs == set(['testmodule:master:20170109091357:9c690d0e',

+                              'testmodule:master:20170109091357:123'])

@@ -191,7 +191,7 @@ 

              resolver.backend = 'db'

              resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357"

              resolver.get_module_build_dependencies.return_value = {

-                 "module-bootstrap-tag": base_mmd}

+                 "module-bootstrap-tag": [base_mmd]}

  

              with patch.object(module_build_service.resolver, 'system_resolver', new=resolver):

                  msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2,
@@ -239,7 +239,7 @@ 

              resolver.backend = 'db'

              resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357"

              resolver.get_module_build_dependencies.return_value = {

-                 "module-bootstrap-tag": base_mmd}

+                 "module-bootstrap-tag": [base_mmd]}

  

              with patch.object(module_build_service.scheduler.handlers.modules.conf,

                                'koji_cg_tag_build', new=koji_cg_tag_build):

@@ -1225,3 +1225,61 @@ 

              assert len(local_modules) == 1

              assert local_modules[0].koji_tag.endswith(

                  "/module-platform-f28-3/results")

+ 

+ 

+ class TestOfflineLocalBuilds:

+ 

+     def setup_method(self):

+         clean_database()

+ 

+     def teardown_method(self):

+         clean_database()

+ 

+     def test_import_fake_base_module(self):

+         module_build_service.utils.import_fake_base_module("platform:foo:1:000000")

+         module_build = models.ModuleBuild.get_build_from_nsvc(

+             db.session, "platform", "foo", 1, "000000")

+         assert module_build

+ 

+         mmd = module_build.mmd()

+         xmd = glib.from_variant_dict(mmd.get_xmd())

+         assert xmd == {

+             'mbs': {

+                 'buildrequires': {},

+                 'commit': 'ref_000000',

+                 'koji_tag': 'repofile://',

+                 'mse': 'true',

+                 'requires': {}}}

+ 

+         profiles = mmd.get_profiles()

+         assert set(profiles.keys()) == set(["buildroot", "srpm-buildroot"])

+ 

+     @patch("module_build_service.utils.general.open", create=True)

+     def test_import_builds_from_local_dnf_repos(self, patched_open):

+         pytest.importorskip("dnf")

+ 

+         with patch("dnf.Base") as dnf_base:

+             repo = mock.MagicMock()

+             repo.repofile = "/etc/yum.repos.d/foo.repo"

+             with open(path.join(BASE_DIR, '..', 'staged_data', 'formatted_testmodule.yaml')) as f:

+                 repo.get_metadata_content.return_value = f.read()

+             base = dnf_base.return_value

+             base.repos = {"reponame": repo}

+ 

+             patched_open.return_value = mock.mock_open(

+                 read_data="FOO=bar\nPLATFORM_ID=platform:x\n").return_value

+ 

+             module_build_service.utils.import_builds_from_local_dnf_repos()

+ 

+             base.read_all_repos.assert_called_once()

+             repo.load.assert_called_once()

+             repo.get_metadata_content.assert_called_once_with("modules")

+ 

+             module_build = models.ModuleBuild.get_build_from_nsvc(

+                 db.session, "testmodule", "master", 20180205135154, "9c690d0e")

+             assert module_build

+             assert module_build.koji_tag == "repofile:///etc/yum.repos.d/foo.repo"

+ 

+             module_build = models.ModuleBuild.get_build_from_nsvc(

+                 db.session, "platform", "x", 1, "000000")

+             assert module_build