#1082 MBS build tagging is optional
Merged 8 months ago by jkaluza. Opened 8 months ago by cqi.
cqi/fm-orchestrator optional-cg-tag-build  into  master

@@ -41,7 +41,7 @@ 

  import koji

  import pungi.arch

  

- from module_build_service import log, build_logs, Modulemd, glib

+ from module_build_service import conf, log, build_logs, Modulemd, glib

  

  logging.basicConfig(level=logging.DEBUG)

  

@@ -734,7 +734,8 @@ 

          try:

              serverdir = self._upload_outputs(session, metadata, file_dir)

              build_info = session.CGImport(metadata, serverdir)

-             self._tag_cg_build()

+             if conf.koji_cg_tag_build:

+                 self._tag_cg_build()

              log.info("Content generator import done.")

              log.debug(json.dumps(build_info, sort_keys=True, indent=4))

  

@@ -423,6 +423,11 @@ 

              'type': dict,

              'default': {},

              'desc': 'Per base-module name:stream Koji arches list.'},

+         'koji_cg_tag_build': {

+             'type': bool,

+             'default': True,

+             'desc': 'Indicate whether tagging build is enabled during importing '

+                     'module to Koji.'},

          'koji_cg_build_tag_template': {

              'type': str,

              'default': "{}-modular-updates-candidate",

@@ -171,6 +171,76 @@ 

          session.commit()

  

  

+ def generate_module_build_koji_tag(build):

+     """Used by wait handler to get module build koji tag

+ 

+     :param build: a module build.

+     :type build: :class:`ModuleBuild`

+     :return: generated koji tag.

+     :rtype: str

+     """

+     log.info('Getting tag for %s:%s:%s', build.name, build.stream, build.version)

+     if conf.system in ['koji', 'test']:

+         return generate_koji_tag(build.name, build.stream, build.version, build.context)

+     else:

+         return '-'.join(['module', build.name, build.stream, build.version])

+ 

+ 

+ @module_build_service.utils.retry(

+     interval=10, timeout=120,

+     wait_on=(ValueError, RuntimeError, ConnectionError))

+ def get_module_build_dependencies(build):

+     """Used by wait handler to get module's build dependencies

+ 

+     :param build: a module build.

+     :type build: :class:`ModuleBuild`

+     :return: the value returned from :meth:`get_module_build_dependencies`

+         according to the configured resolver.

+     :rtype: dict[str, Modulemd.Module]

+     """

+     resolver = module_build_service.resolver.system_resolver

+     if conf.system in ['koji', 'test']:

+         # For Koji backend, query for the module we are going to

+         # build to get the koji_tag and deps from it.

+         log.info('Getting tag for %s:%s:%s', build.name, build.stream, build.version)

+         return resolver.get_module_build_dependencies(

+             build.name, build.stream, build.version, build.context, strict=True)

+     else:

+         # In case of non-koji backend, we want to get the dependencies

+         # of the local module build based on Modulemd.Module, because the

+         # local build is not stored in the external MBS and therefore we

+         # cannot query it using the `query` as for Koji below.

+         return resolver.get_module_build_dependencies(mmd=build.mmd(), strict=True)

+ 

+ 

+ def get_content_generator_build_koji_tag(module_deps):

+     """Used by wait handler to get koji tag for import by content generator

+ 

+     :param module_deps: a mapping from module's koji tag to its module

+         metadata.

+     :type: dict[str, Modulemd.Module]

+     :return: the koji tag.

+     :rtype: str

+     """

+     if conf.system in ['koji', 'test']:

+         # Find out the name of Koji tag to which the module's Content

+         # Generator build should be tagged once the build finishes.

+         module_names_streams = {

+             mmd.get_name(): mmd.get_stream() for mmd in module_deps.values()

+         }

+         for base_module_name in conf.base_module_names:

+             if base_module_name in module_names_streams:

+                 return conf.koji_cg_build_tag_template.format(

+                     module_names_streams[base_module_name])

+ 

+         log.debug('No configured base module is a buildrequire. Hence use'

+                   ' default content generator build koji tag %s',

+                   conf.koji_cg_default_build_tag)

+         return conf.koji_cg_default_build_tag

+     else:

+         return conf.koji_cg_default_build_tag

+ 

+ 

  def wait(config, session, msg):

      """ Called whenever a module enters the 'wait' state.

  

@@ -206,55 +276,8 @@ 

          # This is ok.. it's a race condition we can ignore.

          pass

  

-     resolver = module_build_service.resolver.system_resolver

- 

-     @module_build_service.utils.retry(

-         interval=10, timeout=120,

-         wait_on=(ValueError, RuntimeError, ConnectionError))

-     def _get_deps_and_tag():

-         """

-         Private method to get the dependencies and koji tag of a module we

-         are going to build. We use private method here to allow "retry"

-         on failure.

-         """

-         cg_build_koji_tag = conf.koji_cg_default_build_tag

-         if conf.system not in ['koji', 'test']:

-             # In case of non-koji backend, we want to get the dependencies

-             # of the local module build based on Modulemd.Module, because the

-             # local build is not stored in the external MBS and therefore we

-             # cannot query it using the `query` as for Koji below.

-             dep_koji_tags = resolver.get_module_build_dependencies(

-                 mmd=build.mmd(), strict=True).keys()

- 

-             # We also don't want to get the tag name from the MBS, but just

-             # generate it locally instead.

-             tag = '-'.join(['module', build.name, build.stream, build.version])

-         else:

-             # For Koji backend, query for the module we are going to

-             # build to get the koji_tag and deps from it.

-             nsvc = ':'.join([build.name, build.stream, build.version])

-             log.info("Getting deps for %s" % (nsvc))

-             deps_dict = resolver.get_module_build_dependencies(

-                 build.name, build.stream, build.version, build.context, strict=True)

-             dep_koji_tags = set(deps_dict.keys())

- 

-             # Find out the name of Koji tag to which the module's Content

-             # Generator build should be tagged once the build finishes.

-             module_names_streams = {mmd.get_name(): mmd.get_stream()

-                                     for mmd in deps_dict.values()}

-             for base_module_name in conf.base_module_names:

-                 if base_module_name in module_names_streams:

-                     cg_build_koji_tag = conf.koji_cg_build_tag_template.format(

-                         module_names_streams[base_module_name])

-                     break

- 

-             log.info('Getting tag for {0}'.format(nsvc))

-             tag = generate_koji_tag(build.name, build.stream, build.version, build.context)

- 

-         return dep_koji_tags, tag, cg_build_koji_tag

- 

      try:

-         dep_koji_tags, tag, cg_build_koji_tag = _get_deps_and_tag()

+         build_deps = get_module_build_dependencies(build)

      except ValueError:

          reason = "Failed to get module info from MBS. Max retries reached."

          log.exception(reason)

@@ -262,6 +285,7 @@ 

          session.commit()

          raise

  

+     tag = generate_module_build_koji_tag(build)

      log.debug("Found tag=%s for module %r" % (tag, build))

      # Hang on to this information for later.  We need to know which build is

      # associated with which koji tag, so that when their repos are regenerated

@@ -270,15 +294,22 @@ 

      log.debug("Assigning koji tag=%s to module build" % tag)

      build.koji_tag = tag

  

-     log.debug("Assigning Content Generator build koji tag=%s to module "

-               "build", cg_build_koji_tag)

-     build.cg_build_koji_tag = cg_build_koji_tag

+     if conf.koji_cg_tag_build:

+         cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps)

+         log.debug("Assigning Content Generator build koji tag=%s to module build",

+                   cg_build_koji_tag)

+         build.cg_build_koji_tag = cg_build_koji_tag

+     else:

+         log.debug('It is disabled to tag module build during importing into'

+                   ' Koji by Content Generator.')

+         log.debug('Skip to assign Content Generator build koji tag to module build.')

  

      builder = module_build_service.builder.GenericBuilder.create_from_module(

          session, build, config)

  

-     log.debug("Adding dependencies %s into buildroot for module %s" % (dep_koji_tags, ':'.join(

-         [build.name, build.stream, build.version])))

+     dep_koji_tags = build_deps.keys()

+     log.debug("Adding dependencies %s into buildroot for module %s:%s:%s",

+               dep_koji_tags, build.name, build.stream, build.version)

      builder.buildroot_add_repos(dep_koji_tags)

  

      if not build.component_builds:

@@ -26,6 +26,7 @@ 

  import module_build_service.scheduler.handlers.modules

  import os

  import koji

+ import pytest

  from tests import conf, db, app, scheduler_init_data

  import module_build_service.resolver

  from module_build_service import build_logs, Modulemd

@@ -206,6 +207,10 @@ 

                  module_build = ModuleBuild.query.filter_by(id=2).one()

                  assert module_build.cg_build_koji_tag == "modular-updates-candidate"

  

+     @pytest.mark.parametrize('koji_cg_tag_build,expected_cg_koji_build_tag', [

+         [True, 'f27-modular-updates-candidate'],

+         [False, None],

+     ])

      @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",

             return_value={'build': [], 'srpm-build': []})

      @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder.get_session")

@@ -215,7 +220,8 @@ 

      @patch("module_build_service.config.Config.base_module_names",

             new_callable=mock.PropertyMock, return_value=set(["base-runtime", "platform"]))

      def test_set_cg_build_koji_tag(

-             self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg):

+             self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg,

+             koji_cg_tag_build, expected_cg_koji_build_tag):

          """

          Test that build.cg_build_koji_tag is set.

          """

@@ -243,10 +249,12 @@ 

              resolver.get_module_build_dependencies.return_value = {

                  "module-bootstrap-tag": base_mmd}

  

-             with patch.object(module_build_service.resolver, 'system_resolver', new=resolver):

-                 msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2,

-                                                                module_build_state='some state')

-                 module_build_service.scheduler.handlers.modules.wait(

-                     config=conf, session=db.session, msg=msg)

-                 module_build = ModuleBuild.query.filter_by(id=2).one()

-                 assert module_build.cg_build_koji_tag == "f27-modular-updates-candidate"

+             with patch.object(module_build_service.scheduler.handlers.modules.conf,

+                               'koji_cg_tag_build', new=koji_cg_tag_build):

+                 with patch.object(module_build_service.resolver, 'system_resolver', new=resolver):

+                     msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2,

+                                                                    module_build_state='some state')

+                     module_build_service.scheduler.handlers.modules.wait(

+                         config=conf, session=db.session, msg=msg)

+                     module_build = ModuleBuild.query.filter_by(id=2).one()

+                     assert module_build.cg_build_koji_tag == expected_cg_koji_build_tag

Pretty please pagure-ci rebuild

Space is missing in concatenation

2 new commits added

  • Make it optional to tag build for CG import
  • Refactor _get_deps_and_tag
8 months ago

Pretty please pagure-ci rebuild

+1, run the tests locally in the end...

Pull-Request has been merged by jkaluza

8 months ago