#996 Fill in the 'filtered_rpms' in backend to not access Koji from frontend.
Merged 11 months ago by jkaluza. Opened 11 months ago by jkaluza.
jkaluza/fm-orchestrator move-filters-to-backend  into  master

@@ -1151,15 +1151,19 @@ 

          return weights

  

      @classmethod

-     def get_built_rpms_in_module_build(cls, build):

+     def get_built_rpms_in_module_build(cls, mmd):

          """

-         :param ModuleBuild build: Module build to get the built RPMs from.

+         :param Modulemd mmd: Modulemd to get the built RPMs from.

          :return: list of NVRs

          """

-         koji_session = KojiModuleBuilder.get_session(conf, None)

-         rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]

-         nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)

-         return list(nvrs)

+         with models.make_session(conf) as db_session:

+             build = models.ModuleBuild.get_build_from_nsvc(

+                 db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(),

+                 mmd.get_context())

+             koji_session = KojiModuleBuilder.get_session(conf, None)

+             rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]

+             nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)

+             return list(nvrs)

  

      def finalize(self):

          # Only import to koji CG if the module is "done".

@@ -337,9 +337,9 @@ 

          raise NotImplementedError()

  

      @classmethod

-     def get_built_rpms_in_module_build(cls, build):

+     def get_built_rpms_in_module_build(cls, mmd):

          """

-         :param ModuleBuild build: Module build to get the built RPMs from.

+         :param Modulemd mmd: Modulemd to get the built RPMs from.

          :return: list of NVRs

          """

          raise NotImplementedError()

@@ -22,8 +22,6 @@ 

  # Written by Matt Prahl <mprahl@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

  

- import kobo.rpmlib

- 

  from module_build_service import log

  from module_build_service.resolver.base import GenericResolver

  from module_build_service import models

@@ -173,14 +171,13 @@ 

          """

          Resolves the requires list of N:S or N:S:V:C to a dictionary with keys as

          the module name and the values as a dictionary with keys of ref,

-         stream, version, filtered_rpms.

+         stream, version.

          If there are some modules loaded by utils.load_local_builds(...), these

          local modules will be considered when resolving the requires. A RuntimeError

          is raised on DB lookup errors.

          :param requires: a dictionary with the module name as the key and the stream as the value

          :return: a dictionary

          """

-         from module_build_service.builder import GenericBuilder

          new_requires = {}

          with models.make_session(self.config) as session:

              for nsvc in requires:

@@ -203,11 +200,7 @@ 

                          'ref': None,

                          'stream': local_build.stream,

                          'version': local_build.version,

-                         'context': local_build.context,

-                         # No need to set filtered_rpms for local builds, because MBS

-                         # filters the RPMs automatically when the module build is

-                         # done.

-                         'filtered_rpms': []

+                         'context': local_build.context

                      }

                      continue

  

@@ -222,7 +215,6 @@ 

                      raise UnprocessableEntity('The module {} was not found'.format(nsvc))

  

                  commit_hash = None

-                 filtered_rpms = []

                  mmd = build.mmd()

                  mbs_xmd = mmd.get_xmd().get('mbs')

                  if mbs_xmd and 'commit' in mbs_xmd.keys():

@@ -237,22 +229,11 @@ 

                          'The module "{}" is not built using Module Stream Expansion. '

                          'Please rebuild this module first'.format(nsvc))

  

-                 # Find out the particular NVR of filtered packages

-                 rpm_filter = mmd.get_rpm_filter()

-                 if rpm_filter and rpm_filter.get():

-                     rpm_filter = rpm_filter.get()

-                     built_nvrs = GenericBuilder.get_built_rpms_in_module_build(build)

-                     for nvr in built_nvrs:

-                         parsed_nvr = kobo.rpmlib.parse_nvr(nvr)

-                         if parsed_nvr["name"] in rpm_filter:

-                             filtered_rpms.append(nvr)

- 

                  new_requires[module_name] = {

                      'ref': commit_hash,

                      'stream': module_stream,

                      'version': build.version,

-                     'context': build.context,

-                     'filtered_rpms': filtered_rpms,

+                     'context': build.context

                  }

  

          return new_requires

@@ -266,8 +266,7 @@ 

              "module_name": {

                  "ref": module_commit_hash,

                  "stream": original_module_stream,

-                 "version": module_version,

-                 "filtered_rpms": ["nvr", ...]

+                 "version": module_version

              },

              ...

          }

@@ -32,7 +32,8 @@ 

      attempt_to_reuse_all_components,

      record_component_builds,

      get_rpm_release,

-     generate_koji_tag)

+     generate_koji_tag,

+     record_filtered_rpms)

  from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError

  

  from requests.exceptions import ConnectionError

@@ -151,6 +152,7 @@ 

      try:

          mmd = build.mmd()

          record_component_builds(mmd, build, session=session)

+         mmd = record_filtered_rpms(mmd)

          build.modulemd = mmd.dumps()

          build.transition(conf, models.BUILD_STATES["wait"])

      # Catch custom exceptions that we can expose to the user

@@ -29,6 +29,7 @@ 

  import os

  from multiprocessing.dummy import Pool as ThreadPool

  from datetime import datetime

+ import kobo.rpmlib

  

  from module_build_service import conf, db, log, models, Modulemd

  from module_build_service.errors import (

@@ -38,6 +39,53 @@ 

  from .mse import generate_expanded_mmds

  

  

+ def record_filtered_rpms(mmd):

+     """

+     Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms"

+     list containing the NVRs of filtered RPMs in a buildrequired module.

+ 

+     :param Modulemd mmd: Modulemd of input module.

+     :rtype: Modulemd

+     :return: Modulemd extended with the "filtered_rpms" in XMD section.

+     """

+     # Imported here to allow import of utils in GenericBuilder.

+     from module_build_service.builder import GenericBuilder

+ 

+     new_buildrequires = {}

+     resolver = module_build_service.resolver.GenericResolver.create(conf)

+     for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():

+         # In case this is module resubmit or local build, the filtered_rpms

+         # will already be there, so there is no point in generating them again.

+         if "filtered_rpms" in req_data:

+             continue

+ 

+         # We can just get the first modulemd data from result right here thanks to

+         # strict=True, so in case the module cannot be found, get_module_modulemds

+         # raises an exception.

+         req_mmd = resolver.get_module_modulemds(

+             req_name, req_data["stream"], req_data["version"], req_data["context"], True)[0]

+ 

+         # Find out the particular NVR of filtered packages

+         filtered_rpms = []

+         rpm_filter = req_mmd.get_rpm_filter()

+         if rpm_filter and rpm_filter.get():

+             rpm_filter = rpm_filter.get()

+             built_nvrs = GenericBuilder.backends[conf.system].get_built_rpms_in_module_build(

+                 req_mmd)

+             for nvr in built_nvrs:

+                 parsed_nvr = kobo.rpmlib.parse_nvr(nvr)

+                 if parsed_nvr["name"] in rpm_filter:

+                     filtered_rpms.append(nvr)

+         req_data["filtered_rpms"] = filtered_rpms

+         new_buildrequires[req_name] = req_data

+ 

+     # Replace the old buildrequires with new ones.

+     xmd = glib.from_variant_dict(mmd.get_xmd())

+     xmd["mbs"]["buildrequires"] = new_buildrequires

+     mmd.set_xmd(glib.dict_values(xmd))

+     return mmd

+ 

+ 

  def _scm_get_latest(pkg):

      try:

          # If the modulemd specifies that the 'f25' branch is what

@@ -18,6 +18,9 @@ 

      api:

          rpms:

              - bash

+     xmd:

+         mbs:

+             buildrequires: {}

      components:

          rpms:

              file:

@@ -0,0 +1,45 @@ 

+ document: modulemd

+ version: 1

+ data:

+     summary: A test module in all its beautiful beauty

+     description: >-

+         This module demonstrates how to write simple modulemd files And

+         can be used for testing the build and release pipeline.

+     license:

+         module: [ MIT ]

+     dependencies:

+         buildrequires:

+             platform: f28

+         requires:

+             platform: f28

+     references:

+         community: https://docs.pagure.org/modularity/

+         documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules

+     profiles:

+         default:

+             rpms:

+             - tangerine

+     xmd:

+         mbs:

+             buildrequires:

+                platform:

+                    ref: virtual

+                    stream: f28

+                    version: '3'

+                    context: '00000000'

+     api:

+         rpms:

+         - perl-Tangerine

+         - tangerine

+     components:

+         rpms:

+             perl-List-Compare:

+                 rationale: A dependency of tangerine.

+                 ref: master

+             perl-Tangerine:

+                 rationale: Provides API for this module and is a dependency of tangerine.

+                 ref: master

+             tangerine:

+                 rationale: Provides API for this module.

+                 buildorder: 10

+                 ref: master

@@ -32,7 +32,7 @@ 

  import module_build_service.scheduler.handlers.repos

  import module_build_service.models

  import module_build_service.builder

- from module_build_service import glib

+ from module_build_service import glib, db

  

  import pytest

  from mock import patch, MagicMock

@@ -545,7 +545,17 @@ 

               'size': 878684}], [])

          get_session.return_value = session

  

-         ret = KojiModuleBuilder.get_built_rpms_in_module_build(self.module)

+         # Module builds generated by init_data uses generic modulemd file and

+         # the module's name/stream/version/context does not have to match it.

+         # But for this test, we need it to match.

+         mmd = self.module.mmd()

+         self.module.name = mmd.get_name()

+         self.module.stream = mmd.get_stream()

+         self.module.version = mmd.get_version()

+         self.module.context = mmd.get_context()

+         db.session.commit()

+ 

+         ret = KojiModuleBuilder.get_built_rpms_in_module_build(mmd)

          assert set(ret) == set(

              ['bar-2:1.30-4.el8+1308+551bfa71', 'tar-2:1.30-4.el8+1308+551bfa71'])

  

@@ -120,25 +120,8 @@ 

              ]

              assert set(result) == set(expected)

  

-     @patch("module_build_service.builder.base.GenericBuilder.get_built_rpms_in_module_build")

-     def test_resolve_requires(self, built_rpms):

+     def test_resolve_requires(self):

          build = models.ModuleBuild.query.get(2)

-         mmd = build.mmd()

-         filter_list = Modulemd.SimpleSet()

-         filter_list.add("foo")

-         filter_list.add("bar")

-         mmd.set_rpm_filter(filter_list)

-         build.modulemd = mmd.dumps()

-         db.session.commit()

- 

-         built_rpms.return_value = [

-             "foo-0:2.4.48-3.el8+1308+551bfa71",

-             "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71",

-             "bar-0:2.5.48-3.el8+1308+551bfa71",

-             "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71",

-             "x-0:2.5.48-3.el8+1308+551bfa71",

-             "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"]

- 

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')

          result = resolver.resolve_requires([":".join([

              build.name, build.stream, build.version, build.context])])

@@ -146,10 +129,7 @@ 

          assert result == {

              'testmodule': {

                  'stream': 'master', 'version': '20170109091357', 'context': u'78e4a6fd',

-                 'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba',

-                 'filtered_rpms': [

-                     'foo-0:2.4.48-3.el8+1308+551bfa71',

-                     'bar-0:2.5.48-3.el8+1308+551bfa71']}}

+                 'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba'}}

  

      def test_resolve_profiles(self):

          """

@@ -28,7 +28,7 @@ 

  from tests.test_views.test_views import FakeSCM

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.modules

- from module_build_service import build_logs

+ from module_build_service import build_logs, Modulemd, db

  from module_build_service.models import make_session, ModuleBuild, ComponentBuild

  

  

@@ -39,7 +39,7 @@ 

          self.staged_data_dir = os.path.join(

              os.path.dirname(__file__), '../', 'staged_data')

          testmodule_yml_path = os.path.join(

-             self.staged_data_dir, 'testmodule.yaml')

+             self.staged_data_dir, 'testmodule_init.yaml')

          with open(testmodule_yml_path, 'r') as f:

              yaml = f.read()

          scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77'

@@ -55,19 +55,43 @@ 

          except Exception:

              pass

  

+     @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder."

+            "get_built_rpms_in_module_build")

      @patch('module_build_service.scm.SCM')

-     def test_init_basic(self, mocked_scm):

-         FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',

+     def test_init_basic(self, mocked_scm, built_rpms):

+         FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml',

                  '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

+ 

+         built_rpms.return_value = [

+             "foo-0:2.4.48-3.el8+1308+551bfa71",

+             "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71",

+             "bar-0:2.5.48-3.el8+1308+551bfa71",

+             "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71",

+             "x-0:2.5.48-3.el8+1308+551bfa71",

+             "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"]

+ 

+         platform_build = ModuleBuild.query.get(1)

+         mmd = platform_build.mmd()

+         filter_list = Modulemd.SimpleSet()

+         filter_list.add("foo")

+         filter_list.add("bar")

+         mmd.set_rpm_filter(filter_list)

+         platform_build.modulemd = mmd.dumps()

+         db.session.commit()

+ 

          msg = module_build_service.messaging.MBSModule(

              msg_id=None, module_build_id=2, module_build_state='init')

+ 

          with make_session(conf) as session:

              self.fn(config=conf, session=session, msg=msg)

          build = ModuleBuild.query.filter_by(id=2).one()

          # Make sure the module entered the wait state

          assert build.state == 1, build.state

          # Make sure format_mmd was run properly

-         assert type(build.mmd().get_xmd()['mbs']) is GLib.Variant

+         xmd_mbs = build.mmd().get_xmd()['mbs']

+         assert type(xmd_mbs) is GLib.Variant

+         assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [

+             'foo-0:2.4.48-3.el8+1308+551bfa71', 'bar-0:2.5.48-3.el8+1308+551bfa71']

  

      @patch('module_build_service.scm.SCM')

      def test_init_scm_not_available(self, mocked_scm):

@@ -90,7 +114,7 @@ 

             new_callable=PropertyMock, return_value=True)

      @patch('module_build_service.scm.SCM')

      def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo):

-         FakeSCM(mocked_scm, "includedmodules", ['testmodule.yaml'])

+         FakeSCM(mocked_scm, "includedmodules", ['testmodule_init.yaml'])

          includedmodules_yml_path = os.path.join(

              self.staged_data_dir, 'includedmodules.yaml')

          with open(includedmodules_yml_path, 'r') as f:

Our deployments don't have access to Koji from frontends so far, but the code to fill in the list of filtered RPMs for each buildrequired module tried to access Koji. This failed with kerberos login error.

This PR moves the code from DBResolver to separate function record_filtered_rpms(...) which is called from the handlers.modules.init(...) on backend.

It also changes the get_built_rpms_in_module_build to accept mmd instead of module build. This is needed now, because we might be calling this method for Mock backend on modules for which we don't have the build stored in database.

Pull-Request has been merged by jkaluza

11 months ago