#1324 Add the ability to automatically buildrequire default modules defined by the buildrequired base module
Merged 4 years ago by mprahl. Opened 4 years ago by mprahl.

@@ -72,6 +72,10 @@ 

    module in the disttag of the RPMS being built. If the stream is not the appropriate value, then

    this can be overridden with a custom value using this property. This value can't contain a dash,

    since that is an invalid character in the disttag.

+ - ``default_modules_url`` -  the URL to the list of modules, in the format of ``name:stream``

+   separated by new lines, to include as default modules for any module that buildrequires this

+   module. Any default modules with conflicting streams will be ignored as well as any default module

+   not found in the MBS database. This field only applies to base modules.

  

  

  Virtual Streams

@@ -27,13 +27,13 @@ 

  

  import logging

  import kobo.rpmlib

- import requests

  

  from module_build_service import db, conf

  from module_build_service import models

  from module_build_service.errors import UnprocessableEntity

  from module_build_service.resolver.base import GenericResolver

  from module_build_service.utils.general import import_mmd, load_mmd

+ from module_build_service.utils.request_utils import requests_session

  

  log = logging.getLogger()

  
@@ -44,9 +44,6 @@ 

  

      def __init__(self, config):

          self.mbs_prod_url = config.mbs_url

-         self.session = requests.Session()

-         adapter = requests.adapters.HTTPAdapter(max_retries=3)

-         self.session.mount(self.mbs_prod_url, adapter)

          self._generic_error = "Failed to query MBS with query %r returned HTTP status %s"

  

      def _query_from_nsvc(self, name, stream, version=None, context=None, state="ready"):
@@ -95,7 +92,7 @@ 

          modules = []

  

          while True:

-             res = self.session.get(self.mbs_prod_url, params=query)

+             res = requests_session.get(self.mbs_prod_url, params=query)

              if not res.ok:

                  raise RuntimeError(self._generic_error % (query, res.status_code))

  
@@ -135,7 +132,7 @@ 

          """

          query = {"page": 1, "per_page": 1, "short": True}

          query.update(kwargs)

-         res = self.session.get(self.mbs_prod_url, params=query)

+         res = requests_session.get(self.mbs_prod_url, params=query)

          if not res.ok:

              raise RuntimeError(self._generic_error % (query, res.status_code))

  
@@ -159,7 +156,7 @@ 

              "verbose": True,

              "virtual_stream": virtual_stream,

          }

-         res = self.session.get(self.mbs_prod_url, params=query)

+         res = requests_session.get(self.mbs_prod_url, params=query)

          if not res.ok:

              raise RuntimeError(self._generic_error % (query, res.status_code))

  
@@ -466,7 +463,7 @@ 

          return new_requires

  

      def get_modulemd_by_koji_tag(self, tag):

-         resp = self.session.get(self.mbs_prod_url, params={"koji_tag": tag, "verbose": True})

+         resp = requests_session.get(self.mbs_prod_url, params={"koji_tag": tag, "verbose": True})

          data = resp.json()

          if data["items"]:

              modulemd = data["items"][0]["modulemd"]

@@ -0,0 +1,135 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ import requests

+ 

+ from module_build_service import conf, log, models

+ from module_build_service.errors import UnprocessableEntity

+ from module_build_service.utils.request_utils import requests_session

+ from module_build_service.resolver import system_resolver as resolver

+ 

+ 

+ def add_default_modules(db_session, mmd):

+     """

+     Add default modules as buildrequires to the input modulemd.

+ 

+     The base modules that are buildrequired can optionally link their default modules by specifying

+     a URL to a text file in xmd.mbs.default_modules_url. Any default module that isn't in the

+     database will be logged and ignored.

+ 

+     :param db_session: a SQLAlchemy database session

+     :param Modulemd.ModuleStream mmd: the modulemd of the module to add the module defaults to

+     :raises RuntimeError: if the buildrequired base module isn't in the database or the default

+         modules list can't be downloaded

+     """

+     log.info("Finding the default modules to include as buildrequires")

+     xmd = mmd.get_xmd()

+     buildrequires = xmd["mbs"]["buildrequires"]

+ 

+     for module_name in conf.base_module_names:

+         bm_info = buildrequires.get(module_name)

+         if bm_info is None:

+             log.debug(

+                 "The base module %s is not a buildrequire of the submitted module %s",

+                 module_name, mmd.get_nsvc(),

+             )

+             continue

+ 

+         bm = models.ModuleBuild.get_build_from_nsvc(

+             db_session, module_name, bm_info["stream"], bm_info["version"], bm_info["context"],

+         )

+         bm_nsvc = ":".join([

+             module_name, bm_info["stream"], bm_info["version"], bm_info["context"],

+         ])

+         if not bm:

+             raise RuntimeError("Failed to retrieve the module {} from the database".format(bm_nsvc))

+ 

+         bm_mmd = bm.mmd()

+         bm_xmd = bm_mmd.get_xmd()

+         default_modules_url = bm_xmd.get("mbs", {}).get("default_modules_url")

+         if not default_modules_url:

+             log.debug("The base module %s does not have any default modules", bm_nsvc)

+             continue

+ 

+         try:

+             rv = requests_session.get(default_modules_url, timeout=10)

+         except requests.RequestException:

+             msg = (

+                 "The connection failed when getting the default modules associated with {}"

+                 .format(bm_nsvc)

+             )

+             log.exception(msg)

+             raise RuntimeError(msg)

+ 

+         if not rv.ok:

+             log.error(

+                 "The request to get the default modules associated with %s failed with the status "

+                 'code %d and error "%s"',

+                 bm_nsvc, rv.status_code, rv.text,

+             )

+             raise RuntimeError(

+                 "Failed to retrieve the default modules for {}".format(bm_mmd.get_nsvc())

+             )

+ 

+         default_modules = [m.strip() for m in rv.text.strip().split("\n")]

+         for default_module in default_modules:

+             try:

+                 name, stream = default_module.split(":")

+             except ValueError:

+                 log.error(

+                     'The default module "%s" from %s is in an invalid format',

+                     default_module, rv.url,

+                 )

+                 continue

+ 

+             if name in buildrequires:

+                 conflicting_stream = buildrequires[name]["stream"]

+                 if stream == conflicting_stream:

+                     log.info("The default module %s is already a buildrequire", default_module)

+                     continue

+ 

+                 log.info(

+                     "The default module %s will not be added as a buildrequire since %s:%s "

+                     "is already a buildrequire",

+                     default_module, name, conflicting_stream,

+                 )

+                 continue

+ 

+             try:

+                 # We are reusing resolve_requires instead of directly querying the database since it

+                 # provides the exact format that is needed for mbs.xmd.buildrequires.

+                 #

+                 # Only one default module is processed at a time in resolve_requires so that we

+                 # are aware of which modules are not in the database, and can add those that are as

+                 # buildrequires.

+                 resolved = resolver.resolve_requires([default_module])

Nice idea here :).

+             except UnprocessableEntity:

+                 log.warning(

+                     "The default module %s from %s is not in the database and couldn't be added as "

+                     "a buildrequire",

+                     default_module, bm_nsvc

+                 )

+                 continue

+ 

+             nsvc = ":".join([name, stream, resolved[name]["version"], resolved[name]["context"]])

+             log.info("Adding the default module %s as a buildrequire", nsvc)

+             buildrequires.update(resolved)

+ 

+     mmd.set_xmd(xmd)

@@ -39,6 +39,7 @@ 

  from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError

  from module_build_service.utils.ursine import handle_stream_collision_modules

  from module_build_service.utils.greenwave import greenwave

+ from module_build_service.scheduler.default_modules import add_default_modules

  

  from requests.exceptions import ConnectionError

  from module_build_service.utils import mmd_to_str
@@ -163,6 +164,7 @@ 

      failure_reason = "unspec"

      try:

          mmd = build.mmd()

+         add_default_modules(session, mmd)

          record_module_build_arches(mmd, build, session)

          record_component_builds(mmd, build, session=session)

          # The ursine.handle_stream_collision_modules is Koji specific.

@@ -0,0 +1,49 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ import requests

+ from requests.packages.urllib3.util.retry import Retry

+ 

+ 

+ def get_requests_session(auth=False):

+     """

+     Create a requests session with retries configured.

+ 

+     :return: the configured requests session

+     :rtype: requests.Session

+     """

+     session = requests.Session()

+     retry = Retry(

+         total=3,

+         read=3,

+         connect=3,

+         backoff_factor=0.5,

+         status_forcelist=(500, 502, 503, 504),

+     )

+     retry.BACKOFF_MAX = 2

+     adapter = requests.adapters.HTTPAdapter(max_retries=retry)

+     session.mount("http://", adapter)

+     session.mount("https://", adapter)

+     return session

+ 

+ 

+ # TODO: Use this instead of requests.get directly throughout the codebase

+ requests_session = get_requests_session()

file modified
+62 -64
@@ -34,7 +34,7 @@ 

  

  

  class TestMBSModule:

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_module_modulemds_nsvc(self, mock_session, testmodule_mmd_9c690d0e):

          """ Tests for querying a module from mbs """

          mock_res = Mock()
@@ -52,7 +52,7 @@ 

              "meta": {"next": None},

          }

  

-         mock_session().get.return_value = mock_res

+         mock_session.get.return_value = mock_res

  

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          module_mmds = resolver.get_module_modulemds(
@@ -76,10 +76,10 @@ 

              "state": "ready",

              "virtual_stream": ["f28"],

          }

-         mock_session().get.assert_called_once_with(mbs_url, params=expected_query)

+         mock_session.get.assert_called_once_with(mbs_url, params=expected_query)

          assert nsvcs == expected

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_module_modulemds_partial(

          self, mock_session, testmodule_mmd_9c690d0e, testmodule_mmd_c2c572ed

      ):
@@ -109,7 +109,7 @@ 

              "meta": {"next": None},

          }

  

-         mock_session().get.return_value = mock_res

+         mock_session.get.return_value = mock_res

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          ret = resolver.get_module_modulemds("testmodule", "master", version)

          nsvcs = set(
@@ -131,10 +131,10 @@ 

              "per_page": 10,

              "state": "ready",

          }

-         mock_session().get.assert_called_once_with(mbs_url, params=expected_query)

+         mock_session.get.assert_called_once_with(mbs_url, params=expected_query)

          assert nsvcs == expected

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_module_build_dependencies(

          self, mock_session, platform_mmd, testmodule_mmd_9c690d0e

      ):
@@ -171,7 +171,7 @@ 

              },

          ]

  

-         mock_session().get.return_value = mock_res

+         mock_session.get.return_value = mock_res

          expected = set(["module-f28-build"])

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          result = resolver.get_module_build_dependencies(
@@ -207,11 +207,11 @@ 

              call(mbs_url, params=expected_queries[0]),

              call(mbs_url, params=expected_queries[1]),

          ]

-         mock_session().get.mock_calls = expected_calls

-         assert mock_session().get.call_count == 2

+         mock_session.get.mock_calls = expected_calls

+         assert mock_session.get.call_count == 2

          assert set(result) == expected

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_module_build_dependencies_empty_buildrequires(

          self, mock_session, testmodule_mmd_9c690d0e

      ):
@@ -242,7 +242,7 @@ 

              }

          ]

  

-         mock_session().get.return_value = mock_res

+         mock_session.get.return_value = mock_res

  

          expected = set()

  
@@ -262,10 +262,10 @@ 

              "per_page": 10,

              "state": "ready",

          }

-         mock_session().get.assert_called_once_with(mbs_url, params=expected_query)

+         mock_session.get.assert_called_once_with(mbs_url, params=expected_query)

          assert set(result) == expected

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_resolve_profiles(self, mock_session, formatted_testmodule_mmd, platform_mmd):

  

          mock_res = Mock()
@@ -283,7 +283,7 @@ 

              "meta": {"next": None},

          }

  

-         mock_session().get.return_value = mock_res

+         mock_session.get.return_value = mock_res

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          result = resolver.resolve_profiles(

              formatted_testmodule_mmd, ("buildroot", "srpm-buildroot")
@@ -339,7 +339,7 @@ 

              "state": "ready",

          }

  

-         mock_session().get.assert_called_once_with(mbs_url, params=expected_query)

+         mock_session.get.assert_called_once_with(mbs_url, params=expected_query)

          assert result == expected

  

      @patch(
@@ -363,56 +363,54 @@ 

              expected = {"buildroot": set(["foo"]), "srpm-buildroot": set(["bar"])}

              assert result == expected

  

-     def test_get_empty_buildrequired_modulemds(self):

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

+     def test_get_empty_buildrequired_modulemds(self, mock_session):

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

+         mock_session.get.return_value = Mock(ok=True)

+         mock_session.get.return_value.json.return_value = {"items": [], "meta": {"next": None}}

  

-         with patch.object(resolver, "session") as session:

-             session.get.return_value = Mock(ok=True)

-             session.get.return_value.json.return_value = {"items": [], "meta": {"next": None}}

+         result = resolver.get_buildrequired_modulemds("nodejs", "10", "platform:el8:1:00000000")

+         assert [] == result

  

-             result = resolver.get_buildrequired_modulemds("nodejs", "10", "platform:el8:1:00000000")

-             assert [] == result

- 

-     def test_get_buildrequired_modulemds(self):

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

+     def test_get_buildrequired_modulemds(self, mock_session):

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

+         mock_session.get.return_value = Mock(ok=True)

+         with models.make_session(conf) as db_session:

+             mock_session.get.return_value.json.return_value = {

+                 "items": [

+                     {

+                         "name": "nodejs",

+                         "stream": "10",

+                         "version": 1,

+                         "context": "c1",

+                         "modulemd": mmd_to_str(

+                             tests.make_module(db_session, "nodejs:10:1:c1", store_to_db=False),

+                         ),

+                     },

+                     {

+                         "name": "nodejs",

+                         "stream": "10",

+                         "version": 2,

+                         "context": "c1",

+                         "modulemd": mmd_to_str(

+                             tests.make_module(db_session, "nodejs:10:2:c1", store_to_db=False),

+                         ),

+                     },

+                 ],

+                 "meta": {"next": None},

+             }

  

-         with patch.object(resolver, "session") as session:

-             session.get.return_value = Mock(ok=True)

-             with models.make_session(conf) as db_session:

-                 session.get.return_value.json.return_value = {

-                     "items": [

-                         {

-                             "name": "nodejs",

-                             "stream": "10",

-                             "version": 1,

-                             "context": "c1",

-                             "modulemd": mmd_to_str(

-                                 tests.make_module(db_session, "nodejs:10:1:c1", store_to_db=False),

-                             ),

-                         },

-                         {

-                             "name": "nodejs",

-                             "stream": "10",

-                             "version": 2,

-                             "context": "c1",

-                             "modulemd": mmd_to_str(

-                                 tests.make_module(db_session, "nodejs:10:2:c1", store_to_db=False),

-                             ),

-                         },

-                     ],

-                     "meta": {"next": None},

-                 }

- 

-             result = resolver.get_buildrequired_modulemds("nodejs", "10", "platform:el8:1:00000000")

+         result = resolver.get_buildrequired_modulemds("nodejs", "10", "platform:el8:1:00000000")

  

-             assert 1 == len(result)

-             mmd = result[0]

-             assert "nodejs" == mmd.get_module_name()

-             assert "10" == mmd.get_stream_name()

-             assert 1 == mmd.get_version()

-             assert "c1" == mmd.get_context()

+         assert 1 == len(result)

+         mmd = result[0]

+         assert "nodejs" == mmd.get_module_name()

+         assert "10" == mmd.get_stream_name()

+         assert 1 == mmd.get_version()

+         assert "c1" == mmd.get_context()

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_module_count(self, mock_session):

          mock_res = Mock()

          mock_res.ok.return_value = True
@@ -420,18 +418,18 @@ 

              "items": [{"name": "platform", "stream": "f28", "version": "3", "context": "00000000"}],

              "meta": {"total": 5},

          }

-         mock_session.return_value.get.return_value = mock_res

+         mock_session.get.return_value = mock_res

  

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          count = resolver.get_module_count(name="platform", stream="f28")

  

          assert count == 5

-         mock_session.return_value.get.assert_called_once_with(

+         mock_session.get.assert_called_once_with(

              "https://mbs.fedoraproject.org/module-build-service/1/module-builds/",

              params={"name": "platform", "page": 1, "per_page": 1, "short": True, "stream": "f28"},

          )

  

-     @patch("requests.Session")

+     @patch("module_build_service.resolver.MBSResolver.requests_session")

      def test_get_latest_with_virtual_stream(self, mock_session, platform_mmd):

          mock_res = Mock()

          mock_res.ok.return_value = True
@@ -447,13 +445,13 @@ 

              ],

              "meta": {"total": 5},

          }

-         mock_session.return_value.get.return_value = mock_res

+         mock_session.get.return_value = mock_res

  

          resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")

          mmd = resolver.get_latest_with_virtual_stream("platform", "virtualf28")

  

          assert mmd.get_module_name() == "platform"

-         mock_session.return_value.get.assert_called_once_with(

+         mock_session.get.assert_called_once_with(

              "https://mbs.fedoraproject.org/module-build-service/1/module-builds/",

              params={

                  "name": "platform",

@@ -0,0 +1,145 @@ 

+ # Copyright (c) 2019 Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ 

+ import textwrap

+ 

+ from mock import patch

+ import pytest

+ import requests

+ 

+ from module_build_service.models import ModuleBuild

+ from module_build_service.scheduler.default_modules import add_default_modules

+ from module_build_service.utils.general import load_mmd, mmd_to_str

+ from tests import clean_database, make_module, read_staged_data

+ 

+ 

+ @patch("module_build_service.scheduler.default_modules.requests_session")

+ def test_add_default_modules(mock_requests_session, db_session):

+     """

+     Test that default modules present in the database are added, and the others are ignored.

+     """

+     clean_database()

+     make_module(db_session, "python:3:12345:1")

+     make_module(db_session, "nodejs:11:2345:2")

+     mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

+     xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]

+     assert set(xmd_brs.keys()) == {"platform"}

+ 

+     platform = ModuleBuild.get_build_from_nsvc(

+         db_session,

+         "platform",

+         xmd_brs["platform"]["stream"],

+         xmd_brs["platform"]["version"],

+         xmd_brs["platform"]["context"],

+     )

+     assert platform

+     platform_mmd = platform.mmd()

+     platform_xmd = mmd.get_xmd()

+     default_modules_url = "http://domain.local/default_modules.txt"

+     platform_xmd["mbs"]["default_modules_url"] = default_modules_url

+     platform_mmd.set_xmd(platform_xmd)

+     platform.modulemd = mmd_to_str(platform_mmd)

+     db_session.commit()

+ 

+     mock_requests_session.get.return_value.ok = True

+     # Also ensure that if there's an invalid line, it's just ignored

+     mock_requests_session.get.return_value.text = textwrap.dedent("""\

+         nodejs:11

+         python:3

+         ruby:2.6

+         some invalid stuff

+     """)

+     add_default_modules(db_session, mmd)

+     # Make sure that the default modules were added. ruby:2.6 will be ignored since it's not in

+     # the database

+     assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"nodejs", "platform", "python"}

+     mock_requests_session.get.assert_called_once_with(default_modules_url, timeout=10)

+ 

+ 

+ @patch("module_build_service.scheduler.default_modules.requests_session")

+ def test_add_default_modules_not_linked(mock_requests_session, db_session):

+     """

+     Test that no default modules are added when they aren't linked from the base module.

+     """

+     clean_database()

+     mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

+     assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"}

+     add_default_modules(db_session, mmd)

+     assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"}

+     mock_requests_session.get.assert_not_called()

+ 

+ 

+ @patch("module_build_service.scheduler.default_modules.requests_session")

+ def test_add_default_modules_platform_not_available(mock_requests_session, db_session):

+     """

+     Test that an exception is raised when the platform module that is buildrequired is missing.

+ 

+     This error should never occur in practice.

+     """

+     clean_database(False, False)

+     mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

+ 

+     expected_error = "Failed to retrieve the module platform:f28:3:00000000 from the database"

+     with pytest.raises(RuntimeError, match=expected_error):

+         add_default_modules(db_session, mmd)

+ 

+ 

+ @pytest.mark.parametrize("connection_error", (True, False))

+ @patch("module_build_service.scheduler.default_modules.requests_session")

+ def test_add_default_modules_request_failed(mock_requests_session, connection_error, db_session):

+     """

+     Test that an exception is raised when the request to get the default modules failed.

+     """

+     clean_database()

+     make_module(db_session, "python:3:12345:1")

+     make_module(db_session, "nodejs:11:2345:2")

+     mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

+     xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]

+     assert set(xmd_brs.keys()) == {"platform"}

+ 

+     platform = ModuleBuild.get_build_from_nsvc(

+         db_session,

+         "platform",

+         xmd_brs["platform"]["stream"],

+         xmd_brs["platform"]["version"],

+         xmd_brs["platform"]["context"],

+     )

+     assert platform

+     platform_mmd = platform.mmd()

+     platform_xmd = mmd.get_xmd()

+     default_modules_url = "http://domain.local/default_modules.txt"

+     platform_xmd["mbs"]["default_modules_url"] = default_modules_url

+     platform_mmd.set_xmd(platform_xmd)

+     platform.modulemd = mmd_to_str(platform_mmd)

+     db_session.commit()

+ 

+     if connection_error:

+         mock_requests_session.get.side_effect = requests.ConnectionError("some error")

+         expected_error = (

+             "The connection failed when getting the default modules associated with "

+             "platform:f28:3:00000000"

+         )

+     else:

+         mock_requests_session.get.return_value.ok = False

+         mock_requests_session.get.return_value.text = "some error"

+         expected_error = "Failed to retrieve the default modules for platform:f28:3:00000000"

+ 

+     with pytest.raises(RuntimeError, match=expected_error):

+         add_default_modules(db_session, mmd)

A base module can set xmd.mbs.default_modules_url, which contains a URL to a list of modules in the format of name:stream separated by new lines. When a module buildrequires this base module, the list of default modules are added as buildrequires of the module automatically unless there are conflicting streams or the default module is not in the MBS database.

Please note that the first commit adds a global requests session to reuse rather than duplicate the code in MBSResolver. I recommend reviewing this PR commit by commit.

Build #213 failed (commit: 5b16bc63b69f0d4607df590f647c8e57ac171fc5).
Rebase or make new commits to rebuild.

2 new commits added

  • Add the ability to automatically buildrequire default modules defined by the buildrequired base module
  • Add a global requests session with retry logic configured
4 years ago

Build #215 failed (commit: 9cbcd1486ca3a97cff52ea9ef5252c836003f16d).
Rebase or make new commits to rebuild.

2 new commits added

  • Add the ability to automatically buildrequire default modules defined by the buildrequired base module
  • Add a global requests session with retry logic configured
4 years ago

Pull-Request has been merged by mprahl

4 years ago