From 520a979d3d7cf7bd69d9ffb6305ef92c3b87b263 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Sep 28 2018 03:00:52 +0000 Subject: Add singleton system_resolver system_resolver is created based on loaded configuration, which could avoid calls like `GenericResolver.create(conf)` repeatedly in the code. However, if some cases need to create a specific resolver explicitly, `GenericResolver.create` could be called with addition argument, for example db or mbs is passed to argument backend in tests. Signed-off-by: Chenxiong Qi --- diff --git a/module_build_service/builder/base.py b/module_build_service/builder/base.py index 1fb9e1c..f214c79 100644 --- a/module_build_service/builder/base.py +++ b/module_build_service/builder/base.py @@ -111,7 +111,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): and are implementation-dependent. """ # check if the backend is within allowed backends for the used resolver - resolver = module_build_service.resolver.GenericResolver.create(config) + resolver = module_build_service.resolver.system_resolver if not resolver.is_builder_compatible(backend): raise ValueError("Builder backend '{}' is not compatible with " "resolver backend '{}'. Check your configuration." @@ -305,7 +305,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): try: mmd = module.mmd() - resolver = module_build_service.resolver.GenericResolver.create(conf) + resolver = module_build_service.resolver.system_resolver # Resolve default buildroot groups using the MBS, but only for # non-local modules. diff --git a/module_build_service/resolver/__init__.py b/module_build_service/resolver/__init__.py index e2771c2..82cf80e 100644 --- a/module_build_service/resolver/__init__.py +++ b/module_build_service/resolver/__init__.py @@ -23,6 +23,7 @@ import pkg_resources +from module_build_service import conf from module_build_service.resolver.base import GenericResolver # NOTE: if you are adding a new resolver to MBS please note that you also have to add @@ -32,3 +33,8 @@ for entrypoint in pkg_resources.iter_entry_points('mbs.resolver_backends'): if not GenericResolver.backends: raise ValueError("No resolver plugins are installed or available.") + +# Config has the option of which resolver should be used for current MBS run. +# Hence, create a singleton system wide resolver for use. However, resolver +# could be created with other required arguments in concrete cases. +system_resolver = GenericResolver.create(conf) diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 5ee9863..c996887 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -209,7 +209,7 @@ def wait(config, session, msg): tag = None dependencies = [] - resolver = module_build_service.resolver.GenericResolver.create(config) + resolver = module_build_service.resolver.system_resolver @module_build_service.utils.retry( interval=10, timeout=120, diff --git a/module_build_service/utils/mse.py b/module_build_service/utils/mse.py index 4166462..7f8c1cf 100644 --- a/module_build_service/utils/mse.py +++ b/module_build_service/utils/mse.py @@ -22,7 +22,7 @@ # Written by Ralph Bean # Matt Prahl # Jan Kaluza -from module_build_service import log, models, Modulemd, conf, db +from module_build_service import log, models, Modulemd, db from module_build_service.errors import StreamAmbigous from module_build_service.mmd_resolver import MMDResolver from module_build_service import glib @@ -137,7 +137,7 @@ def _get_mmds_from_requires(session, requires, mmds, recursive=False, # To be able to call itself recursively, we need to store list of mmds # we have added to global mmds list in this particular call. added_mmds = {} - resolver = module_build_service.resolver.GenericResolver.create(conf) + resolver = module_build_service.resolver.system_resolver for name, streams in requires.items(): streams_to_try = streams.get() @@ -364,7 +364,7 @@ def generate_expanded_mmds(session, mmd, raise_if_stream_ambigous=False, default # Resolve the buildrequires and store the result in XMD. if 'mbs' not in xmd: xmd['mbs'] = {} - resolver = module_build_service.resolver.GenericResolver.create(conf) + resolver = module_build_service.resolver.system_resolver xmd['mbs']['buildrequires'] = resolver.resolve_requires(br_list) xmd['mbs']['mse'] = True diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 34b2222..fb9058d 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -33,11 +33,13 @@ from datetime import datetime import kobo.rpmlib import requests +import module_build_service.scm +import module_build_service.resolver + from module_build_service import conf, db, log, models, Modulemd from module_build_service.errors import ( ValidationError, UnprocessableEntity, Forbidden, Conflict) from module_build_service import glib -import module_build_service.scm from .mse import generate_expanded_mmds @@ -54,7 +56,7 @@ def record_filtered_rpms(mmd): from module_build_service.builder import GenericBuilder new_buildrequires = {} - resolver = module_build_service.resolver.GenericResolver.create(conf) + resolver = module_build_service.resolver.system_resolver for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items(): # In case this is module resubmit or local build, the filtered_rpms # will already be there, so there is no point in generating them again. diff --git a/tests/test_builder/test_base.py b/tests/test_builder/test_base.py index 9ad6e89..e4e5153 100644 --- a/tests/test_builder/test_base.py +++ b/tests/test_builder/test_base.py @@ -24,6 +24,7 @@ import mock import module_build_service.models import module_build_service.builder +import module_build_service.resolver from tests import init_data, db @@ -48,31 +49,33 @@ class TestGenericBuilder: resolver = mock.MagicMock() resolver.backend = 'mbs' resolver.resolve_profiles.return_value = mbs_groups - generic_resolver.create.return_value = resolver expected_groups = { "build": [], "srpm-build": [] } - # Call default_buildroot_groups, the result should be cached. - ret = GenericBuilder.default_buildroot_groups(db.session, self.module) - assert ret == expected_groups - resolver.resolve_profiles.assert_called_once() - resolver.resolve_profiles.reset_mock() + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + # Call default_buildroot_groups, the result should be cached. + ret = GenericBuilder.default_buildroot_groups(db.session, self.module) + assert ret == expected_groups + resolver.resolve_profiles.assert_called_once() + resolver.resolve_profiles.reset_mock() # Now try calling it again to verify resolve_profiles is not called, # because it is cached. - ret = GenericBuilder.default_buildroot_groups(db.session, self.module) - assert ret == expected_groups - resolver.resolve_profiles.assert_not_called() - resolver.resolve_profiles.reset_mock() + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + ret = GenericBuilder.default_buildroot_groups(db.session, self.module) + assert ret == expected_groups + resolver.resolve_profiles.assert_not_called() + resolver.resolve_profiles.reset_mock() # And now try clearing the cache and call it again. - GenericBuilder.clear_cache(self.module) - ret = GenericBuilder.default_buildroot_groups(db.session, self.module) - assert ret == expected_groups - resolver.resolve_profiles.assert_called_once() + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + GenericBuilder.clear_cache(self.module) + ret = GenericBuilder.default_buildroot_groups(db.session, self.module) + assert ret == expected_groups + resolver.resolve_profiles.assert_called_once() def test_get_build_weights(self): weights = GenericBuilder.get_build_weights(["httpd", "apr"]) diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index e2a7cc2..8658e1b 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -27,6 +27,7 @@ import module_build_service.scheduler.handlers.modules import os import koji from tests import conf, db, app, scheduler_init_data +import module_build_service.resolver from module_build_service import build_logs, Modulemd from module_build_service.models import ComponentBuild, ModuleBuild @@ -48,8 +49,7 @@ class TestModuleWait: @patch('module_build_service.builder.GenericBuilder.create_from_module') @patch('module_build_service.models.ModuleBuild.from_module_event') - @patch('module_build_service.resolver.GenericResolver') - def test_init_basic(self, resolver, from_module_event, create_builder): + def test_init_basic(self, from_module_event, create_builder): builder = mock.Mock() builder.get_disttag_srpm.return_value = 'some srpm disttag' builder.build.return_value = 1234, 1, "", None @@ -82,7 +82,8 @@ class TestModuleWait: msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, module_build_state='some state') - self.fn(config=self.config, session=self.session, msg=msg) + with patch.object(module_build_service.resolver, 'system_resolver'): + self.fn(config=self.config, session=self.session, msg=msg) @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) @@ -112,13 +113,13 @@ class TestModuleWait: resolver = mock.MagicMock() resolver.backend = 'db' resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357" - generic_resolver.create.return_value = resolver - msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, - module_build_state='some state') - module_build_service.scheduler.handlers.modules.wait( - config=conf, session=db.session, msg=msg) - koji_session.newRepo.assert_called_once_with("module-123-build") + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + koji_session.newRepo.assert_called_once_with("module-123-build") # When module-build-macros is reused, it still has to appear only # once in database. @@ -154,13 +155,13 @@ class TestModuleWait: resolver = mock.MagicMock() resolver.backend = 'db' resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357" - generic_resolver.create.return_value = resolver - msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, - module_build_state='some state') - module_build_service.scheduler.handlers.modules.wait( - config=conf, session=db.session, msg=msg) - assert koji_session.newRepo.called + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + assert koji_session.newRepo.called @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) @@ -196,14 +197,14 @@ class TestModuleWait: resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357" resolver.get_module_build_dependencies.return_value = { "module-bootstrap-tag": base_mmd} - generic_resolver.create.return_value = resolver - msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, - module_build_state='some state') - module_build_service.scheduler.handlers.modules.wait( - config=conf, session=db.session, msg=msg) - module_build = ModuleBuild.query.filter_by(id=2).one() - assert module_build.cg_build_koji_tag == "modular-updates-candidate" + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + module_build = ModuleBuild.query.filter_by(id=2).one() + assert module_build.cg_build_koji_tag == "modular-updates-candidate" @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) @@ -241,11 +242,11 @@ class TestModuleWait: resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357" resolver.get_module_build_dependencies.return_value = { "module-bootstrap-tag": base_mmd} - generic_resolver.create.return_value = resolver - - msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, - module_build_state='some state') - module_build_service.scheduler.handlers.modules.wait( - config=conf, session=db.session, msg=msg) - module_build = ModuleBuild.query.filter_by(id=2).one() - assert module_build.cg_build_koji_tag == "f27-modular-updates-candidate" + + with patch.object(module_build_service.resolver, 'system_resolver', new=resolver): + msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2, + module_build_state='some state') + module_build_service.scheduler.handlers.modules.wait( + config=conf, session=db.session, msg=msg) + module_build = ModuleBuild.query.filter_by(id=2).one() + assert module_build.cg_build_koji_tag == "f27-modular-updates-candidate"