From 244b77f54cdcd68c9b8bc25573d07e028d8acc0f Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Nov 16 2016 20:24:17 +0000 Subject: Add component build throttling --- diff --git a/module_build_service/models.py b/module_build_service/models.py index 65ddd0a..14cf4f6 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -122,16 +122,22 @@ class ModuleBuild(RidaBase): module = db.relationship('Module', backref='module_builds', lazy=False) - def current_batch(self): + def current_batch(self, state=None): """ Returns all components of this module in the current batch. """ if not self.batch: raise ValueError("No batch is in progress: %r" % self.batch) - return [ - component for component in self.component_builds - if component.batch == self.batch - ] + if state: + return [ + component for component in self.component_builds + if component.batch == self.batch and component.state == state + ] + else: + return [ + component for component in self.component_builds + if component.batch == self.batch + ] def mmd(self): mmd = _modulemd.ModuleMetadata() diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py index 9f97a9f..4ead3a7 100644 --- a/module_build_service/scheduler/handlers/repos.py +++ b/module_build_service/scheduler/handlers/repos.py @@ -100,18 +100,24 @@ def done(config, session, msg): # So now we can either start a new batch if there are still some to build # or, if everything is built successfully, then we can bless the module as # complete. - leftover_components = [ + unbuilt_components = [ c for c in module_build.component_builds if (c.state != koji.BUILD_STATES['COMPLETE'] and c.state != koji.BUILD_STATES["FAILED"]) ] - if leftover_components: - module_build.batch += 1 + if unbuilt_components: + # Increment the build batch when no components are being built and all + # have at least attempted a build (even failures) in the current batch + unbuilt_components_in_batch = [ + c for c in module_build.current_batch() + if c.state == koji.BUILD_STATES['BUILDING'] or not c.state + ] + if not unbuilt_components_in_batch: + module_build.batch += 1 + module_build_service.utils.start_build_batch( config, module_build, session, builder) else: module_build.transition(config, state=models.BUILD_STATES['done']) session.commit() - - # And that's it. :) diff --git a/module_build_service/scheduler/main.py b/module_build_service/scheduler/main.py index 2224736..d9c54b3 100644 --- a/module_build_service/scheduler/main.py +++ b/module_build_service/scheduler/main.py @@ -38,6 +38,7 @@ import six.moves.queue as queue import module_build_service.config import module_build_service.messaging +import module_build_service.utils import module_build_service.scheduler.handlers.components import module_build_service.scheduler.handlers.modules import module_build_service.scheduler.handlers.repos @@ -190,8 +191,8 @@ class Poller(threading.Thread): # XXX: detect whether it's really stucked first # self.process_waiting_module_builds(session) self.process_open_component_builds(session) - self.process_lingering_module_builds(session) self.fail_lost_builds(session) + self.process_paused_module_builds(conf, session) log.info("Polling thread sleeping, %rs" % conf.polling_interval) time.sleep(conf.polling_interval) @@ -281,8 +282,21 @@ class Poller(threading.Thread): def process_open_component_builds(self, session): log.warning("process_open_component_builds is not yet implemented...") - def process_lingering_module_builds(self, session): - log.warning("process_lingering_module_builds is not yet implemented...") + def process_paused_module_builds(self, config, session): + if module_build_service.utils.at_concurrent_component_threshold( + config, session): + log.debug('Will not attempt to start paused module builds due to ' + 'the concurrent build threshold being met') + return + # Check to see if module builds that are in build state but don't have + # any component builds being built can be worked on + for module_build in session.query(models.ModuleBuild).filter_by( + state=models.BUILD_STATES['build']).all(): + # If there are no components in the build state on the module build, + # then no possible event will start off new component builds + if not module_build.current_batch(koji.BUILD_STATES['BUILDING']): + module_build_service.utils.start_build_batch( + config, module_build, session, config.system) _work_queue = queue.Queue() diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 1279056..e2e8a34 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -30,7 +30,6 @@ import shutil import tempfile import os import modulemd -import time from module_build_service import log, models from module_build_service.errors import ValidationError, UnprocessableEntity from module_build_service import app, conf, db, log @@ -59,9 +58,28 @@ def retry(timeout=120, interval=30, wait_on=Exception): return wrapper +def at_concurrent_component_threshold(config, session): + """ + Determines if the number of concurrent component builds has reached + the configured threshold + :param config: Module Build Service configuration object + :param session: SQLAlchemy database session + :return: boolean representing if there are too many concurrent builds at + this time + """ + + import koji # Placed here to avoid py2/py3 conflicts... + + if config.num_consecutive_builds and config.num_consecutive_builds <= \ + session.query(models.ComponentBuild).filter_by( + state=koji.BUILD_STATES['BUILDING']).count(): + return True + + return False + + def start_build_batch(config, module, session, builder, components=None): """ Starts a round of the build cycle for a module. """ - import koji # Placed here to avoid py2/py3 conflicts... if any([c.state == koji.BUILD_STATES['BUILDING'] @@ -70,11 +88,11 @@ def start_build_batch(config, module, session, builder, components=None): # The user can either pass in a list of components to 'seed' the batch, or # if none are provided then we just select everything that hasn't - # successfully built yet. - module.batch += 1 + # successfully built yet or isn't currently being built. unbuilt_components = components or [ c for c in module.component_builds if (c.state != koji.BUILD_STATES['COMPLETE'] + and c.state != koji.BUILD_STATES['BUILDING'] and c.batch == module.batch) ] @@ -82,7 +100,12 @@ def start_build_batch(config, module, session, builder, components=None): unbuilt_components)) for c in unbuilt_components: - c.task_id, c.state, c.state_reason, c.nvr = builder.build(artifact_name=c.package, source=c.scmurl) + if at_concurrent_component_threshold(config, session): + log.info('Concurrent build threshold met') + break + + c.task_id, c.state, c.state_reason, c.nvr = builder.build( + artifact_name=c.package, source=c.scmurl) if not c.task_id: module.transition(config, models.BUILD_STATES["failed"], diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index 603d881..0db7a76 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -54,7 +54,9 @@ class TestRepoDone(unittest.TestCase): @mock.patch('module_build_service.builder.KojiModuleBuilder.build') @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect') @mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event') - def test_a_single_match(self, from_repo_done_event, connect, build_fn, config, ready): + @mock.patch('module_build_service.utils.at_concurrent_component_threshold', + return_value=False) + def test_a_single_match(self, threshold, from_repo_done_event, connect, build_fn, config, ready): """ Test that when a repo msg hits us and we have a single match. """ config.return_value = mock.Mock(), "development" @@ -88,7 +90,9 @@ class TestRepoDone(unittest.TestCase): @mock.patch('module_build_service.builder.KojiModuleBuilder.build') @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect') @mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event') - def test_a_single_match_build_fail(self, from_repo_done_event, connect, build_fn, config, ready): + @mock.patch('module_build_service.utils.at_concurrent_component_threshold', + return_value=False) + def test_a_single_match_build_fail(self, threshold, from_repo_done_event, connect, build_fn, config, ready): """ Test that when a KojiModuleBuilder.build fails, the build is marked as failed with proper state_reason. """