PR#1110: In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg. - fm-orchestrator

fm-orchestrator

#1110 In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg.

Merged 5 years ago by jkaluza. Opened 5 years ago by jkaluza.

jkaluza/fm-orchestrator factory-2901 into master

In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg.

Jan Kaluza • 5 years ago

bcfead0

module_build_service/scheduler/producer.py

file modified

+18 -9

		`@@ -194,17 +194,22 @@`
		`log.info(' * {0} components in batch {1}'`
		`.format(n, i))`

		`- def process_waiting_module_builds(self, session):`
		`- log.info('Looking for module builds stuck in the wait state')`
		`- builds = models.ModuleBuild.by_state(session, 'wait')`
		`- log.info(' {0!r} module builds in the wait state...'`
		`- .format(len(builds)))`
		`+ def _nudge_module_builds_in_state(self, session, state, older_than_minutes):`
		`+ """`
		+ Finds all the module builds in the `state` with `time_modified` older
		+ than `older_than_minutes` and adds fake MBSModule message to the
		`+ work queue.`
		`+ """`
		`+ log.info('Looking for module builds stuck in the %s state', state)`
		`+ builds = models.ModuleBuild.by_state(session, state)`
		`+ log.info(' {0!r} module builds in the %s state...'`
		`+ .format(len(builds), state))`
		`now = datetime.utcnow()`
		`- ten_minutes = timedelta(minutes=10)`
		`+ time_modified_threshold = timedelta(minutes=older_than_minutes)`
		`for build in builds:`

		`# Only give builds a nudge if stuck for more than ten minutes`
		`- if (now - build.time_modified) < ten_minutes:`
		`+ if (now - build.time_modified) < time_modified_threshold:`
		`continue`

		`# Pretend the build is modified, so we don't tight spin.`
		`@@ -212,12 +217,16 @@`
		`session.commit()`

		`# Fake a message to kickstart the build anew in the consumer`
		`- state = module_build_service.models.BUILD_STATES['wait']`
		`+ state = module_build_service.models.BUILD_STATES[state]`
		`msg = module_build_service.messaging.MBSModule(`
		`- 'fake message', build.id, state)`
		`+ 'nudge_module_builds_fake_message', build.id, state)`
		`log.info(" Scheduling faked event %r" % msg)`
		`module_build_service.scheduler.consumer.work_queue_put(msg)`

		`+ def process_waiting_module_builds(self, session):`
		`+ for state in ['init', 'wait']:`
		`+ self._nudge_module_builds_in_state(session, state, 10)`
		`+`
		`def process_open_component_builds(self, session):`
		`log.warning('process_open_component_builds is not yet implemented...')`

tests/test_scheduler/test_poller.py

file modified

+6 -4

		`@@ -356,8 +356,9 @@`

		`koji_session.deleteBuildTarget.assert_not_called()`

		`+ @pytest.mark.parametrize('state', ['init', 'wait'])`
		`def test_process_waiting_module_build(`
		`- self, create_builder, global_consumer, dbg):`
		`+ self, create_builder, global_consumer, dbg, state):`
		`""" Test that processing old waiting module builds works. """`

		`consumer = mock.MagicMock()`
		`@@ -370,7 +371,7 @@`
		`# Change the batch to 2, so the module build is in state where`
		`# it is not building anything, but the state is "build".`
		`module_build = models.ModuleBuild.query.filter_by(id=3).one()`
		`- module_build.state = 1`
		`+ module_build.state = models.BUILD_STATES[state]`
		`original = datetime.utcnow() - timedelta(minutes=11)`
		`module_build.time_modified = original`
		`db.session.commit()`
		`@@ -387,8 +388,9 @@`
		`# ensure the time_modified was changed.`
		`assert module_build.time_modified > original`

		`+ @pytest.mark.parametrize('state', ['init', 'wait'])`
		`def test_process_waiting_module_build_not_old_enough(`
		`- self, create_builder, global_consumer, dbg):`
		`+ self, create_builder, global_consumer, dbg, state):`
		`""" Test that we do not process young waiting builds. """`

		`consumer = mock.MagicMock()`
		`@@ -401,7 +403,7 @@`
		`# Change the batch to 2, so the module build is in state where`
		`# it is not building anything, but the state is "build".`
		`module_build = models.ModuleBuild.query.filter_by(id=3).one()`
		`- module_build.state = 1`
		`+ module_build.state = models.BUILD_STATES[state]`
		`original = datetime.utcnow() - timedelta(minutes=9)`
		`module_build.time_modified = original`
		`db.session.commit()`