#1110 In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg.
Merged 7 months ago by jkaluza. Opened 7 months ago by jkaluza.
jkaluza/fm-orchestrator factory-2901  into  master

@@ -194,17 +194,22 @@ 

                          log.info('      * {0} components in batch {1}'

                                   .format(n, i))

  

-     def process_waiting_module_builds(self, session):

-         log.info('Looking for module builds stuck in the wait state')

-         builds = models.ModuleBuild.by_state(session, 'wait')

-         log.info(' {0!r} module builds in the wait state...'

-                  .format(len(builds)))

+     def _nudge_module_builds_in_state(self, session, state, older_than_minutes):

+         """

+         Finds all the module builds in the `state` with `time_modified` older

+         than `older_than_minutes` and adds fake MBSModule message to the

+         work queue.

+         """

+         log.info('Looking for module builds stuck in the %s state', state)

+         builds = models.ModuleBuild.by_state(session, state)

+         log.info(' {0!r} module builds in the %s state...'

+                  .format(len(builds), state))

          now = datetime.utcnow()

-         ten_minutes = timedelta(minutes=10)

+         time_modified_threshold = timedelta(minutes=older_than_minutes)

          for build in builds:

  

              # Only give builds a nudge if stuck for more than ten minutes

-             if (now - build.time_modified) < ten_minutes:

+             if (now - build.time_modified) < time_modified_threshold:

                  continue

  

              # Pretend the build is modified, so we don't tight spin.

@@ -212,12 +217,16 @@ 

              session.commit()

  

              # Fake a message to kickstart the build anew in the consumer

-             state = module_build_service.models.BUILD_STATES['wait']

+             state = module_build_service.models.BUILD_STATES[state]

              msg = module_build_service.messaging.MBSModule(

-                 'fake message', build.id, state)

+                 'nudge_module_builds_fake_message', build.id, state)

              log.info("  Scheduling faked event %r" % msg)

              module_build_service.scheduler.consumer.work_queue_put(msg)

  

+     def process_waiting_module_builds(self, session):

+         for state in ['init', 'wait']:

+             self._nudge_module_builds_in_state(session, state, 10)

+ 

      def process_open_component_builds(self, session):

          log.warning('process_open_component_builds is not yet implemented...')

  

@@ -356,8 +356,9 @@ 

  

              koji_session.deleteBuildTarget.assert_not_called()

  

+     @pytest.mark.parametrize('state', ['init', 'wait'])

      def test_process_waiting_module_build(

-             self, create_builder, global_consumer, dbg):

+             self, create_builder, global_consumer, dbg, state):

          """ Test that processing old waiting module builds works. """

  

          consumer = mock.MagicMock()

@@ -370,7 +371,7 @@ 

          # Change the batch to 2, so the module build is in state where

          # it is not building anything, but the state is "build".

          module_build = models.ModuleBuild.query.filter_by(id=3).one()

-         module_build.state = 1

+         module_build.state = models.BUILD_STATES[state]

          original = datetime.utcnow() - timedelta(minutes=11)

          module_build.time_modified = original

          db.session.commit()

@@ -387,8 +388,9 @@ 

          # ensure the time_modified was changed.

          assert module_build.time_modified > original

  

+     @pytest.mark.parametrize('state', ['init', 'wait'])

      def test_process_waiting_module_build_not_old_enough(

-             self, create_builder, global_consumer, dbg):

+             self, create_builder, global_consumer, dbg, state):

          """ Test that we do not process young waiting builds. """

  

          consumer = mock.MagicMock()

@@ -401,7 +403,7 @@ 

          # Change the batch to 2, so the module build is in state where

          # it is not building anything, but the state is "build".

          module_build = models.ModuleBuild.query.filter_by(id=3).one()

-         module_build.state = 1

+         module_build.state = models.BUILD_STATES[state]

          original = datetime.utcnow() - timedelta(minutes=9)

          module_build.time_modified = original

          db.session.commit()

This fixes issues when UMB message delivery from frontend to backend fails
for whatever reason...

We do the same thing for 'wait' state already, so this commit just extends
it to 'init' state too.

Pull-Request has been merged by jkaluza

7 months ago