#1522 Allow building modules locally without fedmsg-hub.
Merged 4 years ago by jkaluza. Opened 4 years ago by jkaluza.
jkaluza/fm-orchestrator mockbuilder  into  v3

@@ -159,8 +159,6 @@ 

  

      yaml_file_path = os.path.abspath(yaml_file)

  

-     from module_build_service.db_session import db_session

- 

      if offline:

          import_builds_from_local_dnf_repos(platform_id)

      load_local_builds(local_build_nsvs)
@@ -181,10 +179,7 @@ 

  

          module_build_ids = [build.id for build in module_builds]

  

-     stop = module_build_service.scheduler.local.make_simple_stop_condition()

- 

-     # Run the consumer until stop_condition returns True

-     module_build_service.scheduler.local.main([], stop)

+     module_build_service.scheduler.local.main(module_build_ids)

  

      has_failed_module = db_session.query(models.ModuleBuild).filter(

          models.ModuleBuild.id.in_(module_build_ids),

@@ -1,12 +1,14 @@ 

  # -*- coding: utf-8 -*-

  # SPDX-License-Identifier: MIT

  

- import fedmsg

  import logging

- import module_build_service.models

- import moksha.hub

+ from module_build_service import models

  

  from module_build_service.db_session import db_session

+ from module_build_service.scheduler.handlers.modules import init as modules_init_handler

+ from module_build_service.scheduler.handlers.modules import wait as modules_wait_handler

+ from module_build_service.scheduler.handlers.modules import done as modules_done_handler

+ from module_build_service.scheduler.handlers.modules import failed as modules_failed_handler

  

  log = logging.getLogger(__name__)

  
@@ -16,82 +18,48 @@ 

  module build and running tests within test_build.py particularly.

  """

  

- __all__ = ["main", "make_simple_stop_condition"]

+ __all__ = ["main"]

  

  

- def main(initial_messages, stop_condition):

-     """ Run the consumer until some condition is met.

- 

-     Setting stop_condition to None will run the consumer forever.

+ def raise_for_failed_build(module_build_ids):

      """

+     Raises an exception if any module build from `module_build_ids` list is in failed state.

+     This function also calls "failed" handler before raises an exception.

  

-     config = fedmsg.config.load_config()

-     config["mbsconsumer"] = True

-     config["mbsconsumer.stop_condition"] = stop_condition

-     config["mbsconsumer.initial_messages"] = initial_messages

- 

-     # Moksha requires that we subscribe to *something*, so tell it /dev/null

-     # since we'll just be doing in-memory queue-based messaging for this single

-     # build.

-     config["zmq_enabled"] = True

-     config["zmq_subscribe_endpoints"] = "ipc:///dev/null"

- 

-     # Lazy import consumer to avoid potential import cycle.

-     # For example, in some cases, importing event message from events.py would

-     # cause importing the consumer module, which then starts to import relative

-     # code inside handlers module, and the original code is imported eventually.

-     import module_build_service.scheduler.consumer

- 

-     consumers = [module_build_service.scheduler.consumer.MBSConsumer]

- 

-     # Note that the hub we kick off here cannot send any message.  You

-     # should use fedmsg.publish(...) still for that.

-     moksha.hub.main(

-         # Pass in our config dict

-         options=config,

-         # Only run the specified consumers if any are so specified.

-         consumers=consumers,

-         # Do not run default producers.

-         producers=[],

-         # Tell moksha to quiet its logging.

-         framework=False,

-     )

- 

- 

- def make_simple_stop_condition():

-     """ Return a simple stop_condition callable.

- 

-     Intended to be used with the main() function here in manage.py and tests.

- 

-     The stop_condition returns true when the latest module build enters the any

-     of the finished states.

+     :param list module_build_ids: List of module build IDs (int) to build locally.

      """

+     builds = db_session.query(models.ModuleBuild).filter(

+         models.ModuleBuild.id.in_(module_build_ids)).all()

+     has_failed_build = False

+     for build in builds:

+         if build.state == models.BUILD_STATES["failed"]:

+             modules_failed_handler("fake_msg_id", build.id, "failed")

+             has_failed_build = True

+     if has_failed_build:

+         raise ValueError("Local module build failed.")

+ 

+ 

+ def main(module_build_ids):

+     """

+     Build modules locally. The modules have to be stored in the local database before

+     calling this function.

  

-     def stop_condition(message):

-         # XXX - We ignore the message here and instead just query the DB.

- 

-         # Grab the latest module build.

-         module = (

-             db_session.query(module_build_service.models.ModuleBuild)

-             .order_by(module_build_service.models.ModuleBuild.id.desc())

-             .first()

-         )

-         done = (

-             module_build_service.models.BUILD_STATES["failed"],

-             module_build_service.models.BUILD_STATES["ready"],

-             module_build_service.models.BUILD_STATES["done"],

-         )

-         result = module.state in done

-         log.debug("stop_condition checking %r, got %r" % (module, result))

- 

-         # moksha.hub.main starts the hub and runs it in a separate thread. When

-         # the result is True, remove the db_session from that thread local so

-         # that any pending queries in the transaction will not block other

-         # queries made from other threads.

-         # This is useful for testing particularly.

-         if result:

-             db_session.remove()

- 

-         return result

- 

-     return stop_condition

+     :param list module_build_ids: List of module build IDs (int) to build locally.

+     """

+     # The transition between states is normally handled by ModuleBuild.transition, which sends

+     # a message to message bus. The message is then received by the Consumer and handler is called.

+     # But for local builds, we do not have any message bus, so we just call the handlers in

+     # the right order manually.

+     # We only need to ensure that we won't call futher handlers in case the module build failed.

+     for module_build_id in module_build_ids:

+         modules_init_handler("fake_msg_id", module_build_id, "init")

+ 

+     raise_for_failed_build(module_build_ids)

+     for module_build_id in module_build_ids:

+         modules_wait_handler("fake_msg_id", module_build_id, "wait")

+ 

+     raise_for_failed_build(module_build_ids)

+     for module_build_id in module_build_ids:

+         modules_done_handler("fake_msg_id", module_build_id, "done")

+ 

+     raise_for_failed_build(module_build_ids)

@@ -10,16 +10,17 @@ 

  from datetime import datetime, timedelta

  from random import randint

  import hashlib

+ import moksha.hub

+ import fedmsg

  

  import module_build_service.messaging

  import module_build_service.scheduler.consumer

  import module_build_service.scheduler.handlers.repos

  import module_build_service.utils

  from module_build_service.errors import Forbidden

- from module_build_service import models, conf, build_logs

+ from module_build_service import models, conf, build_logs, log

  from module_build_service.db_session import db_session

  from module_build_service.scheduler import events

- from module_build_service.scheduler.local import make_simple_stop_condition

  from module_build_service.scheduler.handlers.tags import tagged as tagged_handler

  from module_build_service.scheduler.handlers.components import (

      build_task_finalize as build_task_finalize_handler)
@@ -44,6 +45,83 @@ 

  user = ("Homer J. Simpson", {"packager"})

  

  

+ def make_simple_stop_condition():

+     """ Return a simple stop_condition callable.

+ 

+     Intended to be used with the main() function here in manage.py and tests.

+ 

+     The stop_condition returns true when the latest module build enters the any

+     of the finished states.

+     """

+     def stop_condition(message):

+         # XXX - We ignore the message here and instead just query the DB.

+ 

+         # Grab the latest module build.

+         module = (

+             db_session.query(models.ModuleBuild)

+             .order_by(models.ModuleBuild.id.desc())

+             .first()

+         )

+         done = (

+             models.BUILD_STATES["failed"],

+             models.BUILD_STATES["ready"],

+             models.BUILD_STATES["done"],

+         )

+         result = module.state in done

+         log.debug("stop_condition checking %r, got %r" % (module, result))

+ 

+         # moksha.hub.main starts the hub and runs it in a separate thread. When

+         # the result is True, remove the db_session from that thread local so

+         # that any pending queries in the transaction will not block other

+         # queries made from other threads.

+         # This is useful for testing particularly.

+         if result:

+             db_session.remove()

+ 

+         return result

+ 

+     return stop_condition

+ 

+ 

+ def main(initial_messages, stop_condition):

+     """ Run the consumer until some condition is met.

+ 

+     Setting stop_condition to None will run the consumer forever.

+     """

+ 

+     config = fedmsg.config.load_config()

+     config["mbsconsumer"] = True

+     config["mbsconsumer.stop_condition"] = stop_condition

+     config["mbsconsumer.initial_messages"] = initial_messages

+ 

+     # Moksha requires that we subscribe to *something*, so tell it /dev/null

+     # since we'll just be doing in-memory queue-based messaging for this single

+     # build.

+     config["zmq_enabled"] = True

+     config["zmq_subscribe_endpoints"] = "ipc:///dev/null"

+ 

+     # Lazy import consumer to avoid potential import cycle.

+     # For example, in some cases, importing event message from events.py would

+     # cause importing the consumer module, which then starts to import relative

+     # code inside handlers module, and the original code is imported eventually.

+     import module_build_service.scheduler.consumer

+ 

+     consumers = [module_build_service.scheduler.consumer.MBSConsumer]

+ 

+     # Note that the hub we kick off here cannot send any message.  You

+     # should use fedmsg.publish(...) still for that.

+     moksha.hub.main(

+         # Pass in our config dict

+         options=config,

+         # Only run the specified consumers if any are so specified.

+         consumers=consumers,

+         # Do not run default producers.

+         producers=[],

+         # Tell moksha to quiet its logging.

+         framework=False,

+     )

+ 

+ 

  class FakeSCM(object):

      def __init__(self, mocked_scm, name, mmd_filename, commit=None, version=20180205135154):

          self.mocked_scm = mocked_scm
@@ -320,7 +398,7 @@ 

  class BaseTestBuild:

  

      def run_scheduler(self, msgs=None, stop_condition=None):

-         module_build_service.scheduler.local.main(

+         main(

              msgs or [],

              stop_condition or make_simple_stop_condition()

          )

file modified
+1 -1
@@ -201,7 +201,7 @@ 

              "--file", staged_data_filename("testmodule-local-build.yaml")

          ]

  

-         def main_side_effect(initial_messages, stop_condition):

+         def main_side_effect(module_build_ids):

              build = db_session.query(models.ModuleBuild).filter(

                  models.ModuleBuild.name == "testmodule-local-build"

              ).first()

We started using events.scheduler to plan fake events instead of internal
queue implemented by fedmsg-hub. Thanks to that, we can stop using fedmsg-hub
completely for local builds.

In this commit, the scheduler.local.main is rewritten to not use fedmsg-hub.
The original scheduler.local.main is still needed by test_build tests and
therefore it is moved there.

Build #620 failed (commit: 74131e90ec8bcc16db75a09ff2e1ce9028dbbe88).
Rebase or make new commits to rebuild.

LGTM and I built a test module locally with this patch sucessfully.
How about include https://pagure.io/fm-orchestrator/c/fc7ecb145f841c222f5f36532048bad458827356?branch=master in this patch?

state => states
ModuleBuild.transition sending => ModuleBuild.transition, which sends a

rebased onto 0d8a616

4 years ago

Pull-Request has been merged by jkaluza

4 years ago