From d6c5f8c13b517d4e40b9de431ce0973853fd96e1 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Jan 16 2020 19:39:45 +0000 Subject: Create config for frontend or backend seperately 1. init_web_config: create Config object for frontend, load configuration from `web_config.py`. 2. init_backend_config: create Config for backend, load configuration from `backend_config.py`. And two new classes inherit from `Config` in config.py: 1. WebConfig: representing the orchestrator frontend web configuration 2. BackendConfig: representing the orchestrator backend workers configuration Before calling init_{web,backend}_config, check sys.argv, if "fedmsg-hub*", "celery" or "build_module_locally" is present, it's running as backend. --- diff --git a/conf/backend_config.py b/conf/backend_config.py new file mode 100644 index 0000000..0491d43 --- /dev/null +++ b/conf/backend_config.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from os import environ, path + +confdir = path.abspath(path.dirname(__file__)) +dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir + + +class BackendConfiguration(object): + # How often should we resort to polling, in seconds + # Set to zero to disable polling + POLLING_INTERVAL = 600 + + # Configs for running tasks asynchronously with Celery + # For details of Celery configs, refer to Celery documentation: + # https://docs.celeryproject.org/en/latest/userguide/configuration.html + # + # Each config name consists of namespace CELERY_ and the new Celery config + # name converted to upper case. For example the broker url, Celery config + # name is broker_url, then as you can see below, the corresponding config + # name in MBS is CELERY_BROKER_URL. + CELERY_BROKER_URL = "" + CELERY_RESULT_BACKEND = "" + CELERY_IMPORTS = [] + + +class TestConfiguration(BackendConfiguration): + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = environ.get( + "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + +class ProdConfiguration(BackendConfiguration): + pass + + +class LocalBuildConfiguration(BackendConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True + CELERY_BROKER_URL = "redis://localhost:6379/0" + CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/config.py b/conf/config.py deleted file mode 100644 index 33c44de..0000000 --- a/conf/config.py +++ /dev/null @@ -1,171 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are -# declared properly somewhere/somehow -confdir = path.abspath(path.dirname(__file__)) -# use parent dir as dbdir else fallback to current dir -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class BaseConfiguration(object): - DEBUG = False - # Make this random (used to generate session keys) - SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - # Where we should run when running "manage.py run" directly. - HOST = "0.0.0.0" - PORT = 5000 - - # Global network-related values, in seconds - NET_TIMEOUT = 120 - NET_RETRY_INTERVAL = 30 - - SYSTEM = "koji" - MESSAGING = "fedmsg" # or amq - MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"] - KOJI_CONFIG = "/etc/module-build-service/koji.conf" - KOJI_PROFILE = "koji" - ARCHES = ["i686", "armv7hl", "x86_64"] - ALLOW_ARCH_OVERRIDE = False - KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos" - KOJI_TAG_PREFIXES = ["module", "scrmod"] - KOJI_ENABLE_CONTENT_GENERATOR = True - CHECK_FOR_EOL = False - PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" - PDC_INSECURE = False - PDC_DEVELOP = True - SCMURLS = ["https://src.fedoraproject.org/modules/"] - YAML_SUBMIT_ALLOWED = False - - # How often should we resort to polling, in seconds - # Set to zero to disable polling - POLLING_INTERVAL = 600 - - # Determines how many builds that can be submitted to the builder - # and be in the build state at a time. Set this to 0 for no restrictions - NUM_CONCURRENT_BUILDS = 5 - - ALLOW_CUSTOM_SCMURLS = False - - RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/" - RPMS_ALLOW_REPOSITORY = False - RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/" - RPMS_ALLOW_CACHE = False - - MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/" - MODULES_ALLOW_REPOSITORY = False - MODULES_ALLOW_SCRATCH = False - - ALLOWED_GROUPS = {"packager"} - - ALLOWED_GROUPS_TO_IMPORT_MODULE = set() - - # Available backends are: console and file - LOG_BACKEND = "console" - - # Path to log file when LOG_BACKEND is set to "file". - LOG_FILE = "module_build_service.log" - - # Available log levels are: debug, info, warn, error. - LOG_LEVEL = "info" - - # Settings for Kerberos - KRB_KEYTAB = None - KRB_PRINCIPAL = None - - # AMQ prefixed variables are required only while using 'amq' as messaging backend - # Addresses to listen to - AMQ_RECV_ADDRESSES = [ - "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji", - "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service", - ] - # Address for sending messages - AMQ_DEST_ADDRESS = \ - "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service" - AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt" - AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key" - AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt" - - # Disable Client Authorization - NO_AUTH = False - - # Configs for running tasks asynchronously with Celery - # For details of Celery configs, refer to Celery documentation: - # https://docs.celeryproject.org/en/latest/userguide/configuration.html - # - # Each config name consists of namespace CELERY_ and the new Celery config - # name converted to upper case. For example the broker url, Celery config - # name is broker_url, then as you can below, the corresponding config name - # in MBS is CELERY_BROKER_URL. - CELERY_BROKER_URL = "" - CELERY_RESULT_BACKEND = "" - CELERY_IMPORTS = [] - - -class TestConfiguration(BaseConfiguration): - BUILD_LOGS_DIR = "/tmp" - BUILD_LOGS_NAME_FORMAT = "build-{id}.log" - LOG_BACKEND = "console" - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get("DATABASE_URI", "sqlite:///:memory:") - DEBUG = True - MESSAGING = "in_memory" - PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - AUTH_METHOD = "oidc" - RESOLVER = "db" - - ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(BaseConfiguration): - pass - - -class LocalBuildConfiguration(BaseConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - ARCH_AUTODETECT = True - ARCH_FALLBACK = "x86_64" - - ALLOW_CUSTOM_SCMURLS = True - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True - LOG_BACKEND = "console" - - CELERY_BROKER_URL = "redis://localhost:6379/0" - CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/web_config.py b/conf/web_config.py new file mode 100644 index 0000000..5503015 --- /dev/null +++ b/conf/web_config.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from os import environ, path + +confdir = path.abspath(path.dirname(__file__)) +dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir + + +class WebConfiguration(object): + # Where we should run when running "manage.py run" directly. + HOST = "0.0.0.0" + PORT = 5000 + + +class TestConfiguration(WebConfiguration): + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = environ.get( + "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + + ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + +class ProdConfiguration(WebConfiguration): + pass + + +class LocalBuildConfiguration(WebConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + ALLOW_CUSTOM_SCMURLS = True + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 236aa8b..fd56dd9 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -18,7 +18,9 @@ for a number of tasks: infrastructure services can pick up the work. """ +import os.path import pkg_resources +import sys from celery import Celery from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy @@ -34,7 +36,7 @@ from module_build_service.logger import init_logging, ModuleBuildLogs, level_fla from module_build_service.errors import ( ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound, Forbidden, json_error) -from module_build_service.config import init_config +from module_build_service.config import init_web_config, init_backend_config from module_build_service.proxy import ReverseProxy try: @@ -46,7 +48,12 @@ api_version = 2 app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) -conf = init_config(app) +backend_commands = ("fedmsg-hub", "celery", "build_module_locally") +if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]): + # running as backend + conf = init_backend_config() +else: + conf = init_web_config(app) celery_app = Celery("module-build-service") # Convert config names specific for Celery like this: diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 1d8eeca..f7734d7 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -473,7 +473,8 @@ class KojiModuleBuilder(GenericBuilder): def get_session(config, login=True): """Create and return a koji.ClientSession object - :param config: the config object returned from :meth:`init_config`. + :param config: the config object returned from :meth:`init_web_config` or + :meth:`init_backend_config`. :type config: :class:`Config` :param bool login: whether to log into the session. To login if True is passed, otherwise not to log into session. diff --git a/module_build_service/config.py b/module_build_service/config.py index 6bffc75..803b053 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -24,11 +24,11 @@ SUPPORTED_RESOLVERS = { } -def init_config(app): +def init_web_config(app): """ Configure MBS and the Flask app """ config_module = None - config_file = "/etc/module-build-service/config.py" + config_file = "/etc/module-build-service/web_config.py" config_section = "DevConfiguration" # automagically detect production environment: @@ -70,9 +70,9 @@ def init_config(app): # TestConfiguration shall only be used for running tests, otherwise... if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): config_section = "TestConfiguration" - from conf import config + from conf import web_config - config_module = config + config_module = web_config # ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word # and overrides anything previously set before! # Again, check Flask app (preferably) or fallback to os.environ. @@ -81,14 +81,14 @@ def init_config(app): elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ: if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options: config_section = "DevConfiguration" - from conf import config + from conf import web_config - config_module = config + config_module = web_config elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: config_section = "DevConfiguration" - from conf import config + from conf import web_config - config_module = config + config_module = web_config # try loading configuration from file if not config_module: try: @@ -98,11 +98,63 @@ def init_config(app): # finally configure MBS and the Flask app config_section_obj = getattr(config_module, config_section) - conf = Config(config_section_obj) + conf = WebConfig(config_section_obj) app.config.from_object(config_section_obj) return conf +def init_backend_config(): + """ Configure MBS and backend workers + """ + config_module = None + config_file = "/etc/module-build-service/backend_config.py" + config_section = "DevConfiguration" + + try: + with open(config_file): + config_section = "ProdConfiguration" + except Exception: + pass + + # Load LocalBuildConfiguration section in case we are building modules + # locally. + if "build_module_locally" in sys.argv: + if "--offline" in sys.argv: + config_section = "OfflineLocalBuildConfiguration" + else: + config_section = "LocalBuildConfiguration" + + # try getting config_file from os.environ + if "MBS_CONFIG_FILE" in os.environ: + config_file = os.environ["MBS_CONFIG_FILE"] + # try getting config_section from os.environ + if "MBS_CONFIG_SECTION" in os.environ: + config_section = os.environ["MBS_CONFIG_SECTION"] + + true_options = ("1", "on", "true", "y", "yes") + # TestConfiguration shall only be used for running tests, otherwise... + if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): + config_section = "TestConfiguration" + from conf import backend_config + + config_module = backend_config + elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: + config_section = "DevConfiguration" + from conf import backend_config + + config_module = backend_config + # try loading configuration from file + if not config_module: + try: + config_module = imp.load_source("mbs_runtime_config", config_file) + except Exception: + raise SystemError("Configuration file {} was not found.".format(config_file)) + + config_section_obj = getattr(config_module, config_section) + conf = BackendConfig(config_section_obj) + return conf + + class Path: """ Config type for paths. Expands the users home directory. @@ -112,7 +164,7 @@ class Path: class Config(object): - """Class representing the orchestrator configuration.""" + """Class representing the orchestrator common configuration.""" _defaults = { "debug": {"type": bool, "default": False, "desc": "Debug mode"}, @@ -249,7 +301,7 @@ class Config(object): "log_level": {"type": str, "default": 0, "desc": "Log level"}, "build_logs_dir": { "type": Path, - "default": "", + "default": tempfile.gettempdir(), "desc": "Directory to store module build logs to.", }, "build_logs_name_format": { @@ -761,7 +813,7 @@ class Config(object): def _setifok_log_backend(self, s): if s is None: - self._log_backend = "console" + s = "console" elif s not in logger.supported_log_backends(): raise ValueError("Unsupported log backend") self._log_backend = str(s) @@ -918,3 +970,23 @@ class Config(object): if i < 1: raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") self._num_threads_for_build_submissions = i + + +class WebConfig(Config): + """Class representing the orchestrator frontend web configuration.""" + _web_defaults = { + } + + def __init__(self, conf_section_obj): + self._defaults.update(self._web_defaults) + super(WebConfig, self).__init__(conf_section_obj) + + +class BackendConfig(Config): + """Class representing the orchestrator backend workers configuration.""" + _backend_defaults = { + } + + def __init__(self, conf_section_obj): + self._defaults.update(self._backend_defaults) + super(BackendConfig, self).__init__(conf_section_obj) diff --git a/module_build_service/models.py b/module_build_service/models.py index 7ec35b6..2e39b92 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -663,8 +663,8 @@ class ModuleBuild(MBSBase): bus. :param db_session: SQLAlchemy session object. - :param conf: MBS config object returned from function :func:`init_config` - which contains loaded configs. + :param conf: MBS config object returned from function :func:`init_web_config` or + :func:`init_backend_config`, which contains loaded configs. :type conf: :class:`Config` :param int state: the state value to transition to. Refer to ``BUILD_STATES``. :param str state_reason: optional reason of why to transform to ``state``. diff --git a/setup.py b/setup.py index ab0dea4..db13d2e 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,8 @@ setup( "/etc/module-build-service/", [ "conf/cacert.pem", - "conf/config.py", + "conf/web_config.py", + "conf/backend_config.py", "conf/koji.conf", "conf/mock.cfg", "conf/yum.conf", diff --git a/tests/__init__.py b/tests/__init__.py index ff9235c..25fa846 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,7 +14,7 @@ import koji import module_build_service from module_build_service import db from module_build_service.utils import get_rpm_release, import_mmd, mmd_to_str -from module_build_service.config import init_config +from module_build_service.config import init_web_config from module_build_service.models import ( ModuleBuild, ModuleArch, ComponentBuild, VirtualStream, BUILD_STATES, @@ -25,7 +25,7 @@ from module_build_service.db_session import db_session base_dir = os.path.dirname(__file__) app = module_build_service.app -conf = init_config(app) +conf = init_web_config(app) def staged_data_filename(filename): diff --git a/tests/test_monitor.py b/tests/test_monitor.py index ff9e322..f45aa55 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -8,7 +8,7 @@ import module_build_service.config as mbs_config import module_build_service.monitor from module_build_service import models from module_build_service.db_session import db_session -from conf.config import TestConfiguration +from conf.web_config import TestConfiguration from six.moves import reload_module from tests import app, init_data, make_module_in_db