From bc68b3b08c4092de9072e4e33313f28d5bfeaf0b Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Jan 16 2020 19:39:45 +0000 Subject: Revert the config split change Revert the changes in a207d97 and 9a2efb3, at this moment, some code relies on both settings from frontend and backend, and it's inconvenient to run unittests with config split up to frontend only and backend only. --- diff --git a/conf/backend_config.py b/conf/backend_config.py deleted file mode 100644 index 43855d6..0000000 --- a/conf/backend_config.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class BackendConfiguration(object): - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # How often should we resort to polling, in seconds - # Set to zero to disable polling - POLLING_INTERVAL = 600 - - # Configs for running tasks asynchronously with Celery - # For details of Celery configs, refer to Celery documentation: - # https://docs.celeryproject.org/en/latest/userguide/configuration.html - # - # Each config name consists of namespace CELERY_ and the new Celery config - # name converted to upper case. For example the broker url, Celery config - # name is broker_url, then as you can see below, the corresponding config - # name in MBS is CELERY_BROKER_URL. - CELERY_BROKER_URL = "" - CELERY_RESULT_BACKEND = "" - CELERY_IMPORTS = [ - "module_build_service.scheduler.handlers.components", - "module_build_service.scheduler.handlers.modules", - "module_build_service.scheduler.handlers.repos", - "module_build_service.scheduler.handlers.tags", - "module_build_service.scheduler.handlers.greenwave", - ] - - -class TestConfiguration(BackendConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(BackendConfiguration): - pass - - -class LocalBuildConfiguration(BackendConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True - CELERY_BROKER_URL = "redis://localhost:6379/0" - CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/config.py b/conf/config.py new file mode 100644 index 0000000..270d994 --- /dev/null +++ b/conf/config.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from os import environ, path + +# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are +# declared properly somewhere/somehow +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir + + +class BaseConfiguration(object): + DEBUG = False + # Make this random (used to generate session keys) + SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" + SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) + SQLALCHEMY_TRACK_MODIFICATIONS = True + # Where we should run when running "manage.py run" directly. + HOST = "0.0.0.0" + PORT = 5000 + + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = "koji" + MESSAGING = "fedmsg" # or amq + MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"] + KOJI_CONFIG = "/etc/module-build-service/koji.conf" + KOJI_PROFILE = "koji" + ARCHES = ["i686", "armv7hl", "x86_64"] + ALLOW_ARCH_OVERRIDE = False + KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos" + KOJI_TAG_PREFIXES = ["module", "scrmod"] + KOJI_ENABLE_CONTENT_GENERATOR = True + CHECK_FOR_EOL = False + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + PDC_INSECURE = False + PDC_DEVELOP = True + SCMURLS = ["https://src.fedoraproject.org/modules/"] + YAML_SUBMIT_ALLOWED = False + + # How often should we resort to polling, in seconds + # Set to zero to disable polling + POLLING_INTERVAL = 600 + + # Determines how many builds that can be submitted to the builder + # and be in the build state at a time. Set this to 0 for no restrictions + NUM_CONCURRENT_BUILDS = 5 + + ALLOW_CUSTOM_SCMURLS = False + + RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/" + RPMS_ALLOW_REPOSITORY = False + RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/" + RPMS_ALLOW_CACHE = False + + MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/" + MODULES_ALLOW_REPOSITORY = False + MODULES_ALLOW_SCRATCH = False + + ALLOWED_GROUPS = {"packager"} + + ALLOWED_GROUPS_TO_IMPORT_MODULE = set() + + # Available backends are: console and file + LOG_BACKEND = "console" + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = "module_build_service.log" + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = "info" + + # Settings for Kerberos + KRB_KEYTAB = None + KRB_PRINCIPAL = None + + # AMQ prefixed variables are required only while using 'amq' as messaging backend + # Addresses to listen to + AMQ_RECV_ADDRESSES = [ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji", + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service", + ] + # Address for sending messages + AMQ_DEST_ADDRESS = \ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service" + AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt" + AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key" + AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt" + + # Disable Client Authorization + NO_AUTH = False + + # Configs for running tasks asynchronously with Celery + # For details of Celery configs, refer to Celery documentation: + # https://docs.celeryproject.org/en/latest/userguide/configuration.html + # + # Each config name consists of namespace CELERY_ and the new Celery config + # name converted to upper case. For example the broker url, Celery config + # name is broker_url, then as you can below, the corresponding config name + # in MBS is CELERY_BROKER_URL. + CELERY_BROKER_URL = "" + CELERY_RESULT_BACKEND = "" + CELERY_IMPORTS = [ + "module_build_service.scheduler.handlers.components", + "module_build_service.scheduler.handlers.modules", + "module_build_service.scheduler.handlers.repos", + "module_build_service.scheduler.handlers.tags", + "module_build_service.scheduler.handlers.greenwave", + ] + + +class TestConfiguration(BaseConfiguration): + BUILD_LOGS_DIR = "/tmp" + BUILD_LOGS_NAME_FORMAT = "build-{id}.log" + LOG_BACKEND = "console" + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = environ.get( + "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + AUTH_METHOD = "oidc" + RESOLVER = "db" + + ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + +class ProdConfiguration(BaseConfiguration): + pass + + +class LocalBuildConfiguration(BaseConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + ARCH_AUTODETECT = True + ARCH_FALLBACK = "x86_64" + + ALLOW_CUSTOM_SCMURLS = True + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True + LOG_BACKEND = "console" + + CELERY_BROKER_URL = "redis://localhost:6379/0" + CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/web_config.py b/conf/web_config.py deleted file mode 100644 index 465d5a8..0000000 --- a/conf/web_config.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class WebConfiguration(object): - SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # Where we should run when running "manage.py run" directly. - HOST = "0.0.0.0" - PORT = 5000 - - -class TestConfiguration(WebConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(WebConfiguration): - pass - - -class LocalBuildConfiguration(WebConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - ALLOW_CUSTOM_SCMURLS = True - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index b2eae40..236aa8b 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -18,9 +18,7 @@ for a number of tasks: infrastructure services can pick up the work. """ -import os.path import pkg_resources -import sys from celery import Celery from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy @@ -48,12 +46,7 @@ api_version = 2 app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) -backend_commands = ("fedmsg-hub", "celery", "build_module_locally") -if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]): - # running as backend - conf = init_config(app, backend=True) -else: - conf = init_config(app) +conf = init_config(app) celery_app = Celery("module-build-service") # Convert config names specific for Celery like this: diff --git a/module_build_service/config.py b/module_build_service/config.py index d50a1e1..e49a8c3 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -24,19 +24,11 @@ SUPPORTED_RESOLVERS = { } -def init_config(app, backend=False): +def init_config(app): """ Configure MBS and the Flask app - By default, this create a config object with WebConfig, if backend is specified, - create a config object with BackendConfig. """ config_module = None - if backend: - config_module_name = "backend_config" - config_file = "/etc/module-build-service/backend_config.py" - else: - config_module_name = "web_config" - config_file = "/etc/module-build-service/web_config.py" - + config_file = "/etc/module-build-service/config.py" config_section = "DevConfiguration" # automagically detect production environment: @@ -74,14 +66,13 @@ def init_config(app, backend=False): if "MBS_CONFIG_SECTION" in app.request.environ: config_section = app.request.environ["MBS_CONFIG_SECTION"] - test_env = False - dev_env = False - true_options = ("1", "on", "true", "y", "yes") # TestConfiguration shall only be used for running tests, otherwise... if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): - test_env = True config_section = "TestConfiguration" + from conf import config + + config_module = config # ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word # and overrides anything previously set before! # Again, check Flask app (preferably) or fallback to os.environ. @@ -89,16 +80,15 @@ def init_config(app, backend=False): # -> /conf/config.py. elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ: if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config + + config_module = config elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config - if test_env or dev_env: - import importlib - config_module = importlib.import_module("conf.%s" % config_module_name) - + config_module = config # try loading configuration from file if not config_module: try: @@ -108,10 +98,7 @@ def init_config(app, backend=False): # finally configure MBS and the Flask app config_section_obj = getattr(config_module, config_section) - if backend: - conf = BackendConfig(config_section_obj) - else: - conf = WebConfig(config_section_obj) + conf = Config(config_section_obj) app.config.from_object(config_section_obj) return conf @@ -125,7 +112,7 @@ class Path: class Config(object): - """Class representing the orchestrator common configuration.""" + """Class representing the orchestrator configuration.""" _defaults = { "debug": {"type": bool, "default": False, "desc": "Debug mode"}, @@ -931,23 +918,3 @@ class Config(object): if i < 1: raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") self._num_threads_for_build_submissions = i - - -class WebConfig(Config): - """Class representing the orchestrator frontend web configuration.""" - _web_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._web_defaults) - super(WebConfig, self).__init__(conf_section_obj) - - -class BackendConfig(Config): - """Class representing the orchestrator backend workers configuration.""" - _backend_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._backend_defaults) - super(BackendConfig, self).__init__(conf_section_obj) diff --git a/module_build_service/models.py b/module_build_service/models.py index 0c602ac..7ec35b6 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -663,7 +663,7 @@ class ModuleBuild(MBSBase): bus. :param db_session: SQLAlchemy session object. - :param conf: MBS config object returned from function :func:`init_config`, + :param conf: MBS config object returned from function :func:`init_config` which contains loaded configs. :type conf: :class:`Config` :param int state: the state value to transition to. Refer to ``BUILD_STATES``. diff --git a/setup.py b/setup.py index f761015..a92cd8e 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,7 @@ setup( "/etc/module-build-service/", [ "conf/cacert.pem", - "conf/web_config.py", - "conf/backend_config.py", + "conf/config.py", "conf/koji.conf", "conf/mock.cfg", "conf/yum.conf", diff --git a/tests/test_monitor.py b/tests/test_monitor.py index f45aa55..ff9e322 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -8,7 +8,7 @@ import module_build_service.config as mbs_config import module_build_service.monitor from module_build_service import models from module_build_service.db_session import db_session -from conf.web_config import TestConfiguration +from conf.config import TestConfiguration from six.moves import reload_module from tests import app, init_data, make_module_in_db