From 98b54d211e3bd8629e8c459590dd476db1514add Mon Sep 17 00:00:00 2001 From: mprahl Date: Mar 03 2020 19:48:47 +0000 Subject: Move config.py to common/config.py --- diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 1e9e4d8..2c02621 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -185,8 +185,9 @@ Logging ------- If you're running module_build_service from scm, then the DevConfiguration -from ``conf/config.py`` which contains ``LOG_LEVEL=debug`` should get applied. See -more about it in ``module_build_service/config.py``, ``app.config.from_object()``. +from ``module_build_service/common/config.py`` which contains ``LOG_LEVEL=debug`` should get +applied. See more about it in ``module_build_service/common/config.py``, +``app.config.from_object()``. Environment ----------- diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 2c7dbb2..c7af037 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -34,7 +34,7 @@ from module_build_service.logger import init_logging, ModuleBuildLogs, level_fla from module_build_service.errors import ( ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound, Forbidden, json_error) -from module_build_service.config import init_config +from module_build_service.common.config import init_config from module_build_service.proxy import ReverseProxy try: diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index b82d9d2..56e4883 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -55,7 +55,7 @@ class KojiContentGenerator(object): def __init__(self, module, config): """ :param module: module_build_service.models.ModuleBuild instance. - :param config: module_build_service.config.Config instance + :param config: module_build_service.common.config.Config instance """ self.owner = module.owner self.module = module diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 974603a..ecce661 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -56,7 +56,7 @@ class KojiModuleBuilder(GenericBuilder): :param db_session: SQLAlchemy session object. :param owner: a string representing who kicked off the builds :param module: module_build_service.models.ModuleBuild instance. - :param config: module_build_service.config.Config instance + :param config: module_build_service.common.config.Config instance :param tag_name: name of tag for given module """ self.db_session = db_session @@ -757,7 +757,7 @@ class KojiModuleBuilder(GenericBuilder): @classmethod def repo_from_tag(cls, config, tag_name, arch): """ - :param config: instance of module_build_service.config.Config + :param config: instance of module_build_service.common.config.Config :param tag_name: Tag for which the repository is returned :param arch: Architecture for which the repository is returned diff --git a/module_build_service/builder/base.py b/module_build_service/builder/base.py index a485007..b39c931 100644 --- a/module_build_service/builder/base.py +++ b/module_build_service/builder/base.py @@ -65,7 +65,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): External Api for builders Example usage: - config = module_build_service.config.Config() + config = module_build_service.common.config.Config() builder = Builder(module="testmodule-1.2-3", backend="koji", config) builder.buildroot_connect() builder.build(artifact_name="bash", @@ -110,7 +110,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): :param owner: a string representing who kicked off the builds :param module: module_build_service.models.ModuleBuild instance. :param backend: a string representing backend e.g. 'koji' - :param config: instance of module_build_service.config.Config + :param config: instance of module_build_service.common.config.Config Any additional arguments are optional extras which can be passed along and are implementation-dependent. @@ -137,7 +137,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): :param db_session: SQLAlchemy database session. :param module: module_build_service.models.ModuleBuild instance. - :param config: module_build_service.config.Config instance. + :param config: module_build_service.common.config.Config instance. :kwarg buildroot_connect: a boolean that determines if the builder should run buildroot_connect on instantiation. """ @@ -160,7 +160,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): def tag_to_repo(cls, backend, config, tag_name, arch): """ :param backend: a string representing the backend e.g. 'koji'. - :param config: instance of module_build_service.config.Config + :param config: instance of module_build_service.common.config.Config :param tag_name: Tag for which the repository is returned :param arch: Architecture for which the repository is returned @@ -291,7 +291,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): @abstractmethod def repo_from_tag(self, config, tag_name, arch): """ - :param config: instance of module_build_service.config.Config + :param config: instance of module_build_service.common.config.Config :param tag_name: Tag for which the repository is returned :param arch: Architecture for which the repository is returned diff --git a/module_build_service/common/config.py b/module_build_service/common/config.py new file mode 100644 index 0000000..0e2c14e --- /dev/null +++ b/module_build_service/common/config.py @@ -0,0 +1,991 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import imp +import logging +import os +import pkg_resources +import re +import sys +import tempfile + +from six import string_types + +from module_build_service import logger +# This avoids creating a circular import by importing log from module_build_service +log = logging.getLogger('module_build_service') + + +# TODO: It'd be nice to reuse this from models.ModuleBuild.rebuild_strategies but models.py +# currently relies on this file, so we can't import it +SUPPORTED_STRATEGIES = ["changed-and-after", "only-changed", "all"] + +SUPPORTED_RESOLVERS = { + "mbs": {"builders": ["mock"]}, + "db": {"builders": ["koji", "mock", "copr"]}, + "local": {"builders": ["mock"]}, + "koji": {"builders": ["koji"]}, +} + + +class BaseConfiguration(object): + DEBUG = False + SECRET_KEY = os.urandom(16) + SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(os.path.join( + os.getcwd(), "module_build_service.db")) + SQLALCHEMY_TRACK_MODIFICATIONS = True + # Where we should run when running "manage.py run" directly. + HOST = "0.0.0.0" + PORT = 5000 + + +class TestConfiguration(BaseConfiguration): + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = os.environ.get( + "DATABASE_URI", "sqlite:///{0}".format(os.path.join(os.getcwd(), "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + + ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + # Ensures task.delay executes locally instead of scheduling a task to a queue. + CELERY_TASK_ALWAYS_EAGER = True + + +class ProdConfiguration(BaseConfiguration): + pass + + +class LocalBuildConfiguration(BaseConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + ALLOW_CUSTOM_SCMURLS = True + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + # Celery tasks will be executed locally for local builds + CELERY_TASK_ALWAYS_EAGER = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True + + +def init_config(): + """ + Create the global MBS configuration. + + :return: a tuple with the first index being the configuration and second index as the + configuration class that was used to initialize the configuration. This can be useful + to configure Flask with. + :rtype: tuple(Config, object) + """ + config_file = os.environ.get("MBS_CONFIG_FILE", "/etc/module-build-service/config.py") + config_section = os.environ.get("MBS_CONFIG_SECTION", "ProdConfiguration") + try: + config_module = imp.load_source("mbs_runtime_config", config_file) + log.info("Using the configuration file at %s", config_file) + except Exception: + log.warning("The configuration file at %s was not present", config_file) + # Default to this file as the configuration module + config_module = sys.modules[__name__] + + true_options = ("1", "on", "true", "y", "yes") + if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): + config_section = "TestConfiguration" + # Get the configuration from this module + config_module = sys.modules[__name__] + elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: + config_section = "DevConfiguration" + # Get the configuration from this module + config_module = sys.modules[__name__] + elif "build_module_locally" in sys.argv: + if "--offline" in sys.argv: + config_section = "OfflineLocalBuildConfiguration" + else: + config_section = "LocalBuildConfiguration" + + if hasattr(config_module, config_section): + log.info("Using the configuration section %s", config_section) + config_section_obj = getattr(config_module, config_section) + else: + log.error( + "The configuration class of %s is not present. Falling back to the default " + "ProdConfiguration class.", + config_section, + ) + config_section_obj = ProdConfiguration + conf = Config(config_section_obj) + return conf, config_section_obj + + +class Path: + """ + Config type for paths. Expands the users home directory. + """ + + pass + + +class Config(object): + """Class representing the orchestrator configuration.""" + + _defaults = { + "debug": {"type": bool, "default": False, "desc": "Debug mode"}, + "system": {"type": str, "default": "koji", "desc": "The buildsystem to use."}, + "db": {"type": str, "default": "", "desc": "RDB URL."}, + "default_dist_tag_prefix": { + "type": str, + "default": "module+", + "desc": "Default dist-tag prefix for built modules.", + }, + "polling_interval": {"type": int, "default": 600, "desc": "Polling interval, in seconds."}, + "cache_dir": { + "type": Path, + "default": os.path.join(tempfile.gettempdir(), "mbs"), + "desc": "Cache directory" + }, + "mbs_url": { + "type": str, + "default": "https://mbs.fedoraproject.org/module-build-service/1/module-builds/", + "desc": "MBS instance url for MBSResolver", + }, + "check_for_eol": { + "type": bool, + "default": False, + "desc": "Flag to determine whether or not MBS should block EOL modules from building.", + }, + "pdc_url": { + "type": str, + "default": "https://pdc.fedoraproject.org/rest_api/v1", + "desc": "PDC URL, used for checking stream EOL.", + }, + "koji_config": { + "type": str, + "default": "/etc/module-build-service/koji.conf", + "desc": "Koji config file." + }, + "koji_profile": {"type": str, "default": "koji", "desc": "Koji config profile."}, + "arches": {"type": list, "default": ["x86_64"], "desc": "Koji architectures."}, + "allow_arch_override": { + "type": bool, + "default": False, + "desc": "Allow to support a custom architecture set", + }, + "koji_build_priority": {"type": int, "default": 10, "desc": ""}, + "koji_repository_url": { + "type": str, + "default": "https://kojipkgs.fedoraproject.org/repos", + "desc": "Koji repository URL." + }, + "koji_build_macros_target": { + "type": str, + "default": "", + "desc": 'Target to build "module-build-macros" RPM in.', + }, + "koji_tag_prefixes": { + "type": list, + "default": ["module", "scrmod"], + "desc": "List of allowed koji tag prefixes.", + }, + "koji_tag_permission": { + "type": str, + "default": "admin", + "desc": "Permission name to require for newly created Koji tags.", + }, + "koji_tag_extra_opts": { + "type": dict, + "default": { + "mock.package_manager": "dnf", + # This is needed to include all the Koji builds (and therefore + # all the packages) from all inherited tags into this tag. + # See https://pagure.io/koji/issue/588 and + # https://pagure.io/fm-orchestrator/issue/660 for background. + "repo_include_all": True, + # Has been requested by Fedora infra in + # https://pagure.io/fedora-infrastructure/issue/7620. + # Disables systemd-nspawn for chroot. + "mock.new_chroot": 0, + # Works around fail-safe mechanism added in DNF 4.2.7 + # https://pagure.io/fedora-infrastructure/issue/8410 + "mock.yum.module_hotfixes": 1, + }, + "desc": "Extra options set for newly created Koji tags.", + }, + "koji_target_delete_time": { + "type": int, + "default": 24 * 3600, + "desc": "Time in seconds after which the Koji target of built module is deleted", + }, + "koji_enable_content_generator": { + "type": bool, + "default": True, + "desc": "Enable or disable imports to koji using content generator api", + }, + "allow_name_override_from_scm": { + "type": bool, + "default": False, + "desc": "Allow modulemd files to override the module name " + "if different from the scm repo name.", + }, + "allow_stream_override_from_scm": { + "type": bool, + "default": False, + "desc": "Allow modulemd files to override the module stream " + "if different from the scm repo branch.", + }, + "allow_custom_scmurls": {"type": bool, "default": False, "desc": "Allow custom scmurls."}, + "rpms_default_repository": { + "type": str, + "default": "https://src.fedoraproject.org/rpms/", + "desc": "RPMs default repository URL.", + }, + "rpms_allow_repository": { + "type": bool, + "default": False, + "desc": "Allow custom RPMs repositories.", + }, + "rpms_default_cache": { + "type": str, + "default": "http://pkgs.fedoraproject.org/repo/pkgs/", + "desc": "RPMs default cache URL.", + }, + "rpms_allow_cache": {"type": bool, "default": False, "desc": "Allow custom RPMs cache."}, + "modules_default_repository": { + "type": str, + "default": "https://src.fedoraproject.org/modules/", + "desc": "Included modules default repository URL.", + }, + "modules_allow_repository": { + "type": bool, + "default": False, + "desc": "Allow custom included modules repositories.", + }, + "allowed_groups": { + "type": set, + "default": {"packager"}, + "desc": "The set of groups allowed to submit builds.", + }, + "allowed_groups_to_import_module": { + "type": set, + "default": set(), + "desc": "The set of groups allowed to import module builds.", + }, + "log_backend": {"type": str, "default": None, "desc": "Log backend"}, + "log_file": {"type": str, "default": "", "desc": "Path to log file"}, + "log_level": {"type": str, "default": "info", "desc": "Log level"}, + "build_logs_dir": { + "type": Path, + "default": tempfile.gettempdir(), + "desc": "Directory to store module build logs to.", + }, + "build_logs_name_format": { + "type": str, + "default": "build-{id}.log", + "desc": ( + "Format of a module build log's name. Use `Build` attributes as formatting " + "kwargs" + ), + }, + "krb_keytab": {"type": None, "default": None, "desc": ""}, + "krb_principal": {"type": None, "default": None, "desc": ""}, + "messaging": {"type": str, "default": "fedmsg", "desc": "The messaging system to use."}, + "messaging_topic_prefix": { + "type": list, + "default": ["org.fedoraproject.prod"], + "desc": "The messaging system topic prefixes which we are interested in.", + }, + "distgits": { + "type": dict, + "default": { + "https://src.fedoraproject.org": ( + "fedpkg clone --anonymous {}", + "fedpkg --release module sources", + ), + "file://": ("git clone {repo_path}", None), + }, + "desc": "Mapping between dist-git and command to ", + }, + "mock_config": {"type": str, "default": "fedora-25-x86_64", "desc": ""}, + "mock_config_file": { + "type": list, + "default": ["/etc/module-build-service/mock.cfg", "conf/mock.cfg"], + "desc": "List of mock config file paths in order of preference.", + }, + "mock_build_srpm_cmd": {"type": str, "default": "fedpkg --release f26 srpm", "desc": ""}, + "mock_resultsdir": { + "type": Path, + "default": "~/modulebuild/builds", + "desc": "Directory for Mock build results.", + }, + "mock_purge_useless_logs": { + "type": bool, + "default": True, + "desc": "Remove empty or otherwise useless log files.", + }, + "arch_autodetect": { + "type": bool, + "default": True, + "desc": "Auto-detect machine arch when configuring builder.", + }, + "arch_fallback": { + "type": str, + "default": "x86_64", + "desc": "Fallback arch if auto-detection is off or unable to determine it.", + }, + "scmurls": { + "type": list, + "default": ["https://src.fedoraproject.org/modules/"], + "desc": "Allowed SCM URLs for submitted module.", + }, + "yaml_submit_allowed": { + "type": bool, + "default": False, + "desc": "Is it allowed to directly submit build by modulemd yaml file?", + }, + "num_concurrent_builds": { + "type": int, + "default": 5, + "desc": "Number of concurrent component builds.", + }, + "net_timeout": { + "type": int, + "default": 120, + "desc": "Global network timeout for read/write operations, in seconds.", + }, + "net_retry_interval": { + "type": int, + "default": 30, + "desc": "Global network retry interval for read/write operations, in seconds.", + }, + "scm_net_timeout": { + "type": int, + "default": 60, + "desc": "Network timeout for SCM operations, in seconds.", + }, + "scm_net_retry_interval": { + "type": int, + "default": 15, + "desc": "Network retry interval for SCM operations, in seconds.", + }, + "no_auth": {"type": bool, "default": False, "desc": "Disable client authentication."}, + "admin_groups": { + "type": set, + "default": set(), + "desc": "The set of groups allowed to manage MBS.", + }, + "yum_config_file": { + "type": list, + "default": ["/etc/module-build-service/yum.conf", "conf/yum.conf"], + "desc": "List of yum config file paths in order of preference.", + }, + "auth_method": { + "type": str, + "default": "oidc", + "desc": "Authentiation method to MBS. Options are oidc or kerberos", + }, + "ldap_uri": { + "type": str, + "default": "", + "desc": "LDAP URI to query for group information when using Kerberos authentication", + }, + "ldap_groups_dn": { + "type": str, + "default": "", + "desc": ( + "The distinguished name of the container or organizational unit containing " + "the groups in LDAP" + ), + }, + "base_module_names": { + "type": list, + "default": ["platform"], + "desc": ( + "List of base module names which define the product version " + "(by their stream) of modules depending on them." + ), + }, + "base_module_arches": { + "type": dict, + "default": {}, + "desc": "Per base-module name:stream Koji arches list.", + }, + "allow_only_compatible_base_modules": { + "type": bool, + "default": True, + "desc": "When True, only modules built on top of compatible base modules are " + "considered by MBS as possible buildrequirement. When False, modules " + "built against any base module stream can be used as a buildrequire.", + }, + "koji_cg_tag_build": { + "type": bool, + "default": True, + "desc": "Indicate whether tagging build is enabled during importing module to Koji.", + }, + "koji_cg_devel_module": { + "type": bool, + "default": True, + "desc": "Indicate whether a devel module should be imported into Koji.", + }, + "koji_cg_build_tag_template": { + "type": str, + "default": "{}-modular-updates-candidate", + "desc": "Name of a Koji tag where the top-level Content Generator " + "build is tagged to. The '{}' string is replaced by a " + "stream name of a base module on top of which the " + "module is built.", + }, + "koji_cg_default_build_tag": { + "type": str, + "default": "modular-updates-candidate", + "desc": "The name of Koji tag which should be used as fallback " + "when koji_cg_build_tag_template tag is not found in " + "Koji.", + }, + "rebuild_strategy": { + "type": str, + "default": "changed-and-after", + "desc": "The module rebuild strategy to use by default.", + }, + "rebuild_strategy_allow_override": { + "type": bool, + "default": False, + "desc": ( + "Allows a user to specify the rebuild strategy they want to use when " + "submitting a module build." + ), + }, + "rebuild_strategies_allowed": { + "type": list, + "default": SUPPORTED_STRATEGIES, + "desc": ( + "The allowed module rebuild strategies. This is only used when " + "REBUILD_STRATEGY_ALLOW_OVERRIDE is True." + ), + }, + "cleanup_failed_builds_time": { + "type": int, + "default": 180, + "desc": ( + "Time in days when to cleanup failed module builds and transition them to " + 'the "garbage" state.' + ), + }, + "cleanup_stuck_builds_time": { + "type": int, + "default": 7, + "desc": ( + "Time in days when to cleanup stuck module builds and transition them to " + 'the "failed" state. The module has to be in a state defined by the ' + '"cleanup_stuck_builds_states" option.' + ), + }, + "cleanup_stuck_builds_states": { + "type": list, + "default": ["init", "build"], + "desc": ( + "States of builds which will be considered to move to failed state when a" + " build is in one of those states longer than the value configured in the " + '"cleanup_stuck_builds_time"' + ), + }, + "resolver": { + "type": str, + "default": "db", + "desc": "Where to look up for modules. Note that this can (and " + "probably will) be builder-specific.", + }, + "koji_external_repo_url_prefix": { + "type": str, + "default": "https://kojipkgs.fedoraproject.org/", + "desc": "URL prefix of base module's external repo.", + }, + "allowed_users": { + "type": set, + "default": set(), + "desc": "The users/service accounts that don't require to be part of a group", + }, + "br_stream_override_module": { + "type": str, + "default": "platform", + "desc": ( + "The module name to override in the buildrequires based on the branch name. " + '"br_stream_override_regexes" must also be set for this to take ' + "effect." + ), + }, + "br_stream_override_regexes": { + "type": list, + "default": [], + "desc": ( + "The list of regexes used to parse the stream override from the branch name. " + '"br_stream_override_module" must also be set for this to take ' + "effect. The regexes can contain multiple capture groups that will be " + "concatenated. Any null capture groups will be ignored. The first regex that " + "matches the branch will be used." + ), + }, + "default_buildroot_packages": { + "type": list, + "default": [ + "bash", + "bzip2", + "coreutils", + "cpio", + "diffutils", + "findutils", + "gawk", + "gcc", + "gcc-c++", + "grep", + "gzip", + "info", + "make", + "patch", + "fedora-release", + "redhat-rpm-config", + "rpm-build", + "sed", + "shadow-utils", + "tar", + "unzip", + "util-linux", + "which", + "xz", + ], + "desc": ("The list packages for offline module build RPM buildroot."), + }, + "default_srpm_buildroot_packages": { + "type": list, + "default": [ + "bash", + "gnupg2", + "fedora-release", + "redhat-rpm-config", + "fedpkg-minimal", + "rpm-build", + "shadow-utils", + ], + "desc": ("The list packages for offline module build RPM buildroot."), + }, + "greenwave_decision_context": { + "type": str, + "default": "", + "desc": "The Greenwave decision context that determines a module's gating status.", + }, + "allowed_privileged_module_names": { + "type": list, + "default": [], + "desc": ( + "List of modules that are allowed to influence the RPM buildroot when " + "buildrequired. These modules can set xmd.mbs.disttag_marking to do change " + "the RPM disttag, or set the xmd.mbs.koji_tag_arches to set the arches " + "for which the modules are built. MBS will use this list order to determine " + "which modules take precedence." + ), + }, + "stream_suffixes": { + "type": dict, + "default": {}, + "desc": "A mapping of platform stream regular expressions and the " + "corresponding suffix added to formatted stream version. " + 'For example, {r"regexp": 0.1, ...}', + }, + "greenwave_url": { + "type": str, + "default": "", + "desc": "The URL of the server where Greenwave is running (should include " + "the root of the API)" + }, + "greenwave_subject_type": { + "type": str, + "default": "", + "desc": "Subject type for Greenwave requests" + }, + "greenwave_timeout": { + "type": int, + "default": 60, + "desc": "Greenwave response timeout" + }, + "modules_allow_scratch": { + "type": bool, + "default": False, + "desc": "Allow module scratch builds", + }, + "scratch_build_only_branches": { + "type": list, + "default": [], + "desc": "The list of regexes used to identify branches from which only the module " + "scratch builds can be built", + }, + "product_pages_url": { + "type": str, + "default": "", + "desc": "The URL to the Product Pages. This is queried to determine if a base module " + "stream has been released. If it has, the stream may be modified automatically " + "to use a different support stream.", + }, + "product_pages_module_streams": { + "type": dict, + "default": {}, + "desc": "The keys are regexes of base module streams that should be checked in the Red " + "Hat Product Pages. The values are tuples. The first value is a string that " + "should be appended to the stream if there is a match and the release the " + "stream represents has been released. The second value is a template string " + "that represents the release in Product Pages and can accept format kwargs of " + "x, y, and z (represents the version). The third value is an optional template " + "string that represent the Product Pages release for major releases " + "(e.g. 8.0.0). After the first match, the rest will be ignored." + }, + "num_threads_for_build_submissions": { + "type": int, + "default": 5, + "desc": "The number of threads when submitting component builds to an external build " + "system.", + }, + "default_modules_scm_url": { + "type": str, + "default": "https://pagure.io/releng/fedora-module-defaults.git", + "desc": "The SCM URL to the default modules repo, which will be used to determine " + "which buildrequires to automatically include. This can be overridden with " + "the xmd.mbs.default_modules_scm_url key in the base module's modulemd.", + }, + "uses_rawhide": { + "type": bool, + "default": True, + "desc": "Denotes if the concept of rawhide exists in the infrastructure of this " + "MBS deployment.", + }, + "rawhide_branch": { + "type": str, + "default": "master", + "desc": "Denotes the branch used for rawhide.", + }, + "dnf_minrate": { + "type": int, + "default": 1024 * 100, # 100KB + "desc": "The minrate configuration on a DNF repo. This configuration will cause DNF to " + "timeout loading a repo if the download speed is below minrate for the " + "duration of the timeout." + }, + "dnf_timeout": { + "type": int, + "default": 30, + "desc": "The timeout configuration for dnf operations, in seconds." + }, + "num_workers": {"type": int, "default": 1, "desc": "Number of Celery workers"}, + "celery_task_always_eager": { + "type": bool, + "default": False, + "desc": "All Celery tasks will be executed locally by blocking until the task returns " + "when this is True", + }, + "celery_task_routes": { + "type": list, + "default": ["module_build_service.route.route_task"], + "desc": "A list of Celery routers. When deciding the final destination queue of a " + "Celery task the routers are consulted in order", + }, + "celery_worker_prefetch_multiplier": { + "type": int, + "default": 1, + "desc": "This defaults to 1 so that the worker doesn't fetch more messages than it can " + "handle at a time. This so that general tasks aren't starved when running " + "a long handler.", + }, + "celery_broker_url": { + "type": str, + "default": "", + "desc": "The broker URL used by the Celery application instance.", + }, + "celery_imports": { + "type": list, + "default": [ + "module_build_service.scheduler.handlers.components", + "module_build_service.scheduler.handlers.modules", + "module_build_service.scheduler.handlers.repos", + "module_build_service.scheduler.handlers.tags", + "module_build_service.scheduler.handlers.greenwave", + "module_build_service.scheduler.producer", + ], + "desc": "The list Python paths for the Celery application to import.", + }, + } + + def __init__(self, conf_section_obj): + """ + Initialize the Config object with defaults and then override them + with runtime values. + """ + + # set defaults + for name, values in self._defaults.items(): + self.set_item(name, values["default"], values["type"]) + + # override defaults + for key in dir(conf_section_obj): + # skip keys starting with underscore + if key.startswith("_"): + continue + # set item (lower key) + self.set_item(key.lower(), getattr(conf_section_obj, key)) + + def set_item(self, key, value, value_type=None): + """ + Set value for configuration item. Creates the self._key = value + attribute and self.key property to set/get/del the attribute. + """ + if key == "set_item" or key.startswith("_"): + raise Exception("Configuration item's name is not allowed: %s" % key) + + # Create the empty self._key attribute, so we can assign to it. + if not hasattr(self, "_" + key): + setattr(self, "_" + key, None) + + # Create self.key property to access the self._key attribute. + # Use the setifok_func if available for the attribute. + setifok_func = "_setifok_{}".format(key) + if hasattr(self, setifok_func): + setx = lambda self, val: getattr(self, setifok_func)(val) + elif value_type == Path: + # For paths, expanduser. + setx = lambda self, val: setattr(self, "_" + key, os.path.expanduser(val)) + else: + setx = lambda self, val: setattr(self, "_" + key, val) + getx = lambda self: getattr(self, "_" + key) + delx = lambda self: delattr(self, "_" + key) + setattr(Config, key, property(getx, setx, delx)) + + # managed/registered configuration items + if key in self._defaults: + # type conversion for configuration item + convert = self._defaults[key]["type"] + if convert in [bool, int, list, str, set, dict]: + try: + # Do no try to convert None... + if value is not None: + value = convert(value) + except Exception: + raise TypeError("Configuration value conversion failed for name: %s" % key) + # unknown type/unsupported conversion, or conversion not needed + elif convert is not None and convert not in [Path]: + raise TypeError( + "Unsupported type %s for configuration item name: %s" % (convert, key)) + + # Set the attribute to the correct value + setattr(self, key, value) + + # + # Register your _setifok_* handlers here + # + + def _setifok_system(self, s): + s = str(s) + if s not in ("koji", "mock"): + raise ValueError("Unsupported buildsystem: %s." % s) + self._system = s + + def _setifok_polling_interval(self, i): + if not isinstance(i, int): + raise TypeError("polling_interval needs to be an int") + if i < 0: + raise ValueError("polling_interval must be >= 0") + self._polling_interval = i + + def _setifok_rpms_default_repository(self, s): + rpm_repo = str(s) + rpm_repo = rpm_repo.rstrip("/") + "/" + self._rpms_default_repository = rpm_repo + + def _setifok_rpms_default_cache(self, s): + rpm_cache = str(s) + rpm_cache = rpm_cache.rstrip("/") + "/" + self._rpms_default_cache = rpm_cache + + def _setifok_log_backend(self, s): + if s is None: + s = "console" + elif s not in logger.supported_log_backends(): + raise ValueError("Unsupported log backend") + self._log_backend = str(s) + + def _setifok_log_file(self, s): + if s is None: + self._log_file = "" + else: + self._log_file = str(s) + + def _setifok_log_level(self, s): + level = str(s).lower() + self._log_level = logger.str_to_log_level(level) + + def _setifok_messaging(self, s): + """ Validate that the specified messaging backend corresponds with one + of the installed plugins. The MBS core provides two such plugins, but + a third-party could install another usable one. + """ + entrypoints = pkg_resources.iter_entry_points("mbs.messaging_backends") + installed_backends = [e.name for e in entrypoints] + s = str(s) + if s not in installed_backends: + raise ValueError( + 'The messaging plugin for "{0}" is not installed.' + " The following are installed: {1}".format(s, ", ".join(installed_backends)) + ) + self._messaging = s + + def _setifok_amq_recv_addresses(self, l): + assert isinstance(l, list) or isinstance(l, tuple) + self._amq_recv_addresses = list(l) + + def _setifok_scmurls(self, l): + if not isinstance(l, list): + raise TypeError("scmurls needs to be a list.") + self._scmurls = [str(x) for x in l] + + def _setifok_num_concurrent_builds(self, i): + if not isinstance(i, int): + raise TypeError("NUM_CONCURRENT_BUILDS needs to be an int") + if i < 0: + raise ValueError("NUM_CONCURRENT_BUILDS must be >= 0") + self._num_concurrent_builds = i + + def _setifok_auth_method(self, s): + s = str(s) + if s.lower() not in ("oidc", "kerberos"): + raise ValueError("Unsupported authentication method") + if s.lower() == "kerberos": + try: + import ldap3 # noqa + except ImportError: + raise ValueError("ldap3 is required for kerberos authz") + self._auth_method = s.lower() + + def _setifok_ldap_uri(self, s): + ldap_uri = str(s) + + if ldap_uri and not re.match(r"^(?:ldap(?:s)?:\/\/.+)$", ldap_uri): + raise ValueError('LDAP_URI is invalid. It must start with "ldap://" or "ldaps://"') + + self._ldap_uri = ldap_uri + + def _setifok_rebuild_strategy(self, strategy): + if strategy not in SUPPORTED_STRATEGIES: + raise ValueError( + 'The strategy "{0}" is not supported. Choose from: {1}'.format( + strategy, ", ".join(SUPPORTED_STRATEGIES) + ) + ) + self._rebuild_strategy = strategy + + def _setifok_base_module_arches(self, data): + if not isinstance(data, dict): + raise ValueError("BASE_MODULE_ARCHES must be a dict") + for ns, arches in data.items(): + if len(ns.split(":")) != 2: + raise ValueError("BASE_MODULE_ARCHES keys must be in 'name:stream' format") + if not isinstance(arches, list): + raise ValueError("BASE_MODULE_ARCHES values must be lists") + self._base_module_arches = data + + def _setifok_rebuild_strategies_allowed(self, strategies): + if not isinstance(strategies, list): + raise ValueError("REBUILD_STRATEGIES_ALLOWED must be a list") + elif not strategies: + raise ValueError( + "REBUILD_STRATEGIES_ALLOWED must contain at least one rebuild strategy") + for strategy in strategies: + if strategy not in SUPPORTED_STRATEGIES: + raise ValueError( + "REBUILD_STRATEGIES_ALLOWED must be one of: {0}".format( + ", ".join(SUPPORTED_STRATEGIES)) + ) + + self._rebuild_strategies_allowed = strategies + + def _setifok_cleanup_failed_builds_time(self, num_days): + if num_days < 1: + raise ValueError("CLEANUP_FAILED_BUILDS_TIME must be set to 1 or more days") + self._cleanup_failed_builds_time = num_days + + def _setifok_resolver(self, s): + if s not in SUPPORTED_RESOLVERS.keys(): + raise ValueError( + 'The resolver "{0}" is not supported. Choose from: {1}'.format( + s, ", ".join(SUPPORTED_RESOLVERS.keys())) + ) + self._resolver = s + + def _setifok_product_pages_module_streams(self, d): + if not isinstance(d, dict): + raise ValueError("PRODUCT_PAGES_MODULE_STREAMS must be a dict") + + for regex, values in d.items(): + try: + re.compile(regex) + except (TypeError, re.error): + raise ValueError( + 'The regex `%r` in the configuration "PRODUCT_PAGES_MODULE_STREAMS" is invalid' + % regex + ) + + if not isinstance(values, list) and not isinstance(values, tuple): + raise ValueError( + 'The values in the configured dictionary for "PRODUCT_PAGES_MODULE_STREAMS" ' + "must be a list or tuple" + ) + + if len(values) != 3: + raise ValueError( + "There must be three entries in each value in the dictionary configured for " + '"PRODUCT_PAGES_MODULE_STREAMS"' + ) + + for i, value in enumerate(values): + if not isinstance(value, string_types): + # The last value is optional + if value is None and i == 2: + continue + + raise ValueError( + 'The value in the %i index of the values in "PRODUCT_PAGES_MODULE_STREAMS" ' + "must be a string" + % i + ) + + self._product_pages_module_streams = d + + def _setifok_num_threads_for_build_submissions(self, i): + if not isinstance(i, int): + raise TypeError("NUM_THREADS_FOR_BUILD_SUBMISSIONS needs to be an int") + if i < 1: + raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") + self._num_threads_for_build_submissions = i diff --git a/module_build_service/config.py b/module_build_service/config.py deleted file mode 100644 index 0e2c14e..0000000 --- a/module_build_service/config.py +++ /dev/null @@ -1,991 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import imp -import logging -import os -import pkg_resources -import re -import sys -import tempfile - -from six import string_types - -from module_build_service import logger -# This avoids creating a circular import by importing log from module_build_service -log = logging.getLogger('module_build_service') - - -# TODO: It'd be nice to reuse this from models.ModuleBuild.rebuild_strategies but models.py -# currently relies on this file, so we can't import it -SUPPORTED_STRATEGIES = ["changed-and-after", "only-changed", "all"] - -SUPPORTED_RESOLVERS = { - "mbs": {"builders": ["mock"]}, - "db": {"builders": ["koji", "mock", "copr"]}, - "local": {"builders": ["mock"]}, - "koji": {"builders": ["koji"]}, -} - - -class BaseConfiguration(object): - DEBUG = False - SECRET_KEY = os.urandom(16) - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(os.path.join( - os.getcwd(), "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - # Where we should run when running "manage.py run" directly. - HOST = "0.0.0.0" - PORT = 5000 - - -class TestConfiguration(BaseConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = os.environ.get( - "DATABASE_URI", "sqlite:///{0}".format(os.path.join(os.getcwd(), "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - # Ensures task.delay executes locally instead of scheduling a task to a queue. - CELERY_TASK_ALWAYS_EAGER = True - - -class ProdConfiguration(BaseConfiguration): - pass - - -class LocalBuildConfiguration(BaseConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - ALLOW_CUSTOM_SCMURLS = True - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - # Celery tasks will be executed locally for local builds - CELERY_TASK_ALWAYS_EAGER = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True - - -def init_config(): - """ - Create the global MBS configuration. - - :return: a tuple with the first index being the configuration and second index as the - configuration class that was used to initialize the configuration. This can be useful - to configure Flask with. - :rtype: tuple(Config, object) - """ - config_file = os.environ.get("MBS_CONFIG_FILE", "/etc/module-build-service/config.py") - config_section = os.environ.get("MBS_CONFIG_SECTION", "ProdConfiguration") - try: - config_module = imp.load_source("mbs_runtime_config", config_file) - log.info("Using the configuration file at %s", config_file) - except Exception: - log.warning("The configuration file at %s was not present", config_file) - # Default to this file as the configuration module - config_module = sys.modules[__name__] - - true_options = ("1", "on", "true", "y", "yes") - if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): - config_section = "TestConfiguration" - # Get the configuration from this module - config_module = sys.modules[__name__] - elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: - config_section = "DevConfiguration" - # Get the configuration from this module - config_module = sys.modules[__name__] - elif "build_module_locally" in sys.argv: - if "--offline" in sys.argv: - config_section = "OfflineLocalBuildConfiguration" - else: - config_section = "LocalBuildConfiguration" - - if hasattr(config_module, config_section): - log.info("Using the configuration section %s", config_section) - config_section_obj = getattr(config_module, config_section) - else: - log.error( - "The configuration class of %s is not present. Falling back to the default " - "ProdConfiguration class.", - config_section, - ) - config_section_obj = ProdConfiguration - conf = Config(config_section_obj) - return conf, config_section_obj - - -class Path: - """ - Config type for paths. Expands the users home directory. - """ - - pass - - -class Config(object): - """Class representing the orchestrator configuration.""" - - _defaults = { - "debug": {"type": bool, "default": False, "desc": "Debug mode"}, - "system": {"type": str, "default": "koji", "desc": "The buildsystem to use."}, - "db": {"type": str, "default": "", "desc": "RDB URL."}, - "default_dist_tag_prefix": { - "type": str, - "default": "module+", - "desc": "Default dist-tag prefix for built modules.", - }, - "polling_interval": {"type": int, "default": 600, "desc": "Polling interval, in seconds."}, - "cache_dir": { - "type": Path, - "default": os.path.join(tempfile.gettempdir(), "mbs"), - "desc": "Cache directory" - }, - "mbs_url": { - "type": str, - "default": "https://mbs.fedoraproject.org/module-build-service/1/module-builds/", - "desc": "MBS instance url for MBSResolver", - }, - "check_for_eol": { - "type": bool, - "default": False, - "desc": "Flag to determine whether or not MBS should block EOL modules from building.", - }, - "pdc_url": { - "type": str, - "default": "https://pdc.fedoraproject.org/rest_api/v1", - "desc": "PDC URL, used for checking stream EOL.", - }, - "koji_config": { - "type": str, - "default": "/etc/module-build-service/koji.conf", - "desc": "Koji config file." - }, - "koji_profile": {"type": str, "default": "koji", "desc": "Koji config profile."}, - "arches": {"type": list, "default": ["x86_64"], "desc": "Koji architectures."}, - "allow_arch_override": { - "type": bool, - "default": False, - "desc": "Allow to support a custom architecture set", - }, - "koji_build_priority": {"type": int, "default": 10, "desc": ""}, - "koji_repository_url": { - "type": str, - "default": "https://kojipkgs.fedoraproject.org/repos", - "desc": "Koji repository URL." - }, - "koji_build_macros_target": { - "type": str, - "default": "", - "desc": 'Target to build "module-build-macros" RPM in.', - }, - "koji_tag_prefixes": { - "type": list, - "default": ["module", "scrmod"], - "desc": "List of allowed koji tag prefixes.", - }, - "koji_tag_permission": { - "type": str, - "default": "admin", - "desc": "Permission name to require for newly created Koji tags.", - }, - "koji_tag_extra_opts": { - "type": dict, - "default": { - "mock.package_manager": "dnf", - # This is needed to include all the Koji builds (and therefore - # all the packages) from all inherited tags into this tag. - # See https://pagure.io/koji/issue/588 and - # https://pagure.io/fm-orchestrator/issue/660 for background. - "repo_include_all": True, - # Has been requested by Fedora infra in - # https://pagure.io/fedora-infrastructure/issue/7620. - # Disables systemd-nspawn for chroot. - "mock.new_chroot": 0, - # Works around fail-safe mechanism added in DNF 4.2.7 - # https://pagure.io/fedora-infrastructure/issue/8410 - "mock.yum.module_hotfixes": 1, - }, - "desc": "Extra options set for newly created Koji tags.", - }, - "koji_target_delete_time": { - "type": int, - "default": 24 * 3600, - "desc": "Time in seconds after which the Koji target of built module is deleted", - }, - "koji_enable_content_generator": { - "type": bool, - "default": True, - "desc": "Enable or disable imports to koji using content generator api", - }, - "allow_name_override_from_scm": { - "type": bool, - "default": False, - "desc": "Allow modulemd files to override the module name " - "if different from the scm repo name.", - }, - "allow_stream_override_from_scm": { - "type": bool, - "default": False, - "desc": "Allow modulemd files to override the module stream " - "if different from the scm repo branch.", - }, - "allow_custom_scmurls": {"type": bool, "default": False, "desc": "Allow custom scmurls."}, - "rpms_default_repository": { - "type": str, - "default": "https://src.fedoraproject.org/rpms/", - "desc": "RPMs default repository URL.", - }, - "rpms_allow_repository": { - "type": bool, - "default": False, - "desc": "Allow custom RPMs repositories.", - }, - "rpms_default_cache": { - "type": str, - "default": "http://pkgs.fedoraproject.org/repo/pkgs/", - "desc": "RPMs default cache URL.", - }, - "rpms_allow_cache": {"type": bool, "default": False, "desc": "Allow custom RPMs cache."}, - "modules_default_repository": { - "type": str, - "default": "https://src.fedoraproject.org/modules/", - "desc": "Included modules default repository URL.", - }, - "modules_allow_repository": { - "type": bool, - "default": False, - "desc": "Allow custom included modules repositories.", - }, - "allowed_groups": { - "type": set, - "default": {"packager"}, - "desc": "The set of groups allowed to submit builds.", - }, - "allowed_groups_to_import_module": { - "type": set, - "default": set(), - "desc": "The set of groups allowed to import module builds.", - }, - "log_backend": {"type": str, "default": None, "desc": "Log backend"}, - "log_file": {"type": str, "default": "", "desc": "Path to log file"}, - "log_level": {"type": str, "default": "info", "desc": "Log level"}, - "build_logs_dir": { - "type": Path, - "default": tempfile.gettempdir(), - "desc": "Directory to store module build logs to.", - }, - "build_logs_name_format": { - "type": str, - "default": "build-{id}.log", - "desc": ( - "Format of a module build log's name. Use `Build` attributes as formatting " - "kwargs" - ), - }, - "krb_keytab": {"type": None, "default": None, "desc": ""}, - "krb_principal": {"type": None, "default": None, "desc": ""}, - "messaging": {"type": str, "default": "fedmsg", "desc": "The messaging system to use."}, - "messaging_topic_prefix": { - "type": list, - "default": ["org.fedoraproject.prod"], - "desc": "The messaging system topic prefixes which we are interested in.", - }, - "distgits": { - "type": dict, - "default": { - "https://src.fedoraproject.org": ( - "fedpkg clone --anonymous {}", - "fedpkg --release module sources", - ), - "file://": ("git clone {repo_path}", None), - }, - "desc": "Mapping between dist-git and command to ", - }, - "mock_config": {"type": str, "default": "fedora-25-x86_64", "desc": ""}, - "mock_config_file": { - "type": list, - "default": ["/etc/module-build-service/mock.cfg", "conf/mock.cfg"], - "desc": "List of mock config file paths in order of preference.", - }, - "mock_build_srpm_cmd": {"type": str, "default": "fedpkg --release f26 srpm", "desc": ""}, - "mock_resultsdir": { - "type": Path, - "default": "~/modulebuild/builds", - "desc": "Directory for Mock build results.", - }, - "mock_purge_useless_logs": { - "type": bool, - "default": True, - "desc": "Remove empty or otherwise useless log files.", - }, - "arch_autodetect": { - "type": bool, - "default": True, - "desc": "Auto-detect machine arch when configuring builder.", - }, - "arch_fallback": { - "type": str, - "default": "x86_64", - "desc": "Fallback arch if auto-detection is off or unable to determine it.", - }, - "scmurls": { - "type": list, - "default": ["https://src.fedoraproject.org/modules/"], - "desc": "Allowed SCM URLs for submitted module.", - }, - "yaml_submit_allowed": { - "type": bool, - "default": False, - "desc": "Is it allowed to directly submit build by modulemd yaml file?", - }, - "num_concurrent_builds": { - "type": int, - "default": 5, - "desc": "Number of concurrent component builds.", - }, - "net_timeout": { - "type": int, - "default": 120, - "desc": "Global network timeout for read/write operations, in seconds.", - }, - "net_retry_interval": { - "type": int, - "default": 30, - "desc": "Global network retry interval for read/write operations, in seconds.", - }, - "scm_net_timeout": { - "type": int, - "default": 60, - "desc": "Network timeout for SCM operations, in seconds.", - }, - "scm_net_retry_interval": { - "type": int, - "default": 15, - "desc": "Network retry interval for SCM operations, in seconds.", - }, - "no_auth": {"type": bool, "default": False, "desc": "Disable client authentication."}, - "admin_groups": { - "type": set, - "default": set(), - "desc": "The set of groups allowed to manage MBS.", - }, - "yum_config_file": { - "type": list, - "default": ["/etc/module-build-service/yum.conf", "conf/yum.conf"], - "desc": "List of yum config file paths in order of preference.", - }, - "auth_method": { - "type": str, - "default": "oidc", - "desc": "Authentiation method to MBS. Options are oidc or kerberos", - }, - "ldap_uri": { - "type": str, - "default": "", - "desc": "LDAP URI to query for group information when using Kerberos authentication", - }, - "ldap_groups_dn": { - "type": str, - "default": "", - "desc": ( - "The distinguished name of the container or organizational unit containing " - "the groups in LDAP" - ), - }, - "base_module_names": { - "type": list, - "default": ["platform"], - "desc": ( - "List of base module names which define the product version " - "(by their stream) of modules depending on them." - ), - }, - "base_module_arches": { - "type": dict, - "default": {}, - "desc": "Per base-module name:stream Koji arches list.", - }, - "allow_only_compatible_base_modules": { - "type": bool, - "default": True, - "desc": "When True, only modules built on top of compatible base modules are " - "considered by MBS as possible buildrequirement. When False, modules " - "built against any base module stream can be used as a buildrequire.", - }, - "koji_cg_tag_build": { - "type": bool, - "default": True, - "desc": "Indicate whether tagging build is enabled during importing module to Koji.", - }, - "koji_cg_devel_module": { - "type": bool, - "default": True, - "desc": "Indicate whether a devel module should be imported into Koji.", - }, - "koji_cg_build_tag_template": { - "type": str, - "default": "{}-modular-updates-candidate", - "desc": "Name of a Koji tag where the top-level Content Generator " - "build is tagged to. The '{}' string is replaced by a " - "stream name of a base module on top of which the " - "module is built.", - }, - "koji_cg_default_build_tag": { - "type": str, - "default": "modular-updates-candidate", - "desc": "The name of Koji tag which should be used as fallback " - "when koji_cg_build_tag_template tag is not found in " - "Koji.", - }, - "rebuild_strategy": { - "type": str, - "default": "changed-and-after", - "desc": "The module rebuild strategy to use by default.", - }, - "rebuild_strategy_allow_override": { - "type": bool, - "default": False, - "desc": ( - "Allows a user to specify the rebuild strategy they want to use when " - "submitting a module build." - ), - }, - "rebuild_strategies_allowed": { - "type": list, - "default": SUPPORTED_STRATEGIES, - "desc": ( - "The allowed module rebuild strategies. This is only used when " - "REBUILD_STRATEGY_ALLOW_OVERRIDE is True." - ), - }, - "cleanup_failed_builds_time": { - "type": int, - "default": 180, - "desc": ( - "Time in days when to cleanup failed module builds and transition them to " - 'the "garbage" state.' - ), - }, - "cleanup_stuck_builds_time": { - "type": int, - "default": 7, - "desc": ( - "Time in days when to cleanup stuck module builds and transition them to " - 'the "failed" state. The module has to be in a state defined by the ' - '"cleanup_stuck_builds_states" option.' - ), - }, - "cleanup_stuck_builds_states": { - "type": list, - "default": ["init", "build"], - "desc": ( - "States of builds which will be considered to move to failed state when a" - " build is in one of those states longer than the value configured in the " - '"cleanup_stuck_builds_time"' - ), - }, - "resolver": { - "type": str, - "default": "db", - "desc": "Where to look up for modules. Note that this can (and " - "probably will) be builder-specific.", - }, - "koji_external_repo_url_prefix": { - "type": str, - "default": "https://kojipkgs.fedoraproject.org/", - "desc": "URL prefix of base module's external repo.", - }, - "allowed_users": { - "type": set, - "default": set(), - "desc": "The users/service accounts that don't require to be part of a group", - }, - "br_stream_override_module": { - "type": str, - "default": "platform", - "desc": ( - "The module name to override in the buildrequires based on the branch name. " - '"br_stream_override_regexes" must also be set for this to take ' - "effect." - ), - }, - "br_stream_override_regexes": { - "type": list, - "default": [], - "desc": ( - "The list of regexes used to parse the stream override from the branch name. " - '"br_stream_override_module" must also be set for this to take ' - "effect. The regexes can contain multiple capture groups that will be " - "concatenated. Any null capture groups will be ignored. The first regex that " - "matches the branch will be used." - ), - }, - "default_buildroot_packages": { - "type": list, - "default": [ - "bash", - "bzip2", - "coreutils", - "cpio", - "diffutils", - "findutils", - "gawk", - "gcc", - "gcc-c++", - "grep", - "gzip", - "info", - "make", - "patch", - "fedora-release", - "redhat-rpm-config", - "rpm-build", - "sed", - "shadow-utils", - "tar", - "unzip", - "util-linux", - "which", - "xz", - ], - "desc": ("The list packages for offline module build RPM buildroot."), - }, - "default_srpm_buildroot_packages": { - "type": list, - "default": [ - "bash", - "gnupg2", - "fedora-release", - "redhat-rpm-config", - "fedpkg-minimal", - "rpm-build", - "shadow-utils", - ], - "desc": ("The list packages for offline module build RPM buildroot."), - }, - "greenwave_decision_context": { - "type": str, - "default": "", - "desc": "The Greenwave decision context that determines a module's gating status.", - }, - "allowed_privileged_module_names": { - "type": list, - "default": [], - "desc": ( - "List of modules that are allowed to influence the RPM buildroot when " - "buildrequired. These modules can set xmd.mbs.disttag_marking to do change " - "the RPM disttag, or set the xmd.mbs.koji_tag_arches to set the arches " - "for which the modules are built. MBS will use this list order to determine " - "which modules take precedence." - ), - }, - "stream_suffixes": { - "type": dict, - "default": {}, - "desc": "A mapping of platform stream regular expressions and the " - "corresponding suffix added to formatted stream version. " - 'For example, {r"regexp": 0.1, ...}', - }, - "greenwave_url": { - "type": str, - "default": "", - "desc": "The URL of the server where Greenwave is running (should include " - "the root of the API)" - }, - "greenwave_subject_type": { - "type": str, - "default": "", - "desc": "Subject type for Greenwave requests" - }, - "greenwave_timeout": { - "type": int, - "default": 60, - "desc": "Greenwave response timeout" - }, - "modules_allow_scratch": { - "type": bool, - "default": False, - "desc": "Allow module scratch builds", - }, - "scratch_build_only_branches": { - "type": list, - "default": [], - "desc": "The list of regexes used to identify branches from which only the module " - "scratch builds can be built", - }, - "product_pages_url": { - "type": str, - "default": "", - "desc": "The URL to the Product Pages. This is queried to determine if a base module " - "stream has been released. If it has, the stream may be modified automatically " - "to use a different support stream.", - }, - "product_pages_module_streams": { - "type": dict, - "default": {}, - "desc": "The keys are regexes of base module streams that should be checked in the Red " - "Hat Product Pages. The values are tuples. The first value is a string that " - "should be appended to the stream if there is a match and the release the " - "stream represents has been released. The second value is a template string " - "that represents the release in Product Pages and can accept format kwargs of " - "x, y, and z (represents the version). The third value is an optional template " - "string that represent the Product Pages release for major releases " - "(e.g. 8.0.0). After the first match, the rest will be ignored." - }, - "num_threads_for_build_submissions": { - "type": int, - "default": 5, - "desc": "The number of threads when submitting component builds to an external build " - "system.", - }, - "default_modules_scm_url": { - "type": str, - "default": "https://pagure.io/releng/fedora-module-defaults.git", - "desc": "The SCM URL to the default modules repo, which will be used to determine " - "which buildrequires to automatically include. This can be overridden with " - "the xmd.mbs.default_modules_scm_url key in the base module's modulemd.", - }, - "uses_rawhide": { - "type": bool, - "default": True, - "desc": "Denotes if the concept of rawhide exists in the infrastructure of this " - "MBS deployment.", - }, - "rawhide_branch": { - "type": str, - "default": "master", - "desc": "Denotes the branch used for rawhide.", - }, - "dnf_minrate": { - "type": int, - "default": 1024 * 100, # 100KB - "desc": "The minrate configuration on a DNF repo. This configuration will cause DNF to " - "timeout loading a repo if the download speed is below minrate for the " - "duration of the timeout." - }, - "dnf_timeout": { - "type": int, - "default": 30, - "desc": "The timeout configuration for dnf operations, in seconds." - }, - "num_workers": {"type": int, "default": 1, "desc": "Number of Celery workers"}, - "celery_task_always_eager": { - "type": bool, - "default": False, - "desc": "All Celery tasks will be executed locally by blocking until the task returns " - "when this is True", - }, - "celery_task_routes": { - "type": list, - "default": ["module_build_service.route.route_task"], - "desc": "A list of Celery routers. When deciding the final destination queue of a " - "Celery task the routers are consulted in order", - }, - "celery_worker_prefetch_multiplier": { - "type": int, - "default": 1, - "desc": "This defaults to 1 so that the worker doesn't fetch more messages than it can " - "handle at a time. This so that general tasks aren't starved when running " - "a long handler.", - }, - "celery_broker_url": { - "type": str, - "default": "", - "desc": "The broker URL used by the Celery application instance.", - }, - "celery_imports": { - "type": list, - "default": [ - "module_build_service.scheduler.handlers.components", - "module_build_service.scheduler.handlers.modules", - "module_build_service.scheduler.handlers.repos", - "module_build_service.scheduler.handlers.tags", - "module_build_service.scheduler.handlers.greenwave", - "module_build_service.scheduler.producer", - ], - "desc": "The list Python paths for the Celery application to import.", - }, - } - - def __init__(self, conf_section_obj): - """ - Initialize the Config object with defaults and then override them - with runtime values. - """ - - # set defaults - for name, values in self._defaults.items(): - self.set_item(name, values["default"], values["type"]) - - # override defaults - for key in dir(conf_section_obj): - # skip keys starting with underscore - if key.startswith("_"): - continue - # set item (lower key) - self.set_item(key.lower(), getattr(conf_section_obj, key)) - - def set_item(self, key, value, value_type=None): - """ - Set value for configuration item. Creates the self._key = value - attribute and self.key property to set/get/del the attribute. - """ - if key == "set_item" or key.startswith("_"): - raise Exception("Configuration item's name is not allowed: %s" % key) - - # Create the empty self._key attribute, so we can assign to it. - if not hasattr(self, "_" + key): - setattr(self, "_" + key, None) - - # Create self.key property to access the self._key attribute. - # Use the setifok_func if available for the attribute. - setifok_func = "_setifok_{}".format(key) - if hasattr(self, setifok_func): - setx = lambda self, val: getattr(self, setifok_func)(val) - elif value_type == Path: - # For paths, expanduser. - setx = lambda self, val: setattr(self, "_" + key, os.path.expanduser(val)) - else: - setx = lambda self, val: setattr(self, "_" + key, val) - getx = lambda self: getattr(self, "_" + key) - delx = lambda self: delattr(self, "_" + key) - setattr(Config, key, property(getx, setx, delx)) - - # managed/registered configuration items - if key in self._defaults: - # type conversion for configuration item - convert = self._defaults[key]["type"] - if convert in [bool, int, list, str, set, dict]: - try: - # Do no try to convert None... - if value is not None: - value = convert(value) - except Exception: - raise TypeError("Configuration value conversion failed for name: %s" % key) - # unknown type/unsupported conversion, or conversion not needed - elif convert is not None and convert not in [Path]: - raise TypeError( - "Unsupported type %s for configuration item name: %s" % (convert, key)) - - # Set the attribute to the correct value - setattr(self, key, value) - - # - # Register your _setifok_* handlers here - # - - def _setifok_system(self, s): - s = str(s) - if s not in ("koji", "mock"): - raise ValueError("Unsupported buildsystem: %s." % s) - self._system = s - - def _setifok_polling_interval(self, i): - if not isinstance(i, int): - raise TypeError("polling_interval needs to be an int") - if i < 0: - raise ValueError("polling_interval must be >= 0") - self._polling_interval = i - - def _setifok_rpms_default_repository(self, s): - rpm_repo = str(s) - rpm_repo = rpm_repo.rstrip("/") + "/" - self._rpms_default_repository = rpm_repo - - def _setifok_rpms_default_cache(self, s): - rpm_cache = str(s) - rpm_cache = rpm_cache.rstrip("/") + "/" - self._rpms_default_cache = rpm_cache - - def _setifok_log_backend(self, s): - if s is None: - s = "console" - elif s not in logger.supported_log_backends(): - raise ValueError("Unsupported log backend") - self._log_backend = str(s) - - def _setifok_log_file(self, s): - if s is None: - self._log_file = "" - else: - self._log_file = str(s) - - def _setifok_log_level(self, s): - level = str(s).lower() - self._log_level = logger.str_to_log_level(level) - - def _setifok_messaging(self, s): - """ Validate that the specified messaging backend corresponds with one - of the installed plugins. The MBS core provides two such plugins, but - a third-party could install another usable one. - """ - entrypoints = pkg_resources.iter_entry_points("mbs.messaging_backends") - installed_backends = [e.name for e in entrypoints] - s = str(s) - if s not in installed_backends: - raise ValueError( - 'The messaging plugin for "{0}" is not installed.' - " The following are installed: {1}".format(s, ", ".join(installed_backends)) - ) - self._messaging = s - - def _setifok_amq_recv_addresses(self, l): - assert isinstance(l, list) or isinstance(l, tuple) - self._amq_recv_addresses = list(l) - - def _setifok_scmurls(self, l): - if not isinstance(l, list): - raise TypeError("scmurls needs to be a list.") - self._scmurls = [str(x) for x in l] - - def _setifok_num_concurrent_builds(self, i): - if not isinstance(i, int): - raise TypeError("NUM_CONCURRENT_BUILDS needs to be an int") - if i < 0: - raise ValueError("NUM_CONCURRENT_BUILDS must be >= 0") - self._num_concurrent_builds = i - - def _setifok_auth_method(self, s): - s = str(s) - if s.lower() not in ("oidc", "kerberos"): - raise ValueError("Unsupported authentication method") - if s.lower() == "kerberos": - try: - import ldap3 # noqa - except ImportError: - raise ValueError("ldap3 is required for kerberos authz") - self._auth_method = s.lower() - - def _setifok_ldap_uri(self, s): - ldap_uri = str(s) - - if ldap_uri and not re.match(r"^(?:ldap(?:s)?:\/\/.+)$", ldap_uri): - raise ValueError('LDAP_URI is invalid. It must start with "ldap://" or "ldaps://"') - - self._ldap_uri = ldap_uri - - def _setifok_rebuild_strategy(self, strategy): - if strategy not in SUPPORTED_STRATEGIES: - raise ValueError( - 'The strategy "{0}" is not supported. Choose from: {1}'.format( - strategy, ", ".join(SUPPORTED_STRATEGIES) - ) - ) - self._rebuild_strategy = strategy - - def _setifok_base_module_arches(self, data): - if not isinstance(data, dict): - raise ValueError("BASE_MODULE_ARCHES must be a dict") - for ns, arches in data.items(): - if len(ns.split(":")) != 2: - raise ValueError("BASE_MODULE_ARCHES keys must be in 'name:stream' format") - if not isinstance(arches, list): - raise ValueError("BASE_MODULE_ARCHES values must be lists") - self._base_module_arches = data - - def _setifok_rebuild_strategies_allowed(self, strategies): - if not isinstance(strategies, list): - raise ValueError("REBUILD_STRATEGIES_ALLOWED must be a list") - elif not strategies: - raise ValueError( - "REBUILD_STRATEGIES_ALLOWED must contain at least one rebuild strategy") - for strategy in strategies: - if strategy not in SUPPORTED_STRATEGIES: - raise ValueError( - "REBUILD_STRATEGIES_ALLOWED must be one of: {0}".format( - ", ".join(SUPPORTED_STRATEGIES)) - ) - - self._rebuild_strategies_allowed = strategies - - def _setifok_cleanup_failed_builds_time(self, num_days): - if num_days < 1: - raise ValueError("CLEANUP_FAILED_BUILDS_TIME must be set to 1 or more days") - self._cleanup_failed_builds_time = num_days - - def _setifok_resolver(self, s): - if s not in SUPPORTED_RESOLVERS.keys(): - raise ValueError( - 'The resolver "{0}" is not supported. Choose from: {1}'.format( - s, ", ".join(SUPPORTED_RESOLVERS.keys())) - ) - self._resolver = s - - def _setifok_product_pages_module_streams(self, d): - if not isinstance(d, dict): - raise ValueError("PRODUCT_PAGES_MODULE_STREAMS must be a dict") - - for regex, values in d.items(): - try: - re.compile(regex) - except (TypeError, re.error): - raise ValueError( - 'The regex `%r` in the configuration "PRODUCT_PAGES_MODULE_STREAMS" is invalid' - % regex - ) - - if not isinstance(values, list) and not isinstance(values, tuple): - raise ValueError( - 'The values in the configured dictionary for "PRODUCT_PAGES_MODULE_STREAMS" ' - "must be a list or tuple" - ) - - if len(values) != 3: - raise ValueError( - "There must be three entries in each value in the dictionary configured for " - '"PRODUCT_PAGES_MODULE_STREAMS"' - ) - - for i, value in enumerate(values): - if not isinstance(value, string_types): - # The last value is optional - if value is None and i == 2: - continue - - raise ValueError( - 'The value in the %i index of the values in "PRODUCT_PAGES_MODULE_STREAMS" ' - "must be a string" - % i - ) - - self._product_pages_module_streams = d - - def _setifok_num_threads_for_build_submissions(self, i): - if not isinstance(i, int): - raise TypeError("NUM_THREADS_FOR_BUILD_SUBMISSIONS needs to be an int") - if i < 1: - raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") - self._num_threads_for_build_submissions = i diff --git a/module_build_service/resolver/base.py b/module_build_service/resolver/base.py index 74e4032..e46744b 100644 --- a/module_build_service/resolver/base.py +++ b/module_build_service/resolver/base.py @@ -6,7 +6,7 @@ import six from abc import ABCMeta, abstractmethod -import module_build_service.config as cfg +import module_build_service.common.config as cfg from module_build_service import conf diff --git a/module_build_service/scheduler/submit.py b/module_build_service/scheduler/submit.py index 6b18921..1e3bebf 100644 --- a/module_build_service/scheduler/submit.py +++ b/module_build_service/scheduler/submit.py @@ -20,7 +20,7 @@ def get_build_arches(mmd, config): Returns the list of architectures for which the module `mmd` should be built. :param mmd: Module MetaData - :param config: config (module_build_service.config.Config instance) + :param config: config (module_build_service.common.config.Config instance) :return list of architectures """ # Imported here to allow import of utils in GenericBuilder. diff --git a/tests/__init__.py b/tests/__init__.py index e1ca19d..3181fec 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -51,10 +51,10 @@ def read_staged_data(yaml_name): def patch_config(): # add test builders for all resolvers with_test_builders = dict() - for k, v in module_build_service.config.SUPPORTED_RESOLVERS.items(): + for k, v in module_build_service.common.config.SUPPORTED_RESOLVERS.items(): v["builders"].extend(["test", "testlocal"]) with_test_builders[k] = v - patch("module_build_service.config.SUPPORTED_RESOLVERS", with_test_builders) + patch("module_build_service.common.config.SUPPORTED_RESOLVERS", with_test_builders) patch_config() diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index fb7a6d4..c798324 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -397,7 +397,9 @@ class BaseTestBuild: @patch("module_build_service.scheduler.handlers.modules.handle_stream_collision_modules") @patch.object( - module_build_service.config.Config, "system", new_callable=PropertyMock, return_value="test" + module_build_service.common.config.Config, "system", + new_callable=PropertyMock, + return_value="test", ) @patch( "module_build_service.builder.GenericBuilder.default_buildroot_groups", @@ -459,7 +461,7 @@ class TestBuild(BaseTestBuild): self.mock_check_gating = self.p_check_gating.start() self.patch_config_broker = patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "celery_broker_url", create=True, new_callable=PropertyMock, @@ -668,7 +670,7 @@ class TestBuild(BaseTestBuild): assert module_build.state_reason == "Gating failed" @patch( - "module_build_service.config.Config.check_for_eol", + "module_build_service.common.config.Config.check_for_eol", new_callable=PropertyMock, return_value=True, ) @@ -710,7 +712,7 @@ class TestBuild(BaseTestBuild): yaml = read_staged_data("testmodule") with patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "yaml_submit_allowed", new_callable=PropertyMock, return_value=False, @@ -733,7 +735,7 @@ class TestBuild(BaseTestBuild): mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4") with patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "yaml_submit_allowed", new_callable=PropertyMock, return_value=True, @@ -855,7 +857,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.num_concurrent_builds", + "module_build_service.common.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1, ) @@ -914,7 +916,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.num_concurrent_builds", + "module_build_service.common.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=2, ) @@ -979,7 +981,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.num_concurrent_builds", + "module_build_service.common.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1, ) @@ -1048,7 +1050,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.num_concurrent_builds", + "module_build_service.common.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1, ) @@ -1629,7 +1631,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @@ -1672,7 +1674,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @@ -1716,7 +1718,7 @@ class TestBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @@ -1873,7 +1875,9 @@ class TestBuild(BaseTestBuild): @patch( - "module_build_service.config.Config.system", new_callable=PropertyMock, return_value="testlocal" + "module_build_service.common.config.Config.system", + new_callable=PropertyMock, + return_value="testlocal", ) class TestLocalBuild(BaseTestBuild): def setup_method(self, test_method): @@ -1896,7 +1900,7 @@ class TestLocalBuild(BaseTestBuild): @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=PropertyMock, return_value=staged_data_filename('local_builds'), ) diff --git a/tests/test_builder/test_builder_utils.py b/tests/test_builder/test_builder_utils.py index 22a613a..a1141f2 100644 --- a/tests/test_builder/test_builder_utils.py +++ b/tests/test_builder/test_builder_utils.py @@ -244,7 +244,7 @@ def test_validate_koji_tag_is_None(): @patch( - "module_build_service.config.Config.allowed_privileged_module_names", + "module_build_service.common.config.Config.allowed_privileged_module_names", new_callable=PropertyMock, return_value=["testmodule"], ) @@ -300,7 +300,7 @@ def test_get_rpm_release_platform_stream_override(): @patch( - "module_build_service.config.Config.allowed_privileged_module_names", + "module_build_service.common.config.Config.allowed_privileged_module_names", new_callable=PropertyMock, return_value=["build"], ) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 898510c..0a86a87 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -876,12 +876,12 @@ class TestKojiBuilder: db_session.commit() with patch( - "module_build_service.config.Config.koji_enable_content_generator", + "module_build_service.common.config.Config.koji_enable_content_generator", new_callable=mock.PropertyMock, return_value=cg_enabled, ): with patch( - "module_build_service.config.Config.koji_cg_devel_module", + "module_build_service.common.config.Config.koji_cg_devel_module", new_callable=mock.PropertyMock, return_value=cg_devel_enabled, ): diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py index 22e45eb..25b4a7a 100644 --- a/tests/test_builder/test_mock.py +++ b/tests/test_builder/test_mock.py @@ -194,7 +194,7 @@ class TestMockModuleBuilderAddRepos: @mock.patch("module_build_service.conf.system", new="mock") @mock.patch( - "module_build_service.config.Config.base_module_repofiles", + "module_build_service.common.config.Config.base_module_repofiles", new_callable=mock.PropertyMock, return_value=["/etc/yum.repos.d/bar.repo", "/etc/yum.repos.d/bar-updates.repo"], create=True, @@ -308,12 +308,14 @@ class TestOfflineLocalBuilds: @mock.patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=mock.PropertyMock, return_value=staged_data_filename("local_builds") ) @mock.patch( - "module_build_service.config.Config.system", new_callable=mock.PropertyMock, return_value="mock" + "module_build_service.common.config.Config.system", + new_callable=mock.PropertyMock, + return_value="mock", ) class TestLocalBuilds: def setup_method(self): diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py new file mode 100644 index 0000000..3ccb7ba --- /dev/null +++ b/tests/test_common/test_config.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import os.path + +from module_build_service import conf + + +class TestConfig: + def test_path_expanduser(self): + test_dir = "~/modulebuild/builds" + conf.mock_resultsdir = test_dir + assert conf.mock_resultsdir == os.path.expanduser(test_dir) + + test_dir = "~/modulebuild/builds" + conf.cache_dir = test_dir + assert conf.cache_dir == os.path.expanduser(test_dir) diff --git a/tests/test_common/test_resolve.py b/tests/test_common/test_resolve.py index efd6277..933b447 100644 --- a/tests/test_common/test_resolve.py +++ b/tests/test_common/test_resolve.py @@ -74,7 +74,7 @@ class TestResolve: assert actual == expected @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", + "module_build_service.common.config.Config.allow_only_compatible_base_modules", new_callable=PropertyMock, return_value=False ) def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg): diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 3ccb7ba..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import os.path - -from module_build_service import conf - - -class TestConfig: - def test_path_expanduser(self): - test_dir = "~/modulebuild/builds" - conf.mock_resultsdir = test_dir - assert conf.mock_resultsdir == os.path.expanduser(test_dir) - - test_dir = "~/modulebuild/builds" - conf.cache_dir = test_dir - assert conf.cache_dir == os.path.expanduser(test_dir) diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 832937d..e8bce9e 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -129,10 +129,12 @@ class TestDBModule: assert set(result) == {"module-f28-build"} @patch( - "module_build_service.config.Config.system", new_callable=PropertyMock, return_value="test" + "module_build_service.common.config.Config.system", + new_callable=PropertyMock, + return_value="test", ) @patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=PropertyMock, return_value=tests.staged_data_filename("local_builds"), ) @@ -250,10 +252,12 @@ class TestDBModule: assert result == expected @patch( - "module_build_service.config.Config.system", new_callable=PropertyMock, return_value="test" + "module_build_service.common.config.Config.system", + new_callable=PropertyMock, + return_value="test", ) @patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=PropertyMock, return_value=tests.staged_data_filename("local_builds") ) diff --git a/tests/test_resolver/test_mbs.py b/tests/test_resolver/test_mbs.py index c770a04..d21b195 100644 --- a/tests/test_resolver/test_mbs.py +++ b/tests/test_resolver/test_mbs.py @@ -317,10 +317,12 @@ class TestMBSModule: assert result == expected @patch( - "module_build_service.config.Config.system", new_callable=PropertyMock, return_value="test" + "module_build_service.common.config.Config.system", + new_callable=PropertyMock, + return_value="test", ) @patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=PropertyMock, return_value=tests.staged_data_filename("local_builds") ) @@ -438,10 +440,12 @@ class TestMBSModule: ) @patch( - "module_build_service.config.Config.system", new_callable=PropertyMock, return_value="test" + "module_build_service.common.config.Config.system", + new_callable=PropertyMock, + return_value="test", ) @patch( - "module_build_service.config.Config.mock_resultsdir", + "module_build_service.common.config.Config.mock_resultsdir", new_callable=PropertyMock, return_value=tests.staged_data_filename("local_builds") ) diff --git a/tests/test_scheduler/test_batches.py b/tests/test_scheduler/test_batches.py index 29db790..b6e6c51 100644 --- a/tests/test_scheduler/test_batches.py +++ b/tests/test_scheduler/test_batches.py @@ -196,7 +196,7 @@ class TestBatches: @patch("module_build_service.scheduler.batches.start_build_component") @patch( - "module_build_service.config.Config.rebuild_strategy", + "module_build_service.common.config.Config.rebuild_strategy", new_callable=mock.PropertyMock, return_value="all", ) @@ -236,7 +236,7 @@ class TestBatches: @patch("module_build_service.scheduler.batches.start_build_component") @patch( - "module_build_service.config.Config.rebuild_strategy", + "module_build_service.common.config.Config.rebuild_strategy", new_callable=mock.PropertyMock, return_value="only-changed", ) diff --git a/tests/test_scheduler/test_greenwave_handler.py b/tests/test_scheduler/test_greenwave_handler.py index 2a1915b..eaf9360 100644 --- a/tests/test_scheduler/test_greenwave_handler.py +++ b/tests/test_scheduler/test_greenwave_handler.py @@ -5,7 +5,7 @@ import pytest from mock import call, patch, PropertyMock, Mock from sqlalchemy import func -import module_build_service.config +import module_build_service.common.config from module_build_service import conf from module_build_service.db_session import db_session from module_build_service.models import BUILD_STATES, ModuleBuild @@ -75,7 +75,7 @@ class TestDecisionUpdateHandler: def setup_method(self, test_method): self.patch_config_broker = patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "celery_broker_url", create=True, new_callable=PropertyMock, diff --git a/tests/test_scheduler/test_module_init.py b/tests/test_scheduler/test_module_init.py index 5e262b0..d8a93ef 100644 --- a/tests/test_scheduler/test_module_init.py +++ b/tests/test_scheduler/test_module_init.py @@ -118,7 +118,7 @@ class TestModuleInit: assert build.state == 4, build.state @patch( - "module_build_service.config.Config.modules_allow_repository", + "module_build_service.common.config.Config.modules_allow_repository", new_callable=PropertyMock, return_value=True, ) diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 3aaa3b9..8711c60 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -194,7 +194,7 @@ class TestModuleWait: @patch("module_build_service.resolver.DBResolver") @patch("module_build_service.resolver.GenericResolver") @patch( - "module_build_service.config.Config.base_module_names", + "module_build_service.common.config.Config.base_module_names", new_callable=mock.PropertyMock, return_value=["base-runtime", "platform"], ) diff --git a/tests/test_scheduler/test_reuse.py b/tests/test_scheduler/test_reuse.py index 1d13ce6..e9fa3f0 100644 --- a/tests/test_scheduler/test_reuse.py +++ b/tests/test_scheduler/test_reuse.py @@ -296,7 +296,7 @@ class TestUtilsModuleReuse: @pytest.mark.parametrize("allow_ocbm", (True, False)) @mock.patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", + "module_build_service.common.config.Config.allow_only_compatible_base_modules", new_callable=mock.PropertyMock, ) def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): @@ -365,12 +365,12 @@ class TestUtilsModuleReuse: @pytest.mark.parametrize("allow_ocbm", (True, False)) @mock.patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", + "module_build_service.common.config.Config.allow_only_compatible_base_modules", new_callable=mock.PropertyMock, ) @mock.patch("koji.ClientSession") @mock.patch( - "module_build_service.config.Config.resolver", + "module_build_service.common.config.Config.resolver", new_callable=mock.PropertyMock, return_value="koji" ) def test_get_reusable_module_koji_resolver( diff --git a/tests/test_scheduler/test_submit.py b/tests/test_scheduler/test_submit.py index 7705005..9a701c7 100644 --- a/tests/test_scheduler/test_submit.py +++ b/tests/test_scheduler/test_submit.py @@ -50,7 +50,7 @@ class TestSubmit: assert set(r) == set(conf.arches) @mock.patch( - "module_build_service.config.Config.allowed_privileged_module_names", + "module_build_service.common.config.Config.allowed_privileged_module_names", new_callable=mock.PropertyMock, return_value=["testmodule"], ) diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 777cd84..dcdcf03 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -33,7 +33,7 @@ from module_build_service.db_session import db_session from module_build_service.errors import UnprocessableEntity from module_build_service.models import ModuleBuild, BUILD_STATES, ComponentBuild from module_build_service import app, version -import module_build_service.config as mbs_config +import module_build_service.common.config as mbs_config import module_build_service.web.submit @@ -496,7 +496,7 @@ class TestViews: mock_session.krb_login.assert_not_called() @patch( - "module_build_service.config.Config.system", + "module_build_service.common.config.Config.system", new_callable=PropertyMock, return_value="invalid_builder", ) @@ -1015,7 +1015,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.rebuild_strategy_allow_override", + "module_build_service.common.config.Config.rebuild_strategy_allow_override", new_callable=PropertyMock, return_value=True, ) @@ -1040,12 +1040,12 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.rebuild_strategies_allowed", + "module_build_service.common.config.Config.rebuild_strategies_allowed", new_callable=PropertyMock, return_value=["all"], ) @patch( - "module_build_service.config.Config.rebuild_strategy_allow_override", + "module_build_service.common.config.Config.rebuild_strategy_allow_override", new_callable=PropertyMock, return_value=True, ) @@ -1300,7 +1300,7 @@ class TestViews: ) def test_cancel_build_admin(self, mocked_get_user): with patch( - "module_build_service.config.Config.admin_groups", + "module_build_service.common.config.Config.admin_groups", new_callable=PropertyMock, return_value={"mbs-admin"}, ): @@ -1314,7 +1314,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=("sammy", {"packager"})) def test_cancel_build_no_admin(self, mocked_get_user): with patch( - "module_build_service.config.Config.admin_groups", + "module_build_service.common.config.Config.admin_groups", new_callable=PropertyMock, return_value={"mbs-admin"}, ): @@ -1431,7 +1431,9 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=anonymous_user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.no_auth", new_callable=PropertyMock, return_value=True + "module_build_service.common.config.Config.no_auth", + new_callable=PropertyMock, + return_value=True, ) def test_submit_build_no_auth_set_owner(self, mocked_conf, mocked_scm, mocked_get_user): FakeSCM( @@ -1451,7 +1453,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=("svc_account", set())) @patch("module_build_service.scm.SCM") - @patch("module_build_service.config.Config.allowed_users", new_callable=PropertyMock) + @patch("module_build_service.common.config.Config.allowed_users", new_callable=PropertyMock) def test_submit_build_allowed_users(self, allowed_users, mocked_scm, mocked_get_user): FakeSCM( mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4") @@ -1467,7 +1469,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=anonymous_user) @patch("module_build_service.scm.SCM") - @patch("module_build_service.config.Config.no_auth", new_callable=PropertyMock) + @patch("module_build_service.common.config.Config.no_auth", new_callable=PropertyMock) def test_patch_set_different_owner(self, mocked_no_auth, mocked_scm, mocked_get_user): FakeSCM( mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4") @@ -1521,7 +1523,10 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") - @patch("module_build_service.config.Config.allow_custom_scmurls", new_callable=PropertyMock) + @patch( + "module_build_service.common.config.Config.allow_custom_scmurls", + new_callable=PropertyMock, + ) def test_submit_custom_scmurl(self, allow_custom_scmurls, mocked_scm, mocked_get_user): FakeSCM( mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4") @@ -1846,7 +1851,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "allowed_groups_to_import_module", new_callable=PropertyMock, return_value=set(), @@ -1897,7 +1902,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=import_module_user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "allow_custom_scmurls", new_callable=PropertyMock, return_value=True, @@ -1922,7 +1927,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=import_module_user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "scmurls", new_callable=PropertyMock, return_value=["file://"], @@ -1961,7 +1966,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=import_module_user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "scmurls", new_callable=PropertyMock, return_value=["file://"], @@ -2002,7 +2007,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=import_module_user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "scmurls", new_callable=PropertyMock, return_value=["file://"], @@ -2025,7 +2030,7 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=import_module_user) @patch.object( - module_build_service.config.Config, + module_build_service.common.config.Config, "scmurls", new_callable=PropertyMock, return_value=["file://"], @@ -2073,7 +2078,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @@ -2135,7 +2140,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=False, ) @@ -2165,12 +2170,12 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=user) @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @patch( - "module_build_service.config.Config.yaml_submit_allowed", + "module_build_service.common.config.Config.yaml_submit_allowed", new_callable=PropertyMock, return_value=True, ) @@ -2229,12 +2234,12 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @patch( - "module_build_service.config.Config.yaml_submit_allowed", + "module_build_service.common.config.Config.yaml_submit_allowed", new_callable=PropertyMock, return_value=True, ) @@ -2263,12 +2268,12 @@ class TestViews: @pytest.mark.parametrize("api_version", [1, 2]) @patch("module_build_service.web.auth.get_user", return_value=user) @patch( - "module_build_service.config.Config.modules_allow_scratch", + "module_build_service.common.config.Config.modules_allow_scratch", new_callable=PropertyMock, return_value=True, ) @patch( - "module_build_service.config.Config.yaml_submit_allowed", + "module_build_service.common.config.Config.yaml_submit_allowed", new_callable=PropertyMock, return_value=False, ) @@ -2301,7 +2306,9 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch.object( - module_build_service.config.Config, "br_stream_override_regexes", new_callable=PropertyMock + module_build_service.common.config.Config, + "br_stream_override_regexes", + new_callable=PropertyMock, ) def test_submit_build_dep_override_from_branch( self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override @@ -2348,7 +2355,9 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch.object( - module_build_service.config.Config, "br_stream_override_regexes", new_callable=PropertyMock + module_build_service.common.config.Config, + "br_stream_override_regexes", + new_callable=PropertyMock, ) def test_submit_build_dep_override_from_branch_br_override( self, mocked_regexes, mocked_scm, mocked_get_user @@ -2422,7 +2431,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.allowed_privileged_module_names", + "module_build_service.common.config.Config.allowed_privileged_module_names", new_callable=PropertyMock, return_value=["build"], ) @@ -2597,11 +2606,11 @@ class TestViews: new_callable=partial(Mock, wraps=datetime), ) @patch( - "module_build_service.config.Config.product_pages_url", + "module_build_service.common.config.Config.product_pages_url", new_callable=PropertyMock, ) @patch( - "module_build_service.config.Config.product_pages_module_streams", + "module_build_service.common.config.Config.product_pages_module_streams", new_callable=PropertyMock, ) @patch("requests.get") @@ -2738,7 +2747,7 @@ class TestViews: @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") @patch( - "module_build_service.config.Config.rebuild_strategy_allow_override", + "module_build_service.common.config.Config.rebuild_strategy_allow_override", new_callable=PropertyMock, return_value=True, ) diff --git a/tests/test_web/test_auth.py b/tests/test_web/test_auth.py index ff9f615..397930b 100644 --- a/tests/test_web/test_auth.py +++ b/tests/test_web/test_auth.py @@ -9,7 +9,7 @@ from mock import patch, PropertyMock, Mock import module_build_service.web.auth import module_build_service.errors -import module_build_service.config as mbs_config +import module_build_service.common.config as mbs_config from module_build_service import app diff --git a/tests/test_web/test_submit.py b/tests/test_web/test_submit.py index 105e152..b52d5f9 100644 --- a/tests/test_web/test_submit.py +++ b/tests/test_web/test_submit.py @@ -108,7 +108,7 @@ class TestUtilsComponentReuse: @mock.patch("module_build_service.web.submit.generate_expanded_mmds") @mock.patch( - "module_build_service.config.Config.scratch_build_only_branches", + "module_build_service.common.config.Config.scratch_build_only_branches", new_callable=mock.PropertyMock, return_value=["^private-.*"], )