From 49030dff19f9fc3fb2ac0eb8fb87ee96fd562da6 Mon Sep 17 00:00:00 2001 From: Lukas Holecek Date: Jun 27 2018 12:41:45 +0000 Subject: [PATCH 1/2] Move functions from utils to avoid circular dependencies --- diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index e5c9084..475f25c 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -1,13 +1,59 @@ # SPDX-License-Identifier: GPL-2.0+ +import os +import logging + from flask import Flask +from flask.config import Config from greenwave.api_v1 import api -from greenwave.utils import json_error, load_config, sha1_mangle_key +from greenwave.policies import load_policies +from greenwave.utils import json_error, sha1_mangle_key from dogpile.cache import make_region from requests import ConnectionError, Timeout from werkzeug.exceptions import default_exceptions +log = logging.getLogger(__name__) + + +def load_config(config_obj=None): + """ + Load Greenwave configuration. It will load the configuration based on how the environment is + configured. + :return: A dict of Greenwave configuration. + """ + # Load default config, then override that with a config file + config = Config(__name__) + if config_obj is None: + if os.getenv('TEST') == 'true': + config_obj = 'greenwave.config.TestingConfig' + elif os.getenv('DEV') == 'true': + config_obj = 'greenwave.config.DevelopmentConfig' + else: + config_obj = 'greenwave.config.ProductionConfig' + + if os.getenv('TEST') == 'true': + default_config_file = os.getcwd() + '/conf/settings.py.example' + elif os.getenv('DEV') == 'true': + default_config_file = os.getcwd() + '/conf/settings.py' + else: + default_config_file = '/etc/greenwave/settings.py' + + log.debug("config: Loading config from %r", config_obj) + config.from_object(config_obj) + + config_file = os.environ.get('GREENWAVE_CONFIG', default_config_file) + log.debug("config: Extending config with %r", config_file) + config.from_pyfile(config_file) + + if os.environ.get('SECRET_KEY'): + config['SECRET_KEY'] = os.environ['SECRET_KEY'] + + log.debug("config: Loading policies from %r", config['POLICIES_DIR']) + config['policies'] = load_policies(config['POLICIES_DIR']) + + return config + # applicaiton factory http://flask.pocoo.org/docs/0.12/patterns/appfactories/ def create_app(config_obj=None): diff --git a/greenwave/policies.py b/greenwave/policies.py index 0f1166c..ba3cd1b 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0+ from fnmatch import fnmatch -import yaml +import glob import logging +import os +import yaml import greenwave.resources log = logging.getLogger(__name__) @@ -32,6 +34,23 @@ def validate_policies(policies, disallowed_rules=None): 'is an instance of %s' % (rule, disallowed_rule)) +def load_policies(policies_dir): + """ + Load Greenwave policies from the given policies directory. + + :param str policies_dir: A path points to the policies directory. + :return: A list of policies. + + """ + policy_pathnames = glob.glob(os.path.join(policies_dir, '*.yaml')) + policies = [] + for policy_pathname in policy_pathnames: + policies.extend(yaml.safe_load_all(open(policy_pathname, 'r'))) + validate_policies(policies) + log.debug("Loaded %i policies from %s", len(policies), policies_dir) + return policies + + def subject_type_identifier_to_item(subject_type, subject_identifier): """ Greenwave < 0.8 included an "item" key in the "unsatisfied_requirements". diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index 7d7cb41..833e2b0 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -12,7 +12,7 @@ from greenwave.policies import ( TestResultFailed, InvalidGatingYaml ) -from greenwave.utils import load_policies +from greenwave.policies import load_policies def test_summarize_answers(): diff --git a/greenwave/utils.py b/greenwave/utils.py index 66b4417..34254b7 100644 --- a/greenwave/utils.py +++ b/greenwave/utils.py @@ -1,18 +1,13 @@ # SPDX-License-Identifier: GPL-2.0+ import functools -import glob import logging -import os import time import hashlib -import yaml from flask import jsonify, current_app, request -from flask.config import Config from requests import ConnectionError, Timeout from werkzeug.exceptions import HTTPException -import greenwave.policies log = logging.getLogger(__name__) @@ -69,62 +64,6 @@ def jsonp(func): return wrapped -def load_config(config_obj=None): - """ - Load Greenwave configuration. It will load the configuration based on how the environment is - configured. - :return: A dict of Greenwave configuration. - """ - # Load default config, then override that with a config file - config = Config(__name__) - if config_obj is None: - if os.getenv('TEST') == 'true': - config_obj = 'greenwave.config.TestingConfig' - elif os.getenv('DEV') == 'true': - config_obj = 'greenwave.config.DevelopmentConfig' - else: - config_obj = 'greenwave.config.ProductionConfig' - - if os.getenv('TEST') == 'true': - default_config_file = os.getcwd() + '/conf/settings.py.example' - elif os.getenv('DEV') == 'true': - default_config_file = os.getcwd() + '/conf/settings.py' - else: - default_config_file = '/etc/greenwave/settings.py' - - log.debug("config: Loading config from %r", config_obj) - config.from_object(config_obj) - - config_file = os.environ.get('GREENWAVE_CONFIG', default_config_file) - log.debug("config: Extending config with %r", config_file) - config.from_pyfile(config_file) - - if os.environ.get('SECRET_KEY'): - config['SECRET_KEY'] = os.environ['SECRET_KEY'] - - log.debug("config: Loading policies from %r", config['POLICIES_DIR']) - config['policies'] = load_policies(config['POLICIES_DIR']) - - return config - - -def load_policies(policies_dir): - """ - Load Greenwave policies from the given policies directory. - - :param str policies_dir: A path points to the policies directory. - :return: A list of policies. - - """ - policy_pathnames = glob.glob(os.path.join(policies_dir, '*.yaml')) - policies = [] - for policy_pathname in policy_pathnames: - policies.extend(yaml.safe_load_all(open(policy_pathname, 'r'))) - greenwave.policies.validate_policies(policies) - log.debug("Loaded %i policies from %s", len(policies), policies_dir) - return policies - - def insert_headers(response): """ Insert the CORS headers for the give reponse if there are any configured for the application. From d04d8f611b82afed000944b9fb795c7704092c73 Mon Sep 17 00:00:00 2001 From: Lukas Holecek Date: Jun 27 2018 13:29:24 +0000 Subject: [PATCH 2/2] tests: Wait for services set up using *_TEST_URL variables --- diff --git a/functional-tests/conftest.py b/functional-tests/conftest.py index 3e258cf..7972c1e 100644 --- a/functional-tests/conftest.py +++ b/functional-tests/conftest.py @@ -2,7 +2,6 @@ import os import sys -import time import textwrap import itertools import json @@ -13,8 +12,10 @@ import pytest import requests from contextlib import contextmanager from sqlalchemy import create_engine +from urllib.parse import urlparse from greenwave.logger import init_logging +from greenwave.utils import retry log = logging.getLogger(__name__) @@ -43,18 +44,19 @@ def drop_and_create_database(dbname): engine.dispose() -def wait_for_listen(port): +def wait_for_listen(address, port, timeout, interval): """ Waits until something is listening on the given TCP port. """ - for attempt in range(50): - try: - s = socket.create_connection(('127.0.0.1', port), timeout=1) - s.close() - return - except socket.error: - time.sleep(0.1) - raise RuntimeError('Gave up waiting for port %s' % port) + @retry(timeout=timeout, interval=interval, wait_on=socket.error) + def _wait_for_listen(): + s = socket.create_connection((address, port), timeout=1) + s.close() + + try: + _wait_for_listen() + except socket.error: + raise RuntimeError('Gave up waiting for {}:{}'.format(address, port)) @contextmanager @@ -72,7 +74,14 @@ def server_subprocess( # creating test process. test_url_env_var = env_var_prefix + '_TEST_URL' if test_url_env_var in os.environ: - yield os.environ[test_url_env_var] + url = os.environ[test_url_env_var] + url_components = urlparse(url) + address = url_components.hostname + port = url_components.port or 80 + assert address is not None + assert port is not None + wait_for_listen(address, port, timeout=30, interval=1) + yield url return if source_path is None: @@ -106,7 +115,7 @@ def server_subprocess( # Start server with subprocess.Popen(start_server_arguments, **subprocess_arguments) as p: log.debug('Started %s server as pid %s', name, p.pid) - wait_for_listen(port) + wait_for_listen('127.0.0.1', port, timeout=5, interval=0.1) yield 'http://localhost:{}/'.format(port) diff --git a/greenwave/utils.py b/greenwave/utils.py index 34254b7..5d4b8fd 100644 --- a/greenwave/utils.py +++ b/greenwave/utils.py @@ -94,8 +94,6 @@ def retry(timeout=None, interval=None, wait_on=Exception): try: return function(*args, **kwargs) except wait_on as e: # pylint: disable=broad-except - log.warning("Exception %r raised from %r. Retry in %rs", - e, function, _interval) time.sleep(_interval) if (time.time() - start) >= _timeout: raise # This re-raises the last exception.