From a76bdfa464e806300ba2113005f10a04a22751e5 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Jun 04 2018 15:52:53 +0000 Subject: Enable grabbing secrets from Vault server --- diff --git a/data/ansible/runner.yml b/data/ansible/runner.yml index 69c4cb6..321ae2a 100644 --- a/data/ansible/runner.yml +++ b/data/ansible/runner.yml @@ -81,6 +81,12 @@ register: result until: result.rc == 0 + - name: Upload secrets + copy: + src: '{{taskotron_secrets_file}}' + dest: '{{taskotron_secrets_file}}' + when: not local + - name: Start heartbeat command: > ./heartbeat.sh start {{ heartbeat_file }} {{ heartbeat_interval }} @@ -95,6 +101,15 @@ static: yes # variable 'task' is registered here + - name: Delete secrets + file: + state: absent + path: '{{taskotron_secrets_file}}' + tags: + - failsafe + when: not local + + - name: Kill heartbeat command: ./heartbeat.sh stop {{ heartbeat_file }} when: taskotron_keepalive_minutes|int > 0 diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py index ebd569c..54c50f9 100644 --- a/libtaskotron/config_defaults.py +++ b/libtaskotron/config_defaults.py @@ -74,6 +74,10 @@ class Config(object): taskotron_master = 'http://localhost/taskmaster' #: artifacts_baseurl = 'http://localhost/artifacts' #: download_cache_enabled = True #: + vault_enabled = False #: + vault_server = 'http://localhost:4999/api/v1' #: + vault_username = 'taskotron' #: + vault_password = 'taskotron' #: tmpdir = '/var/tmp/taskotron' #: logdir = '/var/log/taskotron' #: diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index 469f2ad..9fb226c 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -109,6 +109,18 @@ summary will be printed out into the log, like this:: directive_class = 'ResultsdbDirective' +def git_origin_url(taskdir): + try: + gitconfig_filename = os.path.join(taskdir, '.git/config') + gitconfig = configparser.ConfigParser() + gitconfig.read(gitconfig_filename) + task_repo_url = gitconfig['remote "origin"']['url'] + except TypeError as e: + log.exception(e) + task_repo_url = None + return task_repo_url + + class ResultsdbDirective(BaseDirective): @@ -217,17 +229,11 @@ class ResultsdbDirective(BaseDirective): else: raise TaskotronDirectiveError('No namespace for task %s exists.' % checkname) - try: - taskdir = os.path.dirname(os.path.abspath(arg_data['task'])) - gitconfig_filename = os.path.join(taskdir, '.git/config') - - gitconfig = configparser.ConfigParser() - gitconfig.read(gitconfig_filename) - task_repo_url = gitconfig['remote "origin"']['url'] - except TypeError as e: - log.exception(e) + taskdir = os.path.dirname(os.path.abspath(arg_data['task'])) + task_repo_url = git_origin_url(taskdir) + if not task_repo_url: raise TaskotronDirectiveError("Could not find task's git remote 'origin' url" - "in %s" % gitconfig_filename) + "in %s" % os.path.join(taskdir, '.git/config')) try: if not [ns_repo for ns_repo in ns_repos if task_repo_url.strip().startswith(ns_repo)]: diff --git a/libtaskotron/executor.py b/libtaskotron/executor.py index 3d0aa0a..7378914 100644 --- a/libtaskotron/executor.py +++ b/libtaskotron/executor.py @@ -12,6 +12,10 @@ import yaml import fnmatch import signal import json +import tempfile +import configparser +import requests +import re from libtaskotron import config from libtaskotron import image_utils @@ -181,6 +185,48 @@ class Executor(object): vars_[var] = val return vars_ + def _get_vault_secrets(self, taskdir): + '''Load secrets from the Vault server and store them in a file + + :param str taskdir: path to the directory with test suite (on overlord) + :return: a filename with decrypted secrets + ''' + cfg = config.get_config() + secrets = {} + if cfg.vault_enabled: + task_repo_url = resultsdb_directive.git_origin_url(taskdir) + if task_repo_url: + re_enabler = re.compile(r'taskotron_enable\(([^)]*)\)') + try: + r = requests.get( + "%s/buckets" % cfg.vault_server, + auth=(cfg.vault_username, cfg.vault_password), + ) + except requests.exceptions.RequestException: + r = None + if r and r.ok: + data = r.json()['data'] + valid_buckets = [] + for b in data: + desc = b['description'] + if not desc: + continue + enabled_for = ', '.join(re_enabler.findall(desc)) + if not task_repo_url in enabled_for: + continue + valid_buckets.append(b) + for b in valid_buckets: + secrets[b['uuid']] = b['secrets'] + + if config.get_config().profile == config.ProfileName.TESTING: + return secrets + + fd, fname = tempfile.mkstemp(prefix='taskotron_secrets') + os.close(fd) + with open(fname, 'w') as fd: + fd.write(json.dumps(secrets, indent=2, sort_keys=True)) + return fname + def _create_playbook_vars(self, test_playbook): '''Create and return dictionary containing all variables to be used with our ansible playbook. @@ -230,6 +276,8 @@ class Executor(object): whether VM guest distro has to match taskotron_item taskotron_match_host_release whether VM guest release has to match taskotron_item + taskotron_secrets_file + path to the file with secrets appropriate for the task taskotron_supported_arches list of base architectures supported by Taskotron (e.g. 'armhfp') @@ -274,6 +322,7 @@ class Executor(object): vars_['taskotron_arch'] = self.arg_data['arch'] vars_['taskotron_item'] = self.arg_data['item'] vars_['taskotron_item_type'] = self.arg_data['type'] + vars_['taskotron_secrets_file'] = self._get_vault_secrets(taskdir=vars_['taskdir']) vars_['taskotron_supported_arches'] = cfg.supported_arches vars_['taskotron_supported_binary_arches'] = [binarch for arch in cfg.supported_arches for binarch in arch_utils.Arches.binary[arch]] @@ -446,6 +495,7 @@ class Executor(object): failed = [] for test_playbook in test_playbooks: + playbook_vars = None try: # syntax check self._check_playbook_syntax(os.path.join( @@ -487,7 +537,12 @@ class Executor(object): break else: self.task_vm.teardown() - + try: + if playbook_vars and config.get_config().profile != config.ProfileName.TESTING: + os.remove(playbook_vars['taskotron_secrets_file']) + except OSError: + log.info("Could not delete the secrets file at %r", + playbook_vars['taskotron_secrets_file']) log.info('Playbook execution finished: %s', test_playbook) if failed: diff --git a/testing/functest_executor.py b/testing/functest_executor.py index a7c2a48..40771ee 100644 --- a/testing/functest_executor.py +++ b/testing/functest_executor.py @@ -89,3 +89,44 @@ class TestExecutor(): assert len(matched) == 1 warn_task_line = lines.index(matched[0]) assert 'skipping' in lines[warn_task_line + 1] + + def test_get_vault_secrets(self, monkeypatch): + class MockRequests(object): + def __init__(self, retval): + self.ok = True + self.retval = retval + self.exceptions = mock.Mock() + def get(self, *args, **kwargs): + return self + def json(self, *args, **kwargs): + return self.retval + + self.conf.vault_enabled = True + + mock_git_origin_url = mock.Mock(return_value='fake://repo.git') + monkeypatch.setattr(executor.resultsdb_directive, 'git_origin_url', mock_git_origin_url) + + mock_requests = MockRequests({'data': [ + { + 'description': 'dat description', + 'secrets': 'dem secrets', + 'uuid': 'dat uuid', + } + ]}) + monkeypatch.setattr(executor, 'requests', mock_requests) + secrets = self.executor._get_vault_secrets(taskdir=None) + assert secrets == {} + + + mock_requests = MockRequests({'data': [ + { + 'description': 'dat_description, taskotron_enable(fake://repo.git)', + 'secrets': 'dem_secrets', + 'uuid': 'dat uuid', + } + ]}) + monkeypatch.setattr(executor, 'requests', mock_requests) + secrets = self.executor._get_vault_secrets(taskdir=None) + assert secrets == {'dat uuid': 'dem_secrets'} + +