From c97a6062ff9ae2e1736d836a27dc20c18e1fc719 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Sep 07 2018 10:24:17 +0000 Subject: Import module API --- diff --git a/README.rst b/README.rst index 0d5b217..3f4c611 100644 --- a/README.rst +++ b/README.rst @@ -667,6 +667,76 @@ parameters include: - ``task_id`` +Import module +------------- + +Importing of modules is done via posting the SCM URL of a repository +which contains the generated modulemd YAML file. Name, stream, version, +context and other important information must be present in the metadata. + +:: + + POST /module-build-service/1/import-module/ + +:: + + { + "scmurl": "git://pkgs.fedoraproject.org/modules/foo.git?#21f92fb05572d81d78fd9a27d313942d45055840" + } + + +If all the module build is imported successfully, JSON containing the +most important information is returned from MBS for each build. The JSON +also contains log messages collected during the import. + +:: + + HTTP 201 Created + +:: + + { + "module": { + "component_builds": [], + "context": "00000000", + "id": 3, + "koji_tag": "", + "name": "mariadb", + "owner": "mbs_import", + "rebuild_strategy": "all", + "scmurl": null, + "siblings": [], + "state": 5, + "state_name": "ready", + "state_reason": null, + "stream": "10.2", + "time_completed": "2018-07-24T12:58:14Z", + "time_modified": "2018-07-24T12:58:14Z", + "time_submitted": "2018-07-24T12:58:14Z", + "version": "20180724000000" + }, + "messages": [ + "'koji_tag' is not set in xmd['mbs'] for module mariadb:10.2:20180724000000:00000000", + "Updating existing module build mariadb:10.2:20180724000000:00000000.", + "Module mariadb:10.2:20180724000000:00000000 imported" + ] + } + + +If the module import fails, an error message is returned. + +:: + + HTTP 422 Unprocessable Entity + +:: + + { + "error": "Unprocessable Entity", + "message": "Incomplete NSVC: None:None:0:00000000" + } + + Listing about ------------- diff --git a/conf/config.py b/conf/config.py index 117d29e..6d92b73 100644 --- a/conf/config.py +++ b/conf/config.py @@ -62,6 +62,8 @@ class BaseConfiguration(object): # 'modularity-wg', ]) + ALLOWED_GROUPS_TO_IMPORT_MODULE = set() + # Available backends are: console and file LOG_BACKEND = 'console' @@ -120,6 +122,8 @@ class TestConfiguration(BaseConfiguration): AUTH_METHOD = 'oidc' RESOLVER = 'db' + ALLOWED_GROUPS_TO_IMPORT_MODULE = set(['mbs-import-module']) + class ProdConfiguration(BaseConfiguration): pass diff --git a/module_build_service/config.py b/module_build_service/config.py index 494ab0b..a6c25d0 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -247,6 +247,10 @@ class Config(object): 'type': set, 'default': set(['packager']), 'desc': 'The set of groups allowed to submit builds.'}, + 'allowed_groups_to_import_module': { + 'type': set, + 'default': set(), + 'desc': 'The set of groups allowed to import module builds.'}, 'log_backend': { 'type': str, 'default': None, @@ -350,7 +354,7 @@ class Config(object): 'yaml_submit_allowed': { 'type': bool, 'default': False, - 'desc': 'Is it allowed to directly submit modulemd yaml file?'}, + 'desc': 'Is it allowed to directly submit build by modulemd yaml file?'}, 'num_concurrent_builds': { 'type': int, 'default': 0, diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 4dee488..8b76139 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -29,7 +29,8 @@ import time from datetime import datetime from module_build_service import conf, log, models -from module_build_service.errors import ValidationError, ProgrammingError +from module_build_service.errors import ( + ValidationError, ProgrammingError, UnprocessableEntity) def scm_url_schemes(terse=False): @@ -258,6 +259,10 @@ def import_mmd(session, mmd): the module, we have no idea what build_context or runtime_context is - we only know the resulting "context", but there is no way to store it into do DB. By now, we just ignore mmd.get_context() and use default 00000000 context instead. + + :return: module build (ModuleBuild), + log messages collected during import (list) + :rtype: tuple """ mmd.set_context("00000000") name = mmd.get_name() @@ -265,22 +270,33 @@ def import_mmd(session, mmd): version = str(mmd.get_version()) context = mmd.get_context() + # Log messages collected during import + msgs = [] + # NSVC is used for logging purpose later. - nsvc = ":".join([name, stream, version, context]) + try: + nsvc = ":".join([name, stream, version, context]) + except TypeError: + msg = "Incomplete NSVC: {}:{}:{}:{}".format(name, stream, version, context) + log.error(msg) + raise UnprocessableEntity(msg) # Get the koji_tag. - xmd = mmd.get_xmd() - if "mbs" in xmd.keys() and "koji_tag" in xmd["mbs"].keys(): + try: + xmd = mmd.get_xmd() koji_tag = xmd["mbs"]["koji_tag"] - else: - log.warn("'koji_tag' is not set in xmd['mbs'] for module %s", nsvc) - koji_tag = "" + except KeyError: + msg = "'koji_tag' is not set in xmd['mbs'] for module {}".format(nsvc) + log.error(msg) + raise UnprocessableEntity(msg) # Get the ModuleBuild from DB. build = models.ModuleBuild.get_build_from_nsvc( session, name, stream, version, context) if build: - log.info("Updating existing module build %s.", nsvc) + msg = "Updating existing module build {}.".format(nsvc) + log.info(msg) + msgs.append(msg) else: build = models.ModuleBuild() @@ -298,4 +314,22 @@ def import_mmd(session, mmd): build.time_completed = datetime.utcnow() session.add(build) session.commit() - log.info("Module %s imported", nsvc) + msg = "Module {} imported".format(nsvc) + log.info(msg) + msgs.append(msg) + + return build, msgs + + +def get_mmd_from_scm(url): + """ + Provided an SCM URL, fetch mmd from the corresponding module YAML + file. If ref is specified within the URL, the mmd will be returned + as of the ref. + """ + from module_build_service.utils.submit import _fetch_mmd + + mmd, _ = _fetch_mmd(url, branch=None, allow_local_url=False, + whitelist_url=False, mandatory_checks=False) + + return mmd diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index d2bcec9..41bd879 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -462,7 +462,8 @@ def _is_eol_in_pdc(name, stream): return not results[0]['active'] -def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): +def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False, + mandatory_checks=True): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. import module_build_service.scm @@ -477,7 +478,8 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): else: scm = module_build_service.scm.SCM(url, branch, conf.scmurls, allow_local_url) scm.checkout(td) - scm.verify() + if mandatory_checks: + scm.verify() cofn = scm.get_module_yaml() mmd = load_mmd(cofn, is_file=True) finally: @@ -494,6 +496,9 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): raise ValidationError( 'Module {}:{} is marked as EOL in PDC.'.format(scm.name, scm.branch)) + if not mandatory_checks: + return mmd, scm + # If the name was set in the modulemd, make sure it matches what the scmurl # says it should be if mmd.get_name() and mmd.get_name() != scm.name: diff --git a/module_build_service/utils/views.py b/module_build_service/utils/views.py index ea0f945..de6acd2 100644 --- a/module_build_service/utils/views.py +++ b/module_build_service/utils/views.py @@ -36,10 +36,14 @@ from .general import scm_url_schemes def get_scm_url_re(): + """ + Returns a regular expression for SCM URL extraction and validation. + """ schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) - return re.compile( - r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" - r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") + regex = ( + r"(?P(?P(?:" + schemes_re + r"))://(?P[^/]+)?" + r"(?P/[^\?]+))(?:\?(?P[^#]+)?)?#(?P.+)") + return re.compile(regex) def pagination_metadata(p_query, api_version, request_args): diff --git a/module_build_service/views.py b/module_build_service/views.py index 38ab8a0..d2d0777 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -36,9 +36,10 @@ from module_build_service import app, conf, log, models, db, version, api_versio from module_build_service.utils import ( pagination_metadata, filter_module_builds, filter_component_builds, submit_module_build_from_scm, submit_module_build_from_yaml, - get_scm_url_re, cors_header, validate_api_version) + get_scm_url_re, cors_header, validate_api_version, import_mmd, + get_mmd_from_scm) from module_build_service.errors import ( - ValidationError, Forbidden, NotFound, ProgrammingError) + ValidationError, Forbidden, NotFound, ProgrammingError, UnprocessableEntity) from module_build_service.backports import jsonify @@ -86,6 +87,12 @@ api_routes = { 'options': { 'methods': ['GET'] } + }, + 'import_module': { + 'url': '/module-build-service//import-module/', + 'options': { + 'methods': ['POST'], + } } } @@ -152,6 +159,12 @@ class ModuleBuildAPI(AbstractQueryableBuildAPI): query_filter = staticmethod(filter_module_builds) model = models.ModuleBuild + @staticmethod + def check_groups(username, groups, allowed_groups=conf.allowed_groups): + if allowed_groups and not (allowed_groups & groups): + raise Forbidden("%s is not in any of %r, only %r" % ( + username, allowed_groups, groups)) + # Additional POST and DELETE handlers for modules follow. @validate_api_version() def post(self, api_version): @@ -163,9 +176,7 @@ class ModuleBuildAPI(AbstractQueryableBuildAPI): if conf.no_auth is True and handler.username == "anonymous" and "owner" in handler.data: handler.username = handler.data["owner"] - if conf.allowed_groups and not (conf.allowed_groups & handler.groups): - raise Forbidden("%s is not in any of %r, only %r" % ( - handler.username, conf.allowed_groups, handler.groups)) + self.check_groups(handler.username, handler.groups) handler.validate() modules = handler.post() @@ -193,9 +204,7 @@ class ModuleBuildAPI(AbstractQueryableBuildAPI): elif username == "anonymous": username = r["owner"] - if conf.allowed_groups and not (conf.allowed_groups & groups): - raise Forbidden("%s is not in any of %r, only %r" % ( - username, conf.allowed_groups, groups)) + self.check_groups(username, groups) module = models.ModuleBuild.query.filter_by(id=id).first() if not module: @@ -268,6 +277,43 @@ class RebuildStrategies(MethodView): return jsonify({'items': items}), 200 +class ImportModuleAPI(MethodView): + + @validate_api_version() + def post(self, api_version): + # disable this API endpoint if no groups are defined + if not conf.allowed_groups_to_import_module: + raise Forbidden(( + "Import module API is disabled. Set 'ALLOWED_GROUPS_TO_IMPORT_MODULE'" + " configuration value first.")) + + # auth checks + username, groups = module_build_service.auth.get_user(request) + ModuleBuildAPI.check_groups(username, groups, + allowed_groups=conf.allowed_groups_to_import_module) + + # scmurl processing + data = SCMHandler.load_data(request) + SCMHandler.check_scmurl(data) + + url = data["scmurl"] + + SCMHandler.check_prefix(url) + SCMHandler.check_url_regex(url) + + mmd = get_mmd_from_scm(url) + + if mmd: + build, messages = import_mmd(db.session, mmd) + json_data = {"module": build.json(show_tasks=False), + "messages": messages} + else: + raise UnprocessableEntity("Nothing to import.") + + # return 201 Created if we reach this point + return jsonify(json_data), 201 + + class BaseHandler(object): def __init__(self, request): self.username, self.groups = module_build_service.auth.get_user(request) @@ -304,27 +350,43 @@ class BaseHandler(object): class SCMHandler(BaseHandler): def __init__(self, request): super(SCMHandler, self).__init__(request) + self.data = self.load_data(request) + + @staticmethod + def load_data(request): try: - self.data = json.loads(request.get_data().decode("utf-8")) + return json.loads(request.get_data().decode("utf-8")) except Exception: log.error('Invalid JSON submitted') raise ValidationError('Invalid JSON submitted') - def validate(self): - if "scmurl" not in self.data: + @staticmethod + def check_scmurl(data): + if "scmurl" not in data: log.error('Missing scmurl') raise ValidationError('Missing scmurl') - url = self.data["scmurl"] + @staticmethod + def check_prefix(url): allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls) if not conf.allow_custom_scmurls and not allowed_prefix: log.error("The submitted scmurl %r is not allowed" % url) raise Forbidden("The submitted scmurl %s is not allowed" % url) + @staticmethod + def check_url_regex(url): if not get_scm_url_re().match(url): log.error("The submitted scmurl %r is not valid" % url) raise Forbidden("The submitted scmurl %s is not valid" % url) + def validate(self): + self.check_scmurl(self.data) + + url = self.data["scmurl"] + + self.check_prefix(url) + self.check_url_regex(url) + if "branch" not in self.data: log.error('Missing branch') raise ValidationError('Missing branch') @@ -369,6 +431,7 @@ def register_api(): component_view = ComponentBuildAPI.as_view('component_builds') about_view = AboutAPI.as_view('about') rebuild_strategies_view = RebuildStrategies.as_view('rebuild_strategies') + import_module = ImportModuleAPI.as_view('import_module') for key, val in api_routes.items(): if key.startswith('component_build'): app.add_url_rule(val['url'], @@ -390,6 +453,11 @@ def register_api(): endpoint=key, view_func=rebuild_strategies_view, **val['options']) + elif key == 'import_module': + app.add_url_rule(val['url'], + endpoint=key, + view_func=import_module, + **val['options']) else: raise NotImplementedError("Unhandled api key.") diff --git a/tests/scm_data/mariadb/HEAD b/tests/scm_data/mariadb/HEAD new file mode 100644 index 0000000..cb089cd --- /dev/null +++ b/tests/scm_data/mariadb/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/tests/scm_data/mariadb/config b/tests/scm_data/mariadb/config new file mode 100644 index 0000000..07d359d --- /dev/null +++ b/tests/scm_data/mariadb/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/tests/scm_data/mariadb/description b/tests/scm_data/mariadb/description new file mode 100644 index 0000000..498b267 --- /dev/null +++ b/tests/scm_data/mariadb/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/tests/scm_data/mariadb/info/exclude b/tests/scm_data/mariadb/info/exclude new file mode 100644 index 0000000..a5196d1 --- /dev/null +++ b/tests/scm_data/mariadb/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/tests/scm_data/mariadb/info/refs b/tests/scm_data/mariadb/info/refs new file mode 100644 index 0000000..0d61723 --- /dev/null +++ b/tests/scm_data/mariadb/info/refs @@ -0,0 +1 @@ +9ab5fdeba83eb3382413ee8bc06299344ef4477d refs/heads/master diff --git a/tests/scm_data/mariadb/objects/info/packs b/tests/scm_data/mariadb/objects/info/packs new file mode 100644 index 0000000..3be3c0b --- /dev/null +++ b/tests/scm_data/mariadb/objects/info/packs @@ -0,0 +1,2 @@ +P pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.pack + diff --git a/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.idx b/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.idx new file mode 100644 index 0000000..9874aa7 Binary files /dev/null and b/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.idx differ diff --git a/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.pack b/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.pack new file mode 100644 index 0000000..32249ad Binary files /dev/null and b/tests/scm_data/mariadb/objects/pack/pack-92fbbdbf4fa07dc9cab035120eb248da930e0bd6.pack differ diff --git a/tests/scm_data/mariadb/packed-refs b/tests/scm_data/mariadb/packed-refs new file mode 100644 index 0000000..8cfe6fa --- /dev/null +++ b/tests/scm_data/mariadb/packed-refs @@ -0,0 +1,2 @@ +# pack-refs with: peeled fully-peeled +9ab5fdeba83eb3382413ee8bc06299344ef4477d refs/heads/master diff --git a/tests/scm_data/mariadb/refs/heads/.placeholder b/tests/scm_data/mariadb/refs/heads/.placeholder new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/scm_data/mariadb/refs/heads/.placeholder diff --git a/tests/scm_data/mariadb/refs/tags/.placeholder b/tests/scm_data/mariadb/refs/tags/.placeholder new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/scm_data/mariadb/refs/tags/.placeholder diff --git a/tests/test_scm.py b/tests/test_scm.py index 0b639f3..d79901d 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -30,7 +30,8 @@ from mock import patch import module_build_service.scm from module_build_service.errors import ValidationError, UnprocessableEntity -repo_path = 'file://' + os.path.dirname(__file__) + "/scm_data/testrepo" +base_dir = os.path.join(os.path.dirname(__file__), 'scm_data') +repo_url = 'file://' + base_dir + '/testrepo' class TestSCMModule: @@ -45,14 +46,14 @@ class TestSCMModule: def test_simple_local_checkout(self): """ See if we can clone a local git repo. """ - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) scm.checkout(self.tempdir) files = os.listdir(self.repodir) assert 'foo' in files, "foo not in %r" % files def test_local_get_latest_is_sane(self): """ See that a hash is returned by scm.get_latest. """ - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) latest = scm.get_latest('master') target = '5481faa232d66589e660cc301179867fb00842c9' assert latest == target, "%r != %r" % (latest, target) @@ -62,7 +63,7 @@ class TestSCMModule: https://pagure.io/fm-orchestrator/issue/329 """ - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) assert scm.scheme == 'git', scm.scheme fname = tempfile.mktemp(suffix='mbs-scm-test') try: @@ -71,70 +72,70 @@ class TestSCMModule: assert not os.path.exists(fname), "%r exists! Vulnerable." % fname def test_local_extract_name(self): - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) target = 'testrepo' assert scm.name == target, '%r != %r' % (scm.name, target) def test_local_extract_name_trailing_slash(self): - scm = module_build_service.scm.SCM(repo_path + '/') + scm = module_build_service.scm.SCM(repo_url + '/') target = 'testrepo' assert scm.name == target, '%r != %r' % (scm.name, target) def test_verify(self): - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) scm.checkout(self.tempdir) scm.verify() def test_verify_unknown_branch(self): with pytest.raises(UnprocessableEntity): - module_build_service.scm.SCM(repo_path, "unknown") + module_build_service.scm.SCM(repo_url, "unknown") def test_verify_commit_in_branch(self): target = '7035bd33614972ac66559ac1fdd019ff6027ad21' - scm = module_build_service.scm.SCM(repo_path + "?#" + target, "dev") + scm = module_build_service.scm.SCM(repo_url + "?#" + target, "dev") scm.checkout(self.tempdir) scm.verify() def test_verify_commit_not_in_branch(self): target = '7035bd33614972ac66559ac1fdd019ff6027ad21' - scm = module_build_service.scm.SCM(repo_path + "?#" + target, "master") + scm = module_build_service.scm.SCM(repo_url + "?#" + target, "master") scm.checkout(self.tempdir) with pytest.raises(ValidationError): scm.verify() def test_verify_unknown_hash(self): target = '7035bd33614972ac66559ac1fdd019ff6027ad22' - scm = module_build_service.scm.SCM(repo_path + "?#" + target, "master") + scm = module_build_service.scm.SCM(repo_url + "?#" + target, "master") with pytest.raises(UnprocessableEntity): scm.checkout(self.tempdir) def test_get_module_yaml(self): - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) scm.checkout(self.tempdir) scm.verify() with pytest.raises(UnprocessableEntity): scm.get_module_yaml() def test_get_latest_incorrect_component_branch(self): - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) with pytest.raises(UnprocessableEntity): scm.get_latest('foobar') def test_get_latest_component_branch(self): ref = "5481faa232d66589e660cc301179867fb00842c9" branch = "master" - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) commit = scm.get_latest(branch) assert commit == ref def test_get_latest_component_ref(self): ref = "5481faa232d66589e660cc301179867fb00842c9" - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) commit = scm.get_latest(ref) assert commit == ref def test_get_latest_incorrect_component_ref(self): - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) with pytest.raises(UnprocessableEntity): scm.get_latest('15481faa232d66589e660cc301179867fb00842c9') @@ -146,6 +147,6 @@ class TestSCMModule: 10a651f39911a07d85fe87fcfe91999545e44ae0\trefs/remotes/origin/master """ mock_run.return_value = (0, output, '') - scm = module_build_service.scm.SCM(repo_path) + scm = module_build_service.scm.SCM(repo_url) commit = scm.get_latest(None) assert commit == '58379ef7887cbc91b215bacd32430628c92bc869' diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 06a5937..7ffc2cd 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -33,6 +33,7 @@ import hashlib import pytest from tests import app, init_data, clean_database, reuse_component_init_data +from tests.test_scm import base_dir as scm_base_dir from module_build_service.errors import UnprocessableEntity from module_build_service.models import ModuleBuild from module_build_service import db, version, Modulemd @@ -43,6 +44,7 @@ import module_build_service.scheduler.handlers.modules user = ('Homer J. Simpson', set(['packager'])) other_user = ('some_other_user', set(['packager'])) anonymous_user = ('anonymous', set(['packager'])) +import_module_user = ('Import M. King', set(['mbs-import-module'])) base_dir = dirname(dirname(__file__)) @@ -1191,3 +1193,177 @@ class TestViews: def test_cors_header_decorator(self): rv = self.client.get('/module-build-service/1/module-builds/') assert rv.headers['Access-Control-Allow-Origin'] == '*' + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=user) + @patch.object(module_build_service.config.Config, 'allowed_groups_to_import_module', + new_callable=PropertyMock, return_value=set()) + def test_import_build_disabled(self, mocked_groups, mocked_get_user, api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post(post_url) + data = json.loads(rv.data) + + assert data['error'] == 'Forbidden' + assert data['message'] == ( + 'Import module API is disabled. Set ' + '\'ALLOWED_GROUPS_TO_IMPORT_MODULE\' configuration value first.') + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=user) + def test_import_build_user_not_allowed(self, mocked_get_user, api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post(post_url) + data = json.loads(rv.data) + + assert data['error'] == 'Forbidden' + assert data['message'] == ( + 'Homer J. Simpson is not in any of ' + 'set([\'mbs-import-module\']), only set([\'packager\'])') + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + def test_import_build_scm_invalid_json(self, mocked_get_user, api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post(post_url, data='') + data = json.loads(rv.data) + + assert data['error'] == 'Bad Request' + assert data['message'] == 'Invalid JSON submitted' + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + def test_import_build_scm_url_not_allowed(self, mocked_get_user, api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + '/mariadb'})) + data = json.loads(rv.data) + + assert data['error'] == 'Forbidden' + assert data['message'].startswith('The submitted scmurl ') + assert data['message'].endswith('/tests/scm_data/mariadb is not allowed') + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'allow_custom_scmurls', + new_callable=PropertyMock, return_value=True) + def test_import_build_scm_url_not_in_list(self, mocked_scmurls, mocked_get_user, + api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#b17bea85de2d03558f24d506578abcfcf467e5bc')})) + data = json.loads(rv.data) + + assert data['error'] == 'Forbidden' + assert data['message'].endswith( + '/tests/scm_data/mariadb?#b17bea85de2d03558f24d506578abcfcf467e5bc ' + 'is not in the list of allowed SCMs') + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'scmurls', + new_callable=PropertyMock, return_value=['file://']) + def test_import_build_scm(self, mocked_scmurls, mocked_get_user, api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#7cf8fb26db8dbfea075eb5f898cc053139960250')})) + data = json.loads(rv.data) + + assert 'Module mariadb:10.2:20180724000000:00000000 imported' in data['messages'] + assert data['module']['name'] == 'mariadb' + assert data['module']['stream'] == '10.2' + assert data['module']['version'] == '20180724000000' + assert data['module']['context'] == '00000000' + assert data['module']['owner'] == 'mbs_import' + assert data['module']['state'] == 5 + assert data['module']['state_reason'] is None + assert data['module']['state_name'] == 'ready' + assert data['module']['scmurl'] is None + assert data['module']['component_builds'] == [] + assert data['module']['time_submitted'] == data['module']['time_modified'] == \ + data['module']['time_completed'] + assert data['module']['koji_tag'] == 'mariadb-10.2-20180724000000-00000000' + assert data['module']['siblings'] == [] + assert data['module']['rebuild_strategy'] == 'all' + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'scmurls', + new_callable=PropertyMock, return_value=['file://']) + def test_import_build_scm_another_commit_hash(self, mocked_scmurls, mocked_get_user, + api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#1a43ea22cd32f235c2f119de1727a37902a49f20')})) + data = json.loads(rv.data) + + assert 'Module mariadb:10.2:20180724065109:00000000 imported' in data['messages'] + assert data['module']['name'] == 'mariadb' + assert data['module']['stream'] == '10.2' + assert data['module']['version'] == '20180724065109' + assert data['module']['context'] == '00000000' + assert data['module']['owner'] == 'mbs_import' + assert data['module']['state'] == 5 + assert data['module']['state_reason'] is None + assert data['module']['state_name'] == 'ready' + assert data['module']['scmurl'] is None + assert data['module']['component_builds'] == [] + assert data['module']['time_submitted'] == data['module']['time_modified'] == \ + data['module']['time_completed'] + assert data['module']['koji_tag'] == 'mariadb-10.2-20180724065109-00000000' + assert data['module']['siblings'] == [] + assert data['module']['rebuild_strategy'] == 'all' + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'scmurls', + new_callable=PropertyMock, return_value=['file://']) + def test_import_build_scm_incomplete_nsvc(self, mocked_scmurls, mocked_get_user, + api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#b17bea85de2d03558f24d506578abcfcf467e5bc')})) + data = json.loads(rv.data) + + assert data['error'] == 'Unprocessable Entity' + assert data['message'] == 'Incomplete NSVC: None:None:0:00000000' + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'scmurls', + new_callable=PropertyMock, return_value=['file://']) + def test_import_build_scm_yaml_is_bad(self, mocked_scmurls, mocked_get_user, + api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#cb7cf7069059141e0797ad2cf5a559fb673ef43d')})) + data = json.loads(rv.data) + + assert data['error'] == 'Unprocessable Entity' + assert data['message'].startswith('The following invalid modulemd was encountered') + + @pytest.mark.parametrize('api_version', [1, 2]) + @patch('module_build_service.auth.get_user', return_value=import_module_user) + @patch.object(module_build_service.config.Config, 'scmurls', + new_callable=PropertyMock, return_value=['file://']) + def test_import_build_scm_missing_koji_tag(self, mocked_scmurls, mocked_get_user, + api_version): + post_url = '/module-build-service/{0}/import-module/'.format(api_version) + rv = self.client.post( + post_url, + data=json.dumps({'scmurl': 'file://' + scm_base_dir + ( + '/mariadb?#9ab5fdeba83eb3382413ee8bc06299344ef4477d')})) + data = json.loads(rv.data) + + assert data['error'] == 'Unprocessable Entity' + assert data['message'].startswith('\'koji_tag\' is not set in xmd[\'mbs\'] for module')