#1004 Import module API
Closed 5 years ago by fivaldi. Opened 5 years ago by fivaldi.

Import module API
Filip Valder • 5 years ago  
file modified
+75
@@ -667,6 +667,81 @@ 

  - ``task_id``

  

  

+ Import module

+ -------------

+ 

+ Importing of modules is done via posting the SCM URL of a repository

+ which contains the generated modulemd YAML file. Name, stream, version,

+ context and other important information must be present in the metadata.

+ 

+ ::

+ 

+     POST /module-build-service/1/import-module/

+ 

+ ::

+ 

+     {

+       "scmurl": "git://pkgs.fedoraproject.org/modules/foo.git?#21f92fb05572d81d78fd9a27d313942d45055840"

+     }

+ 

+ 

+ If all the module build is imported successfully, JSON containing the

+ most important information is returned from MBS for each build. The JSON

+ also contains log messages collected during the import.

+ 

+ ::

+ 

+     HTTP 201 Created

+ 

+ ::

+ 

+     {

+       "items": [

+         {

+           "module": {

+             "component_builds": [],

+             "context": "00000000",

+             "id": 3,

+             "koji_tag": "",

+             "name": "mariadb",

+             "owner": "mbs_import",

+             "rebuild_strategy": "all",

+             "scmurl": null,

+             "siblings": [],

+             "state": 5,

+             "state_name": "ready",

+             "state_reason": null,

+             "stream": "10.2",

+             "time_completed": "2018-07-24T12:58:14Z",

+             "time_modified": "2018-07-24T12:58:14Z",

+             "time_submitted": "2018-07-24T12:58:14Z",

+             "version": "20180724000000"

+           },

+           "messages": [

+             "'koji_tag' is not set in xmd['mbs'] for module mariadb:10.2:20180724000000:00000000",

+             "Updating existing module build mariadb:10.2:20180724000000:00000000.",

+             "Module mariadb:10.2:20180724000000:00000000 imported"

+           ]

+         },

+         ...

+       ]

+     }

+ 

+ 

+ If the module import fails, an error message is returned.

+ 

+ ::

+ 

+     HTTP 422 Unprocessable Entity

+ 

+ ::

+ 

+     {

+       "error": "Unprocessable Entity",

+       "message": "Incomplete NSVC: None:None:0:00000000"

+     }

+ 

+ 

  Listing about

  -------------

  

file modified
+4
@@ -61,6 +61,8 @@ 

          # 'modularity-wg',

      ])

  

+     ALLOWED_GROUPS_TO_IMPORT_MODULE = set()

+ 

      # Available backends are: console and file

      LOG_BACKEND = 'console'

  
@@ -119,6 +121,8 @@ 

      AUTH_METHOD = 'oidc'

      RESOLVER = 'db'

  

+     ALLOWED_GROUPS_TO_IMPORT_MODULE = set(['mbs-import-module'])

+ 

  

  class ProdConfiguration(BaseConfiguration):

      pass

@@ -239,6 +239,10 @@ 

              'type': set,

              'default': set(['packager']),

              'desc': 'The set of groups allowed to submit builds.'},

+         'allowed_groups_to_import_module': {

+             'type': set,

+             'default': set(),

+             'desc': 'The set of groups allowed to import module builds.'},

          'log_backend': {

              'type': str,

              'default': None,
@@ -342,7 +346,7 @@ 

          'yaml_submit_allowed': {

              'type': bool,

              'default': False,

-             'desc': 'Is it allowed to directly submit modulemd yaml file?'},

+             'desc': 'Is it allowed to directly submit build by modulemd yaml file?'},

          'num_concurrent_builds': {

              'type': int,

              'default': 0,

@@ -29,7 +29,8 @@ 

  from datetime import datetime

  

  from module_build_service import conf, log, models

- from module_build_service.errors import ValidationError, ProgrammingError

+ from module_build_service.errors import (

+     ValidationError, ProgrammingError, UnprocessableEntity)

  

  

  def scm_url_schemes(terse=False):
@@ -258,6 +259,10 @@ 

      the module, we have no idea what build_context or runtime_context is - we only

      know the resulting "context", but there is no way to store it into do DB.

      By now, we just ignore mmd.get_context() and use default 00000000 context instead.

+ 

+     :return: module build (ModuleBuild),

+              log messages collected during import (list)

+     :rtype: tuple

      """

      mmd.set_context("00000000")

      name = mmd.get_name()
@@ -265,22 +270,33 @@ 

      version = str(mmd.get_version())

      context = mmd.get_context()

  

+     # Log messages collected during import

+     msgs = []

+ 

      # NSVC is used for logging purpose later.

-     nsvc = ":".join([name, stream, version, context])

+     try:

+         nsvc = ":".join([name, stream, version, context])

+     except TypeError:

+         msg = "Incomplete NSVC: {}:{}:{}:{}".format(name, stream, version, context)

+         log.error(msg)

+         raise UnprocessableEntity(msg)

  

      # Get the koji_tag.

-     xmd = mmd.get_xmd()

-     if "mbs" in xmd.keys() and "koji_tag" in xmd["mbs"].keys():

+     try:

+         xmd = mmd.get_xmd()

          koji_tag = xmd["mbs"]["koji_tag"]

-     else:

-         log.warn("'koji_tag' is not set in xmd['mbs'] for module %s", nsvc)

-         koji_tag = ""

+     except KeyError:

+         msg = "'koji_tag' is not set in xmd['mbs'] for module {}".format(nsvc)

+         log.error(msg)

+         raise UnprocessableEntity(msg)

  

      # Get the ModuleBuild from DB.

      build = models.ModuleBuild.get_build_from_nsvc(

          session, name, stream, version, context)

      if build:

-         log.info("Updating existing module build %s.", nsvc)

+         msg = "Updating existing module build {}.".format(nsvc)

+         log.info(msg)

+         msgs.append(msg)

      else:

          build = models.ModuleBuild()

  
@@ -298,4 +314,22 @@ 

      build.time_completed = datetime.utcnow()

      session.add(build)

      session.commit()

-     log.info("Module %s imported", nsvc)

+     msg = "Module {} imported".format(nsvc)

+     log.info(msg)

+     msgs.append(msg)

+ 

+     return build, msgs

+ 

+ 

+ def get_mmd_from_scm(url):

+     """

+     Provided an SCM URL, fetch mmd from the corresponding module YAML

+     file. If ref is specified within the URL, the mmd will be returned

+     as of the ref.

+     """

+     from module_build_service.utils.submit import _fetch_mmd

+ 

+     mmd, _ = _fetch_mmd(url, branch=None, allow_local_url=False,

+                         whitelist_url=False, mandatory_checks=False)

+ 

+     return mmd

@@ -440,7 +440,8 @@ 

      return modules

  

  

- def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):

+ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False,

+                mandatory_checks=True):

      # Import it here, because SCM uses utils methods

      # and fails to import them because of dep-chain.

      import module_build_service.scm
@@ -455,7 +456,8 @@ 

          else:

              scm = module_build_service.scm.SCM(url, branch, conf.scmurls, allow_local_url)

          scm.checkout(td)

-         scm.verify()

+         if mandatory_checks:

+             scm.verify()

          cofn = scm.get_module_yaml()

          mmd = load_mmd(cofn, is_file=True)

      finally:
@@ -467,6 +469,9 @@ 

                  "Failed to remove temporary directory {!r}: {}".format(

                      td, str(e)))

  

+     if not mandatory_checks:

+         return mmd, scm

+ 

      # If the name was set in the modulemd, make sure it matches what the scmurl

      # says it should be

      if mmd.get_name() and mmd.get_name() != scm.name:

@@ -36,10 +36,14 @@ 

  

  

  def get_scm_url_re():

+     """

+     Returns a regular expression for SCM URL extraction and validation.

+     """

      schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True)))

-     return re.compile(

-         r"(?P<giturl>(?:(?P<scheme>(" + schemes_re + r"))://(?P<host>[^/]+))?"

-         r"(?P<repopath>/[^\?]+))\?(?P<modpath>[^#]*)#(?P<revision>.+)")

+     regex = (

+         r"(?P<giturl>(?P<scheme>(?:" + schemes_re + r"))://(?P<host>[^/]+)?"

+         r"(?P<repopath>/[^\?]+))(?:\?(?P<modpath>[^#]+)?)?#(?P<revision>.+)")

+     return re.compile(regex)

  

  

  def pagination_metadata(p_query, api_version, request_args):

file modified
+83 -12
@@ -36,9 +36,10 @@ 

  from module_build_service.utils import (

      pagination_metadata, filter_module_builds, filter_component_builds,

      submit_module_build_from_scm, submit_module_build_from_yaml,

-     get_scm_url_re, cors_header, validate_api_version)

+     get_scm_url_re, cors_header, validate_api_version, import_mmd,

+     get_mmd_from_scm)

  from module_build_service.errors import (

-     ValidationError, Forbidden, NotFound, ProgrammingError)

+     ValidationError, Forbidden, NotFound, ProgrammingError, UnprocessableEntity)

  from module_build_service.backports import jsonify

  

  
@@ -86,6 +87,12 @@ 

          'options': {

              'methods': ['GET']

          }

+     },

+     'import_module': {

+         'url': '/module-build-service/<int:api_version>/import-module/',

+         'options': {

+             'methods': ['POST'],

+         }

      }

  }

  
@@ -152,6 +159,12 @@ 

      query_filter = staticmethod(filter_module_builds)

      model = models.ModuleBuild

  

+     @staticmethod

+     def check_groups(username, groups, allowed_groups=conf.allowed_groups):

+         if allowed_groups and not (allowed_groups & groups):

+             raise Forbidden("%s is not in any of %r, only %r" % (

+                 username, allowed_groups, groups))

+ 

      # Additional POST and DELETE handlers for modules follow.

      @validate_api_version()

      def post(self, api_version):
@@ -163,9 +176,7 @@ 

          if conf.no_auth is True and handler.username == "anonymous" and "owner" in handler.data:

              handler.username = handler.data["owner"]

  

-         if conf.allowed_groups and not (conf.allowed_groups & handler.groups):

-             raise Forbidden("%s is not in any of  %r, only %r" % (

-                 handler.username, conf.allowed_groups, handler.groups))

+         self.check_groups(handler.username, handler.groups)

  

          handler.validate()

          modules = handler.post()
@@ -193,9 +204,7 @@ 

              elif username == "anonymous":

                  username = r["owner"]

  

-         if conf.allowed_groups and not (conf.allowed_groups & groups):

-             raise Forbidden("%s is not in any of  %r, only %r" % (

-                 username, conf.allowed_groups, groups))

+         self.check_groups(username, groups)

  

          module = models.ModuleBuild.query.filter_by(id=id).first()

          if not module:
@@ -268,6 +277,46 @@ 

          return jsonify({'items': items}), 200

  

  

+ class ImportModuleAPI(MethodView):

+ 

+     @validate_api_version()

+     def post(self, api_version):

+         # disable this API endpoint if no groups are defined

+         if not conf.allowed_groups_to_import_module:

+             raise Forbidden((

+                 "Import module API is disabled. Set 'ALLOWED_GROUPS_TO_IMPORT_MODULE'"

+                 " configuration value first."))

+ 

+         # auth checks

+         username, groups = module_build_service.auth.get_user(request)

+         ModuleBuildAPI.check_groups(username, groups,

+                                     allowed_groups=conf.allowed_groups_to_import_module)

+ 

+         # prepare response

+         json_data = {}

+ 

+         # scmurl processing

+         data = SCMHandler.load_data(request)

+         SCMHandler.check_scmurl(data)

+ 

+         url = data["scmurl"]

+ 

+         SCMHandler.check_prefix(url)

+         SCMHandler.check_url_regex(url)

+ 

+         mmd = get_mmd_from_scm(url)

+ 

+         if mmd:

+             build, messages = import_mmd(db.session, mmd)

+             json_data["items"] = {"module": build.json(show_tasks=False),

+                                   "messages": messages}

+         else:

+             raise UnprocessableEntity("Nothing to import.")

+ 

+         # return 201 Created if we reach this point

+         return jsonify(json_data), 201

+ 

+ 

  class BaseHandler(object):

      def __init__(self, request):

          self.username, self.groups = module_build_service.auth.get_user(request)
@@ -304,27 +353,43 @@ 

  class SCMHandler(BaseHandler):

      def __init__(self, request):

          super(SCMHandler, self).__init__(request)

+         self.data = self.load_data(request)

+ 

+     @staticmethod

+     def load_data(request):

          try:

-             self.data = json.loads(request.get_data().decode("utf-8"))

+             return json.loads(request.get_data().decode("utf-8"))

          except Exception:

              log.error('Invalid JSON submitted')

              raise ValidationError('Invalid JSON submitted')

  

-     def validate(self):

-         if "scmurl" not in self.data:

+     @staticmethod

+     def check_scmurl(data):

+         if "scmurl" not in data:

              log.error('Missing scmurl')

              raise ValidationError('Missing scmurl')

  

-         url = self.data["scmurl"]

+     @staticmethod

+     def check_prefix(url):

          allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls)

          if not conf.allow_custom_scmurls and not allowed_prefix:

              log.error("The submitted scmurl %r is not allowed" % url)

              raise Forbidden("The submitted scmurl %s is not allowed" % url)

  

+     @staticmethod

+     def check_url_regex(url):

          if not get_scm_url_re().match(url):

              log.error("The submitted scmurl %r is not valid" % url)

              raise Forbidden("The submitted scmurl %s is not valid" % url)

  

+     def validate(self):

+         self.check_scmurl(self.data)

+ 

+         url = self.data["scmurl"]

+ 

+         self.check_prefix(url)

+         self.check_url_regex(url)

+ 

          if "branch" not in self.data:

              log.error('Missing branch')

              raise ValidationError('Missing branch')
@@ -369,6 +434,7 @@ 

      component_view = ComponentBuildAPI.as_view('component_builds')

      about_view = AboutAPI.as_view('about')

      rebuild_strategies_view = RebuildStrategies.as_view('rebuild_strategies')

+     import_module = ImportModuleAPI.as_view('import_module')

      for key, val in api_routes.items():

          if key.startswith('component_build'):

              app.add_url_rule(val['url'],
@@ -390,6 +456,11 @@ 

                               endpoint=key,

                               view_func=rebuild_strategies_view,

                               **val['options'])

+         elif key == 'import_module':

+             app.add_url_rule(val['url'],

+                              endpoint=key,

+                              view_func=import_module,

+                              **val['options'])

          else:

              raise NotImplementedError("Unhandled api key.")

  

@@ -0,0 +1,1 @@ 

+ ref: refs/heads/master

@@ -0,0 +1,8 @@ 

+ [core]

+ 	repositoryformatversion = 0

+ 	filemode = true

+ 	bare = true

+ [remote "origin"]

+ 	url = ssh://fivaldi@pkgs.fedoraproject.org/forks/fivaldi/modules/mariadb.git

+ 	fetch = +refs/*:refs/*

+ 	mirror = true

@@ -0,0 +1,1 @@ 

+ Unnamed repository; edit this file 'description' to name the repository.

@@ -0,0 +1,6 @@ 

+ # git ls-files --others --exclude-from=.git/info/exclude

+ # Lines that start with '#' are comments.

+ # For a project mostly in C, the following would be a good set of

+ # exclude patterns (uncomment them if you want to use them):

+ # *.[oa]

+ # *~

@@ -0,0 +1,2 @@ 

+ P pack-8854844abfb6597cba3636931a874d809bf91ee7.pack

+ 

@@ -0,0 +1,2 @@ 

+ # pack-refs with: peeled fully-peeled 

+ c7b473af5cd85f29504c874e3308f2c1cae098fa refs/heads/master

@@ -0,0 +1,1 @@ 

+ c7b473af5cd85f29504c874e3308f2c1cae098fa

file modified
+18 -17
@@ -30,7 +30,8 @@ 

  import module_build_service.scm

  from module_build_service.errors import ValidationError, UnprocessableEntity

  

- repo_path = 'file://' + os.path.dirname(__file__) + "/scm_data/testrepo"

+ base_dir = os.path.join(os.path.dirname(__file__), 'scm_data')

+ repo_url = 'file://' + base_dir + '/testrepo'

  

  

  class TestSCMModule:
@@ -45,14 +46,14 @@ 

  

      def test_simple_local_checkout(self):

          """ See if we can clone a local git repo. """

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          scm.checkout(self.tempdir)

          files = os.listdir(self.repodir)

          assert 'foo' in files, "foo not in %r" % files

  

      def test_local_get_latest_is_sane(self):

          """ See that a hash is returned by scm.get_latest. """

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          latest = scm.get_latest('master')

          target = '5481faa232d66589e660cc301179867fb00842c9'

          assert latest == target, "%r != %r" % (latest, target)
@@ -62,7 +63,7 @@ 

  

          https://pagure.io/fm-orchestrator/issue/329

          """

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          assert scm.scheme == 'git', scm.scheme

          fname = tempfile.mktemp(suffix='mbs-scm-test')

          try:
@@ -71,70 +72,70 @@ 

              assert not os.path.exists(fname), "%r exists!  Vulnerable." % fname

  

      def test_local_extract_name(self):

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          target = 'testrepo'

          assert scm.name == target, '%r != %r' % (scm.name, target)

  

      def test_local_extract_name_trailing_slash(self):

-         scm = module_build_service.scm.SCM(repo_path + '/')

+         scm = module_build_service.scm.SCM(repo_url + '/')

          target = 'testrepo'

          assert scm.name == target, '%r != %r' % (scm.name, target)

  

      def test_verify(self):

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          scm.checkout(self.tempdir)

          scm.verify()

  

      def test_verify_unknown_branch(self):

          with pytest.raises(UnprocessableEntity):

-             module_build_service.scm.SCM(repo_path, "unknown")

+             module_build_service.scm.SCM(repo_url, "unknown")

  

      def test_verify_commit_in_branch(self):

          target = '7035bd33614972ac66559ac1fdd019ff6027ad21'

-         scm = module_build_service.scm.SCM(repo_path + "?#" + target, "dev")

+         scm = module_build_service.scm.SCM(repo_url + "?#" + target, "dev")

          scm.checkout(self.tempdir)

          scm.verify()

  

      def test_verify_commit_not_in_branch(self):

          target = '7035bd33614972ac66559ac1fdd019ff6027ad21'

-         scm = module_build_service.scm.SCM(repo_path + "?#" + target, "master")

+         scm = module_build_service.scm.SCM(repo_url + "?#" + target, "master")

          scm.checkout(self.tempdir)

          with pytest.raises(ValidationError):

              scm.verify()

  

      def test_verify_unknown_hash(self):

          target = '7035bd33614972ac66559ac1fdd019ff6027ad22'

-         scm = module_build_service.scm.SCM(repo_path + "?#" + target, "master")

+         scm = module_build_service.scm.SCM(repo_url + "?#" + target, "master")

          with pytest.raises(UnprocessableEntity):

              scm.checkout(self.tempdir)

  

      def test_get_module_yaml(self):

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          scm.checkout(self.tempdir)

          scm.verify()

          with pytest.raises(UnprocessableEntity):

              scm.get_module_yaml()

  

      def test_get_latest_incorrect_component_branch(self):

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          with pytest.raises(UnprocessableEntity):

              scm.get_latest('foobar')

  

      def test_get_latest_component_branch(self):

          ref = "5481faa232d66589e660cc301179867fb00842c9"

          branch = "master"

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          commit = scm.get_latest(branch)

          assert commit == ref

  

      def test_get_latest_component_ref(self):

          ref = "5481faa232d66589e660cc301179867fb00842c9"

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          commit = scm.get_latest(ref)

          assert commit == ref

  

      def test_get_latest_incorrect_component_ref(self):

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          with pytest.raises(UnprocessableEntity):

              scm.get_latest('15481faa232d66589e660cc301179867fb00842c9')

  
@@ -146,6 +147,6 @@ 

  10a651f39911a07d85fe87fcfe91999545e44ae0\trefs/remotes/origin/master

  """

          mock_run.return_value = (0, output, '')

-         scm = module_build_service.scm.SCM(repo_path)

+         scm = module_build_service.scm.SCM(repo_url)

          commit = scm.get_latest(None)

          assert commit == '58379ef7887cbc91b215bacd32430628c92bc869'

@@ -33,6 +33,7 @@ 

  import pytest

  

  from tests import app, init_data, clean_database, reuse_component_init_data

+ from tests.test_scm import base_dir as scm_base_dir

  from module_build_service.errors import UnprocessableEntity

  from module_build_service.models import ModuleBuild

  from module_build_service import db, version, Modulemd
@@ -43,6 +44,7 @@ 

  user = ('Homer J. Simpson', set(['packager']))

  other_user = ('some_other_user', set(['packager']))

  anonymous_user = ('anonymous', set(['packager']))

+ import_module_user = ('Import M. King', set(['mbs-import-module']))

  base_dir = dirname(dirname(__file__))

  

  
@@ -1191,3 +1193,181 @@ 

      def test_cors_header_decorator(self):

          rv = self.client.get('/module-build-service/1/module-builds/')

          assert rv.headers['Access-Control-Allow-Origin'] == '*'

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=user)

+     @patch.object(module_build_service.config.Config, 'allowed_groups_to_import_module',

+                   new_callable=PropertyMock, return_value=set())

+     def test_import_build_disabled(self, mocked_groups, mocked_get_user, api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(post_url)

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Forbidden'

+         assert data['message'] == (

+             'Import module API is disabled. Set '

+             '\'ALLOWED_GROUPS_TO_IMPORT_MODULE\' configuration value first.')

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=user)

+     def test_import_build_user_not_allowed(self, mocked_get_user, api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(post_url)

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Forbidden'

+         assert data['message'] == (

+             'Homer J. Simpson is not in any of '

+             'set([\'mbs-import-module\']), only set([\'packager\'])')

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     def test_import_build_scm_invalid_json(self, mocked_get_user, api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(post_url, data='')

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Bad Request'

+         assert data['message'] == 'Invalid JSON submitted'

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     def test_import_build_scm_url_not_allowed(self, mocked_get_user, api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + '/mariadb'}))

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Forbidden'

+         assert data['message'] == (

+             'The submitted scmurl '

+             'file:///opt/module_build_service/tests/scm_data/mariadb is not allowed')

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'allow_custom_scmurls',

+                   new_callable=PropertyMock, return_value=True)

+     def test_import_build_scm_url_not_in_list(self, mocked_scmurls, mocked_get_user,

+                                               api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#96f4d6d3b7aa311a519e66a1e6eae3ff7411d643')}))

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Forbidden'

+         assert data['message'] == (

+             'file:///opt/module_build_service/tests/scm_data/mariadb'

+             '?#96f4d6d3b7aa311a519e66a1e6eae3ff7411d643 '

+             'is not in the list of allowed SCMs')

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'scmurls',

+                   new_callable=PropertyMock, return_value=['file://'])

+     def test_import_build_scm(self, mocked_scmurls, mocked_get_user, api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#d099cd9fcd921044406f27ba9c64b6297c425492')}))

+         data = json.loads(rv.data)

+ 

+         item = data['items']

+         assert 'Module mariadb:10.2:20180724000000:00000000 imported' in item['messages']

+         assert item['module']['name'] == 'mariadb'

+         assert item['module']['stream'] == '10.2'

+         assert item['module']['version'] == '20180724000000'

+         assert item['module']['context'] == '00000000'

+         assert item['module']['owner'] == 'mbs_import'

+         assert item['module']['state'] == 5

+         assert item['module']['state_reason'] is None

+         assert item['module']['state_name'] == 'ready'

+         assert item['module']['scmurl'] is None

+         assert item['module']['component_builds'] == []

+         assert item['module']['time_submitted'] == item['module']['time_modified'] == \

+             item['module']['time_completed']

+         assert item['module']['koji_tag'] == 'mariadb-10.2-20180724000000-00000000'

+         assert item['module']['siblings'] == []

+         assert item['module']['rebuild_strategy'] == 'all'

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'scmurls',

+                   new_callable=PropertyMock, return_value=['file://'])

+     def test_import_build_scm_another_commit_hash(self, mocked_scmurls, mocked_get_user,

+                                                   api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#8960a704c19c8f0df321ad66e5efc5d2ee99c2bb')}))

+         data = json.loads(rv.data)

+ 

+         item = data['items']

+         assert 'Module mariadb:10.2:20180724065109:00000000 imported' in item['messages']

+         assert item['module']['name'] == 'mariadb'

+         assert item['module']['stream'] == '10.2'

+         assert item['module']['version'] == '20180724065109'

+         assert item['module']['context'] == '00000000'

+         assert item['module']['owner'] == 'mbs_import'

+         assert item['module']['state'] == 5

+         assert item['module']['state_reason'] is None

+         assert item['module']['state_name'] == 'ready'

+         assert item['module']['scmurl'] is None

+         assert item['module']['component_builds'] == []

+         assert item['module']['time_submitted'] == item['module']['time_modified'] == \

+             item['module']['time_completed']

+         assert item['module']['koji_tag'] == 'mariadb-10.2-20180724065109-00000000'

+         assert item['module']['siblings'] == []

+         assert item['module']['rebuild_strategy'] == 'all'

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'scmurls',

+                   new_callable=PropertyMock, return_value=['file://'])

+     def test_import_build_scm_incomplete_nsvc(self, mocked_scmurls, mocked_get_user,

+                                               api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#579b994de17c4e0d18e26f49428e31fa98971012')}))

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Unprocessable Entity'

+         assert data['message'] == 'Incomplete NSVC: None:None:0:00000000'

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'scmurls',

+                   new_callable=PropertyMock, return_value=['file://'])

+     def test_import_build_scm_yaml_is_bad(self, mocked_scmurls, mocked_get_user,

+                                           api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#c4a7e11a3a4c56af125724745d11f4d18e147d67')}))

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Unprocessable Entity'

+         assert data['message'].startswith('The following invalid modulemd was encountered')

+ 

+     @pytest.mark.parametrize('api_version', [1, 2])

+     @patch('module_build_service.auth.get_user', return_value=import_module_user)

+     @patch.object(module_build_service.config.Config, 'scmurls',

+                   new_callable=PropertyMock, return_value=['file://'])

+     def test_import_build_scm_missing_koji_tag(self, mocked_scmurls, mocked_get_user,

+                                                api_version):

+         post_url = '/module-build-service/{0}/import-module/'.format(api_version)

+         rv = self.client.post(

+             post_url,

+             data=json.dumps({'scmurl': 'file://' + scm_base_dir + (

+                 '/mariadb?#c7b473af5cd85f29504c874e3308f2c1cae098fa')}))

+         data = json.loads(rv.data)

+ 

+         assert data['error'] == 'Unprocessable Entity'

+         assert data['message'].startswith('\'koji_tag\' is not set in xmd[\'mbs\'] for module')