From 7dfb647e0880f6a4c8dd73da63058ea6a7a4a655 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Feb 28 2017 14:26:20 +0000 Subject: Merge #347 `Possibility to specify optional parameters when submitting build` --- diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 540d16d..3641c86 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -938,7 +938,15 @@ class CoprModuleBuilder(GenericBuilder): def _get_copr_safe(self): from copr.exceptions import CoprRequestException - kwargs = {"ownername": self.owner, "projectname": CoprModuleBuilder._tag_to_copr_name(self.tag_name)} + + # @TODO it would be nice if the module build object was passed to Builder __init__ + module = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one() + + kwargs = { + "ownername": module.copr_owner or self.owner, + "projectname": module.copr_project or CoprModuleBuilder._tag_to_copr_name(self.tag_name) + } + try: return self._get_copr(**kwargs) except CoprRequestException: @@ -1059,7 +1067,7 @@ class CoprModuleBuilder(GenericBuilder): def finalize(self): modulemd = tempfile.mktemp() - m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).first() + m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one() m1.mmd().dump(modulemd) # Wait until all builds are finished diff --git a/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py b/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py new file mode 100644 index 0000000..c3d6b5b --- /dev/null +++ b/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py @@ -0,0 +1,24 @@ +"""Add optional columns for copr + +Revision ID: 474697622859 +Revises: 0ef60c3ed440 +Create Date: 2017-02-21 11:18:22.304038 + +""" + +# revision identifiers, used by Alembic. +revision = '474697622859' +down_revision = 'a1fc0736bca8' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('module_builds', sa.Column('copr_owner', sa.String(), nullable=True)) + op.add_column('module_builds', sa.Column('copr_project', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('module_builds', 'copr_owner') + op.drop_column('module_builds', 'copr_project') diff --git a/module_build_service/models.py b/module_build_service/models.py index d567a45..cffc9a8 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -101,6 +101,8 @@ class ModuleBuild(RidaBase): state_reason = db.Column(db.String) modulemd = db.Column(db.String, nullable=False) koji_tag = db.Column(db.String) # This gets set after 'wait' + copr_owner = db.Column(db.String) + copr_project = db.Column(db.String) scmurl = db.Column(db.String) owner = db.Column(db.String, nullable=False) time_submitted = db.Column(db.DateTime, nullable=False) @@ -155,7 +157,8 @@ class ModuleBuild(RidaBase): % type(event).__name__) @classmethod - def create(cls, session, conf, name, stream, version, modulemd, scmurl, username): + def create(cls, session, conf, name, stream, version, modulemd, scmurl, username, + copr_owner=None, copr_project=None): now = datetime.utcnow() module = cls( name=name, @@ -165,7 +168,9 @@ class ModuleBuild(RidaBase): modulemd=modulemd, scmurl=scmurl, owner=username, - time_submitted=now + time_submitted=now, + copr_owner=copr_owner, + copr_project=copr_project, ) session.add(module) session.commit() diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 9fb33f7..2be0f0c 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -543,17 +543,17 @@ def record_component_builds(scm, mmd, module, initial_batch = 1): return batch -def submit_module_build_from_yaml(username, yaml): +def submit_module_build_from_yaml(username, yaml, optional_params=None): mmd = load_mmd(yaml) - return submit_module_build(username, None, mmd, None, yaml) + return submit_module_build(username, None, mmd, None, yaml, optional_params) -def submit_module_build_from_scm(username, url, allow_local_url=False): +def submit_module_build_from_scm(username, url, allow_local_url=False, optional_params=None): mmd, scm, yaml = _fetch_mmd(url, allow_local_url) - return submit_module_build(username, url, mmd, scm, yaml) + return submit_module_build(username, url, mmd, scm, yaml, optional_params) -def submit_module_build(username, url, mmd, scm, yaml): +def submit_module_build(username, url, mmd, scm, yaml, optional_params=None): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. import module_build_service.scm @@ -586,7 +586,8 @@ def submit_module_build(username, url, mmd, scm, yaml): version=str(mmd.version), modulemd=yaml, scmurl=url, - username=username + username=username, + **(optional_params or {}) ) record_component_builds(scm, mmd, module) @@ -599,6 +600,18 @@ def submit_module_build(username, url, mmd, scm, yaml): mmd.name, mmd.stream, mmd.version) return module + +def validate_optional_params(params): + forbidden_params = [k for k in params if k not in models.ModuleBuild.__table__.columns] + if forbidden_params: + raise ValidationError('The request contains unspecified parameters: {}'.format(", ".join(forbidden_params))) + + forbidden_params = [k for k in params if k.startswith("copr_")] + if conf.system != "copr" and forbidden_params: + raise ValidationError('The request contains parameters specific to Copr builder: {} even though {} is used' + .format(", ".join(forbidden_params), conf.system)) + + def scm_url_schemes(terse=False): """ Definition of URL schemes supported by both frontend and scheduler. diff --git a/module_build_service/views.py b/module_build_service/views.py index 65fc318..cf414de 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -35,7 +35,7 @@ from module_build_service import app, conf, log from module_build_service import models, db from module_build_service.utils import ( pagination_metadata, filter_module_builds, submit_module_build_from_scm, - submit_module_build_from_yaml, scm_url_schemes, get_scm_url_re) + submit_module_build_from_yaml, scm_url_schemes, get_scm_url_re, validate_optional_params) from module_build_service.errors import ( ValidationError, Unauthorized, NotFound) @@ -97,10 +97,9 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) - if "multipart/form-data" in request.headers.get("Content-Type"): - module = self.post_file(username) - else: - module = self.post_scm(username) + kwargs = {"username": username} + module = (self.post_file(**kwargs) if "multipart/form-data" in request.headers.get("Content-Type", "") else + self.post_scm(**kwargs)) return jsonify(module.json()), 201 @@ -124,18 +123,22 @@ class ModuleBuildAPI(MethodView): log.error("The submitted scmurl %r is not valid" % url) raise Unauthorized("The submitted scmurl %s is not valid" % url) - return submit_module_build_from_scm(username, url, allow_local_url=False) + validate_optional_params(r) + optional_params = {k: v for k, v in r.items() if k != "scmurl"} + return submit_module_build_from_scm(username, url, allow_local_url=False, optional_params=optional_params) def post_file(self, username): if not conf.yaml_submit_allowed: raise Unauthorized("YAML submission is not enabled") + validate_optional_params(request.form) + try: r = request.files["yaml"] except: log.error('Invalid file submitted') raise ValidationError('Invalid file submitted') - return submit_module_build_from_yaml(username, r.read()) + return submit_module_build_from_yaml(username, r.read(), optional_params=request.form.to_dict()) def patch(self, id): username, groups = module_build_service.auth.get_user(request) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 72a0313..9133a5a 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -316,6 +316,24 @@ class TestBuild(unittest.TestCase): @timed(30) @patch('module_build_service.auth.get_user', return_value=user) + def test_submit_build_with_optional_params(self, mocked_get_user): + params = {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'} + + def submit(data): + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(data)) + return json.loads(rv.data) + + data = submit(dict(params.items() + {"not_existing_param": "foo"}.items())) + self.assertIn("The request contains unspecified parameters:", data["message"]) + self.assertIn("not_existing_param", data["message"]) + self.assertEqual(data["status"], 400) + + data = submit(dict(params.items() + {"copr_owner": "foo"}.items())) + self.assertIn("The request contains parameters specific to Copr builder", data["message"]) + + @timed(30) + @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_cancel(self, mocked_scm, mocked_get_user): """