From 1572abed84bd7919a3c244b96c2986e9e2c8098d Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 03 2020 09:24:42 +0000 Subject: Add scratch_build_tasks to generate composes including scratch builds. --- diff --git a/docs/api.rst b/docs/api.rst index 3ec0a9d..1c79769 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -154,6 +154,11 @@ The fields used in the ODCS compose JSON have following meaning: .. note:: If ``target_dir`` is set to non-default value, then the ``result_repofile`` might be an empty string, because ODCS might not have enough data to generate the URL. +.. _scratch_build_tasks: + +*scratch_build_tasks* - ``(white-space separated list of strings)`` + List of Koji task IDs of RPM build scratch builds which will be included in the compose. + .. _scratch_modules: *scratch_modules* - ``(white-space separated list of strings)`` diff --git a/server/conf/pungi.conf b/server/conf/pungi.conf index 57bb79a..798ccdc 100644 --- a/server/conf/pungi.conf +++ b/server/conf/pungi.conf @@ -78,6 +78,14 @@ pkgset_scratch_modules = { } {%- endif %} +{%- if config.scratch_build_tasks %} +pkgset_koji_scratch_tasks = [ +{%- for task_id in config.scratch_build_tasks %} + '{{ task_id }}', +{%- endfor %} + ] +{%- endif %} + {%- if config.source_type_str in ["tag", "build"] and not config.packages %} # In case no package is requested, include all of them. diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 71e0bb0..f46dd76 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -628,6 +628,24 @@ def get_reusable_compose(compose): ) continue + scratch_build_tasks = ( + set(compose.scratch_build_tasks.split(" ")) + if compose.scratch_build_tasks + else set() + ) + old_scratch_build_tasks = ( + set(old_compose.scratch_build_tasks.split(" ")) + if old_compose.scratch_build_tasks + else set() + ) + if scratch_build_tasks != old_scratch_build_tasks: + log.debug( + "%r: Cannot reuse %r - scratch_build_tasks not same", + compose, + old_compose, + ) + continue + parent_pungi_compose_ids = ( set(compose.parent_pungi_compose_ids.split(" ")) if compose.parent_pungi_compose_ids @@ -945,6 +963,7 @@ def generate_pungi_compose(compose): modular_koji_tags=compose.modular_koji_tags, module_defaults_url=compose.module_defaults_url, scratch_modules=compose.scratch_modules, + scratch_build_tasks=compose.scratch_build_tasks, ) if compose.flags & COMPOSE_FLAGS["no_deps"]: pungi_cfg.gather_method = "nodeps" diff --git a/server/odcs/server/migrations/versions/ca08065687c4_.py b/server/odcs/server/migrations/versions/ca08065687c4_.py new file mode 100644 index 0000000..921189b --- /dev/null +++ b/server/odcs/server/migrations/versions/ca08065687c4_.py @@ -0,0 +1,24 @@ +"""Add scratch_build_tasks field. + +Revision ID: ca08065687c4 +Revises: 7b143656694f +Create Date: 2020-09-03 11:20:54.035095 + +""" + +# revision identifiers, used by Alembic. +revision = "ca08065687c4" +down_revision = "7b143656694f" + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column( + "composes", sa.Column("scratch_build_tasks", sa.String(), nullable=True) + ) + + +def downgrade(): + op.drop_column("composes", "scratch_build_tasks") diff --git a/server/odcs/server/models.py b/server/odcs/server/models.py index ea5bfc9..e1e0efc 100644 --- a/server/odcs/server/models.py +++ b/server/odcs/server/models.py @@ -166,6 +166,8 @@ class Compose(ODCSBase): _target_dir = db.Column("target_dir", db.String, nullable=True) # White-space separated list of scratch modules (N:S:V:C) to include in a compose. scratch_modules = db.Column(db.String, nullable=True) + # White-space separated list of RPM scratch builds to include in a compose. + scratch_build_tasks = db.Column(db.String, nullable=True) # White-space separated list of parent Pungi compose IDs (pungi_compose_id). parent_pungi_compose_ids = db.Column(db.String, nullable=True) @@ -215,6 +217,7 @@ class Compose(ODCSBase): target_dir=None, scratch_modules=None, parent_pungi_compose_ids=None, + scratch_build_tasks=None, ): now = datetime.utcnow() compose = cls( @@ -241,6 +244,7 @@ class Compose(ODCSBase): target_dir=target_dir or conf.target_dir, scratch_modules=scratch_modules, parent_pungi_compose_ids=parent_pungi_compose_ids, + scratch_build_tasks=scratch_build_tasks, ) session.add(compose) return compose @@ -288,6 +292,7 @@ class Compose(ODCSBase): target_dir=compose.target_dir, scratch_modules=compose.scratch_modules, parent_pungi_compose_ids=compose.parent_pungi_compose_ids, + scratch_build_tasks=compose.scratch_build_tasks, ) session.add(compose) return compose @@ -424,6 +429,7 @@ class Compose(ODCSBase): "target_dir": target_dir, "scratch_modules": self.scratch_modules, "parent_pungi_compose_ids": self.parent_pungi_compose_ids, + "scratch_build_tasks": self.scratch_build_tasks, } if full: diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index c189a9c..5560b9d 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -229,6 +229,7 @@ class PungiConfig(BasePungiConfig): modular_koji_tags=None, module_defaults_url=None, scratch_modules=None, + scratch_build_tasks=None, ): super(PungiConfig, self).__init__() self.release_name = release_name @@ -247,6 +248,9 @@ class PungiConfig(BasePungiConfig): self.packages = packages or [] self.builds = builds or [] self.scratch_modules = scratch_modules.split(" ") if scratch_modules else [] + self.scratch_build_tasks = ( + scratch_build_tasks.split(" ") if scratch_build_tasks else [] + ) # Store results as list of strings, so it can be used by jinja2 # templates. diff --git a/server/odcs/server/views.py b/server/odcs/server/views.py index f869275..a539898 100644 --- a/server/odcs/server/views.py +++ b/server/odcs/server/views.py @@ -282,6 +282,7 @@ class ODCSAPI(MethodView): :jsonparam list source["koji_event"]: Number defining the :ref:`koji_event`. :jsonparam list source["modular_koji_tags"]: List defining the :ref:`modular_koji_tags`. :jsonparam list source["scratch_modules"]: List defining the :ref:`scratch_modules`. + :jsonparam list source["scratch_build_tasks"]: List defining the :ref:`scratch_build_tasks`. :statuscode 200: Compose request created and returned. :statuscode 401: Request not in valid format. @@ -451,6 +452,10 @@ class ODCSAPI(MethodView): if "scratch_modules" in source_data: scratch_modules = " ".join(source_data["scratch_modules"]) + scratch_build_tasks = None + if "scratch_build_tasks" in source_data: + scratch_build_tasks = " ".join(source_data["scratch_build_tasks"]) + label = data.get("label", None) compose_type = data.get("compose_type", "test") @@ -496,6 +501,7 @@ class ODCSAPI(MethodView): target_dir=target_dir, scratch_modules=scratch_modules, parent_pungi_compose_ids=parent_pungi_compose_ids, + scratch_build_tasks=scratch_build_tasks, ) db.session.add(compose) # Flush is needed, because we use `before_commit` SQLAlchemy event to diff --git a/server/tests/test_models.py b/server/tests/test_models.py index 9d9e084..4a2f6a6 100644 --- a/server/tests/test_models.py +++ b/server/tests/test_models.py @@ -93,6 +93,7 @@ class TestModels(ModelsBaseTest): "toplevel_url": "http://localhost/odcs/odcs-1", "scratch_modules": None, "parent_pungi_compose_ids": None, + "scratch_build_tasks": None, } self.assertEqual(c.json(True), expected_json) diff --git a/server/tests/test_pungi.py b/server/tests/test_pungi.py index cbeeead..a271afb 100644 --- a/server/tests/test_pungi.py +++ b/server/tests/test_pungi.py @@ -242,6 +242,25 @@ class TestPungiConfig(unittest.TestCase): cfg["additional_packages"], [(u"^Temporary$", {u"*": [u"*"]})] ) + def test_get_pungi_conf_scratch_build_tasks(self): + _, mock_path = tempfile.mkstemp() + template_path = os.path.abspath(os.path.join(test_dir, "../conf/pungi.conf")) + shutil.copy2(template_path, mock_path) + + with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path): + pungi_cfg = PungiConfig( + "MBS-512", + "1", + PungiSourceType.KOJI_TAG, + "f26", + scratch_build_tasks="123456 123457", + ) + + cfg = pungi_cfg.get_pungi_config() + self.assertEqual( + set(cfg["pkgset_koji_scratch_tasks"]), set(["123456", "123457"]) + ) + def test_get_pungi_conf_modular_koji_tags(self): _, mock_path = tempfile.mkstemp() template_path = os.path.abspath(os.path.join(test_dir, "../conf/pungi.conf")) diff --git a/server/tests/test_views.py b/server/tests/test_views.py index 9cce883..3c41102 100644 --- a/server/tests/test_views.py +++ b/server/tests/test_views.py @@ -364,6 +364,7 @@ class TestViews(ViewBaseTest): "scratch_modules": None, "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"], "parent_pungi_compose_ids": None, + "scratch_build_tasks": None, } self.assertEqual(data, expected_json) @@ -1247,6 +1248,33 @@ class TestViews(ViewBaseTest): "User unknown not allowed to operate with compose with source_types=tag.", ) + def test_submit_scratch_build_tasks(self): + with self.test_request_context(user="dev"): + flask.g.oidc_scopes = [ + "{0}{1}".format(conf.oidc_base_namespace, "new-compose") + ] + + rv = self.client.post( + "/api/1/composes/", + data=json.dumps( + { + "source": { + "type": "module", + "source": "testmodule:master", + "scratch_build_tasks": [ + "123456", + "123457", + ], + } + } + ), + ) + data = json.loads(rv.get_data(as_text=True)) + self.assertEqual( + data["scratch_build_tasks"], + "123456 123457", + ) + def test_submit_build_scratch_modules(self): with self.test_request_context(user="dev"): flask.g.oidc_scopes = [ @@ -1763,6 +1791,7 @@ class TestViews(ViewBaseTest): "scratch_modules": None, "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"], "parent_pungi_compose_ids": None, + "scratch_build_tasks": None, } self.assertEqual(data, expected_json)