From 56539949b9b57ea201891c6552e89f65ff313612 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Aug 10 2020 08:18:14 +0000 Subject: Support to configure scratch module builds JIRA: RHELCMP-440 Signed-off-by: Haibo Lin --- diff --git a/client/contrib/odcs b/client/contrib/odcs index df4128b..37ac5ac 100755 --- a/client/contrib/odcs +++ b/client/contrib/odcs @@ -125,6 +125,12 @@ KNOWN_ARGS = { metavar="lookaside_repos", help="Specify lookaside repositories.", ), + "--scratch-module": dict( + default=[], + action="append", + metavar="scratch_modules", + help="Scratch modules to be included in the compose with format N:S:V:C", + ), "--label": dict(default=None, help="Label for raw_config compose."), "--compose-type": dict(default=None, help="Compose type for raw_config compose."), "--build": dict( @@ -380,6 +386,7 @@ try: modular_koji_tags=args.modular_tag, module_defaults_url=args.module_defaults_url, module_defaults_commit=args.module_defaults_commit, + scratch_modules=args.scratch_modules, **request_args ) elif args.command == "create-tag": @@ -392,6 +399,7 @@ try: args.modular_tag, args.module_defaults_url, args.module_defaults_commit, + args.scratch_modules, ) result = client.request_compose(source, **request_args) elif args.command == "create-module": @@ -400,6 +408,7 @@ try: args.sigkey, args.module_defaults_url, args.module_defaults_commit, + args.scratch_modules, ) result = client.request_compose(source, **request_args) elif args.command == "create-pulp": diff --git a/client/odcs/client/odcs.py b/client/odcs/client/odcs.py index f446b16..3d1d557 100644 --- a/client/odcs/client/odcs.py +++ b/client/odcs/client/odcs.py @@ -120,6 +120,7 @@ class ComposeSourceGeneric(object): modular_koji_tags=None, module_defaults_url=None, module_defaults_commit=None, + scratch_modules=None, **kwargs ): self.source = {"source": source, "type": source_type} @@ -137,6 +138,8 @@ class ComposeSourceGeneric(object): self.source["module_defaults_url"] = module_defaults_url if module_defaults_commit: self.source["module_defaults_commit"] = module_defaults_commit + if scratch_modules: + self.source["scratch_modules"] = scratch_modules self.source.update(kwargs) @@ -155,6 +158,7 @@ class ComposeSourceTag(ComposeSourceGeneric): modular_koji_tags=None, module_defaults_url=None, module_defaults_commit=None, + scratch_modules=None, **kwargs ): """ @@ -176,6 +180,8 @@ class ComposeSourceTag(ComposeSourceGeneric): :param str module_defaults_url: URL of module defaults repository. :param str module_defaults_commit: Commit or branch name defining particular point module defaults repository. + :param list scratch_modules: List of N:S:V:C of scratch modules to be included in + the compose. """ super(ComposeSourceTag, self).__init__( tag, @@ -187,6 +193,7 @@ class ComposeSourceTag(ComposeSourceGeneric): modular_koji_tags, module_defaults_url, module_defaults_commit, + scratch_modules, **kwargs ) @@ -202,6 +209,7 @@ class ComposeSourceModule(ComposeSourceGeneric): sigkeys=None, module_defaults_url=None, module_defaults_commit=None, + scratch_modules=None, **kwargs ): """ @@ -213,6 +221,8 @@ class ComposeSourceModule(ComposeSourceGeneric): :param str module_defaults_url: URL of module defaults repository. :param str module_defaults_commit: Commit or branch name defining particular point module defaults repository. + :param list scratch_modules: List of N:S:V:C of scratch modules to be included in + the compose. """ super(ComposeSourceModule, self).__init__( " ".join(modules), @@ -220,6 +230,7 @@ class ComposeSourceModule(ComposeSourceGeneric): sigkeys=sigkeys, module_defaults_url=module_defaults_url, module_defaults_commit=module_defaults_commit, + scratch_modules=scratch_modules, **kwargs ) @@ -449,6 +460,7 @@ class ODCS(object): modular_koji_tags=None, module_defaults_url=None, module_defaults_commit=None, + scratch_modules=None, lookaside_repos=None, label=None, compose_type=None, @@ -500,6 +512,8 @@ class ODCS(object): request_data["source"]["module_defaults_url"] = module_defaults_url if module_defaults_commit: request_data["source"]["module_defaults_commit"] = module_defaults_commit + if scratch_modules: + request_data["source"]["scratch_modules"] = scratch_modules if lookaside_repos: request_data["lookaside_repos"] = lookaside_repos if label: diff --git a/server/conf/pungi.conf b/server/conf/pungi.conf index 154a52a..0d234e7 100644 --- a/server/conf/pungi.conf +++ b/server/conf/pungi.conf @@ -67,6 +67,17 @@ pkgset_koji_builds = [ ] {%- endif %} +{%- if config.scratch_modules %} +mbs_api_url = {{ config.mbs_api_url }} +pkgset_scratch_modules = { + '^Temporary$': [ +{%- for nsvc in config.scratch_modules %} + '{{ nsvc }}', +{%- endfor %} + ] +} +{%- endif %} + {%- if config.source_type_str in ["tag", "build"] and not config.packages %} # In case no package is requested, include all of them. diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 5717d6b..f2baa11 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -460,6 +460,8 @@ def get_reusable_compose(compose): Returns the compose in the "done" state which contains the same artifacts and results as the compose `compose` and therefore could be reused instead of generating new one. + + :param models.Compose compose: Instance of models.Compose. """ # RAW_CONFIG composes cannot reuse other composes, we cannot track input @@ -605,6 +607,22 @@ def get_reusable_compose(compose): ) continue + scratch_modules = ( + set(compose.scratch_modules.split(" ")) + if compose.scratch_modules + else set() + ) + old_scratch_modules = ( + set(old_compose.scratch_modules.split(" ")) + if old_compose.scratch_modules + else set() + ) + if scratch_modules != old_scratch_modules: + log.debug( + "%r: Cannot reuse %r - scratch_modules not same", compose, old_compose + ) + continue + target_dir = compose.target_dir old_target_dir = old_compose.target_dir if target_dir != old_target_dir: @@ -842,6 +860,8 @@ def remove_compose_symlink(compose): def generate_pungi_compose(compose): """ Generates the compose of KOJI, TAG, or REPO type using the Pungi tool. + + :param models.Compose compose: Instance of models.Compose. """ koji_tag_cache = KojiTagCache(compose) @@ -887,6 +907,7 @@ def generate_pungi_compose(compose): lookaside_repos=compose.lookaside_repos, modular_koji_tags=compose.modular_koji_tags, module_defaults_url=compose.module_defaults_url, + scratch_modules=compose.scratch_modules, ) if compose.flags & COMPOSE_FLAGS["no_deps"]: pungi_cfg.gather_method = "nodeps" diff --git a/server/odcs/server/migrations/versions/512890e6864d_add_compose_scratch_modules.py b/server/odcs/server/migrations/versions/512890e6864d_add_compose_scratch_modules.py new file mode 100644 index 0000000..cce0faa --- /dev/null +++ b/server/odcs/server/migrations/versions/512890e6864d_add_compose_scratch_modules.py @@ -0,0 +1,26 @@ +"""Add Compose.scratch_modules + +Revision ID: 512890e6864d +Revises: 812f2745248f +Create Date: 2020-07-31 22:40:13.138130 + +""" + +# revision identifiers, used by Alembic. +revision = "512890e6864d" +down_revision = "812f2745248f" + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("composes", sa.Column("scratch_modules", sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("composes", "scratch_modules") + # ### end Alembic commands ### diff --git a/server/odcs/server/models.py b/server/odcs/server/models.py index 0a57083..b9180aa 100644 --- a/server/odcs/server/models.py +++ b/server/odcs/server/models.py @@ -163,6 +163,8 @@ class Compose(ODCSBase): # Target directory in which the compose is stored. This is `conf.target_dir` # by default. _target_dir = db.Column("target_dir", db.String, nullable=True) + # White-space separated list of scratch modules (N:S:V:C) to include in a compose. + scratch_modules = db.Column(db.String, nullable=True) @property def on_default_target_dir(self): @@ -208,6 +210,7 @@ class Compose(ODCSBase): label=None, compose_type=None, target_dir=None, + scratch_modules=None, ): now = datetime.utcnow() compose = cls( @@ -232,6 +235,7 @@ class Compose(ODCSBase): label=label, compose_type=compose_type, target_dir=target_dir or conf.target_dir, + scratch_modules=scratch_modules, ) session.add(compose) return compose @@ -277,6 +281,7 @@ class Compose(ODCSBase): # Also reset celery task_id celery_task_id=None, target_dir=compose.target_dir, + scratch_modules=compose.scratch_modules, ) session.add(compose) return compose @@ -407,6 +412,7 @@ class Compose(ODCSBase): "compose_type": self.compose_type, "pungi_compose_id": self.pungi_compose_id, "target_dir": target_dir, + "scratch_modules": self.scratch_modules, } if full: diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index 24ac76a..91e0693 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -171,6 +171,7 @@ class PungiConfig(BasePungiConfig): lookaside_repos=None, modular_koji_tags=None, module_defaults_url=None, + scratch_modules=None, ): super(PungiConfig, self).__init__() self.release_name = release_name @@ -188,6 +189,7 @@ class PungiConfig(BasePungiConfig): self.arches = conf.arches self.packages = packages or [] self.builds = builds or [] + self.scratch_modules = scratch_modules.split(" ") if scratch_modules else [] # Store results as list of strings, so it can be used by jinja2 # templates. @@ -265,6 +267,10 @@ class PungiConfig(BasePungiConfig): return "repos" return "koji" + @property + def mbs_api_url(self): + return os.path.join(conf.mbs_url, "1") + def get_comps_config(self): if self.source_type == PungiSourceType.MODULE: return "" diff --git a/server/odcs/server/views.py b/server/odcs/server/views.py index 4addd74..eee80ba 100644 --- a/server/odcs/server/views.py +++ b/server/odcs/server/views.py @@ -278,6 +278,7 @@ class ODCSAPI(MethodView): :jsonparam list source["sigkeys"]: List defining the :ref:`sigkeys`. :jsonparam list source["koji_event"]: Number defining the :ref:`koji_event`. :jsonparam list source["modular_koji_tags"]: List defining the :ref:`modular_koji_tags`. + :jsonparam list source["scratch_modules"]: List defining the :ref:`scratch_modules`. :statuscode 200: Compose request created and returned. :statuscode 401: Request not in valid format. @@ -439,6 +440,10 @@ class ODCSAPI(MethodView): elif module_defaults_url and module_defaults_commit: module_defaults = "%s %s" % (module_defaults_url, module_defaults_commit) + scratch_modules = None + if "scratch_modules" in source_data: + scratch_modules = " ".join(source_data["scratch_modules"]) + label = data.get("label", None) compose_type = data.get("compose_type", "test") @@ -482,6 +487,7 @@ class ODCSAPI(MethodView): label=label, compose_type=compose_type, target_dir=target_dir, + scratch_modules=scratch_modules, ) db.session.add(compose) # Flush is needed, because we use `before_commit` SQLAlchemy event to diff --git a/server/tests/test_models.py b/server/tests/test_models.py index 27f7c2c..981ebfd 100644 --- a/server/tests/test_models.py +++ b/server/tests/test_models.py @@ -91,6 +91,7 @@ class TestModels(ModelsBaseTest): "pungi_config_dump": "test", "target_dir": "default", "toplevel_url": "http://localhost/odcs/odcs-1", + "scratch_modules": None, } self.assertEqual(c.json(True), expected_json) diff --git a/server/tests/test_views.py b/server/tests/test_views.py index 272cf7f..0c09e83 100644 --- a/server/tests/test_views.py +++ b/server/tests/test_views.py @@ -361,6 +361,7 @@ class TestViews(ViewBaseTest): "compose_type": "test", "pungi_compose_id": None, "target_dir": "default", + "scratch_modules": None, "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"], } self.assertEqual(data, expected_json) @@ -1245,6 +1246,33 @@ class TestViews(ViewBaseTest): "User unknown not allowed to operate with compose with source_types=tag.", ) + def test_submit_build_scratch_modules(self): + with self.test_request_context(user="dev"): + flask.g.oidc_scopes = [ + "{0}{1}".format(conf.oidc_base_namespace, "new-compose") + ] + + rv = self.client.post( + "/api/1/composes/", + data=json.dumps( + { + "source": { + "type": "module", + "source": "testmodule:master", + "scratch_modules": [ + "foo:bar:20200806:abcdefgh", + "fooo:bar:20200810:abcdefgh", + ], + } + } + ), + ) + data = json.loads(rv.get_data(as_text=True)) + self.assertEqual( + data["scratch_modules"], + "foo:bar:20200806:abcdefgh fooo:bar:20200810:abcdefgh", + ) + def test_query_compose(self): resp = self.client.get("/api/1/composes/1") data = json.loads(resp.get_data(as_text=True)) @@ -1710,6 +1738,7 @@ class TestViews(ViewBaseTest): "compose_type": "test", "pungi_compose_id": None, "target_dir": "default", + "scratch_modules": None, "toplevel_url": "http://localhost/odcs/odcs-%d" % data["id"], } self.assertEqual(data, expected_json)