From 677d3d69c3371dcff39ce5418a6c93aab78159f4 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jan 20 2020 14:33:56 +0000 Subject: [PATCH 1/3] Merge sidetag plugin Originally living in https://pagure.io/sidetag-koji-plugin Fixes: https://pagure.io/koji/issue/1955 --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index e6e775d..28bbf7c 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -9,8 +9,8 @@ Runroot Plugin for running any command in buildroot. -Save Failed Tree Plugin -======================= +Save Failed Tree +================ In some cases developers want to investigate exact environment in which their build failed. Reconstructing this environment via mock needn't end with @@ -66,3 +66,63 @@ TODO ---- * Separate volume/directory on hub * garbage collector + policy for retaining generated tarballs + +Sidetag +======= + +Sidetag plugin is originally work of Mikolaj Izdebski and was pulled into base +koji due to easier integration with rest of the code. + +It is used for managing `sidetags` which are light-weight short-lived build tags +for developer's use. Sidetag creation is governed by hub's policy. + +Hub +--- + +Example for `/etc/koji-hub/hub.conf`: + +.. code-block:: ini + + PluginPath = /usr/lib/koji-hub-plugins + Plugins = sidetag + + [policy] + sidetag = + # allow maximum of 10 sidetags per user for f30-build tag + tag f30-build && compare number_of_tags <= 10 :: allow + # forbid everything else + all :: deny + +Now Sidetag Koji plugin should be installed. To verify that, run +`koji list-api` command -- it should now display `createSideTag` +as one of available API calls. + +Plugin has also its own configuration file +``/etc/koji-hub/plugins/sidetag.conf`` which for now contains the only boolean +option ``remove_empty``. If it is set, sidetag is automatically deleted when +last package is untagged from there. + +CLI +--- + +For convenient handling, also CLI part is provided. Typical session would look +like: + +.. code-block:: shell + + $ koji add-sidetag f30-build --wait + f30-build-side-123456 + Successfully waited 1:36 for a new f30-build-side-123456 repo + + $ koji remove-sidetag f30-build-side-123456 + +API +--- +And in scripts, you can use following calls: + +.. code-block:: python + + import koji + ks = koji.ClientSession('https://koji.fedoraproject.org/kojihub') + ks.gssapi_login() + ks.createSideTag('f30-build') diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py new file mode 100644 index 0000000..c9729d7 --- /dev/null +++ b/plugins/cli/sidetag_cli.py @@ -0,0 +1,90 @@ +# Copyright © 2019 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from __future__ import absolute_import + +from argparse import ArgumentParser + +import koji +from koji.plugin import export_cli +from koji_cli.lib import _, activate_session, watch_tasks +from koji_cli.commands import anon_handle_wait_repo + + +@export_cli +def handle_add_sidetag(options, session, args): + "Create sidetag" + usage = _("usage: %(prog)s add-sidetag [options] ") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = ArgumentParser(usage=usage) + parser.add_argument("basetag", help="name of basetag") + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help=_("Do not print tag name"), + default=options.quiet, + ) + parser.add_argument( + "-w", "--wait", action="store_true", help=_("Wait until repo is ready.") + ) + opts = parser.parse_args(args) + + activate_session(session, options) + + try: + tag = session.createSideTag(basetag) + except koji.ActionNotAllowed: + parser.error(_("Policy violation")) + + if not opts.quiet: + print (tag["name"]) + + if opts.wait: + args = ["--target", tag["name"]] + if opts.quiet: + args.append("--quiet") + anon_handle_wait_repo(options, session, args) + + +@export_cli +def handle_remove_sidetag(options, session, args): + "Remove sidetag" + usage = _("usage: %(prog)s remove-sidetag [options] ...") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = ArgumentParser(usage=usage) + parser.add_argument("sidetags", help="name of sidetag", nargs="+") + opts = parser.parse_args(args) + + activate_session(session, options) + + session.multicall = True + for sidetag in opts.sidetags: + session.removeSideTag(sidetag) + session.multiCall(strict=True) + + +@export_cli +def handle_list_sidetags(options, session, args): + "List sidetags" + usage = _("usage: %(prog)s list-sidetags [options]") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = ArgumentParser(usage=usage) + parser.add_argument("--basetag", action="store", help=_("Filter on basetag")) + parser.add_argument("--user", action="store", help=_("Filter on user")) + parser.add_argument("--mine", action="store_true", help=_("Filter on user")) + + opts = parser.parse_args(args) + + if opts.mine and opts.user: + parser.error(_("Specify only one from --user --mine")) + + if opts.mine: + activate_session(session, options) + user = session.getLoggedInUser()["name"] + else: + user = opts.user + + for tag in session.listSideTags(basetag=opts.basetag, user=user): + print (tag["name"]) diff --git a/plugins/hub/sidetag.conf b/plugins/hub/sidetag.conf new file mode 100644 index 0000000..743f848 --- /dev/null +++ b/plugins/hub/sidetag.conf @@ -0,0 +1,3 @@ +[sidetag] +# automatically remove sidetag on untagging last package +remove_empty = off diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py new file mode 100644 index 0000000..0a5b75a --- /dev/null +++ b/plugins/hub/sidetag_hub.py @@ -0,0 +1,209 @@ +# Copyright © 2019 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from koji.context import context +from koji.plugin import export, callback +import koji +import sys + +CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" +CONFIG = None + +sys.path.insert(0, "/usr/share/koji-hub/") +from kojihub import ( + assert_policy, + get_tag, + get_user, + get_build_target, + _create_tag, + _create_build_target, + _delete_tag, + _delete_build_target, + readTaggedBuilds, + QueryProcessor, + nextval, +) + + +@export +def createSideTag(basetag): + """Create a side tag. + + :param basetag: name or ID of base tag + :type basetag: str or int + """ + + # Any logged-in user is able to request creation of side tags, + # as long the request meets the policy. + context.session.assertLogin() + user = get_user(context.session.user_id, strict=True) + + basetag = get_tag(basetag, strict=True) + + query = QueryProcessor( + tables=["tag_extra"], + clauses=["key='sidetag_user_id'", "value=%(user_id)s", "active IS TRUE"], + columns=["COUNT(*)"], + aliases=["user_tags"], + values={"user_id": str(user["id"])}, + ) + user_tags = query.executeOne() + if user_tags is None: + # should not ever happen + raise koji.GenericError("Unknown db error") + + # Policy is a very flexible mechanism, that can restrict for which + # tags sidetags can be created, or which users can create sidetags etc. + assert_policy( + "sidetag", {"tag": basetag["id"], "number_of_tags": user_tags["user_tags"]} + ) + + # ugly, it will waste one number in tag_id_seq, but result will match with + # id assigned by _create_tag + tag_id = nextval("tag_id_seq") + 1 + sidetag_name = "%s-side-%s" % (basetag["name"], tag_id) + sidetag_id = _create_tag( + sidetag_name, + parent=basetag["id"], + arches=basetag["arches"], + extra={ + "sidetag": True, + "sidetag_user": user["name"], + "sidetag_user_id": user["id"], + }, + ) + _create_build_target(sidetag_name, sidetag_id, sidetag_id) + + return {"name": sidetag_name, "id": sidetag_id} + + +@export +def removeSideTag(sidetag): + """Remove a side tag + + :param sidetag: id or name of sidetag + :type sidetag: int or str + """ + context.session.assertLogin() + user = get_user(context.session.user_id, strict=True) + sidetag = get_tag(sidetag, strict=True) + + # sanity/access + if not sidetag["extra"].get("sidetag"): + raise koji.GenericError("Not a sidetag: %(name)s" % sidetag) + if sidetag["extra"].get("sidetag_user_id") != user["id"]: + if not context.session.hasPerm("admin"): + raise koji.ActionNotAllowed("This is not your sidetag") + _remove_sidetag(sidetag) + + +def _remove_sidetag(sidetag): + # check target + target = get_build_target(sidetag["name"]) + if not target: + raise koji.GenericError("Target is missing for sidetag") + if target["build_tag"] != sidetag["id"] or target["dest_tag"] != sidetag["id"]: + raise koji.GenericError("Target does not match sidetag") + + _delete_build_target(target["id"]) + _delete_tag(sidetag["id"]) + + +@export +def listSideTags(basetag=None, user=None, queryOpts=None): + """List all sidetags with additional filters + + :param basetag: filter by basteag id or name + :type basetag: int or str + :param user: filter by userid or username + :type user: int or str + :param queryOpts: additional query options + {countOnly, order, offset, limit} + :type queryOpts: dict + """ + # te1.sidetag + # te2.user_id + # te3.basetag + if user is not None: + user_id = str(get_user(user, strict=True)["id"]) + else: + user_id = None + if basetag is not None: + basetag_id = get_tag(basetag, strict=True)["id"] + else: + basetag_id = None + + joins = ["LEFT JOIN tag_extra AS te1 ON tag.id = te1.tag_id"] + clauses = ["te1.active IS TRUE", "te1.key = 'sidetag'", "te1.value = 'true'"] + if user_id: + joins.append("LEFT JOIN tag_extra AS te2 ON tag.id = te2.tag_id") + clauses.extend( + [ + "te2.active IS TRUE", + "te2.key = 'sidetag_user_id'", + "te2.value = %(user_id)s", + ] + ) + if basetag_id: + joins.append("LEFT JOIN tag_inheritance ON tag.id = tag_inheritance.tag_id") + clauses.extend( + [ + "tag_inheritance.active IS TRUE", + "tag_inheritance.parent_id = %(basetag_id)s", + ] + ) + + query = QueryProcessor( + tables=["tag"], + clauses=clauses, + columns=["tag.id", "tag.name"], + aliases=["id", "name"], + joins=joins, + values={"basetag_id": basetag_id, "user_id": user_id}, + opts=queryOpts, + ) + return query.execute() + + +def handle_sidetag_untag(cbtype, *args, **kws): + """Remove a side tag when its last build is untagged + + Note, that this is triggered only in case, that some build exists. For + never used tags, some other policy must be applied. Same holds for users + which don't untag their builds. + """ + if "tag" not in kws: + # shouldn't happen, but... + return + tag = get_tag(kws["tag"]["id"], strict=False) + if not tag: + # also shouldn't happen, but just in case + return + if not tag["extra"].get("sidetag"): + # not a side tag + return + # is the tag now empty? + query = QueryProcessor( + tables=["tag_listing"], + clauses=["tag_id = %(tag_id)s", "active IS TRUE"], + values={"tag_id": tag["id"]}, + opts={"countOnly": True}, + ) + if query.execute(): + return + # looks like we've just untagged the last build from a side tag + try: + # XXX: are we double updating tag_listing? + _remove_sidetag(tag) + except koji.GenericError: + pass + + +# read config and register +if not CONFIG: + CONFIG = koji.read_config_files(CONFIG_FILE) + if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean( + "sidetag", "remove_empty" + ): + handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag) From 017b54dbf6308531180bcbebc358166d7f1fee9b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 05 2020 12:41:26 +0000 Subject: [PATCH 2/3] fix basetag argument --- diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py index c9729d7..e119287 100644 --- a/plugins/cli/sidetag_cli.py +++ b/plugins/cli/sidetag_cli.py @@ -34,7 +34,7 @@ def handle_add_sidetag(options, session, args): activate_session(session, options) try: - tag = session.createSideTag(basetag) + tag = session.createSideTag(opts.basetag) except koji.ActionNotAllowed: parser.error(_("Policy violation")) From 8aecb5fb4ae5dee4fe1e449b72452ebf1cb8a164 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 12 2020 09:59:54 +0000 Subject: [PATCH 3/3] remove unused imports --- diff --git a/plugins/cli/sidetag_cli.py b/plugins/cli/sidetag_cli.py index e119287..df10c4e 100644 --- a/plugins/cli/sidetag_cli.py +++ b/plugins/cli/sidetag_cli.py @@ -8,7 +8,7 @@ from argparse import ArgumentParser import koji from koji.plugin import export_cli -from koji_cli.lib import _, activate_session, watch_tasks +from koji_cli.lib import _, activate_session from koji_cli.commands import anon_handle_wait_repo diff --git a/plugins/hub/sidetag_hub.py b/plugins/hub/sidetag_hub.py index 0a5b75a..2f9bd45 100644 --- a/plugins/hub/sidetag_hub.py +++ b/plugins/hub/sidetag_hub.py @@ -1,11 +1,11 @@ # Copyright © 2019 Red Hat, Inc. # # SPDX-License-Identifier: GPL-2.0-or-later +import sys from koji.context import context from koji.plugin import export, callback import koji -import sys CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf" CONFIG = None @@ -20,7 +20,6 @@ from kojihub import ( _create_build_target, _delete_tag, _delete_build_target, - readTaggedBuilds, QueryProcessor, nextval, )