#1956 Merge sidetag plugin
Merged 4 years ago by tkopecek. Opened 4 years ago by tkopecek.
tkopecek/koji issue1955  into  master

file modified
+62 -2
@@ -9,8 +9,8 @@ 

  

  Plugin for running any command in buildroot.

  

- Save Failed Tree Plugin

- =======================

+ Save Failed Tree

+ ================

  

  In some cases developers want to investigate exact environment in which their

  build failed. Reconstructing this environment via mock needn't end with
@@ -66,3 +66,63 @@ 

  ----

   * Separate volume/directory on hub

   * garbage collector + policy for retaining generated tarballs

+ 

+ Sidetag

+ =======

+ 

+ Sidetag plugin is originally work of Mikolaj Izdebski and was pulled into base

+ koji due to easier integration with rest of the code.

+ 

+ It is used for managing `sidetags` which are light-weight short-lived build tags

+ for developer's use. Sidetag creation is governed by hub's policy.

+ 

+ Hub

+ ---

+ 

+ Example for `/etc/koji-hub/hub.conf`:

+ 

+ .. code-block:: ini

+ 

+     PluginPath = /usr/lib/koji-hub-plugins

+     Plugins = sidetag

+ 

+     [policy]

+     sidetag =

+         # allow maximum of 10 sidetags per user for f30-build tag

+         tag f30-build && compare number_of_tags <= 10 :: allow

+         # forbid everything else

+         all :: deny

+ 

+ Now Sidetag Koji plugin should be installed.  To verify that, run

+ `koji list-api` command -- it should now display `createSideTag`

+ as one of available API calls.

+ 

+ Plugin has also its own configuration file

+ ``/etc/koji-hub/plugins/sidetag.conf`` which for now contains the only boolean

+ option ``remove_empty``. If it is set, sidetag is automatically deleted when

+ last package is untagged from there.

+ 

+ CLI

+ ---

+ 

+ For convenient handling, also CLI part is provided. Typical session would look

+ like:

+ 

+ .. code-block:: shell

+ 

+    $ koji add-sidetag f30-build --wait

+    f30-build-side-123456

+    Successfully waited 1:36 for a new f30-build-side-123456 repo

+ 

+    $ koji remove-sidetag f30-build-side-123456

+ 

+ API

+ ---

+ And in scripts, you can use following calls:

+ 

+ .. code-block:: python

+ 

+     import koji

+     ks = koji.ClientSession('https://koji.fedoraproject.org/kojihub')

+     ks.gssapi_login()

+     ks.createSideTag('f30-build')

@@ -0,0 +1,90 @@ 

+ # Copyright Âİ 2019 Red Hat, Inc.

+ #

+ # SPDX-License-Identifier: GPL-2.0-or-later

+ 

+ from __future__ import absolute_import

+ 

+ from argparse import ArgumentParser

+ 

+ import koji

+ from koji.plugin import export_cli

+ from koji_cli.lib import _, activate_session

+ from koji_cli.commands import anon_handle_wait_repo

+ 

+ 

+ @export_cli

+ def handle_add_sidetag(options, session, args):

+     "Create sidetag"

+     usage = _("usage: %(prog)s add-sidetag [options] <basetag>")

+     usage += _("\n(Specify the --help global option for a list of other help options)")

+     parser = ArgumentParser(usage=usage)

+     parser.add_argument("basetag", help="name of basetag")

+     parser.add_argument(

+         "-q",

+         "--quiet",

+         action="store_true",

+         help=_("Do not print tag name"),

+         default=options.quiet,

+     )

+     parser.add_argument(

+         "-w", "--wait", action="store_true", help=_("Wait until repo is ready.")

+     )

+     opts = parser.parse_args(args)

+ 

+     activate_session(session, options)

+ 

+     try:

+         tag = session.createSideTag(opts.basetag)

+     except koji.ActionNotAllowed:

+         parser.error(_("Policy violation"))

+ 

+     if not opts.quiet:

+         print (tag["name"])

+ 

+     if opts.wait:

+         args = ["--target", tag["name"]]

+         if opts.quiet:

+             args.append("--quiet")

+         anon_handle_wait_repo(options, session, args)

+ 

+ 

+ @export_cli

+ def handle_remove_sidetag(options, session, args):

+     "Remove sidetag"

+     usage = _("usage: %(prog)s remove-sidetag [options] <sidetag> ...")

+     usage += _("\n(Specify the --help global option for a list of other help options)")

+     parser = ArgumentParser(usage=usage)

+     parser.add_argument("sidetags", help="name of sidetag", nargs="+")

+     opts = parser.parse_args(args)

+ 

+     activate_session(session, options)

+ 

+     session.multicall = True

+     for sidetag in opts.sidetags:

+         session.removeSideTag(sidetag)

+     session.multiCall(strict=True)

+ 

+ 

+ @export_cli

+ def handle_list_sidetags(options, session, args):

+     "List sidetags"

+     usage = _("usage: %(prog)s list-sidetags [options]")

+     usage += _("\n(Specify the --help global option for a list of other help options)")

+     parser = ArgumentParser(usage=usage)

+     parser.add_argument("--basetag", action="store", help=_("Filter on basetag"))

+     parser.add_argument("--user", action="store", help=_("Filter on user"))

+     parser.add_argument("--mine", action="store_true", help=_("Filter on user"))

+ 

+     opts = parser.parse_args(args)

+ 

+     if opts.mine and opts.user:

+         parser.error(_("Specify only one from --user --mine"))

+ 

+     if opts.mine:

+         activate_session(session, options)

+         user = session.getLoggedInUser()["name"]

+     else:

+         user = opts.user

+ 

+     for tag in session.listSideTags(basetag=opts.basetag, user=user):

+         print (tag["name"])

@@ -0,0 +1,3 @@ 

+ [sidetag]

+ # automatically remove sidetag on untagging last package

+ remove_empty = off

@@ -0,0 +1,208 @@ 

+ # Copyright Âİ 2019 Red Hat, Inc.

+ #

+ # SPDX-License-Identifier: GPL-2.0-or-later

+ import sys

+ 

+ from koji.context import context

+ from koji.plugin import export, callback

+ import koji

+ 

+ CONFIG_FILE = "/etc/koji-hub/plugins/sidetag.conf"

+ CONFIG = None

+ 

+ sys.path.insert(0, "/usr/share/koji-hub/")

+ from kojihub import (

+     assert_policy,

+     get_tag,

+     get_user,

+     get_build_target,

+     _create_tag,

+     _create_build_target,

+     _delete_tag,

+     _delete_build_target,

+     QueryProcessor,

+     nextval,

+ )

+ 

+ 

+ @export

+ def createSideTag(basetag):

+     """Create a side tag.

+ 

+     :param basetag: name or ID of base tag

+     :type basetag: str or int

+     """

+ 

+     # Any logged-in user is able to request creation of side tags,

+     # as long the request meets the policy.

+     context.session.assertLogin()

+     user = get_user(context.session.user_id, strict=True)

+ 

+     basetag = get_tag(basetag, strict=True)

+ 

+     query = QueryProcessor(

+         tables=["tag_extra"],

+         clauses=["key='sidetag_user_id'", "value=%(user_id)s", "active IS TRUE"],

+         columns=["COUNT(*)"],

+         aliases=["user_tags"],

+         values={"user_id": str(user["id"])},

+     )

+     user_tags = query.executeOne()

+     if user_tags is None:

+         # should not ever happen

+         raise koji.GenericError("Unknown db error")

+ 

+     # Policy is a very flexible mechanism, that can restrict for which

+     # tags sidetags can be created, or which users can create sidetags etc.

+     assert_policy(

+         "sidetag", {"tag": basetag["id"], "number_of_tags": user_tags["user_tags"]}

+     )

+ 

+     # ugly, it will waste one number in tag_id_seq, but result will match with

+     # id assigned by _create_tag

+     tag_id = nextval("tag_id_seq") + 1

+     sidetag_name = "%s-side-%s" % (basetag["name"], tag_id)

+     sidetag_id = _create_tag(

+         sidetag_name,

+         parent=basetag["id"],

+         arches=basetag["arches"],

+         extra={

+             "sidetag": True,

+             "sidetag_user": user["name"],

+             "sidetag_user_id": user["id"],

+         },

+     )

+     _create_build_target(sidetag_name, sidetag_id, sidetag_id)

+ 

+     return {"name": sidetag_name, "id": sidetag_id}

+ 

+ 

+ @export

+ def removeSideTag(sidetag):

+     """Remove a side tag

+ 

+     :param sidetag: id or name of sidetag

+     :type sidetag: int or str

+     """

+     context.session.assertLogin()

+     user = get_user(context.session.user_id, strict=True)

+     sidetag = get_tag(sidetag, strict=True)

+ 

+     # sanity/access

+     if not sidetag["extra"].get("sidetag"):

+         raise koji.GenericError("Not a sidetag: %(name)s" % sidetag)

+     if sidetag["extra"].get("sidetag_user_id") != user["id"]:

+         if not context.session.hasPerm("admin"):

+             raise koji.ActionNotAllowed("This is not your sidetag")

+     _remove_sidetag(sidetag)

+ 

+ 

+ def _remove_sidetag(sidetag):

+     # check target

+     target = get_build_target(sidetag["name"])

+     if not target:

+         raise koji.GenericError("Target is missing for sidetag")

+     if target["build_tag"] != sidetag["id"] or target["dest_tag"] != sidetag["id"]:

+         raise koji.GenericError("Target does not match sidetag")

+ 

+     _delete_build_target(target["id"])

+     _delete_tag(sidetag["id"])

+ 

+ 

+ @export

+ def listSideTags(basetag=None, user=None, queryOpts=None):

+     """List all sidetags with additional filters

+ 

+     :param basetag: filter by basteag id or name

+     :type basetag: int or str

+     :param user: filter by userid or username

+     :type user: int or str

+     :param queryOpts: additional query options

+                       {countOnly, order, offset, limit}

+     :type queryOpts: dict

+     """

+     # te1.sidetag

+     # te2.user_id

+     # te3.basetag

+     if user is not None:

+         user_id = str(get_user(user, strict=True)["id"])

+     else:

+         user_id = None

+     if basetag is not None:

+         basetag_id = get_tag(basetag, strict=True)["id"]

+     else:

+         basetag_id = None

+ 

+     joins = ["LEFT JOIN tag_extra AS te1 ON tag.id = te1.tag_id"]

+     clauses = ["te1.active IS TRUE", "te1.key = 'sidetag'", "te1.value = 'true'"]

+     if user_id:

+         joins.append("LEFT JOIN tag_extra AS te2 ON tag.id = te2.tag_id")

+         clauses.extend(

+             [

+                 "te2.active IS TRUE",

+                 "te2.key = 'sidetag_user_id'",

+                 "te2.value = %(user_id)s",

+             ]

+         )

+     if basetag_id:

+         joins.append("LEFT JOIN tag_inheritance ON tag.id = tag_inheritance.tag_id")

+         clauses.extend(

+             [

+                 "tag_inheritance.active IS TRUE",

+                 "tag_inheritance.parent_id = %(basetag_id)s",

+             ]

+         )

+ 

+     query = QueryProcessor(

+         tables=["tag"],

+         clauses=clauses,

+         columns=["tag.id", "tag.name"],

+         aliases=["id", "name"],

+         joins=joins,

+         values={"basetag_id": basetag_id, "user_id": user_id},

+         opts=queryOpts,

+     )

+     return query.execute()

+ 

+ 

+ def handle_sidetag_untag(cbtype, *args, **kws):

+     """Remove a side tag when its last build is untagged

+ 

+     Note, that this is triggered only in case, that some build exists. For

+     never used tags, some other policy must be applied. Same holds for users

+     which don't untag their builds.

+     """

+     if "tag" not in kws:

+         # shouldn't happen, but...

+         return

+     tag = get_tag(kws["tag"]["id"], strict=False)

+     if not tag:

+         # also shouldn't happen, but just in case

+         return

+     if not tag["extra"].get("sidetag"):

+         # not a side tag

+         return

+     # is the tag now empty?

+     query = QueryProcessor(

+         tables=["tag_listing"],

+         clauses=["tag_id = %(tag_id)s", "active IS TRUE"],

+         values={"tag_id": tag["id"]},

+         opts={"countOnly": True},

+     )

+     if query.execute():

+         return

+     # looks like we've just untagged the last build from a side tag

+     try:

+         # XXX: are we double updating tag_listing?

+         _remove_sidetag(tag)

+     except koji.GenericError:

+         pass

+ 

+ 

+ # read config and register

+ if not CONFIG:

+     CONFIG = koji.read_config_files(CONFIG_FILE)

+     if CONFIG.has_option("sidetag", "remove_empty") and CONFIG.getboolean(

+         "sidetag", "remove_empty"

+     ):

+         handle_sidetag_untag = callback("postUntag")(handle_sidetag_untag)

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

4 years ago

So while you are doing this, can you please make creator of side tag to be able to modify the tag & block/unblock packages in it?

1 new commit added

  • fix basetag argument
4 years ago

Metadata Update from @jcupova:
- Pull-request tagged with: testing-done

4 years ago

trivial flake8 check:

./plugins/cli/sidetag_cli.py:11:1: F401 'koji_cli.lib.watch_tasks' imported but unused
./plugins/hub/sidetag_hub.py:8:1: I100 Import statements are in the wrong order. 'import sys' should be before 'import koji' and in a different group.
./plugins/hub/sidetag_hub.py:8:1: I201 Missing newline between import groups. 'import sys' is identified as Stdlib and 'import koji' is identified as Application.
./plugins/hub/sidetag_hub.py:14:1: F401 'kojihub.readTaggedBuilds' imported but unused
make: *** [Makefile:139: flake8] Error 1

1 new commit added

  • remove unused imports
4 years ago

Commit c3cfe38 fixes this pull-request

Pull-Request has been merged by tkopecek

4 years ago