From dc7c2322d1e9af4190fec99773c7d826d75c24fe Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Nov 15 2017 02:26:29 +0000 Subject: PR#699: Add documentation for storage volumes Merges #699 https://pagure.io/koji/pull-request/699 Fixes: #667 https://pagure.io/koji/issue/667 Docs for volume policy --- diff --git a/docs/source/defining_hub_policies.rst b/docs/source/defining_hub_policies.rst index 03c20f1..dd2ddb9 100644 --- a/docs/source/defining_hub_policies.rst +++ b/docs/source/defining_hub_policies.rst @@ -119,6 +119,8 @@ The system currently looks for the following policies requested * package_list : checked when the package list for a tag is modified * channel : consulted when a task is created +* cg_import : consulted during content generator imports +* volume : determine which volume a build should live on These policies are set by assigning a rule set to the given name in the policy section. diff --git a/docs/source/index.rst b/docs/source/index.rst index e44a4f2..87e28df 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -34,6 +34,7 @@ Contents using_the_koji_build_system profiles plugins + volumes writing_a_plugin writing_koji_code content_generators diff --git a/docs/source/volumes.rst b/docs/source/volumes.rst new file mode 100644 index 0000000..2c5c23a --- /dev/null +++ b/docs/source/volumes.rst @@ -0,0 +1,143 @@ +Storage Volumes +=============== + + +Introduction +------------ + +Since version 1.7.0, Koji has had the ability to store builds on +multiple volumes. + +Each build in Koji has a volume field that indicates which volume it is +stored on. The default volume is named ``DEFAULT`` and corresponds to the +original paths under ``/mnt/koji`` that predate this feature. + +Additional volumes correspond to paths under +``/mnt/koji/vol/NAME`` (where NAME is the name of the volume). All builds +associated with such a volume will be stored under this directory. +The expectation is that this directory will map to a different file system. + + +.. hint:: + | If ``koji-1.13.0-1`` is on the DEFAULT volume, its path will be: + | /mnt/koji/packages/koji/1.13.0/1 + + | If ``koji-1.13.0-1`` is on the volume named "test", its path will be: + | /mnt/koji/vol/test/packages/koji/1.13.0/1 + +Adding a new volume +------------------- + +The new volume directory should initially contain a packages/ +subdirectory, and the permissions should be the same as the default +packages directory. + +Assuming you do use a mount for a vol/NAME directory, you will want to +ensure that the same mounts are created on all systems that interface with +``/mnt/koji``, such as builders that run createrepo tasks, hosts running +kojira or similar maintenance, and any hosts that rely on the topdir option +rather than the topurl option. + +Once you have the directory set up, you can tell Koji about it by +running ``koji add-volume NAME``. This call will fail if the hub can't find +the directory. + +Moving builds onto other volumes +-------------------------------- + +By default, all builds live on the DEFAULT volume. + +An admin can move a build to a different volume by using the +``koji set-build-volume`` command, or by using the underlying +``changeBuildVolume`` api call. + +Moving a build across volumes will cause kojira to trigger repo +regenerations, if appropriate. When the new volume is not DEFAULT, Koji will +create a relative symlink to the new build directory on the default +volume. Moving builds across volumes may immediately break repos (until +the regen occurs), so use caution. + +Consider the following example: + +:: + + # mypkg-1.1-20 initially on default volume + $ file /mnt/koji/packages/mypkg/1.1/20 + /mnt/koji/packages/mypkg/1.1/20: directory + + # move it to test volume + $ koji set-build-volume test mypkg-1.1-20 + $ file /mnt/koji/vol/test/packages/mypkg/1.1/20 + /mnt/koji/vol/test/packages/mypkg/1.1/20: directory + + # original location is now a symlink + $ file /mnt/koji/packages/mypkg/1.1/20 + /mnt/koji/packages/mypkg/1.1/20: symbolic link to ../../../vol/test/packages/mypkg/1.1/20 + + +Using the volume in policy checks +--------------------------------- + +Policies involving builds (e.g. gc policy, tag policy), can test a +build's volume with the ``volume`` test. This is a pattern match +test against the volume name. + +Setting a volume policy +----------------------- + +The Koji 1.14.0 release adds the ability to set a volume policy on the hub. +This policy is used at import time to determine which volume the build should +be assigned to. This provides a systematic way to distribute builds +across multiple volumes without manual intervention. + +There is relatively limited data available to the volume policy. Tests that are +expected to work include: + +- user based tests (the user performing the build or running the import) +- package based tests (e.g. ``is_new_package`` or ``package``) +- cg match tests +- the buildtag test + +The action value for the volume policy should be simply the name of the volume +to use. + +The default volume policy is ``all :: DEFAULT``. + +If the volume policy contains errors, or does not return a result, then the +DEFAULT volume is used. + +For more information about Koji policies see: +:doc:`Defining hub policies ` + + +CLI commands +------------ + +``add-volume`` + adds a new volume (directory must already be set up) +``list-volumes`` + prints a list of known volumes +``set-build-volume`` + moves a build to different volume + + +API calls +--------- + +``addVolume(name, strict=True)`` + Add a new storage volume in the database + +``applyVolumePolicy(build, strict=False)`` + Apply the volume policy to a given build + +``changeBuildVolume(build, volume, strict=True)`` + Move a build to a different storage volume + +``getVolume(volume, strict=False)`` + Lookup the given volume + +``listVolumes()`` + List storage volumes + +``removeVolume(volume)`` + Remove unused storage volume from the database diff --git a/hub/kojihub.py b/hub/kojihub.py index dedcbc6..d413cec 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9102,9 +9102,24 @@ class RootExports(object): changeBuildVolume = staticmethod(change_build_volume) def getVolume(self, volume, strict=False): + """Lookup the given volume + + Returns a dictionary containing the name and id of the matching + volume, or None if no match. + If strict is true, raises an error if no match. + """ return lookup_name('volume', volume, strict=strict) def applyVolumePolicy(self, build, strict=False): + """Apply the volume policy to a given build + + The volume policy is normally applied at import time, but it can + also be applied with this call. + + Parameters: + build: the build to apply the policy to + strict: if True, raises on exception on policy issues + """ context.session.assertPerm('admin') build = get_build(build, strict=True) return apply_volume_policy(build, strict)