#4383 handle cross-volume draft symlinks
Merged 6 days ago by tkopecek. Opened a month ago by mikem.
mikem/koji volume-toplink  into  master

@@ -440,7 +440,7 @@ 

  

  * ``mock.bootstrap_image_ready`` - For the specification that we are ready for bootstrapping.

    If we want to specify, that we are ready for bootstrapping, we need to know that these

-   two packages `python3-dnf` and `python3-dnf-plugins-coreare installed in the container.

+   two packages ``python3-dnf`` and ``python3-dnf-plugins-core`` are installed in the container.

  

  * ``mock.module_setup_commands`` - commands for configuring the modules active

    in a buildroot. Available in `mock 2.4

file modified
+11 -3
@@ -73,9 +73,17 @@ 

  Adding a new volume

  -------------------

  

- The new volume directory should initially contain a packages/

- subdirectory, and the permissions should be the same as the default

- packages directory.

+ The new volume directory should initially contain:

+ 

+ * a ``packages`` subdirectory

+ * a ``toplink`` symlink to the primary volume (i.e. /mnt/koji)

+ 

+ The permissions for the ``packages`` directory should be the same as the

+ default packages directory.

+ 

+ The ``toplink`` symlink should normally be an absolute symlink.

+ If you do not create it, then Koji will do so for you when you run the

+ add-volume command.

  

  Assuming you do use a mount for a vol/NAME directory, you will want to

  ensure that the same mounts are created on all systems that interface with

file modified
+75 -7
@@ -6116,6 +6116,22 @@ 

      voldir = koji.pathinfo.volumedir(name)

      if not os.path.isdir(voldir):

          raise koji.GenericError('please create the volume directory first')

+ 

+     # volume directories should have a symlink to default volume, e.g. /mnt/koji

+     toplink = joinpath(voldir, 'toplink')

+     if os.path.islink(toplink):

+         if not os.path.exists(toplink):

+             raise koji.GenericError(f'Broken volume toplink: {toplink}')

+         if not os.path.samefile(toplink, koji.pathinfo.topdir):

+             raise koji.GenericError(f'Invalid volume toplink: {toplink}')

+     elif os.path.exists(toplink):

+         # not a link

+         raise koji.GenericError(f'Not a symlink: {toplink}')

+     else:

+         target = koji.pathinfo.topdir

+         logger.warning('No toplink for volume. Creating {toplink} -> {target}')

+         os.symlink(target, toplink)

+ 

      if strict:

          volinfo = lookup_name('volume', name, strict=False)

          if volinfo:
@@ -6265,6 +6281,64 @@ 

      os.symlink(relpath, basedir)

  

  

+ def ensure_volume_backlink(old_binfo, new_binfo=None):

+     """Ensure we have a link for a build on given non-default volume

+ 

+     We point the symlink at the default volume location, because this path should

+     always be either valid symlink or the actual build dir.

+ 

+     Note: this is tricky!

+     We primarily use relative symlinks under /mnt/koji, however this can break

+     when crossing volumes. In general, /mnt/koji/vol/foo/.. != /mnt/koji/vol

+     However, relpath() assumes this is the case because it is "a path computation"

+     """

+ 

+     # basic checks

+     volname = old_binfo['volume_name']

+     if volname == 'DEFAULT':

+         # nothing to do

+         # default volume symlinks are handled in ensure_volume_symlink()

+         return

+     voldir = koji.pathinfo.volumedir(volname)

+     if not os.path.isdir(voldir):

+         raise koji.GenericError(f'Missing volume dir: {voldir}')

+     toplink = joinpath(voldir, 'toplink')

+     if not os.path.exists(toplink):

+         raise koji.GenericError(f'Missing volume toplink: {toplink}')

+ 

+     # get the old build path (where we will place the symlink)

+     olddir = koji.pathinfo.build(old_binfo)

+ 

+     # construct the relative path in parts

+     # - relpath to voldir

+     # - voldir/toplink is a symlink to topdir

+     # - relpath from topdir to olddir

+     path1 = os.path.relpath(voldir, os.path.dirname(olddir))  # should be ../../..

+     assert path1 == '../../..'  # XXX

+     relpathinfo = koji.PathInfo(topdir='toplink')

+     if new_binfo is not None:

+         base_binfo = new_binfo.copy()

+     else:

+         # call can pass just old_binfo if NVR is not changing

+         base_binfo = old_binfo.copy()

+     base_binfo['volume_name'] = 'DEFAULT'

+     path2 = relpathinfo.build(base_binfo)  # toplink/packages/N/V/R

+ 

+     # check/make the symlink

+     relpath = joinpath(path1, path2)

+     if os.path.islink(olddir):

+         if os.readlink(olddir) == relpath:

+             # already correct

+             return

+         os.unlink(olddir)

+     elif os.path.exists(olddir):

+         raise koji.GenericError('Unexpected build content: %s' % olddir)

+     else:

+         # parent dir might not exist

+         koji.ensuredir(os.path.dirname(olddir))

+     os.symlink(relpath, olddir)

+ 

+ 

  def check_volume_policy(data, strict=False, default=None):

      """Check volume policy for the given data

  
@@ -10681,13 +10755,7 @@ 

  

      # provide a symlink at original draft location

      # we point to the default volume in case the build moves in the future

-     base_vol = lookup_name('volume', 'DEFAULT', strict=True)

-     base_binfo = new_binfo.copy()

-     base_binfo['volume_id'] = base_vol['id']

-     base_binfo['volume_name'] = base_vol['name']

-     basedir = koji.pathinfo.build(base_binfo)

-     relpath = os.path.relpath(basedir, os.path.dirname(oldpath))

-     os.symlink(relpath, oldpath)

+     ensure_volume_backlink(binfo, new_binfo)

  

      # apply volume policy in case it's changed by release update.

      apply_volume_policy(new_binfo, strict=False)

@@ -1,3 +1,6 @@ 

+ import os

+ import shutil

+ import tempfile

  import unittest

  

  from unittest import mock
@@ -9,10 +12,26 @@ 

  class TestAddVolume(unittest.TestCase):

  

      def setUp(self):

+         # set up topdir

+         self.tempdir = tempfile.mkdtemp()

+         self.topdir = self.tempdir + '/koji'

+         self.pathinfo = koji.PathInfo(self.topdir)

+         mock.patch('koji.pathinfo', new=self.pathinfo).start()

+ 

+         # set up test volume

+         vol = self.tempdir + '/vol_test'

+         self.volmount = vol

+         toplink = vol + '/toplink'

+         koji.ensuredir(vol)

+         voldir = self.pathinfo.volumedir('test-volume')

+         koji.ensuredir(os.path.dirname(voldir))  # koji/vol

+         os.symlink(vol, voldir)

+         os.symlink(self.topdir, toplink)

+ 

          self.verify_name_internal = mock.patch('kojihub.kojihub.verify_name_internal').start()

          self.lookup_name = mock.patch('kojihub.kojihub.lookup_name').start()

-         self.isdir = mock.patch('os.path.isdir').start()

-         self.pathinfo_volumedir = mock.patch('koji.pathinfo.volumedir').start()

+         # self.isdir = mock.patch('os.path.isdir').start()

+         # self.pathinfo_volumedir = mock.patch('koji.pathinfo.volumedir').start()

          self.context = mock.patch('kojihub.kojihub.context').start()

          # It seems MagicMock will not automatically handle attributes that

          # start with "assert"
@@ -21,6 +40,7 @@ 

  

      def tearDown(self):

          mock.patch.stopall()

+         shutil.rmtree(self.tempdir)

  

      def test_add_volume_wrong_format(self):

          volume_name = 'volume-name+'
@@ -36,34 +56,88 @@ 

              kojihub.add_volume(volume_name)

  

      def test_non_exist_directory(self):

-         volume_name = 'test-volume'

-         self.isdir.return_value = False

-         self.pathinfo_volumedir.return_value = 'path/to/volume'

+         volume_name = 'no-such-volume'

+ 

          with self.assertRaises(koji.GenericError) as cm:

              kojihub.add_volume(volume_name)

+ 

          self.assertEqual("please create the volume directory first", str(cm.exception))

          self.verify_name_internal.assert_called_once_with(volume_name)

          self.lookup_name.assert_not_called()

  

      def test_valid(self):

          volume_name = 'test-volume'

-         volume_dict = {'id': 0, 'name': volume_name}

-         self.isdir.return_value = True

-         self.pathinfo_volumedir.return_value = 'path/to/volume'

+         volume_dict = {'id': 1, 'name': volume_name}

          self.lookup_name.return_value = volume_dict

+ 

          rv = kojihub.add_volume(volume_name, strict=False)

+ 

          self.assertEqual(rv, volume_dict)

          self.verify_name_internal.assert_called_once_with(volume_name)

          self.lookup_name.assert_called_once_with('volume', volume_name, strict=False, create=True)

  

      def test_volume_exists(self):

          volume_name = 'test-volume'

-         volume_dict = {'id': 0, 'name': volume_name}

-         self.isdir.return_value = True

-         self.pathinfo_volumedir.return_value = 'path/to/volume'

+         volume_dict = {'id': 1, 'name': volume_name}

          self.lookup_name.return_value = volume_dict

+ 

          with self.assertRaises(koji.GenericError) as cm:

              kojihub.add_volume(volume_name, strict=True)

+ 

          self.assertEqual(f'volume {volume_name} already exists', str(cm.exception))

          self.verify_name_internal.assert_called_once_with(volume_name)

          self.lookup_name.assert_called_once_with('volume', volume_name, strict=False)

+ 

+     def test_volume_broken_toplink(self):

+         volume_name = 'test-volume'

+         volume_dict = {'id': 1, 'name': volume_name}

+         self.lookup_name.side_effect = [None, volume_dict]

+         toplink = self.volmount + '/toplink'

+         os.unlink(toplink)

+         os.symlink('BROKEN-LINK', toplink)

+ 

+         with self.assertRaises(koji.GenericError) as cm:

+             kojihub.add_volume(volume_name, strict=True)

+ 

+         assert str(cm.exception).startswith('Broken volume toplink')

+ 

+     def test_volume_invalid_toplink(self):

+         volume_name = 'test-volume'

+         volume_dict = {'id': 1, 'name': volume_name}

+         self.lookup_name.side_effect = [None, volume_dict]

+         toplink = self.volmount + '/toplink'

+         os.unlink(toplink)

+         os.symlink('..', toplink)

+ 

+         with self.assertRaises(koji.GenericError) as cm:

+             kojihub.add_volume(volume_name, strict=True)

+ 

+         assert str(cm.exception).startswith('Invalid volume toplink')

+ 

+     def test_volume_nonlink_toplink(self):

+         volume_name = 'test-volume'

+         volume_dict = {'id': 1, 'name': volume_name}

+         self.lookup_name.side_effect = [None, volume_dict]

+         toplink = self.volmount + '/toplink'

+         os.unlink(toplink)

+         os.mkdir(toplink)

+ 

+         with self.assertRaises(koji.GenericError) as cm:

+             kojihub.add_volume(volume_name, strict=True)

+ 

+         assert str(cm.exception).startswith('Not a symlink')

+ 

+     def test_volume_create_toplink(self):

+         # test that the toplink is automatically created if missing

+         volume_name = 'test-volume'

+         volume_dict = {'id': 1, 'name': volume_name}

+         self.lookup_name.side_effect = [None, volume_dict]

+         toplink = self.volmount + '/toplink'

+         os.unlink(toplink)

+ 

+         kojihub.add_volume(volume_name, strict=True)

+ 

+         self.assertEqual(os.readlink(toplink), self.pathinfo.topdir)

+ 

+ 

+ # the end

@@ -0,0 +1,150 @@ 

+ from unittest import mock

+ import os

+ import os.path

+ import shutil

+ import tempfile

+ import unittest

+ import koji

+ import kojihub

+ 

+ import pytest

+ 

+ class TestEnsureVolumeBacklink(unittest.TestCase):

+ 

+     def setUp(self):

+         self.tempdir = tempfile.mkdtemp()

+         self.topdir = self.tempdir + '/koji'

+         self.pathinfo = koji.PathInfo(self.topdir)

+         mock.patch('koji.pathinfo', new=self.pathinfo).start()

+ 

+         # set up other volume

+         vol = self.tempdir + '/vol_other'

+         self.volmount = vol

+         toplink = vol + '/toplink'

+         koji.ensuredir(vol)

+         voldir = self.pathinfo.volumedir('OTHER')

+         koji.ensuredir(os.path.dirname(voldir))  # koji/vol

+         os.symlink(vol, voldir)

+         os.symlink(self.topdir, toplink)

+ 

+         # mock.patch('kojihub.kojihub.lookup_name', new=self.my_lookup_name).start()

+         self.buildinfo = {

+                 'id': 137,

+                 'task_id': 'TASK_ID',

+                 'name': 'some-image',

+                 'version': '1.2.3.4',

+                 'release': '3',

+                 'epoch': None,

+                 'source': None,

+                 'state': koji.BUILD_STATES['BUILDING'],

+                 # 'volume_id': 1,

+                 'volume_name': 'OTHER',

+                 }

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+         shutil.rmtree(self.tempdir)

+ 

+     def my_lookup_name(self, table, info, **kw):

+         if table != 'volume':

+             raise Exception("Cannot fake call")

+         return {

+                 'id': 'VOLUMEID:%s' % info,

+                 'name': '%s' % info,

+                 }

+ 

+     def test_volume_symlink_no_action(self):

+         # the call should do nothing if the volume is DEFAULT

+         binfo = self.buildinfo.copy()

+         binfo['volume_name'] = 'DEFAULT'

+         files1 = list(find_files(self.topdir))

+ 

+         kojihub.ensure_volume_backlink(binfo)

+ 

+         files2 = list(find_files(self.topdir))

+         self.assertEqual(files1, files2)

+ 

+     def test_volume_symlink_create(self):

+         # verify that backlink is created correctly

+         basedir = self.pathinfo.build(self.buildinfo)  # OTHER volume

+ 

+         kojihub.ensure_volume_backlink(self.buildinfo)

+ 

+         files = list(find_files(self.volmount))

+         expected = [

+                 'packages',

+                 'toplink',

+                 'packages/some-image',

+                 'packages/some-image/1.2.3.4',

+                 'packages/some-image/1.2.3.4/3',

+                 ]

+         self.assertEqual(files, expected)

+         relpath = ('../../../toplink/packages/'

+                    '%(name)s/%(version)s/%(release)s' % self.buildinfo)

+         self.assertEqual(os.readlink(basedir), relpath)

+ 

+     def test_volume_symlink_exists(self):

+         # if an incorrect link is present, it should be replaced

+         basedir = self.pathinfo.build(self.buildinfo)  # OTHER volume

+         oldpath = 'some/other/link'

+         os.makedirs(os.path.dirname(basedir))

+         os.symlink(oldpath, basedir)

+ 

+         kojihub.ensure_volume_backlink(self.buildinfo)

+ 

+         relpath = ('../../../toplink/packages/'

+                    '%(name)s/%(version)s/%(release)s' % self.buildinfo)

+         self.assertEqual(os.readlink(basedir), relpath)

+ 

+     def test_volume_symlink_exists_same(self):

+         # if link is already correct, it should be left alone

+         basedir = self.pathinfo.build(self.buildinfo)  # OTHER volume

+         relpath = ('../../../toplink/packages/'

+                    '%(name)s/%(version)s/%(release)s' % self.buildinfo)

+         os.makedirs(os.path.dirname(basedir))

+         os.symlink(relpath, basedir)

+ 

+         with mock.patch('os.unlink') as unlink:

+             kojihub.ensure_volume_backlink(self.buildinfo)

+             unlink.assert_not_called()

+ 

+         files = list(find_files(self.volmount))

+         expected = [

+                 'packages',

+                 'toplink',

+                 'packages/some-image',

+                 'packages/some-image/1.2.3.4',

+                 'packages/some-image/1.2.3.4/3',

+                 ]

+         self.assertEqual(files, expected)

+ 

+     def test_volume_symlink_exists_error(self):

+         # if the path exists and is not a link, we should error

+         basedir = self.pathinfo.build(self.buildinfo)  # OTHER volume

+         os.makedirs(basedir)

+         files1 = list(find_files(self.tempdir))

+ 

+         with self.assertRaises(koji.GenericError):

+             kojihub.ensure_volume_backlink(self.buildinfo)

+ 

+         files2 = list(find_files(self.tempdir))

+         self.assertEqual(files1, files2)

+ 

+     def test_volume_symlink_exists_error(self):

+         # if the volume dir is bad, we should error

+         basedir = self.pathinfo.build(self.buildinfo)  # OTHER volume

+ 

+         os.unlink(self.volmount + '/toplink')

+         with self.assertRaises(koji.GenericError):

+             kojihub.ensure_volume_backlink(self.buildinfo)

+ 

+         os.rmdir(self.volmount)

+         with self.assertRaises(koji.GenericError):

+             kojihub.ensure_volume_backlink(self.buildinfo)

+ 

+ 

+ def find_files(dirpath):

+     '''Find all files under dir, report relative paths'''

+     for path, dirs, files in os.walk(dirpath):

+         for fn in sorted(dirs + files):

+             yield os.path.relpath(os.path.join(path, fn), dirpath)

@@ -1,5 +1,7 @@ 

  import datetime

- import json

+ import shutil

+ import os.path

+ import tempfile

  from unittest import mock

  import unittest

  
@@ -33,6 +35,7 @@ 

                                                return_value=None).start()

          self.safer_move = mock.patch('kojihub.kojihub.safer_move').start()

          self.ensure_volume_symlink = mock.patch('kojihub.kojihub.ensure_volume_symlink').start()

+         self.ensure_volume_backlink = mock.patch('kojihub.kojihub.ensure_volume_backlink').start()

          self.lookup_name = mock.patch('kojihub.kojihub.lookup_name',

                                        return_value={'id': 1, 'name': 'DEFAULT'}).start()

          self.os_symlink = mock.patch('os.symlink').start()
@@ -93,10 +96,7 @@ 

              '/mnt/koji/vol/X/packages/foo/bar/tgtrel,draft_1',

              '/mnt/koji/vol/X/packages/foo/bar/tgtrel'

          )

-         self.os_symlink.assert_called_once_with(

-             '../../../../../packages/foo/bar/tgtrel',

-             '/mnt/koji/vol/X/packages/foo/bar/tgtrel,draft_1'

-         )

+         self.os_symlink.assert_not_called()

  

      def test_promote_build_not_draft(self):

          self.get_build.return_value = {'draft': False, 'nvr': 'testnvr'}
@@ -177,3 +177,100 @@ 

              'version': 'bar',

              'release': 'tgtrel'

          }, strict=False)

+ 

+ 

+ class TestPromoteBuildFiles(unittest.TestCase):

+     # these tests use a tempdir

+ 

+     def getUpdate(self, *args, **kwargs):

+         update = UP(*args, **kwargs)

+         update.execute = mock.MagicMock()

+         self.updates.append(update)

+         return update

+ 

+     def setUp(self):

+         # set up our dirs

+         self.tempdir = tempfile.mkdtemp()

+         self.topdir = self.tempdir + '/koji'

+         self.pathinfo = koji.PathInfo(self.topdir)

+         mock.patch('koji.pathinfo', new=self.pathinfo).start()

+         # separate dir for volume X

+         vol_x = self.tempdir + '/vol_X'

+         toplink = self.tempdir + '/vol_X/toplink'

+         koji.ensuredir(vol_x)

+         voldir = self.pathinfo.volumedir('X')

+         koji.ensuredir(os.path.dirname(voldir))  # koji/vol

+         os.symlink(vol_x, voldir)

+         os.symlink(self.topdir, toplink)

+ 

+         self.exports = kojihub.RootExports()

+         self.UpdateProcessor = mock.patch('kojihub.kojihub.UpdateProcessor',

+                                           side_effect=self.getUpdate).start()

+         self.updates = []

+         self.context = mock.patch('kojihub.kojihub.context').start()

+         self.context.session.assertLogin = mock.MagicMock()

+         self.user = {'id': 1, 'name': 'jdoe'}

+         self.get_user = mock.patch('kojihub.kojihub.get_user', return_value=self.user).start()

+         self.get_build = mock.patch('kojihub.kojihub.get_build').start()

+         self.assert_policy = mock.patch('kojihub.kojihub.assert_policy').start()

+         self.apply_volume_policy = mock.patch('kojihub.kojihub.apply_volume_policy',

+                                               return_value=None).start()

+         self.lookup_name = mock.patch('kojihub.kojihub.lookup_name',

+                                       return_value={'id': 1, 'name': 'DEFAULT'}).start()

+         self.list_tags = mock.patch('kojihub.kojihub.list_tags',

+                                     return_value=[{'id': 101}]).start()

+         self.set_tag_update = mock.patch('kojihub.kojihub.set_tag_update').start()

+         self._now = datetime.datetime.now()

+         self._datetime = mock.patch('kojihub.kojihub.datetime.datetime').start()

+         self.now = self._datetime.now = mock.MagicMock(return_value=self._now)

+ 

+         self.draft_build = {

+             'id': 1,

+             'name': 'foo',

+             'version': 'bar',

+             'release': 'tgtrel,draft_1',

+             'nvr': 'testnvr',

+             'state': 1,

+             'draft': True,

+             'volume_id': 99,

+             'volume_name': 'X',

+             'task_id': 222

+         }

+ 

+         self.new_build = {

+             # no check on the info

+             'id': 1,

+             'name': 'foo',

+             'version': 'bar',

+             'release': 'tgtrel',

+             'volume_name': 'X'

+         }

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+         shutil.rmtree(self.tempdir)

+ 

+     def test_promote_build_volume_link(self):

+         self.get_build.side_effect = [

+             self.draft_build,

+             None,

+             self.new_build

+         ]

+         orig_bdir = self.pathinfo.build(self.draft_build)

+         koji.ensuredir(orig_bdir)

+         sentinel = 'HELLO 873\n'

+         with open(orig_bdir + '/sentinel.txt', 'wt') as fp:

+             fp.write(sentinel)

+ 

+         # promote

+         ret = self.exports.promoteBuild('a-draft-build')

+ 

+         self.assertEqual(ret, self.new_build)

+         # orig_bdir should be a symlink

+         assert os.path.islink(orig_bdir)

+         # should be accessible via original path

+         with open(orig_bdir + '/sentinel.txt', 'rt') as fp:

+             assert fp.read() == sentinel

+ 

+ 

+ # the end

The convenience symlinks generated when promoting a draft build might not work if the build is not on the default volume.

The solution to this is unfortunately a little complicated.

Fixes https://pagure.io/koji/issue/4386

The added unit test demonstrates the problem. It fails on current master and passes with the current changes.

7 new commits added

  • unrelated docs typo
  • update volume docs
  • update tests for add_volume
  • more unit tests
  • don't require new_binfo
  • require volume toplink to exist
  • check/create volume toplink in add_volume
a month ago

1 new commit added

  • typo
a month ago

OK, I think this might be about ready.

I'm a little unsure about whether auto-creating toplink in add_volume is the best idea, but it's going to be easier for admins. The docs recommend creating it manually.

1 new commit added

  • clean up test dir
a month ago

Longer term, we could also use this function to maintain compat symlinks on the old volume when moving across volumes, but I think it would be best to leave that out for now.

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-custom

a month ago

Metadata Update from @mfilip:
- Pull-request tagged with: testing-done

6 days ago

Commit 573bd41 fixes this pull-request

Pull-Request has been merged by tkopecek

6 days ago