#317 Generate symlinks based on the pungi_compose_id pointing to ODCS composes.
Merged 4 years ago by lsedlar. Opened 4 years ago by jkaluza.
jkaluza/odcs compose-info  into  master

@@ -21,6 +21,7 @@ 

  #

  # Written by Jan Kaluza <jkaluza@redhat.com>

  

+ import errno

  import koji

  import os

  import threading
@@ -218,6 +219,7 @@ 

              compose.transition(COMPOSE_STATES["removed"], state_reason)

              if not compose.reused_id:

                  self._remove_compose_dir(compose.toplevel_dir)

+                 remove_compose_symlink(compose)

  

          # In case of ODCS error, there might be left-over directories

          # belonging to already expired composes. Try to find them in the
@@ -662,6 +664,74 @@ 

      log.info("%r: Compose done", compose)

  

  

+ def generate_compose_symlink(compose):

+     """

+     Generates symlink(s) for compose based on its `compose.pungi_compose_id`.

+ 

+     It generates following symlinks pointing to the compose:

+       - $conf.target_dir/$compose.compose_type/$compose.pungi_compose_id

+       - $conf.target_dir/$compose.compose_type/latest-$name-version

+ 

+     If the latest-* symlink exists, it is replaced with new one pointing to

+     the `composes`.

+     """

+     symlink_dir = os.path.join(conf.target_dir, compose.compose_type)

+     odcs.server.utils.makedirs(symlink_dir)

+ 

+     # Generate the non-latest symlink.

+     compose_dir = os.path.relpath(compose.toplevel_work_dir, symlink_dir)

+     symlink = os.path.join(symlink_dir, compose.pungi_compose_id)

+     log.info("%r: Generating %s symlink.", compose, symlink)

+     os.symlink(compose_dir, symlink)

+ 

+     # Generate the latest-* symlink.

+     latest_name = "latest-%s" % "-".join(compose.pungi_compose_id.split("-")[:2])

+     symlink = os.path.join(symlink_dir, latest_name)

+     try:

+         os.unlink(symlink)

+     except OSError as e:

+         if e.errno != errno.ENOENT:

+             raise

+     log.info("%r: Generating %s symlink.", compose, symlink)

+     os.symlink(compose_dir, symlink)

+ 

+ 

+ def remove_compose_symlink(compose):

+     """

+     Removes non-latest symlink generated by the `generate_compose_symlink`.

+     """

+     # Do not try to remove symlinks from old composes which do not have

+     # the compose_type and pungi_compose_id set.

+     if not compose.compose_type or not compose.pungi_compose_id:

+         return

+ 

+     symlink_dir = os.path.join(conf.target_dir, compose.compose_type)

+     symlink = os.path.join(symlink_dir, compose.pungi_compose_id)

+ 

+     # Check if latest_symlink points to the same directory as the non-latest

+     # symlink. In this case, we will remove the latest-symlink later too.

+     latest_name = "latest-%s" % "-".join(compose.pungi_compose_id.split("-")[:2])

+     latest_symlink = os.path.join(symlink_dir, latest_name)

+     remove_latest_symlink = os.path.realpath(symlink) == os.path.realpath(latest_symlink)

+ 

+     # Remove non-latest symlink.

+     log.info("%r: Removing %s symlink.", compose, symlink)

+     try:

+         os.unlink(symlink)

+     except OSError as e:

+         if e.errno != errno.ENOENT:

+             raise

+ 

+     # Remove latest symlink.

+     if remove_latest_symlink:

+         log.info("%r: Removing %s symlink.", compose, latest_symlink)

+         try:

+             os.unlink(latest_symlink)

+         except OSError as e:

+             if e.errno != errno.ENOENT:

+                 raise

+ 

+ 

  def generate_pungi_compose(compose):

      """

      Generates the compose of KOJI, TAG, or REPO type using the Pungi tool.
@@ -727,6 +797,10 @@ 

      # Raises an exception if invalid

      validate_pungi_compose(compose)

  

+     # Generate symlinks pointing to latest build of raw_config compose.

+     if compose.source_type == PungiSourceType.RAW_CONFIG:

+         generate_compose_symlink(compose)

+ 

      # If there is no exception generated by the pungi.run() and if

      # validation didn't fail, then we know the compose has been

      # successfully generated.

file modified
+18 -3
@@ -24,7 +24,7 @@ 

  import os

  import shutil

  

- from mock import patch, MagicMock

+ from mock import patch, MagicMock, call

  from productmd.rpms import Rpms

  

  from odcs.server import conf, db
@@ -38,7 +38,7 @@ 

                                   koji_get_inherited_tags)

  from odcs.server.utils import makedirs

  import odcs.server.backend

- from .utils import ModelsBaseTest

+ from .utils import ModelsBaseTest, AnyStringWith

  

  from .mbs import mock_mbs

  
@@ -937,10 +937,15 @@ 

                            "url": "git://localhost/test.git",

                            "config_filename": "pungi.conf"}

                    })

-     def test_generate_pungi_compose_raw_config(self):

+     @patch("odcs.server.utils.makedirs")

+     @patch("os.symlink")

+     @patch("os.unlink")

+     def test_generate_pungi_compose_raw_config(self, unlink, symlink, makedirs):

          c = Compose.create(

              db.session, "me", PungiSourceType.RAW_CONFIG, "pungi_cfg#hash",

              COMPOSE_RESULTS["repository"], 60)

+         c.compose_type = "nightly"

+         c.pungi_compose_id = "compose-1-10-2020110.n.0"

          c.id = 1

  

          fake_raw_config_urls = {
@@ -958,6 +963,16 @@ 

              'commit': 'hash'

          })

  

+         makedirs.assert_called_once_with(AnyStringWith("/test_composes/nightly"))

+         symlink.assert_has_calls([

+             call('../odcs-1-2018-1',

+                  AnyStringWith('/test_composes/nightly/compose-1-10-2020110.n.0')),

+             call('../odcs-1-2018-1',

+                  AnyStringWith('/test_composes/nightly/latest-compose-1')),

+         ])

+         unlink.assert_called_with(

+             AnyStringWith('/test_composes/nightly/latest-compose-1'))

+ 

  

  class TestValidatePungiCompose(ModelsBaseTest):

      """Test validate_pungi_compose"""

@@ -67,13 +67,18 @@ 

              c = db.session.query(Compose).filter(Compose.id == 1).one()

              self.assertEqual(c.state, state)

  

-     def test_a_compose_which_state_is_done_is_removed(self):

+     @patch("os.unlink")

+     @patch("os.path.realpath")

+     def test_a_compose_which_state_is_done_is_removed(self, realpath, unlink):

          """

          Test that we do remove a compose in done state.

          """

+         realpath.return_value = "/odcs-real"

          c = db.session.query(Compose).filter(Compose.id == 1).one()

          c.time_to_expire = datetime.utcnow() - timedelta(seconds=120)

          c.state = COMPOSE_STATES["done"]

+         c.compose_type = "nightly"

+         c.pungi_compose_id = "compose-1-10-2020110.n.0"

          db.session.add(c)

          db.session.commit()

          self.thread.do_work()
@@ -81,6 +86,9 @@ 

          c = db.session.query(Compose).filter(Compose.id == 1).one()

          self.assertEqual(c.state, COMPOSE_STATES["removed"])

          self.assertEqual(c.state_reason, 'Compose is expired.')

+         unlink.assert_has_calls([

+             mock.call(AnyStringWith("test_composes/nightly/compose-1-10-2020110.n.0")),

+             mock.call(AnyStringWith("test_composes/nightly/latest-compose-1"))])

  

      def test_a_compose_which_state_is_done_is_removed_keep_state_reason(self):

          """

This commit adds code to generate symlinks for raw_config composes in the following
format:

  • $conf.target_dir/$compose.compose_type/$compose.pungi_compose_id
  • $conf.target_dir/$compose.compose_type/latest-$name-version

The goal is to make it easier for end-user to consume raw_config
composes in the future and to find out the latest successfull
raw_config compose.

Signed-off-by: Jan Kaluza jkaluza@redhat.com

@lsedlar, I have example composes generated with this PR on my testing VM here: http://10.0.145.16/composes/nightly/ (sorry if someone else is reviewing this without access to that network :( )

Do we care that the symlink will be missing for a short time?

We only care about symlink once the compose is done. Before it is done, it can be in whatever state.

Should the latest symlink be removed too if it points to the removed compose?

We only care about symlink once the compose is done. Before it is done, it can be in whatever state.

Ok. The reason for asking is that sooner or later, someone will write a tool that will consume the latest nightly compose, and if it hits the moment when the symlink is updated, it will see it missing. However it's probably not a big deal, since it will most likely make multiple requests, and having the compose replaced with newer version for a single run will possibly cause problems anyway. But this is nothing new, so it should be fine.

1 new commit added

  • Remove also latest-* symlink if it points to the removed compose.
4 years ago

@lsedlar, latest commit removes also latest-* symlink.

Pull-Request has been merged by lsedlar

4 years ago