#327 Add `odcs-promote-compose` server/contrib script.
Merged 4 years ago by lsedlar. Opened 4 years ago by jkaluza.
jkaluza/odcs routing-rules-regexp  into  master

@@ -0,0 +1,216 @@ 

+ #!/usr/bin/env python3

+ 

+ from __future__ import print_function

+ import shutil

+ import argparse

+ import sys

+ import os

+ import stat

+ 

+ from productmd.composeinfo import ComposeInfo

+ from productmd.rpms import Rpms

+ 

+ 

+ class ComposeCheckError(Exception):

+     """"

+     Raised when compose check fails.

+     """

+     pass

+ 

+ 

+ class ComposeCheck(object):

+     """

+     Checks the basic information about compose before promoting it.

+ 

+     This is not real Compose CI, but rather basic sanity check.

+     """

+     def __init__(self, path, target, allow_unsigned=False, allow_finished_incomplete=False):

+         """

+         Creates new ComposeCheck instance.

+ 

+         :param str path: Path to Compose to check.

+         :param str target: Target path where the promoted compose should be

+             copied into.

+         :param bool allow_unsigned: If True, compose with unsigned packages

+             can be promoted.

+         :param bool allow_finished_incomplete: If True, compose in FINISHED_INCOMPLETE

+             state can be promoted.

+         """

+         self.path = path

+         self.target = target

+         self.allow_unsigned = allow_unsigned

+         self.allow_finished_incomplete = allow_finished_incomplete

+ 

+     def check_status(self):

+         """

+         Raises ComposeCheckError if Compose STATUS is not FINISHED.

+         """

+         print("Checking compose STATUS.")

+ 

+         allowed_statuses = ["FINISHED"]

+         if self.allow_finished_incomplete:

+             allowed_statuses.append("FINISHED_INCOMPLETE")

+ 

+         status_path = os.path.join(self.path, "STATUS")

+         with open(status_path, "r") as f:

+             status = f.readline()[:-1]

+             if status not in allowed_statuses:

+                 err_msg = 'Compose is not in %s status.' % (" or ".join(allowed_statuses))

+                 raise ComposeCheckError(err_msg)

+ 

+     def check_compose_info(self):

+         """

+         Raises ComposeCheckError if Compose type is not "production".

+         """

+         print("Checking compose type.")

+         ci = ComposeInfo()

+         ci.load(os.path.join(self.path, "compose", "metadata", "composeinfo.json"))

+         if ci.compose.type != "production":

+             raise ComposeCheckError('Compose type is not "production".')

+ 

+     def check_rpms(self):

+         """

+         Raises ComposeCheckError if there are unsigned packages in the Compose.

+         """

+         if self.allow_unsigned:

+             return

+ 

+         print("Checking for unsigned RPMs.")

+         rpms = Rpms()

+         rpms.load(os.path.join(self.path, "compose", "metadata", "rpms.json"))

+         for per_arch_rpms in rpms.rpms.values():

+             for per_build_rpms in per_arch_rpms.values():

+                 for per_srpm_rpms in per_build_rpms.values():

+                     for rpm in per_srpm_rpms.values():

+                         if not rpm["sigkey"]:

+                             err_msg = "Some RPMs are not signed."

+                             raise ComposeCheckError(err_msg)

+ 

+     def check_symlinks(self):

+         """

+         Raises ComposeCheckError if some symlink in the Compose cannot be resolved

+         or if the symlink's target is not on the same device as Compose target

+         directory.

+         """

+         print("Checking symlinks.")

+         # The `self.target` can consist of multiple non-existing directories. We therefore

+         # need to check parent directories until we hit existing directory. The last possible

+         # path tried is "/" which should always exist.

+         target_dirname = os.path.dirname(self.target)

+         while not os.path.exists(target_dirname):

+             target_dirname = os.path.dirname(target_dirname)

+         target_stat = os.stat(target_dirname)

+ 

+         for root, dirs, files in os.walk(self.path):

+             for p in dirs + files:

+                 path = os.path.join(root, p)

+                 path_stat = os.lstat(path)

+                 if not stat.S_ISLNK(path_stat.st_mode):

+                     continue

+ 

+                 real_path = os.readlink(path)

+                 abspath = os.path.normpath(os.path.join(os.path.dirname(path), real_path))

+                 try:

+                     abspath_stat = os.stat(abspath)

+                 except Exception as e:

+                     err_msg = "Symlink cannot be resolved: %s: %s." % (path, e)

+                     raise ComposeCheckError(err_msg)

+ 

+                 if target_stat.st_dev != abspath_stat.st_dev:

+                     err_msg = ("Symlink's target is on different device than Compose "

+                             "target: %s" % abspath)

+                     raise ComposeCheckError(err_msg)

+ 

+     def run(self):

+         """

+         Runs the compose checks. Raises ComposeCheckError in case of failed check.

+         """

+         self.check_status()

+         self.check_compose_info()

+         self.check_rpms()

+         self.check_symlinks()

+ 

+ 

+ class ComposePromotion(object):

+     """

+     Contains methods and data to promote compose.

+     """

+     def __init__(self, compose, target):

+         """

+         Creates new ComposePromotion instance.

+ 

+         :param str compose: Path to Compose to promote.

+         :param str target: Target path where the promoted compose should be

+             copied into.

+         """

+         self.compose = compose

+         self.target = target

+ 

+         # Tuple in (symlink_path, hardlink_path) format:

+         #  - symlink_path is full path to symlink in the `compose` tree.

+         #  - hardlink_path is full path to new hardlink in the `target` tree.

+         self.symlinks = []

+ 

+     def _copytree_ignore(self, path, names):

+         """

+         Helper method for `shutil.copytree` to ignore symlinks when copying compose.

+ 

+         This method also populates `self.symlinks`.

+         """

+         print("Copying files in %s." % path)

+         ignored = []

+         rel_path = os.path.relpath(path, self.compose)

+         for name in names:

+             file_path = os.path.join(path, name)

+             if os.path.islink(file_path):

+                 ignored.append(name)

+                 hardlink_path = os.path.join(self.target, rel_path, name)

+                 self.symlinks.append((file_path, hardlink_path))

+         return ignored

+ 

+     def _replace_symlinks_with_hardlinks(self):

+         """

+         Copy symlinks from `compose` to `target` and replace them with hardlinks.

+         """

+         print("Replacing %d symlinks with hardlinks." % len(self.symlinks))

+         for symlink, hardlink_path in self.symlinks:

+             real_path = os.readlink(symlink)

+             abspath = os.path.normpath(os.path.join(os.path.dirname(symlink), real_path))

+             os.link(abspath, hardlink_path)

+ 

+     def promote(self):

+         """

+         Promotes the compose.

+         """

+         shutil.copytree(args.compose, args.target, ignore=self._copytree_ignore)

+         self._replace_symlinks_with_hardlinks()

+ 

+ 

+ if __name__ == "__main__":

+     parser = argparse.ArgumentParser(description="Promote ODCS compose.")

+     parser.add_argument("compose", help="Path to compose to promote.")

+     parser.add_argument("target", help="Path to target location")

+     parser.add_argument("--allow-unsigned", action="store_true",

+                         help="Allow unsigned RPMs.")

+     parser.add_argument("--allow-finished-incomplete", action="store_true",

+                         help="Allow compose in FINISHED_INCOMPLETE state.")

+     parser.add_argument("--no-checks", action="store_true",

+                     help="WARN: Promote the compose without any checks.")

+     args = parser.parse_args()

+ 

+     args.compose = os.path.abspath(args.compose)

+     args.target = os.path.abspath(args.target)

+ 

+     if not args.no_checks:

+         compose_check = ComposeCheck(

+             args.compose, args.target, args.allow_unsigned, args.allow_finished_incomplete)

+         try:

+             compose_check.run()

+         except ComposeCheckError as e:

+             print("Compose validation error: %s" % str(e))

+             sys.exit(1)

+ 

+     print("Promoting compose")

+     compose_promotion = ComposePromotion(args.compose, args.target)

+     compose_promotion.promote()

+ 

This script should be used in the future to promote the ODCS compose
to candidate compose which can later be released.

So far the script does the basic sanity checks and copies compose
to target directory. During copying, symlinks are replaced with
hardlinks.

The script might in the future do also other needed Compose metadata
transformations.

Signed-off-by: Jan Kaluza jkaluza@redhat.com

Do you see a use case for promoting FINISHED_INCOMPLETE composes? Should the status whitelist be configurable?

Should this have fallback to copy of the target file if hardlinking fails?

How about using productmd.Compose instead of building paths to the metadata?

This script is currently requiring Python 3. Rest of ODCS supports Python 2.7 still. The incompatibility is going to complicate packaging: setup.py will install the file, but at some point in building RPM all shebangs are replaced to point to specific python executable, and so that is going to break this script.

I was thinking about promoting FINISHED_INCOMPLETE composes and it might be the case for nightly composes. I think I will allow to configure that.

I think for now I would like it to fail if hardlinking fails. With the current design, the promoted compose should be on the same fs as RPMs and if hardlink fails, it should be treated as an error. We can change it later if we have good reason.

@lsedlar, I'm using python3 mainly for easier shutil.copytree with copy_function support. This is not supported in python2 :(. I can probably workaround that by bundling changed version of copytree in the code: https://github.com/python/cpython/blob/master/Lib/shutil.py#L442. But this would make the script more complex.

I will think about some other ways around it. My original idea was to support just python3 in this script - in the end, this script should be used only server side in infrastructure and not by general audience.

1 new commit added

  • Add odcs-promote-compose --allow-finished-incomplete.
4 years ago

Updated to add --allow-finished-incomplete. See the second commit.

2 new commits added

  • Add odcs-promote-compose --allow-finished-incomplete.
  • Add `odcs-promote-compose` server/contrib script.
4 years ago

I think I found good enough way to use shutil.copytree in both python2 and python3 which won't make the code much longer. I will update the PR again during the morning :).

1 new commit added

  • Rewrite compose-copy part of odcs-promote-compose to work with python2.
4 years ago

@lsedlar, this is ready for another review. The new way of copying the compose is compatible with both python2 and python3.

follow_symlinks was added in 3.3; lstat should work the same way in 2.7

3 new commits added

  • Rewrite compose-copy part of odcs-promote-compose to work with python2.
  • Add odcs-promote-compose --allow-finished-incomplete.
  • Add `odcs-promote-compose` server/contrib script.
4 years ago

rebased onto 0c7fbca

4 years ago

Pull-Request has been merged by lsedlar

4 years ago