From 0c7fbca8fed438b02f4d9af2214578b6ad76ad38 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Feb 19 2020 08:34:02 +0000 Subject: [PATCH 1/3] Add `odcs-promote-compose` server/contrib script. This script should be used in the future to promote the ODCS compose to candidate compose which can later be released. So far the script does the basic sanity checks and copies compose to target directory. During copying, symlinks are replaced with hardlinks. The script might in the future do also other needed Compose metadata transformations. Signed-off-by: Jan Kaluza --- diff --git a/server/contrib/odcs-promote-compose b/server/contrib/odcs-promote-compose new file mode 100755 index 0000000..e4a32c4 --- /dev/null +++ b/server/contrib/odcs-promote-compose @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +import shutil +import argparse +import sys +import os +import stat + +from productmd.composeinfo import ComposeInfo +from productmd.rpms import Rpms + + +class ComposeCheckError(Exception): + """" + Raised when compose check fails. + """ + pass + + +class ComposeCheck(object): + """ + Checks the basic information about compose before promoting it. + + This is not real Compose CI, but rather basic sanity check. + """ + def __init__(self, path, target, allow_unsigned=False): + """ + Creates new ComposeCheck instance. + + :param str path: Path to Compose to check. + :param str target: Target path where the promoted compose should be + copied into. + :param bool allow_unsigned: If True, compose with unsigned packages + can be promoted. + """ + self.path = path + self.target = target + self.allow_unsigned = allow_unsigned + + def check_status(self): + """ + Raises ComposeCheckError if Compose STATUS is not FINISHED. + """ + print("Checking compose STATUS.") + status_path = os.path.join(self.path, "STATUS") + with open(status_path, "r") as f: + if f.readline() != "FINISHED\n": + raise ComposeCheckError('Compose is not in "FINISHED" status.') + + def check_compose_info(self): + """ + Raises ComposeCheckError if Compose type is not "production". + """ + print("Checking compose type.") + ci = ComposeInfo() + ci.load(os.path.join(self.path, "compose", "metadata", "composeinfo.json")) + if ci.compose.type != "production": + raise ComposeCheckError('Compose type is not "production".') + + def check_rpms(self): + """ + Raises ComposeCheckError if there are unsigned packages in the Compose. + """ + if self.allow_unsigned: + return + + print("Checking for unsigned RPMs.") + rpms = Rpms() + rpms.load(os.path.join(self.path, "compose", "metadata", "rpms.json")) + for per_arch_rpms in rpms.rpms.values(): + for per_build_rpms in per_arch_rpms.values(): + for per_srpm_rpms in per_build_rpms.values(): + for rpm in per_srpm_rpms.values(): + if not rpm["sigkey"]: + err_msg = "Some RPMs are not signed." + raise ComposeCheckError(err_msg) + + def check_symlinks(self): + """ + Raises ComposeCheckError if some symlink in the Compose cannot be resolved + or if the symlink's target is not on the same device as Compose target + directory. + """ + print("Checking symlinks.") + target_stat = os.stat(os.path.dirname(self.target)) + for root, dirs, files in os.walk(self.path): + for p in dirs + files: + path = os.path.join(root, p) + path_stat = os.stat(path, follow_symlinks=False) + if not stat.S_ISLNK(path_stat.st_mode): + continue + + real_path = os.readlink(path) + abspath = os.path.normpath(os.path.join(os.path.dirname(path), real_path)) + try: + abspath_stat = os.stat(abspath) + except Exception as e: + err_msg = "Symlink cannot be resolved: %s: %s." % (path, e) + raise ComposeCheckError(err_msg) + + if target_stat.st_dev != abspath_stat.st_dev: + err_msg = ("Symlink's target is on different device than Compose " + "target: %s" % abspath) + raise ComposeCheckError(err_msg) + + def run(self): + """ + Runs the compose checks. Raises ComposeCheckError in case of failed check. + """ + self.check_status() + self.check_compose_info() + self.check_rpms() + self.check_symlinks() + + +def copy_and_replace_symlinks(src, dst): + """ + Helper method to be used in `shutil.copytree` as `copy_function`. + + Regular files are copied, but symlinks are replaced with hardlinks. + """ + if os.path.islink(src): + real_path = os.readlink(src) + abspath = os.path.normpath(os.path.join(os.path.dirname(src), real_path)) + os.link(abspath, dst) + else: + shutil.copy2(src, dst) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Promote ODCS compose.") + parser.add_argument("compose", help="Path to compose to promote.") + parser.add_argument("target", help="Path to target location") + parser.add_argument("--allow-unsigned", action="store_true", + help="Allow unsigned RPMs.") + parser.add_argument("--no-checks", action="store_true", + help="WARN: Promote the compose without any checks.") + args = parser.parse_args() + + if not args.no_checks: + compose_check = ComposeCheck(args.compose, args.target, args.allow_unsigned) + try: + compose_check.run() + except ComposeCheckError as e: + print("Compose validation error: %s" % str(e)) + sys.exit(1) + + print("Copying %s to %s." % (args.compose, args.target)) + shutil.copytree(args.compose, args.target, + copy_function=copy_and_replace_symlinks) From 75609565a6e388350c34c54f8a8fe4f0ce54d1a8 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Feb 19 2020 08:34:03 +0000 Subject: [PATCH 2/3] Add odcs-promote-compose --allow-finished-incomplete. This might be needed for nightly composes which can be in FINISHED_INCOMPLETE state but still be promoted as nightlies. --- diff --git a/server/contrib/odcs-promote-compose b/server/contrib/odcs-promote-compose index e4a32c4..21253df 100755 --- a/server/contrib/odcs-promote-compose +++ b/server/contrib/odcs-promote-compose @@ -23,7 +23,7 @@ class ComposeCheck(object): This is not real Compose CI, but rather basic sanity check. """ - def __init__(self, path, target, allow_unsigned=False): + def __init__(self, path, target, allow_unsigned=False, allow_finished_incomplete=False): """ Creates new ComposeCheck instance. @@ -32,20 +32,30 @@ class ComposeCheck(object): copied into. :param bool allow_unsigned: If True, compose with unsigned packages can be promoted. + :param bool allow_finished_incomplete: If True, compose in FINISHED_INCOMPLETE + state can be promoted. """ self.path = path self.target = target self.allow_unsigned = allow_unsigned + self.allow_finished_incomplete = allow_finished_incomplete def check_status(self): """ Raises ComposeCheckError if Compose STATUS is not FINISHED. """ print("Checking compose STATUS.") + + allowed_statuses = ["FINISHED"] + if self.allow_finished_incomplete: + allowed_statuses.append("FINISHED_INCOMPLETE") + status_path = os.path.join(self.path, "STATUS") with open(status_path, "r") as f: - if f.readline() != "FINISHED\n": - raise ComposeCheckError('Compose is not in "FINISHED" status.') + status = f.readline()[:-1] + if status not in allowed_statuses: + err_msg = 'Compose is not in %s status.' % (" or ".join(allowed_statuses)) + raise ComposeCheckError(err_msg) def check_compose_info(self): """ @@ -133,12 +143,15 @@ if __name__ == "__main__": parser.add_argument("target", help="Path to target location") parser.add_argument("--allow-unsigned", action="store_true", help="Allow unsigned RPMs.") + parser.add_argument("--allow-finished-incomplete", action="store_true", + help="Allow compose in FINISHED_INCOMPLETE state.") parser.add_argument("--no-checks", action="store_true", help="WARN: Promote the compose without any checks.") args = parser.parse_args() if not args.no_checks: - compose_check = ComposeCheck(args.compose, args.target, args.allow_unsigned) + compose_check = ComposeCheck( + args.compose, args.target, args.allow_unsigned, args.allow_finished_incomplete) try: compose_check.run() except ComposeCheckError as e: From 9107d2787980c95de1c7402ddd90b58b950bde51 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Feb 19 2020 08:34:03 +0000 Subject: [PATCH 3/3] Rewrite compose-copy part of odcs-promote-compose to work with python2. --- diff --git a/server/contrib/odcs-promote-compose b/server/contrib/odcs-promote-compose index 21253df..2ef821f 100755 --- a/server/contrib/odcs-promote-compose +++ b/server/contrib/odcs-promote-compose @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from __future__ import print_function import shutil import argparse import sys @@ -92,11 +93,18 @@ class ComposeCheck(object): directory. """ print("Checking symlinks.") - target_stat = os.stat(os.path.dirname(self.target)) + # The `self.target` can consist of multiple non-existing directories. We therefore + # need to check parent directories until we hit existing directory. The last possible + # path tried is "/" which should always exist. + target_dirname = os.path.dirname(self.target) + while not os.path.exists(target_dirname): + target_dirname = os.path.dirname(target_dirname) + target_stat = os.stat(target_dirname) + for root, dirs, files in os.walk(self.path): for p in dirs + files: path = os.path.join(root, p) - path_stat = os.stat(path, follow_symlinks=False) + path_stat = os.lstat(path) if not stat.S_ISLNK(path_stat.st_mode): continue @@ -123,18 +131,59 @@ class ComposeCheck(object): self.check_symlinks() -def copy_and_replace_symlinks(src, dst): +class ComposePromotion(object): """ - Helper method to be used in `shutil.copytree` as `copy_function`. - - Regular files are copied, but symlinks are replaced with hardlinks. + Contains methods and data to promote compose. """ - if os.path.islink(src): - real_path = os.readlink(src) - abspath = os.path.normpath(os.path.join(os.path.dirname(src), real_path)) - os.link(abspath, dst) - else: - shutil.copy2(src, dst) + def __init__(self, compose, target): + """ + Creates new ComposePromotion instance. + + :param str compose: Path to Compose to promote. + :param str target: Target path where the promoted compose should be + copied into. + """ + self.compose = compose + self.target = target + + # Tuple in (symlink_path, hardlink_path) format: + # - symlink_path is full path to symlink in the `compose` tree. + # - hardlink_path is full path to new hardlink in the `target` tree. + self.symlinks = [] + + def _copytree_ignore(self, path, names): + """ + Helper method for `shutil.copytree` to ignore symlinks when copying compose. + + This method also populates `self.symlinks`. + """ + print("Copying files in %s." % path) + ignored = [] + rel_path = os.path.relpath(path, self.compose) + for name in names: + file_path = os.path.join(path, name) + if os.path.islink(file_path): + ignored.append(name) + hardlink_path = os.path.join(self.target, rel_path, name) + self.symlinks.append((file_path, hardlink_path)) + return ignored + + def _replace_symlinks_with_hardlinks(self): + """ + Copy symlinks from `compose` to `target` and replace them with hardlinks. + """ + print("Replacing %d symlinks with hardlinks." % len(self.symlinks)) + for symlink, hardlink_path in self.symlinks: + real_path = os.readlink(symlink) + abspath = os.path.normpath(os.path.join(os.path.dirname(symlink), real_path)) + os.link(abspath, hardlink_path) + + def promote(self): + """ + Promotes the compose. + """ + shutil.copytree(args.compose, args.target, ignore=self._copytree_ignore) + self._replace_symlinks_with_hardlinks() if __name__ == "__main__": @@ -149,6 +198,9 @@ if __name__ == "__main__": help="WARN: Promote the compose without any checks.") args = parser.parse_args() + args.compose = os.path.abspath(args.compose) + args.target = os.path.abspath(args.target) + if not args.no_checks: compose_check = ComposeCheck( args.compose, args.target, args.allow_unsigned, args.allow_finished_incomplete) @@ -158,6 +210,7 @@ if __name__ == "__main__": print("Compose validation error: %s" % str(e)) sys.exit(1) - print("Copying %s to %s." % (args.compose, args.target)) - shutil.copytree(args.compose, args.target, - copy_function=copy_and_replace_symlinks) + print("Promoting compose") + compose_promotion = ComposePromotion(args.compose, args.target) + compose_promotion.promote() +