From af3cb6104a8bb8e13e7de488c67409b0d7832f4a Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 06 2020 14:20:50 +0000 Subject: [PATCH 1/6] Additional git testing Signed-off-by: Merlin Mathesius --- diff --git a/tests/test_git.py b/tests/test_git.py index 6e8426b..552b26e 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -139,6 +139,7 @@ class TestGit(unittest.TestCase): [".git", "README", "merged", "newfile"]) commit_after = cccc.git.commit_all(tmpdir, "After update") self.assertRegex(commit_after, GIT_HASH_REGEX) + self.assertEqual(commit_after, self._last_commit(tmpdir)) self._run_cmds(( ["cd", tmpdir], ["git", "checkout", commit_before], From 5db963e18ee47060da0a8ac8f4124be013fa6133 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 07 2020 21:01:42 +0000 Subject: [PATCH 2/6] Move pungi test data to a subdirectory Signed-off-by: Merlin Mathesius --- diff --git a/cccc/pungi.py b/cccc/pungi.py index 45a2429..8a179ac 100644 --- a/cccc/pungi.py +++ b/cccc/pungi.py @@ -84,7 +84,7 @@ class PungiConfig(object): if scm_attr not in self.conf or "repo" not in self.conf[scm_attr]: raise RuntimeError("Attempt to set url for non-existant pungi " "config parameter '%s'" % scm_attr) - if '#' in url: + if "#" in url: (repo, branch) = url.split("#", 1) else: repo = url diff --git a/tests/data/c8.conf b/tests/data/c8.conf deleted file mode 100644 index 79f0592..0000000 --- a/tests/data/c8.conf +++ /dev/null @@ -1,147 +0,0 @@ -from multilib import * -from include-excludes import * -from version import * - -# PRODUCT INFO -release_name = "CentOS Linux" -release_short = "CentOS" -release_type = "updates" -release_version = "8" -release_is_layered = False -version = "8" - -# GENERAL SETTINGS -bootable = True -comps_file = { - 'scm': 'git', - 'repo': 'https://git.centos.org/centos/comps.git', - 'branch': 'master', - 'file': 'comps-centos-8.0.0.1905.xml', -} - -link_type = 'hardlink-or-copy' - -module_defaults_dir = { - 'scm': 'git', - 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', - 'branch': 'c8.0', - 'dir': '.', -} - -multilib = [ - ('^.*$', { - 'x86_64': ['devel', 'runtime'] - }), -] - -variants_file = 'variants.xml' -sigkeys = ['8483C65D'] -hashed_directories = False - -tree_arches = ['aarch64', 'i386', 'ppc64le', 'x86_64'] - -# RUNROOT SETTINGS -runroot = True -runroot_channel = 'image' -runroot_tag = 'dist-c8-build' -runroot_method = 'koji' - -# PKGSET -pkgset_source = "koji" -koji_profile = "mbox" - -# PKGSET - KOJI -pkgset_koji_tag = "dist-c8-compose" -pkgset_koji_module_tag = "modular-updates-candidate" - -filter_system_release_packages = False - -# GATHER -gather_prepopulate = "centos-packages.json" - -gather_method = { - "^(?!(AppStream|PowerTools)).*$":{ - "comps": "deps" - }, - "^(AppStream|PowerTools)$": "hybrid", -} - -gather_backend = "dnf" -check_deps = False -greedy_method = "none" -repoclosure_backend = "dnf" - -# CREATEREPO -createrepo_deltas = False -createrepo_database = True -createrepo_c = True -createrepo_checksum = "sha256" -createrepo_use_xz = True -createrepo_num_threads = 8 -createrepo_num_workers = 4 -createrepo_extra_args = ['--distro=cpe:/o:centos:centos:8,CentOS 8', '--revision=8.0.1905'] - -# CHECKSUMS -media_checksums = ['sha256'] -media_checksum_one_file = True -create_jigdo = False - -# BUILDINSTALL -bootable = True -buildinstall_method = "lorax" - -lorax_options = [ - ("^.*$", { - "*": { - "noupgrade": False, - "rootfs_size": 3, - "version": "8.0.1905" - } - }) -] - -buildinstall_skip = [ - ("^(AppStream|PowerTools)$", { - "*": True - }), -] - -image_name_format = { - "^BaseOS$": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", - ".*": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", -} - -# CREATEISO -create_optional_isos = False - -createiso_skip = [ - ('^(BaseOS|AppStream|PowerTools|)$', { - '*': True, - 'src':True - }), -] - -extra_isos = { - "BaseOS": [{ - "include_variants": ["AppStream"], - "filename": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", - "skip_src": True, - }] -} - -restricted_volid = True - -# LOOKASIDE -variant_as_lookaside = [ - ('AppStream', 'BaseOS'), - ('PowerTools', 'BaseOS'), - ('PowerTools', 'AppStream'), -] - -productimg = False - -translate_paths = [ - ("/mnt/koji", "https://koji.mbox.centos.org/pkgs"), - ("/compose", "http://172.22.0.123/compose"), -] - diff --git a/tests/data/include-excludes.conf b/tests/data/include-excludes.conf deleted file mode 100644 index 3b6040e..0000000 --- a/tests/data/include-excludes.conf +++ /dev/null @@ -1,38 +0,0 @@ -filter_packages = [ - ("^(BaseOS|AppStream)$", { - "*": [ - "python36", - ] - }), - - ("^BaseOS$", { - "*": [ - "compat-openssl10-devel", - "compat-openssl10-pkcs11-helper", - "openldap-servers-debuginfo", - "ongres-scram", - ] - }), -] - -additional_packages = [ - ("^AppStream$", { - "*": [ - "langpacks-*", - ] - }), - - ("^AppStream$", { - "x86_64": [ - - "libreoffice-langpack-*", - "libreoffice-help-*", - ] - }), - - ("^BaseOS$", { - "*": [ - "kernel-doc", - ] - }), -] diff --git a/tests/data/multilib.conf b/tests/data/multilib.conf deleted file mode 100644 index 092fddb..0000000 --- a/tests/data/multilib.conf +++ /dev/null @@ -1,16 +0,0 @@ -# format: {arch|*: [packages]} -multilib_blacklist = { - "*": [ - "libvirt*", - "java-*", - "totem", - ], -} - -# format: {arch|*: [packages]} -multilib_whitelist = { - "*": [ - "valgrind", - "papi", - ], -} diff --git a/tests/data/pungi/c8.conf b/tests/data/pungi/c8.conf new file mode 100644 index 0000000..79f0592 --- /dev/null +++ b/tests/data/pungi/c8.conf @@ -0,0 +1,147 @@ +from multilib import * +from include-excludes import * +from version import * + +# PRODUCT INFO +release_name = "CentOS Linux" +release_short = "CentOS" +release_type = "updates" +release_version = "8" +release_is_layered = False +version = "8" + +# GENERAL SETTINGS +bootable = True +comps_file = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/comps.git', + 'branch': 'master', + 'file': 'comps-centos-8.0.0.1905.xml', +} + +link_type = 'hardlink-or-copy' + +module_defaults_dir = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', + 'branch': 'c8.0', + 'dir': '.', +} + +multilib = [ + ('^.*$', { + 'x86_64': ['devel', 'runtime'] + }), +] + +variants_file = 'variants.xml' +sigkeys = ['8483C65D'] +hashed_directories = False + +tree_arches = ['aarch64', 'i386', 'ppc64le', 'x86_64'] + +# RUNROOT SETTINGS +runroot = True +runroot_channel = 'image' +runroot_tag = 'dist-c8-build' +runroot_method = 'koji' + +# PKGSET +pkgset_source = "koji" +koji_profile = "mbox" + +# PKGSET - KOJI +pkgset_koji_tag = "dist-c8-compose" +pkgset_koji_module_tag = "modular-updates-candidate" + +filter_system_release_packages = False + +# GATHER +gather_prepopulate = "centos-packages.json" + +gather_method = { + "^(?!(AppStream|PowerTools)).*$":{ + "comps": "deps" + }, + "^(AppStream|PowerTools)$": "hybrid", +} + +gather_backend = "dnf" +check_deps = False +greedy_method = "none" +repoclosure_backend = "dnf" + +# CREATEREPO +createrepo_deltas = False +createrepo_database = True +createrepo_c = True +createrepo_checksum = "sha256" +createrepo_use_xz = True +createrepo_num_threads = 8 +createrepo_num_workers = 4 +createrepo_extra_args = ['--distro=cpe:/o:centos:centos:8,CentOS 8', '--revision=8.0.1905'] + +# CHECKSUMS +media_checksums = ['sha256'] +media_checksum_one_file = True +create_jigdo = False + +# BUILDINSTALL +bootable = True +buildinstall_method = "lorax" + +lorax_options = [ + ("^.*$", { + "*": { + "noupgrade": False, + "rootfs_size": 3, + "version": "8.0.1905" + } + }) +] + +buildinstall_skip = [ + ("^(AppStream|PowerTools)$", { + "*": True + }), +] + +image_name_format = { + "^BaseOS$": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", + ".*": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", +} + +# CREATEISO +create_optional_isos = False + +createiso_skip = [ + ('^(BaseOS|AppStream|PowerTools|)$', { + '*': True, + 'src':True + }), +] + +extra_isos = { + "BaseOS": [{ + "include_variants": ["AppStream"], + "filename": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", + "skip_src": True, + }] +} + +restricted_volid = True + +# LOOKASIDE +variant_as_lookaside = [ + ('AppStream', 'BaseOS'), + ('PowerTools', 'BaseOS'), + ('PowerTools', 'AppStream'), +] + +productimg = False + +translate_paths = [ + ("/mnt/koji", "https://koji.mbox.centos.org/pkgs"), + ("/compose", "http://172.22.0.123/compose"), +] + diff --git a/tests/data/pungi/include-excludes.conf b/tests/data/pungi/include-excludes.conf new file mode 100644 index 0000000..3b6040e --- /dev/null +++ b/tests/data/pungi/include-excludes.conf @@ -0,0 +1,38 @@ +filter_packages = [ + ("^(BaseOS|AppStream)$", { + "*": [ + "python36", + ] + }), + + ("^BaseOS$", { + "*": [ + "compat-openssl10-devel", + "compat-openssl10-pkcs11-helper", + "openldap-servers-debuginfo", + "ongres-scram", + ] + }), +] + +additional_packages = [ + ("^AppStream$", { + "*": [ + "langpacks-*", + ] + }), + + ("^AppStream$", { + "x86_64": [ + + "libreoffice-langpack-*", + "libreoffice-help-*", + ] + }), + + ("^BaseOS$", { + "*": [ + "kernel-doc", + ] + }), +] diff --git a/tests/data/pungi/multilib.conf b/tests/data/pungi/multilib.conf new file mode 100644 index 0000000..092fddb --- /dev/null +++ b/tests/data/pungi/multilib.conf @@ -0,0 +1,16 @@ +# format: {arch|*: [packages]} +multilib_blacklist = { + "*": [ + "libvirt*", + "java-*", + "totem", + ], +} + +# format: {arch|*: [packages]} +multilib_whitelist = { + "*": [ + "valgrind", + "papi", + ], +} diff --git a/tests/data/pungi/scm.conf b/tests/data/pungi/scm.conf new file mode 100644 index 0000000..6217a89 --- /dev/null +++ b/tests/data/pungi/scm.conf @@ -0,0 +1,13 @@ +comps_file = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/comps.git', + 'branch': 'master', + 'file': 'comps-centos-8.0.0.1905.xml', +} + +module_defaults_dir = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', + 'branch': 'c8.0', + 'dir': '.', +} diff --git a/tests/data/pungi/simple.conf b/tests/data/pungi/simple.conf new file mode 100644 index 0000000..d5c06b9 --- /dev/null +++ b/tests/data/pungi/simple.conf @@ -0,0 +1,15 @@ +from version import * + +one = "I" +two = "II" +four = "IV" +eight = "VIII" + +list = ["1000", "0010", "0100", "0001"] + +dict = { + "alpha": "A", + "beta": "B", + "gamma": "C", + "delta": "D", +} diff --git a/tests/data/pungi/version.conf b/tests/data/pungi/version.conf new file mode 100644 index 0000000..77ffe72 --- /dev/null +++ b/tests/data/pungi/version.conf @@ -0,0 +1 @@ +CENTOS_VERSION='8.0.1905' diff --git a/tests/data/scm.conf b/tests/data/scm.conf deleted file mode 100644 index 6217a89..0000000 --- a/tests/data/scm.conf +++ /dev/null @@ -1,13 +0,0 @@ -comps_file = { - 'scm': 'git', - 'repo': 'https://git.centos.org/centos/comps.git', - 'branch': 'master', - 'file': 'comps-centos-8.0.0.1905.xml', -} - -module_defaults_dir = { - 'scm': 'git', - 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', - 'branch': 'c8.0', - 'dir': '.', -} diff --git a/tests/data/simple.conf b/tests/data/simple.conf deleted file mode 100644 index d5c06b9..0000000 --- a/tests/data/simple.conf +++ /dev/null @@ -1,15 +0,0 @@ -from version import * - -one = "I" -two = "II" -four = "IV" -eight = "VIII" - -list = ["1000", "0010", "0100", "0001"] - -dict = { - "alpha": "A", - "beta": "B", - "gamma": "C", - "delta": "D", -} diff --git a/tests/data/version.conf b/tests/data/version.conf deleted file mode 100644 index 77ffe72..0000000 --- a/tests/data/version.conf +++ /dev/null @@ -1 +0,0 @@ -CENTOS_VERSION='8.0.1905' diff --git a/tests/test_pungi.py b/tests/test_pungi.py index 02756e1..c8daebe 100644 --- a/tests/test_pungi.py +++ b/tests/test_pungi.py @@ -11,7 +11,8 @@ except ImportError: import unittest -DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data") +DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), + "data", "pungi") class TestPungi(unittest.TestCase): From 243f00dca0e69d7174668ac8cea79ab85fa5cedf Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 18 2020 14:23:29 +0000 Subject: [PATCH 3/6] Add config file loading Signed-off-by: Merlin Mathesius --- diff --git a/cccc/config.py b/cccc/config.py new file mode 100644 index 0000000..449b731 --- /dev/null +++ b/cccc/config.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: MIT + +import os + +from kobo.conf import PyConfigParser + +CCCC_CONFIG_DIR = "/etc/cccc" +CCCC_CONFIG_FILE = os.path.join(CCCC_CONFIG_DIR, "config.conf") + +_DefaultConfig = { + "debug": False, + "git_user_email": "<>", + "git_user_name": "CCCC", + "merged_repo_id_file": os.path.join(CCCC_CONFIG_DIR, "id_file"), + "merged_repo_pungi_config": "cccc.conf", + "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_raw_config_name": "odcs_cccc", + "odcs_server_url": "https://127.0.0.1/", + "verbose": False, +} + +_RHELConfiguration = { + "odcs_server_url": "https://odcs.engineering.redhat.com", +} + +_CentOSConfig = { + "pungi": "https://git.centos.org/centos/pungi-centos", + "comps": "http://git.app.eng.bos.redhat.com/comps.git", + "module_defaults": "https://git.centos.org/centos/centos-module-defaults", +} + +_FedoraConfig = { + "odcs_server_url": "https://odcs.fedoraproject.org", + "pungi": "https://pagure.io/pungi-fedora", + "comps": "https://pagure.io/fedora-comps", + "module_defaults": "https://pagure.io/releng/fedora-module-defaults", +} + + +def init_config(args=None): + """ + Create the global CCCC configuration. + + :return: The configuration. + :rtype: Config + """ + # initialize configuration using default settings + conf = PyConfigParser() + conf.load_from_dict(_DefaultConfig) + + # additional logic is needed here to decide what additional default + # settings to load + # conf.load_from_dict(_FedoraConfig) + + # if config file can be read, merge in settings from file + config_file = os.environ.get("CCCC_CONFIG_FILE", CCCC_CONFIG_FILE) + if os.access(config_file, os.R_OK): + conf.load_from_file(config_file) + + if args: + if args.verbose: + conf["verbose"] = True + if args.debug: + conf["debug"] = True + + if conf["debug"]: + print("DEBUG - configuration values:") + for key in sorted(conf.keys()): + print(" %s = %s" % (key, repr(conf[key]))) + + return conf diff --git a/tests/data/config/config.cfg b/tests/data/config/config.cfg new file mode 100644 index 0000000..c8f893f --- /dev/null +++ b/tests/data/config/config.cfg @@ -0,0 +1 @@ +config_file_loaded = True diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..4f29788 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: MIT + +import cccc.config +import os + +from mock import patch + +try: + import unittest2 as unittest +except ImportError: + import unittest + + +DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data", "config") + + +class TestConfig(unittest.TestCase): + + @patch.dict(os.environ, + {"CCCC_CONFIG_FILE": "/FileThatMustNotExist.cfg"}) + def test_config_file_missing(self): + """ + Test that the config can be initialized to expected default values + when a configuration file does not exist. + """ + expected = { + "debug": False, + "git_user_email": "<>", + "git_user_name": "CCCC", + "merged_repo_id_file": "/etc/cccc/id_file", + "merged_repo_pungi_config": "cccc.conf", + "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_raw_config_name": "odcs_cccc", + "odcs_server_url": "https://127.0.0.1/", + "verbose": False, + } + + conf = cccc.config.init_config() + self.assertDictEqual(expected, conf) + + @patch.dict(os.environ, { + "CCCC_CONFIG_FILE": os.path.join(DATA_DIR, "config.cfg") + }) + def test_config_file(self): + """ + Test that the config can be initialized to expected values when + loading a configuration file. + """ + expected = { + "debug": False, + "git_user_email": "<>", + "git_user_name": "CCCC", + "merged_repo_id_file": "/etc/cccc/id_file", + "merged_repo_pungi_config": "cccc.conf", + "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_raw_config_name": "odcs_cccc", + "odcs_server_url": "https://127.0.0.1/", + "verbose": False, + "config_file_loaded": True, + } + + conf = cccc.config.init_config() + self.assertDictEqual(expected, conf) From 97064fd6fb1fbb56d1d4740b69b54ccd026cd79d Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 18 2020 14:24:40 +0000 Subject: [PATCH 4/6] utils: add ignore_fail parameter to execute_cmd() Signed-off-by: Merlin Mathesius --- diff --git a/cccc/git.py b/cccc/git.py index db8e26e..188adfe 100644 --- a/cccc/git.py +++ b/cccc/git.py @@ -88,11 +88,7 @@ def merge_repo(repodir, url, commit="proposed/master"): """ # make sure remote "proposed" does NOT exist cmd = ["git", "remote", "remove", "proposed"] - try: - execute_cmd(cmd, cwd=repodir) - except RuntimeError: - # there was no remote "proposed" to remove - pass + execute_cmd(cmd, cwd=repodir, ignore_fail=True) # add our merge repo as remote "proposed" cmd = ["git", "remote", "add", "proposed", url] diff --git a/cccc/utils.py b/cccc/utils.py index f9156ac..426b828 100644 --- a/cccc/utils.py +++ b/cccc/utils.py @@ -4,7 +4,8 @@ import subprocess import sys -def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None, env=None): +def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None, env=None, + ignore_fail=False): """ Executes command defined by `args`. If `stdout` or `stderr` is set to Python file object, the stderr/stdout output is redirecter to that file. @@ -23,7 +24,11 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None, env=None :param env: If not None, a mapping that defines environment variables for running the command, used instead the current environment. :type env: dict - :raises RuntimeError: Raised when command exits with non-zero exit code. + :param ignore_fail: If True, don't raise an exception if command exits + with non-zero code. Defaults to False. + :type ignore_fail: bool + :raises RuntimeError: Raised when command exits with non-zero code, + unless `ignore_fail` is True. """ proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) @@ -36,7 +41,7 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None, env=None err_msg = "Command '%s' has taken more time than allowed (%d seconds)" % (args, timeout) raise RuntimeError(err_msg) - if proc.returncode != 0: + if not ignore_fail and proc.returncode != 0: sys.stdout.write(out.decode("utf-8")) sys.stderr.write(err.decode("utf-8")) err_msg = "Command '%s' returned non-zero value %d" % (args, proc.returncode) diff --git a/tests/test_utils.py b/tests/test_utils.py index a87e994..5811b09 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -27,6 +27,10 @@ class TestUtilsExecuteCmd(unittest.TestCase): RuntimeError, "Command .* returned non-zero value .*"): execute_cmd(["/usr/bin/false"]) + def test_execute_cmd_ignore_fail(self): + execute_cmd(["/usr/bin/true"], ignore_fail=True) + execute_cmd(["/usr/bin/false"], ignore_fail=True) + def test_execute_cmd_timeout_called(self): start_time = time.time() with self.assertRaisesRegex( From 6aa06097c406af145e618da30afae8f1743540d3 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 18 2020 14:24:40 +0000 Subject: [PATCH 5/6] main: add new command line parameters --debug and --branch-name Signed-off-by: Merlin Mathesius --- diff --git a/cccc/__main__.py b/cccc/__main__.py index 51207a8..b93e81a 100644 --- a/cccc/__main__.py +++ b/cccc/__main__.py @@ -18,6 +18,18 @@ def main(argv=None): required=False, help="Enable verbose output", ) + parser.add_argument( + "--debug", + action="store_true", + required=False, + help="Enable debugging mode", + ) + parser.add_argument( + "--branch-name", + type=str, + required=False, + help="Git branch name for storing merged pungi configuration", + ) default_group = parser.add_argument_group( "default compose configuration arguments", @@ -57,10 +69,9 @@ def main(argv=None): help="PR module-defaults repository", ) - # at least one of the PR arguments is required - args = parser.parse_args(argv) + # at least one of the PR arguments is required if not (args.pr_pungi_repo or args.pr_comps_repo or args.pr_mod_defaults_repo): From 36825946388edcc56c1204bc7046e403775e42d2 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 18 2020 17:26:17 +0000 Subject: [PATCH 6/6] Add support for merging PR repos with default configuration Signed-off-by: Merlin Mathesius --- diff --git a/cccc/__main__.py b/cccc/__main__.py index b93e81a..f3dbd8d 100644 --- a/cccc/__main__.py +++ b/cccc/__main__.py @@ -3,7 +3,7 @@ import argparse import sys -import cccc +import cccc.cli def main(argv=None): diff --git a/cccc/cli.py b/cccc/cli.py index 12da9ff..cb85c76 100644 --- a/cccc/cli.py +++ b/cccc/cli.py @@ -1,6 +1,203 @@ # SPDX-License-Identifier: MIT +import cccc.config +import cccc.git +import cccc.pungi +import datetime +import os +import shutil +import tempfile + + +def _url_split(url): + """ + Split the provided URL string `url` into 'url' and 'commit'. `url` must + be in 'git_url' or 'git_url#commit_hash_or_branch' format. If + `#commit_hash_or_branch` is not specified, 'commit' will be returned as None. + + :return: [url, commit] + :rtype: list + """ + if "#" in url: + return url.split("#", 1) + else: + return [url, None] + + +def _create_merged_config_branch(conf, args, merged_dir): + """ + Clone the merged configuration repo and create a new empty local branch. + + :param conf: Configuration parameters. + :type conf: dict + :param args: Parsed command line arguments. + :type args: argparse.Namespace object + :param merged_dir: Directory to use for local merged configuration + repo clone. + :type merged_dir: str + :return: Name of new branch. + :rtype: str + """ + if args.branch_name: + merged_branch_name = args.branch_name + else: + now = datetime.datetime.now() + merged_branch_name = now.strftime("merge_%Y%m%d%H%M%S") + + cccc.git.clone_repo(merged_dir, + conf["merged_repo_url"], + id_file=conf["merged_repo_id_file"], + user_name=conf["git_user_name"], + user_email=conf["git_user_email"]) + cccc.git.new_empty_branch(merged_dir, merged_branch_name) + cccc.git.commit_all(merged_dir, "New branch") + + return merged_branch_name + + +def _copy_default_config(conf, args, merged_dir, scratch_dir): + """ + Clone the default configuration repo and commit a copy of that to + the merged configuration repo. + + :param conf: Configuration parameters. + :type conf: dict + :param args: Parsed command line arguments. + :type args: argparse.Namespace object + :param merged_dir: Directory containing local merged configuration + repo clone. + :type merged_dir: str + :param scratch_dir: Directory to use for local default configuration + repo clone. + :type scratch_dir: str + :return: Commit hash of the default configuration. + :rtype: str + """ + (default_url, default_commit) = _url_split(args.default_pungi_repo) + cccc.git.clone_repo(scratch_dir, default_url, commit=default_commit) + cccc.git.import_contents(merged_dir, scratch_dir) + + # load the default pungi config file from the merged repo, then write + # it back out with the file name expected by ODCS + default_pungi_config_file = os.path.join(merged_dir, args.default_pungi_file) + p = cccc.pungi.PungiConfig(default_pungi_config_file) + merged_pungi_config_file = os.path.join(merged_dir, conf["merged_repo_pungi_config"]) + p.config_file_write(merged_pungi_config_file) + + commit = cccc.git.commit_all(merged_dir, "Default configuration") + return commit + + +def _merge_updated_config(conf, args, merged_dir, scratch_dir): + """ + Merge the pungi configuration repo PR changes into the local default + configuration repo and commit a copy of that to the merged configuration repo. + + :param conf: Configuration parameters. + :type conf: dict + :param args: Parsed command line arguments. + :type args: argparse.Namespace object + :param merged_dir: Directory containing local merged configuration + repo clone. + :type merged_dir: str + :param scratch_dir: Directory containing local default configuration + repo clone. + :type scratch_dir: str + :return: Commit hash of the updated configuration. + :rtype: str + """ + # if a pungi configuration change PR repo has been specified... + if args.pr_pungi_repo: + # merge the pungi config PR repo into the default configuration + # repo, then copy that into the merged config repo + (pr_url, pr_commit) = _url_split(args.pr_pungi_repo) + cccc.git.merge_repo(scratch_dir, pr_url, commit=pr_commit) + cccc.git.import_contents(merged_dir, scratch_dir) + + # load the pungi config file from the merged repo + default_pungi_config_file = os.path.join(merged_dir, args.default_pungi_file) + p = cccc.pungi.PungiConfig(default_pungi_config_file) + + # update the pungi comps and/or module-defaults + if args.pr_comps_repo: + p.set_comps_file_url(args.pr_comps_repo) + if args.pr_mod_defaults_repo: + p.set_module_defaults_dir_url(args.pr_mod_defaults_repo) + + # write the updated pungi config file with the file name expected by ODCS + merged_pungi_config_file = os.path.join(merged_dir, conf["merged_repo_pungi_config"]) + p.config_file_write(merged_pungi_config_file) + + # commit the updates to the merged config repo + commit = cccc.git.commit_all(merged_dir, "Updated configuration") + return commit + + +def _push_merged_config(conf, merged_dir): + """ + Push the updates in the local merged configuration repo clone back to the + origin repo. + + :param conf: Configuration parameters. + :type conf: dict + :param merged_dir: Directory containing local merged configuration + repo clone. + :type merged_dir: str + """ + if conf["debug"]: + print("Skipping push of merged_dir!!") + else: + cccc.git.push(merged_dir, id_file=conf["merged_repo_id_file"]) + + +def _odcs_placeholder(conf, default_commit, updated_commit): + """ + Place holder for future calls to ODCS. + + :param conf: Configuration parameters. + :type conf: dict + :param default_commit: Commit hash of the default configuration. + :type default_commit: str + :param updated_commit: Commit hash of the updated configuration. + :type updated_commit: str + """ + odcs_default_config = "%s#%s" % (conf["odcs_raw_config_name"], default_commit) + odcs_updated_config = "%s#%s" % (conf["odcs_raw_config_name"], updated_commit) + + print("odcs default raw_config = %s" % odcs_default_config) + print("odcs updated raw_config = %s" % odcs_updated_config) + def run(args): - print("Run away!") - return 0 + conf = cccc.config.init_config(args) + + merged_dir = tempfile.mkdtemp() + + # clone the merged configuration repo and create a new empty local branch + merged_branch = _create_merged_config_branch(conf, args, merged_dir) + print("Writing merged config to branch [%s]" % merged_branch) + + scratch_dir = tempfile.mkdtemp() + + # clone the default configuration repo and commit a copy of that to + # the merged config repo + merged_default_commit = _copy_default_config(conf, args, merged_dir, scratch_dir) + + # merge pungi configuration change PRs + merged_updated_commit = _merge_updated_config(conf, args, merged_dir, scratch_dir) + + # push the branch back to the merged configuration repo + _push_merged_config(conf, merged_dir) + + # cleanup + if conf["debug"]: + print("Be sure to cleanup merged and scratch directories: " + "rm -rf %s %s" % (merged_dir, scratch_dir)) + else: + shutil.rmtree(merged_dir, ignore_errors=True) + shutil.rmtree(scratch_dir, ignore_errors=True) + + # future ODCS calls, etc. + _odcs_placeholder(conf, merged_default_commit, merged_updated_commit) + + return os.EX_OK diff --git a/test-requirements.txt b/test-requirements.txt index 93253de..bb67c5f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ mock pytest +parameterized diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..cbcb3c9 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: MIT + +from mock import ANY, call, patch + +import cccc.cli +from parameterized import parameterized + +try: + import unittest2 as unittest +except ImportError: + import unittest + +config_defaults = { + "debug": False, + "git_user_email": "<>", + "git_user_name": "CCCC", + "merged_repo_id_file": "/etc/cccc/id_file", + "merged_repo_pungi_config": "cccc.conf", + "merged_repo_url": "https://example.com/cccc-merged-configs.git", + "odcs_raw_config_name": "odcs_cccc", + "odcs_server_url": "https://127.0.0.1/", + "verbose": False, +} + + +class Args(object): + """ + Simple object to serve as fake argparse.ArgumentParser() object + """ + def __init__(self, pr_pungi_repo=None, pr_comps_repo=None, pr_mod_defaults_repo=None): + # create attributes for all of the command line arguments + self.verbose = None + self.debug = None + self.branch_name = "test-branch-name" + self.default_pungi_repo = "https://example.com/default-pungi-repo.git" + self.default_pungi_file = "default-pungi-file.cfg" + self.pr_pungi_repo = pr_pungi_repo + self.pr_comps_repo = pr_comps_repo + self.pr_mod_defaults_repo = pr_mod_defaults_repo + + +class TestCLI(unittest.TestCase): + + # @pytest.mark.parametrize("pr_mod_defaults_repo", + # ["https://example.com/pr-module-defaults-repo.git"]) + # @pytest.mark.parametrize("pr_comps_repo", + # ["https://example.com/pr-comps-repo.git"]) + # @pytest.mark.parametrize("pr_pungi_repo", + # ["https://example.com/pr-pungi-repo.git"]) + @parameterized.expand([ + # (testcase_name, pr_pungi_repo, pr_comps_repo, pr_mod_defaults_repo) + ( + "none", + None, + None, + None, + ), + ( + "mod_defaults only", + None, + None, + "https://example.com/pr-module-defaults-repo.git", + ), + ( + "comps only", + None, + "https://example.com/pr-comps-repo.git", + None, + ), + ( + "comps+mod_defaults", + None, + "https://example.com/pr-comps-repo.git", + "https://example.com/pr-module-defaults-repo.git", + ), + ( + "pungi only", + "https://example.com/pr-pungi-repo.git", + None, + None, + ), + ( + "pungi+mod_defaults", + "https://example.com/pr-pungi-repo.git", + None, + "https://example.com/pr-module-defaults-repo.git", + ), + ( + "pungi+comps", + "https://example.com/pr-pungi-repo.git", + "https://example.com/pr-comps-repo.git", + None, + ), + ( + "pungi+comps+mod_defaults", + "https://example.com/pr-pungi-repo.git", + "https://example.com/pr-comps-repo.git", + "https://example.com/pr-module-defaults-repo.git", + ), + ]) + @patch("cccc.pungi.PungiConfig") + @patch("cccc.git.push") + @patch("cccc.git.merge_repo") + @patch("cccc.git.commit_all") + @patch("cccc.git.new_empty_branch") + @patch("cccc.git.clone_repo") + def test_cli(self, + testcase_name, pr_pungi_repo, pr_comps_repo, pr_mod_defaults_repo, + mock_git_clone, + mock_git_newbranch, + mock_git_commit, + mock_git_merge, + mock_git_push, + mock_pungi_config, + ): + args = Args( + pr_pungi_repo=pr_pungi_repo, + pr_comps_repo=pr_comps_repo, + pr_mod_defaults_repo=pr_mod_defaults_repo, + ) + mock_git_commit.side_effect = ["commit1", "commit2", "commit3"] + + self.assertEqual(cccc.cli.run(args), 0) + + self.assertEqual(mock_git_clone.call_count, 2) + mock_git_clone.assert_has_calls([ + # Note: the first argument to each call is a random temp directory + call(ANY, config_defaults["merged_repo_url"], + id_file=config_defaults["merged_repo_id_file"], + user_name=config_defaults["git_user_name"], + user_email=config_defaults["git_user_email"]), + call(ANY, args.default_pungi_repo, commit=None), + ]) + + self.assertEqual(mock_git_newbranch.call_count, 1) + mock_git_newbranch.assert_has_calls([ + # Note: the first argument to each call is a random temp directory + call(ANY, args.branch_name), + ]) + + self.assertEqual(mock_git_commit.call_count, 3) + mock_git_commit.assert_has_calls([ + # Note: the first argument to each call is a random temp directory + call(ANY, "New branch"), + call(ANY, "Default configuration"), + call(ANY, "Updated configuration"), + ]) + + self.assertEqual(mock_git_push.call_count, 1) + mock_git_push.assert_has_calls([ + # Note: the first argument to each call is a random temp directory + call(ANY, id_file=config_defaults["merged_repo_id_file"]), + ]) + + if pr_pungi_repo: + self.assertEqual(mock_git_merge.call_count, 1) + mock_git_merge.assert_has_calls([ + # Note: the first argument to each call is a random temp directory + call(ANY, args.pr_pungi_repo, commit=None), + ]) + else: + self.assertEqual(mock_git_merge.call_count, 0) + + # PungiConfig.__init__ method should be called twice + self.assertEqual(mock_pungi_config.call_count, 2) + + # PungiConfig.config_file_write method should be called twice + mock_pungi_config.assert_has_calls([ + call().config_file_write(ANY), + ]) + + if pr_comps_repo: + mock_pungi_config.assert_has_calls([ + call().set_comps_file_url(args.pr_comps_repo), + ]) + else: + # mock has no direct way to assert that methods are NOT called + try: + mock_pungi_config.assert_has_calls([ + call().set_comps_file_url(args.pr_comps_repo), + ]) + raise AssertionError("PungiConfig.set_comps_file_url() " + "was called but should NOT have been") + except AssertionError: + pass + + if pr_mod_defaults_repo: + mock_pungi_config.assert_has_calls([ + call().set_module_defaults_dir_url(args.pr_mod_defaults_repo), + ]) + else: + # mock has no direct way to assert that methods are NOT called + try: + mock_pungi_config.assert_has_calls([ + call().set_module_defaults_dir_url(args.pr_mod_defaults_repo), + ]) + raise AssertionError("PungiConfig.set_module_defaults_dir_url() " + "was called but should NOT have been") + except AssertionError: + pass