From 56644fcbc94707bdcde6c9fae6cd29553760b63a Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Apr 23 2020 20:39:37 +0000 Subject: [PATCH 1/2] Add Pungi config file manipulation and tests Signed-off-by: Merlin Mathesius --- diff --git a/cccc/__init__.py b/cccc/__init__.py index 9caa44a..548d2d4 100644 --- a/cccc/__init__.py +++ b/cccc/__init__.py @@ -1,3 +1 @@ -def boom(): - print("Boom!") - return 0 +# SPDX-License-Identifier: MIT diff --git a/cccc/__main__.py b/cccc/__main__.py index b493cd1..51207a8 100644 --- a/cccc/__main__.py +++ b/cccc/__main__.py @@ -1,7 +1,10 @@ +# SPDX-License-Identifier: MIT + import argparse -import cccc import sys +import cccc + def main(argv=None): if argv is None: @@ -63,9 +66,7 @@ def main(argv=None): or args.pr_mod_defaults_repo): parser.error("at least one PR parameter must be provided") - cccc.boom() - - return 0 + return cccc.cli.run(args) if __name__ == "__main__": diff --git a/cccc/cli.py b/cccc/cli.py new file mode 100644 index 0000000..12da9ff --- /dev/null +++ b/cccc/cli.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: MIT + + +def run(args): + print("Run away!") + return 0 diff --git a/cccc/pungi.py b/cccc/pungi.py new file mode 100644 index 0000000..87f0078 --- /dev/null +++ b/cccc/pungi.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: MIT + +import sys + +from kobo.conf import PyConfigParser + + +class PungiConfig(object): + + def __init__(self, cfg_file): + self.config_file_read(cfg_file) + + def config_file_read(self, cfg_file): + """ + Read configuration from file `cfg_file`. + + :param cfg_file: Full file path from which to read. + :type cfg_file: str + """ + print("Reading configuration from file '%s'." % cfg_file, + file=sys.stderr) + self.conf = PyConfigParser() + self.conf.load_from_file(cfg_file) + + def config_file_dumps(self): + """ + Dump current configuration and return as a string. + + :return: Current configuration as a string + :rtype: str + """ + output = "" + for key in sorted(self.conf.keys()): + output += "%s = %s\n" % (key, repr(self.conf[key])) + return output + + def config_file_write(self, cfg_file): + """ + Write current configuration to file `cfg_file`. + + :param cfg_file: Full file path to which to write. + :type cfg_file: str + """ + with open(cfg_file, "w") as f: + print("Writing configuration to file '%s'." % cfg_file, + file=sys.stderr) + f.write(self.config_file_dumps()) + + def _get_scm_url(self, scm_attr): + if scm_attr not in self.conf or "repo" not in self.conf[scm_attr]: + return None + repo = self.conf[scm_attr]["repo"] + if "branch" in self.conf[scm_attr]: + branch = self.conf[scm_attr]["branch"] + else: + branch = "master" + return "%s#%s" % (repo, branch) + + def _set_scm_url(self, scm_attr, url): + if scm_attr not in self.conf or "repo" not in self.conf[scm_attr]: + raise RuntimeError("Attempt to set url for non-existant pungi " + "config parameter '%s'" % scm_attr) + if '#' in url: + (repo, branch) = url.split("#", 1) + else: + repo = url + branch = "master" + self.conf[scm_attr]["repo"] = repo + self.conf[scm_attr]["branch"] = branch + + def get_comps_file_url(self): + return self._get_scm_url("comps_file") + + def set_comps_file_url(self, url): + self._set_scm_url("comps_file", url) + + def get_module_defaults_dir_url(self): + return self._get_scm_url("module_defaults_dir") + + def set_module_defaults_dir_url(self, url): + self._set_scm_url("module_defaults_dir", url) diff --git a/requirements.txt b/requirements.txt index e69de29..c138ac6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1 @@ +kobo diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..93253de --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +mock +pytest diff --git a/tests-requirements.txt b/tests-requirements.txt deleted file mode 100644 index e69de29..0000000 --- a/tests-requirements.txt +++ /dev/null diff --git a/tests/data/c8.conf b/tests/data/c8.conf new file mode 100644 index 0000000..79f0592 --- /dev/null +++ b/tests/data/c8.conf @@ -0,0 +1,147 @@ +from multilib import * +from include-excludes import * +from version import * + +# PRODUCT INFO +release_name = "CentOS Linux" +release_short = "CentOS" +release_type = "updates" +release_version = "8" +release_is_layered = False +version = "8" + +# GENERAL SETTINGS +bootable = True +comps_file = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/comps.git', + 'branch': 'master', + 'file': 'comps-centos-8.0.0.1905.xml', +} + +link_type = 'hardlink-or-copy' + +module_defaults_dir = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', + 'branch': 'c8.0', + 'dir': '.', +} + +multilib = [ + ('^.*$', { + 'x86_64': ['devel', 'runtime'] + }), +] + +variants_file = 'variants.xml' +sigkeys = ['8483C65D'] +hashed_directories = False + +tree_arches = ['aarch64', 'i386', 'ppc64le', 'x86_64'] + +# RUNROOT SETTINGS +runroot = True +runroot_channel = 'image' +runroot_tag = 'dist-c8-build' +runroot_method = 'koji' + +# PKGSET +pkgset_source = "koji" +koji_profile = "mbox" + +# PKGSET - KOJI +pkgset_koji_tag = "dist-c8-compose" +pkgset_koji_module_tag = "modular-updates-candidate" + +filter_system_release_packages = False + +# GATHER +gather_prepopulate = "centos-packages.json" + +gather_method = { + "^(?!(AppStream|PowerTools)).*$":{ + "comps": "deps" + }, + "^(AppStream|PowerTools)$": "hybrid", +} + +gather_backend = "dnf" +check_deps = False +greedy_method = "none" +repoclosure_backend = "dnf" + +# CREATEREPO +createrepo_deltas = False +createrepo_database = True +createrepo_c = True +createrepo_checksum = "sha256" +createrepo_use_xz = True +createrepo_num_threads = 8 +createrepo_num_workers = 4 +createrepo_extra_args = ['--distro=cpe:/o:centos:centos:8,CentOS 8', '--revision=8.0.1905'] + +# CHECKSUMS +media_checksums = ['sha256'] +media_checksum_one_file = True +create_jigdo = False + +# BUILDINSTALL +bootable = True +buildinstall_method = "lorax" + +lorax_options = [ + ("^.*$", { + "*": { + "noupgrade": False, + "rootfs_size": 3, + "version": "8.0.1905" + } + }) +] + +buildinstall_skip = [ + ("^(AppStream|PowerTools)$", { + "*": True + }), +] + +image_name_format = { + "^BaseOS$": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", + ".*": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", +} + +# CREATEISO +create_optional_isos = False + +createiso_skip = [ + ('^(BaseOS|AppStream|PowerTools|)$', { + '*': True, + 'src':True + }), +] + +extra_isos = { + "BaseOS": [{ + "include_variants": ["AppStream"], + "filename": "{release_short}-{version}-{arch}-{disc_type}{disc_num}{suffix}", + "skip_src": True, + }] +} + +restricted_volid = True + +# LOOKASIDE +variant_as_lookaside = [ + ('AppStream', 'BaseOS'), + ('PowerTools', 'BaseOS'), + ('PowerTools', 'AppStream'), +] + +productimg = False + +translate_paths = [ + ("/mnt/koji", "https://koji.mbox.centos.org/pkgs"), + ("/compose", "http://172.22.0.123/compose"), +] + diff --git a/tests/data/include-excludes.conf b/tests/data/include-excludes.conf new file mode 100644 index 0000000..3b6040e --- /dev/null +++ b/tests/data/include-excludes.conf @@ -0,0 +1,38 @@ +filter_packages = [ + ("^(BaseOS|AppStream)$", { + "*": [ + "python36", + ] + }), + + ("^BaseOS$", { + "*": [ + "compat-openssl10-devel", + "compat-openssl10-pkcs11-helper", + "openldap-servers-debuginfo", + "ongres-scram", + ] + }), +] + +additional_packages = [ + ("^AppStream$", { + "*": [ + "langpacks-*", + ] + }), + + ("^AppStream$", { + "x86_64": [ + + "libreoffice-langpack-*", + "libreoffice-help-*", + ] + }), + + ("^BaseOS$", { + "*": [ + "kernel-doc", + ] + }), +] diff --git a/tests/data/multilib.conf b/tests/data/multilib.conf new file mode 100644 index 0000000..092fddb --- /dev/null +++ b/tests/data/multilib.conf @@ -0,0 +1,16 @@ +# format: {arch|*: [packages]} +multilib_blacklist = { + "*": [ + "libvirt*", + "java-*", + "totem", + ], +} + +# format: {arch|*: [packages]} +multilib_whitelist = { + "*": [ + "valgrind", + "papi", + ], +} diff --git a/tests/data/scm.conf b/tests/data/scm.conf new file mode 100644 index 0000000..6217a89 --- /dev/null +++ b/tests/data/scm.conf @@ -0,0 +1,13 @@ +comps_file = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/comps.git', + 'branch': 'master', + 'file': 'comps-centos-8.0.0.1905.xml', +} + +module_defaults_dir = { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', + 'branch': 'c8.0', + 'dir': '.', +} diff --git a/tests/data/simple.conf b/tests/data/simple.conf new file mode 100644 index 0000000..d5c06b9 --- /dev/null +++ b/tests/data/simple.conf @@ -0,0 +1,15 @@ +from version import * + +one = "I" +two = "II" +four = "IV" +eight = "VIII" + +list = ["1000", "0010", "0100", "0001"] + +dict = { + "alpha": "A", + "beta": "B", + "gamma": "C", + "delta": "D", +} diff --git a/tests/data/version.conf b/tests/data/version.conf new file mode 100644 index 0000000..77ffe72 --- /dev/null +++ b/tests/data/version.conf @@ -0,0 +1 @@ +CENTOS_VERSION='8.0.1905' diff --git a/tests/test_cccc.py b/tests/test_cccc.py index 7319da7..c738892 100644 --- a/tests/test_cccc.py +++ b/tests/test_cccc.py @@ -1,13 +1,43 @@ -import unittest +# SPDX-License-Identifier: MIT + +import sys + +from io import StringIO +from mock import patch + from cccc.__main__ import main +try: + import unittest2 as unittest +except ImportError: + import unittest -class TestConsole(unittest.TestCase): - def test_placeholder(self): - self.assertTrue(True) +class TestConsole(unittest.TestCase): - def test_main(self): + @patch("sys.stdout", new=StringIO()) + def test_main_help(self): with self.assertRaises(SystemExit) as cm: - main(['-h']) + main(["-h"]) self.assertEqual(cm.exception.code, 0) + output = sys.stdout.getvalue() + self.assertIn("show this help message and exit", output) + + @patch("sys.stderr", new=StringIO()) + def test_main_no_args(self): + with self.assertRaises(SystemExit) as cm: + main([]) + self.assertNotEqual(cm.exception.code, 0) + output = sys.stderr.getvalue() + self.assertIn("the following arguments are required: " + "--default-pungi-repo, --default-pungi-file", + output) + + @patch("sys.stderr", new=StringIO()) + def test_main_missing_PR_arg(self): + with self.assertRaises(SystemExit) as cm: + main(["--default-pungi-repo", "http://example.com/repo.git", + "--default-pungi-file", "cccc.conf"]) + self.assertNotEqual(cm.exception.code, 0) + output = sys.stderr.getvalue() + self.assertIn("at least one PR parameter must be provided", output) diff --git a/tests/test_pungi.py b/tests/test_pungi.py new file mode 100644 index 0000000..02756e1 --- /dev/null +++ b/tests/test_pungi.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: MIT + +import os.path +import tempfile + +from cccc.pungi import PungiConfig + +try: + import unittest2 as unittest +except ImportError: + import unittest + + +DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data") + + +class TestPungi(unittest.TestCase): + + def test_simple_read(self): + """ + Tests that a simple config file with an include can be read, + and contains the expected values. + """ + expected = { + 'CENTOS_VERSION': '8.0.1905', + 'one': 'I', + 'two': 'II', + 'four': 'IV', + 'eight': 'VIII', + 'dict': {'alpha': 'A', 'beta': 'B', 'delta': 'D', 'gamma': 'C'}, + 'list': ['1000', '0010', '0100', '0001'], + } + + p = PungiConfig(os.path.join(DATA_DIR, "simple.conf")) + self.assertDictEqual(p.conf, expected) + + def test_complex_reread(self): + """ + Tests that a complex config file can be read, written out to a file, + read back in, and the re-read contents are identical to the original. + """ + p = PungiConfig(os.path.join(DATA_DIR, "c8.conf")) + + with tempfile.TemporaryDirectory() as tmpdirname: + tmpfile = os.path.join(tmpdirname, "temp.conf") + + p.config_file_write(tmpfile) + q = PungiConfig(tmpfile) + self.assertDictEqual(p.conf, q.conf) + + def test_scm_read(self): + """ + Tests that a structured config file with our interesting SCM + repo attributes can be read contains the expected values. + """ + expected = { + 'comps_file': { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/comps.git', + 'branch': 'master', + 'file': 'comps-centos-8.0.0.1905.xml', + }, + 'module_defaults_dir': { + 'scm': 'git', + 'repo': 'https://git.centos.org/centos/centos-module-defaults.git', + 'branch': 'c8.0', + 'dir': '.', + }, + } + p = PungiConfig(os.path.join(DATA_DIR, "scm.conf")) + self.assertDictEqual(p.conf, expected) + + def test_url_manipulation(self): + """ + Tests that URL manipulation functions operate as expected. + """ + p = PungiConfig(os.path.join(DATA_DIR, "scm.conf")) + + orig_cf_url = "https://git.centos.org/centos/comps.git#master" + orig_mdd_url = "https://git.centos.org/centos/centos-module-defaults.git#c8.0" + + self.assertEqual(p.get_comps_file_url(), orig_cf_url) + self.assertEqual(p.get_module_defaults_dir_url(), orig_mdd_url) + + new_cf_url = "https://pagure.io/fedora-comps.git#baddf00d" + new_mdd_url = "https://pagure.io/releng/fedora-module-defaults.git" + + p.set_comps_file_url(new_cf_url) + p.set_module_defaults_dir_url(new_mdd_url) + + self.assertEqual(p.get_comps_file_url(), new_cf_url) + self.assertEqual(p.get_module_defaults_dir_url(), new_mdd_url + "#master") + + def test_handling_of_missing_params(self): + """ + Tests that URL manipulation functions fail if attempting to operate on + parameters that do not exist in config file. + """ + p = PungiConfig(os.path.join(DATA_DIR, "simple.conf")) + + self.assertIsNone(p.get_comps_file_url()) + self.assertIsNone(p.get_module_defaults_dir_url()) + + new_url = "https://example.com/repo.git" + with self.assertRaises(RuntimeError) as cm: + p.set_comps_file_url(new_url) + self.assertIn("non-existant pungi config parameter", str(cm.exception)) + + with self.assertRaises(RuntimeError) as cm: + p.set_module_defaults_dir_url(new_url) + self.assertIn("non-existant pungi config parameter", str(cm.exception)) diff --git a/tox.ini b/tox.ini index ecab1d6..4d4e3c3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3,flake8 +envlist = py3,flake8 [flake8] ignore = E731,W503 @@ -7,7 +7,9 @@ max-line-length = 100 exclude = .tox,.git,build,.env [testenv] -deps = pytest +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = pytest setenv = PYTHONPATH = {toxinidir} From 34f43ac2b9ac87932b7f56ea844c26a56342d993 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: Apr 24 2020 15:07:22 +0000 Subject: [PATCH 2/2] Add docstrings for Pungi config get/set methods Signed-off-by: Merlin Mathesius --- diff --git a/cccc/pungi.py b/cccc/pungi.py index 87f0078..45a2429 100644 --- a/cccc/pungi.py +++ b/cccc/pungi.py @@ -47,6 +47,17 @@ class PungiConfig(object): f.write(self.config_file_dumps()) def _get_scm_url(self, scm_attr): + """ + Retrieve URL string from Pungi configuration corresponding to + SCM repo attribute specified by `scm_attr`. + + Returns requested URL as a string in `git_url#commit_hash_or_branch` + format, else None if the requested attribute does not exist or is not + an SCM repo style attribute. + + :return: URL string or None + :rtype: str + """ if scm_attr not in self.conf or "repo" not in self.conf[scm_attr]: return None repo = self.conf[scm_attr]["repo"] @@ -57,6 +68,19 @@ class PungiConfig(object): return "%s#%s" % (repo, branch) def _set_scm_url(self, scm_attr, url): + """ + Store URL string `url` in Pungi configuration corresponding to + SCM repo attribute specified by `scm_attr`. + + Provided URL string `url` must be in `git_url` or + `git_url#commit_hash_or_branch` format. If `#commit_hash_or_branch` + is not specified, branch will default to `master`. + + Throws `RuntimeError` exception if requested attribute does not exist + or is not an SCM repo style attribute. + + :return: None + """ if scm_attr not in self.conf or "repo" not in self.conf[scm_attr]: raise RuntimeError("Attempt to set url for non-existant pungi " "config parameter '%s'" % scm_attr) @@ -69,13 +93,61 @@ class PungiConfig(object): self.conf[scm_attr]["branch"] = branch def get_comps_file_url(self): + """ + Retrieve URL string from Pungi configuration corresponding to + SCM repo attribute `comps_file`. + + Returns requested URL as a string in `git_url#commit_hash_or_branch` + format, else None if the `comps_file` attribute does not exist or is + not an SCM repo style attribute. + + :return: URL string or None + :rtype: str + """ return self._get_scm_url("comps_file") def set_comps_file_url(self, url): + """ + Store URL string `url` in Pungi configuration corresponding to + SCM repo attribute `comps_file`. + + Provided URL string `url` must be in `git_url` or + `git_url#commit_hash_or_branch` format. If `#commit_hash_or_branch` + is not specified, branch will default to `master`. + + Throws `RuntimeError` exception if the `comps_file` attribute does + not exist or is not an SCM repo style attribute. + + :return: None + """ self._set_scm_url("comps_file", url) def get_module_defaults_dir_url(self): + """ + Retrieve URL string from Pungi configuration corresponding to + SCM repo attribute `module_defaults_dir`. + + Returns requested URL as a string in `git_url#commit_hash_or_branch` + format, else None if the `module_defaults_dir` attribute does not + exist or is not an SCM repo style attribute. + + :return: URL string or None + :rtype: str + """ return self._get_scm_url("module_defaults_dir") def set_module_defaults_dir_url(self, url): + """ + Store URL string `url` in Pungi configuration corresponding to + SCM repo attribute `module_defaults_dir`. + + Provided URL string `url` must be in `git_url` or + `git_url#commit_hash_or_branch` format. If `#commit_hash_or_branch` + is not specified, branch will default to `master`. + + Throws `RuntimeError` exception if the `module_defaults_dir` + attribute does not exist or is not an SCM repo style attribute. + + :return: None + """ self._set_scm_url("module_defaults_dir", url)