From 20c3614fb318df95afb95311e97ba6e38c70a1ce Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Nov 05 2019 07:51:26 +0000 Subject: scm: Add backend for downloading archives from Koji Tests and documentation included. JIRA: COMPOSE-3816 Signed-off-by: Lubomír Sedlář --- diff --git a/doc/scm_support.rst b/doc/scm_support.rst index 2a51853..401ef48 100644 --- a/doc/scm_support.rst +++ b/doc/scm_support.rst @@ -17,13 +17,21 @@ which can contain following keys. * ``git`` -- copies files from a Git repository * ``cvs`` -- copies files from a CVS repository * ``rpm`` -- copies files from a package in the compose + * ``koji`` -- downloads archives from a given build in Koji build system -* ``repo`` -- for Git and CVS backends URL to the repository, for RPM a shell - glob for matching package names (or a list of such globs); for ``file`` - backend this option should be empty +* ``repo`` -* ``branch`` -- branch name for Git and CVS backends, with ``master`` and - ``HEAD`` as defaults. Ignored for other backends. + * for Git and CVS backends this should be URL to the repository + * for RPM backend this should be a shell style glob matching package names + (or a list of such globs) + * for file backend this should be empty + * for Koji backend this should be an NVR or package name + +* ``branch`` + + * branch name for Git and CVS backends, with ``master`` and ``HEAD`` as defaults + * Koji tag for koji backend if only package name is given + * otherwise should not be specified * ``file`` -- a list of files that should be exported. @@ -34,6 +42,31 @@ which can contain following keys. needed file (for example to run ``make``). Only supported in Git backend. +Koji examples +------------- + +There are two different ways how to configure the Koji backend. :: + + { + # Download all *.tar files from build my-image-1.0-1. + "scm": "koji", + "repo": "my-image-1.0-1", + "file": "*.tar", + } + + { + # Find latest build of my-image in tag my-tag and take files from + # there. + "scm": "koji", + "repo": "my-image", + "branch": "my-tag", + "file": "*.tar", + } + +Using both tag name and exact NVR will result in error: the NVR would be +interpreted as a package name, and would not match anything. + + ``file`` vs. ``dir`` -------------------- @@ -53,3 +86,7 @@ after ``pkgset`` phase finished. You can't get comps file from a package. Depending on Git repository URL configuration Pungi can only export the requested content using ``git archive``. When a command should run this is not possible and a clone is always needed. + +When using ``koji`` backend, it is required to provide configuration for Koji +profile to be used (``koji_profile``). It is not possible to contact multiple +different Koji instances. diff --git a/pungi/checks.py b/pungi/checks.py index b650217..ed7a5e7 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -446,7 +446,7 @@ def make_schema(): "properties": { "scm": { "type": "string", - "enum": ["file", "cvs", "git", "rpm"], + "enum": ["file", "cvs", "git", "rpm", "koji"], }, "repo": {"type": "string"}, "branch": {"$ref": "#/definitions/optional_string"}, diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index f1f52ca..4666ca9 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -20,15 +20,18 @@ import shutil import glob import six from six.moves import shlex_quote +from six.moves.urllib.request import urlretrieve +from fnmatch import fnmatch import kobo.log from kobo.shortcuts import run, force_list from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir, retry) +from .kojiwrapper import KojiWrapper class ScmBase(kobo.log.LoggingBase): - def __init__(self, logger=None, command=None): + def __init__(self, logger=None, command=None, compose=None): kobo.log.LoggingBase.__init__(self, logger=logger) self.command = command @@ -196,17 +199,70 @@ class RpmScmWrapper(ScmBase): shutil.copy2(src, dst) +class KojiScmWrapper(ScmBase): + def __init__(self, *args, **kwargs): + super(KojiScmWrapper, self).__init__(*args, **kwargs) + try: + profile = kwargs["compose"].conf["koji_profile"] + except KeyError: + raise RuntimeError("Koji profile must be configured") + wrapper = KojiWrapper(profile) + self.koji = wrapper.koji_module + self.proxy = wrapper.koji_proxy + + def export_dir(self, *args, **kwargs): + raise RuntimeError("Only files can be exported from Koji") + + def export_file(self, scm_root, scm_file, target_dir, scm_branch=None): + if scm_branch: + self._get_latest_from_tag(scm_branch, scm_root, scm_file, target_dir) + else: + self._get_from_build(scm_root, scm_file, target_dir) + + def _get_latest_from_tag(self, koji_tag, package, file_pattern, target_dir): + self.log_debug( + "Exporting file %s from latest Koji package %s in tag %s", + file_pattern, + package, + koji_tag, + ) + builds = self.proxy.listTagged(koji_tag, package=package, latest=True) + if len(builds) != 1: + raise RuntimeError("No package %s in tag %s", package, koji_tag) + self._download_build(builds[0], file_pattern, target_dir) + + def _get_from_build(self, build_id, file_pattern, target_dir): + self.log_debug( + "Exporting file %s from Koji build %s", file_pattern, build_id + ) + build = self.proxy.getBuild(build_id) + self._download_build(build, file_pattern, target_dir) + + def _download_build(self, build, file_pattern, target_dir): + for archive in self.proxy.listArchives(build["build_id"]): + filename = archive["filename"] + if not fnmatch(filename, file_pattern): + continue + typedir = self.koji.pathinfo.typedir(build, archive["btype"]) + file_path = os.path.join(typedir, filename) + url = file_path.replace(self.koji.config.topdir, self.koji.config.topurl) + target_file = os.path.join(target_dir, filename) + urlretrieve(url, target_file) + + def _get_wrapper(scm_type, *args, **kwargs): SCM_WRAPPERS = { "file": FileWrapper, "cvs": CvsWrapper, "git": GitWrapper, "rpm": RpmScmWrapper, + "koji": KojiScmWrapper, } try: - return SCM_WRAPPERS[scm_type](*args, **kwargs) + cls = SCM_WRAPPERS[scm_type] except KeyError: raise ValueError("Unknown SCM type: %s" % scm_type) + return cls(*args, **kwargs) def get_file_from_scm(scm_dict, target_path, compose=None): @@ -254,7 +310,7 @@ def get_file_from_scm(scm_dict, target_path, compose=None): command = scm_dict.get('command') logger = compose._logger if compose else None - scm = _get_wrapper(scm_type, logger=logger, command=command) + scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) files_copied = [] for i in force_list(scm_file): @@ -308,7 +364,7 @@ def get_dir_from_scm(scm_dict, target_path, compose=None): command = scm_dict.get("command") logger = compose._logger if compose else None - scm = _get_wrapper(scm_type, logger=logger, command=command) + scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) with temp_dir(prefix="scm_checkout_") as tmp_dir: scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) diff --git a/tests/test_scm.py b/tests/test_scm.py index 81479a1..7058302 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -399,3 +399,106 @@ class CvsSCMTestCase(SCMBaseTest): self.assertEqual( commands, ['/usr/bin/cvs -q -d http://example.com/cvs export -r HEAD subdir']) + + +@mock.patch("pungi.wrappers.scm.urlretrieve") +@mock.patch("pungi.wrappers.scm.KojiWrapper") +class KojiSCMTestCase(SCMBaseTest): + def test_without_koji_profile(self, KW, dl): + compose = mock.Mock(conf={}) + + with self.assertRaises(RuntimeError) as ctx: + scm.get_file_from_scm( + {"scm": "koji", "repo": "my-build-1.0-2", "file": "*"}, + self.destdir, + compose=compose, + ) + self.assertIn("Koji profile must be configured", str(ctx.exception)) + self.assertEqual(KW.mock_calls, []) + self.assertEqual(dl.mock_calls, []) + + def test_doesnt_get_dirs(self, KW, dl): + compose = mock.Mock(conf={"koji_profile": "koji"}) + + with self.assertRaises(RuntimeError) as ctx: + scm.get_dir_from_scm( + {"scm": "koji", "repo": "my-build-1.0-2", "dir": "*"}, + self.destdir, + compose=compose, + ) + self.assertIn("Only files can be exported", str(ctx.exception)) + self.assertEqual(KW.mock_calls, [mock.call("koji")]) + self.assertEqual(dl.mock_calls, []) + + def _setup_koji_wrapper(self, KW, build_id, files): + KW.return_value.koji_module.config.topdir = "/mnt/koji" + KW.return_value.koji_module.config.topurl = "http://koji.local/koji" + KW.return_value.koji_module.pathinfo.typedir.return_value = "/mnt/koji/images" + buildinfo = {"build_id": build_id} + KW.return_value.koji_proxy.getBuild.return_value = buildinfo + KW.return_value.koji_proxy.listArchives.return_value = [ + {"filename": f, "btype": "image"} for f in files + ] + KW.return_value.koji_proxy.listTagged.return_value = [buildinfo] + + def test_get_from_build(self, KW, dl): + compose = mock.Mock(conf={"koji_profile": "koji"}) + + def download(src, dst): + touch(dst) + + dl.side_effect = download + + self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"]) + + retval = scm.get_file_from_scm( + {"scm": "koji", "repo": "my-build-1.0-2", "file": "*.tar"}, + self.destdir, + compose=compose, + ) + self.assertStructure(retval, ["abc.tar"]) + self.assertEqual( + KW.mock_calls, + [ + mock.call("koji"), + mock.call().koji_proxy.getBuild("my-build-1.0-2"), + mock.call().koji_proxy.listArchives(123), + mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"), + ], + ) + self.assertEqual( + dl.call_args_list, + [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)], + ) + + def test_get_from_latest_build(self, KW, dl): + compose = mock.Mock(conf={"koji_profile": "koji"}) + + def download(src, dst): + touch(dst) + + dl.side_effect = download + + self._setup_koji_wrapper(KW, 123, ["abc.out", "abc.tar"]) + + retval = scm.get_file_from_scm( + {"scm": "koji", "repo": "my-build", "file": "*.tar", "branch": "images"}, + self.destdir, + compose=compose, + ) + self.assertStructure(retval, ["abc.tar"]) + self.assertEqual( + KW.mock_calls, + [ + mock.call("koji"), + mock.call().koji_proxy.listTagged( + "images", package="my-build", latest=True + ), + mock.call().koji_proxy.listArchives(123), + mock.call().koji_module.pathinfo.typedir({"build_id": 123}, "image"), + ], + ) + self.assertEqual( + dl.call_args_list, + [mock.call("http://koji.local/koji/images/abc.tar", mock.ANY)], + )