From c51ef772e98398f37471915d9829c4f11b7a3953 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jul 10 2018 11:25:35 +0000 Subject: Get the raw_config using git instead of HTTP. --- diff --git a/server/conf/config.py b/server/conf/config.py index e7ec48c..45f4292 100644 --- a/server/conf/config.py +++ b/server/conf/config.py @@ -137,6 +137,15 @@ class BaseConfiguration(object): MESSAGING_TOPIC = '' INTERNAL_MESSAGING_TOPIC = '' + # Definitions of raw Pungi configs for "raw_config" source_type. + # RAW_CONFIG_URLS = { + # "my_raw_config": { + # "url": "http://example.com/test.git", + # "config_filename": "pungi.conf", + # "path": "some/git/subpath", # optional + # } + # } + class DevConfiguration(BaseConfiguration): DEBUG = True diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 2c7a75a..35953c9 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -516,8 +516,12 @@ def generate_pungi_compose(compose): else: if compose.source_type == PungiSourceType.RAW_CONFIG: source_name, source_hash = compose.source.split("#") - url_template = conf.raw_config_urls[source_name] - pungi_cfg = str(url_template % (source_hash)) + url_data = conf.raw_config_urls[source_name] + # Do not override commit hash by hash from ODCS client if it is + # hardcoded in the config file. + if "commit" not in url_data: + url_data["commit"] = source_hash + pungi_cfg = url_data else: # Generate PungiConfig and run Pungi pungi_cfg = PungiConfig(compose.name, "1", compose.source_type, diff --git a/server/odcs/server/config.py b/server/odcs/server/config.py index b0fa9fd..c134032 100644 --- a/server/odcs/server/config.py +++ b/server/odcs/server/config.py @@ -431,6 +431,19 @@ class Config(object): "list" % (role, user, key)) self._allowed_clients = clients + def _setifok_raw_config_urls(self, raw_config_urls): + if type(raw_config_urls) != dict: + raise TypeError("raw_config_urls must be a dict.") + for name, url_data in raw_config_urls.items(): + if type(url_data) != dict: + raise TypeError("raw_config_urls['%s'] is not a dict." % name) + for key in ["url", "config_filename"]: + if key not in url_data: + raise ValueError( + "raw_config_urls['%s']['%s'] is not defined." + % (name, key)) + self._raw_config_urls = raw_config_urls + def _setifok_log_backend(self, s): if s is None: self._log_backend = "console" diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index f89dfd0..69c04d5 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -35,7 +35,7 @@ import odcs.server.utils from odcs.server import conf, log, db from odcs.server import comps from odcs.common.types import PungiSourceType, COMPOSE_RESULTS -from odcs.server.utils import makedirs, download_file +from odcs.server.utils import makedirs, clone_repo, copytree class PungiConfig(object): @@ -180,20 +180,37 @@ class Pungi(object): self._write_cfg(os.path.join(topdir, "pungi.conf"), main_cfg) self._write_cfg(os.path.join(topdir, "variants.xml"), variants_cfg) self._write_cfg(os.path.join(topdir, "comps.xml"), comps_cfg) - else: + elif type(self.pungi_cfg) == dict: # In case the raw_config wrapper config is set, download the # original pungi.conf as "raw_config.conf" and use # the raw_config wrapper as real "pungi.conf". # The reason is that wrapper config can import raw_config # and override some variables. if conf.raw_config_wrapper_conf_path: - output_path = os.path.join(topdir, "raw_config.conf") + main_cfg_path = os.path.join(topdir, "raw_config.conf") shutil.copy2(conf.raw_config_wrapper_conf_path, os.path.join(topdir, "pungi.conf")) else: - output_path = os.path.join(topdir, "pungi.conf") + main_cfg_path = os.path.join(topdir, "pungi.conf") + + # Clone the git repo with raw_config pungi config files. + repo_dir = os.path.join(topdir, "raw_config_repo") + clone_repo(self.pungi_cfg["url"], repo_dir, + commit=self.pungi_cfg["commit"]) + + # If the 'path' is defined, copy only the files form the 'path' + # to topdir. + if "path" in self.pungi_cfg: + repo_dir = os.path.join(repo_dir, self.pungi_cfg["path"]) - download_file(self.pungi_cfg, output_path) + copytree(repo_dir, topdir) + + # Create the "pungi.conf" from config_filename. + config_path = os.path.join(topdir, self.pungi_cfg["config_filename"]) + if config_path != main_cfg_path: + shutil.copy2(config_path, main_cfg_path) + else: + raise ValueError("Unexpected pungi_conf type: %r" % self.pungi_cfg) if conf.pungi_runroot_koji_conf_path: shutil.copy2(conf.pungi_runroot_koji_conf_path, diff --git a/server/odcs/server/utils.py b/server/odcs/server/utils.py index e2ea706..c8ad4d7 100644 --- a/server/odcs/server/utils.py +++ b/server/odcs/server/utils.py @@ -26,24 +26,13 @@ import os import signal import time import subprocess -import requests +import shutil from distutils.spawn import find_executable from threading import Timer from odcs.server import conf, log -def download_file(url, output_path): - """ - Downloads file from URL `url` to `output_path`. - """ - r = requests.get(url) - r.raise_for_status() - - with open(output_path, 'wb') as f: - f.write(r.content) - - def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception, logger=None): """A decorator that allows to retry a section of code until success or timeout.""" def wrapper(function): @@ -131,6 +120,30 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None, timeout=None): raise RuntimeError(err_msg) +def clone_repo(url, dest, branch='master', commit=None): + cmd = ['git', 'clone', '-b', branch, url, dest] + execute_cmd(cmd) + + if commit: + cmd = ['git', 'checkout', commit] + execute_cmd(cmd, cwd=dest) + + return dest + + +def copytree(src, dst, symlinks=False, ignore=None): + """ + Implementation of shutil.copytree which does not fail when `dst` exists. + """ + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + + def hardlink(dirs, verbose=False): """Run hardlink to consolidates duplicate files in dirs""" hardlink_exe = find_executable('hardlink') diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 904542b..761cc87 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -490,7 +490,11 @@ class TestGeneratePungiCompose(ModelsBaseTest): self.assertEqual(self.pungi_config.pkgset_koji_inherit, False) @patch.object(odcs.server.config.Config, 'raw_config_urls', - new={"pungi_cfg": "http://localhost/pungi.conf#%s"}) + new={ + "pungi_cfg": { + "url": "git://localhost/test.git", + "config_filename": "pungi.conf"} + }) def test_generate_pungi_compose_raw_config(self): c = Compose.create( db.session, "me", PungiSourceType.RAW_CONFIG, "pungi_cfg#hash", @@ -498,7 +502,11 @@ class TestGeneratePungiCompose(ModelsBaseTest): c.id = 1 generate_pungi_compose(c) - self.assertEqual(self.pungi_config, "http://localhost/pungi.conf#hash") + self.assertEqual( + self.pungi_config, + {'url': 'git://localhost/test.git', + 'commit': 'hash', + 'config_filename': 'pungi.conf'}) class TestValidatePungiCompose(ModelsBaseTest): diff --git a/server/tests/test_pungi.py b/server/tests/test_pungi.py index 58987fa..5fe1a1f 100644 --- a/server/tests/test_pungi.py +++ b/server/tests/test_pungi.py @@ -35,6 +35,7 @@ import odcs.server.pungi from odcs.server import conf, db from odcs.server.models import Compose from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS +from odcs.server.utils import makedirs from .utils import ConfigPatcher, AnyStringWith, ModelsBaseTest test_dir = os.path.abspath(os.path.dirname(__file__)) @@ -161,15 +162,24 @@ class TestPungi(unittest.TestCase): def setUp(self): super(TestPungi, self).setUp() - self.patch_download_file = patch("odcs.server.pungi.download_file") - self.download_file = self.patch_download_file.start() + def mocked_clone_repo(url, dest, branch='master', commit=None): + makedirs(dest) + makedirs(os.path.join(dest, "another")) + with open(os.path.join(dest, "pungi.conf"), "w") as fd: + fd.write("fake pungi conf 1") + with open(os.path.join(dest, "another", "pungi.conf"), "w") as fd: + fd.write("fake pungi conf 2") + + self.patch_clone_repo = patch("odcs.server.pungi.clone_repo") + self.clone_repo = self.patch_clone_repo.start() + self.clone_repo.side_effect = mocked_clone_repo self.compose = MagicMock() def tearDown(self): super(TestPungi, self).tearDown() - self.patch_download_file.stop() + self.patch_clone_repo.stop() @patch("odcs.server.utils.execute_cmd") def test_pungi_run(self, execute_cmd): @@ -185,13 +195,48 @@ class TestPungi(unittest.TestCase): @patch("odcs.server.utils.execute_cmd") def test_pungi_run_raw_config(self, execute_cmd): - pungi_cfg = "http://localhost/pungi.conf#hash" + def mocked_execute_cmd(*args, **kwargs): + topdir = kwargs["cwd"] + with open(os.path.join(topdir, "pungi.conf"), "r") as f: + data = f.read() + self.assertTrue("fake pungi conf 1" in data) + execute_cmd.side_effect = mocked_execute_cmd + + pungi_cfg = { + "url": "http://localhost/test.git", + "config_filename": "pungi.conf", + "commit": "hash", + } + pungi = Pungi(pungi_cfg) + pungi.run(self.compose) + + execute_cmd.assert_called_once() + self.clone_repo.assert_called_once_with( + 'http://localhost/test.git', AnyStringWith("/raw_config_repo"), + commit='hash') + + @patch("odcs.server.utils.execute_cmd") + def test_pungi_run_raw_config_subpath(self, execute_cmd): + def mocked_execute_cmd(*args, **kwargs): + topdir = kwargs["cwd"] + with open(os.path.join(topdir, "pungi.conf"), "r") as f: + data = f.read() + self.assertTrue("fake pungi conf 2" in data) + execute_cmd.side_effect = mocked_execute_cmd + + pungi_cfg = { + "url": "http://localhost/test.git", + "config_filename": "pungi.conf", + "commit": "hash", + "path": "another", + } pungi = Pungi(pungi_cfg) pungi.run(self.compose) execute_cmd.assert_called_once() - self.download_file.assert_called_once_with( - "http://localhost/pungi.conf#hash", AnyStringWith("/raw_config.conf")) + self.clone_repo.assert_called_once_with( + 'http://localhost/test.git', AnyStringWith("/raw_config_repo"), + commit='hash') class TestPungiLogs(ModelsBaseTest): @@ -276,12 +321,14 @@ class TestPungiRunroot(unittest.TestCase): unique_path = self.patch_unique_path.start() unique_path.return_value = "odcs/unique_path" - def mocked_download_file(url, output_path): - with open(output_path, "w") as fd: - fd.write("fake pungi.conf") - self.patch_download_file = patch("odcs.server.pungi.download_file") - self.download_file = self.patch_download_file.start() - self.download_file.side_effect = mocked_download_file + def mocked_clone_repo(url, dest, branch='master', commit=None): + makedirs(dest) + with open(os.path.join(dest, "pungi.conf"), "w") as fd: + fd.write("pungi.conf") + + self.patch_clone_repo = patch("odcs.server.pungi.clone_repo") + self.clone_repo = self.patch_clone_repo.start() + self.clone_repo.side_effect = mocked_clone_repo self.compose = MagicMock() @@ -290,7 +337,7 @@ class TestPungiRunroot(unittest.TestCase): self.config_patcher.stop() self.patch_make_koji_session.stop() self.patch_unique_path.stop() - self.patch_download_file.stop() + self.patch_clone_repo.stop() conf_topdir = os.path.join(conf.target_dir, "odcs/unique_path") shutil.rmtree(conf_topdir) @@ -324,7 +371,11 @@ class TestPungiRunroot(unittest.TestCase): def test_pungi_run_runroot_raw_config(self): self.koji_session.getTaskInfo.return_value = {"state": koji.TASK_STATES["CLOSED"]} - pungi_cfg = "http://localhost/pungi.conf#hash" + pungi_cfg = { + "url": "http://localhost/test.git", + "config_filename": "pungi.conf", + "commit": "hash", + } pungi = Pungi(pungi_cfg) pungi.run(self.compose)