#209 Get the raw_config using git instead of HTTP.
Merged 9 months ago by jkaluza. Opened 9 months ago by jkaluza.
jkaluza/odcs raw-config-git  into  master

file modified
+9

@@ -137,6 +137,15 @@ 

      MESSAGING_TOPIC = ''

      INTERNAL_MESSAGING_TOPIC = ''

  

+     # Definitions of raw Pungi configs for "raw_config" source_type.

+     # RAW_CONFIG_URLS = {

+     #   "my_raw_config": {

+     #       "url": "http://example.com/test.git",

+     #       "config_filename": "pungi.conf",

+     #       "path": "some/git/subpath",  # optional

+     #   }

+     # }

+ 

  

  class DevConfiguration(BaseConfiguration):

      DEBUG = True

@@ -516,8 +516,12 @@ 

      else:

          if compose.source_type == PungiSourceType.RAW_CONFIG:

              source_name, source_hash = compose.source.split("#")

-             url_template = conf.raw_config_urls[source_name]

-             pungi_cfg = str(url_template % (source_hash))

+             url_data = conf.raw_config_urls[source_name]

+             # Do not override commit hash by hash from ODCS client if it is

+             # hardcoded in the config file.

+             if "commit" not in url_data:

+                 url_data["commit"] = source_hash

+             pungi_cfg = url_data

          else:

              # Generate PungiConfig and run Pungi

              pungi_cfg = PungiConfig(compose.name, "1", compose.source_type,

@@ -431,6 +431,19 @@ 

                              "list" % (role, user, key))

          self._allowed_clients = clients

  

+     def _setifok_raw_config_urls(self, raw_config_urls):

+         if type(raw_config_urls) != dict:

+             raise TypeError("raw_config_urls must be a dict.")

+         for name, url_data in raw_config_urls.items():

+             if type(url_data) != dict:

+                 raise TypeError("raw_config_urls['%s'] is not a dict." % name)

+             for key in ["url", "config_filename"]:

+                 if key not in url_data:

+                     raise ValueError(

+                         "raw_config_urls['%s']['%s'] is not defined."

+                         % (name, key))

+         self._raw_config_urls = raw_config_urls

+ 

      def _setifok_log_backend(self, s):

          if s is None:

              self._log_backend = "console"

file modified
+22 -5

@@ -35,7 +35,7 @@ 

  from odcs.server import conf, log, db

  from odcs.server import comps

  from odcs.common.types import PungiSourceType, COMPOSE_RESULTS

- from odcs.server.utils import makedirs, download_file

+ from odcs.server.utils import makedirs, clone_repo, copytree

  

  

  class PungiConfig(object):

@@ -180,20 +180,37 @@ 

              self._write_cfg(os.path.join(topdir, "pungi.conf"), main_cfg)

              self._write_cfg(os.path.join(topdir, "variants.xml"), variants_cfg)

              self._write_cfg(os.path.join(topdir, "comps.xml"), comps_cfg)

-         else:

+         elif type(self.pungi_cfg) == dict:

              # In case the raw_config wrapper config is set, download the

              # original pungi.conf as "raw_config.conf" and use

              # the raw_config wrapper as real "pungi.conf".

              # The reason is that wrapper config can import raw_config

              # and override some variables.

              if conf.raw_config_wrapper_conf_path:

-                 output_path = os.path.join(topdir, "raw_config.conf")

+                 main_cfg_path = os.path.join(topdir, "raw_config.conf")

                  shutil.copy2(conf.raw_config_wrapper_conf_path,

                               os.path.join(topdir, "pungi.conf"))

              else:

-                 output_path = os.path.join(topdir, "pungi.conf")

+                 main_cfg_path = os.path.join(topdir, "pungi.conf")

+ 

+             # Clone the git repo with raw_config pungi config files.

+             repo_dir = os.path.join(topdir, "raw_config_repo")

+             clone_repo(self.pungi_cfg["url"], repo_dir,

+                        commit=self.pungi_cfg["commit"])

+ 

+             # If the 'path' is defined, copy only the files form the 'path'

+             # to topdir.

+             if "path" in self.pungi_cfg:

+                 repo_dir = os.path.join(repo_dir, self.pungi_cfg["path"])

  

-             download_file(self.pungi_cfg, output_path)

+             copytree(repo_dir, topdir)

+ 

+             # Create the "pungi.conf" from config_filename.

+             config_path = os.path.join(topdir, self.pungi_cfg["config_filename"])

+             if config_path != main_cfg_path:

+                 shutil.copy2(config_path, main_cfg_path)

+         else:

+             raise ValueError("Unexpected pungi_conf type: %r" % self.pungi_cfg)

  

          if conf.pungi_runroot_koji_conf_path:

              shutil.copy2(conf.pungi_runroot_koji_conf_path,

file modified
+25 -12

@@ -26,24 +26,13 @@ 

  import signal

  import time

  import subprocess

- import requests

+ import shutil

  from distutils.spawn import find_executable

  from threading import Timer

  

  from odcs.server import conf, log

  

  

- def download_file(url, output_path):

-     """

-     Downloads file from URL `url` to `output_path`.

-     """

-     r = requests.get(url)

-     r.raise_for_status()

- 

-     with open(output_path, 'wb') as f:

-         f.write(r.content)

- 

- 

  def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception, logger=None):

      """A decorator that allows to retry a section of code until success or timeout."""

      def wrapper(function):

@@ -131,6 +120,30 @@ 

          raise RuntimeError(err_msg)

  

  

+ def clone_repo(url, dest, branch='master', commit=None):

+     cmd = ['git', 'clone', '-b', branch, url, dest]

+     execute_cmd(cmd)

+ 

+     if commit:

+         cmd = ['git', 'checkout', commit]

+         execute_cmd(cmd, cwd=dest)

+ 

+     return dest

+ 

+ 

+ def copytree(src, dst, symlinks=False, ignore=None):

+     """

+     Implementation of shutil.copytree which does not fail when `dst` exists.

+     """

+     for item in os.listdir(src):

+         s = os.path.join(src, item)

+         d = os.path.join(dst, item)

+         if os.path.isdir(s):

+             shutil.copytree(s, d, symlinks, ignore)

+         else:

+             shutil.copy2(s, d)

+ 

+ 

  def hardlink(dirs, verbose=False):

      """Run hardlink to consolidates duplicate files in dirs"""

      hardlink_exe = find_executable('hardlink')

file modified
+10 -2

@@ -490,7 +490,11 @@ 

          self.assertEqual(self.pungi_config.pkgset_koji_inherit, False)

  

      @patch.object(odcs.server.config.Config, 'raw_config_urls',

-                   new={"pungi_cfg": "http://localhost/pungi.conf#%s"})

+                   new={

+                       "pungi_cfg": {

+                           "url": "git://localhost/test.git",

+                           "config_filename": "pungi.conf"}

+                   })

      def test_generate_pungi_compose_raw_config(self):

          c = Compose.create(

              db.session, "me", PungiSourceType.RAW_CONFIG, "pungi_cfg#hash",

@@ -498,7 +502,11 @@ 

          c.id = 1

  

          generate_pungi_compose(c)

-         self.assertEqual(self.pungi_config, "http://localhost/pungi.conf#hash")

+         self.assertEqual(

+             self.pungi_config,

+             {'url': 'git://localhost/test.git',

+              'commit': 'hash',

+              'config_filename': 'pungi.conf'})

  

  

  class TestValidatePungiCompose(ModelsBaseTest):

file modified
+65 -14

@@ -35,6 +35,7 @@ 

  from odcs.server import conf, db

  from odcs.server.models import Compose

  from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS

+ from odcs.server.utils import makedirs

  from .utils import ConfigPatcher, AnyStringWith, ModelsBaseTest

  

  test_dir = os.path.abspath(os.path.dirname(__file__))

@@ -161,15 +162,24 @@ 

      def setUp(self):

          super(TestPungi, self).setUp()

  

-         self.patch_download_file = patch("odcs.server.pungi.download_file")

-         self.download_file = self.patch_download_file.start()

+         def mocked_clone_repo(url, dest, branch='master', commit=None):

+             makedirs(dest)

+             makedirs(os.path.join(dest, "another"))

+             with open(os.path.join(dest, "pungi.conf"), "w") as fd:

+                 fd.write("fake pungi conf 1")

+             with open(os.path.join(dest, "another", "pungi.conf"), "w") as fd:

+                 fd.write("fake pungi conf 2")

+ 

+         self.patch_clone_repo = patch("odcs.server.pungi.clone_repo")

+         self.clone_repo = self.patch_clone_repo.start()

+         self.clone_repo.side_effect = mocked_clone_repo

  

          self.compose = MagicMock()

  

      def tearDown(self):

          super(TestPungi, self).tearDown()

  

-         self.patch_download_file.stop()

+         self.patch_clone_repo.stop()

  

      @patch("odcs.server.utils.execute_cmd")

      def test_pungi_run(self, execute_cmd):

@@ -185,13 +195,48 @@ 

  

      @patch("odcs.server.utils.execute_cmd")

      def test_pungi_run_raw_config(self, execute_cmd):

-         pungi_cfg = "http://localhost/pungi.conf#hash"

+         def mocked_execute_cmd(*args, **kwargs):

+             topdir = kwargs["cwd"]

+             with open(os.path.join(topdir, "pungi.conf"), "r") as f:

+                 data = f.read()

+                 self.assertTrue("fake pungi conf 1" in data)

+         execute_cmd.side_effect = mocked_execute_cmd

+ 

+         pungi_cfg = {

+             "url": "http://localhost/test.git",

+             "config_filename": "pungi.conf",

+             "commit": "hash",

+         }

+         pungi = Pungi(pungi_cfg)

+         pungi.run(self.compose)

+ 

+         execute_cmd.assert_called_once()

+         self.clone_repo.assert_called_once_with(

+             'http://localhost/test.git', AnyStringWith("/raw_config_repo"),

+             commit='hash')

+ 

+     @patch("odcs.server.utils.execute_cmd")

+     def test_pungi_run_raw_config_subpath(self, execute_cmd):

+         def mocked_execute_cmd(*args, **kwargs):

+             topdir = kwargs["cwd"]

+             with open(os.path.join(topdir, "pungi.conf"), "r") as f:

+                 data = f.read()

+                 self.assertTrue("fake pungi conf 2" in data)

+         execute_cmd.side_effect = mocked_execute_cmd

+ 

+         pungi_cfg = {

+             "url": "http://localhost/test.git",

+             "config_filename": "pungi.conf",

+             "commit": "hash",

+             "path": "another",

+         }

          pungi = Pungi(pungi_cfg)

          pungi.run(self.compose)

  

          execute_cmd.assert_called_once()

-         self.download_file.assert_called_once_with(

-             "http://localhost/pungi.conf#hash", AnyStringWith("/raw_config.conf"))

+         self.clone_repo.assert_called_once_with(

+             'http://localhost/test.git', AnyStringWith("/raw_config_repo"),

+             commit='hash')

  

  

  class TestPungiLogs(ModelsBaseTest):

@@ -276,12 +321,14 @@ 

          unique_path = self.patch_unique_path.start()

          unique_path.return_value = "odcs/unique_path"

  

-         def mocked_download_file(url, output_path):

-             with open(output_path, "w") as fd:

-                 fd.write("fake pungi.conf")

-         self.patch_download_file = patch("odcs.server.pungi.download_file")

-         self.download_file = self.patch_download_file.start()

-         self.download_file.side_effect = mocked_download_file

+         def mocked_clone_repo(url, dest, branch='master', commit=None):

+             makedirs(dest)

+             with open(os.path.join(dest, "pungi.conf"), "w") as fd:

+                 fd.write("pungi.conf")

+ 

+         self.patch_clone_repo = patch("odcs.server.pungi.clone_repo")

+         self.clone_repo = self.patch_clone_repo.start()

+         self.clone_repo.side_effect = mocked_clone_repo

  

          self.compose = MagicMock()

  

@@ -290,7 +337,7 @@ 

          self.config_patcher.stop()

          self.patch_make_koji_session.stop()

          self.patch_unique_path.stop()

-         self.patch_download_file.stop()

+         self.patch_clone_repo.stop()

  

          conf_topdir = os.path.join(conf.target_dir, "odcs/unique_path")

          shutil.rmtree(conf_topdir)

@@ -324,7 +371,11 @@ 

      def test_pungi_run_runroot_raw_config(self):

          self.koji_session.getTaskInfo.return_value = {"state": koji.TASK_STATES["CLOSED"]}

  

-         pungi_cfg = "http://localhost/pungi.conf#hash"

+         pungi_cfg = {

+             "url": "http://localhost/test.git",

+             "config_filename": "pungi.conf",

+             "commit": "hash",

+         }

          pungi = Pungi(pungi_cfg)

          pungi.run(self.compose)

  

Previously, we used RAW_CONFIG_URLS to configure URL to single Pungi configuration file which was later downloaded using HTTP protocol. This cannot be used in practice, because Pungi configuration file can link to other files like "variants.xml", "comps.xml" or even another pungi configuration.

In this PR, the HTTP URL is replaced with git URL and the git repository with raw config and other fileds is cloned instead of downloading single configuration file using HTTP.

rebased onto fe0e6871619549403090ca7c2d5399a6a86d890f

9 months ago

typo here, s/commmit/commit/

If "path" is specified, config_filename is under the 'path' dir in topdir, rather than the topdir directly, which is copied to topdir above. Or I'm mis-reading this that the topdir doesn't exist before the shutil.copytree?

@qwan: I've changed the code a bit, please review again :(.

When path is defined, the files are copied from $topdir/raw_config_repo/$patch to $topdir. If path is not defined, the whole $topdir/raw_config_repo is copied to $topdir.

I've added tests for that.

rebased onto c51ef77

9 months ago

Pull-Request has been merged by jkaluza

9 months ago