From 57d507d6d966cf29594bc3a137f192b6b6a35c91 Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Aug 11 2016 13:13:54 +0000 Subject: copy dir to minion via tarball Differential Revision: https://phab.qadevel.cloud.fedoraproject.org/D965 --- diff --git a/libtaskotron.spec b/libtaskotron.spec index edb1e0a..87d1765 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -31,6 +31,7 @@ Requires: python-requests >= 2.7.0 Requires: python-setuptools Requires: python-xunitparser >= 1.3.3 Requires: PyYAML >= 3.11 +Requires: tar >= 1.28-6 BuildRequires: pytest >= 2.7.3 BuildRequires: python2-devel BuildRequires: python-configparser >= 3.5.0b2 diff --git a/libtaskotron/minion.py b/libtaskotron/minion.py index 5465209..64937ef 100644 --- a/libtaskotron/minion.py +++ b/libtaskotron/minion.py @@ -80,7 +80,9 @@ class BaseMinion(object): # install libtaskotron # FIXME this should only need libtaskotron-core T651. libtaskotron-fedora should be # installed as a check dep. - self.ssh.install_pkgs(['libtaskotron-core', 'libtaskotron-fedora']) + # TODO: remove tar dep once libtaskotron 0.4.16 has been released for long enough + # (tar is here just for backward compatibility, see D965) + self.ssh.install_pkgs(['libtaskotron-core', 'libtaskotron-fedora', 'tar']) log.info('Copying task files onto the minion...') diff --git a/libtaskotron/remote_exec.py b/libtaskotron/remote_exec.py index b62deee..0ffdfe3 100644 --- a/libtaskotron/remote_exec.py +++ b/libtaskotron/remote_exec.py @@ -13,6 +13,8 @@ import socket import stat import time import pipes +import tarfile +import tempfile import paramiko @@ -101,11 +103,12 @@ class ParamikoWrapper(object): self.sftp.close() self.ssh.close() - def cmd(self, cmd): + def cmd(self, cmd, debug=True): '''Execute a command. :param str cmd: A command to be executed. Make sure you escape it properly to prevent shell expansion, in case it is not desired. + :param bool debug: Whether print out debugging log messages about what's happening. :returns: returncode of the command :raise TaskotronRemoteProcessError: If the command has non-zero return code and it isn't a code of the exitcode directive. @@ -113,10 +116,11 @@ class ParamikoWrapper(object): :attr:`TIMEOUT`. ''' - log.debug('Running command on remote host: %s', cmd) - # write the executed command also to stdio log, so that it's clear what's happening when - # reading it - self.outstream.write('$ %s\n' % cmd) + if debug: + log.debug('Running command on remote host: %s', cmd) + # write the executed command also to stdio log, so that it's clear what's happening + # when reading it + self.outstream.write('$ %s\n' % cmd) # bufsize=1 means line buffering, which should improve the chance of receiving complete # lines. get_pty=True allocates a pseudo-terminal, which is needed to have stdout and @@ -276,7 +280,9 @@ class ParamikoWrapper(object): return False def put_dir(self, local_path, remote_path, overwrite=True, debug=True): - '''Copy a directory to a remote path. File permissions are not preserved. + '''Copy a directory to a remote path. This method creates a tarball from contents of + ``local_path``, copies the tarball to remote machine and extracts it to ``remote_path``. + File permissions and symlinks are preserved. :param str remote_path: A path to the remote directory. This directory will be created, if its parent exists. Otherwise you need to create the full tree @@ -286,11 +292,10 @@ class ParamikoWrapper(object): dir). If you choose to not overwrite and ``remote_path`` exists, this method with just immediately return. :param bool debug: Whether print out debugging log messages about what's happening. - :raise TaskotronRemoteError: If the directory could not be uploaded + :raise TaskotronRemoteError: If the directory could not be copied ''' - if debug: - log.debug('Recursively copying %s to %s@%s:%s ...', local_path, self.username, + log.debug('Copying %s to %s@%s:%s ...', local_path, self.username, self.hostname, remote_path) # keep the conditionals in this order to avoid unnecessary remote calls @@ -303,15 +308,21 @@ class ParamikoWrapper(object): if not self._remote_isdir(remote_path): self.sftp.mkdir(remote_path) - files = os.listdir(local_path) + files_to_copy = os.listdir(local_path) + tar_local_path = tempfile.mktemp() + tar_remote_path = os.path.join(remote_path, 'tarred_files.tar') + + with tarfile.open(tar_local_path, 'w') as tar: + for filename in files_to_copy: + tar.add(os.path.join(local_path, filename), arcname=filename) + + self.put_file(tar_local_path, tar_remote_path, debug=False) + self.cmd('tar xf %s -C %s' % (pipes.quote(tar_remote_path), pipes.quote(remote_path)), + debug=False) + + os.remove(tar_local_path) + self.cmd('rm -f %s' % pipes.quote(tar_remote_path), debug=False) - for file_ in files: - remote_file = os.path.join(remote_path, file_) - local_file = os.path.join(local_path, file_) - if os.path.isdir(local_file): - self.put_dir(local_file, remote_file, debug=False) - else: - self.put_file(local_file, remote_file, debug=False) except (OSError, IOError) as e: log.exception(e) raise exc.TaskotronRemoteError('Could not copy dir %s to %s@%s:%s: %s' % diff --git a/testing/test_remote_exec.py b/testing/test_remote_exec.py index fd92984..38dabb2 100644 --- a/testing/test_remote_exec.py +++ b/testing/test_remote_exec.py @@ -222,7 +222,7 @@ class TestRemoteExec(object): def test_put_dir_not_dir(self): with pytest.raises(TaskotronRemoteError): - assert self.remote.put_dir(STUB_FILE, STUB_DIR) is None + self.remote.put_dir(STUB_FILE, STUB_DIR) def test_put_dir_not_dir_dont_overwrite(self): assert self.remote.put_dir(STUB_DIR, STUB_DIR, overwrite=False) is None