From 7090347653b9df7f739758a0dabef0b1e408f808 Mon Sep 17 00:00:00 2001 From: clime Date: Jan 11 2017 12:51:30 +0000 Subject: [backend] switched usage of deprecated ansible Runner for python-paramiko module Also exception handling updates. --- diff --git a/backend/backend/daemons/worker.py b/backend/backend/daemons/worker.py index aa1770e..755c970 100644 --- a/backend/backend/daemons/worker.py +++ b/backend/backend/daemons/worker.py @@ -280,5 +280,7 @@ class Worker(multiprocessing.Process): self.do_job(self.job) except VmError as error: self.log.exception("Building error: {}".format(error)) + except Exception as e: + self.log.exception("Unexpected error: {}".format(e)) finally: self.vm_manager.release_vm(self.vm.vm_name) diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py index 43f7a27..d4f2e29 100644 --- a/backend/backend/exceptions.py +++ b/backend/backend/exceptions.py @@ -26,20 +26,18 @@ class BuilderError(MockRemoteError): return result -class AnsibleResponseError(BuilderError): - pass - - -class AnsibleCallError(BuilderError): - def __init__(self, msg, cmd, module_name, as_root, **kwargs): - self.msg = "{}\n Call cmd: `{}`, module: `{}`, as root: {}".format( - msg, cmd, module_name, as_root +class RemoteCmdError(BuilderError): + def __init__(self, msg, cmd, as_root, rc, stderr, stdout): + self.msg = "{}\n cmd=`{}`, root=`{}`, rc=`{}`, stderr=`{}`, stdout=`{}`".format( + msg, cmd, as_root, rc, stderr, stdout ) - super(AnsibleCallError, self).__init__(self.msg, **kwargs) + super(RemoteCmdError, self).__init__(self.msg, rc, stdout, stderr) self.call_args = dict( cmd=cmd, - module_name=module_name, as_root=as_root, + rc=rc, + stderr=stderr, + stdout=stdout ) diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 10fb09f..139d072 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -299,12 +299,6 @@ class BackendConfigReader(object): opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int") - # ssh options - opts.ssh = Munch() - # TODO: ansible Runner show some magic bugs with transport "ssh", using paramiko - opts.ssh.transport = _get_conf( - cp, "ssh", "transport", "paramiko") - opts.msg_buses = [] for bus_config in glob.glob('/etc/copr/msgbuses/*.conf'): opts.msg_buses.append(pyconffile(bus_config)) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index ae00c60..6772dba 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -155,7 +155,10 @@ class MockRemote(object): if not self.job.chroot: raise MockRemoteError("No chroot specified!") - self.builder.check() + try: + self.builder.check() + except BuilderError as error: + raise MockRemoteError(str(error)) @property def chroot_dir(self): diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 10aa8fa..ab96e26 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -4,12 +4,12 @@ import socket from subprocess import Popen import time from urlparse import urlparse +import paramiko -from ansible.runner import Runner from backend.vm_manage import PUBSUB_INTERRUPT_BUILDER from ..helpers import get_redis_connection, ensure_dir_exists -from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError, VmError +from ..exceptions import BuilderError, BuilderTimeOutError, RemoteCmdError, VmError from ..constants import mockchain, rsync, DEF_BUILD_TIMEOUT @@ -35,8 +35,8 @@ class Builder(object): self.remote_pkg_name = None # if we're at this point we've connected and done stuff on the host - self.conn = self._create_ans_conn() - self.root_conn = self._create_ans_conn(username="root") + self.conn = self._create_ssh_conn() + self.root_conn = self._create_ssh_conn(username="root") self.module_dist_tag = self._load_module_dist_tag() @@ -70,19 +70,14 @@ class Builder(object): create_tmpdir_cmd = "/bin/mktemp -d {0}/{1}-XXXXX".format( self._remote_basedir, "mockremote") - results = self._run_ansible(create_tmpdir_cmd) - - tempdir = None - # TODO: use check_for_ans_error - for _, resdict in results["contacted"].items(): - tempdir = resdict["stdout"] + tempdir = self._run_ssh_cmd(create_tmpdir_cmd)[0].strip() # if still nothing then we"ve broken if not tempdir: raise BuilderError("Could not make tmpdir on {0}".format( self.hostname)) - self._run_ansible("/bin/chmod 755 {0}".format(tempdir)) + self._run_ssh_cmd("/bin/chmod 755 {0}".format(tempdir)) self._remote_tempdir = tempdir return self._remote_tempdir @@ -91,50 +86,39 @@ class Builder(object): def tempdir(self, value): self._remote_tempdir = value - def _create_ans_conn(self, username=None): - ans_conn = Runner(remote_user=username or self.opts.build_user, - host_list=self.hostname + ",", - pattern=self.hostname, - forks=1, - transport=self.opts.ssh.transport, - timeout=self.timeout) - return ans_conn - - def run_ansible_with_check(self, cmd, module_name=None, as_root=False, - err_codes=None, success_codes=None): - - results = self._run_ansible(cmd, module_name, as_root) - - try: - check_for_ans_error( - results, self.hostname, err_codes, success_codes) - except AnsibleResponseError as response_error: - raise AnsibleCallError( - msg="Failed to execute ansible command", - cmd=cmd, module_name=module_name, as_root=as_root, - return_code=response_error.return_code, - stdout=response_error.stdout, stderr=response_error.stderr - ) - - return results + def _create_ssh_conn(self, username=None): + conn = paramiko.SSHClient() + conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + conn.connect(hostname=self.hostname, + username=username or self.opts.build_user) + return conn - def _run_ansible(self, cmd, module_name=None, as_root=False): + def _run_ssh_cmd(self, cmd, as_root=False): """ - Executes single ansible module + Executes single shell command remotely - :param str cmd: module command - :param str module_name: name of the invoked module + :param str cmd: shell command :param bool as_root: - :return: ansible command result + :return: stdout, stderr as strings """ if as_root: conn = self.root_conn else: conn = self.conn - conn.module_name = module_name or "shell" - conn.module_args = str(cmd) - return conn.run() + self.log.info("Running on builder: "+cmd) + + stdin, stdout, stderr = conn.exec_command(cmd) + rc = stdout.channel.recv_exit_status() # blocks + out, err = stdout.read(), stderr.read() + + if rc != 0: + raise RemoteCmdError( + msg="Remote ssh command failed with status {0}".format(rc), + cmd=cmd, as_root=as_root, rc=rc, stderr=stderr, stdout=stdout + ) + + return out, err def _get_remote_results_dir(self): if any(x is None for x in [self.remote_build_dir, @@ -156,13 +140,8 @@ class Builder(object): Packages in buildroot_pkgs are added to minimal buildroot. """ cfg_path = self.get_chroot_config_path(self.job.chroot) - - try: - copy_cmd = "cp /etc/mock/{chroot}.cfg {dest}".format(chroot=self.job.chroot, dest=cfg_path) - self.run_ansible_with_check(copy_cmd, module_name="shell") - except BuilderError as err: - self.log.exception(err) - raise err + copy_cmd = "cp /etc/mock/{chroot}.cfg {dest}".format(chroot=self.job.chroot, dest=cfg_path) + self._run_ssh_cmd(copy_cmd) if ("'{0} '".format(self.buildroot_pkgs) != pipes.quote(str(self.buildroot_pkgs) + ' ')): @@ -171,79 +150,53 @@ class Builder(object): # allowed in packages name raise BuilderError("Do not try this kind of attack on me") - self.log.info("putting {0} into minimal buildroot of {1}" - .format(self.buildroot_pkgs, self.job.chroot)) - - kwargs = { - "cfg_path": cfg_path, - "chroot": self.job.chroot, - "pkgs": self.buildroot_pkgs, - "net_enabled": "True" if self.job.enable_net else "False", - } - buildroot_cmd = ( - "dest={cfg_path}" - " line=\"config_opts['chroot_setup_cmd'] = 'install \\1 {pkgs}'\"" - " regexp=\"^.*chroot_setup_cmd.*(@buildsys-build|@build|buildsys-build buildsys-macros).*$\"" - " backrefs=yes" - ) - buildroot_custom_cmd = ( - "dest={cfg_path}" - " line=\"config_opts['chroot_setup_cmd'] = 'install {pkgs}'\"" - " regexp=\"config_opts\\['chroot_setup_cmd'\\] = ''$\"" - ) - set_networking_cmd = ( - "dest={cfg_path}" - " line=\"config_opts['use_host_resolv'] = {net_enabled}\"" - " regexp=\"^.*use_host_resolv.*$\"" + set_networking_cmd = "echo \"config_opts['use_host_resolv'] = {net_enabled}\" >> {path}".format( + net_enabled=("True" if self.job.enable_net else "False"), path=cfg_path ) - - try: - self.run_ansible_with_check(set_networking_cmd.format(**kwargs), - module_name="lineinfile") - if self.buildroot_pkgs: - if 'custom' in self.job.chroot: - self.run_ansible_with_check(buildroot_custom_cmd.format(**kwargs), - module_name="lineinfile") - else: - self.run_ansible_with_check(buildroot_cmd.format(**kwargs), - module_name="lineinfile") - - if self.module_dist_tag: - set_dist_tag_cmd = ( - "dest={cfg_path}" - " line=\"config_opts['macros']['%dist'] = '{dist_tag}'\"" - " regexp=\"^.*config_opts['macros']['%dist'].*$\"" + self._run_ssh_cmd(set_networking_cmd) + + if self.buildroot_pkgs: + if 'custom' in self.job.chroot: + pattern = "^config_opts['chroot_setup_cmd'] = ''$" + replace_by = "config_opts['chroot_setup_cmd'] = 'install {pkgs}'".format(pkgs=self.buildroot_pkgs) + buildroot_custom_cmd = "sed -i \"s/{pattern}/{replace_by}/\" {path}".format( + pattern=pattern, replace_by=replace_by, path=cfg_path ) - self.run_ansible_with_check(set_dist_tag_cmd.format( - cfg_path=cfg_path, dist_tag=self.module_dist_tag), module_name="lineinfile") + self._run_ssh_cmd(buildroot_custom_cmd) + else: + pattern = "^.*chroot_setup_cmd.*\(@buildsys-build\|@build\|buildsys-build buildsys-macros\).*$" + replace_by = "config_opts['chroot_setup_cmd'] = 'install \\1 {pkgs}'".format(pkgs=self.buildroot_pkgs) + buildroot_cmd = "sed -i \"s/{pattern}/{replace_by}/\" {path}".format( + pattern=pattern, replace_by=replace_by, path=cfg_path + ) + self._run_ssh_cmd(buildroot_cmd) - except Exception as err: - self.log.exception(err) - raise + if self.module_dist_tag: + dist_tag_cmd = "echo \"config_opts['macros']['%dist'] = '{dist_tag}'\" >> {path}".format( + dist_tag=self.module_dist_tag, path=cfg_path + ) + self._run_ssh_cmd(dist_tag_cmd) def collect_built_packages(self): self.log.info("Listing built binary packages") - results = self._run_ansible( + built_packages = self._run_ssh_cmd( "cd {0} && " "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " "done".format(pipes.quote(self._get_remote_results_dir())) - ) - - built_packages = list(results["contacted"].values())[0][u"stdout"] + )[0].strip() self.log.info("Built packages:\n{}".format(built_packages)) return built_packages def check_build_success(self): successfile = os.path.join(self._get_remote_results_dir(), "success") - ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile)) - check_for_ans_error(ansible_test_results, self.hostname) + self._run_ssh_cmd("/usr/bin/test -f {0}".format(successfile)) def download_job_pkg_to_builder(self): repo_url = "{}/{}.git".format(self.opts.dist_git_url, self.job.git_repo) self.log.info("Cloning Dist Git repo {}, branch {}, hash {}".format( self.job.git_repo, self.job.git_branch, self.job.git_hash)) - results = self._run_ansible( + stdout, stderr = self._run_ssh_cmd( "rm -rf /tmp/build_package_repo && " "mkdir /tmp/build_package_repo && " "cd /tmp/build_package_repo && " @@ -261,11 +214,11 @@ class Builder(object): # Wrote: /tmp/.../copr-ping/copr-ping-1-1.fc21.src.rpm try: - self.remote_pkg_path = list(results["contacted"].values())[0][u"stdout"].split("Wrote: ")[1] - self.remote_pkg_name = os.path.basename(self.remote_pkg_path).replace(".src.rpm", "") + self.remote_pkg_path = stdout.split("Wrote: ")[1] + self.remote_pkg_name = os.path.basename(self.remote_pkg_path).replace(".src.rpm", "").strip() except Exception: self.log.exception("Failed to obtain srpm from dist-git") - raise BuilderError("Failed to obtain srpm from dist-git: ansible results {}".format(results)) + raise BuilderError("Failed to obtain srpm from dist-git: stdout: {}, stderr: {}".format(stdout, stderr)) self.log.info("Got srpm to build: {}".format(self.remote_pkg_path)) @@ -307,10 +260,6 @@ class Builder(object): buildcmd += self.remote_pkg_path return buildcmd - def run_build(self, buildcmd): - self.log.info("executing: {0}".format(buildcmd)) - return self._run_ansible(buildcmd) - def setup_pubsub_handler(self): self.rc = get_redis_connection(self.opts) @@ -352,14 +301,14 @@ class Builder(object): # construct the mockchain command buildcmd = self.gen_mockchain_command() - # run the mockchain command async - ansible_build_results = self.run_build(buildcmd) - check_for_ans_error(ansible_build_results, self.hostname) # on error raises AnsibleResponseError + # run the mockchain command + stdout, stderr = self._run_ssh_cmd(buildcmd) # we know the command ended successfully but not if the pkg built # successfully self.check_build_success() - return get_ans_results(ansible_build_results, self.hostname).get("stdout", "") + + return stdout def rsync_call(self, source_path, target_path): ensure_dir_exists(target_path, self.log) @@ -409,80 +358,21 @@ class Builder(object): raise BuilderError("{0} could not be resolved".format(self.hostname)) try: - # check_for_ans_error(res, self.hostname) - self.run_ansible_with_check("/bin/rpm -q mock rsync") - except AnsibleCallError: + self._run_ssh_cmd("/bin/rpm -q mock rsync") + except RemoteCmdError: raise BuilderError(msg="Build host `{0}` does not have mock or rsync installed" .format(self.hostname)) # test for path existence for mockchain and chroot config for this chroot try: - self.run_ansible_with_check("/usr/bin/test -f {0}".format(mockchain)) - except AnsibleCallError: + self._run_ssh_cmd("/usr/bin/test -f {0}".format(mockchain)) + except RemoteCmdError: raise BuilderError(msg="Build host `{}` missing mockchain binary `{}`" .format(self.hostname, mockchain)) try: - self.run_ansible_with_check("/usr/bin/test -f /etc/mock/{}.cfg" - .format(self.job.chroot)) - except AnsibleCallError: + self._run_ssh_cmd("/usr/bin/test -f /etc/mock/{}.cfg" + .format(self.job.chroot)) + except RemoteCmdError: raise BuilderError(msg="Build host `{}` missing mock config for chroot `{}`" .format(self.hostname, self.job.chroot)) - - -def get_ans_results(results, hostname): - if hostname in results["dark"]: - return results["dark"][hostname] - if hostname in results["contacted"]: - return results["contacted"][hostname] - - return {} - - -def check_for_ans_error(results, hostname, err_codes=None, success_codes=None): - """ - dict includes 'msg' - may include 'rc', 'stderr', 'stdout' and any other requested result codes - - :raises AnsibleResponseError: - :raises VmError: - """ - - if err_codes is None: - err_codes = [] - if success_codes is None: - success_codes = [0] - - if ("dark" in results and hostname in results["dark"]) or \ - "contacted" not in results or hostname not in results["contacted"]: - - raise VmError(msg="Error: Could not contact/connect to {}. raw results: {}".format(hostname, results)) - - error = False - err_results = {} - if err_codes or success_codes: - if hostname in results["contacted"]: - if "rc" in results["contacted"][hostname]: - rc = int(results["contacted"][hostname]["rc"]) - err_results["return_code"] = rc - # check for err codes first - if rc in err_codes: - error = True - err_results["msg"] = "rc {0} matched err_codes".format(rc) - elif rc not in success_codes: - error = True - err_results["msg"] = "rc {0} not in success_codes".format(rc) - - elif ("failed" in results["contacted"][hostname] and - results["contacted"][hostname]["failed"]): - - error = True - err_results["msg"] = "results included failed as true" - - if error: - for item in ["stdout", "stderr"]: - if item in results["contacted"][hostname]: - err_results[item] = results["contacted"][hostname][item] - - if error: - raise AnsibleResponseError(**err_results) diff --git a/backend/backend/vm_manage/check.py b/backend/backend/vm_manage/check.py index da1680f..efd17b5 100644 --- a/backend/backend/vm_manage/check.py +++ b/backend/backend/vm_manage/check.py @@ -1,11 +1,10 @@ # coding: utf-8 import json +import paramiko #from setproctitle import setproctitle # from multiprocessing import Process #from threading import Thread -from ansible.runner import Runner - from backend.helpers import get_redis_connection from backend.vm_manage import PUBSUB_MB, EventTopics from backend.vm_manage.executor import Executor @@ -20,22 +19,8 @@ def check_health(opts, vm_name, vm_ip): :param vm_ip: ip address to the newly created VM :raises: :py:class:`~backend.exceptions.CoprWorkerSpawnFailError`: validation fails """ - # setproctitle("check VM: {}".format(vm_ip)) - log = get_redis_logger(opts, "vmm.check_health.detached", "vmm") - runner_options = dict( - remote_user=opts.build_user or "root", - host_list="{},".format(vm_ip), - pattern=vm_ip, - forks=1, - transport=opts.ssh.transport, - timeout=opts.vm_ssh_check_timeout - ) - connection = Runner(**runner_options) - connection.module_name = "shell" - connection.module_args = "echo hello" - result = { "vm_ip": vm_ip, "vm_name": vm_name, @@ -43,17 +28,16 @@ def check_health(opts, vm_name, vm_ip): "result": "OK", "topic": EventTopics.HEALTH_CHECK } + err_msg = None try: - res = connection.run() - if vm_ip not in res.get("contacted", {}): - err_msg = ( - "VM is not responding to the testing playbook." - "Runner options: {}".format(runner_options) + - "Ansible raw response:\n{}".format(res)) - + conn = paramiko.SSHClient() + conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + conn.connect(hostname=vm_ip, username=opts.build_user or "root") + stdin, stdout, stderr = conn.exec_command("echo hello") + stdout.channel.recv_exit_status() except Exception as error: - err_msg = "Failed to check VM ({})due to ansible error: {}".format(vm_ip, error) + err_msg = "Healtcheck failed for VM {} with error {}".format(vm_ip, error) log.exception(err_msg) try: diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 9c1f7f1..db76b0d 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -50,13 +50,8 @@ BuildRequires: pytz # BuildRequires: python-plumbum # BuildRequires: wget -- ??? -%if 0%{?fedora} >= 24 BuildRequires: ansible Requires: ansible -%else -BuildRequires: ansible1.9 -Requires: ansible1.9 -%endif #for doc package BuildRequires: sphinx