From 918104c896abf2c7d95d0633ae650ef85e97c817 Mon Sep 17 00:00:00 2001 From: clime Date: Sep 06 2017 22:32:44 +0000 Subject: [backend] move process_build_results (now get_build_details) logic into worker --- diff --git a/backend/backend/daemons/worker.py b/backend/backend/daemons/worker.py index 72c2529..5729376 100644 --- a/backend/backend/daemons/worker.py +++ b/backend/backend/daemons/worker.py @@ -5,6 +5,7 @@ import time import gzip import shutil import multiprocessing +import pipes from setproctitle import setproctitle from ..exceptions import MockRemoteError, CoprWorkerError, VmError, NoVmAvailable @@ -12,7 +13,7 @@ from ..job import BuildJob from ..mockremote import MockRemote from ..constants import BuildStatus, build_log_format from ..helpers import register_build_result, get_redis_connection, get_redis_logger, \ - local_file_logger + local_file_logger, run_cmd from ..vm_manage import VmStates from ..msgbus import MsgBusStomp, MsgBusFedmsg @@ -208,7 +209,7 @@ class Worker(multiprocessing.Process): mr.download_results() mr.on_success_build() - build_details = mr.process_build_results() + build_details = self.get_build_details(job) job.update(build_details) if self.opts.do_sign: @@ -257,6 +258,33 @@ class Worker(multiprocessing.Process): self.update_process_title(suffix="Task: {} chroot: {} done" .format(job.build_id, job.chroot)) + def collect_built_packages(self, job): + self.log.info("Listing built binary packages in {}" + .format(job.results_dir)) + + cmd = ( + "cd {0} && " + "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" + " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " + "done".format(pipes.quote(job.results_dir)) + ) + built_packages = run_cmd(cmd, shell=True)[0].strip() + + self.log.info("Built packages:\n{}".format(built_packages)) + return built_packages + + def get_build_details(self, job): + """ + :return: dict with build_details + :raises MockRemoteError: Something happened with build itself + """ + try: + build_details = {"built_packages": self.collect_built_packages(job)} + self.log.info("build details: {}".format(build_details)) + except Exception as e: + self.log.exception(str(e)) + raise CoprWorkerError("Error while collecting built packages for {}.".format(job)) + def copy_mock_logs(self, job): if not os.path.isdir(job.results_dir): self.log.info("Job results dir doesn't exists, couldn't copy main log; path: {}" diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index cf00b9a..10a70d7 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -58,27 +58,29 @@ def cmd_debug(cmd, rc, out, err, log): log.info("stderr: {}".format(err)) -def run_cmd(cmd): +def run_cmd(cmd, shell=False): """Runs given command in a subprocess. Params ------ - cmd: list(str) + cmd: list(str) or str if shell==True command to be executed and its arguments + shell: bool + if the command should be interpreted by shell Returns ------- munch.Munch(stdout, stderr, returncode) executed cmd, standard output, error output, and the return code """ - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) (stdout, stderr) = process.communicate() return munch.Munch( - cmd = cmd, - stdout = stdout, - stderr = stderr, - returncode = process.returncode + cmd=cmd, + stdout=stdout, + stderr=stderr, + returncode=process.returncode ) diff --git a/backend/backend/job.py b/backend/backend/job.py index 71f5827..a8b2527 100644 --- a/backend/backend/job.py +++ b/backend/backend/job.py @@ -58,6 +58,7 @@ class BuildJob(object): self.pkg_epoch = None self.pkg_release = None + self.srpm_url = None # TODO: validate update data, user marshmallow for key, val in task_data.items(): diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index d1c060a..fe22d2b 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -312,20 +312,6 @@ class MockRemote(object): except Exception as err: self.log.exception(err) - def process_build_results(self): - """ - :return: dict with build_details - :raises MockRemoteError: Something happened with build itself - """ - try: - build_details = {"built_packages": self.builder.collect_built_packages()} - self.log.info("build details: {}".format(build_details)) - except BuilderError as error: - self.log.error(str(error)) - raise MockRemoteError("Error while collecting built packages for {}.".format(self.job)) - - return build_details - def mark_dir_with_build_id(self): """ Places "build.info" which contains job build_id diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index f222620..07d53bd 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -77,18 +77,6 @@ class Builder(object): cmd, rc, err, out) return out, err - def collect_built_packages(self): - # TODO: cold we delegate this to copr-builder? - self.log.info("Listing built binary packages") - built_packages = self._run_ssh_cmd( - "cd {0} && " - "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" - " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " - "done".format(pipes.quote(self.resultdir)) - )[0].strip() - self.log.info("Built packages:\n{}".format(built_packages)) - return built_packages - def check_build_success(self): successfile = os.path.join(self.resultdir, "success") self._run_ssh_cmd("/usr/bin/test -f {0}".format(successfile)) diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index ed26ffe..8957fa1 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -744,7 +744,7 @@ GROUP BY db.session.add(build_chroot) - for attr in ["results", "built_packages"]: + for attr in ["results", "built_packages", "srpm_url"]: value = upd_dict.get(attr, None) if value: setattr(build, attr, value)