From b77be8117303863a20cb5da3312c65fcb0411958 Mon Sep 17 00:00:00 2001 From: Jan Kaluža Date: May 25 2017 13:16:55 +0000 Subject: Merge #556 `Don't build/obtain SRPM packages on the host` --- diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 6c5bd4e..af3bd68 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -38,7 +38,8 @@ import module_build_service.scheduler.consumer from base import GenericBuilder from utils import (build_from_scm, fake_repo_done_message, - create_local_repo_from_koji_tag, execute_cmd) + create_local_repo_from_koji_tag, execute_cmd, + find_srpm) from KojiModuleBuilder import KojiModuleBuilder from module_build_service.models import ModuleBuild @@ -305,7 +306,7 @@ mdpolicy=group:primary if os.path.exists(old_log): os.rename(old_log, new_log) - def build_srpm(self, artifact_name, source, build_id): + def build_srpm(self, artifact_name, source, build_id, builder): """ Builds the artifact from the SRPM. """ @@ -315,16 +316,6 @@ mdpolicy=group:primary mock_config = os.path.join(self.configdir, "mock-%s.cfg" % str(threading.current_thread().name)) - # Clear resultsdir associated with this thread or in case it does not - # exist, create it. - resultsdir = os.path.join(self.resultsdir, - str(threading.current_thread().name)) - if os.path.exists(resultsdir): - for name in os.listdir(resultsdir): - os.remove(os.path.join(resultsdir, name)) - else: - os.makedirs(resultsdir) - # Open the logs to which we will forward mock stdout/stderr. mock_stdout_log = open(os.path.join(self.resultsdir, artifact_name + "-mock-stdout.log"), "w") @@ -337,17 +328,16 @@ mdpolicy=group:primary stdout=mock_stdout_log, stderr=mock_stderr_log) # Start the build and store results to resultsdir - execute_cmd(["mock", "-v", "-r", mock_config, - "--no-clean", "--rebuild", source, - "--resultdir=%s" % resultsdir], - stdout=mock_stdout_log, stderr=mock_stderr_log) + resultsdir = builder.resultsdir + builder.build(mock_stdout_log, mock_stderr_log) + srpm = find_srpm(resultsdir) # Emit messages simulating complete build. These messages # are put in the scheduler's work queue and are handled # by MBS after the build_srpm() method returns and scope gets # back to scheduler.main.main() method. state = koji.BUILD_STATES['COMPLETE'] - self._send_build_change(state, source, build_id) + self._send_build_change(state, srpm, build_id) with open(os.path.join(resultsdir, "status.log"), 'w') as f: f.write("complete\n") @@ -396,6 +386,7 @@ mdpolicy=group:primary # generate the thread-specific mock config by writing it to fs again. self._load_mock_config() self._write_mock_config() + mock_config = os.path.join(self.configdir, "mock-%s.cfg" % str(threading.current_thread().name)) # Get the build-id in thread-safe manner. build_id = None @@ -403,19 +394,22 @@ mdpolicy=group:primary MockModuleBuilder._build_id += 1 build_id = int(MockModuleBuilder._build_id) + # Clear resultsdir associated with this thread or in case it does not + # exist, create it. + resultsdir = os.path.join(self.resultsdir, + str(threading.current_thread().name)) + if os.path.exists(resultsdir): + for name in os.listdir(resultsdir): + os.remove(os.path.join(resultsdir, name)) + else: + os.makedirs(resultsdir) + # Git sources are treated specially. if source.startswith(("git://", "http://", "https://")): - # Open the srpm-stdout and srpm-stderr logs and build from SCM. - srpm_stdout_fn = os.path.join(self.resultsdir, - artifact_name + "-srpm-stdout.log") - srpm_stderr_fn = os.path.join(self.resultsdir, - artifact_name + "-srpm-stderr.log") - with open(srpm_stdout_fn, "w") as srpm_stdout_log, open(srpm_stderr_fn, "w") as srpm_stderr_log: - return build_from_scm(artifact_name, source, - self.config, self.build_srpm, data=build_id, - stdout=srpm_stdout_log, stderr=srpm_stderr_log) + builder = SCMBuilder(mock_config, resultsdir, source, artifact_name) else: - return self.build_srpm(artifact_name, source, build_id) + builder = LocalBuilder(mock_config, resultsdir, source) + return self.build_srpm(artifact_name, source, build_id, builder) @staticmethod def get_disttag_srpm(disttag, module_build): @@ -429,4 +423,42 @@ mdpolicy=group:primary pass - +class BaseBuilder(object): + def __init__(self, config, resultsdir): + self.config = config + self.resultsdir = resultsdir + self.cmd = ["mock", "-v", "-r", config, + "--no-clean", + "--resultdir=%s" % resultsdir] + + def build(self, stdout, stderr): + execute_cmd(self.cmd, stdout=stdout, stderr=stderr) + + +class LocalBuilder(BaseBuilder): + def __init__(self, config, resultsdir, source): + super(LocalBuilder, self).__init__(config, resultsdir) + self.cmd.extend(["--rebuild", source]) + + +class SCMBuilder(BaseBuilder): + def __init__(self, config, resultsdir, source, artifact_name): + super(SCMBuilder, self).__init__(config, resultsdir) + with open(config, "a") as f: + branch = source.split("?#")[1] + distgit_cmds = self._get_distgit_commands(source) + distgit_get = distgit_cmds[0].format(artifact_name) + f.writelines([ + "config_opts['scm'] = True\n", + "config_opts['scm_opts']['method'] = 'distgit'\n", + "config_opts['scm_opts']['branch'] = '{}'\n".format(branch), + "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), + "config_opts['scm_opts']['distgit_get'] = '{}'\n".format(distgit_get), + "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(distgit_cmds[1]), + ]) + + def _get_distgit_commands(self, source): + for host, cmds in conf.distgits.items(): + if source.startswith(host): + return cmds + raise KeyError("No defined commands for {}".format(source)) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index dbd1352..ffd77ad 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -69,6 +69,12 @@ def build_from_scm(artifact_name, source, config, build_srpm, return ret +def find_srpm(cod): + for f in os.listdir(cod): + if f.endswith(".src.rpm"): + return os.path.join(cod, f) + + def execute_cmd(args, stdout=None, stderr=None, cwd=None): """ Executes command defined by `args`. If `stdout` or `stderr` is set to diff --git a/module_build_service/config.py b/module_build_service/config.py index 392c948..3a1e976 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -286,6 +286,12 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Trusted certificate for ssl connection.'}, + 'distgits': { + 'type': dict, + 'default': { + 'git://pkgs.fedoraproject.org': ('fedpkg clone --anonymous {}', 'fedpkg sources'), + }, + 'desc': 'Mapping between dist-git and command to '}, 'mock_config': { 'type': str, 'default': 'fedora-25-x86_64', @@ -377,7 +383,7 @@ class Config(object): if key in self._defaults: # type conversion for configuration item convert = self._defaults[key]['type'] - if convert in [bool, int, list, str, set]: + if convert in [bool, int, list, str, set, dict]: try: # Do no try to convert None... if value is not None: