From 11a51a0404ed8cb9911a01cc4c7e16523c660b33 Mon Sep 17 00:00:00 2001 From: Martin Curlej Date: Nov 10 2017 17:33:42 +0000 Subject: mbs-buil local now builds only from yaml files --- diff --git a/contrib/mbs-build b/contrib/mbs-build index 2a09699..94fe0c6 100755 --- a/contrib/mbs-build +++ b/contrib/mbs-build @@ -267,7 +267,7 @@ def get_scm_url(scm_url, pyrpkg, local=False): return scm_url -def get_scm_branch(branch): +def get_scm_branch(branch=None): """ If `branch` it not set, returns the branch name based on git repository in the `os.getcwd()`. @@ -317,39 +317,36 @@ def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=Tru return -3, None -def do_local_build(scm_url, branch, skiptests, local_builds_nsvs, log_flag=None, - yaml_file=None, stream=None): +def do_local_build(local_builds_nsvs, log_flag=None, yaml_file=None, stream=None): """ Starts the local build using the 'mbs-manager build_module_locally' command. Returns exit code of that command or None when scm_url or branch are not set and cannot be obtained from the CWD. """ command = ['mbs-manager'] + command.append('build_module_locally') + if local_builds_nsvs: + for build_id in local_builds_nsvs: + command += ['--add-local-build', build_id] + if yaml_file: - command.append('build_module_locally_from_file') - logging.info("Starting local build from yaml file %s" % yaml_file) command.extend(["--file", yaml_file]) - if stream: - command.extend(["--stream", stream]) else: - command.append('build_module_locally') - scm_url = get_scm_url(scm_url, None, local=True) - branch = get_scm_branch(branch) - if not scm_url or not branch: - return None + module_dir = os.getcwd() + module_name = os.path.basename(module_dir) + yaml_file = os.path.join(module_dir, module_name + ".yaml") + command.extend(["--file", yaml_file]) - logging.info("Starting local build of %s, branch %s", scm_url, branch) - if log_flag: - command.append(log_flag) - if skiptests: - command.append('--skiptests') - logging.info("Tests will be skipped due to --skiptests option.") + if stream: + command.extend(["--stream", stream]) + else: + branch = get_scm_branch() + command.extend(["--stream", branch]) - if local_builds_nsvs: - for build_id in local_builds_nsvs: - command += ['--add-local-build', build_id] + if log_flag: + command.append(log_flag) - command.extend([scm_url, branch]) + logging.info("Starting local build of %s, branch %s", yaml_file, branch) process = subprocess.Popen(command) process.communicate() @@ -496,12 +493,8 @@ def main(): "When 'scm_url' or 'branch' is not set, it presumes you are " "executing this command in the directory with the cloned git " "repository with a module.") - parser_local.add_argument("scm_url", nargs='?') - parser_local.add_argument("branch", nargs='?') parser_local.add_argument("--add-local-build", "-l", action='append', dest="local_builds_nsvs", metavar='BUILD_ID') - parser_local.add_argument('--skiptests', dest='skiptests', action='store_true', - help="add macro for skipping check section/phase") parser_local.add_argument('--file', dest='file', action='store', help="Path to the modulemd yaml file") parser_local.add_argument('--stream', dest='stream', action='store', @@ -552,9 +545,7 @@ def main(): else: print("Submitted module build %r" % build_id) elif args.cmd_name == "local": - sys.exit(do_local_build(args.scm_url, args.branch, args.skiptests, - args.local_builds_nsvs, log_flag, args.file, - args.stream)) + sys.exit(do_local_build(args.local_builds_nsvs, log_flag, args.file, args.stream)) elif args.cmd_name == "watch": # Watch the module build. try: diff --git a/module_build_service/manage.py b/module_build_service/manage.py index d1f7c7f..b3777b9 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -32,7 +32,6 @@ from werkzeug.datastructures import FileStorage from module_build_service import app, conf, db, create_app from module_build_service import models from module_build_service.utils import ( - submit_module_build_from_scm, submit_module_build_from_yaml, load_local_builds, ) @@ -90,11 +89,10 @@ def cleardb(): models.ComponentBuild.query.delete() -@manager.option('branch') -@manager.option('url') -@manager.option('--skiptests', action='store_true') +@manager.option('--stream', action='store', dest="stream") +@manager.option('--file', action='store', dest="yaml_file") @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs') -def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False, yaml_file=None, stream=None): +def build_module_locally(local_build_nsvs=None, yaml_file=None, stream=None): """ Performs local module build using Mock """ if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']: @@ -121,25 +119,18 @@ def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False, ya if yaml_file and yaml_file.endswith(".yaml"): yaml_file_path = os.path.abspath(yaml_file) with open(yaml_file_path) as fd: - filename = yaml_file.split("/")[-1] + filename = os.path.basename(yaml_file) handle = FileStorage(fd) handle.filename = filename - submit_module_build_from_yaml(username, handle, stream) + submit_module_build_from_yaml(username, handle, str(stream)) else: - submit_module_build_from_scm(username, url, branch, allow_local_url=True, - skiptests=skiptests) + raise IOError("Provided modulemd file is not a yaml file.") stop = module_build_service.scheduler.make_simple_stop_condition(db.session) # Run the consumer until stop_condition returns True module_build_service.scheduler.main([], stop) -@manager.option('--file', action='store', dest="yaml_file") -@manager.option('--stream', action='store', dest="stream") -def build_module_locally_from_file(yaml_file, stream=None): - build_module_locally(None, None, yaml_file=yaml_file, stream=str(stream)) - - @console_script_help @manager.command def run(host=None, port=None, debug=None): diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 2f83dad..896f303 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -91,9 +91,10 @@ class SCM(object): # non-local bare repositories self.local = False self.bare_repo = True - if url.startswith("file://") and allow_local: + if self.repository.startswith("file://") and allow_local: self.local = True - self.bare_repo = self._is_bare_repo(self.repository[7:]) + abs_repo_path = self.repository[7:] + self.bare_repo = self._is_bare_repo(abs_repo_path) if not self.commit: self.commit = self.get_latest(self.branch) self.latest = True @@ -274,24 +275,24 @@ class SCM(object): def patch_with_uncommited_changes(self, source_dir): """ - This method patches the given tmp git repository with uncommented changes from it - origin git dir. Creates a patch file witch holds result for `git diff` command + This method patches the given tmp git repository with uncommented changes from its + origin git dir. Creates a patch file which holds the result for `git diff` command executed in the origin repo. source_dir (str): path to the temp git repo """ module_diff = ['git', 'diff'] - # striping the self.repository from 'file://' + # stripping the 'file:// from self.repository' _, diff, _ = SCM._run(module_diff, chdir=self.repository[7:]) if diff: try: log.debug("Working with local, non-bare repository. Applying uncommited changes.") - patch_file = source_dir + "/patch" - with open(patch_file, "w+") as fd: + patch_file = os.path.join(source_dir, "patch") + with open(patch_file, "w") as fd: fd.write(diff) module_patch = ['git', 'apply', 'patch'] SCM._run(module_patch, chdir=source_dir) - except Exception as e: + except Exception: log.exception("Failed to update repo %s with uncommited changes." % source_dir) raise diff --git a/tests/test_scm.py b/tests/test_scm.py index 016cccd..a88cb75 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -23,10 +23,8 @@ import os import shutil import tempfile -import subprocess as sp import unittest -from mock import patch from nose.tools import raises import module_build_service.scm @@ -38,17 +36,12 @@ repo_path = 'file://' + os.path.dirname(__file__) + "/scm_data/testrepo" class TestSCMModule(unittest.TestCase): def setUp(self): - # this var holds path to a cloned repo. For some tests we need a working - # tree not only a bare repo - self.temp_cloned_repo = None self.tempdir = tempfile.mkdtemp() self.repodir = self.tempdir + '/testrepo' def tearDown(self): if os.path.exists(self.tempdir): shutil.rmtree(self.tempdir) - if self.temp_cloned_repo and os.path.exists(self.temp_cloned_repo): - shutil.rmtree(self.temp_cloned_repo) def test_simple_local_checkout(self): """ See if we can clone a local git repo. """ @@ -127,75 +120,3 @@ class TestSCMModule(unittest.TestCase): def test_get_latest_incorect_component_branch(self): scm = module_build_service.scm.SCM(repo_path) scm.get_latest(branch='foobar') - - def test_patch_with_uncommited_changes(self): - cloned_repo, repo_link = self._clone_from_bare_repo() - with open(cloned_repo + "/foo", "a") as fd: - fd.write("Winter is comming!") - scm = module_build_service.scm.SCM(repo_link, allow_local=True) - scm.checkout(self.tempdir) - with open(self.repodir + "/foo", "r") as fd: - foo = fd.read() - - assert "Winter is comming!" in foo - - def test_dont_patch_if_commit_ref(self): - target = '7035bd33614972ac66559ac1fdd019ff6027ad21' - cloned_repo, repo_link = self._clone_from_bare_repo() - scm = module_build_service.scm.SCM(repo_link + "?#" + target, "dev", allow_local=True) - with open(cloned_repo + "/foo", "a") as fd: - fd.write("Winter is comming!") - scm.checkout(self.tempdir) - with open(self.repodir + "/foo", "r") as fd: - foo = fd.read() - - assert "Winter is comming!" not in foo - - @patch("module_build_service.scm.open") - @patch("module_build_service.scm.log") - def test_patch_with_exception(self, mock_log, mock_open): - cloned_repo, repo_link = self._clone_from_bare_repo() - with open(cloned_repo + "/foo", "a") as fd: - fd.write("Winter is comming!") - mock_open.side_effect = Exception("Can't write to patch file!") - scm = module_build_service.scm.SCM(repo_link, allow_local=True) - with self.assertRaises(Exception) as ex: - scm.checkout(self.tempdir) - mock_open.assert_called_once_with(self.repodir + "/patch", "w+") - err_msg = "Failed to update repo %s with uncommited changes." % self.repodir - mock_log.assert_called_once_with(err_msg) - assert ex is mock_open.side_effect - assert 0 - - def test_is_bare_repo(self): - scm = module_build_service.scm.SCM(repo_path) - assert scm.bare_repo - - def _clone_from_bare_repo(self): - """ - Helper method which will clone the bare test repo. Also it will create - a dev branch and track it to the remote bare repo. - - Returns: - str: returns the path to the cloned repo - str: returns the file link (file://) to the repo - """ - self.temp_cloned_repo = tempfile.mkdtemp() - cloned_repo = self.temp_cloned_repo + "/testrepo" - clone_cmd = ["git", "clone", "-q", repo_path] - get_dev_branch_cmd = ["git", "branch", "--track", "dev", "origin/dev"] - proc = sp.Popen(clone_cmd, stdout=sp.PIPE, stderr=sp.PIPE, - cwd=self.temp_cloned_repo) - stdout, stderr = proc.communicate() - if stderr: - raise Exception("Failed to clone repo: %s, err code: %s" - % (stderr, proc.returncode)) - proc = sp.Popen(get_dev_branch_cmd, stdout=sp.PIPE, stderr=sp.PIPE, - cwd=cloned_repo) - stdout, stderr = proc.communicate() - if stderr: - raise Exception("Failed to create and track dev branch: %s, err code: %s" - % (stderr, proc.returncode)) - repo_link = "".join(["file://", cloned_repo]) - - return cloned_repo, repo_link