From 77366b94cef9b025303b948a737613ef51e66c48 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Nov 24 2016 13:01:28 +0000 Subject: Try to build testmodule.yaml as unit-test with fake builder backend. --- diff --git a/module_build_service/builder.py b/module_build_service/builder.py index c66ca4e..4001fd5 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -107,6 +107,11 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): """ backend = "generic" + backends = {} + + @classmethod + def register_backend_class(cls, backend_class): + GenericBuilder.backends[backend_class.backend] = backend_class @classmethod def create(cls, owner, module, backend, config, **extra): @@ -123,14 +128,8 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): if isinstance(config.system, Mock): return KojiModuleBuilder(owner=owner, module=module, config=config, **extra) - elif backend == "koji": - return KojiModuleBuilder(owner=owner, module=module, - config=config, **extra) - elif backend == "copr": - return CoprModuleBuilder(owner=owner, module=module, - config=config, **extra) - elif backend == "mock": - return MockModuleBuilder(owner=owner, module=module, + elif backend in GenericBuilder.backends: + return GenericBuilder.backends[backend](owner=owner, module=module, config=config, **extra) else: raise ValueError("Builder backend='%s' not recognized" % backend) @@ -146,10 +145,9 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): Returns URL of repository containing the built artifacts for the tag with particular name and architecture. """ - if backend == "koji": - return KojiModuleBuilder.repo_from_tag(config, tag_name, arch) - if backend == "copr": - return CoprModuleBuilder.repo_from_tag(config, tag_name, arch) + if backend in GenericBuilder.backends: + return GenericBuilder.backends[backend].repo_from_tag( + config, tag_name, arch) else: raise ValueError("Builder backend='%s' not recognized" % backend) @@ -1100,3 +1098,7 @@ class MockModuleBuilder(GenericBuilder): def get_disttag_srpm(disttag): # @FIXME return KojiModuleBuilder.get_disttag_srpm(disttag) + +GenericBuilder.register_backend_class(KojiModuleBuilder) +GenericBuilder.register_backend_class(CoprModuleBuilder) +GenericBuilder.register_backend_class(MockModuleBuilder) diff --git a/module_build_service/scheduler/main.py b/module_build_service/scheduler/main.py index d9c54b3..4254570 100644 --- a/module_build_service/scheduler/main.py +++ b/module_build_service/scheduler/main.py @@ -65,15 +65,21 @@ def module_build_state_from_msg(msg): class MessageIngest(threading.Thread): - def __init__(self, outgoing_work_queue, *args, **kwargs): + def __init__(self, outgoing_work_queue, stop_after_build, *args, **kwargs): self.outgoing_work_queue = outgoing_work_queue super(MessageIngest, self).__init__(*args, **kwargs) + self.stop_after_build = stop_after_build def run(self): for msg in module_build_service.messaging.listen(conf): self.outgoing_work_queue.put(msg) + if type(msg) == module_build_service.messaging.RidaModule: + if (self.stop_after_build and module_build_state_from_msg(msg) + in [models.BUILD_STATES["failed"], models.BUILD_STATES["ready"]]): + break + class MessageWorker(threading.Thread): @@ -130,7 +136,6 @@ class MessageWorker(threading.Thread): if msg is STOP_WORK: log.info("Worker thread received STOP_WORK, shutting down...") - os._exit(0) break try: @@ -183,9 +188,10 @@ class Poller(threading.Thread): def __init__(self, outgoing_work_queue, *args, **kwargs): self.outgoing_work_queue = outgoing_work_queue super(Poller, self).__init__(*args, **kwargs) + self.stop = False def run(self): - while True: + while not self.stop: with models.make_session(conf) as session: self.log_summary(session) # XXX: detect whether it's really stucked first @@ -195,7 +201,10 @@ class Poller(threading.Thread): self.process_paused_module_builds(conf, session) log.info("Polling thread sleeping, %rs" % conf.polling_interval) - time.sleep(conf.polling_interval) + for i in range(0, conf.polling_interval): + time.sleep(1) + if self.stop: + break def fail_lost_builds(self, session): # This function is supposed to be handling only @@ -245,9 +254,6 @@ class Poller(threading.Thread): elif conf.system == "mock": pass - else: - raise NotImplementedError("Buildsystem %r is not supported." % conf.system) - def log_summary(self, session): log.info("Current status:") backlog = self.outgoing_work_queue.qsize() @@ -312,7 +318,7 @@ def main(initial_msgs = [], return_after_build = False): try: # This ingest thread puts work on the queue - messaging_thread = MessageIngest(_work_queue) + messaging_thread = MessageIngest(_work_queue, return_after_build) # This poller does other work, but also sometimes puts work in queue. polling_thread = Poller(_work_queue) # This worker takes work off the queue and handles it. @@ -322,6 +328,9 @@ def main(initial_msgs = [], return_after_build = False): polling_thread.start() worker_thread.start() + worker_thread.join() + polling_thread.stop = True + except KeyboardInterrupt: # FIXME: Make this less brutal os._exit(0) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py new file mode 100644 index 0000000..449cb3a --- /dev/null +++ b/tests/test_build/test_build.py @@ -0,0 +1,205 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import unittest +import munch +import mock +import koji +import xmlrpclib +from os import path, mkdir +from shutil import copyfile + +from module_build_service import db + +import module_build_service.messaging +import module_build_service.scheduler.handlers.repos +from module_build_service import models, conf +from module_build_service.utils import submit_module_build +from module_build_service.messaging import RidaModule + +from mock import patch + +from tests import app, init_data +from tests import conf as test_conf +import json + +from module_build_service.builder import KojiModuleBuilder, GenericBuilder +import module_build_service.scheduler.main + + +class MockedSCM(object): + def __init__(self, mocked_scm, name, mmd_filename): + self.mocked_scm = mocked_scm + self.name = name + self.mmd_filename = mmd_filename + + self.mocked_scm.return_value.checkout = self.checkout + self.mocked_scm.return_value.name = self.name + self.mocked_scm.return_value.get_latest = self.get_latest + + def checkout(self, temp_dir): + scm_dir = path.join(temp_dir, self.name) + mkdir(scm_dir) + base_dir = path.abspath(path.dirname(__file__)) + copyfile(path.join(base_dir, self.mmd_filename), + path.join(scm_dir, self.mmd_filename)) + + return scm_dir + + def get_latest(self, branch = 'master'): + return branch + +class TestModuleBuilder(GenericBuilder): + """ + Test module builder which succeeds for every build. + """ + + backend = "mock" + # Global build_id/task_id we increment when new build is executed. + _build_id = 1 + + def __init__(self, owner, module, config, tag_name): + self.module_str = module + self.tag_name = tag_name + self.config = config + + def buildroot_connect(self, groups): + pass + + def buildroot_prep(self): + pass + + def buildroot_resume(self): + pass + + def buildroot_ready(self, artifacts=None): + return True + + def buildroot_add_dependency(self, dependencies): + pass + + def buildroot_add_artifacts(self, artifacts, install=False): + pass + + def buildroot_add_repos(self, dependencies): + pass + + def _send_repo_done(self): + msg = module_build_service.messaging.KojiRepoChange( + msg_id='a faked internal message', + repo_tag=self.tag_name + "-build", + ) + module_build_service.scheduler.main.outgoing_work_queue_put(msg) + + def _send_build_change(self, state, source, build_id): + # build_id=1 and task_id=1 are OK here, because we are building just + # one RPM at the time. + msg = module_build_service.messaging.KojiBuildChange( + msg_id='a faked internal message', + build_id=build_id, + task_id=build_id, + build_name="name", + build_new_state=state, + build_release="1", + build_version="1" + ) + module_build_service.scheduler.main.outgoing_work_queue_put(msg) + + def build(self, artifact_name, source): + print "Starting building artifact %s: %s" % (artifact_name, source) + + TestModuleBuilder._build_id += 1 + + self._send_repo_done() + self._send_build_change(koji.BUILD_STATES['COMPLETE'], source, + TestModuleBuilder._build_id) + self._send_repo_done() + + state = koji.BUILD_STATES['BUILDING'] + reason = "Submitted %s to Koji" % (artifact_name) + return TestModuleBuilder._build_id, state, reason, None + + @staticmethod + def get_disttag_srpm(disttag): + # @FIXME + return KojiModuleBuilder.get_disttag_srpm(disttag) + +def set_dburi(dburi): + """ + Sets database URI in all places in the middle of test. + """ + conf.set_item("sqlalchemy_database_uri", dburi) + test_conf.set_item("sqlalchemy_database_uri", dburi) + app.config["SQLALCHEMY_DATABASE_URI"] = dburi + +class TestBuild(unittest.TestCase): + + def setUp(self): + GenericBuilder.register_backend_class(TestModuleBuilder) + self.client = app.test_client() + conf.set_item("system", "mock") + + # We need to use real database on fileystem for these tests, because + # there might be multiple threads and processes accessing it + # and in-memory database wouldn't work. + self.orig_dburi = app.config["SQLALCHEMY_DATABASE_URI"] + dbdir = path.abspath(path.dirname(__file__)) + dburi = 'sqlite:///{0}'.format(path.join( + dbdir, '.test_module_build_service.db')) + set_dburi(dburi) + + init_data() + models.ModuleBuild.query.delete() + models.ComponentBuild.query.delete() + + def tearDown(self): + # Set back the original database URI + set_dburi(self.orig_dburi) + conf.set_item("system", "koji") + + @patch('module_build_service.auth.get_username', return_value='Homer J. Simpson') + @patch('module_build_service.auth.assert_is_packager') + @patch('module_build_service.scm.SCM') + def test_submit_build(self, mocked_scm, mocked_assert_is_packager, + mocked_get_username): + """ + Tests the build of testmodule.yaml using TestModuleBuilder which + succeeds everytime. + """ + mocked_scm_obj = MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + + data = json.loads(rv.data) + module_build_id = data['id'] + + msgs = [] + msgs.append(RidaModule("fake msg", 1, 1)) + module_build_service.scheduler.main.main(msgs, True) + + # All components should be built and module itself should be in "done" + # or "ready" state. + for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): + self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) diff --git a/tests/test_build/testmodule.yaml b/tests/test_build/testmodule.yaml new file mode 100644 index 0000000..6118b6d --- /dev/null +++ b/tests/test_build/testmodule.yaml @@ -0,0 +1,39 @@ +document: modulemd +version: 1 +data: + summary: A test module in all its beauty + description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. + name: testmodule + stream: teststream + version: 1 + license: + module: [ MIT ] + dependencies: + buildrequires: + base-runtime: master + requires: + base-runtime: master + references: + community: https://fedoraproject.org/wiki/Modularity + documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules + tracker: https://taiga.fedorainfracloud.org/project/modularity + profiles: + default: + rpms: + - tangerine + api: + rpms: + - perl-Tangerine + - tangerine + components: + rpms: + perl-List-Compare: + rationale: A dependency of tangerine. + ref: f25 + perl-Tangerine: + rationale: Provides API for this module and is a dependency of tangerine. + ref: f25 + tangerine: + rationale: Provides API for this module. + buildorder: 10 + ref: f25