From 32c623769e37d1b09a7bec6b876bfaf2f43d7d24 Mon Sep 17 00:00:00 2001 From: Courtney Pacheco Date: Oct 25 2016 14:03:49 +0000 Subject: Merge #123 `Updating code to allow for componentless module builds` --- diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index 222f0cb..e48fde5 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -29,7 +29,7 @@ import modulemd from pdc_client import PDCClient from copr.client import CoprClient import six -import rida +import module_build_service @@ -155,7 +155,7 @@ def get_module_tag(session, module_info, strict=False): """ return get_module(session, module_info, strict=strict)['koji_tag'] -def get_module_repo(session, module_info, strict=False, config=rida.conf): +def get_module_repo(session, module_info, strict=False, config=module_build_service.conf): """ :param session : PDCClient instance :param module_info: list of module_info dicts diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index c058c32..602476f 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -90,6 +90,13 @@ def _finalize(config, session, msg, state): # Find all of the sibling builds of this particular build. current_batch = parent.current_batch() + # If there are no components, transition module build state to 'done' + if not current_batch: + log.info("Batch is empty. Moving module state to DONE.") + parent.transition(config, module.BUILD_STATES['done']) + session.commit() + return + # Otherwise, check to see if all failed. If so, then we cannot continue on # to a next batch. This module build is doomed. if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]): diff --git a/module_build_service/views.py b/module_build_service/views.py index 6d4cb37..ef10bcc 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -166,66 +166,64 @@ class ModuleBuildAPI(MethodView): # the availability of git URLs paralelly later. full_urls = [] - # List of (pkg_name, git_url) tuples to be used to check - # the availability of git URLs paralelly later. - full_urls = [] - - for pkgname, pkg in mmd.components.rpms.packages.items(): - try: - if pkg.get("repository") and not conf.rpms_allow_repository: - raise Unauthorized( - "Custom component repositories aren't allowed") - if pkg.get("cache") and not conf.rpms_allow_cache: - raise Unauthorized("Custom component caches aren't allowed") - if not pkg.get("repository"): - pkg["repository"] = conf.rpms_default_repository + pkgname - if not pkg.get("cache"): - pkg["cache"] = conf.rpms_default_cache + pkgname - if not pkg.get("commit"): - try: - pkg["commit"] = module_build_service.scm.SCM( - pkg["repository"]).get_latest() - except Exception as e: - raise UnprocessableEntity( - "Failed to get the latest commit: %s" % pkgname) - except Exception: - module.transition(conf, models.BUILD_STATES["failed"]) - db.session.add(module) - db.session.commit() - raise - - full_url = pkg["repository"] + "?#" + pkg["commit"] - full_urls.append((pkgname, full_url)) - - log.debug("Checking scm urls") - # Checks the availability of SCM urls. - pool = ThreadPool(10) - err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0]) - if not module_build_service.scm.SCM(data[1]).is_available() - else None, full_urls) - for err_msg in err_msgs: - if err_msg: - raise UnprocessableEntity(err_msg) - - for pkgname, pkg in mmd.components.rpms.packages.items(): - full_url = pkg["repository"] + "?#" + pkg["commit"] - - existing_build = models.ComponentBuild.query.filter_by( - module_id=module.id, package=pkgname).first() - if (existing_build - and existing_build.state != models.BUILD_STATES['done']): - existing_build.state = models.BUILD_STATES['init'] - db.session.add(existing_build) - else: - # XXX: what about components that were present in previous - # builds but are gone now (component reduction)? - build = models.ComponentBuild( - module_id=module.id, - package=pkgname, - format="rpms", - scmurl=full_url, - ) - db.session.add(build) + # If the modulemd yaml specifies components, then submit them for build + if mmd.components: + for pkgname, pkg in mmd.components.rpms.packages.items(): + try: + if pkg.get("repository") and not conf.rpms_allow_repository: + raise Unauthorized( + "Custom component repositories aren't allowed") + if pkg.get("cache") and not conf.rpms_allow_cache: + raise Unauthorized("Custom component caches aren't allowed") + if not pkg.get("repository"): + pkg["repository"] = conf.rpms_default_repository + pkgname + if not pkg.get("cache"): + pkg["cache"] = conf.rpms_default_cache + pkgname + if not pkg.get("commit"): + try: + pkg["commit"] = module_build_service.scm.SCM( + pkg["repository"]).get_latest() + except Exception as e: + raise UnprocessableEntity( + "Failed to get the latest commit: %s" % pkgname) + except Exception: + module.transition(conf, models.BUILD_STATES["failed"]) + db.session.add(module) + db.session.commit() + raise + + full_url = pkg["repository"] + "?#" + pkg["commit"] + full_urls.append((pkgname, full_url)) + + log.debug("Checking scm urls") + # Checks the availability of SCM urls. + pool = ThreadPool(10) + err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0]) + if not module_build_service.scm.SCM(data[1]).is_available() + else None, full_urls) + for err_msg in err_msgs: + if err_msg: + raise UnprocessableEntity(err_msg) + + for pkgname, pkg in mmd.components.rpms.packages.items(): + full_url = pkg["repository"] + "?#" + pkg["commit"] + + existing_build = models.ComponentBuild.query.filter_by( + module_id=module.id, package=pkgname).first() + if (existing_build + and existing_build.state != models.BUILD_STATES['done']): + existing_build.state = models.BUILD_STATES['init'] + db.session.add(existing_build) + else: + # XXX: what about components that were present in previous + # builds but are gone now (component reduction)? + build = models.ComponentBuild( + module_id=module.id, + package=pkgname, + format="rpms", + scmurl=full_url, + ) + db.session.add(build) module.modulemd = mmd.dumps() module.transition(conf, models.BUILD_STATES["wait"]) diff --git a/test-requirements.txt b/test-requirements.txt index a678696..249626b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ nose mock +copr-backend diff --git a/tests/test_views/fakemodule2.yaml b/tests/test_views/fakemodule2.yaml new file mode 100644 index 0000000..cac1e61 --- /dev/null +++ b/tests/test_views/fakemodule2.yaml @@ -0,0 +1,13 @@ +document: modulemd +version: 0 +data: + name: fakemodule2 + version: 4.3.44 + release: 5 + summary: A fake module containing the bash shell + description: > + A fake module used for testing + license: + module: + - MIT + content: [] diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index d47c3eb..4578847 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -194,6 +194,42 @@ class TestViews(unittest.TestCase): self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') + @patch('module_build_service.auth.get_username', return_value='Homer J. Simpson') + @patch('module_build_service.auth.assert_is_packager') + @patch('module_build_service.scm.SCM') + def test_submit_componentless_build(self, mocked_scm, mocked_assert_is_packager, + mocked_get_username): + def mocked_scm_checkout(temp_dir): + scm_dir = path.join(temp_dir, 'fakemodule2') + mkdir(scm_dir) + base_dir = path.abspath(path.dirname(__file__)) + copyfile(path.join(base_dir, 'fakemodule2.yaml'), + path.join(scm_dir, 'fakemodule2.yaml')) + + return scm_dir + + mocked_scm.return_value.checkout = mocked_scm_checkout + mocked_scm.return_value.name = 'fakemodule2' + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + data = json.loads(rv.data) + + self.assertEquals(data['component_builds'], []) + self.assertEquals(data['name'], 'fakemodule2') + self.assertEquals(data['scmurl'], + ('git://pkgs.stg.fedoraproject.org/modules/testmodule' + '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + self.assertEquals(data['release'], '5') + self.assertTrue(data['time_submitted'] is not None) + self.assertTrue(data['time_modified'] is not None) + self.assertEquals(data['release'], '5') + self.assertEquals(data['time_completed'], None) + self.assertEquals(data['version'], '4.3.44') + self.assertEquals(data['owner'], 'Homer J. Simpson') + self.assertEquals(data['id'], 31) + self.assertEquals(data['state_name'], 'wait') + def test_submit_build_cert_error(self): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'