PR#1145: Make init handler idempotent. - fm-orchestrator

fm-orchestrator

#1145 Make init handler idempotent.

Merged 5 years ago by mprahl. Opened 5 years ago by jkaluza.

jkaluza/fm-orchestrator init-idempotent into master

Jan Kaluza • 5 years ago

7052ea0

docker/Dockerfile-tests

file modified

		`@@ -43,6 +43,7 @@`
		`python-mock \`
		`python-tox \`
		`rpm-build \`
		`+ python2-pyyaml \`
		`&& yum clean all`
		`# We currently require newer versions of these Python packages for the tests.`
		`# more-itertools is required by pytest, but versions 6.0.0 and up aren't Python 2 compatible`

docker/Dockerfile-tests-py3

file modified

		`@@ -33,6 +33,7 @@`
		`python3-mock \`
		`python3-tox \`
		`rpm-build \`
		`+ python3-PyYAML \`
		`&& dnf clean all`
		`VOLUME /src`
		`WORKDIR /src`

module_build_service/utils/submit.py

file modified

+26 -1

		`@@ -159,6 +159,11 @@`
		`xmd['mbs']['rpms'] = {}`
		`# Add missing data in RPM components`
		`for pkgname, pkg in mmd.get_rpm_components().items():`
		`+ # In case of resubmit of existing module which have been`
		`+ # cancelled/failed during the init state, the package`
		`+ # was maybe already handled by MBS, so skip it in this case.`
		`+ if pkgname in xmd['mbs']['rpms']:`
		`+ continue`
		`if pkg.get_repository() and not conf.rpms_allow_repository:`
		`raise Forbidden(`
		`"Custom component repositories aren't allowed. "`
		`@@ -193,7 +198,11 @@`
		`# by real SCM hash and store the result to our private xmd place in modulemd.`
		`pool = ThreadPool(20)`
		`try:`
		`- pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values())`
		`+ # Filter out the packages which we have already resolved in possible`
		`+ # previous runs of this method (can be caused by module build resubmition).`
		`+ pkgs_to_resolve = [pkg for pkg in mmd.get_rpm_components().values()`
		`+ if pkg.get_name() not in xmd['mbs']['rpms']]`
		`+ pkg_dicts = pool.map(_scm_get_latest, pkgs_to_resolve)`
		`finally:`
		`pool.close()`

		`@@ -355,6 +364,22 @@`

		`component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref']`
		`full_url = component.get_repository() + "?#" + component_ref`
		`+`
		`+ # Skip the ComponentBuild if it already exists in database. This can happen`
		`+ # in case of module build resubmition.`
		`+ existing_build = models.ComponentBuild.from_component_name(`
		`+ db.session, component.get_name(), module.id)`
		`+ if existing_build:`
		`+ # Check that the existing build has the same most important attributes.`
		`+ # This should never be a problem, but it's good to be defensive here so`
		`+ # we do not mess things during resubmition.`
		`+ if (existing_build.batch != batch or existing_build.scmurl != full_url or`
		`+ existing_build.ref != component_ref):`
		`+ raise ValidationError(`
		`+ "Module build %s already exists in database, but its attributes "`
		`+ " are different from resubmitted one." % component.get_name())`
		`+ continue`
		`+`
		`build = models.ComponentBuild(`
		`module_id=module.id,`
		`package=component.get_name(),`

test-requirements.txt

file modified

		`@@ -4,3 +4,4 @@`
		`pytest`
		`flake8`
		`tox`
		`+ pyyaml`

tests/test_scheduler/test_module_init.py

file modified

+18

		`@@ -20,6 +20,7 @@`
		`#`

		`import os`
		`+ import yaml`

		`from mock import patch, PropertyMock`
		`from gi.repository import GLib`
		`@@ -93,6 +94,23 @@`
		`assert type(xmd_mbs) is GLib.Variant`
		`assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [`
		`'foo-0:2.4.48-3.el8+1308+551bfa71', 'bar-0:2.5.48-3.el8+1308+551bfa71']`
		`+ return build`
		`+`
		`+ def test_init_called_twice(self):`
		`+ build = self.test_init_basic()`
		`+ old_component_builds = len(build.component_builds)`
		`+ old_mmd = yaml.load(build.modulemd)`
		`+`
		`+ build.state = 4`
		`+ db.session.commit()`
		`+ build = self.test_init_basic()`
		`+ db.session.refresh(build)`
		`+`
		`+ assert build.state == 1`
		`+ assert old_component_builds == len(build.component_builds)`
		`+`
		`+ new_mmd = yaml.load(build.modulemd)`
		`+ assert old_mmd == new_mmd`

		`@patch('module_build_service.scm.SCM')`
		`def test_init_scm_not_available(self, mocked_scm):`