#1145 Make init handler idempotent.
Merged 7 months ago by mprahl. Opened 7 months ago by jkaluza.
jkaluza/fm-orchestrator init-idempotent  into  master

file modified
+1

@@ -43,6 +43,7 @@ 

      python-mock \

      python-tox \

      rpm-build \

+     python2-pyyaml \

      && yum clean all

  # We currently require newer versions of these Python packages for the tests.

  # more-itertools is required by pytest, but versions 6.0.0 and up aren't Python 2 compatible

@@ -33,6 +33,7 @@ 

      python3-mock \

      python3-tox \

      rpm-build \

+     python3-PyYAML \

      && dnf clean all

  VOLUME /src

  WORKDIR /src

@@ -159,6 +159,11 @@ 

              xmd['mbs']['rpms'] = {}

          # Add missing data in RPM components

          for pkgname, pkg in mmd.get_rpm_components().items():

+             # In case of resubmit of existing module which have been

+             # cancelled/failed during the init state, the package

+             # was maybe already handled by MBS, so skip it in this case.

+             if pkgname in xmd['mbs']['rpms']:

+                 continue

              if pkg.get_repository() and not conf.rpms_allow_repository:

                  raise Forbidden(

                      "Custom component repositories aren't allowed.  "

@@ -193,7 +198,11 @@ 

          # by real SCM hash and store the result to our private xmd place in modulemd.

          pool = ThreadPool(20)

          try:

-             pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values())

+             # Filter out the packages which we have already resolved in possible

+             # previous runs of this method (can be caused by module build resubmition).

+             pkgs_to_resolve = [pkg for pkg in mmd.get_rpm_components().values()

+                                if pkg.get_name() not in xmd['mbs']['rpms']]

+             pkg_dicts = pool.map(_scm_get_latest, pkgs_to_resolve)

          finally:

              pool.close()

  

@@ -355,6 +364,22 @@ 

  

          component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref']

          full_url = component.get_repository() + "?#" + component_ref

+ 

+         # Skip the ComponentBuild if it already exists in database. This can happen

+         # in case of module build resubmition.

+         existing_build = models.ComponentBuild.from_component_name(

+             db.session, component.get_name(), module.id)

+         if existing_build:

+             # Check that the existing build has the same most important attributes.

+             # This should never be a problem, but it's good to be defensive here so

+             # we do not mess things during resubmition.

+             if (existing_build.batch != batch or existing_build.scmurl != full_url or

+                     existing_build.ref != component_ref):

+                 raise ValidationError(

+                     "Module build %s already exists in database, but its attributes "

+                     " are different from resubmitted one." % component.get_name())

+             continue

+ 

          build = models.ComponentBuild(

              module_id=module.id,

              package=component.get_name(),

file modified
+1

@@ -4,3 +4,4 @@ 

  pytest

  flake8

  tox

+ pyyaml

@@ -20,6 +20,7 @@ 

  #

  

  import os

+ import yaml

  

  from mock import patch, PropertyMock

  from gi.repository import GLib

@@ -93,6 +94,23 @@ 

          assert type(xmd_mbs) is GLib.Variant

          assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [

              'foo-0:2.4.48-3.el8+1308+551bfa71', 'bar-0:2.5.48-3.el8+1308+551bfa71']

+         return build

+ 

+     def test_init_called_twice(self):

+         build = self.test_init_basic()

+         old_component_builds = len(build.component_builds)

+         old_mmd = yaml.load(build.modulemd)

+ 

+         build.state = 4

+         db.session.commit()

+         build = self.test_init_basic()

+         db.session.refresh(build)

+ 

+         assert build.state == 1

+         assert old_component_builds == len(build.component_builds)

+ 

+         new_mmd = yaml.load(build.modulemd)

+         assert old_mmd == new_mmd

  

      @patch('module_build_service.scm.SCM')

      def test_init_scm_not_available(self, mocked_scm):