#1709 Fixed scratch build suffix bug
Merged 2 years ago by mikem. Opened 2 years ago by mcurlej.
mcurlej/fm-orchestrator scratch_suffix  into  master

@@ -707,11 +707,13 @@ 

              module.build_context, module.runtime_context, module.context, \

                  module.build_context_no_bms = module.contexts_from_mmd(module.modulemd)

  

-             xmd = mmd.get_xmd()

-             if xmd["mbs"].get("static_context"):

+             if static_context:

+                 # if the static_context is True we use the context from defined in the mmd

+                 # and discard the computed one.

                  module.context = mmd.get_context()

- 

-             module.context += context_suffix

+             else:

+                 # if the context is defined by MSE, we need to add a context_suffix if it exists.

+                 module.context += context_suffix

  

              if not conf.allow_dashes_in_svc:

                  if '-' in module.stream:
@@ -786,10 +788,15 @@ 

          return streams, static_context

      else:

          xmd = stream_or_packager.get_xmd()

-         # check if we are handling rebuild of a static context module

+ 

+         # check if the static format is defined through `static_context` field

+         if stream_or_packager.is_static_context():

+             static_context = True

+             return [stream_or_packager], static_context

+ 

+         # check if we are handling rebuild of a static context module defined in xmd

          if "mbs" in xmd:

-             # check if it is a static context

-             if "static_context" in xmd["mbs"] or stream_or_packager.is_static_context():

+             if "static_context" in xmd["mbs"]:

                  static_context = True

                  return [stream_or_packager], static_context

  

@@ -25,10 +25,15 @@ 

      make_module,

      read_staged_data,

      init_data,

+     clean_database,

  )

  

  

  class TestSubmit:

+ 

+     def teardown_method(self, tested_method):

+         clean_database()

+ 

      def test_get_prefixed_version_f28(self):

          scheduler_init_data(1)

          build_one = models.ModuleBuild.get_by_id(db_session, 2)
@@ -144,6 +149,77 @@ 

              assert "mbs_options" not in xmd

              assert xmd["mbs"]["static_context"]

  

+     def test_submit_build_module_scratch_v3_static_context(self):

+         """

+         Test if the static context in the v3 metadata format will contain the correct suffix

+         during a scratch build

+         """

+         init_data(multiple_stream_versions=True)

+         yaml_str = read_staged_data("v3/mmd_packager")

+         mmd = load_mmd(yaml_str)

+         ux_timestamp = "1613048427"

+         version = provide_module_stream_version_from_timestamp(ux_timestamp)

+         params = {"scratch": True}

+ 

+         builds = submit_module_build(db_session, "foo", mmd, params, version)

+ 

+         assert len(builds) == 2

+ 

+         expected_contexts = {"CTX1_1": {}, "CTX2_1": {}}

+ 

+         for build in builds:

+             mmd = build.mmd()

+             context = mmd.get_context()

+             assert context in expected_contexts

+ 

+     def test_submit_build_module_scratch_v2_static_context(self):

+         """

+         Test if the static context in the v2 metadata format will contain

+         the correct suffix during a scratch build

+         """

+         scheduler_init_data(1)

+         yaml_str = read_staged_data("static_context_v2")

+         mmd = load_mmd(yaml_str)

+         ux_timestamp = "1613048427"

+         version = provide_module_stream_version_from_timestamp(ux_timestamp)

+         params = {"scratch": True}

+ 

+         builds = submit_module_build(db_session, "app", mmd, params, version)

+ 

+         assert len(builds) == 2

+ 

+         expected_contexts = {"context1_1": {}, "context2_1": {}}

+ 

+         for build in builds:

+             mmd = build.mmd()

+             context = mmd.get_context()

+             assert context in expected_contexts

+ 

+     def test_submit_build_module_scratch_increment(self):

+         """

+         Test if the context suffix is incremented correctly during a repeated scratch build.

+         """

+         init_data(multiple_stream_versions=True)

+         yaml_str = read_staged_data("v3/mmd_packager")

+         mmd = load_mmd(yaml_str)

+         ux_timestamp = "1613048427"

+         version = provide_module_stream_version_from_timestamp(ux_timestamp)

+         params = {"scratch": True}

+ 

+         builds = submit_module_build(db_session, "foo", mmd, params, version)

+ 

+         assert len(builds) == 2

+ 

+         builds = submit_module_build(db_session, "foo", mmd, params, version)

+ 

+         assert len(builds) == 2

+ 

+         expected_contexts = {"CTX1_2": {}, "CTX2_2": {}}

+         for build in builds:

+             mmd = build.mmd()

+             context = mmd.get_context()

+             assert context in expected_contexts

+ 

  

  class TestProcessModuleContextConfiguration:

      """

When building a scratch build of a module with static context
the scratch suffix was added twice.

@mikem @fivaldi @breilly @ppisar wdyt?

Signed-off-by: Martin Curlej mcurlej@redhat.com

rebased onto 13d0092

2 years ago

Build 13d0092 FAILED!
Rebase or make new commits to rebuild.

Build 13d0092 FAILED!
Rebase or make new commits to rebuild.

So checked the CI log not really sure why it failed.

pretty please pagure-ci rebuild

2 years ago

Our CI is still having some issues, but run-unittests.sh works.
With this change, I'm seeing some test failures:

FAILED tests/test_web/test_submit.py::TestSubmit::test_submit_build_module_scratch_v3_static_context - gi.repository.GLib.GError: modulemd-error-quark: Could not validate st...
FAILED tests/test_web/test_submit.py::TestSubmit::test_submit_build_module_scratch_increment - gi.repository.GLib.GError: modulemd-error-quark: Could not validate stream to ...

Both of the errors are coming from modulemd:

mmd = <Modulemd.ModuleStreamV2 object at 0x7f5ddbf80d80 (ModulemdModuleStreamV2 at 0x55b56911ecf0)>

    def mmd_to_str(mmd):
        """
        Helper method to convert a Modulemd.ModuleStream object to a YAML string.

        :param Modulemd.ModuleStream mmd: the modulemd to convert
        :return: the YAML string of the modulemd
        :rtype: str
        """
        index = Modulemd.ModuleIndex()
        index.add_module_stream(mmd)
>       return to_text_type(index.dump_to_string())
E       gi.repository.GLib.GError: modulemd-error-quark: Could not validate stream to emit: Non-alphanumeric character in stream context 'CTX1_1' (1)

module_build_service/common/utils.py:222: Error

Do we need to reevaluate the context suffix format?

@mikem i had this problem when my fedora image was out of date. try running ./run-unittests.sh --py3 so it will pull the newest image from the repository.

@breilly @mikem Hi, so are there any more reviews to be done on this can be merged?

i had this problem when my fedora image was out of date

Yes, updating the image does solve the issue for the py3 tests. However, the py2 tests (on centos:7) still fail. This is of particular concern since we still have a system running on py2.

Looks like we need a rebuild of libmodulemd2 for epel7

Yeah, @sgallagh needs to upgrade libmodulemd2 in epel7. I have no idea that such package exists in epel7 and I have no permissions for that package.

Unit tests pass with the update

Commit d1f2b85 fixes this pull-request

Pull-Request has been merged by mikem

2 years ago