#1356 Refactor make_module for tests
Merged 4 years ago by jkaluza. Opened 4 years ago by cqi.
cqi/fm-orchestrator refactor-make_module  into  master

file modified
+63 -43
@@ -20,6 +20,7 @@ 

  #

  # Written by Matt Prahl <mprahl@redhat.com

  

+ import functools

  import os

  from datetime import datetime, timedelta

  from mock import patch
@@ -34,7 +35,8 @@ 

  from module_build_service.utils import get_rpm_release, import_mmd, mmd_to_str

  from module_build_service.config import init_config

  from module_build_service.models import (

-     ModuleBuild, ComponentBuild, VirtualStream, make_db_session, BUILD_STATES,

+     ModuleBuild, ModuleArch, ComponentBuild, VirtualStream,

+     make_db_session, BUILD_STATES,

  )

  from module_build_service import Modulemd

  
@@ -448,14 +450,13 @@ 

  

  

  def make_module(

-     db_session,

      nsvc,

-     requires_list=None,

-     build_requires_list=None,

+     dependencies=None,

      base_module=None,

      filtered_rpms=None,

      xmd=None,

-     store_to_db=True,

+     db_session=None,

+     store_to_db=False,

      virtual_streams=None,

      arches=None,

  ):
@@ -463,27 +464,45 @@ 

      Creates new models.ModuleBuild defined by `nsvc` string with requires

      and buildrequires set according to ``requires_list`` and ``build_requires_list``.

  

-     :param db_session: SQLAlchemy database session.

      :param str nsvc: name:stream:version:context of a module.

-     :param list_of_dicts requires_list: List of dictionaries defining the

-         requires in the mmd requires field format.

-     :param list_of_dicts build_requires_list: List of dictionaries defining the

-         build_requires_list in the mmd build_requires_list field format.

+     :param dependencies: list of groups of dependencies (requires and buildrequires).

+         For example, [

+         {"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}},

+         ...

+         ]

+     :type dependencies: list[dict]

+     :param base_module: a base module build required by the new module created.

+     :type base_module: :class:`ModuleBuild`

      :param filtered_rpms: list of filtered RPMs which are added to filter

          section in module metadata.

      :type filtered_rpms: list[str]

      :param dict xmd: a mapping representing XMD section in module metadata. A

          custom xmd could be passed for testing a particular scenario and some

          default key/value pairs are added if not present.

+     :param db_session: SQLAlchemy database session.

      :param bool store_to_db: whether to store created module metadata to the

-         database.

-     :param list virtual_streams: List of virtual streams provided by this module.

-     :param list arches: List of architectures this module is built against.

-         If set to None, ["x86_64"] is used as a default.

+         database. If set to True, ``db_session`` is required.

+     :param virtual_streams: List of virtual streams provided by this module.

+         If set, This requires ``db_session`` and ``store_to_db`` to be set to a

+         session object and True.

+     :type virtual_streams: list[str]

+     :param arches: List of architectures this module is built against. If set

+         to None, ``["x86_64"]`` is used as a default. If set, likewise

+         ``virtual_stream``.

+     :type arches: list[str]

      :return: New Module Build if set to store module metadata to database,

          otherwise the module metadata is returned.

      :rtype: ModuleBuild or Modulemd.Module

      """

+     if store_to_db:

+         assert db_session is not None

+     if base_module:

+         assert db_session is not None

+     if virtual_streams:

+         assert db_session and store_to_db

+     if arches:

+         assert db_session and store_to_db

+ 

      name, stream, version, context = nsvc.split(":")

      mmd = Modulemd.ModuleStreamV2.new(name, stream)

      mmd.set_version(int(version))
@@ -497,29 +516,30 @@ 

          for rpm in filtered_rpms:

              mmd.add_rpm_filter(rpm)

  

-     if requires_list is not None and build_requires_list is not None:

-         if not isinstance(requires_list, list):

-             requires_list = [requires_list]

-         if not isinstance(build_requires_list, list):

-             build_requires_list = [build_requires_list]

- 

-         for requires, build_requires in zip(requires_list, build_requires_list):

-             deps = Modulemd.Dependencies()

-             for req_name, req_streams in requires.items():

-                 if req_streams == []:

-                     deps.set_empty_runtime_dependencies_for_module(req_name)

-                 else:

-                     for req_stream in req_streams:

-                         deps.add_runtime_stream(req_name, req_stream)

- 

-             for req_name, req_streams in build_requires.items():

-                 if req_streams == []:

-                     deps.set_empty_buildtime_dependencies_for_module(req_name)

-                 else:

-                     for req_stream in req_streams:

-                         deps.add_buildtime_stream(req_name, req_stream)

- 

-             mmd.add_dependencies(deps)

+     def _add_require(mmd_deps, require_type, name, streams):

+         assert isinstance(mmd_deps, Modulemd.Dependencies)

+         assert require_type in ("requires", "buildrequires")

+         assert isinstance(streams, (list, tuple))

+ 

+         if require_type == "requires":

+             add_stream = mmd_deps.add_runtime_stream

+             set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module

+         else:

+             add_stream = mmd_deps.add_buildtime_stream

+             set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module

+ 

+         for stream in streams:

+             add_stream(name, stream)

+         else:

+             set_empty_deps(name)

+ 

+     for dep_group in dependencies or []:

+         mmd_deps = Modulemd.Dependencies()

+         # A deps could be {"platform": ["f30"], "python": []}

+         for require_type, deps in dep_group.items():

+             for req_name, req_streams in deps.items():

+                 _add_require(mmd_deps, require_type, req_name, req_streams)

+         mmd.add_dependencies(mmd_deps)

  

      # Caller could pass whole xmd including mbs, but if something is missing,

      # default values are given here.
@@ -577,13 +597,10 @@ 

                  module_build.virtual_streams.append(vs_obj)

                  db_session.commit()

  

-     if arches is None:

-         arches = ["x86_64"]

-     for arch in arches:

-         arch_obj = db_session.query(module_build_service.models.ModuleArch).filter_by(

-             name=arch).first()

+     for arch in arches or ["x86_64"]:

+         arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first()

          if not arch_obj:

-             arch_obj = module_build_service.models.ModuleArch(name=arch)

+             arch_obj = ModuleArch(name=arch)

              db_session.add(arch_obj)

              db_session.commit()

  
@@ -594,6 +611,9 @@ 

      return module_build

  

  

+ make_module_in_db = functools.partial(make_module, store_to_db=True)

+ 

+ 

  def module_build_from_modulemd(yaml):

      """

      Create a ModuleBuild object and return. It is not written into database.

@@ -39,7 +39,7 @@ 

  import pytest

  from mock import patch, MagicMock

  

- from tests import conf, init_data, clean_database, make_module

+ from tests import conf, init_data, clean_database, make_module_in_db

  

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  
@@ -922,10 +922,10 @@ 

      @patch("tempfile.mkdtemp")

      @patch("module_build_service.builder.KojiModuleBuilder.execute_cmd")

      def _build_srpm(self, db_session, execute_cmd, mkdtemp):

-         module_build = make_module(

-             db_session,

+         module_build = make_module_in_db(

              "{name}:{stream}:{version}:{context}".format(**self.module_nsvc),

-             xmd=self.xmd)

+             xmd=self.xmd,

+             db_session=db_session)

  

          mkdtemp.return_value = self.tmp_srpm_build_dir

          return KojiModuleBuilder.get_disttag_srpm("disttag", module_build)

@@ -11,7 +11,7 @@ 

  from module_build_service.models import ModuleBuild, ComponentBuild

  from module_build_service.builder.MockModuleBuilder import MockModuleBuilder

  from module_build_service.utils import import_fake_base_module, mmd_to_str, load_mmd

- from tests import clean_database, make_module, read_staged_data

+ from tests import clean_database, make_module_in_db, read_staged_data

  

  

  class TestMockModuleBuilder:
@@ -202,10 +202,12 @@ 

          import_fake_base_module(db_session, "platform:f29:1:000000")

  

          platform = ModuleBuild.get_last_build_in_stream(db_session, "platform", "f29")

-         foo = make_module(

-             db_session, "foo:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})

-         app = make_module(

-             db_session, "app:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})

+         module_deps = [{

+             "requires": {"platform": ["f29"]},

+             "buildrequires": {"platform": ["f29"]},

+         }]

+         foo = make_module_in_db("foo:1:1:1", module_deps, db_session=db_session)

+         app = make_module_in_db("app:1:1:1", module_deps, db_session=db_session)

  

          patched_open.side_effect = [

              mock.mock_open(read_data="[fake]\nrepofile 1\n").return_value,

@@ -26,8 +26,8 @@ 

  from module_build_service import conf

  from module_build_service.models import ComponentBuild, ComponentBuildTrace, ModuleBuild

  from module_build_service.utils.general import mmd_to_str, load_mmd

- from tests import init_data as init_data_contexts, clean_database, make_module, read_staged_data

- from tests import module_build_from_modulemd

+ from tests import init_data as init_data_contexts, clean_database, read_staged_data

+ from tests import make_module_in_db, module_build_from_modulemd

  

  

  @pytest.mark.usefixtures("model_tests_init_data")
@@ -161,20 +161,27 @@ 

          """

          clean_database(False)

  

-         make_module(

-             db_session, "platform:f29.1.0:10:old_version", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.1.0:15:c11.another", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.1.0:15:c11", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.2.0:0:old_version", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.2.0:1:c11", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.3.0:15:old_version", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.3.0:20:c11", {}, {}, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.1.0:10:old_version",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.1.0:15:c11.another",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.1.0:15:c11",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.2.0:0:old_version",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.2.0:1:c11",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.3.0:15:old_version",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.3.0:20:c11",

+             db_session=db_session, virtual_streams=["f29"])

  

          builds = ModuleBuild.get_last_builds_in_stream_version_lte(

              db_session, "platform", 290200)
@@ -191,8 +198,8 @@ 

  

      def test_get_module_count(self, db_session):

          clean_database(False)

-         make_module(db_session, "platform:f29.1.0:10:c11", {}, {})

-         make_module(db_session, "platform:f29.1.0:10:c12", {}, {})

+         make_module_in_db("platform:f29.1.0:10:c11", db_session=db_session)

+         make_module_in_db("platform:f29.1.0:10:c12", db_session=db_session)

  

          count = ModuleBuild.get_module_count(db_session, name="platform")

          db_session.commit()
@@ -201,12 +208,18 @@ 

      def test_add_virtual_streams_filter(self, db_session):

          clean_database(False)

  

-         make_module(db_session, "platform:f29.1.0:10:c1", {}, {}, virtual_streams=["f29"])

-         make_module(db_session, "platform:f29.1.0:15:c1", {}, {}, virtual_streams=["f29"])

-         make_module(

-             db_session, "platform:f29.3.0:15:old_version", {}, {},

-             virtual_streams=["f28", "f29"])

-         make_module(db_session, "platform:f29.3.0:20:c11", {}, {}, virtual_streams=["f30"])

+         make_module_in_db(

+             "platform:f29.1.0:10:c1",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.1.0:15:c1",

+             db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db(

+             "platform:f29.3.0:15:old_version",

+             db_session=db_session, virtual_streams=["f28", "f29"])

+         make_module_in_db(

+             "platform:f29.3.0:20:c11",

+             db_session=db_session, virtual_streams=["f30"])

  

          query = db_session.query(ModuleBuild).filter_by(name="platform")

          query = ModuleBuild._add_virtual_streams_filter(db_session, query, ["f28", "f29"])

file modified
+13 -3
@@ -30,7 +30,7 @@ 

  from conf.config import TestConfiguration

  

  from six.moves import reload_module

- from tests import app, init_data, make_module

+ from tests import app, init_data, make_module_in_db

  

  num_of_metrics = 18

  
@@ -71,7 +71,12 @@ 

  @mock.patch("module_build_service.monitor.builder_success_counter.inc")

  def test_monitor_state_changing_success(succ_cnt, failed_cnt, db_session):

      conf = mbs_config.Config(TestConfiguration)

-     b = make_module(db_session, "pkg:0.1:1:c1", requires_list={"platform": "el8"})

+     b = make_module_in_db(

+         "pkg:0.1:1:c1", [{

+             "requires": {"platform": ["el8"]},

+             "buildrequires": {"platform": ["el8"]},

+         }],

+         db_session=db_session)

      b.transition(db_session, conf, models.BUILD_STATES["wait"])

      b.transition(db_session, conf, models.BUILD_STATES["build"])

      b.transition(db_session, conf, models.BUILD_STATES["done"])
@@ -85,7 +90,12 @@ 

  def test_monitor_state_changing_failure(succ_cnt, failed_cnt, db_session):

      failure_type = "user"

      conf = mbs_config.Config(TestConfiguration)

-     b = make_module(db_session, "pkg:0.1:1:c1", requires_list={"platform": "el8"})

+     b = make_module_in_db(

+         "pkg:0.1:1:c1", [{

+             "requires": {"platform": ["el8"]},

+             "buildrequires": {"platform": ["el8"]},

+         }],

+         db_session=db_session)

      b.transition(db_session, conf, models.BUILD_STATES["wait"])

      b.transition(db_session, conf, models.BUILD_STATES["build"])

      b.transition(db_session, conf, models.BUILD_STATES["failed"], failure_type=failure_type)

@@ -42,9 +42,7 @@ 

  

          import_mmd(db_session, mmd)

          platform_f300103 = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()

-         mmd = tests.make_module(db_session,

-                                 "testmodule:master:20170109091357:123",

-                                 store_to_db=False)

+         mmd = tests.make_module("testmodule:master:20170109091357:123")

          build = ModuleBuild(

              name="testmodule",

              stream="master",

@@ -380,7 +380,7 @@ 

                      "version": 1,

                      "context": "c1",

                      "modulemd": mmd_to_str(

-                         tests.make_module(db_session, "nodejs:10:1:c1", store_to_db=False),

+                         tests.make_module("nodejs:10:1:c1"),

                      ),

                  },

                  {
@@ -389,7 +389,7 @@ 

                      "version": 2,

                      "context": "c1",

                      "modulemd": mmd_to_str(

-                         tests.make_module(db_session, "nodejs:10:2:c1", store_to_db=False),

+                         tests.make_module("nodejs:10:2:c1"),

                      ),

                  },

              ],

@@ -29,7 +29,7 @@ 

  from module_build_service.models import ModuleBuild

  from module_build_service.scheduler import default_modules

  from module_build_service.utils.general import load_mmd, mmd_to_str

- from tests import clean_database, make_module, read_staged_data

+ from tests import clean_database, make_module_in_db, read_staged_data

  

  

  @patch("module_build_service.scheduler.default_modules._handle_collisions")
@@ -39,8 +39,8 @@ 

      Test that default modules present in the database are added, and the others are ignored.

      """

      clean_database()

-     make_module(db_session, "python:3:12345:1")

-     make_module(db_session, "nodejs:11:2345:2")

+     make_module_in_db("python:3:12345:1", db_session=db_session)

+     make_module_in_db("nodejs:11:2345:2", db_session=db_session)

      mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

      xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]

      assert set(xmd_brs.keys()) == {"platform"}
@@ -112,8 +112,8 @@ 

      Test that an exception is raised when the request to get the default modules failed.

      """

      clean_database()

-     make_module(db_session, "python:3:12345:1")

-     make_module(db_session, "nodejs:11:2345:2")

+     make_module_in_db("python:3:12345:1", db_session=db_session)

+     make_module_in_db("nodejs:11:2345:2", db_session=db_session)

      mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

      xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]

      assert set(xmd_brs.keys()) == {"platform"}

@@ -30,7 +30,7 @@ 

  from module_build_service.scheduler.consumer import MBSConsumer

  from module_build_service.scheduler.handlers.greenwave import get_corresponding_module_build

  from module_build_service.scheduler.handlers.greenwave import decision_update

- from tests import clean_database, make_module

+ from tests import clean_database, make_module_in_db

  

  

  class TestGetCorrespondingModuleBuild:
@@ -123,7 +123,12 @@ 

          clean_database()

  

          # This build should be queried and transformed to ready state

-         module_build = make_module(db_session, "pkg:0.1:1:c1", requires_list={"platform": "el8"})

+         module_build = make_module_in_db(

+             "pkg:0.1:1:c1", [{

+                 "requires": {"platform": ["el8"]},

+                 "buildrequires": {"platform": ["el8"]},

+             }],

+             db_session=db_session)

          module_build.transition(

              db_session, conf, BUILD_STATES["done"], "Move to done directly for running test."

          )

@@ -25,7 +25,7 @@ 

  from mock import patch, Mock

  import pytest

  from module_build_service.utils.greenwave import greenwave

- from tests import clean_database, make_module

+ from tests import clean_database, make_module_in_db

  

  

  class TestGreenwaveQuery():
@@ -59,7 +59,12 @@ 

          response.status_code = resp_status

          mock_requests.post.return_value = response

  

-         fake_build = make_module(db_session, "pkg:0.1:1:c1", requires_list={"platform": "el8"})

+         fake_build = make_module_in_db(

+             "pkg:0.1:1:c1", [{

+                 "requires": {"platform": ["el8"]},

+                 "buildrequires": {"platform": ["el8"]},

+             }],

+             db_session=db_session)

          got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8")

  

          assert got_response == resp_content
@@ -176,7 +181,12 @@ 

          mock_requests.get.return_value = responses[0]

          mock_requests.post.side_effect = responses[1:]

  

-         fake_build = make_module(db_session, "pkg:0.1:1:c1", requires_list={"platform": "el8"})

+         fake_build = make_module_in_db(

+             "pkg:0.1:1:c1", [{

+                 "requires": {"platform": ["el8"]},

+                 "buildrequires": {"platform": ["el8"]},

+             }],

+             db_session=db_session)

          result = greenwave.check_gating(fake_build)

  

          assert result == policies_satisfied

file modified
+22 -24
@@ -21,7 +21,7 @@ 

  

  from module_build_service import conf

  from module_build_service.utils import ursine

- from tests import make_module, clean_database

+ from tests import make_module, make_module_in_db, clean_database

  

  

  class TestFindModuleKojiTags:
@@ -180,12 +180,14 @@ 

          # From the behavior of following code, the reason of the error is

          # mixing use of db.session and make_session, the latter one is called

          # from function ``get_modulemds_from_ursine_content``.

-         mmd_name1s2020c = make_module(

-             db_session,

-             "name1:s:2020:c", xmd={"mbs": {"koji_tag": "module-name1-s-2020-c"}})

-         mmd_name2s2021c = make_module(

-             db_session,

-             "name2:s:2021:c", xmd={"mbs": {"koji_tag": "module-name2-s-2021-c"}})

+         mmd_name1s2020c = make_module_in_db(

+             "name1:s:2020:c",

+             xmd={"mbs": {"koji_tag": "module-name1-s-2020-c"}},

+             db_session=db_session)

+         mmd_name2s2021c = make_module_in_db(

+             "name2:s:2021:c",

+             xmd={"mbs": {"koji_tag": "module-name2-s-2021-c"}},

+             db_session=db_session)

  

          koji_tag = "tag"  # It's ok to use arbitrary tag name.

          with patch.object(conf, "koji_external_repo_url_prefix", new="http://example.com/"):
@@ -210,7 +212,7 @@ 

          self, find_stream_collision_modules, db_session

      ):

          xmd = {"mbs": {"buildrequires": {"modulea": {"stream": "master"}}}}

-         fake_mmd = make_module(db_session, "name1:s:2020:c", xmd=xmd, store_to_db=False)

+         fake_mmd = make_module("name1:s:2020:c", xmd=xmd)

          original_xmd = fake_mmd.get_xmd()

  

          with patch.object(ursine, "log") as log:
@@ -233,7 +235,7 @@ 

                  }

              }

          }

-         fake_mmd = make_module(db_session, "name1:s:2020:c", xmd=xmd, store_to_db=False)

+         fake_mmd = make_module("name1:s:2020:c", xmd=xmd)

          expected_xmd = fake_mmd.get_xmd()

  

          get_modulemds_from_ursine_content.return_value = []
@@ -268,24 +270,20 @@ 

                  }

              }

          }

-         fake_mmd = make_module(db_session, "name1:s:2020:c", xmd=xmd, store_to_db=False)

+         fake_mmd = make_module("name1:s:2020:c", xmd=xmd)

  

          def mock_get_ursine_modulemds(db_session, koji_tag):

              if koji_tag == "module-rhel-8.0-build":

                  return [

                      # This is the one

-                     make_module(

-                         db_session, "modulea:10:20180813041838:5ea3b708", store_to_db=False),

-                     make_module(

-                         db_session, "moduleb:1.0:20180113042038:6ea3b105", store_to_db=False),

+                     make_module("modulea:10:20180813041838:5ea3b708"),

+                     make_module("moduleb:1.0:20180113042038:6ea3b105"),

                  ]

              if koji_tag == "module-project-1.0-build":

                  return [

                      # Both of them are the collided modules

-                     make_module(

-                         db_session, "bar:6:20181013041838:817fa3a8", store_to_db=False),

-                     make_module(

-                         db_session, "foo:2:20180113041838:95f078a1", store_to_db=False),

+                     make_module("bar:6:20181013041838:817fa3a8"),

+                     make_module("foo:2:20180113041838:95f078a1"),

                  ]

  

          get_modulemds_from_ursine_content.side_effect = mock_get_ursine_modulemds
@@ -354,9 +352,9 @@ 

      def test_no_collisions_found(self, get_modulemds_from_ursine_content, db_session):

          xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}}

          get_modulemds_from_ursine_content.return_value = [

-             make_module(db_session, "moduler:1:1:c1", store_to_db=False),

-             make_module(db_session, "modules:2:1:c2", store_to_db=False),

-             make_module(db_session, "modulet:3:1:c3", store_to_db=False),

+             make_module("moduler:1:1:c1"),

+             make_module("modules:2:1:c2"),

+             make_module("modulet:3:1:c3"),

          ]

          assert [] == ursine.find_stream_collision_modules(

              db_session, xmd_mbs_buildrequires, "koji_tag")
@@ -365,9 +363,9 @@ 

      def test_collision_modules_are_found(self, get_modulemds_from_ursine_content, db_session):

          xmd_mbs_buildrequires = {"modulea": {"stream": "master"}, "moduleb": {"stream": "10"}}

          fake_modules = [

-             make_module(db_session, "moduler:1:1:c1", store_to_db=False),

-             make_module(db_session, "moduleb:6:1:c2", store_to_db=False),

-             make_module(db_session, "modulet:3:1:c3", store_to_db=False),

+             make_module("moduler:1:1:c1"),

+             make_module("moduleb:6:1:c2"),

+             make_module("modulet:3:1:c3"),

          ]

          get_modulemds_from_ursine_content.return_value = fake_modules

  

@@ -37,7 +37,7 @@ 

      clean_database,

      init_data,

      scheduler_init_data,

-     make_module,

+     make_module_in_db,

      read_staged_data, staged_data_filename)

  import mock

  import koji
@@ -982,7 +982,7 @@ 

          build adds new MSE build (it means there are new expanded

          buildrequires).

          """

-         build = make_module(db_session, "foo:stream:0:c1", {}, {})

+         build = make_module_in_db("foo:stream:0:c1", db_session=db_session)

          assert build.state == models.BUILD_STATES["ready"]

  

          mmd1 = build.mmd()

@@ -24,7 +24,7 @@ 

  import module_build_service.utils

  from module_build_service import Modulemd, models

  from module_build_service.errors import StreamAmbigous

- from tests import clean_database, make_module, init_data, read_staged_data

+ from tests import clean_database, make_module_in_db, init_data, read_staged_data

  

  

  class TestUtilsModuleStreamExpansion:
@@ -52,34 +52,50 @@ 

          Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the

          platform:f28 and platform:f29 modules.

          """

-         platform_f28 = make_module(db_session, "platform:f28:0:c10", {}, {})

-         platform_f29 = make_module(db_session, "platform:f29:0:c11", {}, {})

-         make_module(db_session, "gtk:1:0:c2", {"platform": ["f28"]}, {}, platform_f28)

-         make_module(db_session, "gtk:1:0:c3", {"platform": ["f29"]}, {}, platform_f29)

-         make_module(db_session, "gtk:2:0:c4", {"platform": ["f28"]}, {}, platform_f28)

-         make_module(db_session, "gtk:2:0:c5", {"platform": ["f29"]}, {}, platform_f29)

-         make_module(db_session, "foo:1:0:c2", {"platform": ["f28"]}, {}, platform_f28)

-         make_module(db_session, "foo:1:0:c3", {"platform": ["f29"]}, {}, platform_f29)

-         make_module(db_session, "foo:2:0:c4", {"platform": ["f28"]}, {}, platform_f28)

-         make_module(db_session, "foo:2:0:c5", {"platform": ["f29"]}, {}, platform_f29)

-         make_module(db_session, "app:1:0:c6", {"platform": ["f29"]}, {}, platform_f29)

+         platform_f28 = make_module_in_db("platform:f28:0:c10", db_session=db_session)

+         platform_f29 = make_module_in_db("platform:f29:0:c11", db_session=db_session)

+         f28_deps = [{

+             "requires": {"platform": ["f28"]},

+             "buildrequires": {"platform": ["f28"]},

+         }]

+         f29_deps = [{

+             "requires": {"platform": ["f29"]},

+             "buildrequires": {"platform": ["f29"]},

+         }]

+         make_module_in_db("gtk:1:0:c2", f28_deps, base_module=platform_f28, db_session=db_session)

+         make_module_in_db("gtk:1:0:c3", f29_deps, base_module=platform_f29, db_session=db_session)

+         make_module_in_db("gtk:2:0:c4", f28_deps, base_module=platform_f28, db_session=db_session)

+         make_module_in_db("gtk:2:0:c5", f29_deps, base_module=platform_f29, db_session=db_session)

+         make_module_in_db("foo:1:0:c2", f28_deps, base_module=platform_f28, db_session=db_session)

+         make_module_in_db("foo:1:0:c3", f29_deps, base_module=platform_f29, db_session=db_session)

+         make_module_in_db("foo:2:0:c4", f28_deps, base_module=platform_f28, db_session=db_session)

+         make_module_in_db("foo:2:0:c5", f29_deps, base_module=platform_f29, db_session=db_session)

+         make_module_in_db("app:1:0:c6", f29_deps, base_module=platform_f29, db_session=db_session)

  

      def test_generate_expanded_mmds_context(self, db_session):

          self._generate_default_modules(db_session)

-         module_build = make_module(

-             db_session, "app:1:0:c1", {"gtk": ["1", "2"]}, {"platform": ["f28"], "gtk": ["1", "2"]}

-         )

+         module_build = make_module_in_db(

+             "app:1:0:c1", [{

+                 "requires": {"gtk": ["1", "2"]},

+                 "buildrequires": {

+                     "platform": ["f28"],

+                     "gtk": ["1", "2"]

+                 },

+             }],

+             db_session=db_session)

          mmds = module_build_service.utils.generate_expanded_mmds(

              db_session, module_build.mmd())

          contexts = set([mmd.get_context() for mmd in mmds])

          assert set(["e1e005fb", "ce132a1e"]) == contexts

  

      @pytest.mark.parametrize(

-         "requires,build_requires,stream_ambigous,expected_xmd,expected_buildrequires",

+         "module_deps,stream_ambigous,expected_xmd,expected_buildrequires",

          [

              (

-                 {"gtk": ["1", "2"]},

-                 {"platform": ["f28"], "gtk": ["1", "2"]},

+                 [{

+                     "requires": {"gtk": ["1", "2"]},

+                     "buildrequires": {"platform": ["f28"], "gtk": ["1", "2"]},

+                 }],

                  True,

                  set(

                      [
@@ -90,8 +106,10 @@ 

                  set([frozenset(["gtk:1", "platform:f28"]), frozenset(["gtk:2", "platform:f28"])]),

              ),

              (

-                 {"foo": ["1"]},

-                 {"platform": ["f28"], "foo": ["1"], "gtk": ["1", "2"]},

+                 [{

+                     "requires": {"foo": ["1"]},

+                     "buildrequires": {"platform": ["f28"], "foo": ["1"], "gtk": ["1", "2"]},

+                 }],

                  True,

                  set(

                      [
@@ -107,43 +125,55 @@ 

                  ),

              ),

              (

-                 {"gtk": ["1"], "foo": ["1"]},

-                 {"platform": ["f28"], "gtk": ["1"], "foo": ["1"]},

+                 [{

+                     "requires": {"gtk": ["1"], "foo": ["1"]},

+                     "buildrequires": {"platform": ["f28"], "gtk": ["1"], "foo": ["1"]},

+                 }],

                  False,

                  set([frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])]),

                  set([frozenset(["foo:1", "gtk:1", "platform:f28"])]),

              ),

              (

-                 {"gtk": ["1"], "foo": ["1"]},

-                 {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]},

+                 [{

+                     "requires": {"gtk": ["1"], "foo": ["1"]},

+                     "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]},

+                 }],

                  False,

                  set([frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])]),

                  set([frozenset(["foo:1", "gtk:1", "platform:f28"])]),

              ),

              (

-                 {"gtk": ["-2"], "foo": ["-2"]},

-                 {"platform": ["f28"], "gtk": ["-2"], "foo": ["-2"]},

+                 [{

+                     "requires": {"gtk": ["-2"], "foo": ["-2"]},

+                     "buildrequires": {"platform": ["f28"], "gtk": ["-2"], "foo": ["-2"]},

+                 }],

                  True,

                  set([frozenset(["foo:1:0:c2", "gtk:1:0:c2", "platform:f28:0:c10"])]),

                  set([frozenset(["foo:1", "gtk:1", "platform:f28"])]),

              ),

              (

-                 {"gtk": ["1"], "foo": ["1"]},

-                 {"platform": ["f28"], "gtk": ["1"]},

+                 [{

+                     "requires": {"gtk": ["1"], "foo": ["1"]},

+                     "buildrequires": {"platform": ["f28"], "gtk": ["1"]},

+                 }],

                  False,

                  set([frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])]),

                  set([frozenset(["gtk:1", "platform:f28"])]),

              ),

              (

-                 {"gtk": []},

-                 {"platform": ["f28"], "gtk": ["1"]},

+                 [{

+                     "requires": {"gtk": []},

+                     "buildrequires": {"platform": ["f28"], "gtk": ["1"]},

+                 }],

                  True,

                  set([frozenset(["gtk:1:0:c2", "platform:f28:0:c10"])]),

                  set([frozenset(["gtk:1", "platform:f28"])]),

              ),

              (

-                 {},

-                 {"platform": ["f29"], "app": ["1"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": ["f29"], "app": ["1"]},

+                 }],

                  False,

                  set([frozenset(["app:1:0:c6", "platform:f29:0:c11"])]),

                  set([frozenset(["app:1", "platform:f29"])]),
@@ -151,11 +181,10 @@ 

          ],

      )

      def test_generate_expanded_mmds_buildrequires(

-         self, requires, build_requires, stream_ambigous, expected_xmd, expected_buildrequires,

-         db_session

+         self, module_deps, stream_ambigous, expected_xmd, expected_buildrequires, db_session

      ):

          self._generate_default_modules(db_session)

-         module_build = make_module(db_session, "app:1:0:c1", requires, build_requires)

+         module_build = make_module_in_db("app:1:0:c1", module_deps, db_session=db_session)

  

          # Check that generate_expanded_mmds raises an exception if stream is ambigous

          # and also that it does not raise an exception otherwise.
@@ -206,38 +235,48 @@ 

          assert buildrequires_per_mmd_buildrequires == expected_buildrequires

  

      @pytest.mark.parametrize(

-         "requires,build_requires,expected",

+         "module_deps,expected",

          [

              (

-                 {"gtk": ["1", "2"]},

-                 {"platform": [], "gtk": ["1", "2"]},

+                 [{

+                     "requires": {"gtk": ["1", "2"]},

+                     "buildrequires": {"platform": [], "gtk": ["1", "2"]},

+                 }],

                  set([frozenset(["gtk:1"]), frozenset(["gtk:2"])]),

              ),

              (

-                 {"gtk": ["1", "2"]},

-                 {"platform": [], "gtk": ["1"]},

+                 [{

+                     "requires": {"gtk": ["1", "2"]},

+                     "buildrequires": {"platform": [], "gtk": ["1"]},

+                 }],

                  set([frozenset(["gtk:1", "gtk:2"])]),

              ),

              (

-                 {"gtk": ["1"], "foo": ["1"]},

-                 {"platform": [], "gtk": ["1"], "foo": ["1"]},

+                 [{

+                     "requires": {"gtk": ["1"], "foo": ["1"]},

+                     "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]},

+                 }],

                  set([frozenset(["foo:1", "gtk:1"])]),

              ),

              (

-                 {"gtk": ["-2"], "foo": ["-2"]},

-                 {"platform": [], "gtk": ["-2"], "foo": ["-2"]},

+                 [{

+                     "requires": {"gtk": ["-2"], "foo": ["-2"]},

+                     "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]},

+                 }],

                  set([frozenset(["foo:1", "gtk:1"])]),

              ),

              (

-                 {"gtk": [], "foo": []},

-                 {"platform": [], "gtk": ["1"], "foo": ["1"]},

+                 [{

+                     "requires": {"gtk": [], "foo": []},

+                     "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]},

+                 }],

                  set([frozenset([])]),

              ),

          ],

      )

-     def test_generate_expanded_mmds_requires(self, requires, build_requires, expected, db_session):

+     def test_generate_expanded_mmds_requires(self, module_deps, expected, db_session):

          self._generate_default_modules(db_session)

-         module_build = make_module(db_session, "app:1:0:c1", requires, build_requires)

+         module_build = make_module_in_db("app:1:0:c1", module_deps, db_session=db_session)

          mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd())

  

          requires_per_mmd = set()
@@ -253,11 +292,10 @@ 

          assert requires_per_mmd == expected

  

      @pytest.mark.parametrize(

-         "requires,build_requires,expected",

+         "module_deps,expected",

          [

              (

-                 {},

-                 {"platform": [], "gtk": ["1", "2"]},

+                 [{"requires": {}, "buildrequires": {"platform": [], "gtk": ["1", "2"]}}],

                  [

                      "platform:f29:0:c11",

                      "gtk:2:0:c4",
@@ -268,8 +306,10 @@ 

                  ],

              ),

              (

-                 {},

-                 {"platform": [], "gtk": ["1"], "foo": ["1"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}

+                 }],

                  [

                      "platform:f28:0:c10",

                      "gtk:1:0:c2",
@@ -280,15 +320,22 @@ 

                  ],

              ),

              (

-                 {},

-                 {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}

+                 }],

                  ["platform:f28:0:c10", "gtk:1:0:c2", "foo:1:0:c2"],

              ),

              (

-                 [{}, {}],

                  [

-                     {"platform": [], "gtk": ["1"], "foo": ["1"]},

-                     {"platform": [], "gtk": ["2"], "foo": ["2"]},

+                     {

+                         "requires": {},

+                         "buildrequires": {"platform": [], "gtk": ["1"], "foo": ["1"]}

+                     },

+                     {

+                         "requires": {},

+                         "buildrequires": {"platform": [], "gtk": ["2"], "foo": ["2"]},

+                     }

                  ],

                  [

                      "foo:1:0:c2",
@@ -304,8 +351,10 @@ 

                  ],

              ),

              (

-                 {},

-                 {"platform": [], "gtk": ["-2"], "foo": ["-2"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": [], "gtk": ["-2"], "foo": ["-2"]},

+                 }],

                  [

                      "foo:1:0:c2",

                      "foo:1:0:c3",
@@ -317,8 +366,8 @@ 

              ),

          ],

      )

-     def test_get_required_modules_simple(self, requires, build_requires, expected, db_session):

-         module_build = make_module(db_session, "app:1:0:c1", requires, build_requires)

+     def test_get_required_modules_simple(self, module_deps, expected, db_session):

+         module_build = make_module_in_db("app:1:0:c1", module_deps, db_session=db_session)

          self._generate_default_modules(db_session)

          nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session)

          assert set(nsvcs) == set(expected)
@@ -329,23 +378,52 @@ 

          and lorem:1 modules which require base:f29 module requiring

          platform:f29 module :).

          """

-         base_module = make_module(db_session, "platform:f29:0:c11", {}, {})

-         make_module(db_session, "gtk:1:0:c2", {"foo": ["unknown"]}, {}, base_module)

-         make_module(db_session, "gtk:1:1:c2", {"foo": ["1"]}, {}, base_module)

-         make_module(db_session, "foo:1:0:c2", {"bar": ["unknown"]}, {}, base_module)

-         make_module(db_session, "foo:1:1:c2", {"bar": ["1"], "lorem": ["1"]}, {}, base_module)

-         make_module(db_session, "bar:1:0:c2", {"base": ["unknown"]}, {}, base_module)

-         make_module(db_session, "bar:1:1:c2", {"base": ["f29"]}, {}, base_module)

-         make_module(db_session, "lorem:1:0:c2", {"base": ["unknown"]}, {}, base_module)

-         make_module(db_session, "lorem:1:1:c2", {"base": ["f29"]}, {}, base_module)

-         make_module(db_session, "base:f29:0:c3", {"platform": ["f29"]}, {}, base_module)

+         base_module = make_module_in_db("platform:f29:0:c11", db_session=db_session)

+         make_module_in_db(

+             "gtk:1:0:c2",

+             [{"requires": {"foo": ["unknown"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "gtk:1:1:c2",

+             [{"requires": {"foo": ["1"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "foo:1:0:c2",

+             [{"requires": {"bar": ["unknown"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "foo:1:1:c2",

+             [{"requires": {"bar": ["1"], "lorem": ["1"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "bar:1:0:c2",

+             [{"requires": {"base": ["unknown"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "bar:1:1:c2",

+             [{"requires": {"base": ["f29"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "lorem:1:0:c2",

+             [{"requires": {"base": ["unknown"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "lorem:1:1:c2",

+             [{"requires": {"base": ["f29"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

+         make_module_in_db(

+             "base:f29:0:c3",

+             [{"requires": {"platform": ["f29"]}, "buildrequires": {}}],

+             base_module=base_module, db_session=db_session)

  

      @pytest.mark.parametrize(

-         "requires,build_requires,expected",

+         "module_deps,expected",

          [

              (

-                 {},

-                 {"platform": [], "gtk": ["1"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": [], "gtk": ["1"]},

+                 }],

                  [

                      "foo:1:1:c2",

                      "base:f29:0:c3",
@@ -356,14 +434,16 @@ 

                  ],

              ),

              (

-                 {},

-                 {"platform": [], "foo": ["1"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": [], "foo": ["1"]},

+                 }],

                  ["foo:1:1:c2", "base:f29:0:c3", "platform:f29:0:c11", "bar:1:1:c2", "lorem:1:1:c2"],

              ),

          ],

      )

-     def test_get_required_modules_recursion(self, requires, build_requires, expected, db_session):

-         module_build = make_module(db_session, "app:1:0:c1", requires, build_requires)

+     def test_get_required_modules_recursion(self, module_deps, expected, db_session):

+         module_build = make_module_in_db("app:1:0:c1", module_deps, db_session=db_session)

          self._generate_default_modules_recursion(db_session)

          nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session)

          assert set(nsvcs) == set(expected)
@@ -374,28 +454,38 @@ 

          and lorem:1 modules which require base:f29 module requiring

          platform:f29 module :).

          """

-         f290000 = make_module(db_session, "platform:f29.0.0:0:c11", {}, {}, virtual_streams=["f29"])

-         f290100 = make_module(db_session, "platform:f29.1.0:0:c11", {}, {}, virtual_streams=["f29"])

-         f290200 = make_module(db_session, "platform:f29.2.0:0:c11", {}, {}, virtual_streams=["f29"])

-         make_module(db_session, "gtk:1:0:c2", {"platform": ["f29"]}, {}, f290000)

-         make_module(db_session, "gtk:1:1:c2", {"platform": ["f29"]}, {}, f290100)

-         make_module(db_session, "gtk:1:2:c2", {"platform": ["f29"]}, {}, f290100)

-         make_module(db_session, "gtk:1:3:c2", {"platform": ["f29"]}, {}, f290200)

+         f29_dep = [{

+             "requires": {"platform": ["f29"]},

+             "buildrequires": {"platform": ["f29"]}

+         }]

+ 

+         f290000 = make_module_in_db(

+             "platform:f29.0.0:0:c11", db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db("gtk:1:0:c2", f29_dep, base_module=f290000, db_session=db_session)

+ 

+         f290100 = make_module_in_db(

+             "platform:f29.1.0:0:c11", db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db("gtk:1:1:c2", f29_dep, base_module=f290100, db_session=db_session)

+         make_module_in_db("gtk:1:2:c2", f29_dep, base_module=f290100, db_session=db_session)

+ 

+         f290200 = make_module_in_db(

+             "platform:f29.2.0:0:c11", db_session=db_session, virtual_streams=["f29"])

+         make_module_in_db("gtk:1:3:c2", f29_dep, base_module=f290200, db_session=db_session)

  

      @pytest.mark.parametrize(

-         "requires,build_requires,expected",

+         "module_deps,expected",

          [

              (

-                 {},

-                 {"platform": ["f29.1.0"], "gtk": ["1"]},

+                 [{

+                     "requires": {},

+                     "buildrequires": {"platform": ["f29.1.0"], "gtk": ["1"]},

+                 }],

                  ["platform:f29.0.0:0:c11", "gtk:1:0:c2", "gtk:1:2:c2", "platform:f29.1.0:0:c11"],

              )

          ],

      )

-     def test_get_required_modules_stream_versions(

-         self, requires, build_requires, expected, db_session

-     ):

-         module_build = make_module(db_session, "app:1:0:c1", requires, build_requires)

+     def test_get_required_modules_stream_versions(self, module_deps, expected, db_session):

+         module_build = make_module_in_db("app:1:0:c1", module_deps, db_session=db_session)

          self._generate_default_modules_modules_multiple_stream_versions(db_session)

          nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session)

          assert set(nsvcs) == set(expected)
@@ -436,7 +526,9 @@ 

          mmd.remove_dependencies(deps)

          mmd.add_dependencies(new_deps)

  

-         make_module(db_session, "platform:lp29.1.1:12:c11", {}, {}, virtual_streams=virtual_streams)

+         make_module_in_db(

+             "platform:lp29.1.1:12:c11",

+             db_session=db_session, virtual_streams=virtual_streams)

  

          mmds = module_build_service.utils.mse._get_base_module_mmds(db_session, mmd)

          if virtual_streams == ["f29"]:

The original motivation for this refactor is to reuse make_module and
drop TestMMDResolver._make_mmd. Some tests require a modulemd created
and some tests also require those modulemd to be stored into database as
a module build. The problem is db_session has to be passed to
make_module even if no need to store into database.

Major changes in this patch:

  • Argument db_session is optional.
  • Arguments requires_list and build_requires_list are replaced by a
    single argument dependencies which is a list of group of requires and
    buildrequires
  • A new make_module_in_db is created for creating and storing the new
    modulemd into database conveniently.
  • Tests are updated with the new make_module and make_module_in_db.

Signed-off-by: Chenxiong Qi cqi@redhat.com

Commit 26c816f fixes this pull-request

Pull-Request has been merged by jkaluza

4 years ago

Pull-Request has been merged by jkaluza

4 years ago