#1039 WIP - Virtual streams support in MMDResolver.
Closed 6 months ago by jkaluza. Opened 6 months ago by jkaluza.
jkaluza/fm-orchestrator virtual-provides  into  master

@@ -27,7 +27,8 @@ 

  import collections

  import itertools

  import solv

- from module_build_service import log

+ import module_build_service.utils

+ from module_build_service import log, conf

  

  

  class MMDResolverPolicy(enum.Enum):

@@ -45,8 +46,11 @@ 

          self.pool.setarch("x86_64")

          self.build_repo = self.pool.add_repo("build")

          self.available_repo = self.pool.add_repo("available")

+         # Solvable objects representing modules stored in a list grouped by

+         # the name:stream. The list is ordered in self.solve() method.

+         self.ordered_solvables = {}

  

-     def _deps2reqs(self, deps):

+     def _deps2reqs(self, deps, base_module_stream_overrides=None):

          """

          Helper method converting dependencies from MMD to solv.Dep instance expressing

          the dependencies in a way libsolv accepts as input.

@@ -56,6 +60,21 @@ 

          The resulting solv.Dep expression will be:

              ((module(gtk) with module(gtk:1)) and (module(foo) with module(foo:1)))

  

+         Base modules are handled in a special way in case when the stream of base module

+         contains version in the "x.y.z" format. For example "el8.0.0" or "el7.6.0".

+         In this case, the resulting solv.Dep expression for such base module will contain verison

+         string computed using get_base_module_version_prefix() method:

+         For example:

+             module(platform) with module(platform:el8) = 80200

+ 

+         The stream used to compute the version can be also override using the

+         `base_module_stream_overrides` dict which has base module name as a key and

+         the stream which will be used to compute the version as a value.

+         This is needed for cases when module requires just "platform:el8", but was

+         in fact built against particular platform stream, for example platform:el8.1.0.

+         In this case, such module should still require platform:el8, but in particular

+         version which is passed to this method using the `base_module_stream_overrides`.

+ 

          The "with" syntax is here to allow depending on "module(gtk)" meaning "any gtk".

          This can happen in case {'gtk': []} is used as an input.

  

@@ -66,6 +85,9 @@ 

              ``Modulemd.Dependencies.get_requires`` or

              ``Modulemd.Dependencies.get_buildrequires`` whose value is

              converted from ``Modulemd.SimpleSet`` to list.

+         :param dict base_module_stream_overrides: The key is base module name, value

+             is the stream string which will be used to compute `version` part of the

+             base module solv.Dep expression.

          :rtype: solv.Dep

          :return: solv.Dep instance with dependencies in form libsolv accepts.

          """

@@ -78,6 +100,8 @@ 

          # or "Requires:".

          # This method creates such solve.Dep.

          stream_dep = lambda n, s: pool.Dep("module(%s:%s)" % (n, s))

+         versioned_stream_dep = lambda n, s, v: pool.Dep("module(%s:%s)" % (n, s)).Rel(

+             solv.REL_EQ, pool.Dep(str(v)))

  

          # There are relations between modules in `deps`. For example:

          #   deps = [{'gtk': ['1'], 'foo': ['1']}]" means "gtk:1 and foo:1" are both required.

@@ -95,6 +119,8 @@ 

              # Contains the solv.Dep requirements for current dict.

              require = None

              for name, streams in dep_dicts.items():

+                 is_base_module = name in conf.base_module_names

+ 

                  # The req_pos will store solv.Dep expression for "positive" requirements.

                  # That is the case of 'gtk': ['1', '2'].

                  # The req_neg will store negative requirements like 'gtk': ['-1', '-2'].

@@ -103,10 +129,40 @@ 

                  # For each stream in `streams` for this dependency, generate the

                  # module(name:stream) solv.Dep and add REL_OR relations between them.

                  for stream in streams:

-                     if stream.startswith("-"):

-                         req_neg = rel_or_dep(req_neg, solv.REL_OR, stream_dep(name, stream[1:]))

+                     if is_base_module:

+                         # Override the stream which is used to compute the stream version in case

+                         # `base_module_stream_overrides` is set.

+                         if base_module_stream_overrides and name in base_module_stream_overrides:

+                             stream_for_version = base_module_stream_overrides[name]

+                         else:

+                             stream_for_version = stream

+ 

+                         if len(stream_for_version.split(".")) < 3:

+                             # In case x.y.z versioning is not used for this base module, do not

+                             # use versions solv.Dep.

+                             if stream.startswith("-"):

+                                 req_neg = rel_or_dep(

+                                     req_neg, solv.REL_OR, stream_dep(name, stream[1:]))

+                             else:

+                                 req_pos = rel_or_dep(

+                                     req_pos, solv.REL_OR, stream_dep(name, stream))

+                         else:

+                             version = module_build_service.utils.get_base_module_version_prefix(

+                                 stream_for_version)

+                             if stream.startswith("-"):

+                                 req_neg = rel_or_dep(

+                                     req_neg, solv.REL_OR,

+                                     versioned_stream_dep(name, stream[1:], version))

+                             else:

+                                 req_pos = rel_or_dep(

+                                     req_pos, solv.REL_OR,

+                                     versioned_stream_dep(name, stream, version))

                      else:

-                         req_pos = rel_or_dep(req_pos, solv.REL_OR, stream_dep(name, stream))

+                         if stream.startswith("-"):

+                             req_neg = rel_or_dep(

+                                 req_neg, solv.REL_OR, stream_dep(name, stream[1:]))

+                         else:

+                             req_pos = rel_or_dep(req_pos, solv.REL_OR, stream_dep(name, stream))

  

                  # Generate the module(name) solv.Dep.

                  req = pool.Dep("module(%s)" % name)

@@ -126,6 +182,55 @@ 

  

          return reqs

  

+     def _add_virtual_provides(self, solvable, mmd):

+         """

+         Adds "virtual_streams" from XMD section of `mmd` to `solvable`.

+ 

+         Base modules like "platform" can contain virtual streams which needs to be considered

+         when resolving dependencies. For example module "platform:el8.1.0" can provide virtual

+         stream "el8". In this case the solvable will have following additional Provides:

+ 

+         - module(platform:el8.1.0) <= 80100 - Modules can require specific platform stream.

+         - module(platform:el8) <= 80100 - Module can also require just platform:el8.

+         """

+         xmd = mmd.get_xmd()

+         # Return in case virtual_streams are not set for this mmd.

+         if "mbs" not in xmd.keys() or "virtual_streams" not in xmd["mbs"].keys():

+             return

+ 

+         # When depsolving, we will need to follow specific rules to choose the right base

+         # module, like sorting the base modules sharing the same virtual streams based on

+         # their "stream version" - For example stream "el8.1" is lower than stream "el8.2"

+         # and so on. We therefore need to convert the stream and version of base module to

+         # integer representation and add "module($name:$stream) <= $stream_based_version"

+         # to Provides.

+         version = module_build_service.utils.get_base_module_version_prefix(mmd.get_stream())

+         dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), mmd.get_stream())).Rel(

+             solv.REL_EQ | solv.REL_LT, self.pool.Dep(str(version)))

+         solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep)

+ 

+         # For each virtual stream, add

+         # "module($name:$stream) <= $virtual_stream_based_version" provide.

+         for stream in xmd["mbs"]["virtual_streams"]:

+             dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), stream)).Rel(

+                 solv.REL_EQ | solv.REL_LT, self.pool.Dep(str(version)))

+             solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep)

+ 

+     def _get_base_module_stream_overrides(self, mmd):

+         """

+         Checks the xmd["mbs"]["buildrequires"] and returns the dict containing

+         base module name as a key and stream of base module against which this

+         module was built. This is later used to override base module streams

+         in the _deps2reqs method.

+         """

+         overrides = {}

+         if "mbs" in mmd.get_xmd().keys():

+             for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():

+                 if req_name not in conf.base_module_names:

+                     continue

+                 overrides[req_name] = req_data["stream"]

+         return overrides

+ 

      def add_modules(self, mmd):

          """

          Adds module represented by `mmd` metadata to MMDResolver. Modules added by this

@@ -177,14 +282,24 @@ 

                                    pool.Dep("module(%s:%s)" % (n, s)).Rel(

                                        solv.REL_EQ, pool.Dep(str(v))))

  

+             self._add_virtual_provides(solvable, mmd)

+             base_module_stream_overrides = self._get_base_module_stream_overrides(mmd)

+ 

              # Fill in the "Requires" of this module, so we can track its dependencies

              # on other modules.

-             requires = self._deps2reqs(normdeps(mmd, "get_requires"))

+             requires = self._deps2reqs(normdeps(mmd, "get_requires"), base_module_stream_overrides)

+             log.debug("Adding module %s with requires: %r", solvable.name, requires)

              solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires)

  

              # Add "Conflicts: module(name)", because TODO, ask ignatenko.

              solvable.add_deparray(solv.SOLVABLE_CONFLICTS, pool.Dep("module(%s)" % n))

              solvables.append(solvable)

+ 

+             # Add solvable to ordered_solvables list. Sorting is done later in the solve method.

+             ns = ":".join([n, s])

+             if ns not in self.ordered_solvables:

+                 self.ordered_solvables[ns] = []

+             self.ordered_solvables[ns].append(solvable)

          else:

              # For input module, we might have multiple buildrequires/requires pairs in the

              # input `mmd`. For example like this:

@@ -225,9 +340,15 @@ 

                  solvable.arch = "src"

  

                  requires = self._deps2reqs([normalized_deps[c]])

+                 log.debug("Adding module %s with requires: %r", solvable.name, requires)

                  solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires)

  

                  solvables.append(solvable)

+                 # Add solvable to ordered_solvables list. Sorting is done later in the solve method.

+                 ns = ":".join([n, s])

+                 if ns not in self.ordered_solvables:

+                     self.ordered_solvables[ns] = []

+                 self.ordered_solvables[ns].append(solvable)

  

          return solvables

  

@@ -267,6 +388,10 @@ 

          # "solvable to n:s"

          s2ns = lambda s: ":".join(s.name.split(":", 2)[:2])

  

+         # Order the solvables.

+         for ns, ordered_solvables in self.ordered_solvables.items():

+             ordered_solvables.sort(key=lambda s: int(s.name.split(":")[2]), reverse=True)

+ 

          # For each solvable object generated from input module, run the solver.

          # For reasons why there might be multiple solvable objects, please read the

          # `add_modules(...)` inline comments.

@@ -359,12 +484,26 @@ 

                  # of our jobs - those are the modules we are looking for.

                  newsolvables = solver.transaction().newsolvables()

                  log.debug("Transaction:")

+ 

                  for s in newsolvables:

                      log.debug("  - %s", s)

+ 

+                 # Skip this alternative in case not all the favored Solvables are in

+                 # the solution. For example if we favored gtk:4:1:c8, but it simply

+                 # cannot be installed because of other dependencies, we know this is not

+                 # possible combination and we should not treat it as alternative.

+                 all_solvables_found = True

+                 for s in opt:

+                     if s not in newsolvables:

+                         all_solvables_found = False

+ 

                  # Append them as an alternative for this src_alternative.

                  # Remember that src_alternatives are grouped by NS or NSVC depending on

                  # MMDResolverPolicy, so there might be more of them.

-                 alternative.append(newsolvables)

+                 if all_solvables_found:

+                     alternative.append(newsolvables)

+                 else:

+                     log.debug("  - ^ Not all input solvables found in the result, skipping.")

  

          # If the MMDResolverPolicy is First, we will check all the alternatives and keep

          # just the "first" one.

@@ -372,15 +511,36 @@ 

              # Prune

              for transactions in alternatives.values():

                  for ns, trans in transactions.items():

-                     try:

-                         # The transation to keep is defined by the name:stream comparison,

-                         # so we always return the same name:stream if the input is the same.

-                         transactions[ns] = [next(t for t in trans

-                                                  if set(ns) <= set(s2ns(s) for s in t))]

-                     except StopIteration:

-                         # No transactions found for requested N:S

-                         del transactions[ns]

-                         continue

+                     # Each transaction in trans lists all the possible working

+                     # combination of solvables. Our goal here is to find out the

+                     # transaction which installs the most latest Solvables - ideally

+                     # always the latest versions of the Solvables we have, but this might

+                     # not be always possible because of dependencies.

+                     #

+                     # We achieve that by generating sorted_trans list in follwing format:

+                     #   [[transaction_id, [solvable1_index, solvable2_index, ...]], [...], ...]

+                     #

+                     # The solvableN_index is a number saying how new the solvable is. We use

+                     # `self.ordered_solvables` to get that number and it is simply index

+                     # of the solvable in the particular self.ordered_solvables[name_stream] list.

+                     # The newest solvable has therefore index 0, the pre-newest solvable index 1

+                     # and so on.

+                     #

+                     # Then we simply sort the `sorted_trans` based on the sum of solvableN_index

+                     # which gives us the transaction with the most recent versions. This is

+                     # used as a solution.

+                     sorted_trans = []

+                     for i, t in enumerate(trans):

+                         idx = []

+                         for s in t:

+                             name_stream = s2ns(s)

+                             if name_stream not in self.ordered_solvables:

+                                 continue

+                             index = self.ordered_solvables[name_stream].index(s)

+                             idx.append(index)

+                         sorted_trans.append([i, idx])

+                     sorted_trans.sort(key=lambda i: sum(i[1]))

+                     transactions[ns] = [trans[sorted_trans[0][0]]]

  

          # Convert the solvables in alternatives to nsvc and return them as set of frozensets.

          return set(frozenset(s2nsvc(s) for s in transactions[0])

@@ -212,6 +212,30 @@ 

      mmd.set_xmd(glib.dict_values(xmd))

  

  

+ def get_base_module_version_prefix(base_module_stream):

+     # The platform version (e.g. prefix1.2.0 => 010200)

+     version_prefix = ''

+     for char in base_module_stream:

+         try:

+             # See if the current character is an integer, signifying the version

+             # has started

+             int(char)

+             version_prefix += char

+         except ValueError:

+             # If version_prefix isn't set, then a digit hasn't been encountered

+             if version_prefix:

+                 # If the character is a period and the version_prefix is set, then

+                 # the loop is still processing the version part of the stream

+                 if char == '.':

+                     version_prefix += '.'

+                 # If the version_prefix is set and the character is not a period or

+                 # digit, then the remainder of the stream is a suffix like "-beta"

+                 else:

+                     break

+     version_prefix = ''.join([section.zfill(2) for section in version_prefix.split('.')])

+     return version_prefix

+ 

+ 

  def get_prefixed_version(mmd):

          """

          Return the prefixed version of the module based on the buildrequired base module stream.

@@ -222,11 +246,13 @@ 

          """

          xmd = mmd.get_xmd()

          version = mmd.get_version()

+ 

          base_module_stream = None

          for base_module in conf.base_module_names:

              # xmd is a GLib Variant and doesn't support .get() syntax

              try:

-                 base_module_stream = xmd['mbs']['buildrequires'].get(base_module, {}).get('stream')

+                 base_module_stream = xmd['mbs']['buildrequires'].get(

+                     base_module, {}).get('stream')

                  if base_module_stream:

                      # Break after finding the first base module that is buildrequired

                      break

@@ -238,28 +264,7 @@ 

                          .format(' or '.join(conf.base_module_names)))

              return version

  

-         # The platform version (e.g. prefix1.2.0 => 010200)

-         version_prefix = ''

-         for char in base_module_stream:

-             try:

-                 # See if the current character is an integer, signifying the version

-                 # has started

-                 int(char)

-                 version_prefix += char

-             except ValueError:

-                 # If version_prefix isn't set, then a digit hasn't been encountered

-                 if version_prefix:

-                     # If the character is a period and the version_prefix is set, then

-                     # the loop is still processing the version part of the stream

-                     if char == '.':

-                         version_prefix += '.'

-                     # If the version_prefix is set and the character is not a period or

-                     # digit, then the remainder of the stream is a suffix like "-beta"

-                     else:

-                         break

- 

-         # Remove the periods and pad the numbers if necessary

-         version_prefix = ''.join([section.zfill(2) for section in version_prefix.split('.')])

+         version_prefix = get_base_module_version_prefix(base_module_stream)

  

          if not version_prefix:

              log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s '

file modified
+99 -1

@@ -28,6 +28,7 @@ 

  

  from module_build_service.mmd_resolver import MMDResolver

  from module_build_service import Modulemd

+ from module_build_service import glib

  

  

  class TestMMDResolver:

@@ -39,7 +40,7 @@ 

          pass

  

      @staticmethod

-     def _make_mmd(nsvc, requires):

+     def _make_mmd(nsvc, requires, xmd_buildrequires=None, virtual_streams=None):

          name, stream, version = nsvc.split(":", 2)

          mmd = Modulemd.Module()

          mmd.set_mdversion(2)

@@ -51,6 +52,17 @@ 

          licenses.add("GPL")

          mmd.set_module_licenses(licenses)

  

+         xmd = glib.from_variant_dict(mmd.get_xmd())

+         xmd["mbs"] = {}

+         xmd["mbs"]["buildrequires"] = {}

+         if xmd_buildrequires:

+             for ns in xmd_buildrequires:

+                 n, s = ns.split(":")

+                 xmd["mbs"]["buildrequires"][n] = {"stream": s}

+         if virtual_streams:

+             xmd["mbs"]["virtual_streams"] = virtual_streams

+         mmd.set_xmd(glib.dict_values(xmd))

+ 

          if ":" in version:

              version, context = version.split(":")

              mmd.set_context(context)

@@ -162,3 +174,89 @@ 

                         for e in exp)

  

          assert expanded == expected

+ 

+     @pytest.mark.parametrize(

+         "buildrequires, expected", (

+             # BR all platform streams -> build for all platform streams.

+             ({"platform": []}, [

+                 [["platform:el8.2.0:0:c0:x86_64"],

+                  ["platform:el8.1.0:0:c0:x86_64"],

+                  ["platform:el8.0.0:0:c0:x86_64"],

+                  ["platform:el7.6.0:0:c0:x86_64"]],

+             ]),

+             # BR "el8" platform stream -> build for all el8 platform streams.

+             ({"platform": ["el8"]}, [

+                 [["platform:el8.2.0:0:c0:x86_64"],

+                  ["platform:el8.1.0:0:c0:x86_64"],

+                  ["platform:el8.0.0:0:c0:x86_64"]],

+             ]),

+             # BR "el8.1.0" platfrom stream -> build just for el8.1.0.

+             ({"platform": ["el8.1.0"]}, [

+                 [["platform:el8.1.0:0:c0:x86_64"]],

+             ]),

+             # BR platform:el8.1.0 and gtk:3, which is not built against el8.1.0,

+             # but it is built only against el8.0.0 -> cherry-pick gtk:3 from el8.0.0

+             # and build once against platform:el8.1.0.

+             ({"platform": ["el8.1.0"], "gtk": ["3"]}, [

+                 [["platform:el8.1.0:0:c0:x86_64", "gtk:3:0:c8:x86_64", ]],

+             ]),

+             # BR platform:el8.2.0 and gtk:3, this time gtk:3 build against el8.2.0 exists

+             # -> use both platform and gtk from el8.2.0 and build once.

+             ({"platform": ["el8.2.0"], "gtk": ["3"]}, [

+                 [["platform:el8.2.0:0:c0:x86_64", "gtk:3:1:c8:x86_64", ]],

+             ]),

+             # BR platform:el8.2.0 and mess:1 which is built against platform:el8.1.0 and

+             # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0

+             # -> Use platform:el8.2.0 and

+             # -> cherry-pick mess:1 from el8.1.0 and

+             # -> use gtk:3:1 from el8.2.0.

+             ({"platform": ["el8.2.0"], "mess": ["1"]}, [

+                 [["platform:el8.2.0:0:c0:x86_64", "mess:1:0:c0:x86_64", "gtk:3:1:c8:x86_64", ]],

+             ]),

+             # BR platform:el8.1.0 and mess:1 which is built against platform:el8.1.0 and

+             # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0

+             # -> Use platform:el8.1.0 and

+             # -> Used mess:1 from el8.1.0 and

+             # -> cherry-pick gtk:3:0 from el8.0.0.

+             ({"platform": ["el8.1.0"], "mess": ["1"]}, [

+                 [["platform:el8.1.0:0:c0:x86_64", "mess:1:0:c0:x86_64", "gtk:3:0:c8:x86_64", ]],

+             ]),

+             # BR platform:el8.0.0 and mess:1 which is built against platform:el8.1.0 and

+             # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0

+             # -> No valid combination, because mess:1 is only available in el8.1.0 and later.

+             ({"platform": ["el8.0.0"], "mess": ["1"]}, []),

+             # This is undefined... it might build just once against latest platform or

+             # against all the platforms... we don't know

+             # ({"platform": ["el8"], "gtk": ["3"]}, [

+             #     [["platform:el8.2.0:0:c0:x86_64", "gtk:3:1:c8:x86_64"]],

+             # ]),

+         )

+     )

+     def test_solve_virtual_streams(self, buildrequires, expected):

+         modules = (

+             # (nsvc, buildrequires, expanded_buildrequires, virtual_streams)

+             ("platform:el8.0.0:0:c0", {}, {}, ["el8"]),

+             ("platform:el8.1.0:0:c0", {}, {}, ["el8"]),

+             ("platform:el8.2.0:0:c0", {}, {}, ["el8"]),

+             ("platform:el7.6.0:0:c0", {}, {}, ["el7"]),

+             ("gtk:3:0:c8", {"platform": ["el8"]}, {"platform:el8.0.0"}, None),

+             ("gtk:3:1:c8", {"platform": ["el8"]}, {"platform:el8.2.0"}, None),

+             ("mess:1:0:c0", [{"gtk": ["3"], "platform": ["el8"]}], {"platform:el8.1.0"}, None),

+         )

+         for n, req, xmd_buildrequires, virtual_streams in modules:

+             self.mmd_resolver.add_modules(self._make_mmd(

+                 n, req, xmd_buildrequires, virtual_streams))

+ 

+         app = self._make_mmd("app:1:0", buildrequires)

+         if not expected:

+             with pytest.raises(RuntimeError):

+                 self.mmd_resolver.solve(app)

+             return

+         else:

+             expanded = self.mmd_resolver.solve(app)

+ 

+         expected = set(frozenset(["app:1:0:%d:src" % c] + e)

+                        for c, exp in enumerate(expected)

+                        for e in exp)

+ 

+         assert expanded == expected

@@ -70,8 +70,8 @@ 

  

          xmd = {

              "mbs": {

-                 "buildrequires": [],

-                 "requires": [],

+                 "buildrequires": {},

+                 "requires": {},

                  "commit": "ref_%s" % context,

                  "mse": "true",

              }

no initial comment

rebased onto 182297961e3bc7f970ab7026efc50954d75f8d00

6 months ago

rebased onto 182297961e3bc7f970ab7026efc50954d75f8d00

6 months ago

rebased onto 92a572bde58eb5b7c8cffb2b22947a5c1a96bbc9

6 months ago

rebased onto 92a572bde58eb5b7c8cffb2b22947a5c1a96bbc9

6 months ago

rebased onto 634d06907cf55499c8187fbd768e5a24eb98ea34

6 months ago

rebased onto 634d06907cf55499c8187fbd768e5a24eb98ea34

6 months ago

rebased onto 5a3e7430528a7218a06e1a2a7ea8f55de4228562

6 months ago

rebased onto fe0640e

6 months ago

Pull-Request has been closed by jkaluza

6 months ago