#72 Use libmodulemd instead of modulemd
Merged 5 years ago by karsten. Opened 6 years ago by otaylor.
modularity/ otaylor/fedmod libmodulemd  into  master

file modified
+1 -1
@@ -57,7 +57,7 @@ 

  Some dependencies aren't currently available from PyPI, and hence need to be

  installed system-wide:

  

-     $ sudo dnf install python3-solv

+     $ sudo dnf install libmodulemd python3-gobject-base python3-solv

  

  ### Additional development dependencies

  

file modified
+2
@@ -13,8 +13,10 @@ 

  BuildRequires:  python3-devel

  BuildRequires:  python3-setuptools

  

+ Requires:       libmodulemd >= 1.2.0

  Requires:       python3-attrs

  Requires:       python3-click

+ Requires:       python3-gobject-base

  Requires:       python3-lxml

  Requires:       python3-modulemd

  Requires:       python3-PyYAML

file modified
+3
@@ -0,0 +1,3 @@ 

+ import gi

+ 

+ gi.require_version('Modulemd', '1.0')

file modified
+15 -9
@@ -6,9 +6,10 @@ 

  from collections import defaultdict

  

  import click

- import modulemd

  import requests

  

+ from gi.repository import Modulemd

+ 

  from attr import attributes, attrib

  from lxml import etree

  from requests_toolbelt.downloadutils.tee import tee_to_file
@@ -155,19 +156,24 @@ 

      repomd_xml = etree.parse(repomd_fname)

      repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules")

      repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd)

-     with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz:

+     with gzip.open(repo_modulemd_fname, "rt") as modules_yaml_gz:

          modules_yaml = modules_yaml_gz.read()

-     modules = modulemd.loads_all(modules_yaml)

      module_forward_lookup = {}

      srpm_reverse_lookup = defaultdict(list)

      rpm_reverse_lookup = defaultdict(list)

-     for module in modules:

-         module_forward_lookup[module.name] = list(set(module.artifacts.rpms))

-         for srpmname in module.components.rpms:

-             srpm_reverse_lookup[srpmname].append(module.name)

-         for rpmname in module.artifacts.rpms:

+ 

+     objects = Modulemd.objects_from_string(modules_yaml)

+     for module in (o for o in objects if isinstance(o,  Modulemd.Module)):

+         module_name = module.props.name

+         artifacts = module.props.rpm_artifacts.get()

+ 

+         module_forward_lookup[module_name] = list(set(artifacts))

+         components = module.props.components_rpm

+         for srpmname in components:

+             srpm_reverse_lookup[srpmname].append(module_name)

+         for rpmname in artifacts:

              rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0]

-             rpm_reverse_lookup[rpmprefix].append(module.name)

+             rpm_reverse_lookup[rpmprefix].append(module_name)

      # Cache the lookup tables as local JSON files

      print("Caching lookup tables")

      _write_cache("_MODULE_FORWARD_LOOKUP_CACHE", module_forward_lookup)

@@ -5,7 +5,6 @@ 

  import requests

  import click

  import logging

- import modulemd

  import solv

  import json

  from ._fetchrepodata import load_cached_repodata

file modified
+23 -12
@@ -1,6 +1,8 @@ 

- import sys

  import modulemd

  import logging

+ 

+ from gi.repository import Modulemd

+ 

  from . import _depchase, _repodata

  

  def _name_only(rpm_name):
@@ -23,7 +25,7 @@ 

      def __init__(self, pkgs):

          self.pkgs = pkgs

          self.core_srpm = None

-         self.mmd = modulemd.ModuleMetadata()

+         self.mmd = Modulemd.Module(mdversion=2)

          self._pool = _depchase.make_pool("x86_64")

  

      def _calculate_dependencies(self):
@@ -63,27 +65,36 @@ 

          are information taken from SPEC file.

          :return:

          """

-         self.mmd.add_module_license("MIT")

+         self.mmd.props.module_licenses.add("MIT")

  

          if self.core_srpm is not None:

-             self.mmd.summary = f"Generated module for {self.core_srpm}"

+             self.mmd.props.summary = f"Generated module for {self.core_srpm}"

          else:

-             self.mmd.summary = f"Generated module for {self.pkgs}"

+             self.mmd.props.summary = f"Generated module for {self.pkgs}"

  

-         self.mmd.description = "Module auto-generated by fedmod"

+         self.mmd.props.description = "Module auto-generated by fedmod"

  

          # Declare the public API

          for pkg in self.api_srpms:

-             self.mmd.api.add_rpm(pkg)

-             self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg))

+             self.mmd.props.rpm_api.add(pkg)

+ 

+             component = Modulemd.ComponentRpm(name=pkg,

+                                               rationale="Package in api",

+                                               buildorder=self._get_build_order(pkg))

+             self.mmd.add_rpm_component(component)

  

          # Declare module level dependencies

+         dependencies = Modulemd.Dependencies()

          for modname in self.module_run_deps:

-             self.mmd.buildrequires[modname] = "f28"

-             self.mmd.requires[modname] = "f28"

+             dependencies.add_buildrequires(modname, [])

+             dependencies.add_requires(modname, [])

+         self.mmd.add_dependencies(dependencies)

  

          for pkg in self.run_srpms:

-             self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg))

+             component = Modulemd.ComponentRpm(name=pkg,

+                                               rationale="Runtime dependencies",

+                                               buildorder=self._get_build_order(pkg))

+             self.mmd.add_rpm_component(component)

  

          # TODO: Always set content licenses appropriately

  
@@ -92,7 +103,7 @@ 

  

  

      def _get_build_order(self, pkg):

-         if pkg in self.mmd.api.rpms:

+         if self.mmd.props.rpm_api.contains(pkg):

              return 10

          else:

              return 0

@@ -1,6 +1,5 @@ 

  import json

  import sys

- import modulemd

  import logging

  import dnf

  from . import _depchase, _repodata

file modified
-1
@@ -24,7 +24,6 @@ 

          ]

      },

      install_requires=[

-         'modulemd',

          'click',

          'requests',

          'requests-toolbelt',

file modified
+34 -20
@@ -3,7 +3,8 @@ 

  import pytest

  from click.testing import CliRunner

  

- import modulemd

+ from gi.repository import Modulemd

+ 

  from _fedmod.cli import _cli_commands

  from _fedmod.module_generator import ModuleGenerator

  
@@ -13,8 +14,7 @@ 

      runner = CliRunner()

      result = runner.invoke(_cli_commands, cmd)

      assert result.exit_code == 0

-     modmd = modulemd.ModuleMetadata()

-     modmd.loads(result.output)

+     modmd = Modulemd.Module.new_from_string(result.output)

      return modmd

  

  
@@ -25,22 +25,30 @@ 

          modmd = _generate_modulemd(input_rpms)

  

          # Expected description for 'grep'

-         assert modmd.summary == "Generated module for grep"

-         assert modmd.description == "Module auto-generated by fedmod"

+         assert modmd.props.summary == "Generated module for grep"

+         assert modmd.props.description == "Module auto-generated by fedmod"

  

          # Expected licenses for 'grep'

-         assert modmd.module_licenses == {'MIT'}

-         assert modmd.content_licenses == set()

+         assert modmd.props.module_licenses.get() == ['MIT']

+         assert modmd.props.content_licenses.get() == []

  

          # Only given modules are listed in the public API

-         assert sorted(modmd.api.rpms) == sorted(input_rpms)

+         assert sorted(modmd.props.rpm_api.get()) == sorted(input_rpms)

  

          # Expected components for grep

-         assert set(modmd.components.rpms) == set(input_rpms)

+         assert set(modmd.props.components_rpm) == set(input_rpms)

  

          # Expected module dependencies for grep

-         assert set(modmd.buildrequires) == {'platform',}

-         assert set(modmd.requires) == {'platform',}

+         dependencies = modmd.props.dependencies

+         assert len(dependencies) == 1

+ 

+         buildrequires = dependencies[0].props.buildrequires

+         assert set(buildrequires) == {'platform',}

+         assert buildrequires['platform'].get() == []

+ 

+         requires = dependencies[0].props.requires

+         assert set(requires) == {'platform',}

+         assert requires['platform'].get() == []

  

  

  class TestMultiplePackageInput(object):
@@ -50,22 +58,28 @@ 

          modmd = _generate_modulemd(input_rpms)

  

          # Can't generate descriptive metadata when given multiple RPMs

-         assert modmd.summary == "Generated module for ('grep', 'haproxy')"

-         assert modmd.description == "Module auto-generated by fedmod"

+         assert modmd.props.summary == "Generated module for ('grep', 'haproxy')"

+         assert modmd.props.description == "Module auto-generated by fedmod"

  

          # Expected licenses for grep + haproxy

-         assert modmd.module_licenses == {'MIT'}

-         assert len(modmd.content_licenses) == 0 # This doesn't seem right...

+         assert modmd.props.module_licenses.get() == ['MIT']

+         assert modmd.props.content_licenses.get() == [] # This doesn't seem right...

  

          # Only given modules are listed in the public API

-         assert sorted(modmd.api.rpms) == sorted(input_rpms)

+         assert sorted(modmd.props.rpm_api.get()) == sorted(input_rpms)

  

          # Expected components for grep + haproxy

          expected_components = set(input_rpms)

-         assert set(modmd.components.rpms) == expected_components

+         assert set(modmd.props.components_rpm) == expected_components

  

-         # Expected module dependencies for grep + haproxy

-         assert set(modmd.buildrequires) == {'platform',}

-         assert set(modmd.requires) == {'platform',}

+         # Expected module dependencies for grep

+         dependencies = modmd.props.dependencies

+         assert len(dependencies) == 1

  

+         buildrequires = dependencies[0].props.buildrequires

+         assert set(buildrequires) == {'platform',}

+         assert buildrequires['platform'].get() == []

  

+         requires = dependencies[0].props.requires

+         assert set(requires) == {'platform',}

+         assert requires['platform'].get() == []

Use the libmodulemd library via gobject-introspection. libmodulemd
has support for version 2 of the modulemd specification, and produces
better formatted output with human-readable ordering.

why not to do this one... in __init__.py?

I deprecated this function in libmodulemd 1.2.0 because it's confusing. It returns both Module and Defaults objects, and as such should not be a method on the Modules object.

I recommend using the newer, preferred interface Modulemd.objects_from_string()

You need to check for whether you've got a module or a defaults object before this or else this line is going to throw an exception.

Maybe this loop should be a list comprehension instead that checks the type.

Do we actually want this to be explicitly f28 here? Should it instead be using the module stream expansion form?
dependencies.add_buildrequires(modname, [])

Oh, and the other question is whether it makes sense to have 'f28' be the default here, since really that only applies to the platform module?

Is this intended to only provide a starting point? Seems odd that buildorder is always only 0 or 10.

why not to do this one... in __init__.py?

My thought was "this is the generic recipe for requiring libmodulemd" and hiding part of the recipe in some other file would be confusing if someone wanted to borrow code. But I agree that it would make things a little shorter and cleaner looking to put it in a central place, and also repeating does give the mistaken impression that you could the version in only one file and have it work.

Is this intended to only provide a starting point? Seems odd that buildorder is always only 0 or 10.

That's orthogonal to this patch :-) The existing fedmod code builds all the implicit dependencies of what was specified on the rpm2flatpak command first, and then the explicitly specified packages afterwards.

I have code to do more real build-ordering (discussion in issue #48)

Is this intended to only provide a starting point? Seems odd that buildorder is always only 0 or 10.

That's orthogonal to this patch :-) The existing fedmod code builds all the implicit dependencies of what was specified on the rpm2flatpak command first, and then the explicitly specified packages afterwards.
I have code to do more real build-ordering (discussion in issue #48)

Sure, I just spotted that in the stream of work and was curious. Don't treat that as blocking for the review.

Do we actually want this to be explicitly f28 here? Should it instead be using the module stream expansion form? dependencies.add_buildrequires(modname, [])
Oh, and the other question is whether it makes sense to have 'f28' be the default here, since really that only applies to the platform module?

This is obviously a policy question, but that sounds like it corresponds to where I understand the policy is going for general modules. I'll change it that way. Flatpaks are different - they target a particular fedora-versioned Flatpak runtime, but I'll handle that in the flatpak2rpm patch.

rebased onto 96d5f20e563d6c3bb30b19d91023bb00934351f1

6 years ago

This is correct because the content licenses should not be provided by the module creator; they're added by the build process by detecting the licenses in the component RPMs.

One more comment: please update fedmod.spec and README.md to reference the change in dependency from python3-modulemd to libmodulemd and python3-gobject-introspection.

rebased onto bd61e72fde1c24dc18638cd4da81ce2754923068

6 years ago

One more comment: please update fedmod.spec and README.md to reference the change in dependency from python3-modulemd to libmodulemd and python3-gobject-introspection.

Fixed those and setup.py

not needed

Thanks for catching the leftovers. Fixed.

rebased onto cd8826b

6 years ago

This PR breaks some tests:

pytest -vv tests

================================================================================================================= FAILURES ==================================================================================================================
_____________ TestSinglePackageInput.test_generated_modulemd_file _____________

self = <tests.test_module_generator.TestSinglePackageInput object at 0x7fb4ad302cc0>

def test_generated_modulemd_file(self):
    input_rpms = ('grep',)
    modmd = _generate_modulemd(input_rpms)

    # Expected description for 'grep'
    assert modmd.props.summary == "Generated module for grep"
    assert modmd.props.description == "Module auto-generated by fedmod"

    # Expected licenses for 'grep'
  assert modmd.props.module_licenses == {'MIT'}

E AssertionError: assert <Modulemd.SimpleSet object at 0x7fb4ad26ec18 (ModulemdSimpleSet at 0x55d7d4145c20)> == {'MIT'}
E + where <Modulemd.SimpleSet object at 0x7fb4ad26ec18 (ModulemdSimpleSet at 0x55d7d4145c20)> = <gi._gi.GProps object at 0x7fb4b6c5e0b8>.module_licenses
E + where <gi._gi.GProps object at 0x7fb4b6c5e0b8> = <Modulemd.Module object at 0x7fb4ad26eee8 (ModulemdModule at 0x55d7d463aaf0)>.props

tests/test_module_generator.py:33: AssertionError
_____________ TestMultiplePackageInput.test_generated_modulemd_file _____________

self = <tests.test_module_generator.TestMultiplePackageInput object at 0x7fb4ad307be0>

def test_generated_modulemd_file(self):
    input_rpms = ('grep', 'haproxy')
    modmd = _generate_modulemd(input_rpms)

    # Can't generate descriptive metadata when given multiple RPMs
    assert modmd.props.summary == "Generated module for ('grep', 'haproxy')"
    assert modmd.props.description == "Module auto-generated by fedmod"

    # Expected licenses for grep + haproxy
  assert modmd.props.module_licenses == {'MIT'}

E AssertionError: assert <Modulemd.SimpleSet object at 0x7fb4ad230dc8 (ModulemdSimpleSet at 0x55d7d4145d20)> == {'MIT'}
E + where <Modulemd.SimpleSet object at 0x7fb4ad230dc8 (ModulemdSimpleSet at 0x55d7d4145d20)> = <gi._gi.GProps object at 0x7fb4ad302e10>.module_licenses
E + where <gi._gi.GProps object at 0x7fb4ad302e10> = <Modulemd.Module object at 0x7fb4ad303f30 (ModulemdModule at 0x55d7d463a910)>.props

tests/test_module_generator.py:66: AssertionError

Sorry for the slow response - was on PTO for a few weeks. I'm not sure where the code you are hitting the failures above comes from

  assert modmd.props.module_licenses == {'MIT'}

In this PR, that code is:

  assert modmd.props.module_licenses.get() == ['MIT']

Maybe there was some sort of merge conflict with local changes that you fixed when you were testing this?

Pull-Request has been merged by karsten

5 years ago