#78 Add rpm2flatpak and flatpak-report subcommands
Closed 5 years ago by nphilipp. Opened 5 years ago by otaylor.
modularity/ otaylor/fedmod flatpak-generator  into  master

file modified
+6
@@ -14,6 +14,12 @@ 

  * `fedmod rpm2module`: generates a draft modulemd file based on

    the given RPM name (multiple RPM names can be given, but the resulting

    draft module will lack any descriptive metadata in that case)

+ * `fedmod rpm2flatpak`: generates a draft modulemd file and container.yaml

+   the given RPM name

+ * `fedmod flatpak-report`': generates a JSON report about the packages in the

+   Flatpak runtime and not in the Flatpak runtime that would be required for

+   turning the specified list of rpms into Flatpaks. This is used for maintaining

+   the Fedora Flatpak runtime, but is likely not very useful otherwise.

  * `fedmod fetch-metadata`: download the F28 package and module metadata needed

    to generate draft module definitions (the metadata sets to use are not yet

    configurable)

file modified
+1
@@ -17,6 +17,7 @@ 

  Requires:       python3-attrs

  Requires:       python3-click

  Requires:       python3-gobject-base

+ Requires:       python3-koji

  Requires:       python3-lxml

  Requires:       python3-modulemd

  Requires:       python3-PyYAML

file modified
+19
@@ -2,6 +2,7 @@ 

  import click

  import logging

  

+ from .flatpak_generator import FlatpakGenerator, do_flatpak_report

  from .module_generator import ModuleGenerator

  from .module_repoquery import ModuleRepoquery

  
@@ -57,6 +58,24 @@ 

      mg = ModuleGenerator(pkgs)

      mg.run(output)

  

+ # modulemd generation for Flatpaks

+ @_cli_commands.command()

+ @click.option("--output-modulemd", metavar="FILE", help="Write modulemd to FILE instead of <pkg>.yaml.")

+ @click.option("--output-containerspec", metavar="FILE", help="Write container specification to FILE instead of container.yaml")

+ @click.option("--force", "-f", is_flag=True, help="Overwriting existing output files")

+ @click.argument("pkg", metavar='PKG', required=True)

+ def rpm2flatpak(pkg, output_modulemd, output_containerspec, force):

+     """Generat modulemd from an RPM"""

+     fg = FlatpakGenerator(pkg)

+     fg.run(output_modulemd, output_containerspec, force=force)

+ 

+ @_cli_commands.command('flatpak-report')

+ @click.option("--quiet", "-q", is_flag=True,

+               help="Don't print progress messages to stderr.")

+ @click.argument("pkgs", metavar='PKGS', nargs=-1, required=True)

+ def flatpak_report(pkgs, quiet):

+     do_flatpak_report(pkgs, quiet=quiet)

+ 

  # Checking availability of dependencies through module streams

  @_cli_commands.command('resolve-deps')

  @click.option("--module-dependency", "-m", multiple=True, metavar="MODULE",

@@ -0,0 +1,151 @@ 

+ import click

+ import json

+ import koji

+ import sys

+ import logging

+ import os

+ from textwrap import dedent

+ 

+ import gi

+ from gi.repository import Modulemd

+ 

+ from .get_module_builds import get_module_builds

+ from .module_generator import ModuleGenerator

+ from .util import rpm_name_only

+ from . import _depchase

+ 

+ 

+ FLATPAK_RUNTIME_STREAM = 'f28'

+ 

+ 

+ def _get_runtime_packages():

+     builds = get_module_builds('flatpak-runtime', FLATPAK_RUNTIME_STREAM)

+     # Each flatpak-runtime stream should be built against a single context

+     assert len(builds) == 1

+     build = builds[0]

+ 

+     mmd_str = build['extra']['typeinfo']['module']['modulemd_str']

+     mmd = Modulemd.Module.new_from_string(mmd_str)

+ 

+     return set(mmd.props.profiles['runtime'].props.rpms.get())

+ 

+ 

+ class FlatpakGenerator(ModuleGenerator):

+     def __init__(self, pkg):

+         super().__init__([pkg])

+ 

+     def _calculate_dependencies(self):

+         pkg = self.pkgs[0]

+         pool = self._pool

+ 

+         self.api_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, pkg))}

+ 

+         runtime_packages = _get_runtime_packages()

+         all_needed_packages = _depchase.ensure_installable(pool, [pkg], hints=runtime_packages)

+ 

+         pkgs = all_needed_packages - runtime_packages

+ 

+         run_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs}

+         self.run_srpms = run_srpms - self.api_srpms

+ 

+         self.module_run_deps = {'flatpak-runtime': [FLATPAK_RUNTIME_STREAM]}

+ 

+     def _update_module_md(self):

+         super()._update_module_md()

+ 

+         default_profile = Modulemd.Profile(name="default")

+         for pkg in self.pkgs:

+             default_profile.add_rpm(pkg)

+ 

+         self.mmd.add_profile(default_profile)

+ 

+     def _write_container_yaml(self, output_fname):

+         pkg = self.pkgs[0]

+         command = pkg

+ 

+         container_yaml = dedent(f'''\

+             compose:

+                 modules:

+                     - {pkg}:stable

+             flatpak:

+                 # Derived from the project's domain name

+                 id: org.example.MyApp

+                 branch: stable

+                 # Binary to execute to run the app

+                 command: {command}

+                 tags: []

+                 # Not sandboxed. See 'man flatpak-build-finish'

+                 finish-args: >

+                     --filesystem=host

+                     --share=ipc

+                     --socket=x11

+                     --socket=wayland

+                     --socket=session-bus

+             ''')

+ 

+         with open(output_fname, 'w') as f:

+             f.write(container_yaml)

+ 

+         print('Generated container specification: %r. Please edit appropriately.' % output_fname)

+ 

+     def run(self, output_modulemd, output_containerspec, force=False):

+         if output_modulemd is None:

+             pkg = self.pkgs[0]

+             output_modulemd = pkg + '.yaml'

+         if output_containerspec is None:

+             output_containerspec = 'container.yaml'

+ 

+         if not force:

+             if os.path.exists(output_modulemd):

+                 raise click.ClickException(f"{output_modulemd} exists. Pass --force to overwrite")

+             if os.path.exists(output_containerspec):

+                 raise click.ClickException(f"{output_modulemd} exists. Pass --force to overwrite")

+ 

+         super().run(output_modulemd)

+ 

+         self._write_container_yaml(output_containerspec)

+ 

+ 

+ def do_flatpak_report(pkgs, quiet=False):

+     if not quiet:

+         print("Initializing", file=sys.stderr)

+     runtime_packages = _get_runtime_packages()

+     pool = _depchase.make_pool("x86_64")

+ 

+     packages = {}

+     flatpaks = {}

+ 

+     for p in runtime_packages:

+         packages[p] = {

+             'name': p,

+             'runtime': True,

+             'used_by': []

+         }

+ 

+     for pkg in pkgs:

+         if not quiet:

+             print("Calculating deps for", pkg, file=sys.stderr)

+         all_needed_packages = _depchase.ensure_installable(pool, [pkg], hints=sorted(runtime_packages))

+         extra_packages = all_needed_packages - runtime_packages - set([pkg])

+         used_runtime_packages = all_needed_packages - extra_packages - set([pkg])

+         flatpaks[pkg] = {

+             'runtime': sorted(used_runtime_packages),

+             'extra': sorted(extra_packages)

+         }

+         for p in all_needed_packages:

+             if p == pkg:

+                 continue

+             data = packages.get(p, None)

+             if data is None:

+                 data = {

+                     'name': p,

+                     'runtime': False,

+                     'used_by': []

+                 }

+                 packages[p] = data

+             data['used_by'].append(pkg)

+ 

+     json.dump({

+         'packages': packages,

+         'flatpaks': flatpaks

+     }, sys.stdout, indent=4)

@@ -0,0 +1,109 @@ 

+ import koji

+ import re

+ 

+ FEDORA_TAG_PATTERNS = [(re.compile(p), s) for p, s in [

+     (r'^(f\d+)-modular$', 'stable'),

+     (r'^(f\d+)-modular-pending$', 'pending'),

+     (r'^(f\d+)-modular-signing-pending$', 'signing-pending'),

+     (r'^(f\d+)-modular-updates$', 'stable'),

+     (r'^(f\d+)-modular-updates-candidate$', 'candidate'),

+     (r'^(f\d+)-modular-updates-pending$', 'pending'),

+     (r'^(f\d+)-modular-updates-testing$', 'testing'),

+ ]]

+ 

+ STATUSES = sorted({s for p, s in FEDORA_TAG_PATTERNS})

+ 

+ def _add_status_and_base_version(session, build):

+     # Find out where a build is tagged to find its base Fedora version and status

+     tags = session.listTags(build=build['build_id'])

+     for t in tags:

+         for p, s in FEDORA_TAG_PATTERNS:

+             m = p.match(t['name'])

+             if m:

+                 build['fedmod_status'] = s

+                 build['fedmod_base_version'] = m.group(1)

+                 break

+ 

+ def get_module_builds(module_name, stream,

+                       version=None,

+                       base_version=None,

+                       status=None,

+                       koji_config=None,

+                       koji_profile='koji'):

+ 

+     """Return a list of Koji build objects for the specified, or latest

+     version of a module. All the returned builds will have the same version,

+     but multiple builds with different contexts may be returned due to

+     stream expansion.

+ 

+     module_name -- the name of the module

+     stream -- the stream of the module

+     version -- the version of the module. If not specified, the latest

+        version will be used.

+     base_version -- the base Fedora version that the module was built for

+        (corresponds to the stream of the 'platform' module). If None

+        builds for all base versions will be returned

+     status -- the status of the module in Fedora - can be

+        'stable', 'testing', 'candidate', 'pending', or 'signing-ending'. If None,

+        builds with all statuses will be returned

+     koji_config -- alternate koji config file to read

+     koji_profile -- alternate koji profile to use

+     """

+ 

+     options = koji.read_config(profile_name=koji_profile, user_config=koji_config)

+     session_opts = koji.grab_session_options(options)

+     session = koji.ClientSession(options['server'], session_opts)

+ 

+     package_id = session.getPackageID(module_name)

+ 

+     # List all builds of the module

+     builds = session.listBuilds(packageID=package_id, type='module',

+                                 state=koji.BUILD_STATES['COMPLETE'])

+     # For convenience, add keys that turn the NVR into module terms

+     for b in builds:

+         b['fedmod_stream'] = b['version']

+         if '.' in b['release']:

+             b['fedmod_version'], b['fedmod_context'] = b['release'].split('.', 1)

+         else:

+             b['fedmod_version'], b['fedmod_context'] = b['release'], None

+ 

+         # Will fill in later

+         b['fedmod_status'] = None

+         b['fedmod_base_version'] = None

+ 

+     def filter_build(b):

+         if b['fedmod_stream'] != stream:

+             return False

+         if version is not None and b['fedmod_version'] != version:

+             return False

+         return True

+ 

+     matching_builds = [b for b in builds if filter_build(b)]

+ 

+     if base_version is not None or status is not None:

+         for b in matching_builds:

+             _add_status_and_base_version(session, b)

+ 

+         def filter_build_2(b):

+             if base_version is not None and b['fedmod_base_version'] != base_version:

+                 return False

+             if status is not None and b['fedmod_status'] != status:

+                 return False

+             return True

+ 

+         matching_builds = [b for b in matching_builds if filter_build_2(b)]

+ 

+     if version is None and len(matching_builds) > 0:

+         # OK, we've limited the builds to the ones that match the search criteria, find

+         # the most recent one, based on the module version.

+         latest_version = max(b['fedmod_version'] for b in matching_builds)

+         result = [b for b in matching_builds if b['fedmod_version'] == latest_version]

+     else:

+         result = matching_builds

+ 

+     # Look up tag information if we didn't do so earlier to filter

+     if not (base_version is not None or status is not None):

+         for b in result:

+             _add_status_and_base_version(session, b)

+ 

+     return result

@@ -4,10 +4,7 @@ 

  from gi.repository import Modulemd

  

  from . import _depchase, _repodata

- 

- def _name_only(rpm_name):

-     name, version, release = rpm_name.rsplit("-", 2)

-     return name

+ from .util import rpm_name_only

  

  def _unwrap_description(desc):

      def replace(m):
@@ -39,14 +36,16 @@ 

      def _calculate_dependencies(self):

          pkgs = self.pkgs

          pool = self._pool

-         self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs}

+         self.api_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs}

          run_deps = _depchase.ensure_installable(pool, pkgs)

          module_run_deps, rpm_run_deps = _categorise_deps(run_deps)

          # The platform module provides any other runtime dependencies - we expect we'll

          # always depend on it, but check for the heck of it.

          if len(rpm_run_deps) > 0:

              module_run_deps.add('platform')

-         self.module_run_deps = module_run_deps

+         # We default to 'all available streams' for all dependencies we've found. The module

+         # author may then edit this to exclude streams or restrict to specific streams.

+         self.module_run_deps = { m: [] for m in module_run_deps }

          self.run_srpms = set()

  

      def _save_module_md(self, output_fname):
@@ -97,9 +96,9 @@ 

  

          # Declare module level dependencies

          dependencies = Modulemd.Dependencies()

-         for modname in self.module_run_deps:

-             dependencies.add_buildrequires(modname, [])

-             dependencies.add_requires(modname, [])

+         for modname, streams in self.module_run_deps.items():

+             dependencies.add_buildrequires(modname, streams)

+             dependencies.add_requires(modname, streams)

          self.mmd.add_dependencies(dependencies)

  

          for pkg in self.run_srpms:

@@ -3,10 +3,7 @@ 

  import logging

  import dnf

  from . import _depchase, _repodata

- 

- def _name_only(rpm_name):

-     name, version, release = rpm_name.rsplit("-", 2)

-     return name

+ from .util import rpm_name_only

  

  class ModuleRepoquery(object):

  
@@ -29,14 +26,14 @@ 

                  if full_nevra:

                      print(name)

                  else:

-                     print(_name_only(name))

+                     print(rpm_name_only(name))

  

      def list_pkg_deps(self, pkgs, module_deps, json_output=False):

          pkgs_in_modules = set()

          if module_deps:

              for module in module_deps:

                  rpm_names = _repodata.get_rpms_in_module(module)

-                 pkgs_in_modules |= set(map(lambda x: _name_only(x), rpm_names))

+                 pkgs_in_modules |= set(map(lambda x: rpm_name_only(x), rpm_names))

  

          pool = _depchase.make_pool("x86_64")

          if json_output:
@@ -73,7 +70,7 @@ 

          pool = _depchase.make_pool("x86_64")

          srpm = _depchase.get_srpm_for_rpm(pool, pkg)

          if srpm:

-             print(_name_only(srpm))

+             print(rpm_name_only(srpm))

  

      def get_rpms_for_srpm(self, pkg):

          pool = _depchase.make_pool("x86_64")

file added
+2
@@ -0,0 +1,2 @@ 

+ def rpm_name_only(rpm_name):

+     return rpm_name.rsplit("-", 2)[0]

@@ -0,0 +1,83 @@ 

+ """In-process tests for the flatpak generator"""

+ 

+ import logging

+ import os

+ import pytest

+ import shutil

+ import tempfile

+ import yaml

+ 

+ from click.testing import CliRunner

+ 

+ from gi.repository import Modulemd

+ 

+ from _fedmod.cli import _cli_commands

+ from _fedmod.flatpak_generator import FlatpakGenerator

+ 

+ log = logging.getLogger(__name__)

+ 

+ def _generate_flatpak(rpm):

+     cmd = ['rpm2flatpak']

+     cmd.append(rpm)

+ 

+     workdir = tempfile.mkdtemp()

+     try:

+         os.chdir(workdir)

+         runner = CliRunner()

+         result = runner.invoke(_cli_commands, cmd)

+         assert result.exit_code == 0

+ 

+         modulemd_fname = rpm + '.yaml'

+         with open(rpm + '.yaml') as f:

+             contents = f.read()

+ 

+         log.info('%s:\n%s\n', modulemd_fname, contents)

+ 

+         objs = Modulemd.objects_from_string(contents)

+         assert len(objs) == 1

+         assert isinstance(objs[0], Modulemd.Module)

+         modmd = objs[0]

+ 

+         with open('container.yaml') as f:

+             contents = f.read()

+ 

+         log.info('container.yaml:\n%s\n',contents)

+         container_yaml = yaml.safe_load(contents)

+     finally:

+         shutil.rmtree(workdir)

+ 

+     return modmd, container_yaml

+ 

+ 

+ class TestFlatpak(object):

+ 

+     def test_generated_flatpak_files(self):

+         modmd, container_yaml = _generate_flatpak('eog')

+ 

+         # Expected description for 'grep'

+         assert modmd.props.summary == "Eye of GNOME image viewer"

+         assert modmd.props.description.startswith("The Eye of GNOME image viewer (eog) is")

+ 

+         # Expected licenses for 'grep'

+         assert modmd.props.module_licenses.get() == ['MIT']

+         assert modmd.props.content_licenses.get() == []

+ 

+         # Only given modules are listed in the public API

+         assert sorted(modmd.props.rpm_api.get()) == ['eog']

+ 

+         # Expected components

+         assert set(modmd.props.components_rpm) == set([

+             'bubblewrap', 'eog', 'exempi', 'gnome-desktop3', 'libpeas'

+         ])

+ 

+         # Expected module dependencies for grep

+         dependencies = modmd.props.dependencies

+         assert len(dependencies) == 1

+ 

+         buildrequires = dependencies[0].props.buildrequires

+         assert set(buildrequires) == {'flatpak-runtime',}

+         assert buildrequires['flatpak-runtime'].get() == ['f28']

+ 

+         requires = dependencies[0].props.requires

+         assert set(requires) == {'flatpak-runtime',}

+         assert requires['flatpak-runtime'].get() == ['f28']

@@ -0,0 +1,50 @@ 

+ """In-process tests for the flatpak-report subcommand"""

+ 

+ import json

+ import logging

+ import pytest

+ 

+ from click.testing import CliRunner

+ 

+ from gi.repository import Modulemd

+ 

+ from _fedmod.cli import _cli_commands

+ from _fedmod.flatpak_generator import FlatpakGenerator

+ 

+ log = logging.getLogger(__name__)

+ 

+ def _generate_flatpak_report(rpms):

+     cmd = ['flatpak-report', '--quiet']

+     cmd.extend(rpms)

+     runner = CliRunner()

+     result = runner.invoke(_cli_commands, cmd)

+     assert result.exit_code == 0

+ 

+     return json.loads(result.output)

+ 

+ 

+ def test_flatpak_report():

+     output = _generate_flatpak_report(['eog', 'gedit'])

+     print(output.keys())

+ 

+     packages = output['packages']

+ 

+     # A package required by both specified rpms, in the runtime

+     assert packages['gtk3'] == {

+         'name': 'gtk3',

+         'runtime': True,

+         'used_by': ['eog', 'gedit']

+     }

+ 

+     # An extra package required only by eog

+     assert packages['exempi'] == {

+         'name': 'exempi',

+         'runtime': False,

+         'used_by': ['eog']

+     }

+ 

+     flatpaks = output['flatpaks']

+     assert sorted(flatpaks.keys()) == ['eog', 'gedit']

+ 

+     assert 'gtk3' in flatpaks['eog']['runtime']

+     assert 'exempi' in flatpaks['eog']['extra']

This PR adds new subcommands:

rpm2flatpak generates an appropriate modulemd file and container.yaml (for OSBS) file to package up a RPM of a graphical application with dependencies as a Flatpak.

flatpak-report is a specialized subcommand that basically is just used in the scripts for maintaining the flatpak-runtime module - the idea is that you pass in a list of RPMs and it creates a report that shows what packages from the runtime and what additional packages are needed if you ran rpm2flatpak on all the RPMs one-by-one. But it's much faster than doing that, and also, the rpm2flatpak output doesn't actually include information about what required rpms are satisfied by the runtime - something we want when maintaining the runtime to see if packages that we put in the runtime are actually required by lots of different applications.

Note that this will need to be rebased after #77 is merged (or vice-versa) to fix a test failure because the expected summary/description will change. The patch can be independently reviewed, however.

Well, it's still operational in Fedora, but yeah, removing PDC usage from
here got lost on my to-do list. I'll redo that part of the patch to use the
mbs API later and see how it goes.

(Getting flatpak-runtime added to the modular updates composes also
possible, then the flatpak-runtime modulemd could just be read from there,
but it's a bit questionable since a user would never want to enable that
module.)

On Wed, May 16, 2018, 1:20 PM Igor Gnatenko pagure@pagure.io wrote:

ignatenkobrain commented on the pull-request: Add rpm2flatpak and flatpak-report subcommands that you are following:
pdc is dead..

To reply, visit the link below or just reply to this email
https://pagure.io/modularity/fedmod/pull-request/78

rebased onto 2ed3edd4eda6390e6a18debf80c6adee80c573e8

5 years ago

Pushed a new PDC-free version that talks to Koji instead.

rebased onto 3b89033f18dcaf31516de97c5ec8e788d6515923

5 years ago

rebased onto 2ffa2f2

5 years ago

I pushed a new rebased version that also:

Any chance of getting this reviewed and merged?

Took me a while because we discussed going from "one tool with many verbs" to "one library plus multiple tools" which would make enhancements like this one easier (new tools like this one wouldn't need to go through a PR to be accepted into the "core).

I'll rebase/merge for now, we can make this its own command when that change happens.

Pull-Request has been closed by nphilipp

5 years ago