From 2ffa2f226d0179796f733adcbb79aab693c45a39 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: May 24 2018 18:38:49 +0000 Subject: [PATCH 1/3] Factor out a rpm_name_only utility functio We had '_name_only' cut-and-pasted to two files. Rename to util.rpm_name_only() and streamline the tiny implementation further. --- diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index f90c17c..c47f746 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -4,10 +4,7 @@ import re from gi.repository import Modulemd from . import _depchase, _repodata - -def _name_only(rpm_name): - name, version, release = rpm_name.rsplit("-", 2) - return name +from .util import rpm_name_only def _unwrap_description(desc): def replace(m): @@ -39,7 +36,7 @@ class ModuleGenerator(object): def _calculate_dependencies(self): pkgs = self.pkgs pool = self._pool - self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} + self.api_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} run_deps = _depchase.ensure_installable(pool, pkgs) module_run_deps, rpm_run_deps = _categorise_deps(run_deps) # The platform module provides any other runtime dependencies - we expect we'll diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 83bda27..94ec26e 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -3,10 +3,7 @@ import sys import logging import dnf from . import _depchase, _repodata - -def _name_only(rpm_name): - name, version, release = rpm_name.rsplit("-", 2) - return name +from .util import rpm_name_only class ModuleRepoquery(object): @@ -29,14 +26,14 @@ class ModuleRepoquery(object): if full_nevra: print(name) else: - print(_name_only(name)) + print(rpm_name_only(name)) def list_pkg_deps(self, pkgs, module_deps, json_output=False): pkgs_in_modules = set() if module_deps: for module in module_deps: rpm_names = _repodata.get_rpms_in_module(module) - pkgs_in_modules |= set(map(lambda x: _name_only(x), rpm_names)) + pkgs_in_modules |= set(map(lambda x: rpm_name_only(x), rpm_names)) pool = _depchase.make_pool("x86_64") if json_output: @@ -73,7 +70,7 @@ class ModuleRepoquery(object): pool = _depchase.make_pool("x86_64") srpm = _depchase.get_srpm_for_rpm(pool, pkg) if srpm: - print(_name_only(srpm)) + print(rpm_name_only(srpm)) def get_rpms_for_srpm(self, pkg): pool = _depchase.make_pool("x86_64") diff --git a/src/_fedmod/util.py b/src/_fedmod/util.py new file mode 100644 index 0000000..83ec263 --- /dev/null +++ b/src/_fedmod/util.py @@ -0,0 +1,3 @@ +def rpm_name_only(rpm_name): + return rpm_name.rsplit("-", 2)[0] + From 4e9dae1e9f7284ff9830aac708b9eada8fa367e5 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: May 24 2018 18:38:49 +0000 Subject: [PATCH 2/3] Add rpm2flatpak Add 'fedmod rpm2flatpak' that generates an appopriate module definition file and 'container.yaml' for packaging an existing RPM in Fedora as a Flatpak. --- diff --git a/README.md b/README.md index ed56a90..3fc92da 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Currently, this consists of: * `fedmod rpm2module`: generates a draft modulemd file based on the given RPM name (multiple RPM names can be given, but the resulting draft module will lack any descriptive metadata in that case) +* `fedmod rpm2flatpak`: generates a draft modulemd file and container.yaml + the given RPM name * `fedmod fetch-metadata`: download the F28 package and module metadata needed to generate draft module definitions (the metadata sets to use are not yet configurable) diff --git a/fedmod.spec b/fedmod.spec index b87c929..7c583d5 100644 --- a/fedmod.spec +++ b/fedmod.spec @@ -17,6 +17,7 @@ Requires: libmodulemd >= 1.2.0 Requires: python3-attrs Requires: python3-click Requires: python3-gobject-base +Requires: python3-koji Requires: python3-lxml Requires: python3-modulemd Requires: python3-PyYAML diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index f8e1bae..6b03ed4 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -2,6 +2,7 @@ import sys import click import logging +from .flatpak_generator import FlatpakGenerator from .module_generator import ModuleGenerator from .module_repoquery import ModuleRepoquery @@ -57,6 +58,17 @@ def rpm2module(pkgs, output): mg = ModuleGenerator(pkgs) mg.run(output) +# modulemd generation for Flatpaks +@_cli_commands.command() +@click.option("--output-modulemd", metavar="FILE", help="Write modulemd to FILE instead of .yaml.") +@click.option("--output-containerspec", metavar="FILE", help="Write container specification to FILE instead of container.yaml") +@click.option("--force", "-f", is_flag=True, help="Overwriting existing output files") +@click.argument("pkg", metavar='PKG', required=True) +def rpm2flatpak(pkg, output_modulemd, output_containerspec, force): + """Generat modulemd from an RPM""" + fg = FlatpakGenerator(pkg) + fg.run(output_modulemd, output_containerspec, force=force) + # Checking availability of dependencies through module streams @_cli_commands.command('resolve-deps') @click.option("--module-dependency", "-m", multiple=True, metavar="MODULE", diff --git a/src/_fedmod/flatpak_generator.py b/src/_fedmod/flatpak_generator.py new file mode 100644 index 0000000..45baa52 --- /dev/null +++ b/src/_fedmod/flatpak_generator.py @@ -0,0 +1,105 @@ +import click +import koji +import sys +import logging +import os +from textwrap import dedent + +import gi +from gi.repository import Modulemd + +from .get_module_builds import get_module_builds +from .module_generator import ModuleGenerator +from .util import rpm_name_only +from . import _depchase + + +FLATPAK_RUNTIME_STREAM = 'f28' + + +def _get_runtime_packages(): + builds = get_module_builds('flatpak-runtime', FLATPAK_RUNTIME_STREAM) + # Each flatpak-runtime stream should be built against a single context + assert len(builds) == 1 + build = builds[0] + + mmd_str = build['extra']['typeinfo']['module']['modulemd_str'] + mmd = Modulemd.Module.new_from_string(mmd_str) + + return set(mmd.props.profiles['runtime'].props.rpms.get()) + + +class FlatpakGenerator(ModuleGenerator): + def __init__(self, pkg): + super().__init__([pkg]) + + def _calculate_dependencies(self): + pkg = self.pkgs[0] + pool = self._pool + + self.api_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, pkg))} + + runtime_packages = _get_runtime_packages() + all_needed_packages = _depchase.ensure_installable(pool, [pkg], hints=runtime_packages) + + pkgs = all_needed_packages - runtime_packages + + run_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} + self.run_srpms = run_srpms - self.api_srpms + + self.module_run_deps = {'flatpak-runtime': [FLATPAK_RUNTIME_STREAM]} + + def _update_module_md(self): + super()._update_module_md() + + default_profile = Modulemd.Profile(name="default") + for pkg in self.pkgs: + default_profile.add_rpm(pkg) + + self.mmd.add_profile(default_profile) + + def _write_container_yaml(self, output_fname): + pkg = self.pkgs[0] + command = pkg + + container_yaml = dedent(f'''\ + compose: + modules: + - {pkg}:stable + flatpak: + # Derived from the project's domain name + id: org.example.MyApp + branch: stable + # Binary to execute to run the app + command: {command} + tags: [] + # Not sandboxed. See 'man flatpak-build-finish' + finish-args: > + --filesystem=host + --share=ipc + --socket=x11 + --socket=wayland + --socket=session-bus + ''') + + with open(output_fname, 'w') as f: + f.write(container_yaml) + + print('Generated container specification: %r. Please edit appropriately.' % output_fname) + + def run(self, output_modulemd, output_containerspec, force=False): + if output_modulemd is None: + pkg = self.pkgs[0] + output_modulemd = pkg + '.yaml' + if output_containerspec is None: + output_containerspec = 'container.yaml' + + if not force: + if os.path.exists(output_modulemd): + raise click.ClickException(f"{output_modulemd} exists. Pass --force to overwrite") + if os.path.exists(output_containerspec): + raise click.ClickException(f"{output_modulemd} exists. Pass --force to overwrite") + + super().run(output_modulemd) + + self._write_container_yaml(output_containerspec) diff --git a/src/_fedmod/get_module_builds.py b/src/_fedmod/get_module_builds.py new file mode 100644 index 0000000..ddb65c9 --- /dev/null +++ b/src/_fedmod/get_module_builds.py @@ -0,0 +1,109 @@ +import koji +import re + +FEDORA_TAG_PATTERNS = [(re.compile(p), s) for p, s in [ + (r'^(f\d+)-modular$', 'stable'), + (r'^(f\d+)-modular-pending$', 'pending'), + (r'^(f\d+)-modular-signing-pending$', 'signing-pending'), + (r'^(f\d+)-modular-updates$', 'stable'), + (r'^(f\d+)-modular-updates-candidate$', 'candidate'), + (r'^(f\d+)-modular-updates-pending$', 'pending'), + (r'^(f\d+)-modular-updates-testing$', 'testing'), +]] + +STATUSES = sorted({s for p, s in FEDORA_TAG_PATTERNS}) + +def _add_status_and_base_version(session, build): + # Find out where a build is tagged to find its base Fedora version and status + tags = session.listTags(build=build['build_id']) + for t in tags: + for p, s in FEDORA_TAG_PATTERNS: + m = p.match(t['name']) + if m: + build['fedmod_status'] = s + build['fedmod_base_version'] = m.group(1) + break + +def get_module_builds(module_name, stream, + version=None, + base_version=None, + status=None, + koji_config=None, + koji_profile='koji'): + + """Return a list of Koji build objects for the specified, or latest + version of a module. All the returned builds will have the same version, + but multiple builds with different contexts may be returned due to + stream expansion. + + module_name -- the name of the module + stream -- the stream of the module + version -- the version of the module. If not specified, the latest + version will be used. + base_version -- the base Fedora version that the module was built for + (corresponds to the stream of the 'platform' module). If None + builds for all base versions will be returned + status -- the status of the module in Fedora - can be + 'stable', 'testing', 'candidate', 'pending', or 'signing-ending'. If None, + builds with all statuses will be returned + koji_config -- alternate koji config file to read + koji_profile -- alternate koji profile to use + """ + + options = koji.read_config(profile_name=koji_profile, user_config=koji_config) + session_opts = koji.grab_session_options(options) + session = koji.ClientSession(options['server'], session_opts) + + package_id = session.getPackageID(module_name) + + # List all builds of the module + builds = session.listBuilds(packageID=package_id, type='module', + state=koji.BUILD_STATES['COMPLETE']) + # For convenience, add keys that turn the NVR into module terms + for b in builds: + b['fedmod_stream'] = b['version'] + if '.' in b['release']: + b['fedmod_version'], b['fedmod_context'] = b['release'].split('.', 1) + else: + b['fedmod_version'], b['fedmod_context'] = b['release'], None + + # Will fill in later + b['fedmod_status'] = None + b['fedmod_base_version'] = None + + def filter_build(b): + if b['fedmod_stream'] != stream: + return False + if version is not None and b['fedmod_version'] != version: + return False + return True + + matching_builds = [b for b in builds if filter_build(b)] + + if base_version is not None or status is not None: + for b in matching_builds: + _add_status_and_base_version(session, b) + + def filter_build_2(b): + if base_version is not None and b['fedmod_base_version'] != base_version: + return False + if status is not None and b['fedmod_status'] != status: + return False + return True + + matching_builds = [b for b in matching_builds if filter_build_2(b)] + + if version is None and len(matching_builds) > 0: + # OK, we've limited the builds to the ones that match the search criteria, find + # the most recent one, based on the module version. + latest_version = max(b['fedmod_version'] for b in matching_builds) + result = [b for b in matching_builds if b['fedmod_version'] == latest_version] + else: + result = matching_builds + + # Look up tag information if we didn't do so earlier to filter + if not (base_version is not None or status is not None): + for b in result: + _add_status_and_base_version(session, b) + + return result diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index c47f746..cf7aa54 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -43,7 +43,9 @@ class ModuleGenerator(object): # always depend on it, but check for the heck of it. if len(rpm_run_deps) > 0: module_run_deps.add('platform') - self.module_run_deps = module_run_deps + # We default to 'all available streams' for all dependencies we've found. The module + # author may then edit this to exclude streams or restrict to specific streams. + self.module_run_deps = { m: [] for m in module_run_deps } self.run_srpms = set() def _save_module_md(self, output_fname): @@ -94,9 +96,9 @@ class ModuleGenerator(object): # Declare module level dependencies dependencies = Modulemd.Dependencies() - for modname in self.module_run_deps: - dependencies.add_buildrequires(modname, []) - dependencies.add_requires(modname, []) + for modname, streams in self.module_run_deps.items(): + dependencies.add_buildrequires(modname, streams) + dependencies.add_requires(modname, streams) self.mmd.add_dependencies(dependencies) for pkg in self.run_srpms: diff --git a/src/_fedmod/util.py b/src/_fedmod/util.py index 83ec263..45d7d5b 100644 --- a/src/_fedmod/util.py +++ b/src/_fedmod/util.py @@ -1,3 +1,2 @@ def rpm_name_only(rpm_name): return rpm_name.rsplit("-", 2)[0] - diff --git a/tests/test_flatpak_generator.py b/tests/test_flatpak_generator.py new file mode 100644 index 0000000..caeefed --- /dev/null +++ b/tests/test_flatpak_generator.py @@ -0,0 +1,83 @@ +"""In-process tests for the flatpak generator""" + +import logging +import os +import pytest +import shutil +import tempfile +import yaml + +from click.testing import CliRunner + +from gi.repository import Modulemd + +from _fedmod.cli import _cli_commands +from _fedmod.flatpak_generator import FlatpakGenerator + +log = logging.getLogger(__name__) + +def _generate_flatpak(rpm): + cmd = ['rpm2flatpak'] + cmd.append(rpm) + + workdir = tempfile.mkdtemp() + try: + os.chdir(workdir) + runner = CliRunner() + result = runner.invoke(_cli_commands, cmd) + assert result.exit_code == 0 + + modulemd_fname = rpm + '.yaml' + with open(rpm + '.yaml') as f: + contents = f.read() + + log.info('%s:\n%s\n', modulemd_fname, contents) + + objs = Modulemd.objects_from_string(contents) + assert len(objs) == 1 + assert isinstance(objs[0], Modulemd.Module) + modmd = objs[0] + + with open('container.yaml') as f: + contents = f.read() + + log.info('container.yaml:\n%s\n',contents) + container_yaml = yaml.safe_load(contents) + finally: + shutil.rmtree(workdir) + + return modmd, container_yaml + + +class TestFlatpak(object): + + def test_generated_flatpak_files(self): + modmd, container_yaml = _generate_flatpak('eog') + + # Expected description for 'grep' + assert modmd.props.summary == "Eye of GNOME image viewer" + assert modmd.props.description.startswith("The Eye of GNOME image viewer (eog) is") + + # Expected licenses for 'grep' + assert modmd.props.module_licenses.get() == ['MIT'] + assert modmd.props.content_licenses.get() == [] + + # Only given modules are listed in the public API + assert sorted(modmd.props.rpm_api.get()) == ['eog'] + + # Expected components + assert set(modmd.props.components_rpm) == set([ + 'bubblewrap', 'eog', 'exempi', 'gnome-desktop3', 'libpeas' + ]) + + # Expected module dependencies for grep + dependencies = modmd.props.dependencies + assert len(dependencies) == 1 + + buildrequires = dependencies[0].props.buildrequires + assert set(buildrequires) == {'flatpak-runtime',} + assert buildrequires['flatpak-runtime'].get() == ['f28'] + + requires = dependencies[0].props.requires + assert set(requires) == {'flatpak-runtime',} + assert requires['flatpak-runtime'].get() == ['f28'] From d6cb6aab0faa3267aecb63a2eb3a50a03ca898f4 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: May 24 2018 18:38:49 +0000 Subject: [PATCH 3/3] Add flatpak-report command 'fedmod flatpak-report' generates a JSON output that describes how the packages listed on the command line correlate with the Flatpak runtime - what packages would be in the 'rpm2flatpak' generated modules for each package. --- diff --git a/README.md b/README.md index 3fc92da..24f8eb2 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ Currently, this consists of: draft module will lack any descriptive metadata in that case) * `fedmod rpm2flatpak`: generates a draft modulemd file and container.yaml the given RPM name +* `fedmod flatpak-report`': generates a JSON report about the packages in the + Flatpak runtime and not in the Flatpak runtime that would be required for + turning the specified list of rpms into Flatpaks. This is used for maintaining + the Fedora Flatpak runtime, but is likely not very useful otherwise. * `fedmod fetch-metadata`: download the F28 package and module metadata needed to generate draft module definitions (the metadata sets to use are not yet configurable) diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index 6b03ed4..7572f51 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -2,7 +2,7 @@ import sys import click import logging -from .flatpak_generator import FlatpakGenerator +from .flatpak_generator import FlatpakGenerator, do_flatpak_report from .module_generator import ModuleGenerator from .module_repoquery import ModuleRepoquery @@ -69,6 +69,13 @@ def rpm2flatpak(pkg, output_modulemd, output_containerspec, force): fg = FlatpakGenerator(pkg) fg.run(output_modulemd, output_containerspec, force=force) +@_cli_commands.command('flatpak-report') +@click.option("--quiet", "-q", is_flag=True, + help="Don't print progress messages to stderr.") +@click.argument("pkgs", metavar='PKGS', nargs=-1, required=True) +def flatpak_report(pkgs, quiet): + do_flatpak_report(pkgs, quiet=quiet) + # Checking availability of dependencies through module streams @_cli_commands.command('resolve-deps') @click.option("--module-dependency", "-m", multiple=True, metavar="MODULE", diff --git a/src/_fedmod/flatpak_generator.py b/src/_fedmod/flatpak_generator.py index 45baa52..521be44 100644 --- a/src/_fedmod/flatpak_generator.py +++ b/src/_fedmod/flatpak_generator.py @@ -1,4 +1,5 @@ import click +import json import koji import sys import logging @@ -103,3 +104,48 @@ class FlatpakGenerator(ModuleGenerator): super().run(output_modulemd) self._write_container_yaml(output_containerspec) + + +def do_flatpak_report(pkgs, quiet=False): + if not quiet: + print("Initializing", file=sys.stderr) + runtime_packages = _get_runtime_packages() + pool = _depchase.make_pool("x86_64") + + packages = {} + flatpaks = {} + + for p in runtime_packages: + packages[p] = { + 'name': p, + 'runtime': True, + 'used_by': [] + } + + for pkg in pkgs: + if not quiet: + print("Calculating deps for", pkg, file=sys.stderr) + all_needed_packages = _depchase.ensure_installable(pool, [pkg], hints=sorted(runtime_packages)) + extra_packages = all_needed_packages - runtime_packages - set([pkg]) + used_runtime_packages = all_needed_packages - extra_packages - set([pkg]) + flatpaks[pkg] = { + 'runtime': sorted(used_runtime_packages), + 'extra': sorted(extra_packages) + } + for p in all_needed_packages: + if p == pkg: + continue + data = packages.get(p, None) + if data is None: + data = { + 'name': p, + 'runtime': False, + 'used_by': [] + } + packages[p] = data + data['used_by'].append(pkg) + + json.dump({ + 'packages': packages, + 'flatpaks': flatpaks + }, sys.stdout, indent=4) diff --git a/tests/test_flatpak_report.py b/tests/test_flatpak_report.py new file mode 100644 index 0000000..5e8a69b --- /dev/null +++ b/tests/test_flatpak_report.py @@ -0,0 +1,50 @@ +"""In-process tests for the flatpak-report subcommand""" + +import json +import logging +import pytest + +from click.testing import CliRunner + +from gi.repository import Modulemd + +from _fedmod.cli import _cli_commands +from _fedmod.flatpak_generator import FlatpakGenerator + +log = logging.getLogger(__name__) + +def _generate_flatpak_report(rpms): + cmd = ['flatpak-report', '--quiet'] + cmd.extend(rpms) + runner = CliRunner() + result = runner.invoke(_cli_commands, cmd) + assert result.exit_code == 0 + + return json.loads(result.output) + + +def test_flatpak_report(): + output = _generate_flatpak_report(['eog', 'gedit']) + print(output.keys()) + + packages = output['packages'] + + # A package required by both specified rpms, in the runtime + assert packages['gtk3'] == { + 'name': 'gtk3', + 'runtime': True, + 'used_by': ['eog', 'gedit'] + } + + # An extra package required only by eog + assert packages['exempi'] == { + 'name': 'exempi', + 'runtime': False, + 'used_by': ['eog'] + } + + flatpaks = output['flatpaks'] + assert sorted(flatpaks.keys()) == ['eog', 'gedit'] + + assert 'gtk3' in flatpaks['eog']['runtime'] + assert 'exempi' in flatpaks['eog']['extra']