From ebe361b3a25dea4cddfbd0a431938bca47f7f209 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Jan 16 2019 20:09:11 +0000 Subject: [PATCH 1/2] rpm2flatpak: add a --flatpak-common option The --flatpak-common option makes the generated modulemd depend on the flatpak-common module and use RPMs found in that module rather than starting from scratch. For now, it's off by default because flatpak-common is still experimental. --- diff --git a/_fedmod/cli.py b/_fedmod/cli.py index 906a27d..e909de8 100644 --- a/_fedmod/cli.py +++ b/_fedmod/cli.py @@ -78,6 +78,8 @@ def rpm2module(pkgs, output): @_cli_commands.command() @click.option("--flathub", metavar="ID_OR_SEARCH_TERM", help="Initialize from a Flathub Flatpak.") +@click.option("--flatpak-common", "-c", is_flag=True, + help="Use depenencies from the flatpak-common module") @click.option("--output-modulemd", metavar="FILE", help="Write modulemd to FILE instead of .yaml.") @click.option("--output-containerspec", metavar="FILE", @@ -86,10 +88,11 @@ def rpm2module(pkgs, output): @click.option("--force", "-f", is_flag=True, help="Overwriting existing output files") @click.argument("pkg", metavar='PKG', required=True) -def rpm2flatpak(pkg, output_modulemd, output_containerspec, force, flathub): +def rpm2flatpak(pkg, output_modulemd, output_containerspec, force, flathub, flatpak_common): """Generate modulemd from an RPM""" fg = FlatpakGenerator(pkg) - fg.run(output_modulemd, output_containerspec, force=force, flathub=flathub) + fg.run(output_modulemd, output_containerspec, force=force, flathub=flathub, + include_flatpak_common=flatpak_common) @_cli_commands.command('flatpak-report') diff --git a/_fedmod/flatpak_generator.py b/_fedmod/flatpak_generator.py index 5dad063..8099476 100644 --- a/_fedmod/flatpak_generator.py +++ b/_fedmod/flatpak_generator.py @@ -70,6 +70,15 @@ def _get_runtime_packages(): return set(mmd.peek_profiles()['runtime'].peek_rpms().get()) +def _get_common_packages(): + builds = get_module_builds('flatpak-common', dataset_release_branch(), include_rpms=True) + # Each flatpak-common stream should be built against a single context + assert len(builds) == 1 + build = builds[0] + + return {r['name'] for r in build['fedmod_rpms']} + + def _load_flathub_manifest(search_term): response = requests.get("https://flathub.org/api/v1/apps") response.raise_for_status() @@ -125,17 +134,22 @@ class FlatpakGenerator(ModuleGenerator): self.api_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, pkg))} - runtime_packages = _get_runtime_packages() + hint_packages = _get_runtime_packages() + if self.include_flatpak_common: + hint_packages.update(_get_common_packages()) + all_needed_packages = _depchase.ensure_installable( - pool, [pkg], hints=runtime_packages) + pool, [pkg], hints=hint_packages) - pkgs = all_needed_packages - runtime_packages + pkgs = all_needed_packages - hint_packages run_srpms = {rpm_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} self.run_srpms = run_srpms - self.api_srpms self.module_run_deps = {'flatpak-runtime': [dataset_release_branch()]} + if self.include_flatpak_common: + self.module_run_deps['flatpak-common'] = [dataset_release_branch()] def _update_module_md(self): super()._update_module_md() @@ -213,9 +227,11 @@ class FlatpakGenerator(ModuleGenerator): f" Please edit appropriately.") def run(self, output_modulemd, output_containerspec, - force=False, flathub=None): + force=False, flathub=None, include_flatpak_common=False): flathub_manifest = _load_flathub_manifest(flathub) if flathub else None + self.include_flatpak_common = include_flatpak_common + if output_modulemd is None: pkg = self.pkgs[0] output_modulemd = pkg + '.yaml' diff --git a/_fedmod/get_module_builds.py b/_fedmod/get_module_builds.py index 8f740ef..0b4d0bb 100644 --- a/_fedmod/get_module_builds.py +++ b/_fedmod/get_module_builds.py @@ -34,7 +34,8 @@ def get_module_builds(module_name, stream, base_version=None, status=None, koji_config=None, - koji_profile='koji'): + koji_profile='koji', + include_rpms=False): """Return a list of Koji build objects for the specified, or latest version of a module. All the returned builds will have the same version, @@ -53,6 +54,7 @@ def get_module_builds(module_name, stream, If None, builds with all statuses will be returned koji_config -- alternate koji config file to read koji_profile -- alternate koji profile to use + include_rpms -- if true, include a list of rpms built for each build """ options = koji.read_config(profile_name=koji_profile, @@ -116,4 +118,15 @@ def get_module_builds(module_name, stream, for b in result: _add_status_and_base_version(session, b) + if include_rpms: + for b in result: + archives = session.listArchives(buildID=b['build_id']) + # The RPM list for the 'modulemd.txt' archive has all the RPMs, recent + # versions of MBS also write upload 'modulemd..txt' archives with + # architecture subsets. + archives = [a for a in archives if a['filename'] == 'modulemd.txt'] + assert len(archives) == 1 + + b['fedmod_rpms'] = session.listRPMs(imageID=archives[0]['id']) + return result diff --git a/tests/test_flatpak_generator.py b/tests/test_flatpak_generator.py index ee52775..a0cb475 100644 --- a/tests/test_flatpak_generator.py +++ b/tests/test_flatpak_generator.py @@ -54,6 +54,24 @@ def mock_koji(): session.listTags = Mock(return_value=[ {'name': 'f29-modular-updates-candidate'} ]) + # listArchives/listRPMs are set up to return the RPMs for a mocked + # flatpak-common module - the generator code *either* looks up + # the modulemd or it looks up the RPM list, not both, so we don't + # have to pay attention to arguments passed. + session.listArchives = Mock(return_value=[ + {'btype': 'module', + 'build_id': 1149156, + 'filename': 'modulemd.txt', + 'id': 160909} + ]) + session.listRPMs = Mock(return_value=[ + {'arch': 'x86_64', + 'epoch': None, + 'id': 12345678, + 'name': 'gnome-desktop3', + 'release': '1.module_2256+753d1eef', + 'version': '3.30.2'}, + ]) p1 = patch('koji.read_config', return_value={'server': @@ -72,11 +90,13 @@ def mock_koji(): p3.stop() -def _generate_flatpak(rpm, flathub=None, expected_error_output=None): +def _generate_flatpak(rpm, flathub=None, common=False, expected_error_output=None): cmd = ['rpm2flatpak'] cmd.append(rpm) if flathub: cmd += ['--flathub', flathub] + if common: + cmd += ['--flatpak-common'] prevdir = os.getcwd() with tempfile.TemporaryDirectory() as workdir: @@ -203,3 +223,27 @@ class TestFlatpak(object): else: _generate_flatpak('eog', flathub=search_term, expected_error_output=expected_error) + + def test_flatpak_with_common(self): + with mock_koji(): + modmd, container_yaml = _generate_flatpak('eog', common=True) + + # Expected components - does not include gnome-desktop3, since that's + # in the mocked flatpak-common module + assert set(modmd.props.components_rpm) == set([ + 'bubblewrap', 'eog', 'exempi', 'libpeas' + ]) + + # Expected module dependencies for eog + dependencies = modmd.props.dependencies + assert len(dependencies) == 1 + + buildrequires = dependencies[0].props.buildrequires + assert set(buildrequires) == {'flatpak-runtime', 'flatpak-common'} + assert buildrequires['flatpak-common'].get() == ['f29'] + assert buildrequires['flatpak-runtime'].get() == ['f29'] + + requires = dependencies[0].props.requires + assert set(requires) == {'flatpak-runtime', 'flatpak-common'} + assert requires['flatpak-common'].get() == ['f29'] + assert requires['flatpak-runtime'].get() == ['f29'] From f8a98261995de38269f3cabe2844f3a2e5ee5237 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Jan 16 2019 20:09:11 +0000 Subject: [PATCH 2/2] rpm2flatpak: Add 'platform' as a dependency Recent versions of the module-build-service require that the base_module is an explicit build requirement of all modules, even if it is indirectly pulled in. (See https://pagure.io/fm-orchestrator/pull-request/1053) --- diff --git a/_fedmod/flatpak_generator.py b/_fedmod/flatpak_generator.py index 8099476..b8386e5 100644 --- a/_fedmod/flatpak_generator.py +++ b/_fedmod/flatpak_generator.py @@ -150,6 +150,7 @@ class FlatpakGenerator(ModuleGenerator): self.module_run_deps = {'flatpak-runtime': [dataset_release_branch()]} if self.include_flatpak_common: self.module_run_deps['flatpak-common'] = [dataset_release_branch()] + self.module_run_deps['platform'] = [dataset_release_branch()] def _update_module_md(self): super()._update_module_md() diff --git a/tests/test_flatpak_generator.py b/tests/test_flatpak_generator.py index a0cb475..356744a 100644 --- a/tests/test_flatpak_generator.py +++ b/tests/test_flatpak_generator.py @@ -163,12 +163,14 @@ class TestFlatpak(object): assert len(dependencies) == 1 buildrequires = dependencies[0].props.buildrequires - assert set(buildrequires) == {'flatpak-runtime'} + assert set(buildrequires) == {'flatpak-runtime', 'platform'} assert buildrequires['flatpak-runtime'].get() == ['f29'] + assert buildrequires['platform'].get() == ['f29'] requires = dependencies[0].props.requires - assert set(requires) == {'flatpak-runtime'} + assert set(requires) == {'flatpak-runtime', 'platform'} assert requires['flatpak-runtime'].get() == ['f29'] + assert requires['platform'].get() == ['f29'] @responses.activate @pytest.mark.needs_metadata @@ -239,11 +241,13 @@ class TestFlatpak(object): assert len(dependencies) == 1 buildrequires = dependencies[0].props.buildrequires - assert set(buildrequires) == {'flatpak-runtime', 'flatpak-common'} + assert set(buildrequires) == {'flatpak-runtime', 'flatpak-common', 'platform'} assert buildrequires['flatpak-common'].get() == ['f29'] assert buildrequires['flatpak-runtime'].get() == ['f29'] + assert buildrequires['platform'].get() == ['f29'] requires = dependencies[0].props.requires - assert set(requires) == {'flatpak-runtime', 'flatpak-common'} + assert set(requires) == {'flatpak-runtime', 'flatpak-common', 'platform'} assert requires['flatpak-common'].get() == ['f29'] assert requires['flatpak-runtime'].get() == ['f29'] + assert requires['platform'].get() == ['f29']