#30 WIP: Add rpm2flatpak subcommand
Merged 2 years ago by otaylor. Opened 2 years ago by yselkowitz.
yselkowitz/flatpak-module-tools rpm2flatpak  into  main

@@ -18,6 +18,7 @@ 

  from .flatpak_builder import (FLATPAK_METADATA_ANNOTATIONS,

                                FLATPAK_METADATA_BOTH,

                                FLATPAK_METADATA_LABELS)

+ from .flatpak_generator import FlatpakGenerator

  from .koji_utils import watch_koji_task

  from .installer import Installer

  from .rpm_builder import RpmBuilder
@@ -484,3 +485,31 @@ 

      builder.build_rpms_local(

          manual_packages, manual_repos, auto=auto, allow_outdated=allow_outdated

      )

+ 

+ 

+ @cli.command()

+ @click.option("--flathub", metavar="ID_OR_SEARCH_TERM",

+               help="Initialize from a Flathub Flatpak.")

+ @click.option("--runtime-name", metavar="RUNTIME",

+               help="Specify runtime-name, defaults to 'flatpak-runtime'")

+ @click.option("--runtime-version", metavar="VERSION",

+               help="Specify runtime-version, defaults to latest stable release")

+ @click.option("--output-containerspec", metavar="FILE",

+               help="Write container specification to FILE"

+                    " instead of container.yaml.")

+ @click.option("--force", "-f", is_flag=True,

+               help="Overwriting existing output files")

+ @click.argument("package", metavar='PACKAGE', required=True)

+ @click.pass_context

+ def init(

+     ctx,

+     output_containerspec: Optional[Path],

+     flathub: Optional[str],

+     force: bool,

+     runtime_name: Optional[str],

+     runtime_version: Optional[str],

+     package: List[str]

+ ):

+     """Generate container.yaml from an RPM"""

+     fg = FlatpakGenerator(package)

+     fg.run(output_containerspec, force=force, flathub=flathub, runtime_name=runtime_name, runtime_version=runtime_version)

@@ -0,0 +1,193 @@ 

+ import json

+ import os

+ import re

+ import sys

+ from textwrap import dedent

+ 

+ import click

+ import requests

+ import yaml

+ 

+ 

+ # Some PyYAML magic to get the output we want for container.yaml

+ 

+ class LiteralScalar(str):

+     """String subclass that gets dumped into yaml as a scalar"""

+     pass

+ 

+ 

+ def _represent_literal_scalar(dumper, s):

+     return dumper.represent_scalar(tag=u'tag:yaml.org,2002:str',

+                                    value=s,

+                                    style='|')

+ 

+ 

+ yaml.add_representer(LiteralScalar, _represent_literal_scalar)

+ 

+ 

+ class NoSortMapping(dict):

+     """dict subclass that dumped into yaml as a scalar without sorting keys"""

+     pass

+ 

+ 

+ def _represent_no_sort_mapping(dumper, d):

+     return yaml.MappingNode(tag='tag:yaml.org,2002:map',

+                             value=[(dumper.represent_data(k),

+                                     dumper.represent_data(v))

+                                    for k, v in d.items()],

+                             flow_style=False)

+ 

+ 

+ yaml.add_representer(NoSortMapping, _represent_no_sort_mapping)

+ 

+ 

+ def _load_flathub_manifest(search_term):

+     response = requests.get("https://flathub.org/api/v1/apps")

+     response.raise_for_status()

+     apps = response.json()

+ 

+     matches = []

+     search_lower = search_term.lower()

+     for app in apps:

+         if (search_lower in app['flatpakAppId'].lower() or

+                 search_lower in app['name'].lower()):

+             matches.append((app['flatpakAppId'], app['name']))

+ 

+     if len(matches) > 1:

+         max_id_len = max([len(app_id) for app_id, _ in matches])

+         for app_id, name in matches:

+             print(app_id + (' ' * (max_id_len - len(app_id)) + ' ' + name))

+         raise click.ClickException("Multiple matches found on flathub.org")

+     elif len(matches) == 0:

+         raise click.ClickException("No match found on flathub.org")

+ 

+     app_id = matches[0][0]

+ 

+     for fname, is_yaml in [

+             (f"{app_id}.json", False),

+             (f"{app_id}.yaml", True),

+             (f"{app_id}.yml", True)]:

+         url = f"https://raw.githubusercontent.com/flathub/{app_id}/master/{fname}"

+         response = requests.get(url)

+         if response.status_code == 404:

+             continue

+         else:

+             break

+ 

+     response.raise_for_status()

+ 

+     if is_yaml:

+         return yaml.safe_load(response.text)

+     else:

+         # flatpak-builder supports non-standard comments in the manifest, strip

+         # them out. (Ignore the possibility of C comments embedded in strings.)

+         #

+         # Regex explanation: matches /*<something>*/ (multiline)

+         #    <something> DOES NOT contains "/*" substring

+         no_comments = re.sub(r'/\*((?!/\*).)*?\*/', '', response.text, flags=re.DOTALL)

+         return json.loads(no_comments)

+ 

+ 

+ class FlatpakGenerator(str):

+     def __init__(self, pkg):

+         self.pkg = pkg

+ 

+     def _flathub_container_yaml(self, manifest, runtime_name, runtime_version):

+         app_id = manifest.get('app-id')

+         if app_id is None:

+             app_id = manifest['id']

+         yml = NoSortMapping({

+             'flatpak': NoSortMapping({

+                 'id': app_id,

+                 'branch': 'stable',

+                 'runtime-name': runtime_name,

+                 'runtime-version': 'f' + str(runtime_version),

+             })

+         })

+ 

+         yml['flatpak']['packages'] = [self.pkg]

+ 

+         for key in ['command',

+                     'appstream-license',

+                     'appstream-compose',

+                     'desktop-file-name-prefix',

+                     'desktop-file-name-suffix',

+                     'rename-appdata-file',

+                     'rename-desktop-file',

+                     'rename-icon',

+                     'copy-icon']:

+             if key in manifest:

+                 yml['flatpak'][key] = manifest[key]

+ 

+         if 'finish-args' in manifest:

+             yml['flatpak']['finish-args'] = LiteralScalar('\n'.join(manifest['finish-args']))

+ 

+         return yaml.dump(yml, default_flow_style=False, indent=4)

+ 

+     def _default_container_yaml(self, runtime_name, runtime_version):

+         pkg = self.pkg

+         command = pkg

+         branch = 'f' + str(runtime_version)

+ 

+         container_yaml = dedent(f'''\

+             flatpak:

+                 # Derived from the project's domain name

+                 id: org.example.MyApp

+                 branch: stable

+                 runtime-name: {runtime_name}

+                 runtime-version: {branch}

+                 # RPM package(s) to install, main package first

+                 packages:

+                 - {pkg}

+                 # Binary to execute to run the app

+                 command: {command}

+                 # Not sandboxed. See 'man flatpak-build-finish'

+                 finish-args: |-

+                     --device=dri

+                     --filesystem=host

+                     --share=ipc

+                     --socket=x11

+                     --socket=wayland

+                     --socket=session-bus

+             ''')

+ 

+         return container_yaml

+ 

+     def _write_container_yaml(self, output_fname, flathub_manifest, runtime_name, runtime_version):

+         if flathub_manifest:

+             container_yaml = self._flathub_container_yaml(flathub_manifest, runtime_name, runtime_version)

+         else:

+             container_yaml = self._default_container_yaml(runtime_name, runtime_version)

+ 

+         with open(output_fname, 'w') as f:

+             f.write(container_yaml)

+ 

+         print(f"Generated container specification: {output_fname!r}."

+               f" Please edit appropriately.")

+ 

+     def run(self, output_containerspec,

+             force=False, flathub=None, runtime_name=None, runtime_version=None):

+         flathub_manifest = _load_flathub_manifest(flathub) if flathub else None

+ 

+         if output_containerspec is None:

+             output_containerspec = 'container.yaml'

+ 

+         if not force:

+             if os.path.exists(output_containerspec):

+                 raise click.ClickException(f"{output_containerspec} exists."

+                                            f" Pass --force to overwrite.")

+ 

+         if runtime_name is None:

+             runtime_name = 'flatpak-runtime'

+ 

+         if runtime_version is None:

+             response = requests.get("https://bodhi.fedoraproject.org/releases/?state=current")

+             response.raise_for_status()

+ 

+             runtime_version = max(int(

+                 r["version"])

+                     for r in response.json()["releases"]

+                     if r["id_prefix"] == "FEDORA-FLATPAK"

+             )

+ 

+         self._write_container_yaml(output_containerspec, flathub_manifest, runtime_name, runtime_version)

@@ -0,0 +1,7 @@ 

+ [{"flatpakAppId":"org.gnome.eog",

+  "name":"Eye of GNOME",

+  "summary":"Browse and rotate images",

+  "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.eog.png"},

+ {"flatpakAppId":"org.gnome.FeedReader",

+  "name":"FeedReader","summary":"RSS client for various webservices",

+  "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.FeedReader.png"}]

@@ -0,0 +1,16 @@ 

+ {

+   "id": "org.gnome.eog",

+   "runtime": "org.gnome.Platform",

+   "runtime-version": "3.30",

+   "sdk": "org.gnome.Sdk",

+   "branch": "stable",

+   "command": "eog",

+   "rename-desktop-file": "eog.desktop",

+   "rename-appdata-file": "eog.appdata.xml",

+   "rename-icon": "eog",

+   "copy-icon": true,

+   "finish-args":

+   /* X11 + XShm access */

+   ["--share=ipc",

+    "--socket=x11"]

+ }

@@ -0,0 +1,15 @@ 

+ # eog.json has id so that we test both

+ app-id: org.gnome.eog

+ runtime: org.gnome.Platform

+ runtime-version: '3.30'

+ sdk: org.gnome.Sdk

+ branch: stable

+ command: eog

+ rename-desktop-file: eog.desktop

+ rename-appdata-file: eog.appdata.xml

+ rename-icon: eog

+ copy-icon: true

+ finish-args:

+   # X11 + XShm access

+   - --share=ipc

+   - --socket=x11

@@ -0,0 +1,30 @@ 

+ {

+   "releases": [

+     {

+       "name": "F38F",

+       "long_name": "Fedora 38 Flatpaks",

+       "version": "38",

+       "id_prefix": "FEDORA-FLATPAK",

+       "branch": "f38",

+       "dist_tag": "f38-flatpak",

+       "stable_tag": "f38-flatpak-updates",

+       "testing_tag": "f38-flatpak-updates-testing",

+       "candidate_tag": "f38-flatpak-updates-candidate",

+       "pending_signing_tag": "",

+       "pending_testing_tag": "f38-flatpak-updates-testing-pending",

+       "pending_stable_tag": "f38-flatpak-updates-pending",

+       "override_tag": "f38-flatpak-override",

+       "mail_template": "fedora_errata_template",

+       "state": "current",

+       "composed_by_bodhi": true,

+       "create_automatic_updates": false,

+       "package_manager": "unspecified",

+       "testing_repository": null,

+       "eol": null

+     }

+   ],

+   "page": 1,

+   "pages": 1,

+   "rows_per_page": 20,

+   "total": 1

+ }

@@ -0,0 +1,125 @@ 

+ """In-process tests for the flatpak generator"""

+ 

+ import logging

+ import os

+ import tempfile

+ 

+ from click.testing import CliRunner

+ import pytest

+ import responses

+ import yaml

+ 

+ from flatpak_module_tools.cli import cli

+ 

+ 

+ log = logging.getLogger(__name__)

+ 

+ testfiles_dir = os.path.join(os.path.dirname(__file__), 'files', 'generator')

+ 

+ with open(os.path.join(testfiles_dir, 'apps.json')) as f:

+     APPS_JSON = f.read()

+ 

+ with open(os.path.join(testfiles_dir, 'eog.yaml')) as f:

+     EOG_YAML = f.read()

+ 

+ with open(os.path.join(testfiles_dir, 'eog.json')) as f:

+     EOG_JSON = f.read()

+ 

+ with open(os.path.join(testfiles_dir, 'releases.json')) as f:

+     RELEASES_JSON = f.read()

+ 

+ 

+ def _generate_flatpak(rpm, flathub=None, runtime_name=None, runtime_version=None, expected_error_output=None):

+     cmd = ['init']

+     cmd.append(rpm)

+     if flathub:

+         cmd += ['--flathub', flathub]

+ 

+     prevdir = os.getcwd()

+     with tempfile.TemporaryDirectory() as workdir:

+         try:

+             os.chdir(workdir)

+             runner = CliRunner()

+             result = runner.invoke(cli, cmd, catch_exceptions=False)

+             if expected_error_output is not None:

+                 assert result.exit_code != 0

+                 assert expected_error_output in result.output

+                 return

+             else:

+                 assert result.exit_code == 0

+ 

+             with open('container.yaml') as f:

+                 contents = f.read()

+ 

+             log.info('container.yaml:\n%s\n', contents)

+             container_yaml = yaml.safe_load(contents)

+         finally:

+             os.chdir(prevdir)

+ 

+     return container_yaml

+ 

+ 

+ class TestFlatpak(object):

+     @pytest.mark.filterwarnings('ignore::DeprecationWarning:koji')

+     @pytest.mark.filterwarnings('ignore::PendingDeprecationWarning:koji')

+     @pytest.mark.needs_metadata

+     def test_generated_flatpak_files(self):

+         container_yaml = _generate_flatpak('eog')

+ 

+     @responses.activate

+     @pytest.mark.needs_metadata

+     @pytest.mark.parametrize(('search_term', 'extension', 'expected_error'),

+                              [

+                                  ('org.gnome.eog', 'yaml', None),

+                                  ('org.gnome.eog', 'yml', None),

+                                  ('org.gnome.eog', 'json', None),

+                                  ('eYe of gNome', 'yaml', None),

+                                  ('org.gnome', 'yaml',

+                                   'Multiple matches found on flathub.org'),

+                                  ('notexist', 'yaml',

+                                   'No match found on flathub.org'),

+                              ])

+     def test_flatpak_from_flathub(self, search_term, extension,

+                                   expected_error):

+         responses.add(responses.GET, 'https://flathub.org/api/v1/apps',

+                       body=APPS_JSON, content_type='application/json')

+         responses.add(responses.GET, 'https://bodhi.fedoraproject.org/releases/?state=current',

+                       body=RELEASES_JSON, content_type='application/json')

+ 

+         app_id = 'org.gnome.eog'

+         base = 'https://raw.githubusercontent.com/flathub'

+ 

+         for ext, content_type, body in [

+                 ('yml', 'application/x-yaml', EOG_YAML),

+                 ('yaml', 'application/x-yaml', EOG_YAML),

+                 ('json', 'application/json', EOG_JSON)]:

+             if extension == ext:

+                 responses.add(responses.GET,

+                               f"{base}/{app_id}/master/{app_id}.{ext}",

+                               body=body, content_type=content_type)

+             else:

+                 responses.add(responses.GET,

+                               f"{base}/{app_id}/master/{app_id}.{ext}",

+                               body='Not found', status=404)

+ 

+         if expected_error is None:

+             container_yaml = \

+                 _generate_flatpak('eog',

+                                   flathub=search_term,

+                                   expected_error_output=expected_error)

+ 

+             f = container_yaml['flatpak']

+ 

+             assert f['id'] == 'org.gnome.eog'

+             assert f['command'] == 'eog'

+             assert f['runtime-name'] == 'flatpak-runtime'

+             assert f['runtime-version'] == 'f38'

+             assert f['rename-desktop-file'] == 'eog.desktop'

+             assert f['rename-appdata-file'] == 'eog.appdata.xml'

+             assert f['rename-icon'] == 'eog'

+             assert f['copy-icon'] is True

+             assert f['finish-args'] == '--share=ipc\n--socket=x11'

+             assert f['packages'][0] == 'eog'

+         else:

+             _generate_flatpak('eog', flathub=search_term,

+                                   expected_error_output=expected_error)

WARNING this is probably not ready yet.

This is based on and replaces fedmod rpm2flatpak, as we have moved away from modularity in flatpak builds from F39.

Maybe this should be --kde5? Maybe it should be--runtime=flatpak-kde5-runtime ? Maybe it's not necessary at all and the user should just edit the result? (Different from the fedmod case, since fedmod needed to know the runtime). I'd probably skip the -K shortcut in any case, since it's not like this will be typed all the time.

I'm thinking this gets wrapped by fedpkg as flatpak-init, to go along with flatpak-build flatpak-build-local flatpak-build-rpms flatpak-build-rpms-local So maybe just 'flatpak-module init' ?

This variable should be 'runtime_version'. I think we initialize it by current Fedora stable release. Adam Williamson's recommendation was to use bodhi for that. Something like:

response = requests.get("https://bodhi.fedoraproject.org/releases/?state=current")
response.raise_for_status()

return max(int(
    r["version"])
          for r in response.json()["releases"]
          if r["id_prefix"] == "FEDORA-FLATPAK"
))

(Call fromrun() not __init__()). I don't know if this needs to be overridable, other than by editing, but the command ends up with a --runtime argument, it could have a --runtime-version argument too.

Generally looks fine to me, other than the naming / cli questions above.

rebased onto 5f8207821288654639e9f7d01e549a9b74037d17

2 years ago

rebased onto 6e947d20e88f0ee61a0fb56ca5ed2528ce6b608c

2 years ago

rebased onto 7f675b94774cae604a2c219ea8d29574c1c61df7

2 years ago

A bit stray in this patch, if an end-user hits this, they are still going to be puzzled :-) ... but sure.

A bit stray in this patch, if an end-user hits this, they are still going to be puzzled :-) ... but sure.

Sorry, had that locally, but it was meant for a separate PR. Do you want me to drop that here?

rebased onto 768ada635fdd298824126a17d5c7f6db02ff124a

2 years ago

This generally looks great. The style of the tests is clearly not the same as the other tests, but that is fine since they are moving from fedmod - if we want to make things consistent later, we can do that. The only request I would make here is to move things from tests/files/flatpak to tests/files/generator - since everything is flatpak in flatpak-module-tools.

rebased onto cf20a2e

2 years ago

Yes this were taken straight from fedmod, with the modulemd stripped out and adjusted the integration to the rest of the package, but didn't consider code style.

Pull-Request has been merged by otaylor

2 years ago