PR#16: Allow overriding the architecture that FlatpakBuilder uses - flatpak-module-tools

		`@@ -29,6 +29,44 @@`
		`from xml.etree import ElementTree`


		`+ class Arch:`
		`+ def __init__(self, oci, flatpak, rpm):`
		`+ self.oci = oci`
		`+ self.flatpak = flatpak`
		`+ self.rpm = rpm`
		`+`
		`+`
		`+ ARCHES = {`
		`+ arch.oci: arch for arch in [`
		`+ Arch(oci="amd64", flatpak="x86_64", rpm="x86_64"),`
		`+ Arch(oci="arm64", flatpak="aarch64", rpm="aarch64"),`
		`+ Arch(oci="s390x", flatpak="s390x", rpm="s390x"),`
		`+ Arch(oci="ppc64le", flatpak="ppc64le", rpm="ppc64le"),`
		`+ # This is used in tests to test the case where the Flatpak and RPM names are`
		`+ # different - this does not happen naturally at the moment as far as I know.`
		`+ Arch(oci="testarch", flatpak="testarch", rpm="testarch_rpm"),`
		`+ ]`
		`+ }`
		`+`
		`+`
		`+ @functools.lru_cache(maxsize=None)`
		`+ def _get_flatpak_arch():`
		`+ return subprocess.check_output(`
		`+ ['flatpak', '--default-arch'], universal_newlines=True`
		`+ ).strip()`
		`+`
		`+ def get_arch(oci_arch=None):`
		`+ if oci_arch:`
		`+ return ARCHES[oci_arch]`
		`+ else:`
		`+ flatpak_arch = _get_flatpak_arch()`
		`+ for arch in ARCHES.values():`
		`+ if arch.flatpak == flatpak_arch:`
		`+ return arch`
		`+`
		`+ raise RuntimeError("Unknown flatpak arch '{}'".format(flatpak_arch))`
		`+`
		`+`
		`FLATPAK_METADATA_LABELS = "labels"`
		`FLATPAK_METADATA_ANNOTATIONS = "annotations"`
		`FLATPAK_METADATA_BOTH = "both"`
		`@@ -56,21 +94,6 @@`
		`CP = configparser.RawConfigParser`


		`- @functools.lru_cache(maxsize=None)`
		`- def get_flatpak_arch():`
		`- """Return Flatpak's name for the current architecture"""`
		`- return subprocess.check_output(['flatpak', '--default-arch'],`
		`- universal_newlines=True).strip()`
		`-`
		`-`
		`- # Returns flatpak's name for the current arch`
		`- @functools.lru_cache(maxsize=None)`
		`- def get_rpm_arch():`
		`- """Return RPM's name for the current architecture"""`
		`- return subprocess.check_output(['rpm', '--eval', '%{_arch}'],`
		`- universal_newlines=True).strip()`
		`-`
		`-`
		`# flatpak build-init requires the sdk and runtime to be installed on the`
		`# build system (so that subsequent build steps can execute things with`
		`# the SDK). While it isn't impossible to download the runtime image and`
		`@@ -78,7 +101,7 @@`
		`# since our build step is just unpacking the filesystem we've already`
		`# created. This is a stub implementation of 'flatpak build-init' that`
		`# doesn't check for the SDK or use it to set up the build filesystem.`
		`- def build_init(directory, appname, sdk, runtime, runtime_branch, tags=[]):`
		`+ def build_init(directory, appname, sdk, runtime, runtime_branch, arch, tags=[]):`
		`if not os.path.isdir(directory):`
		`os.mkdir(directory)`
		`with open(os.path.join(directory, "metadata"), "w") as f:`
		`@@ -91,7 +114,7 @@`
		`sdk=sdk,`
		`runtime=runtime,`
		`runtime_branch=runtime_branch,`
		`- arch=get_flatpak_arch())))`
		`+ arch=arch.flatpak)))`
		`if tags:`
		`f.write("tags=" + ";".join(tags) + "\n")`
		`os.mkdir(os.path.join(directory, "files"))`
		`@@ -105,10 +128,10 @@`
		`self.mmd = mmd`
		`self.rpms = rpms`

		`- def get_profile_packages(self, profile):`
		`+ def get_profile_packages(self, profile, arch):`
		`result = self.mmd.get_profile(profile).get_rpms()`

		`- arch_profile = profile + '-' + get_rpm_arch()`
		`+ arch_profile = profile + '-' + arch.rpm`
		`if arch_profile in self.mmd.get_profile_names():`
		`result.extend(self.mmd.get_profile(arch_profile).get_rpms())`

		`@@ -389,7 +412,7 @@`

		`class FlatpakBuilder(object):`
		`def __init__(self, source, workdir, root, parse_manifest=None,`
		`- flatpak_metadata=FLATPAK_METADATA_ANNOTATIONS):`
		`+ flatpak_metadata=FLATPAK_METADATA_ANNOTATIONS, oci_arch=None):`
		`self.source = source`
		`self.workdir = workdir`
		`self.root = root`
		`@@ -401,6 +424,8 @@`
		`raise ValueError("Bad flatpak_metadata value %s" % flatpak_metadata)`
		`self.flatpak_metadata = flatpak_metadata`

		`+ self.arch = get_arch(oci_arch)`
		`+`
		`self.log = logging.getLogger(__name__)`

		`self.extra_labels = {}`
		`@@ -443,7 +468,7 @@`

		`def get_install_packages(self):`
		`source = self.source`
		`- packages = source.base_module.get_profile_packages(source.profile)`
		`+ packages = source.base_module.get_profile_packages(source.profile, self.arch)`
		`if not source.runtime:`
		`# The flatpak-runtime-config package is needed when building an application`
		`# Flatpak because it includes file triggers for files in /app. (Including just`
		`@@ -481,7 +506,9 @@`

		`if not source.runtime:`
		`runtime_module = source.runtime_module`
		`- available_packages = sorted(runtime_module.get_profile_packages('runtime'))`
		`+ available_packages = sorted(`
		`+ runtime_module.get_profile_packages('runtime', self.arch)`
		`+ )`

		`for m in source.app_modules:`
		`# Strip off the '.rpm' suffix from the filename to get something`
		`@@ -489,7 +516,9 @@`
		`available_packages.extend(x[:-4] for x in m.rpms)`
		`else:`
		`base_module = source.base_module`
		`- available_packages = sorted(base_module.get_profile_packages(source.profile))`
		`+ available_packages = sorted(`
		`+ base_module.get_profile_packages(source.profile, self.arch)`
		`+ )`

		`return available_packages`

		`@@ -714,7 +743,7 @@`
		`'id': id_,`
		`'runtime_id': runtime_id,`
		`'sdk_id': sdk_id,`
		`- 'arch': get_flatpak_arch(),`
		`+ 'arch': self.arch.flatpak,`
		`'branch': branch`
		`}`

		`@@ -755,9 +784,13 @@`
		`subprocess.check_call(['ostree', 'commit'] + commit_args)`
		`subprocess.check_call(['ostree', 'summary', '-u', '--repo', repo])`

		`- subprocess.check_call(['flatpak', 'build-bundle', repo,`
		`- '--oci', '--runtime',`
		`- outfile, id_, branch])`
		`+ subprocess.check_call([`
		`+ 'flatpak', 'build-bundle', repo,`
		`+ '--oci',`
		`+ '--runtime',`
		`+ '--arch', self.arch.flatpak,`
		`+ outfile, id_, branch`
		`+ ])`

		`return runtime_ref`

		`@@ -786,10 +819,12 @@`
		`# See comment for build_init() for why we can't use 'flatpak build-init'`
		`# subprocess.check_call(['flatpak', 'build-init',`
		`# builddir, app_id, runtime_id, runtime_id, runtime_version])`
		`- build_init(builddir, app_id, sdk_id, runtime_id, runtime_version, tags=info.get('tags', []))`
		`+ build_init(`
		`+ builddir, app_id, sdk_id, runtime_id, runtime_version, self.arch, tags=info.get('tags', [])`
		`+ )`

		`# with gzip'ed tarball, tar is several seconds faster than tarfile.extractall`
		`- subprocess.check_call(['tar', 'xCfz', builddir, tarred_filesystem])`
		`+ subprocess.check_call(['tar', 'xvCfz', builddir, tarred_filesystem])`

		`processor = FileTreeProcessor(builddir, info)`
		`processor.process()`
		`@@ -811,7 +846,7 @@`
		`raise`

		`def try_export(disable_sandbox):`
		`- args = ['flatpak', 'build-export', repo, builddir, app_branch]`
		`+ args = ['flatpak', 'build-export', '-v', repo, builddir, app_branch]`
		`if disable_sandbox:`
		`args += ['--disable-sandbox']`
		`if 'end-of-life' in info:`
		`@@ -836,11 +871,15 @@`
		`self.log.info('Retrying without --disable-sandbox')`
		`try_export(disable_sandbox=False)`

		`- subprocess.check_call(['flatpak', 'build-bundle', repo, '--oci',`
		`- outfile, app_id, app_branch])`
		`+ subprocess.check_call([`
		`+ 'flatpak', 'build-bundle', repo,`
		`+ '--oci',`
		`+ '--arch', self.arch.flatpak,`
		`+ outfile, app_id, app_branch`
		`+ ])`

		`app_ref = 'app/{app_id}/{arch}/{branch}'.format(app_id=app_id,`
		`- arch=get_flatpak_arch(),`
		`+ arch=self.arch.flatpak,`
		`branch=app_branch)`

		`return app_ref`

tests/test_flatpak_builder.py

file modified

+131 -45

		`@@ -1,5 +1,8 @@`
		`+ from io import TextIOWrapper`
		`+ import json`
		`import os`
		`from subprocess import check_call`
		`+ import tarfile`
		`from textwrap import dedent`

		`import gi`
		`@@ -7,10 +10,11 @@`
		`from gi.repository import Modulemd`

		`import pytest`
		`+ from six.moves import configparser`
		`import yaml`

		`from flatpak_module_tools.flatpak_builder import (`
		`- FlatpakBuilder, FlatpakSourceInfo, FLATPAK_METADATA_BOTH, get_rpm_arch, ModuleInfo`
		`+ FlatpakBuilder, FlatpakSourceInfo, FLATPAK_METADATA_BOTH, get_arch, ModuleInfo`
		`)`


		`@@ -142,36 +146,58 @@`
		`- flatpak-runtime:f33`
		`flatpak:`
		`id: org.fedoraproject.Platform`
		`- branch: stable`
		`+ sdk: org.fedoraproject.Sdk`
		`+ branch: f33`
		`end-of-life: Fedora 33 is no longer supported`
		`end-of-life-rebase: org.fedoraproject.NewPlatform`
		`"""`


		`+ # We repeat the test with and without overriding the architectures`
		`+ @pytest.fixture(params=(None, "arm64", "testarch"))`
		`+ def oci_arch_override(request):`
		`+ return request.param`
		`+`
		`+`
		`+ # This is the Arch object corresponding to oci_arch_override`
		`+ @pytest.fixture`
		`+ def arch(oci_arch_override):`
		`+ return get_arch(oci_arch_override)`
		`+`
		`+`
		`+ # We need to substitute on the arch a lot, so define another fixture to do that`
		`+ # concisely`
		`+ @pytest.fixture`
		`+ def R(arch):`
		`+ def r(str):`
		`+ return str.format(flatpak_arch=arch.flatpak, rpm_arch=arch.rpm)`
		`+`
		`+ return r`
		`+`
		`+`
		`@pytest.fixture`
		`- def runtime_module():`
		`+ def runtime_module(R):`
		`runtime_mmd = Modulemd.ModuleStream.read_string(FLATPAK_RUNTIME_MMD, True)`
		`yield ModuleInfo(runtime_mmd.get_module_name(),`
		`runtime_mmd.get_stream_name(),`
		`runtime_mmd.get_version(),`
		`runtime_mmd,`
		`- ['flatpak-rpm-macros-0:32-2.x86_64.rpm',`
		`- 'flatpak-runtime-config-0:29-5.x86_64.rpm'])`
		`+ [R('flatpak-rpm-macros-0:32-2.{rpm_arch}.rpm'),`
		`+ R('flatpak-runtime-config-0:29-5.{rpm_arch}.rpm')])`


		`@pytest.fixture`
		`- def testapp_module():`
		`- testapp_mmd = TESTAPP_MMD.replace("@ARCH@",`
		`- get_rpm_arch())`
		`+ def testapp_module(arch, R):`
		`+ testapp_mmd = TESTAPP_MMD.replace("@ARCH@", arch.rpm)`

		`testapp_mmd = Modulemd.ModuleStream.read_string(testapp_mmd, True)`
		`yield ModuleInfo(testapp_mmd.get_module_name(),`
		`testapp_mmd.get_stream_name(),`
		`testapp_mmd.get_version(),`
		`testapp_mmd,`
		`- ['testapp-0:1-1.x86_64.rpm',`
		`- 'testapp-fancymath-0:1-1.x86_64.rpm',`
		`- 'libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.x86_64.rpm'])`
		`+ [R('testapp-0:1-1.{rpm_arch}.rpm'),`
		`+ R('testapp-fancymath-0:1-1.{rpm_arch}.rpm'),`
		`+ R('libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.{rpm_arch}.rpm')])`


		`@pytest.fixture`
		`@@ -246,7 +272,6 @@`
		`manifest_lines = [l + "\n" for l in manifest.strip().split("\n")]`

		`expected_components = [c for c in parse_manifest(manifest_lines) if c['include']]`
		`- print(expected_components)`
		`components = builder.get_components(manifestfile)`

		`def flatten(components):`
		`@@ -256,13 +281,73 @@`
		`assert flatten(components) == flatten(expected_components)`


		`- def test_app_basic(testapp_source, tmpdir):`
		`+ def check_exported_oci(oci_outdir, arch, R, runtime=False):`
		`+ with open(os.path.join(oci_outdir, "index.json")) as f:`
		`+ index = json.load(f)`
		`+`
		`+ def descriptor_to_path(descriptor):`
		`+ digest = descriptor["digest"]`
		`+ assert digest.startswith("sha256:")`
		`+ return os.path.join(oci_outdir, "blobs", "sha256", digest[7:])`
		`+`
		`+ # Load the manifest that describes the container`
		`+ with open(descriptor_to_path(index["manifests"][0])) as f:`
		`+ manifest = json.load(f)`
		`+`
		`+ # Get the labels for the container`
		`+ with open(descriptor_to_path(manifest["config"])) as f:`
		`+ config = json.load(f)`
		`+`
		`+ print(config)`
		`+ assert config['architecture'] == arch.oci`
		`+`
		`+ labels = config["config"]["Labels"]`
		`+`
		`+ if runtime:`
		`+ assert labels["org.flatpak.ref"] == \`
		`+ R("runtime/org.fedoraproject.Platform/{flatpak_arch}/f33")`
		`+ else:`
		`+ assert labels["org.flatpak.ref"] == \`
		`+ R("app/org.fedoraproject.TestApp/{flatpak_arch}/stable")`
		`+`
		`+ # Do some basic checks on the metadata label`
		`+ metadata_from_labels = labels["org.flatpak.metadata"]`
		`+ cp = configparser.RawConfigParser()`
		`+ cp.read_string(metadata_from_labels)`
		`+`
		`+ if runtime:`
		`+ assert cp.get("Runtime", "runtime") == \`
		`+ R("org.fedoraproject.Platform/{flatpak_arch}/f33")`
		`+ assert cp.get("Runtime", "sdk") == \`
		`+ R("org.fedoraproject.Sdk/{flatpak_arch}/f33")`
		`+ else:`
		`+ assert cp.get("Application", "runtime") == \`
		`+ R("org.fedoraproject.Platform/{flatpak_arch}/f33")`
		`+`
		`+ # Now get the contents we built`
		`+ tar = tarfile.open(descriptor_to_path(manifest["layers"][0]), "r:gz")`
		`+`
		`+ # Check that that has the same metadata as the labels - the metadata`
		`+ # file here will be used after installation`
		`+ extracted = tar.extractfile("metadata")`
		`+ assert extracted`
		`+ metadata_stream = TextIOWrapper(extracted)`
		`+ metadata_from_tarfile = metadata_stream.read()`
		`+ metadata_stream.close()`
		`+ assert metadata_from_tarfile == metadata_from_labels`
		`+`
		`+ # And check that the bin/hello file we add for both the runtime and app is there`
		`+ assert tar.getmember("files/bin/hello") is not None`
		`+`
		`+`
		`+ def test_app_basic(testapp_source, tmpdir, oci_arch_override, arch, R):`
		`workdir = str(tmpdir / "work")`
		`os.mkdir(workdir)`

		`builder = FlatpakBuilder(testapp_source, workdir, "root",`
		`parse_manifest=parse_manifest,`
		`- flatpak_metadata=FLATPAK_METADATA_BOTH)`
		`+ flatpak_metadata=FLATPAK_METADATA_BOTH,`
		`+ oci_arch=oci_arch_override)`

		`assert set(builder.get_install_packages()) == set([`
		`"testapp", "testapp-fancymath", "flatpak-runtime-config"`
		`@@ -271,9 +356,9 @@`
		`"glibc",`
		`"flatpak-runtime-config",`
		`"libfoo",`
		`- "libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.x86_64",`
		`- "testapp-0:1-1.x86_64",`
		`- "testapp-fancymath-0:1-1.x86_64"`
		`+ R("libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.{rpm_arch}"),`
		`+ R("testapp-0:1-1.{rpm_arch}"),`
		`+ R("testapp-fancymath-0:1-1.{rpm_arch}")`
		`])`

		`bindir = tmpdir / "root/app/bin"`
		`@@ -282,42 +367,43 @@`
		`with open(bindir / "hello", "w") as f:`
		`os.fchmod(f.fileno(), 0o0755)`

		`- check_call(["tar", "cfv", "export.tar", "-H", "pax", "--sort=name",`
		`+ check_call(["tar", "cf", "export.tar", "-H", "pax", "--sort=name",`
		`"root"], cwd=tmpdir)`

		`with open(tmpdir / "export.tar", "rb") as f:`
		`outfile, manifest_file = (builder._export_from_stream(f, close_stream=False))`

		`- check_call(["gzip", tmpdir / "export.tar"])`
		`-`
		`- builder.build_container(str(tmpdir / "export.tar.gz"))`
		`+ ref_name, oci_outdir, tarred_oci_outdir = builder.build_container(outfile)`

		`# libfoo from the module should be listed in builder.get_components()`
		`- check_get_components(builder, tmpdir, dedent("""\`
		`- false flatpak-runtime-config-0:29-5.x86_64`
		`- false glibc-0:2.32-4.fc33.x86_64`
		`- true libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.x86_64`
		`- true testapp-0:1-1.x86_64`
		`- true testapp-fancymath-0:1-1.x86_64`
		`- """))`
		`+ check_get_components(builder, tmpdir, R(dedent("""\`
		`+ false flatpak-runtime-config-0:29-5.{rpm_arch}`
		`+ false glibc-0:2.32-4.fc33.{rpm_arch}`
		`+ true libfoo-0:1.2.4-1.module_f33+11439+4b44cd2d.{rpm_arch}`
		`+ true testapp-0:1-1.{rpm_arch}`
		`+ true testapp-fancymath-0:1-1.{rpm_arch}`
		`+ """)))`

		`# But libfoo from the runtime should not`
		`- check_get_components(builder, tmpdir, dedent("""\`
		`- false flatpak-runtime-config-0:29-5.x86_64`
		`- false glibc-0:2.32-4.fc33.x86_64`
		`- false libfoo-0:1.2.3-1.fc33.x86_64`
		`- true testapp-0:1-1.x86_64`
		`- true testapp-fancymath-0:1-1.x86_64`
		`- """))`
		`+ check_get_components(builder, tmpdir, R(dedent("""\`
		`+ false flatpak-runtime-config-0:29-5.{rpm_arch}`
		`+ false glibc-0:2.32-4.fc33.{rpm_arch}`
		`+ false libfoo-0:1.2.3-1.fc33.{rpm_arch}`
		`+ true testapp-0:1-1.{rpm_arch}`
		`+ true testapp-fancymath-0:1-1.{rpm_arch}`
		`+ """)))`

		`+ check_exported_oci(oci_outdir, arch, R, runtime=False)`

		`- def test_runtime_basic(runtime_source, tmpdir):`
		`+`
		`+ def test_runtime_basic(runtime_source, tmpdir, oci_arch_override, arch, R):`
		`workdir = str(tmpdir / "work")`
		`os.mkdir(workdir)`

		`builder = FlatpakBuilder(runtime_source, workdir, "root",`
		`parse_manifest=parse_manifest,`
		`- flatpak_metadata=FLATPAK_METADATA_BOTH)`
		`+ flatpak_metadata=FLATPAK_METADATA_BOTH,`
		`+ oci_arch=oci_arch_override)`

		`assert set(builder.get_install_packages()) == set([`
		`"glibc", "flatpak-runtime-config", "libfoo"`
		`@@ -338,16 +424,16 @@`
		`with open(tmpdir / "export.tar", "rb") as f:`
		`outfile, manifest_file = (builder._export_from_stream(f, close_stream=False))`

		`- check_call(["gzip", tmpdir / "export.tar"])`
		`-`
		`- builder.build_container(str(tmpdir / "export.tar.gz"))`
		`+ refname, oci_outdir, tarred_oci_outdir = builder.build_container(outfile)`

		`# builder.get_components() should not filter out any packages`
		`- check_get_components(builder, tmpdir, dedent("""\`
		`- true flatpak-runtime-config-0:29-5.x86_64`
		`- true glibc-0:2.32-4.fc33.x86_64`
		`- true libfoo-0:1.2.3-1.fc33.x86_64`
		`- """))`
		`+ check_get_components(builder, tmpdir, R(dedent("""\`
		`+ true flatpak-runtime-config-0:29-5.{rpm_arch}`
		`+ true glibc-0:2.32-4.fc33.{rpm_arch}`
		`+ true libfoo-0:1.2.3-1.fc33.{rpm_arch}`
		`+ """)))`
		`+`
		`+ check_exported_oci(oci_outdir, arch, R, runtime=True)`


		`def test_export_long_filenames(testapp_source, tmpdir):`

otaylor commented 2 years ago

This supercedes #15

We want to allow cross-building and generating a Flatpak OCI for an architecture other than the current one - to enable this, allow FlatpakBuilder(..., oci_arch=<other arch).

Because we need to maintain a mapping between OCI/RPM/Flatpak architecture names, new architectures will require an addition here.

Tests are extended to check the generated OCI content, and then updated to handle metadata changes based on architecture - looking at the individual commits will clarify what is going on.