From d03de37a60df35bea84c6129cd603a1099c9addb Mon Sep 17 00:00:00 2001 From: Simon Steinbeiss Date: Apr 29 2022 14:58:47 +0000 Subject: Initial blah --- diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3406ead --- /dev/null +++ b/Makefile @@ -0,0 +1,303 @@ +# +# Maintenance Helpers +# +# This makefile contains targets used for development, as well as helpers to +# aid automatization of maintenance. Unless a target is documented in +# `make help`, it is not supported and is only meant to be used by developers +# to aid their daily development work. +# +# All supported targets honor the `SRCDIR` variable to find the source-tree. +# For most unsupported targets, you are expected to have the source-tree as +# your working directory. To specify a different source-tree, simply override +# the variable via `SRCDIR=` on the commandline. While you can also +# override `BUILDDIR`, you are usually expected to have the build output +# directory as working directory. +# + +BUILDDIR ?= . +SRCDIR ?= . + +MKDIR ?= mkdir +PYTHON3 ?= python3 +RST2MAN ?= rst2man +TAR ?= tar +WGET ?= wget + +SHELL = /bin/bash + +# +# Automatic Variables +# +# This section contains a bunch of automatic variables used all over the place. +# They mostly try to fetch information from the repository sources to avoid +# hard-coding them in this makefile. +# +# Most of the variables here are pre-fetched so they will only ever be +# evaluated once. This, however, means they are always executed regardless of +# which target is run. +# +# VERSION: +# This evaluates the `version` field of `setup.py`. Therefore, it will +# be set to the latest version number of this repository without any +# prefix (just a plain number). +# +# COMMIT: +# This evaluates to the latest git commit sha. This will not work if +# the source is not a git checkout. Hence, this variable is not +# pre-fetched but evaluated at time of use. +# + +VERSION := $(shell (cd "$(SRCDIR)" && python3 setup.py --version)) +COMMIT = $(shell (cd "$(SRCDIR)" && git rev-parse HEAD)) + +# +# Generic Targets +# +# The following is a set of generic targets used across the makefile. The +# following targets are defined: +# +# help +# This target prints all supported targets. It is meant as +# documentation of targets we support and might use outside of this +# repository. +# This is also the default target. +# +# $(BUILDDIR)/ +# $(BUILDDIR)/%/ +# This target simply creates the specified directory. It is limited to +# the build-dir as a safety measure. Note that this requires you to use +# a trailing slash after the directory to not mix it up with regular +# files. Lastly, you mostly want this as order-only dependency, since +# timestamps on directories do not affect their content. +# +# FORCE +# Dummy target to force .PHONY behavior. This is required if .PHONY is +# not an option (e.g., due to implicit targets). +# + +.PHONY: help +help: + @echo "make [TARGETS...]" + @echo + @echo "This is the maintenance makefile of osbuild. The following" + @echo "targets are available:" + @echo + @echo " help: Print this usage information." + @echo " man: Generate all man-pages" + @echo + @echo " coverity-download: Force a new download of the coverity tool" + @echo " coverity-check: Run the coverity test suite" + @echo " coverity-submit: Run coverity and submit the results" + @echo + @echo " test-all: Run all tests" + @echo " test-data: Generate test data" + @echo " test-module: Run all module unit-tests" + @echo " test-run: Run all osbuild pipeline tests" + @echo " test-src: Run all osbuild source tests" + +$(BUILDDIR)/: + mkdir -p "$@" + +$(BUILDDIR)/%/: + mkdir -p "$@" + +FORCE: + +# +# Documentation +# +# The following targets build the included documentation. This includes the +# packaged man-pages, but also all other kinds of documentation that needs to +# be generated. Note that these targets are relied upon by automatic +# deployments to our website, as well as package manager scripts. +# + +MANPAGES_RST = $(wildcard $(SRCDIR)/docs/*.[0123456789].rst) +MANPAGES_TROFF = $(patsubst $(SRCDIR)/%.rst,$(BUILDDIR)/%,$(MANPAGES_RST)) + +$(MANPAGES_TROFF): $(BUILDDIR)/docs/%: $(SRCDIR)/docs/%.rst | $(BUILDDIR)/docs/ + $(RST2MAN) "$<" "$@" + +.PHONY: man +man: $(MANPAGES_TROFF) + +# +# Coverity +# +# Download the coverity analysis tool and run it on the repository, archive the +# analysis result and upload it to coverity. The target to do all of that is +# `coverity-submit`. +# +# Individual targets exist for the respective steps. +# +# Needs COVERITY_TOKEN and COVERITY_EMAIL to be set for downloading +# the analysis tool and submitting the final results. +# + +COVERITY_URL = https://scan.coverity.com/download/linux64 +COVERITY_TARFILE = coverity-tool.tar.gz + +COVERITY_BUILDDIR = $(BUILDDIR)/coverity +COVERITY_TOOLTAR = $(COVERITY_BUILDDIR)/$(COVERITY_TARFILE) +COVERITY_TOOLDIR = $(COVERITY_BUILDDIR)/cov-analysis-linux64 +COVERITY_ANALYSIS = $(COVERITY_BUILDDIR)/cov-analysis-osbuild.xz + +.PHONY: coverity-token +coverity-token: + $(if $(COVERITY_TOKEN),,$(error COVERITY_TOKEN must be set)) + +.PHONY: coverity-email +coverity-email: + $(if $(COVERITY_EMAIL),,$(error COVERITY_EMAIL must be set)) + +.PHONY: coverity-download +coverity-download: | coverity-token $(COVERITY_BUILDDIR)/ + @$(RM) -rf "$(COVERITY_TOOLDIR)" "$(COVERITY_TOOLTAR)" + @echo "Downloading $(COVERITY_TARFILE) from $(COVERITY_URL)..." + @$(WGET) -q "$(COVERITY_URL)" --post-data "project=osbuild&token=$(COVERITY_TOKEN)" -O "$(COVERITY_TOOLTAR)" + @echo "Extracting $(COVERITY_TARFILE)..." + @$(MKDIR) -p "$(COVERITY_TOOLDIR)" + @$(TAR) -xzf "$(COVERITY_TOOLTAR)" --strip 1 -C "$(COVERITY_TOOLDIR)" + +$(COVERITY_TOOLTAR): | $(COVERITY_BUILDDIR)/ + @$(MAKE) --no-print-directory coverity-download + +.PHONY: coverity-check +coverity-check: $(COVERITY_TOOLTAR) + @echo "Running coverity suite..." + @$(COVERITY_TOOLDIR)/bin/cov-build \ + --dir "$(COVERITY_BUILDDIR)/cov-int" \ + --no-command \ + --fs-capture-search "$(SRCDIR)" \ + --fs-capture-search-exclude-regex "$(COVERITY_BUILDDIR)" + @echo "Compressing analysis results..." + @$(TAR) -caf "$(COVERITY_ANALYSIS)" -C "$(COVERITY_BUILDDIR)" "cov-int" + +$(COVERITY_ANALYSIS): | $(COVERITY_BUILDDIR)/ + @$(MAKE) --no-print-directory coverity-check + +.PHONY: coverity-submit +coverity-submit: $(COVERITY_ANALYSIS) | coverity-email coverity-token + @echo "Submitting $(COVERITY_ANALYSIS)..." + @curl --form "token=$(COVERITY_TOKEN)" \ + --form "email=$(COVERITY_EMAIL)" \ + --form "file=@$(COVERITY_ANALYSIS)" \ + --form "version=main" \ + --form "description=$$(git describe)" \ + https://scan.coverity.com/builds?project=osbuild + +.PHONY: coverity-clean +coverity-clean: + @$(RM) -rfv "$(COVERITY_BUILDDIR)/cov-int" "$(COVERITY_ANALYSIS)" + +.PHONY: coverity-clean-all +coverity-clean-all: coverity-clean + @$(RM) -rfv "$(COVERITY_BUILDDIR)" + +# +# Test Suite +# +# We use the python `unittest` module for all tests. All the test-sources are +# located in the `./test/` top-level directory, with `./test/mod/` for module +# unittests, `./test/run/` for osbuild pipeline runtime tests, and `./test/src/` +# for linters and other tests on the source code. +# + +TEST_MANIFESTS_MPP = $(filter-out $(SRCDIR)/test/data/manifests/fedora-build.mpp.json, \ + $(wildcard $(SRCDIR)/test/data/manifests/*.mpp.json)) \ + $(wildcard $(SRCDIR)/test/data/stages/*/*.mpp.json) +TEST_MANIFESTS_GEN = $(TEST_MANIFESTS_MPP:%.mpp.json=%.json) + +.PHONY: $(TEST_MANIFESTS_GEN) +$(TEST_MANIFESTS_GEN): %.json: %.mpp.json + $(SRCDIR)/tools/osbuild-mpp -I "$(SRCDIR)/test/data/manifests" "$<" "$@" + +$(SRCDIR)/test/data/manifests/f34-base.json: $(SRCDIR)/test/data/manifests/f34-build.json +$(SRCDIR)/test/data/manifests/fedora-boot.json: $(SRCDIR)/test/data/manifests/f34-build.json +$(SRCDIR)/test/data/manifests/filesystem.json: $(SRCDIR)/test/data/manifests/f34-build.json +$(SRCDIR)/test/data/manifests/fedora-container.json: $(SRCDIR)/test/data/manifests/f34-build-v2.json +$(SRCDIR)/test/data/manifests/fedora-ostree-container.json: $(SRCDIR)/test/data/manifests/f34-build-v2.json + +.PHONY: test-data +test-data: $(TEST_MANIFESTS_GEN) + +.PHONY: test-module +test-module: + @$(PYTHON3) -m pytest \ + $(SRCDIR)/test/mod \ + --rootdir=$(SRCDIR) \ + -v + +.PHONY: test-run +test-run: + @[[ $${EUID} -eq 0 ]] || (echo "Error: Root privileges required!"; exit 1) + @$(PYTHON3) -m pytest \ + $(SRCDIR)/test/run \ + --rootdir=$(SRCDIR) \ + -v + +.PHONY: test-src +test-src: + @$(PYTHON3) -m pytest \ + $(SRCDIR)/test/src \ + --rootdir=$(SRCDIR) \ + -v + +.PHONY: test-all +test-all: + @$(PYTHON3) -m pytest \ + $(SRCDIR)/test \ + --rootdir=$(SRCDIR) \ + -v + +# +# Building packages +# +# The following rules build osbuild packages from the current HEAD commit, +# based on the spec file in this directory. The resulting packages have the +# commit hash in their version, so that they don't get overwritten when calling +# `make rpm` again after switching to another branch. +# +# All resulting files (spec files, source rpms, rpms) are written into +# ./rpmbuild, using rpmbuild's usual directory structure. +# + +.PHONY: git-diff-check +git-diff-check: + @git diff --exit-code + @git diff --cached --exit-code + +RPM_SPECFILE=rpmbuild/SPECS/osbuild-$(COMMIT).spec +RPM_TARBALL=rpmbuild/SOURCES/osbuild-$(COMMIT).tar.gz + +$(RPM_SPECFILE): + mkdir -p $(CURDIR)/rpmbuild/SPECS + (echo "%global commit $(COMMIT)"; git show HEAD:osbuild.spec) > $(RPM_SPECFILE) + +$(RPM_TARBALL): + mkdir -p $(CURDIR)/rpmbuild/SOURCES + git archive --prefix=osbuild-$(COMMIT)/ --format=tar.gz HEAD > $(RPM_TARBALL) + +.PHONY: srpm +srpm: git-diff-check $(RPM_SPECFILE) $(RPM_TARBALL) + rpmbuild -bs \ + --define "_topdir $(CURDIR)/rpmbuild" \ + $(RPM_SPECFILE) + +.PHONY: rpm +rpm: git-diff-check $(RPM_SPECFILE) $(RPM_TARBALL) + rpmbuild -bb \ + --define "_topdir $(CURDIR)/rpmbuild" \ + $(RPM_SPECFILE) + +# +# Releasing +# + +NEXT_VERSION := $(shell expr "$(VERSION)" + 1) + +.PHONY: bump-version +bump-version: + sed -i "s|Version:\(\s*\)$(VERSION)|Version:\1$(NEXT_VERSION)|" osbuild.spec + sed -i "s|Release:\(\s*\)[[:digit:]]\+|Release:\11|" osbuild.spec + sed -i "s|version=\"$(VERSION)\"|version=\"$(NEXT_VERSION)\"|" setup.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..c69f3b7 --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +OSBuild +======= + +Build-Pipelines for Operating System Artifacts + +OSBuild is a pipeline-based build system for operating system artifacts. It +defines a universal pipeline description and a build system to execute them, +producing artifacts like operating system images, working towards an image +build pipeline that is more comprehensible, reproducible, and extendable. + +See the `osbuild(1)` man-page for details on how to run osbuild, the definition +of the pipeline description, and more. + +### Project + + * **Website**: + * **Bug Tracker**: + * **IRC**: #osbuild on [Libera.Chat](https://libera.chat/) + * **Changelog**: + +#### Contributing + +Please refer to the [developer guide](https://www.osbuild.org/guides/developer-guide/developer-guide.html) to learn about our workflow, code style and more. + +### Requirements + +The requirements for this project are: + + * `bubblewrap >= 0.4.0` + * `python >= 3.7` + +Additionally, the built-in stages require: + + * `bash >= 5.0` + * `coreutils >= 8.31` + * `curl >= 7.68` + * `qemu-img >= 4.2.0` + * `rpm >= 4.15` + * `tar >= 1.32` + * `util-linux >= 235` + * `skopeo` + +At build-time, the following software is required: + + * `python-docutils >= 0.13` + * `pkg-config >= 0.29` + +Testing requires additional software: + + * `pytest` + +### Install + +Installing `osbuild` requires to not only install the `osbuild` module, but also +additional artifacts such as tools (i.e: `osbuild-mpp`) sources, stages, schemas +and SELinux policies. + +For this reason, doing an installation from source is not trivial and the easier +way to install it is to create the set of RPMs that contain all these components. + +This can be done with the `rpm` make target, i.e: + +```sh +make rpm +``` + +A set of RPMs will be created in the `./rpmbuild/RPMS/noarch/` directory and can +be installed in the system using the distribution package manager, i.e: + +```sh +sudo dnf install ./rpmbuild/RPMS/noarch/*.rpm +``` + +### Repository: + + - **web**: + - **https**: `https://github.com/osbuild/osbuild.git` + - **ssh**: `git@github.com:osbuild/osbuild.git` + +### License: + + - **Apache-2.0** + - See LICENSE file for details. diff --git a/__pycache__/github.cpython-39.pyc b/__pycache__/github.cpython-39.pyc new file mode 100644 index 0000000..0245dd7 Binary files /dev/null and b/__pycache__/github.cpython-39.pyc differ diff --git a/assemblers/org.osbuild.error b/assemblers/org.osbuild.error new file mode 100755 index 0000000..8091d69 --- /dev/null +++ b/assemblers/org.osbuild.error @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +""" +Return an error + +Error assembler. Return the given error. Very much like the error stage this +is useful for testing, debugging, and wasting time. +""" + + +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"properties": { + "returncode": { + "description": "What to return code to use", + "type": "number", + "default": 255 + } +} +""" + + +def main(options): + errno = options.get("returncode", 255) + print(f"Error assembler will now return error: {errno}") + return errno + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args.get("options", {})) + sys.exit(r) diff --git a/assemblers/org.osbuild.noop b/assemblers/org.osbuild.noop new file mode 100755 index 0000000..94b85da --- /dev/null +++ b/assemblers/org.osbuild.noop @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +""" +No-op assembler + +No-op assembler. Produces no output, just prints a JSON dump of its options +and then exits. +""" + + +import json +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false +""" + + +def main(_tree, _output_dir, options): + print("Not doing anything with these options:", json.dumps(options)) + + +if __name__ == '__main__': + args = osbuild.api.arguments() + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args.get("options", {})) + sys.exit(r) diff --git a/assemblers/org.osbuild.oci-archive b/assemblers/org.osbuild.oci-archive new file mode 100755 index 0000000..1662587 --- /dev/null +++ b/assemblers/org.osbuild.oci-archive @@ -0,0 +1,278 @@ +#!/usr/bin/python3 +""" +Assemble an OCI image archive + +Assemble an Open Container Initiative[1] image[2] archive, i.e. a +tarball whose contents is in the OCI image layout. + +Currently the only required options are `filename` and `architecture`. +The execution parameters for the image, which then should form the base +for the container, can be given via `config`. They have the same format +as the `config` option for the "OCI Image Configuration" (see [2]), +except those that map to the "Go type map[string]struct{}", which are +represented as array of strings. + +The final resulting tarball, aka a "oci-archive", can be imported via +podman[3] with `podman pull oci-archive:`. + +[1] https://www.opencontainers.org/ +[2] https://github.com/opencontainers/image-spec/ +[3] https://podman.io/ +""" + + +import datetime +import json +import os +import subprocess +import sys +import tempfile + +import osbuild.api + + +DEFAULT_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + + +SCHEMA = """ +"additionalProperties": false, +"required": ["architecture", "filename"], +"properties": { + "architecture": { + "description": "The CPU architecture of the image", + "type": "string" + }, + "filename": { + "description": "Resulting image filename", + "type": "string" + }, + "config": { + "description": "The execution parameters", + "type": "object", + "additionalProperties": false, + "properties": { + "Cmd": { + "type": "array", + "default": ["sh"], + "items": { + "type": "string" + } + }, + "Env": { + "type": "array", + "default": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], + "items": { + "type": "string" + } + }, + "ExposedPorts": { + "type": "array", + "items": { + "type": "string" + } + }, + "User": { + "type": "string" + }, + "Labels": { + "type": "object", + "additionalProperties": true + }, + "StopSiganl": { + "type": "string" + }, + "Volumes": { + "type": "array", + "items": { + "type": "string" + } + }, + "WorkingDir": { + "type": "string" + } + } + } +} +""" + + +MEDIA_TYPES = { + "layer": "application/vnd.oci.image.layer.v1.tar", + "manifest": "application/vnd.oci.image.manifest.v1+json", + "config": "application/vnd.oci.image.config.v1+json" +} + + +def sha256sum(path: str) -> str: + ret = subprocess.run(["sha256sum", path], + stdout=subprocess.PIPE, + encoding="utf-8", + check=True) + + return ret.stdout.strip().split(" ")[0] + + +def blobs_add_file(blobs: str, path: str, mtype: str): + digest = sha256sum(path) + size = os.stat(path).st_size + + os.rename(path, os.path.join(blobs, digest)) + info = { + "digest": "sha256:" + digest, + "size": size, + "mediaType": MEDIA_TYPES[mtype] + } + + print(f"blobs: +{mtype} ({size}, {digest})") + return info + + +def blobs_add_json(blobs: str, js: str, mtype: str): + js_file = os.path.join(blobs, "temporary.js") + with open(js_file, "w") as f: + json.dump(js, f) + + return blobs_add_file(blobs, js_file, mtype) + + +def blobs_add_layer(blobs: str, tree: str): + compression = "gzip" + + layer_file = os.path.join(blobs, "layer.tar") + + command = [ + "tar", + "--no-selinux", + "--acls", + "--xattrs", + "-cf", layer_file, + "-C", tree, + ] + os.listdir(tree) + + print("creating layer") + subprocess.run(command, + stdout=subprocess.DEVNULL, + check=True) + + digest = "sha256:" + sha256sum(layer_file) + + print("compressing layer") + suffix = ".compressed" + subprocess.run([compression, + "-S", suffix, + layer_file], + stdout=subprocess.DEVNULL, + check=True) + + layer_file += suffix + + info = blobs_add_file(blobs, layer_file, "layer") + info["mediaType"] += "+" + compression + + return digest, info + + +def config_from_options(options): + command = options.get("Cmd", ["sh"]) + env = options.get("Env", ["PATH=" + DEFAULT_PATH]) + + config = { + "Env": env, + "Cmd": command + } + + for name in ["User", "Labels", "StopSignal", "WorkingDir"]: + item = options.get(name) + if item: + config[name] = item + + for name in ["ExposedPorts", "Volumes"]: + item = options.get(name) + if item: + config[name] = {x: {} for x in item} + + print(config) + return config + + +def create_oci_dir(tree, output_dir, options): + architecture = options["architecture"] + + config = { + "created": datetime.datetime.utcnow().isoformat() + "Z", + "architecture": architecture, + "os": "linux", + "config": config_from_options(options["config"]), + "rootfs": { + "type": "layers", + "diff_ids": [] + } + } + + manifest = { + "schemaVersion": 2, + "config": None, + "layers": [] + } + + index = { + "schemaVersion": 2, + "manifests": [] + } + + blobs = os.path.join(output_dir, "blobs", "sha256") + os.makedirs(blobs) + + ## layers / rootfs + + digest, info = blobs_add_layer(blobs, tree) + + config["rootfs"]["diff_ids"] = [digest] + manifest["layers"].append(info) + + ## write config + info = blobs_add_json(blobs, config, "config") + manifest["config"] = info + + # manifest + info = blobs_add_json(blobs, manifest, "manifest") + index["manifests"].append(info) + + # index + print("writing index") + with open(os.path.join(output_dir, "index.json"), "w") as f: + json.dump(index, f) + + # oci-layout tag + with open(os.path.join(output_dir, "oci-layout"), "w") as f: + json.dump({"imageLayoutVersion": "1.0.0"}, f) + + +def main(tree, output_dir, options): + filename = options["filename"] + + with tempfile.TemporaryDirectory(dir=output_dir) as tmpdir: + workdir = os.path.join(tmpdir, "output") + os.makedirs(workdir) + + create_oci_dir(tree, workdir, options) + + command = [ + "tar", + "--remove-files", + "-cf", os.path.join(output_dir, filename), + f"--directory={workdir}", + ] + os.listdir(workdir) + + print("creating final archive") + subprocess.run(command, + stdout=subprocess.DEVNULL, + check=True) + + +if __name__ == '__main__': + args = osbuild.api.arguments() + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"]) + sys.exit(r) diff --git a/assemblers/org.osbuild.ostree.commit b/assemblers/org.osbuild.ostree.commit new file mode 100755 index 0000000..1b5c9fb --- /dev/null +++ b/assemblers/org.osbuild.ostree.commit @@ -0,0 +1,191 @@ +#!/usr/bin/python3 +""" +Assemble a file system tree into a ostree commit + +Takes a file system tree that is already conforming to the ostree +system layout[1] and commits it to an archive repository. + +The repository is located at the `/repo` directory and additional +metadata is stored in `/compose.json` which contains the commit +compose information. + +Alternatively, if the `tar` option is supplied, the repository and +the `compose.json` will be archived in a file named via the +`tar.filename` option. This supports auto-compression via the file +extension (see the tar man page). Requires the `tar` command to be +in the build root. + +[1] https://ostree.readthedocs.io/en/stable/manual/adapting-existing/ +""" + + +import json +import os +import subprocess +import sys +import tempfile + +from osbuild import api +from osbuild.util import ostree + + +SCHEMA = """ +"additionalProperties": false, +"required": ["ref"], +"properties": { + "ref": { + "description": "OStree ref to create for the commit", + "type": "string", + "default": "" + }, + "os_version": { + "description": "Set the version of the OS as commit metadata", + "type": "string" + }, + "tmp-is-dir": { + "description": "Create a regular directory for /tmp", + "type": "boolean", + "default": true + }, + "parent": { + "description": "commit id of the parent commit", + "type": "string" + }, + "tar": { + "description": "Emit a tarball as the result", + "type": "object", + "additionalProperties": false, + "required": ["filename"], + "properties": { + "filename": { + "description": "File-name of the tarball to create. Compression is determined by the extension.", + "type": "string" + } + } + } +} +""" + + +TOPLEVEL_DIRS = ["dev", "proc", "run", "sys", "sysroot", "var"] +TOPLEVEL_LINKS = { + "home": "var/home", + "media": "run/media", + "mnt": "var/mnt", + "opt": "var/opt", + "ostree": "sysroot/ostree", + "root": "var/roothome", + "srv": "var/srv", +} + + +def copy(name, source, dest): + subprocess.run(["cp", "--reflink=auto", "-a", + os.path.join(source, name), + "-t", os.path.join(dest)], + check=True) + + +def init_rootfs(root, tmp_is_dir): + """Initialize a pristine root file-system""" + + fd = os.open(root, os.O_DIRECTORY) + + os.fchmod(fd, 0o755) + + for d in TOPLEVEL_DIRS: + os.mkdir(d, mode=0o755, dir_fd=fd) + os.chmod(d, mode=0o755, dir_fd=fd) + + for l, t in TOPLEVEL_LINKS.items(): + # /l -> t + os.symlink(t, l, dir_fd=fd) + + if tmp_is_dir: + os.mkdir("tmp", mode=0o1777, dir_fd=fd) + os.chmod("tmp", mode=0o1777, dir_fd=fd) + else: + os.symlink("tmp", "sysroot/tmp", dir_fd=fd) + + +def main(tree, output_dir, options, meta): + ref = options["ref"] + os_version = options.get("os_version", None) + tmp_is_dir = options.get("tmp-is-dir", True) + parent = options.get("parent", None) + tar = options.get("tar", None) + + with tempfile.TemporaryDirectory(dir=output_dir) as root: + print("Initializing root filesystem", file=sys.stderr) + init_rootfs(root, tmp_is_dir) + + print("Copying data", file=sys.stderr) + copy("usr", tree, root) + copy("boot", tree, root) + copy("var", tree, root) + + for name in ["bin", "lib", "lib32", "lib64", "sbin"]: + if os.path.lexists(f"{tree}/{name}"): + copy(name, tree, root) + + repo = os.path.join(output_dir, "repo") + + subprocess.run(["ostree", + "init", + "--mode=archive", + f"--repo={repo}"], + stdout=sys.stderr, + check=True) + + treefile = ostree.Treefile() + treefile["ref"] = ref + + argv = ["rpm-ostree", "compose", "commit"] + argv += [f"--repo={repo}"] + + if parent: + argv += [f"--parent={parent}"] + + if os_version: + argv += [ + f"--add-metadata-string=version={os_version}", + ] + + argv += [ + f"--add-metadata-string=rpmostree.inputhash={meta['id']}", + f"--write-composejson-to={output_dir}/compose.json" + ] + + with treefile.as_tmp_file() as path: + argv += [path, root] + + subprocess.run(argv, + stdout=sys.stderr, + check=True) + + with open(os.path.join(output_dir, "compose.json"), "r") as f: + compose = json.load(f) + + api.metadata({"compose": compose}) + + if tar: + filename = tar["filename"] + command = [ + "tar", + "--remove-files", + "--auto-compress", + "-cf", os.path.join(output_dir, filename), + "-C", output_dir, + "repo", "compose.json" + ] + subprocess.run(command, + stdout=sys.stderr, + check=True) + + +if __name__ == '__main__': + args = api.arguments() + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"], args["meta"]) + sys.exit(r) diff --git a/assemblers/org.osbuild.qemu b/assemblers/org.osbuild.qemu new file mode 100755 index 0000000..ac536b4 --- /dev/null +++ b/assemblers/org.osbuild.qemu @@ -0,0 +1,722 @@ +#!/usr/bin/python3 +""" +Assemble a bootable partitioned disk image with qemu-img + +Assemble a bootable partitioned disk image using `qemu-img`. + +Creates a sparse partitioned disk image of type `pttype` of a given `size`, +with a partition table according to `partitions` or a MBR partitioned disk +having a single bootable partition containing the root filesystem if the +`pttype` property is absent. + +If the partition type is MBR it installs GRUB2 (using the buildhost's +`/usr/lib/grub/i386-pc/boot.img` etc.) as the bootloader. + +Copies the tree contents into the root filesystem and then converts the raw +sparse image into the format requested with the `fmt` option. + +Buildhost commands used: `truncate`, `mount`, `umount`, `sfdisk`, +`grub2-mkimage`, `mkfs.ext4` or `mkfs.xfs`, `qemu-img`. +""" + + +import contextlib +import json +import os +import shutil +import struct +import subprocess +import sys +import tempfile +from typing import List, BinaryIO + +import osbuild.api +import osbuild.remoteloop as remoteloop + + +SCHEMA = """ +"additionalProperties": false, +"required": ["format", "filename", "ptuuid", "size"], +"oneOf": [{ + "required": ["root_fs_uuid"] +},{ + "required": ["pttype", "partitions"] +}], +"properties": { + "bootloader": { + "description": "Options specific to the bootloader", + "type": "object", + "properties": { + "type": { + "description": "What bootloader to install", + "type": "string", + "enum": ["grub2", "zipl"] + } + } + }, + "format": { + "description": "Image file format to use", + "type": "string", + "enum": ["raw", "raw.xz", "qcow2", "vdi", "vmdk", "vpc", "vhdx"] + }, + "qcow2_compat": { + "description": "The qcow2-compatibility-version to use", + "type": "string" + }, + "filename": { + "description": "Image filename", + "type": "string" + }, + "partitions": { + "description": "Partition layout ", + "type": "array", + "items": { + "description": "Description of one partition", + "type": "object", + "properties": { + "bootable": { + "description": "Mark the partition as bootable (dos)", + "type": "boolean" + }, + "name": { + "description": "The partition name (GPT)", + "type": "string" + }, + "size": { + "description": "The size of this partition", + "type": "integer" + }, + "start": { + "description": "The start offset of this partition", + "type": "integer" + }, + "type": { + "description": "The partition type (UUID or identifier)", + "type": "string" + }, + "uuid": { + "description": "UUID of the partition (GPT)", + "type": "string" + }, + "filesystem": { + "description": "Description of the filesystem", + "type": "object", + "required": ["mountpoint", "type", "uuid"], + "properties": { + "label": { + "description": "Label for the filesystem", + "type": "string" + }, + "mountpoint": { + "description": "Where to mount the partition", + "type": "string" + }, + "type": { + "description": "Type of the filesystem", + "type": "string", + "enum": ["ext4", "xfs", "vfat", "btrfs"] + }, + "uuid": { + "description": "UUID for the filesystem", + "type": "string" + } + } + } + } + } + }, + "ptuuid": { + "description": "UUID for the disk image's partition table", + "type": "string" + }, + "pttype": { + "description": "The type of the partition table", + "type": "string", + "enum": ["mbr", "dos", "gpt"] + }, + "root_fs_uuid": { + "description": "UUID for the root filesystem", + "type": "string" + }, + "size": { + "description": "Virtual disk size", + "type": "integer" + }, + "root_fs_type": { + "description": "Type of the root filesystem", + "type": "string", + "enum": ["ext4", "xfs", "btrfs"], + "default": "ext4" + } +} +""" + + +@contextlib.contextmanager +def mount(source, dest): + subprocess.run(["mount", source, dest], check=True) + try: + yield dest + finally: + subprocess.run(["umount", "-R", dest], check=True) + + +def mkfs_ext4(device, uuid, label): + opts = [] + if label: + opts = ["-L", label] + subprocess.run(["mkfs.ext4", "-U", uuid] + opts + [device], + input="y", encoding='utf-8', check=True) + + +def mkfs_xfs(device, uuid, label): + opts = [] + if label: + opts = ["-L", label] + subprocess.run(["mkfs.xfs", "-m", f"uuid={uuid}"] + opts + [device], + encoding='utf-8', check=True) + + +def mkfs_btrfs(device, uuid, label): + opts = [] + if label: + opts = ["-L", label] + subprocess.run(["mkfs.btrfs", "-U", uuid] + opts + [device], + encoding='utf-8', check=True) + + +def mkfs_vfat(device, uuid, label): + volid = uuid.replace('-', '') + opts = [] + if label: + opts = ["-n", label] + subprocess.run(["mkfs.vfat", "-i", volid] + opts + [device], encoding='utf-8', check=True) + + +class Filesystem: + def __init__(self, + fstype: str, + uuid: str, + mountpoint: str, + label: str = None): + self.type = fstype + self.uuid = uuid + self.mountpoint = mountpoint + self.label = label + + def make_at(self, device: str): + fs_type = self.type + if fs_type == "ext4": + maker = mkfs_ext4 + elif fs_type == "xfs": + maker = mkfs_xfs + elif fs_type == "vfat": + maker = mkfs_vfat + elif fs_type == "btrfs": + maker = mkfs_btrfs + else: + raise ValueError(f"Unknown filesystem type '{fs_type}'") + maker(device, self.uuid, self.label) + + +class Partition: + def __init__(self, + pttype: str = None, + start: int = None, + size: int = None, + bootable: bool = False, + name: str = None, + uuid: str = None, + filesystem: Filesystem = None): + self.type = pttype + self.start = start + self.size = size + self.bootable = bootable + self.name = name + self.uuid = uuid + self.filesystem = filesystem + self.index = None + + @property + def start_in_bytes(self): + return (self.start or 0) * 512 + + @property + def size_in_bytes(self): + return (self.size or 0) * 512 + + @property + def mountpoint(self): + if self.filesystem is None: + return None + return self.filesystem.mountpoint + + @property + def fs_type(self): + if self.filesystem is None: + return None + return self.filesystem.type + + @property + def fs_uuid(self): + if self.filesystem is None: + return None + return self.filesystem.uuid + + +class PartitionTable: + def __init__(self, label, uuid, partitions): + self.label = label + self.uuid = uuid + self.partitions = partitions or [] + + def __getitem__(self, key) -> Partition: + return self.partitions[key] + + def partitions_with_filesystems(self) -> List[Partition]: + """Return partitions with filesystems sorted by hierarchy""" + def mountpoint_len(p): + return len(p.mountpoint) + parts_fs = filter(lambda p: p.filesystem is not None, self.partitions) + return sorted(parts_fs, key=mountpoint_len) + + def partition_containing_root(self) -> Partition: + """Return the partition containing the root filesystem""" + for p in self.partitions: + if p.mountpoint and p.mountpoint == "/": + return p + return None + + def partition_containing_boot(self) -> Partition: + """Return the partition containing /boot""" + for p in self.partitions_with_filesystems(): + if p.mountpoint == "/boot": + return p + # fallback to the root partition + return self.partition_containing_root() + + def find_prep_partition(self) -> Partition: + """Find the PReP partition'""" + if self.label == "dos": + prep_type = "41" + elif self.label == "gpt": + prep_type = "9E1A2D38-C612-4316-AA26-8B49521E5A8B" + + for part in self.partitions: + if part.type.upper() == prep_type: + return part + return None + + def find_bios_boot_partition(self) -> Partition: + """Find the BIOS-boot Partition""" + bb_type = "21686148-6449-6E6F-744E-656564454649" + for part in self.partitions: + if part.type.upper() == bb_type: + return part + return None + + def write_to(self, target, sync=True): + """Write the partition table to disk""" + # generate the command for sfdisk to create the table + command = f"label: {self.label}\nlabel-id: {self.uuid}" + for partition in self.partitions: + fields = [] + for field in ["start", "size", "type", "name", "uuid"]: + value = getattr(partition, field) + if value: + fields += [f'{field}="{value}"'] + if partition.bootable: + fields += ["bootable"] + command += "\n" + ", ".join(fields) + + subprocess.run(["sfdisk", "-q", target], + input=command, + encoding='utf-8', + check=True) + + if sync: + self.update_from(target) + + def update_from(self, target): + """Update and fill in missing information from disk""" + r = subprocess.run(["sfdisk", "--json", target], + stdout=subprocess.PIPE, + encoding='utf-8', + check=True) + disk_table = json.loads(r.stdout)["partitiontable"] + disk_parts = disk_table["partitions"] + + assert len(disk_parts) == len(self.partitions) + for i, part in enumerate(self.partitions): + part.index = i + part.start = disk_parts[i]["start"] + part.size = disk_parts[i]["size"] + part.type = disk_parts[i].get("type") + part.name = disk_parts[i].get("name") + + +def filesystem_from_json(js) -> Filesystem: + return Filesystem(js["type"], js["uuid"], js["mountpoint"], js.get("label")) + + +def partition_from_json(js) -> Partition: + p = Partition(pttype=js.get("type"), + start=js.get("start"), + size=js.get("size"), + bootable=js.get("bootable"), + name=js.get("name"), + uuid=js.get("uuid")) + fs = js.get("filesystem") + if fs: + p.filesystem = filesystem_from_json(fs) + return p + + +def partition_table_from_options(options) -> PartitionTable: + ptuuid = options["ptuuid"] + pttype = options.get("pttype", "dos") + partitions = options.get("partitions") + + if pttype == "mbr": + pttype = "dos" + + if partitions is None: + # legacy mode, create a correct + root_fs_uuid = options["root_fs_uuid"] + root_fs_type = options.get("root_fs_type", "ext4") + partitions = [{ + "bootable": True, + "type": "83", + "filesystem": { + "type": root_fs_type, + "uuid": root_fs_uuid, + "mountpoint": "/" + } + }] + parts = [partition_from_json(p) for p in partitions] + return PartitionTable(pttype, ptuuid, parts) + + +def grub2_write_boot_image(boot_f: BinaryIO, + image_f: BinaryIO, + core_location: int): + """Write the boot image (grub2 stage 1) to the MBR""" + + # The boot.img file is 512 bytes, but we must only copy the first 440 + # bytes, as these contain the bootstrapping code. The rest of the + # first sector contains the partition table, and must not be + # overwritten. + image_f.seek(0) + image_f.write(boot_f.read(440)) + + # Additionally, write the location (in sectors) of + # the grub core image, into the boot image, so the + # latter can find the former. To exact location is + # taken from grub2's "boot.S": + # GRUB_BOOT_MACHINE_KERNEL_SECTOR 0x5c (= 92) + image_f.seek(0x5c) + image_f.write(struct.pack(" -1: + os.close(dir_fd) + + res = { + "path": self.lo.devname, + "node": { + "major": self.lo.LOOP_MAJOR, + "minor": self.lo.minor, + } + } + + return res + + def close(self): + # Calling `close` is valid on closed + # `LoopControl` and `Loop` objects + self.ctl.close() + + if self.lo: + # Flush the buffer cache of the loop device. This + # seems to be required when clearing the fd of the + # loop device (as of kernel 5.13.8) or otherwise + # it leads to data loss. + self.lo.flush_buf() + + # clear the fd. Since it might not immediately be + # cleared (due to a race with udev or some other + # process still having a reference to the loop dev) + # we give it some time and wait for the clearing + self.lo.clear_fd_wait(self.fd, 30) + self.lo.close() + self.lo = None + + if self.fd is not None: + fd = self.fd + self.fd = None + try: + os.fsync(fd) + finally: + os.close(fd) + + +def main(): + service = LoopbackService.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/devices/org.osbuild.luks2 b/devices/org.osbuild.luks2 new file mode 100755 index 0000000..95638e8 --- /dev/null +++ b/devices/org.osbuild.luks2 @@ -0,0 +1,157 @@ +#!/usr/bin/python3 +""" +Host service for Linux Unified Key Setup (LUKS, format 2) devices. + +This will open a LUKS container, given the path of the parent +device and the corresponding passphrase. + +NB: This will use the custom osbuild udev rule inhibitor mechanism +to suppress certain udev rules. See `osbuil.util.udev.UdevInhibitor` +for details. + +Host commands used: `cryptsetup`, `dmsetup` +""" + +import argparse +import contextlib +import os +import stat +import subprocess +import sys +import uuid + +from typing import Dict + +from osbuild import devices +from osbuild.util.udev import UdevInhibitor + + +SCHEMA = """ +"additionalProperties": false, +"required": ["passphrase"], +"properties": { + "passphrase": { + "description": "Passphrase to use", + "default": "", + "type": "string" + } +} + +""" + + +class CryptDeviceService(devices.DeviceService): + + def __init__(self, args: argparse.Namespace): + super().__init__(args) + self.devname = None + self.lock = None + self.check = False + + def dminfo(self, name=None): + """Return the major, minor and open count for the device""" + res = subprocess.run(["dmsetup", "info", "-c", + "-o", "major,minor,open", + "--noheadings", + "--separator", ":", + name or self.devname], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="UTF-8") + + if res.returncode != 0: + data = res.stdout.strip() + msg = f"Failed to find the device node: {data}" + raise RuntimeError(msg) + + data = res.stdout.strip() + data = list(map(int, data.split(":"))) + assert len(data) == 3 + major, minor, count = data[0], data[1], data[2] + return major, minor, count + + def open_count(self, name=None): + count = 0 + with contextlib.suppress(RuntimeError): + _, _, count = self.dminfo(name) + return count + + def open(self, devpath: str, parent: str, tree: str, options: Dict): + passphrase = options.get("passphrase", "") + + parent_dev = os.path.join("/dev", parent) + + # Generate a random name for it, since this is a temporary name + # that is not store in the device at all + devname = "osbuild-luks-" + str(uuid.uuid4()) + self.devname = devname + + # This employs the custom osbuild udev rule inhibitor mechanism + self.lock = UdevInhibitor.for_dm_name(devname) + + # Make sure the logical volume is activated + res = subprocess.run(["cryptsetup", "-q", "open", parent_dev, devname], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + input=passphrase, + encoding="UTF-8") + + if res.returncode != 0: + data = res.stdout.strip() + msg = f"Failed to open crypt device: {data}" + raise RuntimeError(msg) + + print(f"opened as {devname}") + + # Now that device is successfully opened, we check on close + self.check = True + + # Now that the crypt device is open, find its major/minor numbers + major, minor, _ = self.dminfo() + + subpath = os.path.join("mapper", devname) + fullpath = os.path.join(devpath, subpath) + os.makedirs(os.path.join(devpath, "mapper"), exist_ok=True) + os.mknod(fullpath, 0o666 | stat.S_IFBLK, os.makedev(major, minor)) + + data = { + "path": subpath, + "name": devname, + "node": { + "major": major, + "minor": minor + } + } + return data + + def close(self): + if not self.devname: + return + + _, _, opencount = self.dminfo() + print(f"closing (opencount: {opencount})") + + self.lock.release() + self.lock = None + + name = self.devname + self.devname = None + + # finally close the device + res = subprocess.run(["cryptsetup", "-q", "close", name], + check=self.check, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="UTF-8") + + +def main(): + service = CryptDeviceService.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + r = main() + sys.exit(r) diff --git a/devices/org.osbuild.lvm2.lv b/devices/org.osbuild.lvm2.lv new file mode 100755 index 0000000..af8eeda --- /dev/null +++ b/devices/org.osbuild.lvm2.lv @@ -0,0 +1,192 @@ +#!/usr/bin/python3 +""" +Host service for providing access to LVM2 logical volumes + +This host service can be used to activate logical volumes +so that they can be accessed from osbuild stages. +The parent volume group is identified via the physical +device which must be passed to this service as parent. + +Example usage, where `lvm` here is the lvm partition and +`root` then the logical volume named `root`: +``` +"lvm": { + "type": "org.osbuild.loopback", + "options": { + "filename": "disk.img", + "start": ..., +}, +"root": { + "type": "org.osbuild.lvm2.lv", + "parent": "lvm", + "options": { + "volume": "root" + } +} +``` + +Required host tools: lvchange, pvdisplay, lvdisplay +""" + +import json +import os +import stat +import subprocess +import sys +import time + +from typing import Dict, Tuple + +from osbuild import devices + + +SCHEMA = """ +"additionalProperties": false, +"required": ["volume"], +"properties": { + "volume": { + "type": "string", + "description": "Logical volume to active" + } +} +""" + + +class LVService(devices.DeviceService): + + def __init__(self, args): + super().__init__(args) + self.fullname = None + self.target = None + + @staticmethod + def lv_set_active(fullname: str, status: bool): + mode = "y" if status else "n" + cmd = [ + "lvchange", "--activate", mode, fullname + ] + res = subprocess.run(cmd, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + encoding="UTF-8") + + if res.returncode != 0: + data = res.stderr.strip() + msg = f"Failed to set LV device ({fullname}) status: {data}" + raise RuntimeError(msg) + + @staticmethod + def volume_group_for_device(device: str) -> str: + # Find the volume group that belongs to the device specified via `parent` + vg_name = None + count = 0 + + cmd = [ + "pvdisplay", "-C", "--noheadings", "-o", "vg_name", device + ] + + while True: + res = subprocess.run(cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8") + + if res.returncode == 5: + if count == 10: + raise RuntimeError("Could not find parent device") + time.sleep(1*count) + count += 1 + continue + + if res.returncode != 0: + json.dump({"error": res.stderr.strip()}, sys.stdout) + return 1 + + vg_name = res.stdout.strip() + if vg_name: + break + + return vg_name + + @staticmethod + def device_for_logical_volume(vg_name: str, volume: str) -> Tuple[int, int]: + # Now that we have the volume group, find the specified logical volume and its device path + + cmd = [ + "lvdisplay", "-C", "--noheadings", + "-o", "lv_kernel_major,lv_kernel_minor", + "--separator", ";", + "-S", f"lv_name={volume}", + vg_name + ] + + res = subprocess.run(cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8") + + if res.returncode != 0: + raise RuntimeError(res.stderr.strip()) + + data = res.stdout.strip() + devnum = list(map(int, data.split(";"))) + assert len(devnum) == 2 + major, minor = devnum[0], devnum[1] + + return major, minor + + def open(self, devpath: str, parent: str, tree: str, options: Dict): + lv = options["volume"] + + assert not parent.startswith("/") + parent_path = os.path.join("/dev", parent) + + # Find the volume group that belongs to the device specified + # via `parent` + vg = self.volume_group_for_device(parent_path) + + # Activate the logical volume + self.fullname = f"{vg}/{lv}" + self.lv_set_active(self.fullname, True) + + # Now that we have the volume group, find the major and minor + # device numbers for the logical volume + major, minor = self.device_for_logical_volume(vg, lv) + + # Create the device node for the LV in the build root's /dev + devname = os.path.join(vg, lv) + fullpath = os.path.join(devpath, devname) + + os.makedirs(os.path.join(devpath, vg), exist_ok=True) + os.mknod(fullpath, 0o666 | stat.S_IFBLK, os.makedev(major, minor)) + + data = { + "path": devname, + "node": { + "major": major, + "minor": minor + }, + "lvm": { + "vg_name": vg, + "lv_name": lv + } + } + return data + + def close(self): + if self.fullname: + self.lv_set_active(self.fullname, False) + self.fullname = None + + +def main(): + service = LVService.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + r = main() + sys.exit(r) diff --git a/docs/osbuild-manifest.5.rst b/docs/osbuild-manifest.5.rst new file mode 100644 index 0000000..37d5f00 --- /dev/null +++ b/docs/osbuild-manifest.5.rst @@ -0,0 +1,203 @@ +================ +osbuild-manifest +================ + +----------------------- +OSBuild Manifest Format +----------------------- + +:Manual section: 5 +:Manual group: File Formats Manual + +SYNOPSIS +======== + +| +| { +| "**pipeline**": { +| "**build**": { ... }, +| "**stages**": [ ... ], +| "**assembler**": { ... } +| }, +| +| "**sources**": { +| "org.osbuild.files": { +| "**urls**": { +| ... +| } +| } +| } +| } +| + +DESCRIPTION +=========== + +The osbuild manifest format describes to ``osbuild``\(1) which build pipeline +to execute and which resources to make available to it. A manifest is always +formatted as a single `JSON` document and must contain all information needed +to execute a specified pipeline. Furthermore, a manifest must be +authoritative in that data passed to ``osbuild``\(1) via other means than the +manifest must not affect the outcome of the pipeline. Therefore, the content of +a manifest deterministicly describes the expected output. + +The exact schema of the manifest is available online as the OSBuild JSON +Schema [#]_. + +A manifest consists of a fixed amount of top-level sections. These include +sections that describe the steps of the pipeline to execute, but also external +resources to make available to the pipeline execution. The following manual +describe the different sections available in the manifest, as well as short +examples how these sections can look like. + +PIPELINES +========= + +The `pipeline` section describes the pipeline to execute. This includes a +description of the build system to use for the execution, the stages to +execute, and the final assemblers whichproduce the desired output format. + +| +| "**pipeline**": { +| "**build**": { ... }, +| "**stages**": [ ... ], +| "**assembler**": { ... } +| } +| + +PIPELINE: build +--------------- + +The `build` section specifies the system to use when executing stages of a +pipeline. The definition of this section is recursive, in that it requires a +pipeline definition as its value. The build system is created by recursively +executing its pipeline definition first. It is then used as the build system +to execute the pipeline that defined this build system. + +Additionally to the pipeline description, a build definition must also define +the runner to use when executing applications in this system. These runners are +shipped with ``osbuild``\(1) and perform environment setup before executing a +stage. + +| +| "**build**": { +| "**pipeline**": { +| "**build**": { ... }, +| "**stages**": [ ... ], +| "**assembler**": { ... } +| }, +| "**runner**": "org.osbuild.linux" +| } +| + +PIPELINE: stages +---------------- + +The `stages` section is an array of stages to execute as part of the pipeline. +Together they produce a file system tree that forms the output of the pipeline. +Each stage can modify the tree, alter its content, or add new data to it. All +stages are executed in sequence, each taking the output of the previous stage +as their input. + +| +| "**stages**": [ +| { +| "**name**": "", +| "**options**": { +| ... +| } +| }, +| { +| "**name**": "", +| "**options**": { +| ... +| } +| } +| ] +| + +Stages are shipped together with ``osbuild``\(1). The manifest can optionally +contain options that are passed to the respective stage. + +PIPELINE: assembler +------------------- + +The assembler is the final stage of a pipeline. It is similar to a `stage` but +always executed last. Furthermore, it is not allowed to modify the file system +tree. Instead, it is expected to take the file system tree and produce a +desired output format for consumption by the user. + +| +| "**assembler**": { +| "**name**": "", +| "**options**": { +| ... +| } +| } +| + +Assemblers are shipped together with ``osbuild``\(1). The manifest can +optionally contain options that are passed to the respective assembler. + +SOURCES +======= + +The `sources` section describes external resources that are needed to execute a +pipeline. Specified sources do not have to be used by a pipeline execution. +Hence, it is not an error to specify more resources than actually required. + +Note: + The pipeline executor might prefetch resources before executing a + pipeline. Therefore, you should only specify resources that are + actually required to execute a pipeline. + +The `sources` section thus allows to hide from the pipeline execution where an +external resource comes from and how it is fetched. Instead, it provides an +internal API to the pipeline to access these external resources in a common +way. Depending on which pipeline `stages` are defined, they required different +source types to provide configured resources. + +The following sub-sections describe the different available source types. To +configure a specific source type, you would use something like the following: + +| +| "**sources**": { +| "": { +| ... +| }, +| "": { +| ... +| } +| } +| + +SOURCE: org.osbuild.files +------------------------- + +The `org.osbuild.files` source type allows to provide external files to the +pipeline execution. The argument to this type is a dictionary of file names and +their corresponding resource URIs. The file name must be the hash of the +expected file, prefixed with the hash-type. + +The following example shows how you could provide two files to a pipeline +execution via the `org.osbuild.files` source type: + +| +| "**sources**": { +| "org.osbuild.files": { +| "sha256:": "https://example.com/some-file-A", +| "sha256:": "https://example.com/some-file-B" +| } +| } +| + +SEE ALSO +======== + +``osbuild``\(1), ``osbuild-composer``\(1) + +NOTES +===== + +.. [#] OSBuild JSON Schema: + https://osbuild.org/schemas/osbuild1.json diff --git a/docs/osbuild.1.rst b/docs/osbuild.1.rst new file mode 100644 index 0000000..6f637e7 --- /dev/null +++ b/docs/osbuild.1.rst @@ -0,0 +1,118 @@ +======= +osbuild +======= + +---------------------------------------------- +Build-Pipelines for Operating System Artifacts +---------------------------------------------- + +:Manual section: 1 +:Manual group: User Commands + +SYNOPSIS +======== + +| ``osbuild`` [ OPTIONS ] PIPELINE +| ``osbuild`` [ OPTIONS ] - +| ``osbuild`` ``--help`` + +DESCRIPTION +=========== + +**osbuild** is a build-system for operating system artifacts. It takes a +pipeline description as input and produces file-system trees, images, or other +artifacts as output. Its pipeline description gives comprehensive control over +the individual steps to execute as part of a pipeline. **osbuild** provides +isolation from the host system as well as caching capabilities, and thus +ensures that pipeline builds will be deterministic and efficient. + +OPTIONS +======= + +**osbuild** reads the pipeline description from the file passed on the +command-line. To make **osbuild** read the pipeline description from standard +input, pass ``-``. + +The following command-line options are supported. If an option is passed, which +is not listed here, **osbuild** will deny startup and exit with an error. + +-h, --help print usage information and exit immediately +--store=DIR directory where intermediary file system trees + are stored +--secrets=PATH json file containing a dictionary of secrets + that are passed to sources +-l DIR, --libdir=DIR directory containing stages, assemblers, and + the osbuild library +--checkpoint=CHECKPOINT stage to commit to the object store during + build (can be passed multiple times) +--export=OBJECT object to export (can be passed multiple times) +--json output results in JSON format +--output-directory=DIR directory where result objects are stored +--inspect return the manifest in JSON format including + all the ids + +NB: If neither ``--output-directory`` nor ``--checkpoint`` is specified, no +attempt to build the manifest will be made. + +MANIFEST +======== + +The input to **osbuild** is a description of the pipeline to execute, as well +as required parameters to each pipeline stage. This data must be *JSON* +encoded. It is read from the file specified on the command-line, or, if ``-`` +is passed, from standard input. + +The format of the manifest is described in ``osbuild-manifest``\(5). The formal +schema of the manifest is available online as the OSBuild JSON Schema [#]_. + +EXAMPLES +======== + +The following sub-sections contain examples on running **osbuild**. Generally, +**osbuild** must be run with superuser privileges, since this is required to +create file-system images. + +Example 1: Run an empty pipeline +-------------------------------- + +To verify your **osbuild** setup, you can run it on an empty pipeline which +produces no output: + + | + | # echo {} | osbuild - + | + +Example 1: Build a Fedora 30 qcow2 image +---------------------------------------- + +To build a basic qcow2 image of Fedora 30, use: + + | + | # osbuild ./samples/base-qcow2.json + | + +The pipeline definition ``./samples/base-rpm-qcow2.json`` is provided in the +upstream source repository of **osbuild**. + +Example 2: Run from a local checkout +------------------------------------ + +To run **osbuild** from a local checkout, use: + + | + | # python3 -m osbuild --libdir . samples/base-rpm-qcow2.json + | + +This will make sure to execute the **osbuild** module from the current +directory, as well as use it to search for stages, assemblers, and more. + +SEE ALSO +======== + +``osbuild-manifest``\(5), ``osbuild-composer``\(1) + +NOTES +===== + +.. [#] OSBuild JSON Schema: + https://osbuild.org/schemas/osbuild1.json diff --git a/inputs/org.osbuild.containers b/inputs/org.osbuild.containers new file mode 100755 index 0000000..5abca60 --- /dev/null +++ b/inputs/org.osbuild.containers @@ -0,0 +1,178 @@ +#!/usr/bin/python3 +"""Inputs for container images + +This reads images from the `org.osbuild.containers` directory in the +sources store, or from oci-archive files in pipelines (typically +created by `org.osbuild.oci-archive`). + +The store is indexed by the "container image id", which is the digest +of the container configuration file (rather than the outer manifest) +and is what will be shown in the "podman images" output when the image +is installed. This digest is stable as opposed to the manifest digest +which can change during transfer and storage due to +e.g. recompression. + +When using pipeline sources, the first file (alphabetically) in the +root of the tree is used as the oci archive file to install. + +""" + +import os +import sys +import pathlib + +from osbuild import inputs + + +SCHEMA = r""" +"definitions": { + "source-options": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The name to use for the image" + } + } + }, + "source-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/source-options" + } + } + }, + "source-origin": { + "type": "string", + "description": "When the origin of the input is a source", + "enum": ["org.osbuild.source"] + }, + "pipeline-options": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { + "type": "string", + "description": "The name to use for the image" + } + } + }, + "pipeline-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/pipeline-options" + } + } + }, + "pipeline-origin": { + "type": "string", + "description": "When the origin of the input is a pipeline", + "enum": ["org.osbuild.pipeline"] + } +}, +"additionalProperties": true, +"oneOf": [ + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.containers"] + }, + "origin": { + "description": "The org.osbuild.source origin case", + "$ref": "#/definitions/source-origin" + }, + "references": { + "description": "Container image id", + "$ref": "#/definitions/source-object-ref" + } + } + }, + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.containers"] + }, + "origin": { + "description": "The org.osbuild.source origin case", + "$ref": "#/definitions/pipeline-origin" + }, + "references": { + "description": "References to pipelines", + "$ref": "#/definitions/pipeline-object-ref" + } + } + } +] +""" + + +class ContainersInput(inputs.InputService): + + @staticmethod + def map_source_ref(source, ref, data, target): + cache_dir = os.path.join(source, ref) + os.link(os.path.join(cache_dir, "container-image.tar"), os.path.join(target, ref)) + + return ref, "docker-archive" + + @staticmethod + def map_pipeline_ref(store, ref, data, target): + # prepare the mount point + os.makedirs(target, exist_ok=True) + print("target", target) + + store.read_tree_at(ref, target) + + # Find the archive file in target, we use the first alphabetical regular file + files = sorted(filter(lambda f: os.path.isfile(os.path.join(target, f)), + os.listdir(target))) + if len(files) == 0: + raise RuntimeError("No archive files in source") + + return files[0], "oci-archive" + + def map(self, store, origin, refs, target, _options): + source = store.source("org.osbuild.containers") + images = {} + + for ref, data in refs.items(): + if origin == "org.osbuild.source": + ref, container_format = self.map_source_ref(source, ref, data, target) + else: + ref, container_format = self.map_pipeline_ref(store, ref, data, target) + + images[ref] = { + "format": container_format, + "name": data["name"] + } + images[ref]["name"] = data["name"] + + reply = { + "path": target, + "data": { + "archives": images + } + } + return reply + + +def main(): + service = ContainersInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/inputs/org.osbuild.files b/inputs/org.osbuild.files new file mode 100755 index 0000000..2cad387 --- /dev/null +++ b/inputs/org.osbuild.files @@ -0,0 +1,189 @@ +#!/usr/bin/python3 +""" +Inputs for individual files + +Provides all the files, named via their content hash, specified +via `references` in a new directory. + +The returned data in `files` is a dictionary where the keys are +paths to the provided files and the values dictionaries with +metadata for it. The input itself currently does not set any +metadata itself, but will forward any metadata set via the +`metadata` property. Keys in that must start with a prefix, +like `rpm.` to avoid namespace clashes. This is enforced via +schema validation. +""" + +import os +import pathlib +import sys + +from osbuild import inputs + + +SCHEMA = r""" +"definitions": { + "metadata": { + "description": "Additional metadata to forward to the stage", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^\\w+[.]{1}\\w+$": { + "additionalProperties": false + } + } + }, + "file": { + "description": "File to access with in a pipeline", + "type": "string" + }, + "plain-ref": { + "type": "array", + "items": { + "type": "string" + } + }, + "source-options": { + "type": "object", + "additionalProperties": false, + "properties": { + "metadata": { + "$ref": "#/definitions/metadata" + } + } + }, + "source-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/source-options" + } + } + }, + "source-origin": { + "type": "string", + "description": "When the origin of the input is a source", + "enum": ["org.osbuild.source"] + }, + "pipeline-options": { + "type": "object", + "additionalProperties": false, + "properties": { + "metadata": { + "$ref": "#/definitions/metadata" + }, + "file": { + "$ref": "#/definitions/file" + } + } + }, + "pipeline-object-ref": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "$ref": "#/definitions/pipeline-options" + } + } + }, + "pipeline-origin": { + "type": "string", + "description": "When the origin of the input is a pipeline", + "enum": ["org.osbuild.pipeline"] + } +}, +"additionalProperties": true, +"oneOf": [ + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.files"] + }, + "origin": { + "description": "The org.osbuild.source origin case", + "$ref": "#/definitions/source-origin" + }, + "references": { + "description": "Checksums of files to use as files input", + "oneOf": [ + {"$ref": "#/definitions/plain-ref"}, + {"$ref": "#/definitions/source-object-ref"} + ] + } + } + }, + { + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { + "enum": ["org.osbuild.files"] + }, + "origin": { + "description": "The org.osbuild.pipeline origin case", + "$ref": "#/definitions/pipeline-origin" + }, + "references": { + "description": "References to pipelines", + "$ref": "#/definitions/pipeline-object-ref" + } + } + } +] +""" + + +class FilesInput(inputs.InputService): + + @staticmethod + def map_pipeline_ref(store, ref, data, target): + filepath = data["file"].lstrip("/") + + # prepare the mount point + filename = pathlib.Path(target, filepath) + os.makedirs(filename.parent, exist_ok=True) + filename.touch() + + store.read_tree_at(ref, filename, filepath) + + return filepath, data.get("metadata", {}) + + @staticmethod + def map_source_ref(source, ref, data, target): + os.link(f"{source}/{ref}", f"{target}/{ref}") + data = data.get("metadata", {}) + return ref, data + + def map(self, store, origin, refs, target, _options): + + source = store.source("org.osbuild.files") + files = {} + + for ref, data in refs.items(): + if origin == "org.osbuild.source": + ref, data = self.map_source_ref(source, ref, data, target) + else: + ref, data = self.map_pipeline_ref(store, ref, data, target) + files[ref] = data + + reply = { + "path": target, + "data": { + "files": files + } + } + return reply + + +def main(): + service = FilesInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/inputs/org.osbuild.noop b/inputs/org.osbuild.noop new file mode 100755 index 0000000..2fafd85 --- /dev/null +++ b/inputs/org.osbuild.noop @@ -0,0 +1,44 @@ +#!/usr/bin/python3 +""" +No-op inputs + +Does nothing with the supplied data but just forwards +it to the stage. +""" + + +import os +import sys +import uuid + +from osbuild import inputs + +SCHEMA = """ +"additionalProperties": true +""" + + +class NoopInput(inputs.InputService): + + def map(self, _store, _origin, refs, target, _options): + + uid = str(uuid.uuid4()) + path = os.path.join(target, uid) + os.makedirs(path) + + reply = { + "path": target, + "data": { + "refs": refs + } + } + return reply + + +def main(): + service = NoopInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/inputs/org.osbuild.ostree b/inputs/org.osbuild.ostree new file mode 100755 index 0000000..e0e3fc4 --- /dev/null +++ b/inputs/org.osbuild.ostree @@ -0,0 +1,129 @@ +#!/usr/bin/python3 +""" +Inputs for ostree commits + +Pull the commits specified by `references` into a newly created +repository. Optionally, if `ref` was specified, create an new +reference for that commit. + +The returned data in `refs` is a dictionary where the keys are +commit ids and the values are dictionries. The latter will +contain `ref` it was specified. +""" + + +import os +import json +import sys +import subprocess + +from osbuild import inputs + + +SCHEMA = """ +"additionalProperties": false, +"required": ["type", "origin", "references"], +"properties": { + "type": { + "enum": ["org.osbuild.ostree"] + }, + "origin": { + "description": "The origin of the input (pipeline or source)", + "type": "string", + "enum": ["org.osbuild.source", "org.osbuild.pipeline"] + }, + "references": { + "description": "Commit identifier", + "oneOf": [{ + "type": "array", + "items": { + "type": "string" + } + }, { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false, + "properties": { + "ref": { + "type": "string", + "description": "OSTree reference to create for this commit" + } + } + } + } + }] + } +} +""" + + +def ostree(*args, _input=None, **kwargs): + args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + print("ostree " + " ".join(args), file=sys.stderr) + subprocess.run(["ostree"] + args, + encoding="utf-8", + stdout=sys.stderr, + input=_input, + check=True) + + +def export(checksums, cache, output): + repo_cache = os.path.join(cache, "repo") + + repo_out = os.path.join(output, "repo") + ostree("init", mode="archive", repo=repo_out) + + refs = {} + for commit, options in checksums.items(): + # Transfer the commit: remote → cache + print(f"exporting {commit}", file=sys.stderr) + + ostree("pull-local", repo_cache, commit, + repo=repo_out) + + ref = options.get("ref") + if ref: + ostree("refs", "--create", ref, commit, + repo=repo_out) + + refs[commit] = options + + reply = { + "path": repo_out, + "data": { + "refs": refs + } + } + + return reply + + +class OSTreeInput(inputs.InputService): + + def map(self, store, origin, refs, target, _options): + + if origin == "org.osbuild.pipeline": + for ref, options in refs.items(): + source = store.read_tree(ref) + with open(os.path.join(source, "compose.json"), "r") as f: + compose = json.load(f) + commit_id = compose["ostree-commit"] + reply = export({commit_id: options}, source, target) + else: + source = store.source("org.osbuild.ostree") + reply = export(refs, source, target) + + return reply + + +def main(): + service = OSTreeInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/inputs/org.osbuild.ostree.checkout b/inputs/org.osbuild.ostree.checkout new file mode 100755 index 0000000..e6f7b26 --- /dev/null +++ b/inputs/org.osbuild.ostree.checkout @@ -0,0 +1,114 @@ +#!/usr/bin/python3 +""" +Inputs for checkouts of ostree commits + +This input takes a number of commits and will check them out to a +temporary directory. The name of the directory is the commit id. +Internally uses `ostree checkout` +""" + + +import os +import json +import sys +import subprocess + +from osbuild import inputs + + +SCHEMA = """ +"additionalProperties": false, +"required": ["type", "origin", "references"], +"properties": { + "type": { + "enum": ["org.osbuild.ostree.checkout"] + }, + "origin": { + "description": "The origin of the input", + "type": "string", + "enum": ["org.osbuild.source", "org.osbuild.pipeline"] + }, + "references": { + "description": "Commit identifier to check out", + "oneOf": [{ + "type": "array", + "items": { + "type": "string" + } + }, { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false + } + } + }] + } +} +""" + + +def ostree(*args, _input=None, **kwargs): + args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + print("ostree " + " ".join(args), file=sys.stderr) + subprocess.run(["ostree"] + args, + encoding="utf-8", + stdout=sys.stderr, + input=_input, + check=True) + + +def checkout(checksums, cache, output): + repo_cache = os.path.join(cache, "repo") + + refs = [] + for commit in checksums: + print(f"checkout {commit}", file=sys.stderr) + + dest = os.path.join(output, commit) + + ostree("checkout", commit, dest, + repo=repo_cache) + + refs.append(commit) + + return refs + + +class OSTreeCheckoutInput(inputs.InputService): + + def map(self, store, origin, refs, target, _options): + + ids = [] + + if origin == "org.osbuild.pipeline": + for ref, options in refs.items(): + source = store.read_tree(ref) + with open(os.path.join(source, "compose.json"), "r") as f: + compose = json.load(f) + commit_id = compose["ostree-commit"] + ids.append(checkout({commit_id: options}, source, target)) + else: + source = store.source("org.osbuild.ostree") + ids = checkout(refs, source, target) + + reply = { + "path": target, + "data": { + "refs": {i: {"path": i} for i in ids} + } + } + + return reply + + +def main(): + service = OSTreeCheckoutInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/inputs/org.osbuild.tree b/inputs/org.osbuild.tree new file mode 100755 index 0000000..42249a2 --- /dev/null +++ b/inputs/org.osbuild.tree @@ -0,0 +1,78 @@ +#!/usr/bin/python3 +""" +Tree inputs + +Open the tree produced by the pipeline supplied via the +first and only entry in `references`. The tree is opened +in read only mode. If the id is `null` or the empty +string it returns an empty tree. +""" + +import sys + +from osbuild import inputs + + +SCHEMA = """ +"additionalProperties": false, +"required": ["type", "origin", "references"], +"properties": { + "type": { + "enum": ["org.osbuild.tree"] + }, + "origin": { + "description": "The origin of the input (must be 'org.osbuild.pipeline')", + "type": "string", + "enum": ["org.osbuild.pipeline"] + }, + "references": { + "description": "Exactly one pipeline identifier to ues as tree input", + "oneOf": [{ + "type": "array", + "additionalItems": false, + "items": [{ + "type": "string" + }] + }, { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false + } + }, + "minProperties": 1, + "maxProperties": 1 + }] + } +} +""" + + +class TreeInput(inputs.InputService): + + def map(self, store, _origin, refs, target, _options): + + # input verification *must* have been done via schema + # verification. It is expected that origin is a pipeline + # and we have exactly one reference, i.e. a pipeline id + pid, _ = refs.popitem() + + if pid: + path = store.read_tree_at(pid, target) + + if not path: + raise ValueError(f"Unknown pipeline '{pid}'") + + reply = {"path": target} + return reply + + +def main(): + service = TreeInput.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.btrfs b/mounts/org.osbuild.btrfs new file mode 100755 index 0000000..7a1ee1c --- /dev/null +++ b/mounts/org.osbuild.btrfs @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +""" +btrfs mount service + +Mount a btrfs filesystem at the given location. + +Host commands used: mount +""" + +import sys +from typing import Dict + +from osbuild import mounts + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type", "source", "target"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } +} +""" + + +class BtrfsMount(mounts.FileSystemMountService): + + def translate_options(self, _options: Dict): + return ["-t", "btrfs"] + + +def main(): + service = BtrfsMount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.ext4 b/mounts/org.osbuild.ext4 new file mode 100755 index 0000000..2fb7624 --- /dev/null +++ b/mounts/org.osbuild.ext4 @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +""" +ext4 mount service + +Mount a ext4 filesystem at the given location. + +Host commands used: mount +""" + +import sys +from typing import Dict + +from osbuild import mounts + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type", "source", "target"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } +} +""" + + +class Ext4Mount(mounts.FileSystemMountService): + + def translate_options(self, _options: Dict): + return ["-t", "ext4"] + + +def main(): + service = Ext4Mount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.fat b/mounts/org.osbuild.fat new file mode 100755 index 0000000..8a5cba0 --- /dev/null +++ b/mounts/org.osbuild.fat @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +""" +FAT mount service + +Mount a FAT filesystem at the given location. + +Host commands used: mount +""" + +import sys +from typing import Dict + +from osbuild import mounts + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type", "source", "target"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } +} +""" + + +class XfsMount(mounts.FileSystemMountService): + + def translate_options(self, _options: Dict): + return ["-t", "vfat"] + + +def main(): + service = XfsMount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.noop b/mounts/org.osbuild.noop new file mode 100755 index 0000000..a59d959 --- /dev/null +++ b/mounts/org.osbuild.noop @@ -0,0 +1,64 @@ +#!/usr/bin/python3 +""" +No-op mount service + +Does not mount anything, but only creates an empty directory. +Useful for testing. + +Host commands used: mount +""" + +import os +import sys +from typing import Dict + +from osbuild import mounts + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type", "source", "target"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } +} +""" + + +class NoOpMount(mounts.MountService): + + def mount(self, args: Dict): + root = args["root"] + target = args["target"] + + mountpoint = os.path.join(root, target.lstrip("/")) + + os.makedirs(mountpoint, exist_ok=True) + self.mountpoint = mountpoint + + return mountpoint + + def umount(self): + self.mountpoint = None + + def sync(self): + pass + + +def main(): + service = NoOpMount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment new file mode 100755 index 0000000..9a21f67 --- /dev/null +++ b/mounts/org.osbuild.ostree.deployment @@ -0,0 +1,128 @@ +#!/usr/bin/python3 +""" +OSTree deployment mount service + +This mount service will setup all needed bind mounts so +that a given `tree` will look like an active OSTree +deployment, very much as OSTree does during early boot. + +More specifically it will: + - setup the sysroot bindmount to the deployment + - setup the shared var directory + - bind the boot directory into the deployment + +Host commands used: mount +""" + +import os +import sys +import subprocess +from typing import Dict + +from osbuild import mounts +from osbuild.util import ostree + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "options": { + "type": "object", + "required": ["deployment"], + "properties": { + "deployment": { + "type": "object", + "additionalProperties": false, + "required": ["osname", "ref"], + "properties": { + "osname": { + "description": "Name of the stateroot to be used in the deployment", + "type": "string" + }, + "ref": { + "description": "OStree ref to create and use for deployment", + "type": "string" + }, + "serial": { + "description": "The deployment serial (usually '0')", + "type": "number", + "default": 0 + } + } + } + } + } +} +""" + + +class OSTreeDeploymentMount(mounts.MountService): + + def __init__(self, args): + super().__init__(args) + + self.mountpoint = None + self.check = False + + @staticmethod + def bind_mount(source, target): + subprocess.run([ + "mount", "--bind", "--make-private", source, target, + ], check=True) + + def mount(self, args: Dict): + + tree = args["tree"] + options = args["options"] + + deployment = options["deployment"] + osname = deployment["osname"] + ref = deployment["ref"] + serial = deployment.get("serial", 0) + + root = ostree.deployment_path(tree, osname, ref, serial) + + print(f"Deployment root at '{os.path.relpath(root, tree)}'") + + var = os.path.join(tree, "ostree", "deploy", osname, "var") + boot = os.path.join(tree, "boot") + + self.mountpoint = root + self.bind_mount(root, root) # prepare to move it later + + self.bind_mount(tree, os.path.join(root, "sysroot")) + self.bind_mount(var, os.path.join(root, "var")) + self.bind_mount(boot, os.path.join(root, "boot")) + + subprocess.run([ + "mount", "--move", root, tree, + ], check=True) + + self.mountpoint = tree + self.check = True + + return None + + def umount(self): + + if not self.mountpoint: + return + + subprocess.run(["sync", "-f", self.mountpoint], + check=self.check) + + subprocess.run(["umount", "-R", self.mountpoint], + check=self.check) + self.mountpoint = None + + +def main(): + service = OSTreeDeploymentMount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/mounts/org.osbuild.xfs b/mounts/org.osbuild.xfs new file mode 100755 index 0000000..5529ad6 --- /dev/null +++ b/mounts/org.osbuild.xfs @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +""" +XFS mount service + +Mount a XFS filesystem at the given location. + +Host commands used: mount +""" + +import sys +from typing import Dict + +from osbuild import mounts + + +SCHEMA_2 = """ +"additionalProperties": false, +"required": ["name", "type", "source", "target"], +"properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } +} +""" + + +class XfsMount(mounts.FileSystemMountService): + + def translate_options(self, _options: Dict): + return ["-t", "xfs"] + + +def main(): + service = XfsMount.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/osbuild.spec b/osbuild.spec new file mode 100644 index 0000000..4ef999d --- /dev/null +++ b/osbuild.spec @@ -0,0 +1,258 @@ +%global forgeurl https://github.com/osbuild/osbuild +%global selinuxtype targeted + +Version: 54 + +%forgemeta + +%global pypi_name osbuild +%global pkgdir %{_prefix}/lib/%{pypi_name} + +Name: %{pypi_name} +Release: 1%{?dist} +License: ASL 2.0 + +URL: %{forgeurl} + +Source0: %{forgesource} +BuildArch: noarch +Summary: A build system for OS images + +BuildRequires: make +BuildRequires: python3-devel +BuildRequires: python3-docutils +BuildRequires: systemd + +Requires: bash +Requires: bubblewrap +Requires: coreutils +Requires: curl +Requires: dnf +Requires: e2fsprogs +Requires: glibc +Requires: policycoreutils +Requires: qemu-img +Requires: systemd +Requires: tar +Requires: util-linux +Requires: python3-%{pypi_name} = %{version}-%{release} +Requires: (%{name}-selinux if selinux-policy-%{selinuxtype}) + +# Turn off dependency generators for runners. The reason is that runners are +# tailored to the platform, e.g. on RHEL they are using platform-python. We +# don't want to pick up those dependencies on other platform. +%global __requires_exclude_from ^%{pkgdir}/(runners)/.*$ + +# Turn off shebang mangling on RHEL. brp-mangle-shebangs (from package +# redhat-rpm-config) is run on all executables in a package after the `install` +# section runs. The below macro turns this behavior off for: +# - runners, because they already have the correct shebang for the platform +# they're meant for, and +# - stages and assemblers, because they are run within osbuild build roots, +# which are not required to contain the same OS as the host and might thus +# have a different notion of "platform-python". +# RHEL NB: Since assemblers and stages are not excluded from the dependency +# generator, this also means that an additional dependency on /usr/bin/python3 +# will be added. This is intended and needed, so that in the host build root +# /usr/bin/python3 is present so stages and assemblers can be run. +%global __brp_mangle_shebangs_exclude_from ^%{pkgdir}/(assemblers|runners|stages)/.*$ + +%{?python_enable_dependency_generator} + +%description +A build system for OS images + +%package -n python3-%{pypi_name} +Summary: %{summary} +%{?python_provide:%python_provide python3-%{pypi_name}} + +%description -n python3-%{pypi_name} +A build system for OS images + +%package lvm2 +Summary: LVM2 support +Requires: %{name} = %{version}-%{release} +Requires: lvm2 + +%description lvm2 +Contains the necessary stages and device host +services to build LVM2 based images. + +%package luks2 +Summary: LUKS2 support +Requires: %{name} = %{version}-%{release} +Requires: cryptsetup + +%description luks2 +Contains the necessary stages and device host +services to build LUKS2 encrypted images. + +%package ostree +Summary: OSTree support +Requires: %{name} = %{version}-%{release} +Requires: ostree +Requires: rpm-ostree + +%description ostree +Contains the necessary stages, assembler and source +to build OSTree based images. + +%package selinux +Summary: SELinux policies +Requires: %{name} = %{version}-%{release} +BuildRequires: selinux-policy +BuildRequires: selinux-policy-devel +%{?selinux_requires} + +%description selinux +Contains the necessary SELinux policies that allows +osbuild to use labels unknown to the host inside the +containers it uses to build OS artifacts. + +%package tools +Summary: Extra tools and utilities +Requires: %{name} = %{version}-%{release} +Requires: python3-pyyaml + +%description tools +Contains additional tools and utilities for development of +manifests and osbuild. + +%prep +%forgesetup + +%build +%py3_build +make man + +# SELinux +make -f /usr/share/selinux/devel/Makefile osbuild.pp +bzip2 -9 osbuild.pp + +%pre +%selinux_relabel_pre -s %{selinuxtype} + +%install +%py3_install + +mkdir -p %{buildroot}%{pkgdir}/stages +install -p -m 0755 $(find stages -type f) %{buildroot}%{pkgdir}/stages/ + +mkdir -p %{buildroot}%{pkgdir}/assemblers +install -p -m 0755 $(find assemblers -type f) %{buildroot}%{pkgdir}/assemblers/ + +mkdir -p %{buildroot}%{pkgdir}/runners +install -p -m 0755 $(find runners -type f -or -type l) %{buildroot}%{pkgdir}/runners + +mkdir -p %{buildroot}%{pkgdir}/sources +install -p -m 0755 $(find sources -type f) %{buildroot}%{pkgdir}/sources + +mkdir -p %{buildroot}%{pkgdir}/devices +install -p -m 0755 $(find devices -type f) %{buildroot}%{pkgdir}/devices + +mkdir -p %{buildroot}%{pkgdir}/inputs +install -p -m 0755 $(find inputs -type f) %{buildroot}%{pkgdir}/inputs + +mkdir -p %{buildroot}%{pkgdir}/mounts +install -p -m 0755 $(find mounts -type f) %{buildroot}%{pkgdir}/mounts + +# mount point for bind mounting the osbuild library +mkdir -p %{buildroot}%{pkgdir}/osbuild + +# schemata +mkdir -p %{buildroot}%{_datadir}/osbuild/schemas +install -p -m 0644 $(find schemas/*.json) %{buildroot}%{_datadir}/osbuild/schemas +ln -s %{_datadir}/osbuild/schemas %{buildroot}%{pkgdir}/schemas + +# documentation +mkdir -p %{buildroot}%{_mandir}/man1 +mkdir -p %{buildroot}%{_mandir}/man5 +install -p -m 0644 -t %{buildroot}%{_mandir}/man1/ docs/*.1 +install -p -m 0644 -t %{buildroot}%{_mandir}/man5/ docs/*.5 + +# SELinux +install -D -m 0644 -t %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype} %{name}.pp.bz2 +install -D -m 0644 -t %{buildroot}%{_mandir}/man8 selinux/%{name}_selinux.8 + +# Udev rules +mkdir -p %{buildroot}%{_udevrulesdir} +install -p -m 0755 data/10-osbuild-inhibitor.rules %{buildroot}%{_udevrulesdir} + +%check +exit 0 +# We have some integration tests, but those require running a VM, so that would +# be an overkill for RPM check script. + +%files +%license LICENSE +%{_bindir}/osbuild +%{_mandir}/man1/%{name}.1* +%{_mandir}/man5/%{name}-manifest.5* +%{_datadir}/osbuild/schemas +%{pkgdir} +%{_udevrulesdir}/*.rules +# the following files are in the lvm2 sub-package +%exclude %{pkgdir}/devices/org.osbuild.lvm2* +%exclude %{pkgdir}/stages/org.osbuild.lvm2* +# the following files are in the luks2 sub-package +%exclude %{pkgdir}/devices/org.osbuild.luks2* +%exclude %{pkgdir}/stages/org.osbuild.crypttab +%exclude %{pkgdir}/stages/org.osbuild.luks2* +# the following files are in the ostree sub-package +%exclude %{pkgdir}/assemblers/org.osbuild.ostree* +%exclude %{pkgdir}/inputs/org.osbuild.ostree* +%exclude %{pkgdir}/sources/org.osbuild.ostree* +%exclude %{pkgdir}/stages/org.osbuild.ostree* +%exclude %{pkgdir}/stages/org.osbuild.rpm-ostree + +%files -n python3-%{pypi_name} +%license LICENSE +%doc README.md +%{python3_sitelib}/%{pypi_name}-*.egg-info/ +%{python3_sitelib}/%{pypi_name}/ + +%files lvm2 +%{pkgdir}/devices/org.osbuild.lvm2* +%{pkgdir}/stages/org.osbuild.lvm2* + +%files luks2 +%{pkgdir}/devices/org.osbuild.luks2* +%{pkgdir}/stages/org.osbuild.crypttab +%{pkgdir}/stages/org.osbuild.luks2* + +%files ostree +%{pkgdir}/assemblers/org.osbuild.ostree* +%{pkgdir}/inputs/org.osbuild.ostree* +%{pkgdir}/sources/org.osbuild.ostree* +%{pkgdir}/stages/org.osbuild.ostree* +%{pkgdir}/stages/org.osbuild.rpm-ostree + +%files selinux +%{_datadir}/selinux/packages/%{selinuxtype}/%{name}.pp.bz2 +%{_mandir}/man8/%{name}_selinux.8.* +%ghost %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{name} + +%post selinux +%selinux_modules_install -s %{selinuxtype} %{_datadir}/selinux/packages/%{selinuxtype}/%{name}.pp.bz2 + +%postun selinux +if [ $1 -eq 0 ]; then + %selinux_modules_uninstall -s %{selinuxtype} %{name} +fi + +%posttrans selinux +%selinux_relabel_post -s %{selinuxtype} + +%files tools +%{_bindir}/osbuild-mpp + + +%changelog +* Mon Aug 19 2019 Miro Hrončok - 1-3 +- Rebuilt for Python 3.8 + +* Mon Jul 29 2019 Martin Sehnoutka - 1-2 +- update upstream URL to the new Github organization + +* Wed Jul 17 2019 Martin Sehnoutka - 1-1 +- Initial package diff --git a/osbuild/__init__.py b/osbuild/__init__.py new file mode 100644 index 0000000..e6d5529 --- /dev/null +++ b/osbuild/__init__.py @@ -0,0 +1,18 @@ +"""OSBuild Module + +The `osbuild` module provides access to the internal features of OSBuild. It +provides parsers for the input and output formats of osbuild, access to shared +infrastructure of osbuild stages, as well as a pipeline executor. + +The utility module `osbuild.util` provides access to common functionality +independent of osbuild but used across the osbuild codebase. +""" + +from .pipeline import Manifest, Pipeline, Stage + + +__all__ = [ + "Manifest", + "Pipeline", + "Stage", +] diff --git a/osbuild/__main__.py b/osbuild/__main__.py new file mode 100755 index 0000000..f5275b0 --- /dev/null +++ b/osbuild/__main__.py @@ -0,0 +1,14 @@ +"""OSBuild Main + +This specifies the entrypoint of the osbuild module when run as executable. For +compatibility we will continue to run the CLI. +""" + +import sys + +from osbuild.main_cli import osbuild_cli as main + + +if __name__ == "__main__": + r = main() + sys.exit(r) diff --git a/osbuild/api.py b/osbuild/api.py new file mode 100644 index 0000000..8ef3521 --- /dev/null +++ b/osbuild/api.py @@ -0,0 +1,216 @@ +import abc +import asyncio +import contextlib +import io +import json +import os +import sys +import tempfile +import traceback +import threading +from typing import Dict, Optional +from .util.types import PathLike +from .util import jsoncomm + + +__all__ = [ + "API" +] + + +class BaseAPI(abc.ABC): + """Base class for all API providers + + This base class provides the basic scaffolding for setting + up API endpoints, normally to be used for bi-directional + communication from and to the sandbox. It is to be used as + a context manager. The communication channel will only be + established on entering the context and will be shut down + when the context is left. + + New messages are delivered via the `_message` method, that + needs to be implemented by deriving classes. + + Optionally, the `_cleanup` method can be implemented, to + clean up resources after the context is left and the + communication channel shut down. + + On incoming messages, first the `_dispatch` method will be + called; the default implementation will receive the message + call `_message.` + """ + + def __init__(self, socket_address: Optional[PathLike] = None): + self.socket_address = socket_address + self.barrier = threading.Barrier(2) + self.event_loop = None + self.thread = None + self._socketdir = None + + @property + @classmethod + @abc.abstractmethod + def endpoint(cls): + """The name of the API endpoint""" + + @abc.abstractmethod + def _message(self, msg: Dict, fds: jsoncomm.FdSet, sock: jsoncomm.Socket): + """Called for a new incoming message + + The file descriptor set `fds` will be closed after the call. + Use the `FdSet.steal()` method to extract file descriptors. + """ + + def _cleanup(self): + """Called after the event loop is shut down""" + + @classmethod + def _make_socket_dir(cls, rundir: PathLike = "/run/osbuild"): + """Called to create the temporary socket dir""" + os.makedirs(rundir, exist_ok=True) + return tempfile.TemporaryDirectory(prefix="api-", dir=rundir) + + def _dispatch(self, sock: jsoncomm.Socket): + """Called when data is available on the socket""" + msg, fds, _ = sock.recv() + if msg is None: + # Peer closed the connection + self.event_loop.remove_reader(sock) + return + self._message(msg, fds, sock) + fds.close() + + def _accept(self, server): + client = server.accept() + if client: + self.event_loop.add_reader(client, self._dispatch, client) + + def _run_event_loop(self): + with jsoncomm.Socket.new_server(self.socket_address) as server: + server.blocking = False + server.listen() + self.barrier.wait() + self.event_loop.add_reader(server, self._accept, server) + asyncio.set_event_loop(self.event_loop) + self.event_loop.run_forever() + self.event_loop.remove_reader(server) + + @property + def running(self): + return self.event_loop is not None + + def __enter__(self): + # We are not re-entrant, so complain if re-entered. + assert not self.running + + if not self.socket_address: + self._socketdir = self._make_socket_dir() + address = os.path.join(self._socketdir.name, self.endpoint) + self.socket_address = address + + self.event_loop = asyncio.new_event_loop() + self.thread = threading.Thread(target=self._run_event_loop) + + self.barrier.reset() + self.thread.start() + self.barrier.wait() + + return self + + def __exit__(self, *args): + self.event_loop.call_soon_threadsafe(self.event_loop.stop) + self.thread.join() + self.event_loop.close() + + # Give deriving classes a chance to clean themselves up + self._cleanup() + + self.thread = None + self.event_loop = None + + if self._socketdir: + self._socketdir.cleanup() + self._socketdir = None + self.socket_address = None + + +class API(BaseAPI): + """The main OSBuild API""" + + endpoint = "osbuild" + + def __init__(self, *, socket_address=None): + super().__init__(socket_address) + self.metadata = {} + self.error = None + + def _set_metadata(self, message, fds): + fd = message["metadata"] + with os.fdopen(fds.steal(fd), encoding="utf-8") as f: + data = json.load(f) + self.metadata.update(data) + + def _get_exception(self, message): + self.error = { + "type": "exception", + "data": message["exception"], + } + + def _message(self, msg, fds, sock): + if msg["method"] == 'add-metadata': + self._set_metadata(msg, fds) + elif msg["method"] == 'exception': + self._get_exception(msg) + + +def exception(e, path="/run/osbuild/api/osbuild"): + """Send exception to osbuild""" + traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr) + with jsoncomm.Socket.new_client(path) as client: + with io.StringIO() as out: + traceback.print_tb(e.__traceback__, file=out) + stacktrace = out.getvalue() + msg = { + "method": "exception", + "exception": { + "type": type(e).__name__, + "value": str(e), + "traceback": stacktrace + } + } + client.send(msg) + + sys.exit(2) + + +# pylint: disable=broad-except +@contextlib.contextmanager +def exception_handler(path="/run/osbuild/api/osbuild"): + try: + yield + except Exception as e: + exception(e, path) + + +def arguments(path="/run/osbuild/api/arguments"): + """Retrieve the input arguments that were supplied to API""" + with open(path, "r", encoding="utf-8") as fp: + data = json.load(fp) + return data + + +def metadata(data: Dict, path="/run/osbuild/api/osbuild"): + """Update metadata for the current module""" + + def data_to_file(): + with tempfile.TemporaryFile() as f: + f.write(json.dumps(data).encode('utf-8')) + # re-open the file to get a read-only file descriptor + return open(f"/proc/self/fd/{f.fileno()}", "r") + + with jsoncomm.Socket.new_client(path) as client, data_to_file() as f: + msg = { + "method": "add-metadata", + "metadata": 0 + } + client.send(msg, fds=[f.fileno()]) diff --git a/osbuild/buildroot.py b/osbuild/buildroot.py new file mode 100644 index 0000000..8dfd8b9 --- /dev/null +++ b/osbuild/buildroot.py @@ -0,0 +1,347 @@ +"""Build Roots + +This implements the file-system environment available to osbuild modules. It +uses `bubblewrap` to contain osbuild modules in a private environment with as +little access to the outside as possible. +""" + +import contextlib +import importlib +import importlib.util +import io +import os +import select +import stat +import subprocess +import tempfile +import time + + +__all__ = [ + "BuildRoot", +] + + +class CompletedBuild: + """The result of a `BuildRoot.run` + + Contains the actual `process` that was executed but also has + convenience properties to quickly access the `returncode` and + `output`. The latter is also provided via `stderr`, `stdout` + properties, making it a drop-in replacement for `CompletedProcess`. + """ + + def __init__(self, proc: subprocess.CompletedProcess, output: str): + self.process = proc + self.output = output + + @property + def returncode(self): + return self.process.returncode + + @property + def stdout(self): + return self.output + + @property + def stderr(self): + return self.output + + +class ProcOverrides: + """Overrides for /proc inside the buildroot""" + + def __init__(self, path) -> None: + self.path = path + self.overrides = set() + + @property + def cmdline(self) -> str: + with open(os.path.join(self.path, "cmdline"), "r") as f: + return f.read().strip() + + @cmdline.setter + def cmdline(self, value) -> None: + with open(os.path.join(self.path, "cmdline"), "w") as f: + f.write(value + "\n") + self.overrides.add("cmdline") + + +# pylint: disable=too-many-instance-attributes +class BuildRoot(contextlib.AbstractContextManager): + """Build Root + + This class implements a context-manager that maintains a root file-system + for contained environments. When entering the context, the required + file-system setup is performed, and it is automatically torn down when + exiting. + + The `run()` method allows running applications in this environment. Some + state is persistent across runs, including data in `/var`. It is deleted + only when exiting the context manager. + """ + + def __init__(self, root, runner, libdir, var, *, rundir="/run/osbuild"): + self._exitstack = None + self._rootdir = root + self._rundir = rundir + self._vardir = var + self._libdir = libdir + self._runner = runner + self._apis = [] + self.dev = None + self.var = None + self.proc = None + self.tmp = None + self.mount_boot = True + + @staticmethod + def _mknod(path, name, mode, major, minor): + os.mknod(os.path.join(path, name), + mode=(stat.S_IMODE(mode) | stat.S_IFCHR), + device=os.makedev(major, minor)) + + def __enter__(self): + self._exitstack = contextlib.ExitStack() + with self._exitstack: + # We create almost everything directly in the container as temporary + # directories and mounts. However, for some things we need external + # setup. For these, we create temporary directories which are then + # bind-mounted into the container. + # + # For now, this includes: + # + # * We create a tmpfs instance *without* `nodev` which we then use + # as `/dev` in the container. This is required for the container + # to create device nodes for loop-devices. + # + # * We create a temporary directory for variable data and then use + # it as '/var' in the container. This allows the container to + # create throw-away data that it does not want to put into a + # tmpfs. + + os.makedirs(self._rundir, exist_ok=True) + dev = tempfile.TemporaryDirectory(prefix="osbuild-dev-", dir=self._rundir) + self.dev = self._exitstack.enter_context(dev) + + os.makedirs(self._vardir, exist_ok=True) + tmp = tempfile.TemporaryDirectory(prefix="osbuild-tmp-", dir=self._vardir) + self.tmp = self._exitstack.enter_context(tmp) + + self.var = os.path.join(self.tmp, "var") + os.makedirs(self.var, exist_ok=True) + + proc = os.path.join(self.tmp, "proc") + os.makedirs(proc) + self.proc = ProcOverrides(proc) + self.proc.cmdline = "root=/dev/osbuild" + + subprocess.run(["mount", "-t", "tmpfs", "-o", "nosuid", "none", self.dev], check=True) + self._exitstack.callback(lambda: subprocess.run(["umount", "--lazy", self.dev], check=True)) + + self._mknod(self.dev, "full", 0o666, 1, 7) + self._mknod(self.dev, "null", 0o666, 1, 3) + self._mknod(self.dev, "random", 0o666, 1, 8) + self._mknod(self.dev, "urandom", 0o666, 1, 9) + self._mknod(self.dev, "tty", 0o666, 5, 0) + self._mknod(self.dev, "zero", 0o666, 1, 5) + + # Prepare all registered API endpoints + for api in self._apis: + self._exitstack.enter_context(api) + + self._exitstack = self._exitstack.pop_all() + + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self._exitstack.close() + self._exitstack = None + + def register_api(self, api: "BaseAPI"): + """Register an API endpoint. + + The context of the API endpoint will be bound to the context of + this `BuildRoot`. + """ + self._apis.append(api) + + if self._exitstack: + self._exitstack.enter_context(api) + + def run(self, argv, monitor, timeout=None, binds=None, readonly_binds=None, extra_env=None): + """Runs a command in the buildroot. + + Takes the command and arguments, as well as bind mounts to mirror + in the build-root for this command. + + This must be called from within an active context of this buildroot + context-manager. + + Returns a `CompletedBuild` object. + """ + + if not self._exitstack: + raise RuntimeError("No active context") + + mounts = [] + + # Import directories from the caller-provided root. + imports = ["usr"] + if self.mount_boot: + imports.insert(0, "boot") + + for p in imports: + source = os.path.join(self._rootdir, p) + if os.path.isdir(source) and not os.path.islink(source): + mounts += ["--ro-bind", source, os.path.join("/", p)] + + # Create /usr symlinks. + mounts += ["--symlink", "usr/lib", "/lib"] + mounts += ["--symlink", "usr/lib64", "/lib64"] + mounts += ["--symlink", "usr/bin", "/bin"] + mounts += ["--symlink", "usr/sbin", "/sbin"] + + # Setup /dev. + mounts += ["--dev-bind", self.dev, "/dev"] + mounts += ["--tmpfs", "/dev/shm"] + + # Setup temporary/data file-systems. + mounts += ["--dir", "/etc"] + mounts += ["--tmpfs", "/run"] + mounts += ["--tmpfs", "/tmp"] + mounts += ["--bind", self.var, "/var"] + + # Setup API file-systems. + mounts += ["--proc", "/proc"] + mounts += ["--ro-bind", "/sys", "/sys"] + mounts += ["--ro-bind-try", "/sys/fs/selinux", "/sys/fs/selinux"] + + # There was a bug in mke2fs (fixed in versionv 1.45.7) where mkfs.ext4 + # would fail because the default config, created on the fly, would + # contain a syntax error. Therefore we bind mount the config from + # the build root, if it exists + mounts += ["--ro-bind-try", + os.path.join(self._rootdir, "etc/mke2fs.conf"), + "/etc/mke2fs.conf"] + + # Skopeo needs things like /etc/containers/policy.json, so take them from buildroot + mounts += ["--ro-bind-try", + os.path.join(self._rootdir, "etc/containers"), + "/etc/containers"] + + # We execute our own modules by bind-mounting them from the host into + # the build-root. We have minimal requirements on the build-root, so + # these modules can be executed. Everything else we provide ourselves. + # In case `libdir` contains the python module, it must be self-contained + # and we provide nothing else. Otherwise, we additionally look for + # the installed `osbuild` module and bind-mount it as well. + mounts += ["--ro-bind", f"{self._libdir}", "/run/osbuild/lib"] + if not os.listdir(os.path.join(self._libdir, "osbuild")): + modorigin = importlib.util.find_spec("osbuild").origin + modpath = os.path.dirname(modorigin) + mounts += ["--ro-bind", f"{modpath}", "/run/osbuild/lib/osbuild"] + + # Setup /proc overrides + for override in self.proc.overrides: + mounts += [ + "--ro-bind", + os.path.join(self.proc.path, override), + os.path.join("/proc", override) + ] + + # Make caller-provided mounts available as well. + for b in binds or []: + mounts += ["--bind"] + b.split(":") + for b in readonly_binds or []: + mounts += ["--ro-bind"] + b.split(":") + + # Prepare all registered API endpoints: bind mount the address with + # the `endpoint` name, provided by the API, into the well known path + mounts += ["--dir", "/run/osbuild/api"] + for api in self._apis: + api_path = "/run/osbuild/api/" + api.endpoint + mounts += ["--bind", api.socket_address, api_path] + + cmd = [ + "bwrap", + "--cap-add", "CAP_MAC_ADMIN", + "--chdir", "/", + "--die-with-parent", + "--new-session", + "--unshare-ipc", + "--unshare-pid", + "--unshare-net" + ] + + cmd += mounts + cmd += ["--", f"/run/osbuild/lib/runners/{self._runner}"] + cmd += argv + + # Setup a new environment for the container. + env = { + "container": "bwrap-osbuild", + "LC_CTYPE": "C.UTF-8", + "PATH": "/usr/sbin:/usr/bin", + "PYTHONPATH": "/run/osbuild/lib", + "PYTHONUNBUFFERED": "1", + "TERM": os.getenv("TERM", "dumb"), + } + if extra_env: + env.update(extra_env) + + proc = subprocess.Popen(cmd, + bufsize=0, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=True) + + data = io.StringIO() + start = time.monotonic() + READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR + poller = select.poll() + poller.register(proc.stdout.fileno(), READ_ONLY) + while True: + buf = self.read_with_timeout(proc, poller, start, timeout) + if not buf: + break + + txt = buf.decode("utf-8") + data.write(txt) + monitor.log(txt) + + poller.unregister(proc.stdout.fileno()) + buf, _ = proc.communicate() + txt = buf.decode("utf-8") + monitor.log(txt) + data.write(txt) + output = data.getvalue() + data.close() + + return CompletedBuild(proc, output) + + @classmethod + def read_with_timeout(cls, proc, poller, start, timeout): + fd = proc.stdout.fileno() + if timeout is None: + return os.read(fd, 32768) + + # convert timeout to milliseconds + remaining = (timeout * 1000) - (time.monotonic() - start) + if remaining <= 0: + proc.terminate() + raise TimeoutError + + buf = None + events = poller.poll(remaining) + if not events: + proc.terminate() + raise TimeoutError + for fd, flag in events: + if flag & (select.POLLIN | select.POLLPRI): + buf = os.read(fd, 32768) + if flag & (select.POLLERR | select.POLLHUP): + proc.terminate() + return buf diff --git a/osbuild/devices.py b/osbuild/devices.py new file mode 100644 index 0000000..9cd6ecb --- /dev/null +++ b/osbuild/devices.py @@ -0,0 +1,127 @@ +""" +Device Handling for pipeline stages + +Specific type of artifacts require device support, such as +loopback devices or device mapper. Since stages are always +run in a container and are isolated from the host, they do +not have direct access to devices and specifically can not +setup new ones. +Therefore device handling is done at the osbuild level with +the help of a device host services. Device specific modules +provide the actual functionality and thus the core device +support in osbuild itself is abstract. +""" + +import abc +import hashlib +import json +import os + +from typing import Dict, Optional + +from osbuild import host + + +class Device: + """ + A single device with its corresponding options + """ + + def __init__(self, name, info, parent, options: Dict): + self.name = name + self.info = info + self.parent = parent + self.options = options or {} + self.id = self.calc_id() + + def calc_id(self): + # NB: Since the name of the device is arbitrary or prescribed + # by the stage, it is not included in the id calculation. + m = hashlib.sha256() + + m.update(json.dumps(self.info.name, sort_keys=True).encode()) + if self.parent: + m.update(json.dumps(self.parent.id, sort_keys=True).encode()) + m.update(json.dumps(self.options, sort_keys=True).encode()) + return m.hexdigest() + + +class DeviceManager: + """Manager for Devices + + Uses a `host.ServiceManager` to open `Device` instances. + """ + + def __init__(self, mgr: host.ServiceManager, devpath: str, tree: str) -> Dict: + self.service_manager = mgr + self.devpath = devpath + self.tree = tree + self.devices = {} + + def device_relpath(self, dev: Optional[Device]) -> Optional[str]: + if dev is None: + return None + return self.devices[dev.name]["path"] + + def device_abspath(self, dev: Optional[Device]) -> Optional[str]: + relpath = self.device_relpath(dev) + if relpath is None: + return None + return os.path.join(self.devpath, relpath) + + def open(self, dev: Device) -> Dict: + + parent = self.device_relpath(dev.parent) + + args = { + # global options + "dev": self.devpath, + "tree": self.tree, + + "parent": parent, + + # per device options + "options": dev.options, + } + + mgr = self.service_manager + + client = mgr.start(f"device/{dev.name}", dev.info.path) + res = client.call("open", args) + + self.devices[dev.name] = res + return res + + +class DeviceService(host.Service): + """Device host service""" + + @abc.abstractmethod + def open(self, devpath: str, parent: str, tree: str, options: Dict): + """Open a specific device + + This method must be implemented by the specific device service. + It should open the device and create a device node in `devpath`. + The return value must contain the relative path to the device + node. + """ + + @abc.abstractmethod + def close(self): + """Close the device""" + + def stop(self): + self.close() + + def dispatch(self, method: str, args, _fds): + if method == "open": + r = self.open(args["dev"], + args["parent"], + args["tree"], + args["options"]) + return r, None + if method == "close": + r = self.close() + return r, None + + raise host.ProtocolError("Unknown method") diff --git a/osbuild/formats/__init__.py b/osbuild/formats/__init__.py new file mode 100644 index 0000000..8eeb5e1 --- /dev/null +++ b/osbuild/formats/__init__.py @@ -0,0 +1,3 @@ +""" +Concrete representation of manifest descriptions +""" diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py new file mode 100644 index 0000000..b9b8522 --- /dev/null +++ b/osbuild/formats/v1.py @@ -0,0 +1,287 @@ +""" Version 1 of the manifest description + +This is the first version of the osbuild manifest description, +that has a "main" pipeline that consists of zero or more stages +to create a tree and optionally one assembler that assembles +the created tree into an artefact. The pipeline can have any +number of nested build pipelines. A sources section is used +to fetch resources. +""" +from typing import Dict +from osbuild.meta import Index, ValidationResult +from ..pipeline import Manifest, Pipeline, detect_host_runner + + +VERSION = "1" + + +def describe(manifest: Manifest, *, with_id=False) -> Dict: + """Create the manifest description for the pipeline""" + def describe_stage(stage): + description = {"name": stage.name} + if stage.options: + description["options"] = stage.options + if with_id: + description["id"] = stage.id + return description + + def describe_pipeline(pipeline: Pipeline) -> Dict: + description = {} + if pipeline.build: + build = manifest[pipeline.build] + description["build"] = { + "pipeline": describe_pipeline(build), + "runner": pipeline.runner + } + + if pipeline.stages: + stages = [describe_stage(s) for s in pipeline.stages] + description["stages"] = stages + + return description + + def get_source_name(source): + name = source.info.name + if name == "org.osbuild.curl": + name = "org.osbuild.files" + return name + + pipeline = describe_pipeline(manifest["tree"]) + + assembler = manifest.get("assembler") + if assembler: + description = describe_stage(assembler.stages[0]) + pipeline["assembler"] = description + + description = {"pipeline": pipeline} + + if manifest.sources: + sources = { + get_source_name(s): s.options + for s in manifest.sources + } + description["sources"] = sources + + return description + + +def load_assembler(description: Dict, index: Index, manifest: Manifest): + pipeline = manifest["tree"] + + build, base, runner = pipeline.build, pipeline.id, pipeline.runner + name, options = description["name"], description.get("options", {}) + + # Add a pipeline with one stage for our assembler + pipeline = manifest.add_pipeline("assembler", runner, build) + + info = index.get_module_info("Assembler", name) + + stage = pipeline.add_stage(info, options, {}) + info = index.get_module_info("Input", "org.osbuild.tree") + ip = stage.add_input("tree", info, "org.osbuild.pipeline") + ip.add_reference(base) + return pipeline + + +def load_build(description: Dict, index: Index, manifest: Manifest, n: int): + pipeline = description.get("pipeline") + if pipeline: + build_pipeline = load_pipeline(pipeline, index, manifest, n + 1) + else: + build_pipeline = None + + return build_pipeline, description["runner"] + + +def load_stage(description: Dict, index: Index, pipeline: Pipeline): + name = description["name"] + opts = description.get("options", {}) + info = index.get_module_info("Stage", name) + + stage = pipeline.add_stage(info, opts) + + if stage.name == "org.osbuild.rpm": + info = index.get_module_info("Input", "org.osbuild.files") + ip = stage.add_input("packages", info, "org.osbuild.source") + for pkg in stage.options["packages"]: + options = None + if isinstance(pkg, dict): + gpg = pkg.get("check_gpg") + if gpg: + options = {"metadata": {"rpm.check_gpg": gpg}} + pkg = pkg["checksum"] + ip.add_reference(pkg, options) + elif stage.name == "org.osbuild.ostree": + info = index.get_module_info("Input", "org.osbuild.ostree") + ip = stage.add_input("commits", info, "org.osbuild.source") + commit, ref = opts["commit"], opts.get("ref") + options = {"ref": ref} if ref else None + ip.add_reference(commit, options) + + +def load_source(name: str, description: Dict, index: Index, manifest: Manifest): + if name == "org.osbuild.files": + name = "org.osbuild.curl" + + info = index.get_module_info("Source", name) + + if name == "org.osbuild.curl": + items = description["urls"] + elif name == "org.osbuild.ostree": + items = description["commits"] + else: + raise ValueError(f"Unknown source type: {name}") + + # NB: the entries, i.e. `urls`, `commits` are left in the + # description dict, although the sources are not using + # it anymore. The reason is that it makes `describe` work + # without any special casing + + manifest.add_source(info, items, description) + + +def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = 0) -> Pipeline: + build = description.get("build") + if build: + build_pipeline, runner = load_build(build, index, manifest, n) + else: + build_pipeline, runner = None, detect_host_runner() + + # the "main" pipeline is called `tree`, since it is building the + # tree that will later be used by the `assembler`. Nested build + # pipelines will get call "build", and "build-build-...", where + # the number of repetitions is equal their level of nesting + if not n: + name = "tree" + else: + name = "-".join(["build"] * n) + + build_id = build_pipeline and build_pipeline.id + pipeline = manifest.add_pipeline(name, runner, build_id) + + for stage in description.get("stages", []): + load_stage(stage, index, pipeline) + + return pipeline + + +def load(description: Dict, index: Index) -> Manifest: + """Load a manifest description""" + + pipeline = description.get("pipeline", {}) + sources = description.get("sources", {}) + + manifest = Manifest() + + load_pipeline(pipeline, index, manifest) + + # load the assembler, if any + assembler = pipeline.get("assembler") + if assembler: + load_assembler(assembler, index, manifest) + + # load the sources + for name, desc in sources.items(): + load_source(name, desc, index, manifest) + + for pipeline in manifest.pipelines.values(): + for stage in pipeline.stages: + stage.sources = sources + + return manifest + + +def output(manifest: Manifest, res: Dict) -> Dict: + """Convert a result into the v1 format""" + + def result_for_pipeline(pipeline): + # The pipeline might not have been built one of its + # dependencies, i.e. its build pipeline, failed to + # build. We thus need to be tolerant of a missing + # result but still need to to recurse + current = res.get(pipeline.id, {}) + retval = { + "success": current.get("success", True) + } + + if pipeline.build: + build = manifest[pipeline.build] + retval["build"] = result_for_pipeline(build) + retval["success"] = retval["build"]["success"] + + stages = current.get("stages") + if stages: + retval["stages"] = stages + return retval + + result = result_for_pipeline(manifest["tree"]) + + assembler = manifest.get("assembler") + if not assembler: + return result + + current = res.get(assembler.id) + # if there was an error before getting to the assembler + # pipeline, there might not be a result present + if not current: + return result + + result["assembler"] = current["stages"][0] + if not result["assembler"]["success"]: + result["success"] = False + + return result + + +def validate(manifest: Dict, index: Index) -> ValidationResult: + """Validate a OSBuild manifest + + This function will validate a OSBuild manifest, including + all its stages and assembler and build manifests. It will + try to validate as much as possible and not stop on errors. + The result is a `ValidationResult` object that can be used + to check the overall validation status and iterate all the + individual validation errors. + """ + + schema = index.get_schema("Manifest") + result = schema.validate(manifest) + + # main pipeline + pipeline = manifest.get("pipeline", {}) + + # recursively validate the build pipeline as a "normal" + # pipeline in order to validate its stages and assembler + # options; for this it is being re-parented in a new plain + # {"pipeline": ...} dictionary. NB: Any nested structural + # errors might be detected twice, but de-duplicated by the + # `ValidationResult.merge` call + build = pipeline.get("build", {}).get("pipeline") + if build: + res = validate({"pipeline": build}, index=index) + result.merge(res, path=["pipeline", "build"]) + + stages = pipeline.get("stages", []) + for i, stage in enumerate(stages): + name = stage["name"] + schema = index.get_schema("Stage", name) + res = schema.validate(stage) + result.merge(res, path=["pipeline", "stages", i]) + + asm = pipeline.get("assembler", {}) + if asm: + name = asm["name"] + schema = index.get_schema("Assembler", name) + res = schema.validate(asm) + result.merge(res, path=["pipeline", "assembler"]) + + # sources + sources = manifest.get("sources", {}) + for name, source in sources.items(): + if name == "org.osbuild.files": + name = "org.osbuild.curl" + schema = index.get_schema("Source", name) + res = schema.validate(source) + result.merge(res, path=["sources", name]) + + return result diff --git a/osbuild/formats/v2.py b/osbuild/formats/v2.py new file mode 100644 index 0000000..824a19f --- /dev/null +++ b/osbuild/formats/v2.py @@ -0,0 +1,495 @@ +""" Version 2 of the manifest description + +Second, and current, version of the manifest description +""" +from typing import Dict +from osbuild.meta import Index, ModuleInfo, ValidationResult +from ..inputs import Input +from ..pipeline import Manifest, Pipeline, Stage, detect_host_runner +from ..sources import Source + + +VERSION = "2" + + +# pylint: disable=too-many-statements +def describe(manifest: Manifest, *, with_id=False) -> Dict: + + # Undo the build, runner pairing introduce by the loading + # code. See the comment there for more details + runners = { + p.build: p.runner for p in manifest.pipelines.values() + if p.build + } + + def pipeline_ref(pid): + if with_id: + return pid + + pl = manifest[pid] + return f"name:{pl.name}" + + def describe_device(dev): + desc = { + "type": dev.info.name + } + + if dev.options: + desc["options"] = dev.options + + return desc + + def describe_devices(devs: Dict): + desc = { + name: describe_device(dev) + for name, dev in devs.items() + } + return desc + + def describe_input(ip: Input): + origin = ip.origin + desc = { + "type": ip.info.name, + "origin": origin, + } + if ip.options: + desc["options"] = ip.options + + refs = {} + for name, ref in ip.refs.items(): + if origin == "org.osbuild.pipeline": + name = pipeline_ref(name) + refs[name] = ref + + if refs: + desc["references"] = refs + + return desc + + def describe_inputs(ips: Dict[str, Input]): + desc = { + name: describe_input(ip) + for name, ip in ips.items() + } + return desc + + def describe_mount(mnt): + desc = { + "name": mnt.name, + "type": mnt.info.name, + "target": mnt.target + } + + if mnt.device: + desc["source"] = mnt.device.name + + if mnt.options: + desc["options"] = mnt.options + return desc + + def describe_mounts(mounts: Dict): + desc = [ + describe_mount(mnt) + for mnt in mounts.values() + ] + return desc + + def describe_stage(s: Stage): + desc = { + "type": s.info.name + } + + if with_id: + desc["id"] = s.id + + if s.options: + desc["options"] = s.options + + devs = describe_devices(s.devices) + if devs: + desc["devices"] = devs + + mounts = describe_mounts(s.mounts) + if mounts: + desc["mounts"] = mounts + + ips = describe_inputs(s.inputs) + if ips: + desc["inputs"] = ips + + return desc + + def describe_pipeline(p: Pipeline): + desc = { + "name": p.name + } + + if p.build: + desc["build"] = pipeline_ref(p.build) + + runner = runners.get(p.id) + if runner: + desc["runner"] = runner + + stages = [ + describe_stage(stage) + for stage in p.stages + ] + + if stages: + desc["stages"] = stages + + return desc + + def describe_source(s: Source): + desc = { + "items": s.items + } + + return desc + + pipelines = [ + describe_pipeline(pipeline) + for pipeline in manifest.pipelines.values() + ] + + sources = { + source.info.name: describe_source(source) + for source in manifest.sources + } + + description = { + "version": VERSION, + "pipelines": pipelines + } + + if sources: + description["sources"] = sources + + return description + + +def resolve_ref(name: str, manifest: Manifest) -> str: + ref = name[5:] + target = manifest.pipelines.get(ref) + if not target: + raise ValueError(f"Unknown pipeline reference: name:{ref}") + return target.id + + +def sort_devices(devices: Dict) -> Dict: + """Sort the devices so that dependencies are in the correct order + + We need to ensure that parents are sorted before the devices that + depend on them. For this we keep a list of devices that need to + be processed and iterate over that list as long as it has devices + in them and we make progress, i.e. the length changes. + """ + result = {} + todo = list(devices.keys()) + + while todo: + before = len(todo) + + for i, name in enumerate(todo): + desc = devices[name] + + parent = desc.get("parent") + if parent and not parent in result: + # if the parent is not in the `result` list, it must + # be in `todo`; otherwise it is missing + if parent not in todo: + msg = f"Missing parent device '{parent}' for '{name}'" + raise ValueError(msg) + + continue + + # no parent, or parent already present, ok to add to the + # result and "remove" from the todo list, by setting the + # contents to `None`. + result[name] = desc + todo[i] = None + + todo = list(filter(bool, todo)) + if len(todo) == before: + # we made no progress, which means that all devices in todo + # depend on other devices in todo, hence we have a cycle + raise ValueError("Cycle detected in 'devices'") + + return result + + +def load_device(name: str, description: Dict, index: Index, stage: Stage): + device_type = description["type"] + options = description.get("options", {}) + parent = description.get("parent") + + if parent: + device = stage.devices.get(parent) + if not parent: + raise ValueError(f"Unknown parent device: {parent}") + parent = device + + info = index.get_module_info("Device", device_type) + + if not info: + raise TypeError(f"Missing meta information for {device_type}") + stage.add_device(name, info, parent, options) + + +def load_input(name: str, description: Dict, index: Index, stage: Stage, manifest: Manifest, source_refs: set): + input_type = description["type"] + origin = description["origin"] + options = description.get("options", {}) + + info = index.get_module_info("Input", input_type) + ip = stage.add_input(name, info, origin, options) + + refs = description.get("references", {}) + + if isinstance(refs, list): + refs = {r: {} for r in refs} + + if origin == "org.osbuild.pipeline": + resolved = {} + for r, desc in refs.items(): + if not r.startswith("name:"): + continue + target = resolve_ref(r, manifest) + resolved[target] = desc + refs = resolved + elif origin == "org.osbuild.source": + unknown_refs = set(refs.keys()) - source_refs + if unknown_refs: + raise ValueError(f"Unknown source reference(s) {unknown_refs}") + + for r, desc in refs.items(): + ip.add_reference(r, desc) + + +def load_mount(description: Dict, index: Index, stage: Stage): + mount_type = description["type"] + info = index.get_module_info("Mount", mount_type) + + name = description["name"] + + if name in stage.mounts: + raise ValueError(f"Duplicated mount '{name}'") + + source = description.get("source") + target = description.get("target") + + options = description.get("options", {}) + + device = None + if source: + device = stage.devices.get(source) + if not device: + raise ValueError(f"Unknown device '{source}' for mount '{name}'") + + stage.add_mount(name, info, device, target, options) + + +def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Manifest, source_refs): + stage_type = description["type"] + opts = description.get("options", {}) + info = index.get_module_info("Stage", stage_type) + + stage = pipeline.add_stage(info, opts) + + devs = description.get("devices", {}) + devs = sort_devices(devs) + + for name, desc in devs.items(): + load_device(name, desc, index, stage) + + ips = description.get("inputs", {}) + for name, desc in ips.items(): + load_input(name, desc, index, stage, manifest, source_refs) + + mounts = description.get("mounts", []) + for mount in mounts: + load_mount(mount, index, stage) + + return stage + + +def load_pipeline(description: Dict, index: Index, manifest: Manifest, source_refs: set): + name = description["name"] + build = description.get("build") + runner = description.get("runner") + source_epoch = description.get("source-epoch") + + if build and build.startswith("name:"): + target = resolve_ref(build, manifest) + build = target + + pl = manifest.add_pipeline(name, runner, build, source_epoch) + + for desc in description.get("stages", []): + load_stage(desc, index, pl, manifest, source_refs) + + +def load(description: Dict, index: Index) -> Manifest: + """Load a manifest description""" + + sources = description.get("sources", {}) + pipelines = description.get("pipelines", []) + + manifest = Manifest() + source_refs = set() + + # load the sources + for name, desc in sources.items(): + info = index.get_module_info("Source", name) + items = desc.get("items", {}) + options = desc.get("options", {}) + manifest.add_source(info, items, options) + source_refs.update(items.keys()) + + for desc in pipelines: + load_pipeline(desc, index, manifest, source_refs) + + # The "runner" property in the manifest format is the + # runner to the run the pipeline with. In osbuild the + # "runner" property belongs to the "build" pipeline, + # i.e. is what runner to use for it. This we have to + # go through the pipelines and fix things up + pipelines = manifest.pipelines.values() + + host_runner = detect_host_runner() + runners = { + pl.id: pl.runner for pl in pipelines + } + + for pipeline in pipelines: + if not pipeline.build: + pipeline.runner = host_runner + continue + + runner = runners[pipeline.build] + pipeline.runner = runner + + return manifest + + +#pylint: disable=too-many-branches +def output(manifest: Manifest, res: Dict) -> Dict: + """Convert a result into the v2 format""" + + if not res["success"]: + last = list(res.keys())[-1] + failed = res[last]["stages"][-1] + + result = { + "type": "error", + "success": False, + "error": { + "type": "org.osbuild.error.stage", + "details": { + "stage": { + "id": failed["id"], + "type": failed["name"], + "output": failed["output"], + "error": failed["error"] + } + } + } + } + else: + result = { + "type": "result", + "success": True, + "metadata": {} + } + + # gather all the metadata + for p in manifest.pipelines.values(): + data = {} + r = res.get(p.id, {}) + for stage in r.get("stages", []): + md = stage.get("metadata") + if not md: + continue + name = stage["name"] + val = data.setdefault(name, {}) + val.update(md) + + if data: + result["metadata"][p.name] = data + + # generate the log + result["log"] = {} + for p in manifest.pipelines.values(): + r = res.get(p.id, {}) + log = [] + + for stage in r.get("stages", []): + data = { + "id": stage["id"], + "type": stage["name"], + "output": stage["output"] + } + if not stage["success"]: + data["success"] = stage["success"] + if stage["error"]: + data["error"] = stage["error"] + + log.append(data) + + if log: + result["log"][p.name] = log + + return result + + +def validate(manifest: Dict, index: Index) -> ValidationResult: + + schema = index.get_schema("Manifest", version="2") + result = schema.validate(manifest) + + def validate_module(mod, klass, path): + name = mod.get("type") + if not name: + return + schema = index.get_schema(klass, name, version="2") + res = schema.validate(mod) + result.merge(res, path=path) + + def validate_stage_modules(klass, stage, path): + group = ModuleInfo.MODULES[klass] + items = stage.get(group, {}) + + if isinstance(items, list): + items = {i["name"]: i for i in items} + + for name, mod in items.items(): + validate_module(mod, klass, path + [group, name]) + + def validate_stage(stage, path): + name = stage["type"] + schema = index.get_schema("Stage", name, version="2") + res = schema.validate(stage) + result.merge(res, path=path) + + for mod in ("Device", "Input", "Mount"): + validate_stage_modules(mod, stage, path) + + def validate_pipeline(pipeline, path): + stages = pipeline.get("stages", []) + for i, stage in enumerate(stages): + validate_stage(stage, path + ["stages", i]) + + # sources + sources = manifest.get("sources", {}) + for name, source in sources.items(): + schema = index.get_schema("Source", name, version="2") + res = schema.validate(source) + result.merge(res, path=["sources", name]) + + # pipelines + pipelines = manifest.get("pipelines", []) + for i, pipeline in enumerate(pipelines): + validate_pipeline(pipeline, path=["pipelines", i]) + + return result diff --git a/osbuild/host.py b/osbuild/host.py new file mode 100644 index 0000000..5d278af --- /dev/null +++ b/osbuild/host.py @@ -0,0 +1,540 @@ +""" +Functionality provided by the host + +The biggest functionality this module provides are so called host +services: + +Stages run inside a container to isolate them from the host which +the build is run on. This means that the stages do not have direct +access to certain features offered by the host system, like access +to the network, devices as well as the osbuild store itself. + +Host services are a way to provide functionality to stages that is +restricted to the host and not directly available in the container. + +A service itself is an executable that gets spawned by osbuild on- +demand and communicates with osbuild via a simple JSON based IPC +protocol. To ease the development of such services the `Service` +class of this module can be used, which sets up and handles the +communication with the host. + +On the host side a `ServiceManager` can be used to spawn and manage +concrete services. Specifically it functions as a context manager +and will shut down services when the context exits. + +The `ServiceClient` class provides a client for the services and can +thus be used to interact with the service from the host side. + +A note about host service lifetimes: The host service lifetime is +meant to be bound to the service it provides, e.g. when the service +provides data to a stage, it is meant that this data is accessible +for exactly as long as the binary is run and all resources must be +freed when the service is stopped. +The idea behind this design is to ensure that no resources get +leaked because only the host service itself is responsible for +their clean up, independent of any control of osbuild. +""" + +import abc +import argparse +import asyncio +import fcntl +import importlib +import io +import os +import signal +import subprocess +import sys +import threading +import traceback +from collections import OrderedDict +from typing import Any, Dict, List, Optional, Tuple, Callable + +from osbuild.util.jsoncomm import FdSet, Socket + + +class ProtocolError(Exception): + """Errors concerning the communication between host and service""" + + +class RemoteError(Exception): + """A RemoteError indicates an unexpected error in the service""" + + def __init__(self, name, value, stack) -> None: + self.name = name + self.value = value + self.stack = stack + msg = f"{name}: {value}\n {stack}" + super().__init__(msg) + + +class ServiceProtocol: + """ + Wire protocol between host and service + + The ServicePortocol specifies the wire protocol between the host + and the service. It contains methods to translate messages into + their wire format and back. + """ + + @staticmethod + def decode_message(msg: Dict) -> Tuple[str, Dict]: + if not msg: + raise ProtocolError("message empty") + + t = msg.get("type") + if not t: + raise ProtocolError("'type' field missing") + + d = msg.get("data") + if not d: + raise ProtocolError("'data' field missing") + return t, d + + @staticmethod + def encode_method(name: str, arguments: List): + msg = { + "type": "method", + "data": { + "name": name, + "args": arguments, + } + } + return msg + + @staticmethod + def decode_method(data: Dict): + name = data.get("name") + if not name: + raise ProtocolError("'name' field missing") + + args = data.get("args", []) + return name, args + + @staticmethod + def encode_reply(reply: Any): + msg = { + "type": "reply", + "data": { + "reply": reply + } + } + return msg + + @staticmethod + def decode_reply(msg: Dict) -> Any: + if "reply" not in msg: + raise ProtocolError("'reply' field missing") + + data = msg["reply"] + # NB: This is the returned data of the remote + # method call, which can also be `None` + return data + + @staticmethod + def encode_signal(sig: Any): + msg = { + "type": "signal", + "data": { + "reply": sig + } + } + return msg + + @staticmethod + def encode_exception(value, tb): + backtrace = "".join(traceback.format_tb(tb)) + msg = { + "type": "exception", + "data": { + "name": value.__class__.__name__, + "value": str(value), + "backtrace": backtrace + } + } + return msg + + @staticmethod + def decode_exception(data): + name = data["name"] + value = data["value"] + tb = data["backtrace"] + + return RemoteError(name, value, tb) + + +class Service(abc.ABC): + """ + Host service + + This abstract base class provides all the base functionality to + implement a host service. Specifically, it handles the setup of + the service itself and the communication with osbuild. + + The `dispatch` method needs to be implemented by deriving + classes to handle remote method calls. + + The `stop` method should be implemented to tear down state and + free resources. + """ + + protocol = ServiceProtocol + + def __init__(self, args: argparse.Namespace): + + self.sock = Socket.new_from_fd(args.service_fd) + self.id = args.service_id + + @classmethod + def from_args(cls, argv): + """Create a service object given an argument vector""" + + parser = cls.prepare_argument_parser() + args = parser.parse_args(argv) + return cls(args) + + @classmethod + def prepare_argument_parser(cls): + """Prepare the command line argument parser""" + + name = __class__.__name__ + + desc = f"osbuild {name} host service" + parser = argparse.ArgumentParser(description=desc) + + parser.add_argument("--service-fd", metavar="FD", type=int, + help="service file descriptor") + parser.add_argument("--service-id", metavar="ID", type=str, + help="service identifier") + return parser + + @abc.abstractmethod + def dispatch(self, method: str, args: Any, fds: FdSet): + """Handle remote method calls + + This method must be overridden in order to handle remote + method calls. The incoming arguments are the method name, + `method` and its arguments, `args`, together with a set + of file descriptors (optional). The reply to this method + will form the return value of the remote call. + """ + + def stop(self): + """Service is stopping + + This method will be called when the service is stopping, + and should be overridden to tear down state and free + resources allocated by the service. + + NB: This method might be called at any time due to signals, + even during the handling method calls. + """ + + def main(self): + """Main service entry point + + This method should be invoked in the service executable + to actually run the service. After additional setup this + will call the `serve` method to wait for remote method + calls. + """ + + # We ignore `SIGTERM` and `SIGINT` here, so that the + # controlling process (osbuild) can shut down all host + # services in a controlled fashion and in the correct + # order by closing the communication socket. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + + try: + self.serve() + finally: + self.stop() + + def serve(self): + """Serve remote requests + + Wait for remote method calls and translate them into + calls to `dispatch`. + """ + + while True: + msg, fds, _ = self.sock.recv() + if not msg: + break + + reply_fds = None + try: + reply, reply_fds = self._handle_message(msg, fds) + + # Catch invalid file descriptors early so that + # we send an error reply instead of throwing + # an exception in `sock.send` later. + self._check_fds(reply_fds) + + except: # pylint: disable=bare-except + reply_fds = self._close_all(reply_fds) + _, val, tb = sys.exc_info() + reply = self.protocol.encode_exception(val, tb) + + finally: + fds.close() + + try: + self.sock.send(reply, fds=reply_fds) + except BrokenPipeError: + break + finally: + self._close_all(reply_fds) + + def _handle_message(self, msg, fds): + """ + Internal method called by `service` to handle new messages + """ + + kind, data = self.protocol.decode_message(msg) + + if kind != "method": + raise ProtocolError(f"unknown message type: {kind}") + + name, args = self.protocol.decode_method(data) + ret, fds = self.dispatch(name, args, fds) + msg = self.protocol.encode_reply(ret) + + return msg, fds + + def emit_signal(self, data: Any, fds: Optional[list] = None): + self._check_fds(fds) + self.sock.send(self.protocol.encode_signal(data), fds=fds) + + @staticmethod + def _close_all(fds: Optional[List[int]]): + if not fds: + return [] + + for fd in fds: + try: + os.close(fd) + except OSError as e: + print(f"error closing fd '{fd}': {e!s}") + return [] + + @staticmethod + def _check_fds(fds: Optional[List[int]]): + if not fds: + return + + for fd in fds: + fcntl.fcntl(fd, fcntl.F_GETFD) + + +class ServiceClient: + """ + Host service client + + Can be used to remotely call methods on the host services. Normally + returned from the `ServiceManager` when starting a new host service. + """ + protocol = ServiceProtocol + + def __init__(self, uid, proc, sock): + self.uid = uid + self.proc = proc + self.sock = sock + + def call(self, method: str, args: Optional[Any] = None) -> Any: + """Remotely call a method and return the result""" + + ret, _ = self.call_with_fds(method, args) + return ret + + def call_with_fds(self, method: str, + args: Optional[Any] = None, + fds: Optional[List] = None, + on_signal: Callable[[Any, FdSet], None] = None) -> Tuple[Any, FdSet]: + """ + Remotely call a method and return the result, including file + descriptors. + """ + + msg = self.protocol.encode_method(method, args) + + self.sock.send(msg, fds=fds) + + while True: + ret, fds, _ = self.sock.recv() + kind, data = self.protocol.decode_message(ret) + if kind == "signal": + ret = self.protocol.decode_reply(data) + on_signal(ret, fds) + if kind == "reply": + ret = self.protocol.decode_reply(data) + return ret, fds + if kind == "exception": + error = self.protocol.decode_exception(data) + raise error + + raise ProtocolError(f"unknown message type: {kind}") + + def stop(self): + """ + Stop the host service associated with this client. + """ + + self.sock.close() + self.proc.wait() + + +class ServiceManager: + """ + Host service manager + + Manager, i.e. `start` and `stop` host services. Must be used as a + context manager. When the context is active, host services can be + started via the `start` method. + + When a `monitor` is provided, stdout and stderr of the service will + be forwarded to the monitor via `monitor.log`, otherwise sys.stdout + is used. + """ + + def __init__(self, *, monitor=None): + self.services = OrderedDict() + self.monitor = monitor + + self.barrier = threading.Barrier(2) + self.event_loop = None + self.thread = None + + @property + def running(self): + """Return whether the service manager is running""" + return self.event_loop is not None + + @staticmethod + def make_env(): + # We want the `osbuild` python package that contains this + # very module, which might be different from the system wide + # installed one, to be accessible to the Input programs so + # we detect our origin and set the `PYTHONPATH` accordingly + modorigin = importlib.util.find_spec("osbuild").origin + modpath = os.path.dirname(modorigin) + env = os.environ.copy() + env["PYTHONPATH"] = os.path.dirname(modpath) + env["PYTHONUNBUFFERED"] = "1" + return env + + def start(self, uid, cmd, extra_args=None) -> ServiceClient: + """ + Start a new host service + + Create a new host service with the unique identifier `uid` by + spawning the executable provided via `cmd` with optional extra + arguments `extra_args`. + + The return value is a `ServiceClient` instance that is already + connected to the service and can thus be used to call methods. + + NB: Must be called with an active context + """ + + if not self.running: + raise RuntimeError("ServiceManager not running") + + if uid in self.services: + raise ValueError(f"{uid} already started") + + ours, theirs = Socket.new_pair() + env = self.make_env() + + try: + fd = theirs.fileno() + argv = [ + cmd, + "--service-id", uid, + "--service-fd", str(fd) + ] + + if extra_args: + argv += extra_args + + proc = subprocess.Popen(argv, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + pass_fds=(fd, ), + close_fds=True) + + service = ServiceClient(uid, proc, ours) + self.services[uid] = service + ours = None + + stdout = io.TextIOWrapper(proc.stdout, + encoding="utf-8", + line_buffering=True) + + name = os.path.basename(cmd) + + def reader(): + return self._stdout_ready(name, uid, stdout) + + self.event_loop.add_reader(stdout, reader) + + finally: + if ours: + ours.close() + + return service + + def stop(self, uid): + """ + Stop a service given its unique identifier, `uid` + """ + + service = self.services.get(uid) + if not service: + raise ValueError(f"unknown service: {uid}") + + service.stop() + + def _stdout_ready(self, name, uid, stdout): + txt = stdout.readline() + if not txt: + self.event_loop.remove_reader(stdout) + return + + msg = f"{uid} ({name}): {txt}" + if self.monitor: + self.monitor.log(msg) + else: + print(msg, end="") + + def _thread_main(self): + self.barrier.wait() + asyncio.set_event_loop(self.event_loop) + self.event_loop.run_forever() + + def __enter__(self): + # We are not re-entrant, so complain if re-entered. + assert not self.running + + self.event_loop = asyncio.new_event_loop() + self.thread = threading.Thread(target=self._thread_main) + + self.barrier.reset() + self.thread.start() + self.barrier.wait() + + return self + + def __exit__(self, *args): + # Stop all registered services + while self.services: + _, srv = self.services.popitem() + srv.stop() + + self.event_loop.call_soon_threadsafe(self.event_loop.stop) + self.thread.join() + self.event_loop.close() diff --git a/osbuild/inputs.py b/osbuild/inputs.py new file mode 100644 index 0000000..4a029a9 --- /dev/null +++ b/osbuild/inputs.py @@ -0,0 +1,121 @@ +""" +Pipeline inputs + +A pipeline input provides data in various forms to a `Stage`, like +files, OSTree commits or trees. The content can either be obtained +via a `Source` or have been built by a `Pipeline`. Thus an `Input` +is the bridge between various types of content that originate from +different types of sources. + +The acceptable origin of the data is determined by the `Input` +itself. What types of input are allowed and required is determined +by the `Stage`. + +To osbuild itself this is all transparent. The only data visible to +osbuild is the path. The input options are just passed to the +`Input` as is and the result is forwarded to the `Stage`. +""" + +import abc +import hashlib +import json +import os + +from typing import Dict, Optional, Tuple + +from osbuild import host +from osbuild.util.types import PathLike +from .objectstore import StoreClient, StoreServer + + +class Input: + """ + A single input with its corresponding options. + """ + + def __init__(self, name, info, origin: str, options: Dict): + self.name = name + self.info = info + self.origin = origin + self.refs = {} + self.options = options or {} + self.id = self.calc_id() + + def add_reference(self, ref, options: Optional[Dict] = None): + self.refs[ref] = options or {} + self.id = self.calc_id() + + def calc_id(self): + + # NB: The input `name` is not included here on purpose since it + # is either prescribed by the stage itself and thus not actual + # parameter or arbitrary and chosen by the manifest generator + # and thus can be changed without affecting the contents + m = hashlib.sha256() + m.update(json.dumps(self.info.name, sort_keys=True).encode()) + m.update(json.dumps(self.origin, sort_keys=True).encode()) + m.update(json.dumps(self.refs, sort_keys=True).encode()) + m.update(json.dumps(self.options, sort_keys=True).encode()) + return m.hexdigest() + + def map(self, + mgr: host.ServiceManager, + storeapi: StoreServer, + root: PathLike) -> Tuple[str, Dict]: + + target = os.path.join(root, self.name) + os.makedirs(target) + + args = { + # mandatory bits + "origin": self.origin, + "refs": self.refs, + + "target": target, + + # global options + "options": self.options, + + # API endpoints + "api": { + "store": storeapi.socket_address + } + } + + client = mgr.start(f"input/{self.name}", self.info.path) + reply = client.call("map", args) + + path = reply["path"] + + if not path.startswith(root): + raise RuntimeError(f"returned {path} has wrong prefix") + + reply["path"] = os.path.relpath(path, root) + + return reply + + +class InputService(host.Service): + """Input host service""" + + @abc.abstractmethod + def map(self, store, origin, refs, target, options): + pass + + def unmap(self): + pass + + def stop(self): + self.unmap() + + def dispatch(self, method: str, args, _fds): + if method == "map": + store = StoreClient(connect_to=args["api"]["store"]) + r = self.map(store, + args["origin"], + args["refs"], + args["target"], + args["options"]) + return r, None + + raise host.ProtocolError("Unknown method") diff --git a/osbuild/loop.py b/osbuild/loop.py new file mode 100644 index 0000000..391523d --- /dev/null +++ b/osbuild/loop.py @@ -0,0 +1,617 @@ +import contextlib +import ctypes +import errno +import fcntl +import os +import stat +import time +from typing import Callable, Optional + +from .util import linux + +__all__ = [ + "Loop", + "LoopControl", + "UnexpectedDevice" +] + + +class UnexpectedDevice(Exception): + def __init__(self, expected_minor, rdev, mode): + super().__init__() + self.expected_minor = expected_minor + self.rdev = rdev + self.mode = mode + + +class LoopInfo(ctypes.Structure): + _fields_ = [ + ('lo_device', ctypes.c_uint64), + ('lo_inode', ctypes.c_uint64), + ('lo_rdevice', ctypes.c_uint64), + ('lo_offset', ctypes.c_uint64), + ('lo_sizelimit', ctypes.c_uint64), + ('lo_number', ctypes.c_uint32), + ('lo_encrypt_type', ctypes.c_uint32), + ('lo_encrypt_key_size', ctypes.c_uint32), + ('lo_flags', ctypes.c_uint32), + ('lo_file_name', ctypes.c_uint8 * 64), + ('lo_crypt_name', ctypes.c_uint8 * 64), + ('lo_encrypt_key', ctypes.c_uint8 * 32), + ('lo_init', ctypes.c_uint64 * 2) + ] + + @property + def autoclear(self) -> bool: + """Return if `LO_FLAGS_AUTOCLEAR` is set in `lo_flags`""" + return bool(self.lo_flags & Loop.LO_FLAGS_AUTOCLEAR) + + def is_bound_to(self, info: os.stat_result) -> bool: + """Return if the loop device is bound to the file `info`""" + return (self.lo_device == info.st_dev and + self.lo_inode == info.st_ino) + + +class Loop: + """Loopback device + + A class representing a Linux loopback device, typically found at + /dev/loop{minor}. + + Methods + ------- + set_fd(fd) + Bind a file descriptor to the loopback device + clear_fd() + Unbind the file descriptor from the loopback device + change_fd(fd) + Replace the bound file descriptor + set_capacity() + Re-read the capacity of the backing file + set_status(offset=None, sizelimit=None, autoclear=None, partscan=None) + Set properties of the loopback device + mknod(dir_fd, mode=0o600) + Create a secondary device node + """ + + LOOP_MAJOR = 7 + + LO_FLAGS_READ_ONLY = 1 + LO_FLAGS_AUTOCLEAR = 4 + LO_FLAGS_PARTSCAN = 8 + LO_FLAGS_DIRECT_IO = 16 + + LOOP_SET_FD = 0x4C00 + LOOP_CLR_FD = 0x4C01 + LOOP_SET_STATUS64 = 0x4C04 + LOOP_GET_STATUS64 = 0x4C05 + LOOP_CHANGE_FD = 0x4C06 + LOOP_SET_CAPACITY = 0x4C07 + LOOP_SET_DIRECT_IO = 0x4C08 + LOOP_SET_BLOCK_SIZE = 0x4C09 + + def __init__(self, minor, dir_fd=None): + """ + Parameters + ---------- + minor + the minor number of the underlying device + dir_fd : int, optional + A directory file descriptor to a filesystem containing the + underlying device node, or None to use /dev (default is None) + + Raises + ------ + UnexpectedDevice + If the file in the expected device node location is not the + expected device node + """ + + self.devname = f"loop{minor}" + self.minor = minor + self.on_close: Optional[Callable[["Loop"], None]] = None + + with contextlib.ExitStack() as stack: + if not dir_fd: + dir_fd = os.open("/dev", os.O_DIRECTORY) + stack.callback(lambda: os.close(dir_fd)) + self.fd = os.open(self.devname, os.O_RDWR, dir_fd=dir_fd) + + info = os.stat(self.fd) + if ((not stat.S_ISBLK(info.st_mode)) or + (not os.major(info.st_rdev) == self.LOOP_MAJOR) or + (not os.minor(info.st_rdev) == minor)): + raise UnexpectedDevice(minor, info.st_rdev, info.st_mode) + + def __del__(self): + self.close() + + def close(self): + """Close this loop device. + + No operations on this object are valid after this call. + """ + fd, self.fd = self.fd, -1 + if fd >= 0: + if callable(self.on_close): + self.on_close(self) # pylint: disable=not-callable + os.close(fd) + self.devname = "" + + def flock(self, op: int) -> None: + """Add or remove an advisory lock on the loopback device + + Perform a lock operation on the loopback device via `flock(2)`. + + The locks are per file-descriptor and thus duplicated fds share + the same lock. The lock is automatically released when all of + those duplicated fds are closed or an explicit `LOCK_UN` call + was made on any of them. + + NB: These locks are advisory only and are not preventing anyone + from actually accessing the device, but they will prevent udev + probing the device, see https://systemd.io/BLOCK_DEVICE_LOCKING + + If the file is already locked any attempt to lock it again via + a different (non-duped) fd will block or, if `fcntl.LOCK_NB` + is specified, will raise a `BlockingIOError`. + + Parameters + ---------- + op : int + the lock operation to perform; one, or a combination, of: + `fcntl.LOCK_EX`: exclusive lock + `fcntl.LOCK_SH`: shared lock + `fcntl.LOCK_NB`: don't block on lock acquisition + `fcntl.LOCK_UN`: unlock + """ + + fcntl.flock(self.fd, op) + + def flush_buf(self) -> None: + """Flush the buffer cache of the loopback device + + This function might be required to be called before the usage + of `clear_fd`. It seems that the kernel (as of version 5.13.8) + is not clearing the buffer cache of the block device layer in + case the fd is manually cleared. + + NB: This function needs the `CAP_SYS_ADMIN` capability. + """ + + linux.ioctl_blockdev_flushbuf(self.fd) + + def set_fd(self, fd): + """Bind a file descriptor to the loopback device + + The loopback device must be unbound. The backing file must be + either a regular file or a block device. If the backing file is + itself a loopback device, then a cycle must not be created. If + the backing file is opened read-only, then the resulting + loopback device will be read-only too. + + Parameters + ---------- + fd : int + the file descriptor to bind + """ + + fcntl.ioctl(self.fd, self.LOOP_SET_FD, fd) + + def clear_fd(self): + """Unbind the file descriptor from the loopback device + + The loopback device must be bound. The device is then marked + to be cleared, so once nobody holds it open any longer the + backing file is unbound and the device returns to the unbound + state. + """ + + fcntl.ioctl(self.fd, self.LOOP_CLR_FD) + + def clear_fd_wait(self, fd: int, timeout: float, wait: float = 0.1) -> None: + """Wait until the file descriptor is cleared + + When clearing the file descriptor of the loopback device the + kernel will check if the loop device has a reference count + greater then one(!), i.e. if another fd besied the one trying + to clear the loopback device is open. If so it will only set + the `LO_FLAGS_AUTOCLEAR` flag and wait until the the device + is released. This means we cannot be sure the loopback device + is actually cleared. + To alleviated this situation we wait until the the loop is not + bound anymore or not bound to `fd` anymore (in case someone + else bound it between checks). + + Raises a `TimeoutError` if the file descriptor when `timeout` + is reached. + + Parameters + ---------- + fd : int + the file descriptor to wait for + timeout : float + the maximum time to wait in seconds + wait : float + the time to wait between each check in seconds + """ + + file_info = os.fstat(fd) + endtime = time.monotonic() + timeout + + # wait until the loop device is unbound, which means calling + # `get_status` will fail with `ENXIO` or if someone raced us + # and bound the loop device again, it is not backed by "our" + # file descriptor specified via `fd` anymore + while True: + + try: + self.clear_fd() + loop_info = self.get_status() + + except OSError as err: + + # check if the loop is still bound + if err.errno == errno.ENXIO: + return + + # check if it is backed by the fd + if not loop_info.is_bound_to(file_info): + return + + if time.monotonic() > endtime: + raise TimeoutError("waiting for loop device timed out") + + time.sleep(wait) + + def change_fd(self, fd): + """Replace the bound filedescriptor + + Atomically replace the backing filedescriptor of the loopback + device, even if the device is held open. + + The effective size (taking sizelimit into account) of the new + and existing backing file descriptors must be the same, and + the loopback device must be read-only. The loopback device will + remain read-only, even if the new file descriptor was opened + read-write. + + Parameters + ---------- + fd : int + the file descriptor to change to + """ + + fcntl.ioctl(self.fd, self.LOOP_CHANGE_FD, fd) + + def is_bound_to(self, fd: int) -> bool: + """Check if the loopback device is bound to `fd` + + Checks if the loopback device is bound and, if so, whether the + backing file refers to the same file as `fd`. The latter is + done by comparing the device and inode information. + + Parameters + ---------- + fd : int + the file descriptor to check + + Returns + ------- + bool + True if the loopback device is bound to the file descriptor + """ + + try: + loop_info = self.get_status() + except OSError as err: + + # raised if the loopback is bound at all + if err.errno == errno.ENXIO: + return False + + file_info = os.fstat(fd) + + # it is bound, check if it is bound by `fd` + return loop_info.is_bound_to(file_info) + + def set_status(self, offset=None, sizelimit=None, autoclear=None, partscan=None): + """Set properties of the loopback device + + The loopback device must be bound, and the properties will be + cleared once the device is unbound, but preserved by changing + the backing file descriptor. + + Note that this operation is not atomic: All the current properties + are read out, the ones specified in this function call are modified, + and then they are written back. For this reason, concurrent + modification of the properties must be avoided. + + Setting sizelimit means the size of the loopback device is taken + to be the max of the size of the backing file and the limit. A + limit of 0 is taken to mean unlimited. + + Enabling autoclear has the same effect as calling clear_fd(). + + When partscan is first enabled, the partition table of the + device is scanned, and new blockdevices potentially added for + the partitions. + + Parameters + ---------- + offset : int, optional + The offset in bytes from the start of the backing file, or + None to leave unchanged (default is None) + sizelimit : int, optional + The max size in bytes to make the loopback device, or None + to leave unchanged (default is None) + autoclear : bool, optional + Whether or not to enable autoclear, or None to leave unchanged + (default is None) + partscan : bool, optional + Whether or not to enable partition scanning, or None to leave + unchanged (default is None) + """ + + info = self.get_status() + if offset: + info.lo_offset = offset + if sizelimit: + info.lo_sizelimit = sizelimit + if autoclear is not None: + if autoclear: + info.lo_flags |= self.LO_FLAGS_AUTOCLEAR + else: + info.lo_flags &= ~self.LO_FLAGS_AUTOCLEAR + if partscan is not None: + if partscan: + info.lo_flags |= self.LO_FLAGS_PARTSCAN + else: + info.lo_flags &= ~self.LO_FLAGS_PARTSCAN + fcntl.ioctl(self.fd, self.LOOP_SET_STATUS64, info) + + def get_status(self) -> LoopInfo: + """Get properties of the loopback device + + Return a `LoopInfo` structure with the information of this + loopback device. See loop(4) for more information. + """ + + info = LoopInfo() + fcntl.ioctl(self.fd, self.LOOP_GET_STATUS64, info) + return info + + def set_direct_io(self, dio=True): + """Set the direct-IO property on the loopback device + + Enabling direct IO allows one to avoid double caching, which + should improve performance and memory usage. + + Parameters + ---------- + dio : bool, optional + Whether or not to enable direct IO (default is True) + """ + + fcntl.ioctl(self.fd, self.LOOP_SET_DIRECT_IO, dio) + + def mknod(self, dir_fd, mode=0o600): + """Create a secondary device node + + Create a device node with the correct name, mode, minor and major + number in the provided directory. + + Note that the device node will survive even if a device is + unbound and rebound, so anyone with access to the device node + will have access to any future devices with the same minor + number. The intended use of this is to first bind a file + descriptor to a loopback device, then mknod it where it should + be accessed from, and only after the destination directory is + ensured to have been destroyed/made inaccessible should the the + loopback device be unbound. + + Note that the provided directory should not be devtmpfs, as the + device node is guaranteed to already exist there, and the call + would hence fail. + + Parameters + ---------- + dir_fd : int + Target directory file descriptor + mode : int, optional + Access mode on the created device node (0o600 is default) + """ + + os.mknod(self.devname, + mode=(stat.S_IMODE(mode) | stat.S_IFBLK), + device=os.makedev(self.LOOP_MAJOR, self.minor), + dir_fd=dir_fd) + + +class LoopControl: + """Loopback control device + + A class representing the Linux loopback control device, typically + found at /dev/loop-control. It allows the creation and destruction + of loopback devices. + + A loopback device may be bound, which means that a file descriptor + has been attached to it as its backing file. Otherwise, it is + considered unbound. + + Methods + ------- + add(minor) + Add a new loopback device + remove(minor) + Remove an existing loopback device + get_unbound() + Get or create the first unbound loopback device + """ + + LOOP_CTL_ADD = 0x4C80 + LOOP_CTL_REMOVE = 0x4C81 + LOOP_CTL_GET_FREE = 0x4C82 + + def __init__(self, dir_fd=None): + """ + Parameters + ---------- + dir_fd : int, optional + A directory filedescriptor to a devtmpfs filesystem, + or None to use /dev (default is None) + """ + + with contextlib.ExitStack() as stack: + if not dir_fd: + dir_fd = os.open("/dev", os.O_DIRECTORY) + stack.callback(lambda: os.close(dir_fd)) + + self.fd = os.open("loop-control", os.O_RDWR, dir_fd=dir_fd) + + def __del__(self): + self.close() + + def _check_open(self): + if self.fd < 0: + raise RuntimeError("LoopControl closed") + + def close(self): + """Close the loop control file-descriptor + + No operations on this object are valid after this call, + with the exception of this `close` method which then + is a no-op. + """ + if self.fd >= 0: + os.close(self.fd) + self.fd = -1 + + def add(self, minor=-1): + """Add a new loopback device + + Add a new, unbound loopback device. If a minor number is given + and it is positive, a loopback device with that minor number + is added. Otherwise, if there are no unbound devices, a device + using the first unused minor number is created. + + Parameters + ---------- + minor : int, optional + The requested minor number, or a negative value for + unspecified (default is -1) + + Returns + ------- + int + The minor number of the created device + """ + + self._check_open() + return fcntl.ioctl(self.fd, self.LOOP_CTL_ADD, minor) + + def remove(self, minor=-1): + """Remove an existing loopback device + + Removes an unbound and unopen loopback device. If a minor + number is given and it is positive, the loopback device + with that minor number is removed. Otherwise, the first + unbound device is attempted removed. + + Parameters + ---------- + minor : int, optional + The requested minor number, or a negative value for + unspecified (default is -1) + """ + + self._check_open() + fcntl.ioctl(self.fd, self.LOOP_CTL_REMOVE, minor) + + def get_unbound(self): + """Get or create an unbound loopback device + + If an unbound loopback device exists, returns it. + Otherwise, create a new one. + + Returns + ------- + int + The minor number of the returned device + """ + + self._check_open() + return fcntl.ioctl(self.fd, self.LOOP_CTL_GET_FREE) + + def loop_for_fd(self, + fd: int, + lock: bool = False, + setup: Optional[Callable[[Loop], None]] = None, + **kwargs): + """ + Get or create an unbound loopback device and bind it to an fd + + Getting an unbound loopback device, attaching a backing file + descriptor and setting the loop device status is racy so this + method will retry until it succeeds or it fails to get an + unbound loop device. + + If `lock` is set, an exclusive advisory lock will be taken + on the device before the device gets configured. If this + fails, the next loop device will be tried. + Locking the device can be helpful to prevent systemd-udevd from + reacting to changes to the device, like processing udev rules. + See https://systemd.io/BLOCK_DEVICE_LOCKING/ + + A callback can be specified via `setup` that will be invoked + after the loop device is opened but before any other operation + is done, such as setting the backing file. + + All given keyword arguments except `lock` are forwarded to the + `Loop.set_status` call. + """ + + self._check_open() + + if fd < 0: + raise ValueError(f"Invalid file descriptor '{fd}'") + + while True: + lo = Loop(self.get_unbound()) + + # if a setup callback is specified invoke it now + if callable(setup): + try: + setup(lo) + except: + lo.close() + raise + + # try to lock the device if requested and use a + # different one if it fails + if lock: + try: + lo.flock(fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + lo.close() + continue + + try: + lo.set_fd(fd) + except OSError as e: + lo.close() + if e.errno == errno.EBUSY: + continue + raise e + + # `set_status` returns EBUSY when the pages from the + # previously bound file have not been fully cleared yet. + try: + lo.set_status(**kwargs) + except BlockingIOError: + lo.clear_fd() + lo.close() + continue + break + + return lo diff --git a/osbuild/main_cli.py b/osbuild/main_cli.py new file mode 100644 index 0000000..786d8a7 --- /dev/null +++ b/osbuild/main_cli.py @@ -0,0 +1,189 @@ +"""Entrypoints for osbuild + +This module contains the application and API entrypoints of `osbuild`, the +command-line-interface to osbuild. The `osbuild_cli()` entrypoint can be safely +used from tests to run the cli. +""" + + +import argparse +import json +import os +import sys + +import osbuild +import osbuild.meta +import osbuild.monitor +from osbuild.objectstore import ObjectStore + + +RESET = "\033[0m" +BOLD = "\033[1m" +RED = "\033[31m" +GREEN = "\033[32m" + + +def parse_manifest(path): + if path == "-": + manifest = json.load(sys.stdin) + else: + with open(path) as f: + manifest = json.load(f) + + return manifest + + +def show_validation(result, name): + if name == "-": + name = "" + + print(f"{BOLD}{name}{RESET} ", end='') + + if result: + print(f"is {BOLD}{GREEN}valid{RESET}") + return + + print(f"has {BOLD}{RED}errors{RESET}:") + print("") + + for error in result: + print(f"{BOLD}{error.id}{RESET}:") + print(f" {error.message}\n") + + +def export(name_or_id, output_directory, store, manifest): + pipeline = manifest[name_or_id] + obj = store.get(pipeline.id) + dest = os.path.join(output_directory, name_or_id) + + os.makedirs(dest, exist_ok=True) + obj.export(dest) + + +def parse_arguments(sys_argv): + parser = argparse.ArgumentParser(description="Build operating system images") + + parser.add_argument("manifest_path", metavar="MANIFEST", + help="json file containing the manifest that should be built, or a '-' to read from stdin") + parser.add_argument("--store", metavar="DIRECTORY", type=os.path.abspath, + default=".osbuild", + help="directory where intermediary os trees are stored") + parser.add_argument("-l", "--libdir", metavar="DIRECTORY", type=os.path.abspath, default="/usr/lib/osbuild", + help="the directory containing stages, assemblers, and the osbuild library") + parser.add_argument("--checkpoint", metavar="ID", action="append", type=str, default=None, + help="stage to commit to the object store during build (can be passed multiple times)") + parser.add_argument("--export", metavar="ID", action="append", type=str, default=[], + help="object to export, can be passed multiple times") + parser.add_argument("--json", action="store_true", + help="output results in JSON format") + parser.add_argument("--output-directory", metavar="DIRECTORY", type=os.path.abspath, + help="directory where result objects are stored") + parser.add_argument("--inspect", action="store_true", + help="return the manifest in JSON format including all the ids") + parser.add_argument("--monitor", metavar="NAME", default=None, + help="Name of the monitor to be used") + parser.add_argument("--monitor-fd", metavar="FD", type=int, default=sys.stdout.fileno(), + help="File descriptor to be used for the monitor") + parser.add_argument("--stage-timeout", type=int, default=None, + help="set the maximal time (in seconds) each stage is allowed to run") + + return parser.parse_args(sys_argv[1:]) + + +# pylint: disable=too-many-branches,too-many-return-statements,too-many-statements +def osbuild_cli(): + args = parse_arguments(sys.argv) + desc = parse_manifest(args.manifest_path) + + index = osbuild.meta.Index(args.libdir) + + # detect the format from the manifest description + info = index.detect_format_info(desc) + if not info: + print("Unsupported manifest format") + return 2 + fmt = info.module + + # first thing is validation of the manifest + res = fmt.validate(desc, index) + if not res: + if args.json or args.inspect: + json.dump(res.as_dict(), sys.stdout) + sys.stdout.write("\n") + else: + show_validation(res, args.manifest_path) + return 2 + + manifest = fmt.load(desc, index) + + exports = set(args.export) + unresolved = [e for e in exports if e not in manifest] + if unresolved: + for name in unresolved: + print(f"Export {BOLD}{name}{RESET} not found!") + print(f"{RESET}{BOLD}{RED}Failed{RESET}") + return 1 + + if args.checkpoint: + missed = manifest.mark_checkpoints(args.checkpoint) + if missed: + for checkpoint in missed: + print(f"Checkpoint {BOLD}{checkpoint}{RESET} not found!") + print(f"{RESET}{BOLD}{RED}Failed{RESET}") + return 1 + + if args.inspect: + result = fmt.describe(manifest, with_id=True) + json.dump(result, sys.stdout) + sys.stdout.write("\n") + return 0 + + output_directory = args.output_directory + + if exports and not output_directory: + print("Need --output-directory for --export") + return 1 + + monitor_name = args.monitor + if not monitor_name: + monitor_name = "NullMonitor" if args.json else "LogMonitor" + monitor = osbuild.monitor.make(monitor_name, args.monitor_fd) + + try: + with ObjectStore(args.store) as object_store: + stage_timeout = args.stage_timeout + + pipelines = manifest.depsolve(object_store, exports) + + manifest.download(object_store, monitor, args.libdir) + + r = manifest.build( + object_store, + pipelines, + monitor, + args.libdir, + stage_timeout=stage_timeout + ) + + if r["success"] and exports: + for pid in exports: + export(pid, output_directory, object_store, manifest) + + except KeyboardInterrupt: + print() + print(f"{RESET}{BOLD}{RED}Aborted{RESET}") + return 130 + + if args.json: + r = fmt.output(manifest, r) + json.dump(r, sys.stdout) + sys.stdout.write("\n") + else: + if r["success"]: + for name, pl in manifest.pipelines.items(): + print(f"{name + ':': <10}\t{pl.id}") + else: + print() + print(f"{RESET}{BOLD}{RED}Failed{RESET}") + + return 0 if r["success"] else 1 diff --git a/osbuild/meta.py b/osbuild/meta.py new file mode 100644 index 0000000..3c083a6 --- /dev/null +++ b/osbuild/meta.py @@ -0,0 +1,559 @@ +"""Introspection and validation for osbuild + +This module contains utilities that help to introspect parts +that constitute the inner parts of osbuild, i.e. its stages, +assemblers and sources. Additionally, it provides classes and +functions to do schema validation of OSBuild manifests and +module options. + +A central `Index` class can be used to obtain stage and schema +information. For the former a `ModuleInfo` class is returned via +`Index.get_module_info`, which contains meta-information about +the individual stages. Schemata, obtained via `Index.get_schema` +is represented via a `Schema` class that can in turn be used +to validate the individual components. +Additionally, the `Index` also provides meta information about +the different formats and version that are supported to read +manifest descriptions and write output data. Fir this a class +called `FormatInfo` together with `Index.get_format_inf` and +`Index.list_formats` is provided. A `FormatInfo` can also be +inferred for a specific manifest description via a helper +method called `detect_format_info` +""" +import ast +import contextlib +import copy +import importlib.util +import os +import pkgutil +import json +import sys +from collections import deque +from typing import Dict, Iterable, List, Optional + +import jsonschema + + +FAILED_TITLE = "JSON Schema validation failed" +FAILED_TYPEURI = "https://osbuild.org/validation-error" + + +class ValidationError: + """Describes a single failed validation + + Consists of a `message` member describing the error + that occurred and a `path` that points to the element + that caused the error. + Implements hashing, equality and less-than and thus + can be sorted and used in sets and dictionaries. + """ + + def __init__(self, message: str): + self.message = message + self.path = deque() + + @classmethod + def from_exception(cls, ex): + err = cls(ex.message) + err.path = ex.absolute_path + return err + + @property + def id(self): + if not self.path: + return "." + + result = "" + for p in self.path: + if isinstance(p, str): + if " " in p: + p = f"'{p}'" + result += "." + p + elif isinstance(p, int): + result += f"[{p}]" + else: + raise AssertionError("new type") + + return result + + def as_dict(self): + """Serializes this object as a dictionary + + The `path` member will be serialized as a list of + components (string or integer) and `message` the + human readable message string. + """ + return { + "message": self.message, + "path": list(self.path) + } + + def rebase(self, path: Iterable[str]): + """Prepend the `path` to `self.path`""" + rev = reversed(path) + self.path.extendleft(rev) + + def __hash__(self): + return hash((self.id, self.message)) + + def __eq__(self, other: "ValidationError"): + if not isinstance(other, ValidationError): + raise ValueError("Need ValidationError") + + if self.id != other.id: + return False + return self.message == other.message + + def __lt__(self, other: "ValidationError"): + if not isinstance(other, ValidationError): + raise ValueError("Need ValidationError") + + return self.id < other.id + + def __str__(self): + return f"ValidationError: {self.message} [{self.id}]" + + +class ValidationResult: + """Result of a JSON Schema validation""" + + def __init__(self, origin: Optional[str]): + self.origin = origin + self.errors = set() + + def fail(self, msg: str) -> ValidationError: + """Add a new `ValidationError` with `msg` as message""" + err = ValidationError(msg) + self.errors.add(err) + return err + + def add(self, err: ValidationError): + """Add a `ValidationError` to the set of errors""" + self.errors.add(err) + return self + + def merge(self, result: "ValidationResult", *, path=None): + """Merge all errors of `result` into this + + Merge all the errors of in `result` into this, + adjusting their the paths be pre-pending the + supplied `path`. + """ + for err in result: + err = copy.deepcopy(err) + err.rebase(path or []) + self.errors.add(err) + + def as_dict(self): + """Represent this result as a dictionary + + If there are not errors, returns an empty dict; + otherwise it will contain a `type`, `title` and + `errors` field. The `title` is a human readable + description, the `type` is a URI identifying + the validation error type and errors is a list + of `ValueErrors`, in turn serialized as dict. + Additionally, a `success` member is provided to + be compatible with pipeline build results. + """ + errors = [e.as_dict() for e in self] + if not errors: + return {} + + return { + "type": FAILED_TYPEURI, + "title": FAILED_TITLE, + "success": False, + "errors": errors + } + + @property + def valid(self): + """Returns `True` if there are zero errors""" + return len(self) == 0 + + def __iadd__(self, error: ValidationError): + return self.add(error) + + def __bool__(self): + return self.valid + + def __len__(self): + return len(self.errors) + + def __iter__(self): + return iter(sorted(self.errors)) + + def __str__(self): + return f"ValidationResult: {len(self)} error(s)" + + def __getitem__(self, key): + if not isinstance(key, str): + raise ValueError("Only string keys allowed") + + lst = list(filter(lambda e: e.id == key, self)) + if not lst: + raise IndexError(f"{key} not found") + + return lst + + +class Schema: + """JSON Schema representation + + Class that represents a JSON schema. The `data` attribute + contains the actual schema data itself. The `klass` and + (optional) `name` refer to entity this schema belongs to. + The schema information can be used to validate data via + the `validate` method. + + The class can be created with empty schema data. In that + case it represents missing schema information. Any call + to `validate` will then result in a failure. + + The truth value of this objects corresponds to it having + schema data. + """ + + def __init__(self, schema: str, name: Optional[str] = None): + self.data = schema + self.name = name + self._validator = None + + def check(self) -> ValidationResult: + """Validate the `schema` data itself""" + res = ValidationResult(self.name) + + # validator is assigned if and only if the schema + # itself passes validation (see below). Therefore + # this can be taken as an indicator for a valid + # schema and thus we can and should short-circuit + if self._validator: + return res + + if not self.data: + res.fail("missing schema information") + return res + + try: + Validator = jsonschema.Draft4Validator + Validator.check_schema(self.data) + self._validator = Validator(self.data) + except jsonschema.exceptions.SchemaError as err: + res += ValidationError.from_exception(err) + + return res + + def validate(self, target) -> ValidationResult: + """Validate the `target` against this schema + + If the schema information itself is missing, it + will return a `ValidationResult` in failed state, + with 'missing schema information' as the reason. + """ + res = self.check() + if not res: + return res + + for error in self._validator.iter_errors(target): + res += ValidationError.from_exception(error) + + return res + + def __bool__(self): + return self.check().valid + + +class ModuleInfo: + """Meta information about a stage + + Represents the information about a osbuild pipeline + modules, like a stage, assembler or source. + Contains the short description (`desc`), a longer + description (`info`) and the raw schema data for + its valid options (`opts`). To use the schema data + the `get_schema` method can be used to obtain a + `Schema` object. + + Normally this class is instantiated via its `load` method. + """ + + # Known modules and their corresponding directory name + MODULES = { + "Assembler": "assemblers", + "Device": "devices", + "Input": "inputs", + "Mount": "mounts", + "Source": "sources", + "Stage": "stages", + } + + def __init__(self, klass: str, name: str, path: str, info: Dict): + self.name = name + self.type = klass + self.path = path + + self.info = info["info"] + self.desc = info["desc"] + self.opts = info["schema"] + + def _load_opts(self, version, fallback=None): + raw = self.opts[version] + if not raw and fallback: + raw = self.opts[fallback] + if not raw: + raise ValueError(f"Unsupported version: {version}") + return raw + + def _make_options(self, version): + if version == "2": + raw = self.opts["2"] + if not raw: + return self._make_options("1") + elif version == "1": + raw = {"options": self.opts["1"]} + else: + raise ValueError(f"Unsupported version: {version}") + + return raw + + def get_schema(self, version="1"): + schema = { + "title": f"Pipeline {self.type}", + "type": "object", + "additionalProperties": False, + } + + if self.type in ("Stage", "Assembler"): + type_id = "type" if version == "2" else "name" + opts = self._make_options(version) + schema["properties"] = { + type_id: {"enum": [self.name]}, + **opts, + } + if "mounts" not in schema["properties"]: + schema["properties"]["mounts"] = { + "type": "array" + } + schema["required"] = [type_id] + elif self.type in ("Device"): + schema["additionalProperties"] = True + opts = self._load_opts(version, "1") + schema["properties"] = { + "type": {"enum": [self.name]}, + "options": opts + } + elif self.type in ("Mount"): + opts = self._load_opts("2") + schema.update(opts) + schema["properties"]["type"] = { + "enum": [self.name], + } + else: + opts = self._load_opts(version, "1") + schema.update(opts) + + # if there are is a definitions node, it needs to be at + # the top level schema node, since the schema inside the + # stages is written as-if they were the root node and + # so are the references + props = schema.get("properties", {}) + if "definitions" in props: + schema["definitions"] = props["definitions"] + del props["definitions"] + + options = props.get("options", {}) + if "definitions" in options: + schema["definitions"] = options["definitions"] + del options["definitions"] + + return schema + + @classmethod + def _parse_schema(cls, klass, name, node): + if not node: + return {} + + value = node.value + if not isinstance(value, ast.Str): + return {} + + try: + return json.loads("{" + value.s + "}") + except json.decoder.JSONDecodeError as e: + msg = "Invalid schema: " + e.msg + line = e.doc.splitlines()[e.lineno - 1] + fullname = cls.MODULES[klass] + "/" + name + lineno = e.lineno + node.lineno - 1 + detail = fullname, lineno, e.colno, line + raise SyntaxError(msg, detail) from None + + @classmethod + def load(cls, root, klass, name) -> Optional["ModuleInfo"]: + names = ["SCHEMA", "SCHEMA_2"] + + def filter_type(lst, target): + return [x for x in lst if isinstance(x, target)] + + def targets(a): + return [t.id for t in filter_type(a.targets, ast.Name)] + + base = cls.MODULES.get(klass) + if not base: + raise ValueError(f"Unsupported type: {klass}") + + path = os.path.join(root, base, name) + try: + with open(path) as f: + data = f.read() + except FileNotFoundError: + return None + + tree = ast.parse(data, name) + + docstring = ast.get_docstring(tree) + doclist = docstring.split("\n") + + assigns = filter_type(tree.body, ast.Assign) + values = { + t: a + for a in assigns + for t in targets(a) + if t in names + } + + def parse_schema(node): + return cls._parse_schema(klass, name, node) + + info = { + 'schema': { + "1": parse_schema(values.get("SCHEMA")), + "2": parse_schema(values.get("SCHEMA_2")), + }, + 'desc': doclist[0], + 'info': "\n".join(doclist[1:]) + } + return cls(klass, name, path, info) + + +class FormatInfo: + """Meta information about a format + + Class the can be used to get meta information about + the the different formats in which osbuild accepts + manifest descriptions and writes results. + """ + + def __init__(self, module): + self.module = module + self.version = getattr(module, "VERSION") + docs = getattr(module, "__doc__") + info, desc = docs.split("\n", 1) + self.info = info.strip() + self.desc = desc.strip() + + @classmethod + def load(cls, name): + mod = sys.modules.get(name) + if not mod: + mod = importlib.import_module(name) + if not mod: + raise ValueError(f"Could not load module {name}") + return cls(mod) + + +class Index: + """Index of modules and formats + + Class that can be used to get the meta information about + osbuild modules as well as JSON schemata. + """ + + def __init__(self, path: str): + self.path = path + self._module_info = {} + self._format_info = {} + self._schemata = {} + + @staticmethod + def list_formats() -> List[str]: + """List all known formats for manifest descriptions""" + base = "osbuild.formats" + spec = importlib.util.find_spec(base) + locations = spec.submodule_search_locations + modinfo = [ + mod for mod in pkgutil.walk_packages(locations) + if not mod.ispkg + ] + + return [base + "." + m.name for m in modinfo] + + def get_format_info(self, name) -> FormatInfo: + """Get the `FormatInfo` for the format called `name`""" + info = self._format_info.get(name) + if not info: + info = FormatInfo.load(name) + self._format_info[name] = info + return info + + def detect_format_info(self, data) -> Optional[FormatInfo]: + """Obtain a `FormatInfo` for the format that can handle `data`""" + formats = self.list_formats() + version = data.get("version", "1") + for fmt in formats: + info = self.get_format_info(fmt) + if info.version == version: + return info + return None + + def list_modules_for_class(self, klass: str) -> List[str]: + """List all available modules for the given `klass`""" + module_path = ModuleInfo.MODULES.get(klass) + + if not module_path: + raise ValueError(f"Unsupported nodule class: {klass}") + + path = os.path.join(self.path, module_path) + modules = filter(lambda f: os.path.isfile(f"{path}/{f}"), + os.listdir(path)) + return list(modules) + + def get_module_info(self, klass, name) -> Optional[ModuleInfo]: + """Obtain `ModuleInfo` for a given stage or assembler""" + + if (klass, name) not in self._module_info: + + info = ModuleInfo.load(self.path, klass, name) + self._module_info[(klass, name)] = info + + return self._module_info[(klass, name)] + + def get_schema(self, klass, name=None, version="1") -> Schema: + """Obtain a `Schema` for `klass` and `name` (optional) + + Returns a `Schema` for the entity identified via `klass` + and `name` (if given). Always returns a `Schema` even if + no schema information could be found for the entity. In + that case the actual schema data for `Schema` will be + `None` and any validation will fail. + """ + schema = self._schemata.get((klass, name, version)) + if schema is not None: + return schema + + if klass == "Manifest": + path = f"{self.path}/schemas/osbuild{version}.json" + with contextlib.suppress(FileNotFoundError): + with open(path, "r") as f: + schema = json.load(f) + elif klass in ModuleInfo.MODULES: + info = self.get_module_info(klass, name) + if info: + schema = info.get_schema(version) + else: + raise ValueError(f"Unknown klass: {klass}") + + schema = Schema(schema, name or klass) + self._schemata[(klass, name, version)] = schema + + return schema diff --git a/osbuild/monitor.py b/osbuild/monitor.py new file mode 100644 index 0000000..c87c4a3 --- /dev/null +++ b/osbuild/monitor.py @@ -0,0 +1,139 @@ +""" +Monitor pipeline activity + +The osbuild `Pipeline` class supports monitoring of its activities +by providing a monitor object that implements the `BaseMonitor` +interface. During the execution of the pipeline various functions +are called on the monitor object at certain events. Consult the +`BaseMonitor` class for the description of all available events. +""" + +import abc +import json +import os +import sys +import time + +from typing import Dict + +import osbuild + + +RESET = "\033[0m" +BOLD = "\033[1m" + + +class TextWriter: + """Helper class for writing text to file descriptors""" + def __init__(self, fd: int): + self.fd = fd + self.isatty = os.isatty(fd) + + def term(self, text, *, clear=False): + """Write text if attached to a terminal.""" + if not self.isatty: + return + + if clear: + self.write(RESET) + + self.write(text) + + def write(self, text: str): + """Write all of text to the log file descriptor""" + data = text.encode("utf-8") + n = len(data) + while n: + k = os.write(self.fd, data) + n -= k + if n: + data = data[n:] + + +class BaseMonitor(abc.ABC): + """Base class for all pipeline monitors""" + + def __init__(self, fd: int): + """Logging will be done to file descriptor `fd`""" + self.out = TextWriter(fd) + + def begin(self, pipeline: osbuild.Pipeline): + """Called once at the beginning of a build""" + + def finish(self, result: Dict): + """Called at the very end of the build""" + + def stage(self, stage: osbuild.Stage): + """Called when a stage is being built""" + + def assembler(self, assembler: osbuild.Stage): + """Called when an assembler is being built""" + + def result(self, result: osbuild.pipeline.BuildResult): + """Called when a module is done with its result""" + + def log(self, message: str): + """Called for all module log outputs""" + + +class NullMonitor(BaseMonitor): + """Monitor class that does not report anything""" + + +class LogMonitor(BaseMonitor): + """Monitor that follows show the log output of modules + + This monitor will print a header with `name: id` followed + by the options for each module as it is being built. The + full log messages of the modules will be print as soon as + they become available. + The constructor argument `fd` is a file descriptor, where + the log will get written to. If `fd` is a `TTY`, escape + sequences will be used to highlight sections of the log. + """ + def result(self, result): + duration = int(time.time() - self.timer_start) + self.out.write(f"\n⏱ Duration: {duration}s\n") + + def begin(self, pipeline): + self.out.term(BOLD, clear=True) + self.out.write(f"Pipeline {pipeline.name}: {pipeline.id}") + self.out.term(RESET) + self.out.write("\n") + + def stage(self, stage): + self.module(stage) + + def assembler(self, assembler): + self.out.term(BOLD, clear=True) + self.out.write("Assembler ") + self.out.term(RESET) + + self.module(assembler) + + def module(self, module): + options = module.options or {} + title = f"{module.name}: {module.id}" + + self.out.term(BOLD, clear=True) + self.out.write(title) + self.out.term(RESET) + self.out.write(" ") + + json.dump(options, self.out, indent=2) + self.out.write("\n") + + self.timer_start = time.time() + + def log(self, message): + self.out.write(message) + + +def make(name, fd): + module = sys.modules[__name__] + monitor = getattr(module, name, None) + if not monitor: + raise ValueError(f"Unknown monitor: {name}") + if not issubclass(monitor, BaseMonitor): + raise ValueError(f"Invalid monitor: {name}") + return monitor(fd) diff --git a/osbuild/mounts.py b/osbuild/mounts.py new file mode 100644 index 0000000..7f46b89 --- /dev/null +++ b/osbuild/mounts.py @@ -0,0 +1,168 @@ +""" +Mount Handling for pipeline stages + +Allows stages to access file systems provided by devices. +This makes mount handling transparent to the stages, i.e. +the individual stages do not need any code for different +file system types and the underlying devices. +""" + +import abc +import hashlib +import json +import os +import subprocess + +from typing import Dict + +from osbuild import host +from osbuild.devices import DeviceManager + + +class Mount: + """ + A single mount with its corresponding options + """ + + def __init__(self, name, info, device, target, options: Dict): + self.name = name + self.info = info + self.device = device + self.target = target + self.options = options + self.id = self.calc_id() + + def calc_id(self): + m = hashlib.sha256() + m.update(json.dumps(self.info.name, sort_keys=True).encode()) + if self.device: + m.update(json.dumps(self.device.id, sort_keys=True).encode()) + if self.target: + m.update(json.dumps(self.target, sort_keys=True).encode()) + m.update(json.dumps(self.options, sort_keys=True).encode()) + return m.hexdigest() + + +class MountManager: + """Manager for Mounts + + Uses a `host.ServiceManager` to activate `Mount` instances. + Takes a `DeviceManager` to access devices and a directory + called `root`, which is the root of all the specified mount + points. + """ + + def __init__(self, devices: DeviceManager, root: str) -> None: + self.devices = devices + self.root = root + self.mounts = {} + + def mount(self, mount: Mount) -> Dict: + + source = self.devices.device_abspath(mount.device) + + args = { + "source": source, + "target": mount.target, + + "root": self.root, + "tree": self.devices.tree, + + "options": mount.options, + } + + mgr = self.devices.service_manager + + client = mgr.start(f"mount/{mount.name}", mount.info.path) + path = client.call("mount", args) + + if not path: + res = {} + self.mounts[mount.name] = res + return res + + if not path.startswith(self.root): + raise RuntimeError(f"returned path '{path}' has wrong prefix") + + path = os.path.relpath(path, self.root) + + self.mounts[mount.name] = path + + return {"path": path} + + +class MountService(host.Service): + """Mount host service""" + + @abc.abstractmethod + def mount(self, args: Dict): + """Mount a device""" + + @abc.abstractmethod + def umount(self): + """Unmount all mounted resources""" + + def stop(self): + self.umount() + + def dispatch(self, method: str, args, _fds): + if method == "mount": + r = self.mount(args) + return r, None + + raise host.ProtocolError("Unknown method") + + +class FileSystemMountService(MountService): + """Specialized mount host service for file system mounts""" + + def __init__(self, args): + super().__init__(args) + + self.mountpoint = None + self.check = False + + @abc.abstractmethod + def translate_options(self, options: Dict): + return [] + + def mount(self, args: Dict): + + source = args["source"] + target = args["target"] + root = args["root"] + options = args["options"] + + mountpoint = os.path.join(root, target.lstrip("/")) + args = self.translate_options(options) + + os.makedirs(mountpoint, exist_ok=True) + self.mountpoint = mountpoint + + subprocess.run( + ["mount"] + + args + [ + "--source", source, + "--target", mountpoint + ], + check=True) + + self.check = True + return mountpoint + + def umount(self): + if not self.mountpoint: + return + + self.sync() + + print("umounting") + + # We ignore errors here on purpose + subprocess.run(["umount", self.mountpoint], + check=self.check) + self.mountpoint = None + + def sync(self): + subprocess.run(["sync", "-f", self.mountpoint], + check=self.check) diff --git a/osbuild/objectstore.py b/osbuild/objectstore.py new file mode 100644 index 0000000..f4deca3 --- /dev/null +++ b/osbuild/objectstore.py @@ -0,0 +1,513 @@ +import contextlib +import os +import subprocess +import tempfile +import uuid +from typing import Optional + +from osbuild.util.types import PathLike +from osbuild.util import jsoncomm, rmrf +from . import api + + +__all__ = [ + "ObjectStore", +] + + +def mount(source, target, bind=True, ro=True, private=True, mode="0755"): + options = [] + if ro: + options += ["ro"] + if mode: + options += [mode] + + args = [] + if bind: + args += ["--rbind"] + if private: + args += ["--make-rprivate"] + if options: + args += ["-o", ",".join(options)] + + r = subprocess.run(["mount"] + args + [source, target], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + encoding="utf-8", + check=False) + + if r.returncode != 0: + code = r.returncode + msg = r.stdout.strip() + raise RuntimeError(f"{msg} (code: {code})") + + +def umount(target, lazy=False): + args = [] + if lazy: + args += ["--lazy"] + subprocess.run(["sync", "-f", target], check=True) + subprocess.run(["umount", "-R"] + args + [target], check=True) + + +class Object: + def __init__(self, store: "ObjectStore"): + self._init = True + self._readers = 0 + self._writer = False + self._base = None + self._workdir = None + self._tree = None + self.id = None + self.store = store + self.reset() + + def init(self) -> None: + """Initialize the object with content of its base""" + self._check_writable() + self._check_readers() + self._check_writer() + if self._init: + return + + with self.store.new(self._base) as obj: + obj.export(self._tree) + self._init = True + + @property + def base(self) -> Optional[str]: + return self._base + + @base.setter + def base(self, base_id: Optional[str]): + self._init = not base_id + self._base = base_id + self.id = base_id + + @property + def _path(self) -> str: + if self._base and not self._init: + path = self.store.resolve_ref(self._base) + else: + path = self._tree + return path + + @contextlib.contextmanager + def write(self) -> str: + """Return a path that can be written to""" + self._check_writable() + self._check_readers() + self._check_writer() + self.init() + self.id = None + with self.tempdir("writer") as target: + mount(self._path, target, ro=False) + try: + self._writer = True + yield target + finally: + umount(target) + self._writer = False + + @contextlib.contextmanager + def read(self) -> str: + with self.tempdir("reader") as target: + with self.read_at(target) as path: + yield path + + @contextlib.contextmanager + def read_at(self, target: PathLike, path: str = "/") -> str: + """Read the object or a part of it at given location + + Map the tree or a part of it specified via `path` at the + specified path `target`. + """ + self._check_writable() + self._check_writer() + + path = os.path.join(self._path, path.lstrip("/")) + + mount(path, target) + try: + self._readers += 1 + yield target + finally: + umount(target) + self._readers -= 1 + + def store_tree(self): + """Store the tree with a fresh name and reset itself + + Moves the tree atomically by using rename(2), to a + randomly generated unique name. Afterwards it resets + itself and can be used as if it was new. + """ + self._check_writable() + self._check_readers() + self._check_writer() + self.init() + destination = str(uuid.uuid4()) + os.rename(self._tree, os.path.join(self.store.objects, destination)) + self.reset() + return destination + + def reset(self): + self.cleanup() + self._workdir = self.store.tempdir(suffix="object") + self._tree = os.path.join(self._workdir.name, "tree") + os.makedirs(self._tree, mode=0o755, exist_ok=True) + self._init = not self._base + + def cleanup(self): + self._check_readers() + self._check_writer() + if self._tree: + # manually remove the tree, it might contain + # files with immutable flag set, which will + # throw off standard Python 3 tempdir cleanup + rmrf.rmtree(self._tree) + self._tree = None + if self._workdir: + self._workdir.cleanup() + self._workdir = None + self.id = None + + def _check_readers(self): + """Internal: Raise a ValueError if there are readers""" + if self._readers: + raise ValueError("Read operation is ongoing") + + def _check_writable(self): + """Internal: Raise a ValueError if not writable""" + if not self._workdir: + raise ValueError("Object is not writable") + + def _check_writer(self): + """Internal: Raise a ValueError if there is a writer""" + if self._writer: + raise ValueError("Write operation is ongoing") + + @contextlib.contextmanager + def _open(self): + """Open the directory and return the file descriptor""" + with self.read() as path: + fd = os.open(path, os.O_DIRECTORY) + try: + yield fd + finally: + os.close(fd) + + def tempdir(self, suffix=None): + workdir = self._workdir.name + if suffix: + suffix = "-" + suffix + return tempfile.TemporaryDirectory(dir=workdir, + suffix=suffix) + + def __enter__(self): + self._check_writable() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def export(self, to_directory: PathLike): + """Copy object into an external directory""" + with self.read() as from_directory: + subprocess.run( + [ + "cp", + "--reflink=auto", + "-a", + os.fspath(from_directory) + "/.", + os.fspath(to_directory), + ], + check=True, + ) + + +class HostTree: + """Read-only access to the host file system + + An object that provides the same interface as + `objectstore.Object` that can be used to read + the host file-system. + """ + + def __init__(self, store): + self.store = store + + @staticmethod + def write(): + raise ValueError("Cannot write to host") + + @contextlib.contextmanager + def read(self): + with self.store.tempdir() as tmp: + # Create a bare bones root file system + # with just /usr mounted from the host + usr = os.path.join(tmp, "usr") + os.makedirs(usr) + + mount(tmp, tmp) # ensure / is read-only + mount("/usr", usr) + try: + yield tmp + finally: + umount(tmp) + + def cleanup(self): + pass # noop for the host + + +class ObjectStore(contextlib.AbstractContextManager): + def __init__(self, store: PathLike): + self.store = store + self.objects = os.path.join(store, "objects") + self.refs = os.path.join(store, "refs") + self.tmp = os.path.join(store, "tmp") + os.makedirs(self.store, exist_ok=True) + os.makedirs(self.objects, exist_ok=True) + os.makedirs(self.refs, exist_ok=True) + os.makedirs(self.tmp, exist_ok=True) + self._objs = set() + + def _get_floating(self, object_id: str) -> Optional[Object]: + """Internal: get a non-committed object""" + for obj in self._objs: + if obj.id == object_id: + return obj + return None + + def contains(self, object_id): + if not object_id: + return False + + if self._get_floating(object_id): + return True + + return os.access(self.resolve_ref(object_id), os.F_OK) + + def resolve_ref(self, object_id: Optional[str]) -> Optional[str]: + """Returns the path to the given object_id""" + if not object_id: + return None + return os.path.join(self.refs, object_id) + + def tempdir(self, prefix=None, suffix=None): + """Return a tempfile.TemporaryDirectory within the store""" + return tempfile.TemporaryDirectory(dir=self.tmp, + prefix=prefix, + suffix=suffix) + + def get(self, object_id): + obj = self._get_floating(object_id) + if obj: + return obj + + if not self.contains(object_id): + return None + + obj = self.new(base_id=object_id) + return obj + + def new(self, base_id=None): + """Creates a new temporary `Object`. + + It returns a temporary instance of `Object`, the base + optionally set to `base_id`. It can be used to interact + with the store. + If changes to the object's content were made (by calling + `Object.write`), these must manually be committed to the + store via `commit()`. + """ + + obj = Object(self) + + if base_id: + # if we were given a base id then this is the base + # content for the new object + # NB: `Object` has copy-on-write semantics, so no + # copying of the data takes places at this point + obj.base = base_id + + self._objs.add(obj) + + return obj + + def commit(self, obj: Object, object_id: str) -> str: + """Commits a Object to the object store + + Move the contents of the obj (Object) to object directory + of the store with a universally unique name. Creates a + symlink to that ('objects/{hash}') in the references + directory with the object_id as the name ('refs/{object_id}). + If the link already exists, it will be atomically replaced. + + Returns: The name of the object + """ + + # the object is stored in the objects directory using its unique + # name. This means that eatch commit will always result in a new + # object in the store, even if an identical one exists. + object_name = obj.store_tree() + + # symlink the object_id (config hash) in the refs directory to the + # object name in the objects directory. If a symlink by that name + # already exists, atomically replace it, but leave the backing object + # in place (it may be in use). + with self.tempdir() as tmp: + link = f"{tmp}/link" + os.symlink(f"../objects/{object_name}", link) + os.replace(link, self.resolve_ref(object_id)) + + # the reference that is pointing to `object_name` is now the base + # of `obj`. It is not actively initialized but any subsequent calls + # to `obj.write()` will initialize it again + obj.base = object_id + + return object_name + + def cleanup(self): + """Cleanup all created Objects that are still alive""" + for obj in self._objs: + obj.cleanup() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + +class StoreServer(api.BaseAPI): + + endpoint = "store" + + def __init__(self, store: ObjectStore, *, socket_address=None): + super().__init__(socket_address) + self.store = store + self.tmproot = store.tempdir(prefix="store-server-") + self._stack = contextlib.ExitStack() + + def _cleanup(self): + self.tmproot.cleanup() + self.tmproot = None + self._stack.close() + self._stack = None + + def _read_tree(self, msg, sock): + object_id = msg["object-id"] + obj = self.store.get(object_id) + if not obj: + sock.send({"path": None}) + return + + reader = obj.read() + path = self._stack.enter_context(reader) + sock.send({"path": path}) + + def _read_tree_at(self, msg, sock): + object_id = msg["object-id"] + target = msg["target"] + subtree = msg["subtree"] + + obj = self.store.get(object_id) + if not obj: + sock.send({"path": None}) + return + + try: + reader = obj.read_at(target, subtree) + path = self._stack.enter_context(reader) + # pylint: disable=broad-except + except Exception as e: + sock.send({"error": str(e)}) + return + + sock.send({"path": path}) + + def _mkdtemp(self, msg, sock): + args = { + "suffix": msg.get("suffix"), + "prefix": msg.get("prefix"), + "dir": self.tmproot.name + } + + path = tempfile.mkdtemp(**args) + sock.send({"path": path}) + + def _source(self, msg, sock): + name = msg["name"] + base = self.store.store + path = os.path.join(base, "sources", name) + sock.send({"path": path}) + + def _message(self, msg, _fds, sock): + if msg["method"] == "read-tree": + self._read_tree(msg, sock) + elif msg["method"] == "read-tree-at": + self._read_tree_at(msg, sock) + elif msg["method"] == "mkdtemp": + self._mkdtemp(msg, sock) + elif msg["method"] == "source": + self._source(msg, sock) + else: + raise ValueError("Invalid RPC call", msg) + + +class StoreClient: + def __init__(self, connect_to="/run/osbuild/api/store"): + self.client = jsoncomm.Socket.new_client(connect_to) + + def __del__(self): + if self.client is not None: + self.client.close() + + def mkdtemp(self, suffix=None, prefix=None): + msg = { + "method": "mkdtemp", + "suffix": suffix, + "prefix": prefix + } + + self.client.send(msg) + msg, _, _ = self.client.recv() + + return msg["path"] + + def read_tree(self, object_id: str): + msg = { + "method": "read-tree", + "object-id": object_id + } + + self.client.send(msg) + msg, _, _ = self.client.recv() + + return msg["path"] + + def read_tree_at(self, object_id: str, target: str, path="/"): + msg = { + "method": "read-tree-at", + "object-id": object_id, + "target": os.fspath(target), + "subtree": os.fspath(path) + } + + self.client.send(msg) + msg, _, _ = self.client.recv() + + err = msg.get("error") + if err: + raise RuntimeError(err) + + return msg["path"] + + def source(self, name: str) -> str: + msg = { + "method": "source", + "name": name + } + + self.client.send(msg) + msg, _, _ = self.client.recv() + + return msg["path"] diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py new file mode 100644 index 0000000..4bcc30f --- /dev/null +++ b/osbuild/pipeline.py @@ -0,0 +1,481 @@ +import collections +import contextlib +import hashlib +import json +import os +from typing import Dict, Generator, Iterable, Iterator, List, Optional + +from .api import API +from . import buildroot +from . import host +from . import objectstore +from . import remoteloop +from .devices import Device, DeviceManager +from .inputs import Input +from .mounts import Mount, MountManager +from .sources import Source +from .util import osrelease + + +def cleanup(*objs): + """Call cleanup method for all objects, filters None values out""" + _ = map(lambda o: o.cleanup(), filter(None, objs)) + + +class BuildResult: + def __init__(self, origin, returncode, output, metadata, error): + self.name = origin.name + self.id = origin.id + self.options = origin.options + self.success = returncode == 0 + self.output = output + self.metadata = metadata + self.error = error + + def as_dict(self): + return vars(self) + + +class Stage: + def __init__(self, info, source_options, build, base, options, source_epoch): + self.info = info + self.sources = source_options + self.build = build + self.base = base + self.options = options + self.source_epoch = source_epoch + self.checkpoint = False + self.inputs = {} + self.devices = {} + self.mounts = {} + + @property + def name(self): + return self.info.name + + @property + def id(self): + m = hashlib.sha256() + m.update(json.dumps(self.name, sort_keys=True).encode()) + m.update(json.dumps(self.build, sort_keys=True).encode()) + m.update(json.dumps(self.base, sort_keys=True).encode()) + m.update(json.dumps(self.options, sort_keys=True).encode()) + if self.source_epoch is not None: + m.update(json.dumps(self.source_epoch, sort_keys=True).encode()) + if self.inputs: + data = {n: i.id for n, i in self.inputs.items()} + m.update(json.dumps(data, sort_keys=True).encode()) + return m.hexdigest() + + @property + def dependencies(self) -> Generator[str, None, None]: + """Return a list of pipeline ids this stage depends on""" + + for ip in self.inputs.values(): + + if ip.origin != "org.osbuild.pipeline": + continue + + for ref in ip.refs: + yield ref + + def add_input(self, name, info, origin, options=None): + ip = Input(name, info, origin, options or {}) + self.inputs[name] = ip + return ip + + def add_device(self, name, info, parent, options): + dev = Device(name, info, parent, options) + self.devices[name] = dev + return dev + + def add_mount(self, name, info, device, target, options): + mount = Mount(name, info, device, target, options) + self.mounts[name] = mount + return mount + + def prepare_arguments(self, args, location): + args["options"] = self.options + args["meta"] = meta = { + "id": self.id, + } + + if self.source_epoch is not None: + meta["source-epoch"] = self.source_epoch + + # Root relative paths: since paths are different on the + # host and in the container they need to be mapped to + # their path within the container. For all items that + # have registered roots, re-root their path entries here + for name, root in args.get("paths", {}).items(): + group = args.get(name) + if not group or not isinstance(group, dict): + continue + for item in group.values(): + path = item.get("path") + if not path: + continue + item["path"] = os.path.join(root, path) + + with open(location, "w", encoding="utf-8") as fp: + json.dump(args, fp) + + def run(self, tree, runner, build_tree, store, monitor, libdir, timeout=None): + with contextlib.ExitStack() as cm: + + build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp) + cm.enter_context(build_root) + + # if we have a build root, then also bind-mount the boot + # directory from it, since it may contain efi binaries + build_root.mount_boot = bool(self.build) + + tmpdir = store.tempdir(prefix="buildroot-tmp-") + tmpdir = cm.enter_context(tmpdir) + + inputs_tmpdir = os.path.join(tmpdir, "inputs") + os.makedirs(inputs_tmpdir) + inputs_mapped = "/run/osbuild/inputs" + inputs = {} + + devices_mapped = "/dev" + devices = {} + + mounts_tmpdir = os.path.join(tmpdir, "mounts") + os.makedirs(mounts_tmpdir) + mounts_mapped = "/run/osbuild/mounts" + mounts = {} + + os.makedirs(os.path.join(tmpdir, "api")) + args_path = os.path.join(tmpdir, "api", "arguments") + + args = { + "tree": "/run/osbuild/tree", + "paths": { + "devices": devices_mapped, + "inputs": inputs_mapped, + "mounts": mounts_mapped, + }, + "devices": devices, + "inputs": inputs, + "mounts": mounts, + } + + ro_binds = [ + f"{self.info.path}:/run/osbuild/bin/{self.name}", + f"{inputs_tmpdir}:{inputs_mapped}", + f"{args_path}:/run/osbuild/api/arguments" + ] + + binds = [ + os.fspath(tree) + ":/run/osbuild/tree", + f"{mounts_tmpdir}:{mounts_mapped}" + ] + + storeapi = objectstore.StoreServer(store) + cm.enter_context(storeapi) + + mgr = host.ServiceManager(monitor=monitor) + cm.enter_context(mgr) + + for key, ip in self.inputs.items(): + data = ip.map(mgr, storeapi, inputs_tmpdir) + inputs[key] = data + + devmgr = DeviceManager(mgr, build_root.dev, tree) + for name, dev in self.devices.items(): + devices[name] = devmgr.open(dev) + + mntmgr = MountManager(devmgr, mounts_tmpdir) + for key, mount in self.mounts.items(): + data = mntmgr.mount(mount) + mounts[key] = data + + self.prepare_arguments(args, args_path) + + api = API() + build_root.register_api(api) + + rls = remoteloop.LoopServer() + build_root.register_api(rls) + + extra_env = {} + if self.source_epoch is not None: + extra_env["SOURCE_DATE_EPOCH"] = str(self.source_epoch) + + r = build_root.run([f"/run/osbuild/bin/{self.name}"], + monitor, + timeout=timeout, + binds=binds, + readonly_binds=ro_binds, + extra_env=extra_env) + + return BuildResult(self, r.returncode, r.output, api.metadata, api.error) + + +class Pipeline: + def __init__(self, name: str, runner=None, build=None, source_epoch=None): + self.name = name + self.build = build + self.runner = runner + self.stages = [] + self.assembler = None + self.source_epoch = source_epoch + + @property + def id(self): + """ + Pipeline id: corresponds to the `id` of the last stage + + In contrast to `name` this identifies the pipeline via + the tree, i.e. the content, it produces. Therefore two + pipelines that produce the same `tree`, i.e. have the + same exact stages and build pipeline, will have the + same `id`; thus the `id`, in contrast to `name` does + not uniquely identify a pipeline. + In case a Pipeline has no stages, its `id` is `None`. + """ + return self.stages[-1].id if self.stages else None + + def add_stage(self, info, options, sources_options=None): + stage = Stage(info, sources_options, self.build, + self.id, options or {}, self.source_epoch) + self.stages.append(stage) + if self.assembler: + self.assembler.base = stage.id + return stage + + def build_stages(self, object_store, monitor, libdir, stage_timeout=None): + results = {"success": True} + + # We need a build tree for the stages below, which is either + # another tree that needs to be built with the build pipeline + # or the host file system if no build pipeline is specified + # NB: the very last level of nested build pipelines is always + # build on the host + + if not self.build: + build_tree = objectstore.HostTree(object_store) + else: + build_tree = object_store.get(self.build) + + if not build_tree: + raise AssertionError(f"build tree {self.build} not found") + + # If there are no stages, just return build tree we just + # obtained and a new, clean `tree` + if not self.stages: + tree = object_store.new() + return results, build_tree, tree + + # Check if the tree that we are supposed to build does + # already exist. If so, short-circuit here + tree = object_store.get(self.id) + + if tree: + return results, build_tree, tree + + # Not in the store yet, need to actually build it, but maybe + # an intermediate checkpoint exists: Find the last stage that + # already exists in the store and use that as the base. + tree = object_store.new() + base_idx = -1 + for i in reversed(range(len(self.stages))): + if object_store.contains(self.stages[i].id): + tree.base = self.stages[i].id + base_idx = i + break + + # If two run() calls race each-other, two trees will get built + # and it is nondeterministic which of them will end up + # referenced by the `tree_id` in the content store if they are + # both committed. However, after the call to commit all the + # trees will be based on the winner. + results["stages"] = [] + + for stage in self.stages[base_idx + 1:]: + with build_tree.read() as build_path, tree.write() as path: + + monitor.stage(stage) + + r = stage.run(path, + self.runner, + build_path, + object_store, + monitor, + libdir, + stage_timeout) + + monitor.result(r) + + results["stages"].append(r.as_dict()) + if not r.success: + cleanup(build_tree, tree) + results["success"] = False + return results, None, None + + # The content of the tree now corresponds to the stage that + # was build and this can can be identified via the id of it + tree.id = stage.id + + if stage.checkpoint: + object_store.commit(tree, stage.id) + + return results, build_tree, tree + + def run(self, store, monitor, libdir, stage_timeout=None): + results = {"success": True} + + monitor.begin(self) + + # If the final result is already in the store, no need to attempt + # building it. Just fetch the cached information. If the associated + # tree exists, we return it as well, but we do not care if it is + # missing, since it is not a mandatory part of the result and would + # usually be needless overhead. + obj = store.get(self.id) + + if not obj: + results, _, obj = self.build_stages(store, monitor, libdir, stage_timeout) + + if not results["success"]: + return results + + monitor.finish(results) + + return results + + +class Manifest: + """Representation of a pipeline and its sources""" + + def __init__(self): + self.pipelines = collections.OrderedDict() + self.sources: List[Source] = [] + + def add_pipeline(self, name: str, runner: str, build: str, source_epoch: Optional[int] = None) -> Pipeline: + pipeline = Pipeline(name, runner, build, source_epoch) + if name in self.pipelines: + raise ValueError(f"Name {name} already exists") + self.pipelines[name] = pipeline + return pipeline + + def add_source(self, info, items: List, options: Dict) -> Source: + source = Source(info, items, options) + self.sources.append(source) + return source + + def download(self, store, monitor, libdir): + with host.ServiceManager(monitor=monitor) as mgr: + for source in self.sources: + source.download(mgr, store, libdir) + + def depsolve(self, store, targets: Iterable[str]) -> List[str]: + """Return the list of pipelines that need to be built + + Given a list of target pipelines, return the names + of all pipelines and their dependencies that are not + already present in the store. + """ + + # A stack of pipelines to check if they need to be built + check = list(map(self.get, targets)) + + # The ordered result "set", will be reversed at the end + build = collections.OrderedDict() + + while check: + pl = check.pop() # get the last(!) item + + if store.contains(pl.id): + continue + + # The store does not have this pipeline, it needs to + # be built, add it to the ordered result set and + # ensure it is at the end, i.e. built before previously + # checked items. NB: the result set is reversed before + # it gets returned. This ensures that a dependency that + # gets checked multiple times, like a build pipeline, + # always gets built before its dependent pipeline. + build[pl.id] = pl + build.move_to_end(pl.id) + + # Add all dependencies to the stack of things to check, + # starting with the build pipeline, if there is one + if pl.build: + check.append(self.get(pl.build)) + + # Stages depend on other pipeline via pipeline inputs. + # We check in reversed order until we hit a checkpoint + for stage in reversed(pl.stages): + + # we stop if we have a checkpoint, i.e. we don't + # need to build any stages after that checkpoint + if store.contains(stage.id): + break + + pls = map(self.get, stage.dependencies) + check.extend(pls) + + return list(map(lambda x: x.name, reversed(build.values()))) + + def build(self, store, pipelines, monitor, libdir, stage_timeout=None): + results = {"success": True} + + for pl in map(self.get, pipelines): + res = pl.run(store, monitor, libdir, stage_timeout) + results[pl.id] = res + if not res["success"]: + results["success"] = False + return results + + return results + + def mark_checkpoints(self, checkpoints): + points = set(checkpoints) + + def mark_stage(stage): + c = stage.id + if c in points: + stage.checkpoint = True + points.remove(c) + + def mark_pipeline(pl): + if pl.name in points and pl.stages: + pl.stages[-1].checkpoint = True + points.remove(pl.name) + + for stage in pl.stages: + mark_stage(stage) + + for pl in self.pipelines.values(): + mark_pipeline(pl) + + return points + + def get(self, name_or_id: str) -> Optional[Pipeline]: + pl = self.pipelines.get(name_or_id) + if pl: + return pl + for pl in self.pipelines.values(): + if pl.id == name_or_id: + return pl + return None + + def __contains__(self, name_or_id: str) -> bool: + return self.get(name_or_id) is not None + + def __getitem__(self, name_or_id: str) -> Pipeline: + pl = self.get(name_or_id) + if pl: + return pl + raise KeyError(f"'{name_or_id}' not found") + + def __iter__(self) -> Iterator[Pipeline]: + return iter(self.pipelines.values()) + + +def detect_host_runner(): + """Use os-release(5) to detect the runner for the host""" + osname = osrelease.describe_os(*osrelease.DEFAULT_PATHS) + return "org.osbuild." + osname diff --git a/osbuild/remoteloop.py b/osbuild/remoteloop.py new file mode 100644 index 0000000..29eadf1 --- /dev/null +++ b/osbuild/remoteloop.py @@ -0,0 +1,125 @@ +import contextlib +import errno +import os +from . import api +from . import loop +from .util import jsoncomm + +__all__ = [ + "LoopClient", + "LoopServer" +] + + +class LoopServer(api.BaseAPI): + """Server for creating loopback devices + + The server listens for requests on a AF_UNIX/SOCK_DRGAM sockets. + + A request should contain SCM_RIGHTS of two filedescriptors, one + that sholud be the backing file for the new loopdevice, and a + second that should be a directory file descriptor where the new + device node will be created. + + The payload should be a JSON object with the mandatory arguments + @fd which is the offset in the SCM_RIGHTS array for the backing + file descriptor and @dir_fd which is the offset for the output + directory. Optionally, @offset and @sizelimit in bytes may also + be specified. + + The server respods with a JSON object containing the device name + of the new device node created in the output directory. + + The created loopback device is guaranteed to be bound to the + given backing file descriptor for the lifetime of the LoopServer + object. + """ + + endpoint = "remoteloop" + + def __init__(self, *, socket_address=None): + super().__init__(socket_address) + self.devs = [] + self.ctl = loop.LoopControl() + + def _create_device(self, fd, dir_fd, offset=None, sizelimit=None): + while True: + # Getting an unbound loopback device and attaching a backing + # file descriptor to it is racy, so we must use a retry loop + lo = loop.Loop(self.ctl.get_unbound()) + try: + lo.set_fd(fd) + except OSError as e: + lo.close() + if e.errno == errno.EBUSY: + continue + raise e + # `set_status` returns EBUSY when the pages from the previously + # bound file have not been fully cleared yet. + try: + lo.set_status(offset=offset, sizelimit=sizelimit, autoclear=True) + except BlockingIOError: + lo.clear_fd() + lo.close() + continue + break + + lo.mknod(dir_fd) + # Pin the Loop objects so they are only released when the LoopServer + # is destroyed. + self.devs.append(lo) + return lo.devname + + def _message(self, msg, fds, sock): + fd = fds[msg["fd"]] + dir_fd = fds[msg["dir_fd"]] + offset = msg.get("offset") + sizelimit = msg.get("sizelimit") + + devname = self._create_device(fd, dir_fd, offset, sizelimit) + sock.send({"devname": devname}) + + def _cleanup(self): + for lo in self.devs: + lo.close() + self.ctl.close() + + +class LoopClient: + client = None + + def __init__(self, connect_to): + self.client = jsoncomm.Socket.new_client(connect_to) + + def __del__(self): + if self.client is not None: + self.client.close() + + @contextlib.contextmanager + def device(self, filename, offset=None, sizelimit=None): + req = {} + fds = [] + + fd = os.open(filename, os.O_RDWR) + dir_fd = os.open("/dev", os.O_DIRECTORY) + + fds.append(fd) + req["fd"] = 0 + fds.append(dir_fd) + req["dir_fd"] = 1 + + if offset: + req["offset"] = offset + if sizelimit: + req["sizelimit"] = sizelimit + + self.client.send(req, fds=fds) + os.close(dir_fd) + os.close(fd) + + payload, _, _ = self.client.recv() + path = os.path.join("/dev", payload["devname"]) + try: + yield path + finally: + os.unlink(path) diff --git a/osbuild/sources.py b/osbuild/sources.py new file mode 100644 index 0000000..956e841 --- /dev/null +++ b/osbuild/sources.py @@ -0,0 +1,67 @@ +import abc +import contextlib +import os +import json +import tempfile + +from . import host +from .objectstore import ObjectStore +from .util.types import PathLike + + +class Source: + """ + A single source with is corresponding options. + """ + + def __init__(self, info, items, options) -> None: + self.info = info + self.items = items or {} + self.options = options + + def download(self, mgr: host.ServiceManager, store: ObjectStore, libdir: PathLike): + source = self.info.name + cache = os.path.join(store.store, "sources") + + args = { + "options": self.options, + "cache": cache, + "output": None, + "checksums": [], + "libdir": os.fspath(libdir) + } + + client = mgr.start(f"source/{source}", self.info.path) + + with self.make_items_file(store.tmp) as fd: + fds = [fd] + reply = client.call_with_fds("download", args, fds) + + return reply + + @contextlib.contextmanager + def make_items_file(self, tmp): + with tempfile.TemporaryFile("w+", dir=tmp, encoding="utf-8") as f: + json.dump(self.items, f) + f.seek(0) + yield f.fileno() + + +class SourceService(host.Service): + """Source host service""" + + @abc.abstractmethod + def download(self, items, cache, options): + pass + + def dispatch(self, method: str, args, fds): + if method == "download": + with os.fdopen(fds.steal(0)) as f: + items = json.load(f) + + r = self.download(items, + args["cache"], + args["options"]) + return r, None + + raise host.ProtocolError("Unknown method") diff --git a/osbuild/util/__init__.py b/osbuild/util/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/osbuild/util/__init__.py diff --git a/osbuild/util/checksum.py b/osbuild/util/checksum.py new file mode 100644 index 0000000..6350a80 --- /dev/null +++ b/osbuild/util/checksum.py @@ -0,0 +1,50 @@ +"""Checksum Utilities + +Small convenience functions to work with checksums. +""" +import hashlib +import os + +from .types import PathLike + + +# How many bytes to read in one go. Taken from coreutils/gnulib +BLOCKSIZE = 32768 + + +def hexdigest_file(path: PathLike, algorithm: str) -> str: + """Return the hexdigest of the file at `path` using `algorithm` + + Will stream the contents of file to the hash `algorithm` and + return the hexdigest. If the specified `algorithm` is not + supported a `ValueError` will be raised. + """ + hasher = hashlib.new(algorithm) + + with open(path, "rb") as f: + + os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL) + + while True: + data = f.read(BLOCKSIZE) + if not data: + break + + hasher.update(data) + + return hasher.hexdigest() + + +def verify_file(path: PathLike, checksum: str) -> bool: + """Hash the file and return if the specified `checksum` matches + + Uses `hexdigest_file` to hash the contents of the file at + `path` and return if the hexdigest matches the one specified + in `checksum`, where `checksum` consist of the algorithm used + and the digest joined via `:`, e.g. `sha256:abcd...`. + """ + algorithm, want = checksum.split(":", 1) + + have = hexdigest_file(path, algorithm) + + return have == want diff --git a/osbuild/util/ctx.py b/osbuild/util/ctx.py new file mode 100644 index 0000000..68c8ff4 --- /dev/null +++ b/osbuild/util/ctx.py @@ -0,0 +1,35 @@ +"""ContextManager Utilities + +This module implements helpers around python context-managers, with-statements, +and RAII. It is meant as a supplement to `contextlib` from the python standard +library. +""" + +import contextlib + + +__all__ = [ + "suppress_oserror", +] + + +@contextlib.contextmanager +def suppress_oserror(*errnos): + """Suppress OSError Exceptions + + This is an extension to `contextlib.suppress()` from the python standard + library. It catches any `OSError` exceptions and suppresses them. However, + it only catches the exceptions that match the specified error numbers. + + Parameters + ---------- + errnos + A list of error numbers to match on. If none are specified, this + function has no effect. + """ + + try: + yield + except OSError as e: + if e.errno not in errnos: + raise e diff --git a/osbuild/util/jsoncomm.py b/osbuild/util/jsoncomm.py new file mode 100644 index 0000000..6ab82c6 --- /dev/null +++ b/osbuild/util/jsoncomm.py @@ -0,0 +1,405 @@ +"""JSON Communication + +This module implements a client/server communication method based on JSON +serialization. It uses unix-domain-datagram-sockets and provides a simple +unicast message transmission. +""" + + +import array +import contextlib +import errno +import json +import os +import socket +from typing import Any +from typing import Optional +from .types import PathLike + + +class FdSet: + """File-Descriptor Set + + This object wraps an array of file-descriptors. Unlike a normal integer + array, this object owns the file-descriptors and therefore closes them once + the object is released. + + File-descriptor sets are initialized once. From then one, the only allowed + operation is to query it for information, or steal file-descriptors from + it. If you close a set, all remaining file-descriptors are closed and + removed from the set. It will then be an empty set. + """ + + _fds = array.array("i") + + def __init__(self, *, rawfds): + for i in rawfds: + if not isinstance(i, int) or i < 0: + raise ValueError() + + self._fds = rawfds + + def __del__(self): + self.close() + + def close(self): + """Close All Entries + + This closes all stored file-descriptors and clears the set. Once this + returns, the set will be empty. It is safe to call this multiple times. + Note that a set is automatically closed when it is garbage collected. + """ + + for i in self._fds: + if i >= 0: + os.close(i) + + self._fds = array.array("i") + + @classmethod + def from_list(cls, l: list): + """Create new Set from List + + This creates a new file-descriptor set initialized to the same entries + as in the given list. This consumes the file-descriptors. The caller + must not assume ownership anymore. + """ + + fds = array.array("i") + fds.fromlist(l) + return cls(rawfds=fds) + + def __len__(self): + return len(self._fds) + + def __getitem__(self, key: Any): + if self._fds[key] < 0: + raise IndexError + return self._fds[key] + + def steal(self, key: Any): + """Steal Entry + + Retrieve the entry at the given position, but drop it from the internal + file-descriptor set. The caller will now own the file-descriptor and it + can no longer be accessed through the set. + + Note that this does not reshuffle the set. All indices stay constant. + """ + + v = self[key] + self._fds[key] = -1 + return v + + +class Socket(contextlib.AbstractContextManager): + """Communication Socket + + This socket object represents a communication channel. It allows sending + and receiving JSON-encoded messages. It uses unix-domain sequenced-packet + sockets as underlying transport. + """ + + _socket = None + _unlink = None + + def __init__(self, sock, unlink): + self._socket = sock + self._unlink = unlink + + def __del__(self): + self.close() + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() + return False + + @property + def blocking(self): + """Get the current blocking mode of the socket. + + This is related to the socket's timeout, i.e. if no time out is set + the socket is in blocking mode; otherwise it is non-blocking. + """ + timeout = self._socket.gettimeout() + return timeout is not None + + @blocking.setter + def blocking(self, value: bool): + """Set the blocking mode of the socket.""" + self._socket.setblocking(value) + + def accept(self) -> Optional["Socket"]: + """Accept a new connection on the socket. + + See python's `socket.accept` for more information. + """ + # Since, in the kernel, for AF_UNIX, new connection requests, + # i.e. clients connecting, are directly put on the receive + # queue of the listener socket, accept here *should* always + # return a socket and not block, even if the client meanwhile + # disconnected; we don't rely on that kernel behavior though + try: + conn, _ = self._socket.accept() + except (socket.timeout, BlockingIOError): + return None + return Socket(conn, None) + + def listen(self, backlog: Optional[int] = 2**16): + """Enable accepting of incoming connections. + + See python's `socket.listen` for details. + """ + + # `Socket.listen` accepts an `int` or no argument, but not `None` + args = [backlog] if backlog is not None else [] + self._socket.listen(*args) + + def close(self): + """Close Socket + + Close the socket and all underlying resources. This can be called + multiple times. + """ + + # close the socket if it is set + if self._socket is not None: + self._socket.close() + self._socket = None + + # unlink the file-system entry, if pinned + if self._unlink is not None: + try: + os.unlink(self._unlink[1], dir_fd=self._unlink[0]) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + os.close(self._unlink[0]) + self._unlink = None + + @classmethod + def new_client(cls, connect_to: Optional[PathLike] = None): + """Create Client + + Create a new client socket. + + Parameters + ---------- + connect_to + If not `None`, the client will use the specified address as the + default destination for all send operations. + """ + + sock = None + + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + + # Trigger an auto-bind. If you do not do this, you might end up with + # an unbound unix socket, which cannot receive messages. + # Alternatively, you can also set `SO_PASSCRED`, but this has + # side-effects. + sock.bind("") + + # Connect the socket. This has no effect other than specifying the + # default destination for send operations. + if connect_to is not None: + sock.connect(os.fspath(connect_to)) + except: + if sock is not None: + sock.close() + raise + + return cls(sock, None) + + @classmethod + def new_server(cls, bind_to: PathLike): + """Create Server + + Create a new listener socket. Returned socket is in non-blocking + mode by default. See `blocking` property. + + Parameters + ---------- + bind_to + The socket-address to listen on for incoming client requests. + """ + + sock = None + unlink = None + path = os.path.split(bind_to) + + try: + # We bind the socket and then open a directory-fd on the target + # socket. This allows us to properly unlink the socket when the + # server is closed. Note that sockets are never automatically + # cleaned up on linux, nor can you bind to existing sockets. + # We use a dirfd to guarantee this works even when you change + # your mount points in-between. + # Yeah, this is racy when mount-points change between the socket + # creation and open. But then your entire socket creation is racy + # as well. We do not guarantee atomicity, so you better make sure + # you do not rely on it. + sock = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) + sock.bind(os.fspath(bind_to)) + unlink = os.open(os.path.join(".", path[0]), os.O_CLOEXEC | os.O_PATH) + sock.setblocking(False) + except: + if unlink is not None: + os.close(unlink) + if sock is not None: + sock.close() + raise + + return cls(sock, (unlink, path[1])) + + @classmethod + def new_pair(cls, *, blocking=True): + """Create a connected socket pair + + Create a pair of connected sockets and return both as a tuple. + + Parameters + ---------- + blocking + The blocking mode for the socket pair. + """ + a, b = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) + + a.setblocking(blocking) + b.setblocking(blocking) + + return cls(a, None), cls(b, None) + + @classmethod + def new_from_fd(cls, fd: int, *, blocking=True, close_fd=True): + """Create a socket for an existing file descriptor + + Duplicate the file descriptor and return a `Socket` for it. + The blocking mode can be set via `blocking`. If `close_fd` + is True (the default) `fd` will be closed. + + Parameters + ---------- + fd + The file descriptor to use. + blocking + The blocking mode for the socket pair. + """ + sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_SEQPACKET) + sock.setblocking(blocking) + if close_fd: + os.close(fd) + return cls(sock, None) + + def fileno(self) -> int: + assert self._socket is not None + return self._socket.fileno() + + def recv(self): + """Receive a Message + + This receives the next pending message from the socket. This operation + is synchronous. + + A tuple consisting of the deserialized message payload, the auxiliary + file-descriptor set, and the socket-address of the sender is returned. + + In case the peer closed the connection, A tuple of `None` values is + returned. + """ + + # On `SOCK_SEQPACKET`, packets might be arbitrarily sized. There is no + # hard-coded upper limit, since it is only restricted by the size of + # the kernel write buffer on sockets (which itself can be modified via + # sysctl). The only real maximum is probably something like 2^31-1, + # since that is the maximum of that sysctl datatype. + # Anyway, `MSG_TRUNC+MSG_PEEK` usually allows us to easily peek at the + # incoming buffer. Unfortunately, the python `recvmsg()` wrapper + # discards the return code and we cannot use that. Instead, we simply + # loop until we know the size. This is slightly awkward, but seems fine + # as long as you do not put this into a hot-path. + size = 4096 + while True: + peek = self._socket.recvmsg(size, 0, socket.MSG_PEEK) + if not peek[0]: + # Connection was closed + return None, None, None + if not (peek[2] & socket.MSG_TRUNC): + break + size *= 2 + + # Fetch a packet from the socket. On linux, the maximum SCM_RIGHTS array + # size is hard-coded to 253. This allows us to size the ancillary buffer + # big enough to receive any possible message. + fds = array.array("i") + msg = self._socket.recvmsg(size, socket.CMSG_LEN(253 * fds.itemsize)) + + # First thing we do is always to fetch the CMSG FDs into an FdSet. This + # guarantees that we do not leak FDs in case the message handling fails + # for other reasons. + for level, ty, data in msg[1]: + if level == socket.SOL_SOCKET and ty == socket.SCM_RIGHTS: + assert len(data) % fds.itemsize == 0 + fds.frombytes(data) + fdset = FdSet(rawfds=fds) + + # Check the returned message flags. If the message was truncated, we + # have to discard it. This shouldn't happen, but there is no harm in + # handling it. However, `CTRUNC` can happen, since it is also triggered + # when LSMs reject FD transmission. Treat it the same as a parser error. + flags = msg[2] + if flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): + raise BufferError + + try: + payload = json.loads(msg[0]) + except json.JSONDecodeError as e: + raise BufferError from e + + return (payload, fdset, msg[3]) + + def send(self, payload: object, *, fds: Optional[list] = None): + """Send Message + + Send a new message via this socket. This operation is synchronous. The + maximum message size depends on the configured send-buffer on the + socket. An `OSError` with `EMSGSIZE` is raised when it is exceeded. + + Parameters + ---------- + payload + A python object to serialize as JSON and send via this socket. See + `json.dump()` for details about the serialization involved. + destination + The destination to send to. If `None`, the default destination is + used (if none is set, this will raise an `OSError`). + fds + A list of file-descriptors to send with the message. + + Raises + ------ + OSError + If the socket cannot be written, a matching `OSError` is raised. + TypeError + If the payload cannot be serialized, a type error is raised. + """ + + serialized = json.dumps(payload).encode() + cmsg = [] + if fds: + cmsg.append((socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))) + + n = self._socket.sendmsg([serialized], cmsg, 0) + assert n == len(serialized) + + def send_and_recv(self, payload: object, *, fds: Optional[list] = None): + """Send a message and wait for a reply + + This is a convenience helper that combines `send` and `recv`. + See the individual methods for details about the parameters. + """ + + self.send(payload, fds=fds) + return self.recv() diff --git a/osbuild/util/linux.py b/osbuild/util/linux.py new file mode 100644 index 0000000..8308a7e --- /dev/null +++ b/osbuild/util/linux.py @@ -0,0 +1,123 @@ +"""Linux API Access + +This module provides access to linux system-calls and other APIs, in particular +those not provided by the python standard library. The idea is to provide +universal wrappers with broad access to linux APIs. Convenience helpers and +higher-level abstractions are beyond the scope of this module. + +In some cases it is overly complex to provide universal access to a specific +API. Hence, the API might be restricted to a reduced subset of its +functionality, just to make sure we can actually implement the wrappers in a +reasonable manner. +""" + + +import array +import fcntl +import platform + + +__all__ = [ + "ioctl_get_immutable", + "ioctl_toggle_immutable", +] + + +# NOTE: These are wrong on at least ALPHA and SPARC. They use different +# ioctl number setups. We should fix this, but this is really awkward +# in standard python. +# Our tests will catch this, so we will not accidentally run into this +# on those architectures. +FS_IOC_GETFLAGS = 0x80086601 +FS_IOC_SETFLAGS = 0x40086602 + +FS_IMMUTABLE_FL = 0x00000010 + + +if platform.machine() == "ppc64le": + BLK_IOC_FLSBUF = 0x20001261 +else: + BLK_IOC_FLSBUF = 0x00001261 + + +def ioctl_get_immutable(fd: int): + """Query FS_IMMUTABLE_FL + + This queries the `FS_IMMUTABLE_FL` flag on a specified file. + + Arguments + --------- + fd + File-descriptor to operate on. + + Returns + ------- + bool + Whether the `FS_IMMUTABLE_FL` flag is set or not. + + Raises + ------ + OSError + If the underlying ioctl fails, a matching `OSError` will be raised. + """ + + if not isinstance(fd, int) or fd < 0: + raise ValueError() + + flags = array.array('L', [0]) + fcntl.ioctl(fd, FS_IOC_GETFLAGS, flags, True) + return bool(flags[0] & FS_IMMUTABLE_FL) + + +def ioctl_toggle_immutable(fd: int, set_to: bool): + """Toggle FS_IMMUTABLE_FL + + This toggles the `FS_IMMUTABLE_FL` flag on a specified file. It can both set + and clear the flag. + + Arguments + --------- + fd + File-descriptor to operate on. + set_to + Whether to set the `FS_IMMUTABLE_FL` flag or not. + + Raises + ------ + OSError + If the underlying ioctl fails, a matching `OSError` will be raised. + """ + + if not isinstance(fd, int) or fd < 0: + raise ValueError() + + flags = array.array('L', [0]) + fcntl.ioctl(fd, FS_IOC_GETFLAGS, flags, True) + if set_to: + flags[0] |= FS_IMMUTABLE_FL + else: + flags[0] &= ~FS_IMMUTABLE_FL + fcntl.ioctl(fd, FS_IOC_SETFLAGS, flags, False) + + +def ioctl_blockdev_flushbuf(fd: int): + """Flush the block device buffer cache + + NB: This function needs the `CAP_SYS_ADMIN` capability. + + Arguments + --------- + fd + File-descriptor of a block device to operate on. + + Raises + ------ + OSError + If the underlying ioctl fails, a matching `OSError` + will be raised. + """ + + if not isinstance(fd, int) or fd < 0: + raise ValueError(f"Invalid file descriptor: '{fd}'") + + fcntl.ioctl(fd, BLK_IOC_FLSBUF, 0) diff --git a/osbuild/util/lorax.py b/osbuild/util/lorax.py new file mode 100644 index 0000000..b8bb114 --- /dev/null +++ b/osbuild/util/lorax.py @@ -0,0 +1,205 @@ +#!/usr/bin/python3 +""" +Lorax related utilities: Template parsing and execution + +This module contains a re-implementation of the Lorax +template engine, but for osbuild. Not all commands in +the original scripting language are support, but all +needed to run the post install and cleanup scripts. +""" + +import contextlib +import glob +import os +import re +import shlex +import shutil +import subprocess + +import mako.template + + +def replace(target, patterns): + finder = [(re.compile(p), s) for p, s in patterns] + newfile = target + ".replace" + + with open(target, "r") as i, open(newfile, "w") as o: + for line in i: + for p, s in finder: + line = p.sub(s, line) + o.write(line) + os.rename(newfile, target) + + +def rglob(pathname, *, fatal=False): + seen = set() + for f in glob.iglob(pathname): + if f not in seen: + seen.add(f) + yield f + if fatal and not seen: + raise IOError(f"nothing matching {pathname}") + + +class Script: + + # all built-in commands in a name to method map + commands = {} + + # helper decorator to register builtin methods + class command: + def __init__(self, fn): + self.fn = fn + + def __set_name__(self, owner, name): + bultins = getattr(owner, "commands") + bultins[name] = self.fn + setattr(owner, name, self.fn) + + # Script class starts here + def __init__(self, script, build, tree): + self.script = script + self.tree = tree + self.build = build + + def __call__(self): + for i, line in enumerate(self.script): + cmd, args = line[0], line[1:] + ignore_error = False + if cmd.startswith("-"): + cmd = cmd[1:] + ignore_error = True + + method = self.commands.get(cmd) + + if not method: + raise ValueError(f"Unknown command: '{cmd}'") + + try: + method(self, *args) + except Exception: + if ignore_error: + continue + print(f"Error on line: {i} " + str(line)) + raise + + def tree_path(self, target): + dest = os.path.join(self.tree, target.lstrip("/")) + return dest + + @command + def append(self, filename, data): + target = self.tree_path(filename) + dirname = os.path.dirname(target) + os.makedirs(dirname, exist_ok=True) + print(f"append '{target}' '{data}'") + with open(target, "a", encoding="utf-8") as f: + f.write(bytes(data, "utf8").decode("unicode_escape")) + f.write("\n") + + @command + def mkdir(self, *dirs): + for d in dirs: + print(f"mkdir '{d}'") + os.makedirs(self.tree_path(d), exist_ok=True) + + @command + def move(self, src, dst): + src = self.tree_path(src) + dst = self.tree_path(dst) + + if os.path.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + + print(f"move '{src}' -> '{dst}'") + os.rename(src, dst) + + @command + def install(self, src, dst): + dst = self.tree_path(dst) + for s in rglob(os.path.join(self.build, src.lstrip("/")), fatal=True): + with contextlib.suppress(shutil.Error): + print(f"install {s} -> {dst}") + shutil.copy2(os.path.join(self.build, s), dst) + + @command + def remove(self, *files): + for g in files: + for f in rglob(self.tree_path(g)): + if os.path.isdir(f) and not os.path.islink(f): + shutil.rmtree(f) + else: + os.unlink(f) + print(f"remove '{f}'") + + @command + def replace(self, pat, repl, *files): + found = False + for g in files: + for f in rglob(self.tree_path(g)): + found = True + print(f"replace {f}: {pat} -> {repl}") + replace(f, [(pat, repl)]) + + if not found: + assert found, f"No match for {pat} in {' '.join(files)}" + + @command + def runcmd(self, *args): + print("run ", " ".join(args)) + subprocess.run(args, cwd=self.tree, check=True) + + @command + def symlink(self, source, dest): + target = self.tree_path(dest) + if os.path.exists(target): + self.remove(dest) + print(f"symlink '{source}' -> '{target}'") + os.symlink(source, target) + + @command + def systemctl(self, verb, *units): + assert verb in ('enable', 'disable', 'mask') + self.mkdir("/run/systemd/system") + cmd = ['systemctl', '--root', self.tree, '--no-reload', verb] + + for unit in units: + with contextlib.suppress(subprocess.CalledProcessError): + args = cmd + [unit] + self.runcmd(*args) + + +def brace_expand(s): + if not ('{' in s and ',' in s and '}' in s): + return [s] + + result = [] + right = s.find('}') + left = s[:right].rfind('{') + prefix, choices, suffix = s[:left], s[left+1:right], s[right+1:] + for choice in choices.split(','): + result.extend(brace_expand(prefix+choice+suffix)) + + return result + + +def brace_expand_line(line): + return [after for before in line for after in brace_expand(before)] + + +def render_template(path, args): + """Render a template at `path` with arguments `args`""" + + with open(path, "r") as f: + data = f.read() + + tlp = mako.template.Template(text=data, filename=path) + txt = tlp.render(**args) + + lines = map(lambda l: l.strip(), txt.splitlines()) + lines = filter(lambda l: l and not l.startswith("#"), lines) + commands = map(shlex.split, lines) + commands = map(brace_expand_line, commands) + + result = list(commands) + return result diff --git a/osbuild/util/lvm2.py b/osbuild/util/lvm2.py new file mode 100644 index 0000000..1aba0cb --- /dev/null +++ b/osbuild/util/lvm2.py @@ -0,0 +1,628 @@ +#!/usr/bin/python3 +""" +Utility functions to read and write LVM metadata. + +This module provides a `Disk` class that can be used +to read in LVM images and explore and manipulate its +metadata directly, i.e. it reads and writes the data +and headers directly. This allows one to rename an +volume group without having to involve the kernel, +which does not like to have two active LVM volume +groups with the same name. + +The struct definitions have been taken from upstream +LVM2 sources[1], specifically: + - `lib/format_text/layout.h` + - `lib/format_text/format-text.c` + +[1] https://github.com/lvmteam/lvm2 (commit 8801a86) +""" + +import abc +import binascii +import io +import json +import os +import re +import struct +import sys + +from collections import OrderedDict +from typing import BinaryIO, Dict, Union + +PathLike = Union[str, bytes, os.PathLike] + +INITIAL_CRC = 0xf597a6cf +MDA_HEADER_SIZE = 512 + + +def _calc_crc(buf, crc=INITIAL_CRC): + crc = crc ^ 0xFFFFFFFF + crc = binascii.crc32(buf, crc) + return crc ^ 0xFFFFFFFF + + +class CStruct: + class Field: + def __init__(self, name: str, ctype: str, position: int): + self.name = name + self.type = ctype + self.pos = position + + def __init__(self, mapping: Dict, byte_order="<"): + fmt = byte_order + self.fields = [] + for pos, name in enumerate(mapping): + ctype = mapping[name] + fmt += ctype + field = self.Field(name, ctype, pos) + self.fields.append(field) + self.struct = struct.Struct(fmt) + + @property + def size(self): + return self.struct.size + + def unpack(self, data): + up = self.struct.unpack_from(data) + res = { + field.name: up[idx] + for idx, field in enumerate(self.fields) + } + return res + + def read(self, fp): + pos = fp.tell() + data = fp.read(self.size) + + if len(data) < self.size: + return None + + res = self.unpack(data) + res["_position"] = pos + return res + + def pack(self, data): + values = [ + data[field.name] for field in self.fields + ] + data = self.struct.pack(*values) + return data + + def write(self, fp, data: Dict, *, offset=None): + packed = self.pack(data) + + save = None + if offset: + save = fp.tell() + fp.seek(offset) + + fp.write(packed) + + if save: + fp.seek(save) + + def __getitem__(self, name): + for f in self.fields: + if f.name == f: + return f + raise KeyError(f"Unknown field '{name}'") + + def __contains__(self, name): + return any(field.name == name for field in self.fields) + + +class Header: + """Abstract base class for all headers""" + + @property + @classmethod + @abc.abstractmethod + def struct(cls) -> struct.Struct: + """Definition of the underlying struct data""" + + def __init__(self, data): + self.data = data + + def __getitem__(self, name): + assert name in self.struct + return self.data[name] + + def __setitem__(self, name, value): + assert name in self.struct + self.data[name] = value + + def pack(self): + return self.struct.pack(self.data) + + @classmethod + def read(cls, fp): + data = cls.struct.read(fp) # pylint: disable=no-member + return cls(data) + + def write(self, fp): + raw = self.pack() + fp.write(raw) + + def __str__(self) -> str: + msg = f"{self.__class__.__name__}:" + for f in self.struct.fields: + msg += f"\n\t{f.name}: {self[f.name]}" + return msg + + +class LabelHeader(Header): + + struct = CStruct({ # 32 bytes on disk + "id": "8s", # int8_t[8] // LABELONE + "sector": "Q", # uint64_t // Sector number of this label + "crc": "L", # uint32_t // From next field to end of sector + "offset": "L", # uint32_t // Offset from start of struct to contents + "type": "8s" # int8_t[8] // LVM2 00 + }) + + LABELID = b"LABELONE" + + # scan sector 0 to 3 inclusive + LABEL_SCAN_SECTORS = 4 + + def __init__(self, data): + super().__init__(data) + self.sector_size = 512 + + @classmethod + def search(cls, fp, *, sector_size=512): + fp.seek(0, io.SEEK_SET) + for _ in range(cls.LABEL_SCAN_SECTORS): + raw = fp.read(sector_size) + if raw[0:len(cls.LABELID)] == cls.LABELID: + data = cls.struct.unpack(raw) + return LabelHeader(data) + return None + + def read_pv_header(self, fp): + sector = self.data["sector"] + offset = self.data["offset"] + offset = sector * self.sector_size + offset + fp.seek(offset) + return PVHeader.read(fp) + + +class DiskLocN(Header): + + struct = CStruct({ + "offset": "Q", # uint64_t // Offset in bytes to start sector + "size": "Q" # uint64_t // Size in bytes + }) + + @property + def offset(self): + return self.data["offset"] + + @property + def size(self): + return self.data["size"] + + def read_data(self, fp: BinaryIO): + fp.seek(self.offset) + data = fp.read(self.size) + return io.BytesIO(data) + + @classmethod + def read_array(cls, fp): + while True: + data = cls.struct.read(fp) + + if not data or data["offset"] == 0: + break + + yield DiskLocN(data) + + +class PVHeader(Header): + + ID_LEN = 32 + struct = CStruct({ + "uuid": "32s", # int8_t[ID_LEN] + "disk_size": "Q" # uint64_t // size in bytes + }) + # followed by two NULL terminated list of data areas + # and metadata areas of type `DiskLocN` + + def __init__(self, data, data_areas, meta_areas): + super().__init__(data) + self.data_areas = data_areas + self.meta_areas = meta_areas + + @property + def uuid(self): + return self.data["uuid"] + + @property + def disk_size(self): + return self.data["disk_size"] + + @classmethod + def read(cls, fp): + data = cls.struct.read(fp) + + data_areas = list(DiskLocN.read_array(fp)) + meta_areas = list(DiskLocN.read_array(fp)) + + return cls(data, data_areas, meta_areas) + + def __str__(self): + msg = super().__str__() + if self.data_areas: + msg += "\nData: \n\t" + "\n\t".join(map(str, self.data_areas)) + if self.meta_areas: + msg += "\nMeta: \n\t" + "\n\t".join(map(str, self.meta_areas)) + return msg + + +class RawLocN(Header): + struct = CStruct({ + "offset": "Q", # uint64_t // Offset in bytes to start sector + "size": "Q", # uint64_t // Size in bytes + "checksum": "L", # uint32_t // Checksum of data + "flags": "L", # uint32_t // Flags + }) + + IGNORED = 0x00000001 + + @classmethod + def read_array(cls, fp: BinaryIO): + while True: + loc = cls.struct.read(fp) + + if not loc or loc["offset"] == 0: + break + + yield cls(loc) + + +class MDAHeader(Header): + struct = CStruct({ + "checksum": "L", # uint32_t // Checksum of data + "magic": "16s", # int8_t[16] // Allows to scan for metadata + "version": "L", # uint32_t + "start": "Q", # uint64_t // Absolute start byte of itself + "size": "Q" # uint64_t // Size of metadata area + }) + # followed by a null termiated list of type `RawLocN` + + LOC_COMMITTED = 0 + LOC_PRECOMMITTED = 1 + + HEADER_SIZE = MDA_HEADER_SIZE + + def __init__(self, data, raw_locns): + super().__init__(data) + self.raw_locns = raw_locns + + @property + def checksum(self): + return self.data["checksum"] + + @property + def magic(self): + return self.data["magic"] + + @property + def version(self): + return self.data["version"] + + @property + def start(self): + return self.data["start"] + + @property + def size(self): + return self.data["size"] + + @classmethod + def read(cls, fp): + data = cls.struct.read(fp) + raw_locns = list(RawLocN.read_array(fp)) + return cls(data, raw_locns) + + def read_metadata(self, fp) -> "Metadata": + loc = self.raw_locns[self.LOC_COMMITTED] + offset = self.start + loc["offset"] + fp.seek(offset) + data = fp.read(loc["size"]) + md = Metadata.decode(data) + return md + + def write_metadata(self, fp, data: "Metadata"): + raw = data.encode() + + loc = self.raw_locns[self.LOC_COMMITTED] + offset = self.start + loc["offset"] + fp.seek(offset) + + n = fp.write(raw) + loc["size"] = n + loc["checksum"] = _calc_crc(raw) + self.write(fp) + + def write(self, fp): + data = self.struct.pack(self.data) + + fr = io.BytesIO() + fr.write(data) + + for loc in self.raw_locns: + loc.write(fr) + + l = fr.tell() + fr.write(b"\0" * (self.HEADER_SIZE - l)) + + raw = fr.getvalue() + + cs = struct.Struct(" None: + self._vg_name = vg_name + self.data = data + + @property + def vg_name(self) -> str: + return self._vg_name + + @vg_name.setter + def vg_name(self, vg_name: str) -> None: + self.rename_vg(vg_name) + + def rename_vg(self, new_name): + # Replace the corresponding key in the dict and + # ensure it is always the first key + name = self.vg_name + d = self.data[name] + del self.data[name] + self.data[new_name] = d + self.data.move_to_end(new_name, last=False) + + @classmethod + def decode(cls, data: bytes) -> "Metadata": + data = data.decode("utf-8") + name, md = Metadata.decode_data(data) + return cls(name, md) + + def encode(self) -> bytes: + data = Metadata.encode_data(self.data) + return data.encode("utf-8") + + def __str__(self) -> str: + return json.dumps(self.data, indent=2) + + @staticmethod + def decode_data(raw): + substitutions = { + r"#.*\n": "", + r"\[": "[ ", + r"\]": " ]", + r'"': ' " ', + r"[=,]": "", + r"\s+": " ", + r"\0$": "", + } + + data = raw + for pattern, repl in substitutions.items(): + data = re.sub(pattern, repl, data) + + data = data.split() + + DICT_START = '{' + DICT_END = '}' + ARRAY_START = '[' + ARRAY_END = ']' + STRING_START = '"' + STRING_END = '"' + + def next_token(): + if not data: + return None + return data.pop(0) + + def parse_str(val): + result = "" + + while val != STRING_END: + result = f"{result} {val}" + val = next_token() + + return result.strip() + + def parse_type(val): + # type = integer | float | string + # integer = [0-9]* + # float = [0-9]*'.'[0-9]* + # string = '"'.*'"' + + if val == STRING_START: + return parse_str(next_token()) + if "." in val: + return float(val) + return int(val) + + def parse_array(val): + result = [] + + while val != ARRAY_END: + val = parse_type(val) + result.append(val) + val = next_token() + + return result + + def parse_section(val): + result = OrderedDict() + + while val and val != DICT_END: + result[val] = parse_value() + val = next_token() + + return result + + def parse_value(): + val = next_token() + + if val == DICT_START: + return parse_section(next_token()) + if val == ARRAY_START: + return parse_array(next_token()) + + return parse_type(val) + + name = next_token() + obj = parse_section(name) + + return name, obj + + @staticmethod + def encode_data(data): + + def encode_dict(d): + s = "" + for k, v in d.items(): + s += k + if not isinstance(v, dict): + s += " = " + else: + s += " " + s += encode_val(v) + "\n" + return s + + def encode_val(v): + if isinstance(v, int): + s = str(v) + elif isinstance(v, str): + s = f'"{v}"' + elif isinstance(v, list): + s = "[" + ", ".join(encode_val(x) for x in v) + "]" + elif isinstance(v, dict): + s = '{\n' + s += encode_dict(v) + s += '}\n' + return s + + return encode_dict(data) + "\0" + + +class Disk: + def __init__(self, fp, path: PathLike) -> None: + self.fp = fp + self.path = path + + self.lbl_hdr = None + self.pv_hdr = None + self.ma_headers = [] + self.metadata = None + + try: + self._init_headers() + except: # pylint: disable=broad-except + self.fp.close() + raise + + def _init_headers(self): + fp = self.fp + lbl = LabelHeader.search(fp) + + if not lbl: + raise RuntimeError("Could not find label header") + + self.lbl_hdr = lbl + self.pv_hdr = lbl.read_pv_header(fp) + + pv = self.pv_hdr + + for ma in pv.meta_areas: + data = ma.read_data(self.fp) + hdr = MDAHeader.read(data) + self.ma_headers.append(hdr) + + if not self.ma_headers: + raise RuntimeError("Could not find metadata header") + + md = self.ma_headers[0].read_metadata(fp) + self.metadata = md + + @classmethod + def open(cls, path: PathLike, *, read_only=False) -> None: + mode = "rb" + if not read_only: + mode += "+" + + fp = open(path, mode) + + return cls(fp, path) + + def flush_metadata(self): + for ma in self.ma_headers: + ma.write_metadata(self.fp, self.metadata) + + def rename_vg(self, new_name): + """Rename the volume group""" + self.metadata.rename_vg(new_name) + + def set_description(self, desc: str) -> None: + """Set the description of in the metadata block""" + self.metadata.data["description"] = desc + + def set_creation_time(self, t: int) -> None: + """Set the creation time of the volume group""" + self.metadata.data["creation_time"] = t + + def set_creation_host(self, host: str) -> None: + """Set the host that created the volume group""" + self.metadata.data["creation_host"] = host + + def dump(self): + print(self.path) + print(self.lbl_hdr) + print(self.pv_hdr) + print(self.metadata) + + def __enter__(self): + assert self.fp, "Disk not open" + return self + + def __exit__(self, *exc_details): + if self.fp: + self.fp.flush() + self.fp.close() + self.fp = None + + +def main(): + + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} DISK") + sys.exit(1) + + with Disk.open(sys.argv[1]) as disk: + disk.dump() + + +if __name__ == "__main__": + main() diff --git a/osbuild/util/osrelease.py b/osbuild/util/osrelease.py new file mode 100644 index 0000000..24e003b --- /dev/null +++ b/osbuild/util/osrelease.py @@ -0,0 +1,58 @@ +"""OS-Release Information + +This module implements handlers for the `/etc/os-release` type of files. The +related documentation can be found in `os-release(5)`. +""" + +import os + + +# The default paths where os-release is located, as per os-release(5) +DEFAULT_PATHS = [ + "/etc/os-release", + "/usr/lib/os-release" +] + + +def parse_files(*paths): + """Read Operating System Information from `os-release` + + This creates a dictionary with information describing the running operating + system. It reads the information from the path array provided as `paths`. + The first available file takes precedence. It must be formatted according + to the rules in `os-release(5)`. + """ + osrelease = {} + + path = next((p for p in paths if os.path.exists(p)), None) + if path: + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + if line[0] == "#": + continue + key, value = line.split("=", 1) + osrelease[key] = value.strip('"') + + return osrelease + + +def describe_os(*paths): + """Read the Operating System Description from `os-release` + + This creates a string describing the running operating-system name and + version. It uses `parse_files()` underneath to acquire the requested + information. + + The returned string uses the format `${ID}${VERSION_ID}` with all dots + stripped. + """ + osrelease = parse_files(*paths) + + # Fetch `ID` and `VERSION_ID`. Defaults are defined in `os-release(5)`. + osrelease_id = osrelease.get("ID", "linux") + osrelease_version_id = osrelease.get("VERSION_ID", "") + + return osrelease_id + osrelease_version_id.replace(".", "") diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py new file mode 100644 index 0000000..ad40947 --- /dev/null +++ b/osbuild/util/ostree.py @@ -0,0 +1,188 @@ +import contextlib +import json +import os +import subprocess +import tempfile +import typing + +from typing import List + +from .types import PathLike + + +class Param: + """rpm-ostree Treefile parameter""" + + def __init__(self, value_type, mandatory=False): + self.type = value_type + self.mandatory = mandatory + + def check(self, value): + origin = getattr(self.type, "__origin__", None) + if origin: + self.typecheck(value, origin) + if origin is list or origin is typing.List: + self.check_list(value, self.type) + else: + raise NotImplementedError(origin) + else: + self.typecheck(value, self.type) + + @staticmethod + def check_list(value, tp): + inner = tp.__args__ + for x in value: + Param.typecheck(x, inner) + + @staticmethod + def typecheck(value, tp): + if isinstance(value, tp): + return + raise ValueError(f"{value} is not of {tp}") + + +class Treefile: + """Representation of an rpm-ostree Treefile + + The following parameters are currently supported, + presented together with the rpm-ostree compose + phase that they are used in. + - ref: commit + - repos: install + - selinux: install, postprocess, commit + - boot-location: postprocess + - etc-group-members: postprocess + - machineid-compat + + NB: 'ref' and 'repos' are mandatory and must be + present, even if they are not used in the given + phase; they therefore have defaults preset. + """ + + parameters = { + "ref": Param(str, True), + "repos": Param(List[str], True), + "selinux": Param(bool), + "boot-location": Param(str), + "etc-group-members": Param(List[str]), + "machineid-compat": Param(bool), + "initramfs-args": Param(List[str]), + } + + def __init__(self): + self._data = {} + self["ref"] = "osbuild/devel" + self["repos"] = ["osbuild"] + + def __getitem__(self, key): + param = self.parameters.get(key) + if not param: + raise ValueError(f"Unknown param: {key}") + return self._data[key] + + def __setitem__(self, key, value): + param = self.parameters.get(key) + if not param: + raise ValueError(f"Unknown param: {key}") + param.check(value) + self._data[key] = value + + def dumps(self): + return json.dumps(self._data) + + def dump(self, fp): + return json.dump(self._data, fp) + + @contextlib.contextmanager + def as_tmp_file(self): + name = None + try: + fd, name = tempfile.mkstemp(suffix=".json", + text=True) + + with os.fdopen(fd, "w+") as f: + self.dump(f) + + yield name + finally: + if name: + os.unlink(name) + + +def rev_parse(repo: PathLike, ref: str) -> str: + """Resolve an OSTree reference `ref` in the repository at `repo`""" + + repo = os.fspath(repo) + + r = subprocess.run(["ostree", "rev-parse", ref, f"--repo={repo}"], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False) + + msg = r.stdout.strip() + if r.returncode != 0: + raise RuntimeError(msg) + + return msg + + +def deployment_path(root: PathLike, osname: str, ref: str, serial: int): + """Return the path to a deployment given the parameters""" + + base = os.path.join(root, "ostree") + + repo = os.path.join(base, "repo") + stateroot = os.path.join(base, "deploy", osname) + + commit = rev_parse(repo, ref) + sysroot = f"{stateroot}/deploy/{commit}.{serial}" + + return sysroot + + +class PasswdLike: + """Representation of a file with structure like /etc/passwd + + If each line in a file contains a key-value pair separated by the + first colon on the line, it can be considered "passwd"-like. This + class can parse the the list, manipulate it, and export it to file + again. + """ + def __init__(self): + """Initialize an empty PasswdLike object""" + self.db = dict() + + @classmethod + def from_file(cls, path: PathLike, allow_missing_file: bool=False): + """Initialize a PasswdLike object from an existing file""" + ret = cls() + if allow_missing_file: + if not os.path.isfile(path): + return ret + + with open(path, "r") as p: + ret.db = cls._passwd_lines_to_dict(p.readlines()) + return ret + + def merge_with_file(self, path: PathLike, allow_missing_file: bool=False): + """Extend the database with entries from another file""" + if allow_missing_file: + if not os.path.isfile(path): + return + + with open(path, "r") as p: + additional_passwd_dict = self._passwd_lines_to_dict(p.readlines()) + for name, passwd_line in additional_passwd_dict.items(): + if name not in self.db: + self.db[name] = passwd_line + + def dump_to_file(self, path: PathLike): + """Write the current database to a file""" + with open(path, "w") as p: + p.writelines(list(self.db.values())) + + @staticmethod + def _passwd_lines_to_dict(lines): + """Take a list of passwd lines and produce a "name": "line" dictionary""" + return {line.split(':')[0]: line for line in lines} diff --git a/osbuild/util/path.py b/osbuild/util/path.py new file mode 100644 index 0000000..b25d29c --- /dev/null +++ b/osbuild/util/path.py @@ -0,0 +1,14 @@ +"""Path handling utility functions""" +import os.path + +from .types import PathLike + + +def in_tree(path: PathLike, tree: PathLike, must_exist=False) -> bool: + """Return whether the canonical location of 'path' is under 'tree'. + If 'must_exist' is True, the file must also exist for the check to succeed. + """ + path = os.path.abspath(path) + if path.startswith(tree): + return not must_exist or os.path.exists(path) + return False diff --git a/osbuild/util/rhsm.py b/osbuild/util/rhsm.py new file mode 100644 index 0000000..3ab1729 --- /dev/null +++ b/osbuild/util/rhsm.py @@ -0,0 +1,109 @@ +"""Red Hat Subscription Manager support module + +This module implements utilities that help with interactions +with the subscriptions attached to the host machine. +""" + +import configparser +import contextlib +import glob +import os +import re + + +class Subscriptions: + def __init__(self, repositories): + self.repositories = repositories + # These are used as a fallback if the repositories don't + # contain secrets for a requested URL. + self.secrets = None + + def get_fallback_rhsm_secrets(self): + rhsm_secrets = { + 'ssl_ca_cert': "/etc/rhsm/ca/redhat-uep.pem", + 'ssl_client_key': "", + 'ssl_client_cert': "" + } + + keys = glob.glob("/etc/pki/entitlement/*-key.pem") + for key in keys: + # The key and cert have the same prefix + cert = key.rstrip("-key.pem") + ".pem" + # The key is only valid if it has a matching cert + if os.path.exists(cert): + rhsm_secrets['ssl_client_key'] = key + rhsm_secrets['ssl_client_cert'] = cert + # Once the dictionary is complete, assign it to the object + self.secrets = rhsm_secrets + + raise RuntimeError("no matching rhsm key and cert") + + @classmethod + def from_host_system(cls): + """Read redhat.repo file and process the list of repositories in there.""" + ret = cls(None) + with contextlib.suppress(FileNotFoundError): + with open("/etc/yum.repos.d/redhat.repo", "r") as fp: + ret = cls.parse_repo_file(fp) + + with contextlib.suppress(RuntimeError): + ret.get_fallback_rhsm_secrets() + + if not ret.repositories and not ret.secrets: + raise RuntimeError("No RHSM secrets found on this host.") + + return ret + + @staticmethod + def _process_baseurl(input_url): + """Create a regex from a baseurl. + + The osbuild manifest format does not contain information about repositories. + It only includes URLs of each RPM. In order to make this RHSM support work, + osbuild needs to find a relation between a "baseurl" in a *.repo file and the + URL given in the manifest. To do so, it creates a regex from all baseurls + found in the *.repo file and matches them against the URL. + """ + # First escape meta characters that might occur in a URL + input_url = re.escape(input_url) + + # Now replace variables with regexes (see man 5 yum.conf for the list) + for variable in ["\\$releasever", "\\$arch", "\\$basearch", "\\$uuid"]: + input_url = input_url.replace(variable, "[^/]*") + + return re.compile(input_url) + + @classmethod + def parse_repo_file(cls, fp): + """Take a file object and reads its content assuming it is a .repo file.""" + parser = configparser.ConfigParser() + parser.read_file(fp) + + repositories = dict() + for section in parser.sections(): + current = { + "matchurl": cls._process_baseurl(parser.get(section, "baseurl")) + } + for parameter in ["sslcacert", "sslclientkey", "sslclientcert"]: + current[parameter] = parser.get(section, parameter) + + repositories[section] = current + + return cls(repositories) + + def get_secrets(self, url): + # Try to find a matching URL from redhat.repo file first + if self.repositories is not None: + for parameters in self.repositories.values(): + if parameters["matchurl"].match(url) is not None: + return { + "ssl_ca_cert": parameters["sslcacert"], + "ssl_client_key": parameters["sslclientkey"], + "ssl_client_cert": parameters["sslclientcert"] + } + + # In case there is no matching URL, try the fallback + if self.secrets: + return self.secrets + + raise RuntimeError(f"There are no RHSM secret associated with {url}") diff --git a/osbuild/util/rmrf.py b/osbuild/util/rmrf.py new file mode 100644 index 0000000..7a1c015 --- /dev/null +++ b/osbuild/util/rmrf.py @@ -0,0 +1,110 @@ +"""Recursive File System Removal + +This module implements `rm -rf` as a python function. Its core is the +`rmtree()` function, which takes a file-system path and then recursively +deletes everything it finds on that path, until eventually the path entry +itself is dropped. This is modeled around `shutil.rmtree()`. + +This function tries to be as thorough as possible. That is, it tries its best +to modify permission bits and other flags to make sure directory entries can be +removed. +""" + + +import os +import shutil + +import osbuild.util.linux as linux + + +__all__ = [ + "rmtree", +] + + +def rmtree(path: str): + """Recursively Remove from File System + + This removes the object at the given path from the file-system. It + recursively iterates through its content and removes them, before removing + the object itself. + + This function is modeled around `shutil.rmtree()`, but extends its + functionality with a more aggressive approach. It tries much harder to + unlink file system objects. This includes immutable markers and more. + + Note that this function can still fail. In particular, missing permissions + can always prevent this function from succeeding. However, a caller should + never assume that they can intentionally prevent this function from + succeeding. In other words, this function might be extended in any way in + the future, to be more powerful and successful in removing file system + objects. + + Parameters + --------- + path + A file system path pointing to the object to remove. + + Raises + ------ + Exception + This raises the same exceptions as `shutil.rmtree()` (since that + function is used internally). Consult its documentation for details. + """ + + def fixperms(p): + fd = None + try: + + # if we can't open the file, we just return and let the unlink + # fail (again) with `EPERM`. + # A notable case of why open would fail is symlinks; since we + # want the symlink and not the target we pass the `O_NOFOLLOW` + # flag, but this will result in `ELOOP`, thus we never change + # symlinks. This should be fine though since "on Linux, the + # permissions of an ordinary symbolic link are not used in any + # operations"; see symlinks(7). + try: + fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW) + except OSError: + return + + # The root-only immutable flag prevents files from being unlinked + # or modified. Clear it, so we can unlink the file-system tree. + try: + linux.ioctl_toggle_immutable(fd, False) + except OSError: + pass + + # If we do not have sufficient permissions on a directory, we + # cannot traverse it, nor unlink its content. Make sure to set + # sufficient permissions up front. + try: + os.fchmod(fd, 0o777) + except OSError: + pass + finally: + if fd is not None: + os.close(fd) + + def unlink(p): + try: + os.unlink(p) + except IsADirectoryError: + rmtree(p) + except FileNotFoundError: + pass + + def on_error(_fn, p, exc_info): + e = exc_info[0] + if issubclass(e, FileNotFoundError): + pass + elif issubclass(e, PermissionError): + if p != path: + fixperms(os.path.dirname(p)) + fixperms(p) + unlink(p) + else: + raise e + + shutil.rmtree(path, onerror=on_error) diff --git a/osbuild/util/selinux.py b/osbuild/util/selinux.py new file mode 100644 index 0000000..46c1e6a --- /dev/null +++ b/osbuild/util/selinux.py @@ -0,0 +1,84 @@ +"""SELinux utility functions""" + +import errno +import os +import subprocess + +from typing import Dict, TextIO + +# Extended attribute name for SELinux labels +XATTR_NAME_SELINUX = b"security.selinux" + + +def parse_config(config_file: TextIO): + """Parse an SELinux configuration file""" + config = {} + for line in config_file: + line = line.strip() + if not line: + continue + if line.startswith('#'): + continue + k, v = line.split('=', 1) + config[k.strip()] = v.strip() + return config + + +def config_get_policy(config: Dict[str, str]): + """Return the effective SELinux policy + + Checks if SELinux is enabled and if so returns the + policy; otherwise `None` is returned. + """ + enabled = config.get('SELINUX', 'disabled') + if enabled not in ['enforcing', 'permissive']: + return None + return config.get('SELINUXTYPE', None) + + +def setfiles(spec_file: str, root: str, *paths): + """Initialize the security context fields for `paths` + + Initialize the security context fields (extended attributes) + on `paths` using the given specification in `spec_file`. The + `root` argument determines the root path of the file system + and the entries in `path` are interpreted as relative to it. + Uses the setfiles(8) tool to actually set the contexts. + """ + for path in paths: + subprocess.run(["setfiles", "-F", + "-r", root, + spec_file, + f"{root}{path}"], + check=True) + + +def getfilecon(path: str) -> str: + """Get the security context associated with `path`""" + label = os.getxattr(path, XATTR_NAME_SELINUX, + follow_symlinks=False) + return label.decode().strip('\n\0') + + +def setfilecon(path: str, context: str) -> None: + """ + Set the security context associated with `path` + + Like `setfilecon`(3), but does not attempt to translate + the context via `selinux_trans_to_raw_context`. + """ + + try: + os.setxattr(path, XATTR_NAME_SELINUX, + context.encode(), + follow_symlinks=True) + except OSError as err: + # in case we get a not-supported error, check if + # the context we want to set is already set and + # ignore the error in that case. This follows the + # behavior of `setfilecon(3)`. + if err.errno == errno.ENOTSUP: + have = getfilecon(path) + if have == context: + return + raise diff --git a/osbuild/util/types.py b/osbuild/util/types.py new file mode 100644 index 0000000..7462901 --- /dev/null +++ b/osbuild/util/types.py @@ -0,0 +1,11 @@ +# +# Define some useful typing abbreviations +# + +import os + +from typing import Union + + +#: Represents a file system path. See also `os.fspath`. +PathLike = Union[str, bytes, os.PathLike] diff --git a/osbuild/util/udev.py b/osbuild/util/udev.py new file mode 100644 index 0000000..b72de08 --- /dev/null +++ b/osbuild/util/udev.py @@ -0,0 +1,59 @@ +"""userspace /dev device manager (udev) utilities""" + +import contextlib +import pathlib + + +# The default lock dir to use +LOCKDIR = "/run/osbuild/locks/udev" + + +class UdevInhibitor: + """ + Inhibit execution of certain udev rules for block devices + + This is the osbuild side of the custom mechanism that + allows us to inhibit certain udev rules for block devices. + + For each device a lock file is created in a well known + directory (LOCKDIR). A custom udev rule set[1] checks + for the said lock file and inhibits other udev rules from + being executed. + See the aforementioned rules file for more information. + + [1] 10-osbuild-inhibitor.rules + """ + + def __init__(self, path: pathlib.Path): + self.path = path + path.parent.mkdir(parents=True, exist_ok=True) + + def inhibit(self) -> None: + self.path.touch() + + def release(self) -> None: + with contextlib.suppress(FileNotFoundError): + self.path.unlink() + + @property + def active(self) -> bool: + return self.path.exists() + + def __str__(self): + return f"UdevInhibtor at '{self.path}'" + + @classmethod + def for_dm_name(cls, name: str, lockdir=LOCKDIR): + """Inhibit a Device Mapper device with the given name""" + path = pathlib.Path(lockdir, f"dm-{name}") + ib = cls(path) + ib.inhibit() + return ib + + @classmethod + def for_device(cls, major: int, minor: int, lockdir=LOCKDIR): + """Inhibit a device given its major and minor number""" + path = pathlib.Path(lockdir, f"device-{major}:{minor}") + ib = cls(path) + ib.inhibit() + return ib diff --git a/release.md b/release.md new file mode 100644 index 0000000..e029ab4 --- /dev/null +++ b/release.md @@ -0,0 +1,40 @@ +CHANGES WITH 41: +---------------- + + * `stages/authconfig`: apply default authconfig settings (#871) + + * `stages/yum.config`: add a new stage, allow configuring YUM langpacks plugin (#872, #874) + + * `stages/selinux`: ability to force an auto-relabel (#875) + + * `stages/pwquality`: set pwquality configuration (#870) + + * `stages/rhsm`: add support to configure yum plugins (#876) + + * `stages/cron.script`: add new stage (#873) + + * `stages/grub2`: add support for terminal and serial config (#865) + + * `stages/sshd.config`: set sshd configuration (#862) + + * `stages/modprobe`: support 'install' command (#867) + + * `stages/lvm2.create`: fix 'size' and add 'extents' (#864) + + * Tech-preview: support for building rhel7 images (#845) + + * New options for cloud-init: Azure datasource and reporting/logging (#866) + + * Transparent support for ostree deployments (#847) + + * formats/v2: fix describe for mount without source (#878) + + * Do not bind-mount the `/boot` directory from the host into the build root (#860) + + * mpp: add mpp-eval, mpp-join and fix long options (#848, #851) + +Contributions from: Achilleas Koutsou, Alexander Larsson, Christian Kellner, + Martin Sehnoutka, Ondřej Budai, Simon Steinbeiss, Tom Gundersen, + Tomas Hozza + +— Vöcklabruck, 2021-11-08 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..598596e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +jsonschema +pytest diff --git a/runners/org.osbuild.arch b/runners/org.osbuild.arch new file mode 120000 index 0000000..2a7391d --- /dev/null +++ b/runners/org.osbuild.arch @@ -0,0 +1 @@ +org.osbuild.linux \ No newline at end of file diff --git a/runners/org.osbuild.centos8 b/runners/org.osbuild.centos8 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.centos8 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.centos9 b/runners/org.osbuild.centos9 new file mode 100755 index 0000000..963e2bc --- /dev/null +++ b/runners/org.osbuild.centos9 @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys + +import osbuild.api + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + subprocess.run(["touch", "/etc/ld.so.conf"], check=True) + subprocess.run(["ldconfig"], check=True) + + +def sysusers(): + try: + subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stdout) + sys.exit(1) + + +def tmpfiles(): + # Allow systemd-tmpfiles to return non-0. Some packages want to create + # directories owned by users that are not set up with systemd-sysusers. + subprocess.run(["systemd-tmpfiles", "--create"], check=False) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +if __name__ == "__main__": + with osbuild.api.exception_handler(): + ldconfig() + sysusers() + tmpfiles() + nsswitch() + + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.fedora30 b/runners/org.osbuild.fedora30 new file mode 100755 index 0000000..963e2bc --- /dev/null +++ b/runners/org.osbuild.fedora30 @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys + +import osbuild.api + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + subprocess.run(["touch", "/etc/ld.so.conf"], check=True) + subprocess.run(["ldconfig"], check=True) + + +def sysusers(): + try: + subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stdout) + sys.exit(1) + + +def tmpfiles(): + # Allow systemd-tmpfiles to return non-0. Some packages want to create + # directories owned by users that are not set up with systemd-sysusers. + subprocess.run(["systemd-tmpfiles", "--create"], check=False) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +if __name__ == "__main__": + with osbuild.api.exception_handler(): + ldconfig() + sysusers() + tmpfiles() + nsswitch() + + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.fedora31 b/runners/org.osbuild.fedora31 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora31 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora32 b/runners/org.osbuild.fedora32 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora32 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora33 b/runners/org.osbuild.fedora33 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora33 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora34 b/runners/org.osbuild.fedora34 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora34 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora35 b/runners/org.osbuild.fedora35 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora35 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora36 b/runners/org.osbuild.fedora36 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora36 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.fedora37 b/runners/org.osbuild.fedora37 new file mode 120000 index 0000000..d46777b --- /dev/null +++ b/runners/org.osbuild.fedora37 @@ -0,0 +1 @@ +org.osbuild.fedora30 \ No newline at end of file diff --git a/runners/org.osbuild.linux b/runners/org.osbuild.linux new file mode 100755 index 0000000..a023e68 --- /dev/null +++ b/runners/org.osbuild.linux @@ -0,0 +1,9 @@ +#!/usr/bin/python3 + +import subprocess +import sys + + +if __name__ == "__main__": + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.rhel7 b/runners/org.osbuild.rhel7 new file mode 100755 index 0000000..9399a6c --- /dev/null +++ b/runners/org.osbuild.rhel7 @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys + +import osbuild.api + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + with open("/etc/ld.so.conf", "w", ) as f: + # qemu-img needs `libiscsi`, which is located in /usr/lib64/iscsi + f.write("/usr/lib64/iscsi\n") + f.flush() + subprocess.run(["ldconfig"], check=True) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +if __name__ == "__main__": + with osbuild.api.exception_handler(): + ldconfig() + nsswitch() + + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.rhel81 b/runners/org.osbuild.rhel81 new file mode 100755 index 0000000..925326e --- /dev/null +++ b/runners/org.osbuild.rhel81 @@ -0,0 +1,84 @@ +#!/usr/libexec/platform-python + +import os +import subprocess +import sys + +import osbuild.api + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + subprocess.run(["touch", "/etc/ld.so.conf"], check=True) + subprocess.run(["ldconfig"], check=True) + + +def sysusers(): + try: + subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stdout) + sys.exit(1) + + +def tmpfiles(): + # Allow systemd-tmpfiles to return non-0. Some packages want to create + # directories owned by users that are not set up with systemd-sysusers. + subprocess.run(["systemd-tmpfiles", "--create"], check=False) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +def os_release(): + """/usr/lib/os-release doesn't exist. The `redhat-release` package + generates `/etc/os-release directly. To work around this, do the same here. + + https://bugzilla.redhat.com/show_bug.cgi?id=1766754 + """ + + # remove the symlink that systemd-nspawn creates + os.remove("/etc/os-release") + with open("/etc/os-release", "w") as f: + f.write('NAME="Red Hat Enterprise Linux"\n') + f.write('VERSION="8.1 (Ootpa)"\n') + f.write('ID="rhel"\n') + f.write('ID_LIKE="fedora"\n') + f.write('VERSION_ID="8.1"\n') + f.write('PLATFORM_ID="platform:el8"\n') + f.write('PRETTY_NAME="Red Hat Enterprise Linux 8.1 (Ootpa)"\n') + f.write('ANSI_COLOR="0;31"\n') + f.write('CPE_NAME="cpe:/o:redhat:enterprise_linux:8.1:GA"\n') + f.write('HOME_URL="https://www.redhat.com/"\n') + f.write('BUG_REPORT_URL="https://bugzilla.redhat.com/"\n') + + +def python_alternatives(): + """/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points + to /usr/bin/python3.6 by default. Recreate the link in /etc, so that + shebang lines in stages and assemblers work. + """ + os.makedirs("/etc/alternatives", exist_ok=True) + try: + os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3") + except FileExistsError: + pass + +if __name__ == "__main__": + with osbuild.api.exception_handler(): + ldconfig() + sysusers() + tmpfiles() + nsswitch() + os_release() + python_alternatives() + + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.rhel82 b/runners/org.osbuild.rhel82 new file mode 100755 index 0000000..3d8f5fd --- /dev/null +++ b/runners/org.osbuild.rhel82 @@ -0,0 +1,83 @@ +#!/usr/libexec/platform-python + +import os +import platform +import subprocess +import sys + +import osbuild.api + + +def quirks(): + # Platform specific quirks + env = os.environ.copy() + + if platform.machine() == "aarch64": + # Work around a bug in qemu-img on aarch64 that can lead to qemu-img + # hangs when more then one coroutine is use (which is the default) + # See https://bugs.launchpad.net/qemu/+bug/1805256 + env["OSBUILD_QEMU_IMG_COROUTINES"] = "1" + + return env + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + subprocess.run(["touch", "/etc/ld.so.conf"], check=True) + subprocess.run(["ldconfig"], check=True) + + +def sysusers(): + try: + subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stdout) + sys.exit(1) + + +def tmpfiles(): + # Allow systemd-tmpfiles to return non-0. Some packages want to create + # directories owned by users that are not set up with systemd-sysusers. + subprocess.run(["systemd-tmpfiles", "--create"], check=False) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +def python_alternatives(): + """/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points + to /usr/bin/python3.6 by default. Recreate the link in /etc, so that + shebang lines in stages and assemblers work. + """ + os.makedirs("/etc/alternatives", exist_ok=True) + try: + os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3") + except FileExistsError: + pass + + +def main(): + with osbuild.api.exception_handler(): + ldconfig() + sysusers() + tmpfiles() + nsswitch() + python_alternatives() + + env = quirks() + + r = subprocess.run(sys.argv[1:], + env=env, + check=False) + sys.exit(r.returncode) + + +if __name__ == "__main__": + main() diff --git a/runners/org.osbuild.rhel83 b/runners/org.osbuild.rhel83 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.rhel83 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.rhel84 b/runners/org.osbuild.rhel84 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.rhel84 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.rhel85 b/runners/org.osbuild.rhel85 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.rhel85 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.rhel86 b/runners/org.osbuild.rhel86 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.rhel86 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.rhel87 b/runners/org.osbuild.rhel87 new file mode 120000 index 0000000..fe28a86 --- /dev/null +++ b/runners/org.osbuild.rhel87 @@ -0,0 +1 @@ +org.osbuild.rhel82 \ No newline at end of file diff --git a/runners/org.osbuild.rhel90 b/runners/org.osbuild.rhel90 new file mode 120000 index 0000000..ffc7052 --- /dev/null +++ b/runners/org.osbuild.rhel90 @@ -0,0 +1 @@ +org.osbuild.centos9 \ No newline at end of file diff --git a/runners/org.osbuild.rhel91 b/runners/org.osbuild.rhel91 new file mode 120000 index 0000000..70c2aaf --- /dev/null +++ b/runners/org.osbuild.rhel91 @@ -0,0 +1 @@ +org.osbuild.rhel90 \ No newline at end of file diff --git a/runners/org.osbuild.ubuntu1804 b/runners/org.osbuild.ubuntu1804 new file mode 100755 index 0000000..963e2bc --- /dev/null +++ b/runners/org.osbuild.ubuntu1804 @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys + +import osbuild.api + + +def ldconfig(): + # ld.so.conf must exist, or `ldconfig` throws a warning + subprocess.run(["touch", "/etc/ld.so.conf"], check=True) + subprocess.run(["ldconfig"], check=True) + + +def sysusers(): + try: + subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stdout) + sys.exit(1) + + +def tmpfiles(): + # Allow systemd-tmpfiles to return non-0. Some packages want to create + # directories owned by users that are not set up with systemd-sysusers. + subprocess.run(["systemd-tmpfiles", "--create"], check=False) + + +def nsswitch(): + # the default behavior is fine, but using nss-resolve does not + # necessarily work in a non-booted container, so make sure that + # is not configured. + try: + os.remove("/etc/nsswitch.conf") + except FileNotFoundError: + pass + + +if __name__ == "__main__": + with osbuild.api.exception_handler(): + ldconfig() + sysusers() + tmpfiles() + nsswitch() + + r = subprocess.run(sys.argv[1:], check=False) + sys.exit(r.returncode) diff --git a/runners/org.osbuild.ubuntu2004 b/runners/org.osbuild.ubuntu2004 new file mode 120000 index 0000000..317be18 --- /dev/null +++ b/runners/org.osbuild.ubuntu2004 @@ -0,0 +1 @@ +org.osbuild.ubuntu1804 \ No newline at end of file diff --git a/samples b/samples new file mode 120000 index 0000000..691a31b --- /dev/null +++ b/samples @@ -0,0 +1 @@ +test/data/manifests/ \ No newline at end of file diff --git a/schemas/osbuild1.json b/schemas/osbuild1.json new file mode 100644 index 0000000..76f8699 --- /dev/null +++ b/schemas/osbuild1.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://osbuild.org/schemas/osbuild1.json", + + "title": "OSBuild Manifest", + "description": "OSBuild manifest describing a pipeline and all parameters", + "type": "object", + "additionalProperties": false, + "properties": { + "pipeline": { "$ref": "#/definitions/pipeline" }, + "sources": { "$ref": "#/definitions/sources" } + }, + + "definitions": { + "assembler": { + "title": "Pipeline Assembler", + "description": "Final stage of a pipeline that assembles the result", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "options": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ "name" ] + }, + + "build": { + "title": "Build Pipeline", + "description": "Description of the build pipeline required to run stages", + "type": "object", + "additionalProperties": false, + "properties": { + "pipeline": { "$ref": "#/definitions/pipeline" }, + "runner": { "type": "string" } + }, + "required": [ "pipeline", "runner" ] + }, + + "pipeline": { + "title": "Pipeline Description", + "description": "Full description of a pipeline to execute", + "type": "object", + "additionalProperties": false, + "properties": { + "assembler": { "$ref": "#/definitions/assembler" }, + "build": { "$ref": "#/definitions/build" }, + "stages": { "$ref": "#/definitions/stages" } + } + }, + + "source": { + "title": "External Source", + "description": "External source to be passed to the pipeline", + "type": "object", + "additionalProperties": true + }, + + "sources": { + "title": "Collection of External Sources", + "description": "Collection of external sources to be passed to the pipeline", + "type": "object", + "additionalProperties": { "$ref": "#/definitions/source" } + }, + + "stage": { + "title": "Pipeline Stage", + "description": "Single stage of a pipeline executing one step", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "options": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ "name" ] + }, + + "stages": { + "type": "array", + "items": { "$ref": "#/definitions/stage" } + } + } +} diff --git a/schemas/osbuild2.json b/schemas/osbuild2.json new file mode 100644 index 0000000..d9f0d53 --- /dev/null +++ b/schemas/osbuild2.json @@ -0,0 +1,169 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://osbuild.org/schemas/osbuild2.json", + + "title": "OSBuild Manifest", + "description": "OSBuild manifest describing a pipeline and all parameters", + "type": "object", + "additionalProperties": false, + "properties": { + "pipelines": { "$ref": "#/definitions/pipelines" }, + "sources": { "$ref": "#/definitions/sources" }, + "version": { + "enum": ["2"] + } + }, + + "definitions": { + + "devices": { + "title": "Collection of devices for a stage", + "additionalProperties": { + "$ref": "#/definitions/device" + } + }, + + "device": { + "title": "Device for a stage", + "additionalProperties": false, + "required": ["type"], + "properties": { + "type": { "type": "string" }, + "parent": { "type": "string" }, + "options": { + "type": "object", + "additionalProperties": true + } + } + }, + + "inputs": { + "title": "Collection of inputs for a stage", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z][a-zA-Z0-9_\\-\\.]{0,254}": { + "$ref": "#/definitions/input" + } + } + }, + + "input": { + "title": "Single input for a stage", + "additionalProperties": false, + "required": ["type", "origin", "references"], + "properties": { + "type": { "type": "string" }, + "origin": { "enum": ["org.osbuild.source", "org.osbuild.pipeline"] }, + "references": { "$ref": "#/definitions/reference" }, + "options": { + "type": "object", + "additionalProperties": true + } + } + }, + + "mounts": { + "title": "Collection of mount points for a stage", + "type": "array", + "items": { "$ref": "#/definitions/mount"} + }, + + "mount": { + "title": "Mount point for a stage", + "additionalProperties": false, + "required": ["name", "type"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } + } + }, + + "pipelines": { + "title": "Collection of pipelines to execute", + "description": "Array of pipelines to execute one after another", + "type": "array", + "items": { "$ref": "#/definitions/pipeline" } + }, + + "pipeline": { + "title": "Pipeline Description", + "description": "Full description of a pipeline to execute", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type:": "string" }, + "build": { "type": "string" }, + "runner": { "type": "string" }, + "source-epoch": { "type": "integer" }, + "stages": { "$ref": "#/definitions/stages" } + } + }, + + "reference": { + "oneOf":[ + { + "type": "array", + "items": { "type": "string" } + },{ + "type": "object", + "additionalProperties": true + } + ] + }, + + "source": { + "title": "External Source", + "description": "External source to be passed to the pipeline", + "type": "object", + "additionalProperties": false, + "properties": { + "items": { "$ref": "#/definitions/reference" }, + "options": { + "type": "object", + "additionalProperties": true + } + }, + "required": ["items"] + }, + + "sources": { + "title": "Collection of External Sources", + "description": "Collection of external sources to be passed to the pipeline", + "type": "object", + "additionalProperties": { "$ref": "#/definitions/source" } + }, + + "stage": { + "title": "Pipeline Stage", + "description": "Single stage of a pipeline executing one step", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "type": "string" }, + "devices": { "$ref": "#/definitions/devices" }, + "inputs": {"$ref": "#/definitions/inputs" }, + "mounts": {"$ref": "#/definitions/mounts" }, + "options": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ "type" ] + }, + + "stages": { + "type": "array", + "items": { "$ref": "#/definitions/stage" } + } + } +} diff --git a/schutzbot/RH-IT-Root-CA.keystore b/schutzbot/RH-IT-Root-CA.keystore new file mode 100644 index 0000000..f6a60ad Binary files /dev/null and b/schutzbot/RH-IT-Root-CA.keystore differ diff --git a/schutzbot/ci_details.sh b/schutzbot/ci_details.sh new file mode 100755 index 0000000..28879ef --- /dev/null +++ b/schutzbot/ci_details.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Dumps details about the instance running the CI job. + +PRIMARY_IP=$(ip route get 8.8.8.8 | head -n 1 | cut -d' ' -f7) +EXTERNAL_IP=$(curl --retry 5 -s -4 icanhazip.com) +PTR=$(curl --retry 5 -s -4 icanhazptr.com) +CPUS=$(nproc) +MEM=$(free -m | grep -oP '\d+' | head -n 1) +DISK=$(df --output=size -h / | sed '1d;s/[^0-9]//g') +HOSTNAME=$(uname -n) + +echo -e "\033[0;36m" +cat << EOF +------------------------------------------------------------------------------ +CI MACHINE SPECS +------------------------------------------------------------------------------ + + Hostname: ${HOSTNAME} + Primary IP: ${PRIMARY_IP} + External IP: ${EXTERNAL_IP} + Reverse DNS: ${PTR} + CPUs: ${CPUS} + RAM: ${MEM} GB + DISK: ${DISK} GB + +------------------------------------------------------------------------------ +EOF +echo -e "\033[0m" + +echo "List of installed packages:" +rpm -qa | sort +echo "------------------------------------------------------------------------------" + +# Ensure cloud-init has completely finished on the instance. This ensures that +# the instance is fully ready to go. +while true; do + if [[ -f /var/lib/cloud/instance/boot-finished ]]; then + break + fi + echo -e "\n🤔 Waiting for cloud-init to finish running..." + sleep 5 +done diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh new file mode 100755 index 0000000..1586d2f --- /dev/null +++ b/schutzbot/deploy.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -euxo pipefail + +DNF_REPO_BASEURL=http://osbuild-composer-repos.s3.amazonaws.com + +# The osbuild-composer commit to run reverse-dependency test against. +# Currently: test: add image test cases for Fedora 34 and 35 +OSBUILD_COMPOSER_COMMIT=1401a7a6595f86812008b6bfdacdf7289945a6f3 + +# Get OS details. +source /etc/os-release +ARCH=$(uname -m) + +# Add osbuild team ssh keys. +cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null + +# Distro version that this script is running on. +DISTRO_VERSION=${ID}-${VERSION_ID} + +if [[ "$ID" == rhel ]] && sudo subscription-manager status; then + # If this script runs on subscribed RHEL, install content built using CDN + # repositories. + DISTRO_VERSION=rhel-${VERSION_ID%.*}-cdn +fi + +# Set up dnf repositories with the RPMs we want to test +sudo tee /etc/yum.repos.d/osbuild.repo << EOF +[osbuild] +name=osbuild ${CI_COMMIT_SHA} +baseurl=${DNF_REPO_BASEURL}/osbuild/${DISTRO_VERSION}/${ARCH}/${CI_COMMIT_SHA} +enabled=1 +gpgcheck=0 +# Default dnf repo priority is 99. Lower number means higher priority. +priority=5 + +[osbuild-composer] +name=osbuild-composer ${OSBUILD_COMPOSER_COMMIT} +baseurl=${DNF_REPO_BASEURL}/osbuild-composer/${DISTRO_VERSION}/${ARCH}/${OSBUILD_COMPOSER_COMMIT} +enabled=1 +gpgcheck=0 +# Give this a slightly lower priority, because we used to have osbuild in this repo as well. +priority=10 +EOF + +if [[ $ID == rhel || $ID == centos ]] && ! rpm -q epel-release; then + # Set up EPEL repository (for ansible and koji) + sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION_ID%.*}.noarch.rpm +fi + +# Install the Image Builder packages. +# Note: installing only -tests to catch missing dependencies +sudo dnf -y install osbuild-composer-tests + +# Set up a directory to hold repository overrides. +sudo mkdir -p /etc/osbuild-composer/repositories + +# Temp fix until composer gains these dependencies +sudo dnf -y install osbuild-luks2 osbuild-lvm2 diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh new file mode 100755 index 0000000..88fd1d2 --- /dev/null +++ b/schutzbot/mockbuild.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -euo pipefail + +# Colorful output. +function greenprint { + echo -e "\033[1;32m${1}\033[0m" +} + +# Get OS and architecture details. +source /etc/os-release +ARCH=$(uname -m) + +# Register RHEL if we are provided with a registration script and intend to do that. +REGISTER="${REGISTER:-'false'}" +if [[ $REGISTER == "true" && -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then + greenprint "🪙 Registering RHEL instance" + sudo chmod +x "$RHN_REGISTRATION_SCRIPT" + sudo "$RHN_REGISTRATION_SCRIPT" +fi + +# Mock configuration file to use for building RPMs. +MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)" + +if [[ $ID == centos ]]; then + MOCK_CONFIG="centos-stream-${VERSION_ID%.*}-$(uname -m)" +fi + +# The commit this script operates on. +COMMIT=$(git rev-parse HEAD) + +# Bucket in S3 where our artifacts are uploaded +REPO_BUCKET=osbuild-composer-repos + +# Public URL for the S3 bucket with our artifacts. +MOCK_REPO_BASE_URL="http://${REPO_BUCKET}.s3.amazonaws.com" + +# Distro version in whose buildroot was the RPM built. +DISTRO_VERSION=${ID}-${VERSION_ID} + +if [[ "$ID" == rhel ]] && sudo subscription-manager status; then + # If this script runs on a subscribed RHEL, the RPMs are actually built + # using the latest CDN content, therefore rhel-*-cdn is used as the distro + # version. + DISTRO_VERSION=rhel-${VERSION_ID%.*}-cdn +fi + +# Relative path of the repository – used for constructing both the local and +# remote paths below, so that they're consistent. +REPO_PATH=osbuild/${DISTRO_VERSION}/${ARCH}/${COMMIT} + +# Directory to hold the RPMs temporarily before we upload them. +REPO_DIR=repo/${REPO_PATH} + +# Full URL to the RPM repository after they are uploaded. +REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_PATH} + +# Don't rerun the build if it already exists +if curl --silent --fail --head --output /dev/null "${REPO_URL}/repodata/repomd.xml"; then + greenprint "🎁 Repository already exists. Exiting." + exit 0 +fi + +# Mock and s3cmd is only available in EPEL for RHEL. +if [[ $ID == rhel || $ID == centos ]] && ! rpm -q epel-release; then + greenprint "📦 Setting up EPEL repository" + curl -Ls --retry 5 --output /tmp/epel.rpm \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION_ID%.*}.noarch.rpm + sudo rpm -Uvh /tmp/epel.rpm +fi + +# Install requirements for building RPMs in mock. +greenprint "📦 Installing mock requirements" +sudo dnf -y install createrepo_c make mock python3-pip rpm-build s3cmd + +# Print some data. +greenprint "🧬 Using mock config: ${MOCK_CONFIG}" +greenprint "📦 SHA: ${COMMIT}" +greenprint "📤 RPMS will be uploaded to: ${REPO_URL}" + +# Build source RPMs. +greenprint "🔧 Building source RPMs." +make srpm + +if [[ "$ID" == rhel && ${VERSION_ID%.*} == 8 ]] && ! sudo subscription-manager status; then + greenprint "📋 Updating RHEL 8 mock template with the latest nightly repositories" + # strip everything after line with # repos + sudo sed -i '/# repos/q' /etc/mock/templates/rhel-8.tpl + # remove the subscription check + sudo sed -i "s/config_opts\['redhat_subscription_required'\] = True/config_opts['redhat_subscription_required'] = False/" /etc/mock/templates/rhel-8.tpl + # reuse redhat.repo + cat /etc/yum.repos.d/rhel8internal.repo | sudo tee -a /etc/mock/templates/rhel-8.tpl > /dev/null + # We need triple quotes at the end of the template to mark the end of the repo list. + echo '"""' | sudo tee -a /etc/mock/templates/rhel-8.tpl +elif [[ $VERSION_ID == 9.0 ]]; then + greenprint "📋 Inserting RHEL 9 mock template" + sudo cp schutzbot/rhel-9-mock-configs/templates/rhel-9.tpl /etc/mock/templates/ + sudo cp schutzbot/rhel-9-mock-configs/*.cfg /etc/mock/ +fi + +# Compile RPMs in a mock chroot +greenprint "🎁 Building RPMs with mock" +sudo mock -r $MOCK_CONFIG --no-bootstrap-chroot \ + --resultdir $REPO_DIR \ + rpmbuild/SRPMS/*.src.rpm +sudo chown -R $USER ${REPO_DIR} + +# Change the ownership of all of our repo files from root to our CI user. +sudo chown -R "$USER" "${REPO_DIR%%/*}" + +greenprint "🧹 Remove logs from mock build" +rm "${REPO_DIR}"/*.log + +# Create a repo of the built RPMs. +greenprint "⛓️ Creating dnf repository" +createrepo_c "${REPO_DIR}" + +# Upload repository to S3. +greenprint "☁ Uploading RPMs to S3" +pushd repo + AWS_ACCESS_KEY_ID="$V2_AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$V2_AWS_SECRET_ACCESS_KEY" \ + s3cmd --acl-public put --recursive . s3://${REPO_BUCKET}/ +popd diff --git a/schutzbot/rhel-9-mock-configs/rhel-9-aarch64.cfg b/schutzbot/rhel-9-mock-configs/rhel-9-aarch64.cfg new file mode 100644 index 0000000..786f30a --- /dev/null +++ b/schutzbot/rhel-9-mock-configs/rhel-9-aarch64.cfg @@ -0,0 +1,4 @@ +include('templates/rhel-9.tpl') + +config_opts['target_arch'] = 'aarch64' +config_opts['legal_host_arches'] = ('aarch64',) diff --git a/schutzbot/rhel-9-mock-configs/rhel-9-x86_64.cfg b/schutzbot/rhel-9-mock-configs/rhel-9-x86_64.cfg new file mode 100644 index 0000000..477c1a3 --- /dev/null +++ b/schutzbot/rhel-9-mock-configs/rhel-9-x86_64.cfg @@ -0,0 +1,4 @@ +include('templates/rhel-9.tpl') + +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) diff --git a/schutzbot/rhel-9-mock-configs/templates/rhel-9.tpl b/schutzbot/rhel-9-mock-configs/templates/rhel-9.tpl new file mode 100644 index 0000000..eb5f131 --- /dev/null +++ b/schutzbot/rhel-9-mock-configs/templates/rhel-9.tpl @@ -0,0 +1,55 @@ +# inspired by https://gitlab.com/redhat/centos-stream/ci-cd/zuul/jobs/-/blob/master/playbooks/files/centos-stream9-x86_64.cfg + +config_opts['root'] = 'rhel-9-{{ target_arch }}' + + +config_opts['chroot_setup_cmd'] = 'install tar gcc-c++ redhat-rpm-config redhat-release which xz sed make bzip2 gzip gcc coreutils unzip shadow-utils diffutils cpio bash gawk rpm-build info patch util-linux findutils grep' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['extra_chroot_dirs'] = [ '/run/lock', ] +config_opts['bootstrap_image'] = 'registry-proxy.engineering.redhat.com/rh-osbs/ubi9' + +config_opts['dnf.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= +mdpolicy=group:primary +best=1 +protected_packages= +module_platform_id=platform:el9 +user_agent={{ user_agent }} + +[rhel9-baseos] +name=RHEL 9 BaseOS +baseurl=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9/compose/BaseOS/$basearch/os/ +enabled=1 +gpgcheck=0 + +[rhel9-appstream] +name=RHEL 9 AppStream +baseurl=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9/compose/AppStream/$basearch/os/ +enabled=1 +gpgcheck=0 + +[rhel9-crb] +name=RHEL 9 CRB +baseurl=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9/compose/CRB/$basearch/os/ +enabled=1 +gpgcheck=0 + +[rhel9-buildroot] +name=RHEL 9 Buildroot +baseurl=http://download.eng.bos.redhat.com/rhel-9/nightly/BUILDROOT-9/latest-BUILDROOT-9-RHEL-9/compose/Buildroot/$basearch/os +enabled=1 +gpgcheck=0 +""" + diff --git a/schutzbot/sonarqube.sh b/schutzbot/sonarqube.sh new file mode 100755 index 0000000..e3f01e1 --- /dev/null +++ b/schutzbot/sonarqube.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -euxo pipefail + +SONAR_SCANNER_CLI_VERSION=${SONAR_SCANNER_CLI_VERSION:-4.6.2.2472} + +export SONAR_SCANNER_OPTS="-Djavax.net.ssl.trustStore=schutzbot/RH-IT-Root-CA.keystore -Djavax.net.ssl.trustStorePassword=$KEYSTORE_PASS" +sudo dnf install -y unzip +curl "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_CLI_VERSION-linux.zip" -o sonar-scanner-cli.zip +unzip -q sonar-scanner-cli.zip + +SONAR_SCANNER_CMD="sonar-scanner-$SONAR_SCANNER_CLI_VERSION-linux/bin/sonar-scanner" +SCANNER_OPTS="-Dsonar.projectKey=osbuild:osbuild -Dsonar.sources=. -Dsonar.host.url=https://sonarqube.corp.redhat.com -Dsonar.login=$SONAR_SCANNER_TOKEN" + +# add options for branch analysis if not running on main +if [ "$CI_COMMIT_BRANCH" != "main" ];then + SCANNER_OPTS="$SCANNER_OPTS -Dsonar.pullrequest.branch=$CI_COMMIT_BRANCH -Dsonar.pullrequest.key=$CI_COMMIT_SHA -Dsonar.pullrequest.base=main" +fi + +# run the sonar-scanner +eval "$SONAR_SCANNER_CMD $SCANNER_OPTS" + +SONARQUBE_URL="https://sonarqube.corp.redhat.com/dashboard?id=osbuild%3Aosbuild&pullRequest=$CI_COMMIT_SHA" +# Report back to GitHub +curl \ + -u "${SCHUTZBOT_LOGIN}" \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/osbuild/osbuild/statuses/${CI_COMMIT_SHA}" \ + -d '{"state":"success", "description": "SonarQube scan sent for analysis", "context": "SonarQube", "target_url": "'"${SONARQUBE_URL}"'"}' diff --git a/schutzbot/team_ssh_keys.txt b/schutzbot/team_ssh_keys.txt new file mode 100644 index 0000000..d3317c0 --- /dev/null +++ b/schutzbot/team_ssh_keys.txt @@ -0,0 +1,10 @@ +# SSH keys from members of the osbuild team that are used in CI. +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDE+WCsyhgXLEBvcNKmEO5w9cYsqlcTUpQLgtHAO8+Ub1bw3UTuiJ/Qd3QmAr6gnb1US0Zs8srTZ5W34jOmYcupmmujLEUUrc/Lj2UzIDWqYi04GD3nzGM/oRWT2glJUXBe63bdy38/GfGdNqp9ZkVgaOEkwVyaxAuNcgNXoEJroSWMStvnDki9SvmXn972QFn+vfQdSt+6z18YFqyE0UhNVX+VE6rezNFRzARCrd+nCc8fVdTRd2vQ+0McButDLzwRXEwLjNOPfXCvoI5EH8+uhOWzLlR9ChfsMcbsipDRHyee0hKWhqg/C8j2DCC9r0zHArPcvLi+qGNIxdSE+KXjWRInr/DSttN/MwDulhJ4uHQQA/PDM1W8yddZpfBv1PaDO+wsm6s2bMN1mzgUPAOZK+i6gaYuoVc89f6+aHJVEPgtqT9Zp+6PD13RWDV6jfi0pfmwRi/CXLlc588oU5D8LEmlUNnUvSNmyV7OJayIK+K6+e7hWGC6/UWess+WSBc= larskarlitski@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQR4bv/n0rVI0ZHV4QoEjNrnHsUFFAcLJ6FWnnJyI31aFXWjjPf3NkbynPqqv3ksk9mj6jJzIBnlo2lZ0kLKIlnblJAyz0GVctxPsBQjzijgLPWTWXS/cLoyLZNS7AsqyTe9rzUATDHmBSje5FaJ6Shas2fybiD5V56fVekgen+sKVBWyFAKsxlWV1EytH5WLn0X0H6K50eCA7sNDfNlGs8k8EXmQPmLOEV55nGI4xBxLmAwx/dn9F3t2EhBwGzw1B6Zc4HA/ayWtJcoARO3gNiazTHKZUz37AAoJ2MnLB698L39aYZ/M55zduSLcyUqF+DBHMfzHH3QRsG0kzv+X9 tgunders@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDV/HNj7nVUp+Yh2hAhHgU+R6X9tLAnApUALhPK8oqH/WG6SjXDiLLEoCSisqu+3+WQ3/UcJeoRl5jpyRAjhxnsz/o5tfLXFDJBZ1/njE9T+e73tjfgS6s+BRpCdEAv6/BUVCxz4B5yI7+5mph0JFpv/soe8uhR6MlxAVxAhaLla4S2UPcn6SciyVznIzrTWgfsSHk/o5hIDVfJ68qYVPWqZWmV86hjE/VhPVgmiqFFh2YsJO/FlH36+wtABcKHbovkCjqQ9PkUqfns+82CNjP/XQ6GJZVK5xpYT59ILjlFwA5s9SYkLcjI+Xy2GGNm4ftNqtoF57q33pfDkxQsYJ+D obudai@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAw6IgsAlMJQlOtJXvtlY1racZPntLiy4+iDwrPMCgbYbsylY5TI2S4JCzC3OsnOF/abozKOhTrX04KOSOPkG8iZjBEUsMX4rQXtdViyec8pAdKOimzN9tdlfC2joW8jPlr/wpKMnMRCQmNDUZIOl1ujyTeY592JE8sj9TTqyc+fk= bcl@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDCk/bA7cpp5ynNr+0SGFFjfdJY1M4EUGqbqbcEQ3uU/CX7AhDEHB2pZMAoCPenoIlB0tz3qTvhQqn9jskiT70qU7bDIRbouXit3Jo0YheYZB7+eD/zxAHropUqjA3Bs6JSoBf2z48FumiFfP+eI0UFJ601Zux2fJCTpLuiboEscegWGyzJrl8/b7XCppNusewM1POyErZDlgHh/2gD7rsupMQKxv3rDOexbGMsA3aGgdrPkvg966tCN5Fz7eBM1RQ3o2ywojTgX9I3U/VeZRJ3w1T1IRDAF/DIMh/dA0afsSXn/7PJUx4NiTRstaBbuYYOMs+PQYSE6o2MLdRXZMQA2Dve/dARkg8LO1gvgNEtNUnDcmVPBrKvULDY9viHrGym9Q0Uo6eaxkqQMjzunV5ozUK6pmuYrMK0dODsVDiB4Ja5CQ6pybAOi+i45FqTT8Wc032KbvydWFpnNCvq31vXfVIysqtGASbhWyvry/Qjr04fkcwQhv6Lnph0pAJr5x9ow5rQrQBJZA23hRXDRuUQqpmnFvzK9TWxb1YQSdyQtpuPplbFUfr5JvyCDwHSBzmPZQUESgddmwCmsO6j0KZVmyJkSbXcWW69LMKGmpb1+Fgg2oo9kAW3e89XWyC7VfLIw5Qog08GtlgcpW92w75nSu3wPMi3ww2g78aG5zgLGw== daherrma@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEApQ0yn8aUPRXoJgGZpWXcOjMXjJgmq9KN/tg0iyK9nuR3uRfdiNGTpgKr7DY9HfdP07R09e/uZ8qnLcXUrXYfJj087JNTMHjT1Ifb0KUB9Mzy82RDUFuefds356arhzhW6YS4cOOgY2GmEt2KhftjdHRic/eElZW9I5mA3Oz/uDk= ckellner@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQqwlSYW+lD0qlxrol3JQgcN0QSaT9sm3vmhxX18OnP/3XyxZwgnEiJtIDnoZ2FAWOEKTQ9PP0Ab9gfko9mWl6OP5Nj0wT37ZeWYAJoOA3qLw1PCUF8caHs9NcgTEc2Gd+yKIxObQo2xGZQFLBW7owI4S9Rl1a30+5oLFRZJXx8Yd5+b1xFUU/4oOSkc3birHlwTZrFrW6H+AYT4u9ioSvBKtmj7KCqzbRW7p6Sk40p0EoCYU83ZsevzYXTHTAFRHD/gsRS4N7SOUClz+ZnakVBV7fa0ETLkfSmVplhlklBQjBM7OVLeghjlSRXhAcVV63iJfTMNlTjM47MB/C48L+AlH9KMYmbR1ur44NsNGFKTe08B3Q1YfcSR4no6B5iR+zgb6fpDolklAcW5qbHiOOad3Gd8M7VGpDrHZzQe7QGe6fVMKjVq7xfJFgvOaW44F7RTgL6FnT3bNoi6k26enl1O7WlM0v8j/SwtJlSl3I0HpJXLWP/sgr7U590hZSRgG2U5hoXV9YXUCyLFy4pom82CSkAPT2DybdbnhCf4FAnW2CuZMr7KXuVrHC11LA+S2HLN9Fc9f8SA745+0gO5GTvoSWEt6oAgqPWgK6xj3aeHqWN4WQvMHzdftLh4ZxjHd+c7VxreEMQgZJ124W5nUttn4S0xpJHFMADJ44Kjxg6w== jkozol-1@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMk4+D+GkL/se9RYdQ6mmv0v7cG3gXPiZdLvsbRFk+JMgnLEFUu4uwRN9+huMWdTXYLVcCZVw1l9kmDAOwPKYCGwetOwYI73nYt4n7i9zSQqKknxEO2w+16PvWy6kbgk7zQp9YEqJlIeZbVXiDuFeFyqEH5TxdAfDLu0VG/Db0M1Cor3v7g0/utHWVfS0QiLMQ5xLZ3jAbn/uTjOzsvtA219BTfdx7IVxKmG9/zeLUqyHBcOquI2rUjgHj1FN4yMCob2pSmAfV2u1iQZYHghXlw/kzh4BODjOSWCO9v+621eBdDJ36hdMHWizevxLz9Tj7X1Pwz+4TS/WABXs5MXBFcxx65wpSBr84LfZfWebe+vFe3Kkmdop3FPt7uuZyl8xdRsvfoIG3qZg2izX4IAtXo9tR++yIsEkP/5k/NVwkq158molT0HUruuK9regPRRvf7j6psAH4bMhV3o430eD7ibC5g1yT3kKbGnbJJX+HHL0+lt8IKMPCHWEWOT8uR0x9VUbT4RYFXt53I/ZXHXop+eypEctoKlsgMv1rpPGwSnrhtK1hgf399ph7GDjixZiuzcTP+Ulmj3atB32UHR7mRjarnaCI1w7PJt8pmg9K+8/XUa51q7bap1PGwsJS0lZ3zJEXFPdJfqO8/fkhDb4NYPlX0/bg/LNI9YqPg8CWSw== jkozol-2@redhat.com +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY/ylCrPBzil4TnZR4tWULpz3QgfBMQyEnMOHDAJNp/FK70hD+PUiRm3UY96pmGXonQvqiDoyPuVh025FkWshPK91Dyq8QD8h25q5C5Cg6kMgBpdGzbX44ksms1KyOHmSZ48MpWw3PFOrlNP1vysr6Imjz9Jixmx4sOZvqKnrbsbOW04gowVzpZM8m048lvf6/KhqeImfeSRc9Rtpos8GqEQVlwRevE1qBON963V1QtFOrm9weoQgb369SdqRRdxaGNAymNh3d78DneOWXmEyBflLSpIDx5I2s/1NB1Dp95Bp3VvlV3CH1HC7LAFKYi+xsz3/KHdgtvgShX6LFSdsp rvykydal@dhcp-lab-144.englab.brq.redhat.com diff --git a/schutzbot/terraform b/schutzbot/terraform new file mode 100644 index 0000000..450dfcc --- /dev/null +++ b/schutzbot/terraform @@ -0,0 +1 @@ +7993024714534f70cd412e3181b2ab08082ecd86 diff --git a/schutzbot/update_github_status.sh b/schutzbot/update_github_status.sh new file mode 100755 index 0000000..79d42cd --- /dev/null +++ b/schutzbot/update_github_status.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [[ $1 == "start" ]]; then + GITHUB_NEW_STATE="pending" + GITHUB_NEW_DESC="I'm currently testing this commit, be patient." +elif [[ $1 == "finish" ]]; then + GITHUB_NEW_STATE="success" + GITHUB_NEW_DESC="I like this commit!" +elif [[ $1 == "update" ]]; then + if [[ $CI_JOB_STATUS == "canceled" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="Someone told me to cancel this test run." + elif [[ $CI_JOB_STATUS == "failed" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="I'm sorry, something is odd about this commit." + else + exit 0 + fi +else + echo "unknown command" + exit 1 +fi + +curl \ + -u "${SCHUTZBOT_LOGIN}" \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/osbuild/osbuild/statuses/${CI_COMMIT_SHA}" \ + -d '{"state":"'"${GITHUB_NEW_STATE}"'", "description": "'"${GITHUB_NEW_DESC}"'", "context": "Schutzbot on GitLab", "target_url": "'"${CI_PIPELINE_URL}"'"}' diff --git a/selinux/README.md b/selinux/README.md new file mode 100644 index 0000000..02e23d6 --- /dev/null +++ b/selinux/README.md @@ -0,0 +1,65 @@ +# SELiunx and osbuild + +SELinux labels for files are store as extended attributes under the +`security.selinux` prefix. + +## File system tree labelling +All stages, including the `org.osbuild.rpm` stage are run inside a +container which will indicate to all tools, including rpm scriptles +that SELinux is disabled. + +Labels are manually applied to the file system tree via a specialised +`org.osbuild.selinux` stage. This stage should therefore be at the +very end of the pipeline that is building the tree so that all files +are properly labelled. + +## Container peculiarities and policy differences + +SELinux is not namespaced which means there is only one global +policy inside the Linux kernel. Since the kernel is shared by all +containers, the policy that is loaded in the kernel applies to all +containers as well. + +Labels are verified against the active policy in the kernel when +writing (`setxattr`) but also when reading them (`getxattr`) as +long as selinux is activated for the kernel (i.e. on the host). + +To read or write labels that are not included in the currently +active policy, the `CAP_MAC_ADMIN` capability(7) is needed. If +a process does not have this policy the following will happen +when trying to write or read the label: + +When trying to write a label that is unknown to the currently +active policy, the kernel will reject it and the call to +`setxattr` will fail with `EINVAL` resulting in "Invalid argument" +errors from the corresponding tooling. + +When trying to read a label that is unknown to the currently +active policy, the kernel will "pretend" the file is not labelled and +return `system_u:object_r:unlabeled_t:s0` as label. Thus a file with +an unknown label (unknown to the host kernel) is indistinguishable +from an unlabelled file. + +In RHEL and Fedora's SELinux policy, only very few programs can +gain or retain the`CAP_MAC_ADMIN` capability, even if the current +user is `unconfined` or `sysadm`. Normal tools like `cp`, `ls`, +`stat`, or `tar` do *not* have this capability meaning that +inspecting the labels for files and folders will result in +`unlabeled_t` for unknown (to the host) labels. + +### Custom OSBuild SElinux Policy + +On RHEL and Fedora, the SELinux policy has a few contexts that +allow `CAP_MAC_ADMIN`, most notably `install_t` and `setfiles_mac`. +The latter is a policy for the `setfiles` binary, which is used +by the`org.osbuild.selinux` stage to label files. But to be able +to transition into `setfiles_mac`, the calling program must have a +special transition rule allowing this. Therefore osbuild uses a +custom policy with specialised labels for osbuild executables such +as stages, runners and the main binary: `osbuild_t`. Then a domain +transition rule is enabled that allows `setfiles` to transition to +`setfiles_mac` from `osbuild`. From `selinux/osbuild.te`: + + # execute setfiles in the setfiles_mac domain + # when in the osbuild_t domain + seutil_domtrans_setfiles_mac(osbuild_t) diff --git a/selinux/osbuild.fc b/selinux/osbuild.fc new file mode 100644 index 0000000..d9cb686 --- /dev/null +++ b/selinux/osbuild.fc @@ -0,0 +1,4 @@ +/usr/bin/osbuild -- gen_context(system_u:object_r:osbuild_exec_t,s0) +/usr/lib/osbuild/assemblers/.* -- gen_context(system_u:object_r:osbuild_exec_t,s0) +/usr/lib/osbuild/stages/.* -- gen_context(system_u:object_r:osbuild_exec_t,s0) +/usr/lib/osbuild/sources/.* -- gen_context(system_u:object_r:osbuild_exec_t,s0) diff --git a/selinux/osbuild.if b/selinux/osbuild.if new file mode 100644 index 0000000..48d099f --- /dev/null +++ b/selinux/osbuild.if @@ -0,0 +1,114 @@ + +## policy for osbuild + +######################################## +## +## Execute osbuild_exec_t in the osbuild domain. +## +## +## +## Domain allowed to transition. +## +## +# +interface(`osbuild_domtrans',` + gen_require(` + type osbuild_t, osbuild_exec_t; + ') + + corecmd_search_bin($1) + domtrans_pattern($1, osbuild_exec_t, osbuild_t) +') + +###################################### +## +## Execute osbuild in the caller domain. +## +## +## +## Domain allowed access. +## +## +# +interface(`osbuild_exec',` + gen_require(` + type osbuild_exec_t; + ') + + corecmd_search_bin($1) + can_exec($1, osbuild_exec_t) +') + +######################################## +## +## Execute osbuild in the osbuild domain, and +## allow the specified role the osbuild domain. +## +## +## +## Domain allowed to transition +## +## +## +## +## The role to be allowed the osbuild domain. +## +## +# +interface(`osbuild_run',` + gen_require(` + type osbuild_t; + attribute_role osbuild_roles; + ') + + osbuild_domtrans($1) + roleattribute $2 osbuild_roles; +') + +######################################## +## +## Role access for osbuild +## +## +## +## Role allowed access +## +## +## +## +## User domain for the role +## +## +# +interface(`osbuild_role',` + gen_require(` + type osbuild_t; + attribute_role osbuild_roles; + ') + + roleattribute $1 osbuild_roles; + + osbuild_domtrans($2) + + ps_process_pattern($2, osbuild_t) + allow $2 osbuild_t:process { signull signal sigkill }; +') + +######################################## +## +## osbuild nnp / nosuid transitions to domain +## +## +## +## Domain to be allowed to transition into. +## +## +# +interface(`osbuild_nnp_nosuid_trans',` + gen_require(` + type osbuild_t; + class process2 { nnp_transition nosuid_transition }; + ') + + allow osbuild_t $1:process2 {nnp_transition nosuid_transition}; +') diff --git a/selinux/osbuild.te b/selinux/osbuild.te new file mode 100644 index 0000000..e4a0c7d --- /dev/null +++ b/selinux/osbuild.te @@ -0,0 +1,68 @@ +policy_module(osbuild, 1.0.0) + +######################################## +# +# Declarations +# + +attribute_role osbuild_roles; +roleattribute system_r osbuild_roles; + +type osbuild_t; +type osbuild_exec_t; +application_domain(osbuild_t, osbuild_exec_t) +role osbuild_roles types osbuild_t; + +######################################## +# +# osbuild local policy +# + +allow osbuild_t self:fifo_file manage_fifo_file_perms; +allow osbuild_t self:unix_stream_socket create_stream_socket_perms; + +# ##################################### +# Customization +# + +# make an osbuild_t unconfined domain +unconfined_domain(osbuild_t) + +# execute setfiles in the setfiles_mac domain +# when in the osbuild_t domain +seutil_domtrans_setfiles_mac(osbuild_t) +osbuild_nnp_nosuid_trans(setfiles_mac_t) + +# Allow sysadm and unconfined to run osbuild +optional_policy(` + gen_require(` + type sysadm_t; + role sysadm_r; + ') + + osbuild_run(sysadm_t, sysadm_r) +') + +optional_policy(` + gen_require(` + type unconfined_t; + role unconfined_r; + ') + + osbuild_run(unconfined_t, unconfined_r) +') + +optional_policy(` + gen_require(` + type unconfined_service_t; + role system_r; + ') + + osbuild_run(unconfined_service_t, system_r) +') + +# allow transitioning to install_t (for ostree) +optional_policy(` + anaconda_domtrans_install(osbuild_t) + osbuild_nnp_nosuid_trans(install_t) +') diff --git a/selinux/osbuild_selinux.8 b/selinux/osbuild_selinux.8 new file mode 100644 index 0000000..3c727a0 --- /dev/null +++ b/selinux/osbuild_selinux.8 @@ -0,0 +1,147 @@ +.TH "osbuild_selinux" "8" "20-06-09" "osbuild" "SELinux Policy osbuild" +.SH "NAME" +osbuild_selinux \- Security Enhanced Linux Policy for the osbuild processes +.SH "DESCRIPTION" + +Security-Enhanced Linux secures the osbuild processes via flexible mandatory access control. + +The osbuild processes execute with the osbuild_t SELinux type. You can check if you have these processes running by executing the \fBps\fP command with the \fB\-Z\fP qualifier. + +For example: + +.B ps -eZ | grep osbuild_t + + +.SH "ENTRYPOINTS" + +The osbuild_t SELinux type can be entered via the \fBosbuild_exec_t\fP file type. + +The default entrypoint paths for the osbuild_t domain are the following: + +/usr/lib/osbuild/stages/*, /usr/lib/osbuild/sources/*, /usr/lib/osbuild/assemblers/*, /usr/bin/osbuild +.SH PROCESS TYPES +SELinux defines process types (domains) for each process running on the system +.PP +You can see the context of a process using the \fB\-Z\fP option to \fBps\bP +.PP +Policy governs the access confined processes have to files. +SELinux osbuild policy is very flexible allowing users to setup their osbuild processes in as secure a method as possible. +.PP +The following process types are defined for osbuild: + +.EX +.B osbuild_t +.EE +.PP +Note: +.B semanage permissive -a osbuild_t +can be used to make the process type osbuild_t permissive. SELinux does not deny access to permissive process types, but the AVC (SELinux denials) messages are still generated. + +.SH BOOLEANS +SELinux policy is customizable based on least access required. osbuild policy is extremely flexible and has several booleans that allow you to manipulate the policy and run osbuild with the tightest access possible. + + +.PP +If you want to deny user domains applications to map a memory region as both executable and writable, this is dangerous and the executable should be reported in bugzilla, you must turn on the deny_execmem boolean. Disabled by default. + +.EX +.B setsebool -P deny_execmem 1 + +.EE + +.PP +If you want to control the ability to mmap a low area of the address space, as configured by /proc/sys/vm/mmap_min_addr, you must turn on the mmap_low_allowed boolean. Disabled by default. + +.EX +.B setsebool -P mmap_low_allowed 1 + +.EE + +.PP +If you want to disable kernel module loading, you must turn on the secure_mode_insmod boolean. Disabled by default. + +.EX +.B setsebool -P secure_mode_insmod 1 + +.EE + +.PP +If you want to allow unconfined executables to make their heap memory executable. Doing this is a really bad idea. Probably indicates a badly coded executable, but could indicate an attack. This executable should be reported in bugzilla, you must turn on the selinuxuser_execheap boolean. Disabled by default. + +.EX +.B setsebool -P selinuxuser_execheap 1 + +.EE + +.PP +If you want to allow unconfined executables to make their stack executable. This should never, ever be necessary. Probably indicates a badly coded executable, but could indicate an attack. This executable should be reported in bugzilla, you must turn on the selinuxuser_execstack boolean. Enabled by default. + +.EX +.B setsebool -P selinuxuser_execstack 1 + +.EE + +.SH "MANAGED FILES" + +The SELinux process type osbuild_t can manage files labeled with the following file types. The paths listed are the default paths for these file types. Note the processes UID still need to have DAC permissions. + +.br +.B file_type + + all files on the system +.br + +.SH FILE CONTEXTS +SELinux requires files to have an extended attribute to define the file type. +.PP +You can see the context of a file using the \fB\-Z\fP option to \fBls\bP +.PP +Policy governs the access confined processes have to these files. +SELinux osbuild policy is very flexible allowing users to setup their osbuild processes in as secure a method as possible. +.PP + +.I The following file types are defined for osbuild: + + +.EX +.PP +.B osbuild_exec_t +.EE + +- Set files with the osbuild_exec_t type, if you want to transition an executable to the osbuild_t domain. + +.br +.TP 5 +Paths: +/usr/lib/osbuild/stages/*, /usr/lib/osbuild/sources/*, /usr/lib/osbuild/assemblers/*, /usr/bin/osbuild + +.PP +Note: File context can be temporarily modified with the chcon command. If you want to permanently change the file context you need to use the +.B semanage fcontext +command. This will modify the SELinux labeling database. You will need to use +.B restorecon +to apply the labels. + +.SH "COMMANDS" +.B semanage fcontext +can also be used to manipulate default file context mappings. +.PP +.B semanage permissive +can also be used to manipulate whether or not a process type is permissive. +.PP +.B semanage module +can also be used to enable/disable/install/remove policy modules. + +.B semanage boolean +can also be used to manipulate the booleans + +.PP +.B system-config-selinux +is a GUI tool available to customize SELinux policy settings. + +.SH AUTHOR +This manual page was auto-generated using +.B "sepolicy manpage". + +.SH "SEE ALSO" +selinux(8), osbuild(8), semanage(8), restorecon(8), chcon(1), sepolicy(8), setsebool(8) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9e7774b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[pylint.MASTER] +disable=missing-docstring,too-few-public-methods,invalid-name,duplicate-code,superfluous-parens,too-many-locals,attribute-defined-outside-init,too-many-arguments,consider-using-with,consider-using-from-import + +[pylint.TYPECHECK] +ignored-classes=osbuild.loop.LoopInfo + +[pylint.DESIGN] +max-attributes=10 +max-statements=75 + +[pycodestyle] +max-line-length = 120 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7a17bd0 --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +import setuptools + +setuptools.setup( + name="osbuild", + version="54", + description="A build system for OS images", + packages=["osbuild", "osbuild.formats", "osbuild.util"], + license='Apache-2.0', + install_requires=[ + "jsonschema", + ], + entry_points={ + "console_scripts": [ + "osbuild = osbuild.main_cli:osbuild_cli" + ] + }, + scripts=[ + "tools/osbuild-mpp" + ], +) diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl new file mode 100755 index 0000000..d46cd6a --- /dev/null +++ b/sources/org.osbuild.curl @@ -0,0 +1,181 @@ +#!/usr/bin/python3 +""" +Source for downloading files from URLs. + +The files are indexed by their content hash. Can download files +that require secrets. The only secret provider currently supported +is `org.osbuild.rhsm` for downloading Red Hat content that requires +a subscriptions. + +Internally use curl to download the files; the files are cached in +an internal cache. Multiple parallel connections are used to speed +up the download. +""" + + +import concurrent.futures +import itertools +import os +import subprocess +import sys +import tempfile + +from osbuild import sources + +from osbuild.util.checksum import verify_file +from osbuild.util.rhsm import Subscriptions + + +SCHEMA = """ +"additionalProperties": false, +"definitions": { + "item": { + "description": "The files to fetch indexed their content checksum", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": { + "oneOf": [ + { + "type": "string", + "description": "URL to download the file from." + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "url" + ], + "properties": { + "url": { + "type": "string", + "description": "URL to download the file from." + }, + "secrets": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the secrets provider." + } + } + } + } + } + ] + } + } + } +}, +"properties": { + "items": {"$ref": "#/definitions/item"}, + "urls": {"$ref": "#/definitions/item"} +}, +"oneOf": [{ + "required": ["items"] +}, { + "required": ["urls"] +}] +""" + + +def fetch(url, checksum, directory): + secrets = url.get("secrets") + url_path = url.get("url") + # Download to a temporary directory until we have verified the checksum. Use a + # subdirectory, so we avoid copying across block devices. + with tempfile.TemporaryDirectory(prefix="osbuild-unverified-file-", dir=directory) as tmpdir: + # some mirrors are sometimes broken. retry manually, because we could be + # redirected to a different, working, one on retry. + return_code = 0 + for _ in range(10): + curl_command = [ + "curl", + "--silent", + "--speed-limit", "1000", + "--connect-timeout", "30", + "--fail", + "--location", + "--output", checksum, + ] + if secrets: + if secrets.get('ssl_ca_cert'): + curl_command.extend(["--cacert", secrets.get('ssl_ca_cert')]) + if secrets.get('ssl_client_cert'): + curl_command.extend(["--cert", secrets.get('ssl_client_cert')]) + if secrets.get('ssl_client_key'): + curl_command.extend(["--key", secrets.get('ssl_client_key')]) + # url must follow options + curl_command.append(url_path) + + curl = subprocess.run(curl_command, encoding="utf-8", cwd=tmpdir, check=False) + return_code = curl.returncode + if return_code == 0: + break + else: + raise RuntimeError(f"curl: error downloading {url}: error code {return_code}") + + if not verify_file(f"{tmpdir}/{checksum}", checksum): + raise RuntimeError(f"checksum mismatch: {checksum} {url}") + + # The checksum has been verified, move the file into place. in case we race + # another download of the same file, we simply ignore the error as their + # contents are guaranteed to be the same. + try: + os.rename(f"{tmpdir}/{checksum}", f"{directory}/{checksum}") + except FileExistsError: + pass + + +def download(items, cache): + with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: + requested_urls = [] + requested_checksums = [] + subscriptions = None + + for (checksum, url) in items.items(): + + # Invariant: all files in @directory must be named after their (verified) checksum. + # Check this before secrets so that if everything is pre-downloaded we don't need secrets + if os.path.isfile(f"{cache}/{checksum}"): + continue + + if not isinstance(url, dict): + url = {"url": url} + + # check if url needs rhsm secrets + if url.get("secrets", {}).get("name") == "org.osbuild.rhsm": + # rhsm secrets only need to be retrieved once and can then be reused + if subscriptions is None: + subscriptions = Subscriptions.from_host_system() + url["secrets"] = subscriptions.get_secrets(url.get("url")) + + requested_urls.append(url) + requested_checksums.append(checksum) + + results = executor.map(fetch, requested_urls, requested_checksums, itertools.repeat(cache)) + + for _ in results: + pass + + +class CurlSource(sources.SourceService): + + def download(self, items, cache, _options): + cache = os.path.join(cache, "org.osbuild.files") + os.makedirs(cache, exist_ok=True) + + download(items, cache) + + +def main(): + service = CurlSource.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/sources/org.osbuild.inline b/sources/org.osbuild.inline new file mode 100755 index 0000000..7420679 --- /dev/null +++ b/sources/org.osbuild.inline @@ -0,0 +1,98 @@ +#!/usr/bin/python3 +"""Source for binary data encoded inline in the manifest + +This source can be used to transport data in the source +section of the manifest. Each resource is ascii-encoded +in the `data` property, where the encoding is specified +in the `encoding` property. The resources is content +addressed via the hash value of the raw data before the +ascii encoding. This hash value is verified after the +resource is decoded and written to the store. +""" + + +import base64 +import contextlib +import os +import sys +import tempfile + +from typing import Dict + +from osbuild import sources +from osbuild.util.checksum import verify_file + + +SCHEMA = """ +"definitions": { + "item": { + "description": "Inline data indexed by their checksum", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{32,128}": { + "type": "object", + "additionalProperties": false, + "required": ["encoding", "data"], + "properties": { + "encoding": { + "description": "The specific encoding of `data`", + "enum": ["base64"] + }, + "data": { + "description": "The ascii encoded raw data", + "type": "string" + } + } + } + } + } +}, +"additionalProperties": false, +"required": ["items"], +"properties": { + "items": {"$ref": "#/definitions/item"} +} +""" + + +def process(items: Dict, cache: str, tmpdir): + for checksum, item in items.items(): + target = os.path.join(cache, checksum) + floating = os.path.join(tmpdir, checksum) + + if os.path.isfile(target): + return + + data = base64.b64decode(item["data"]) + + # Write the bits to disk and then verify the checksum + # This ensures that 1) the data is ok and that 2) we + # wrote them correctly as well + with open(floating, "wb") as f: + f.write(data) + + if not verify_file(floating, checksum): + raise RuntimeError("Checksum mismatch for {}".format(checksum)) + + with contextlib.suppress(FileExistsError): + os.rename(floating, target) + + +class InlineSource(sources.SourceService): + + def download(self, items, cache, _options): + cache = os.path.join(cache, "org.osbuild.files") + os.makedirs(cache, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix=".unverified-", dir=cache) as tmpdir: + process(items, cache, tmpdir) + + +def main(): + service = InlineSource.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/sources/org.osbuild.ostree b/sources/org.osbuild.ostree new file mode 100755 index 0000000..07ec493 --- /dev/null +++ b/sources/org.osbuild.ostree @@ -0,0 +1,130 @@ +#!/usr/bin/python3 +"""Fetch OSTree commits from an repository + +Uses ostree to pull specific commits from (remote) repositories +at the provided `url`. Can verify the commit, if one or more +gpg keys are provided via `gpgkeys`. +""" + + +import os +import sys +import subprocess +import uuid + +from osbuild import sources + + +SCHEMA = """ +"additionalProperties": false, +"definitions": { + "item": { + "description": "The commits to fetch indexed their checksum", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "[0-9a-f]{5,64}": { + "type": "object", + "additionalProperties": false, + "required": ["remote"], + "properties": { + "remote": { + "type": "object", + "additionalProperties": false, + "required": ["url"], + "properties": { + "url": { + "type": "string", + "description": "URL of the repository." + }, + "gpgkeys": { + "type": "array", + "items": { + "type": "string", + "description": "GPG keys to verify the commits" + } + } + } + } + } + } + } + } +}, +"properties": { + "items": {"$ref": "#/definitions/item"}, + "commits": {"$ref": "#/definitions/item"} +}, +"oneOf": [{ + "required": ["items"] +}, { + "required": ["commits"] +}] +""" + + +def ostree(*args, _input=None, **kwargs): + args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + print("ostree " + " ".join(args), file=sys.stderr) + subprocess.run(["ostree"] + args, + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + input=_input, + check=True) + + +def download(items, cache): + # Prepare the cache and the output repo + repo_cache = os.path.join(cache, "repo") + ostree("init", mode="archive", repo=repo_cache) + + # Make sure the cache repository uses locks to protect the metadata during + # shared access. This is the default since `2018.5`, but lets document this + # explicitly here. + ostree("config", "set", "repo.locking", "true", repo=repo_cache) + + for commit, item in items.items(): + remote = item["remote"] + url = remote["url"] + gpg = remote.get("gpgkeys", []) + uid = str(uuid.uuid4()) + + verify_args = [] + if not gpg: + verify_args = ["--no-gpg-verify"] + + ostree("remote", "add", + uid, url, + *verify_args, + repo=repo_cache) + + for key in gpg: + ostree("remote", "gpg-import", "--stdin", uid, + repo=repo_cache, _input=key) + + # Transfer the commit: remote → cache + print(f"pulling {commit}", file=sys.stderr) + ostree("pull", uid, commit, repo=repo_cache) + + # Remove the temporary remotes again + ostree("remote", "delete", uid, + repo=repo_cache) + + +class OSTreeSource(sources.SourceService): + + def download(self, items, cache, _options): + cache = os.path.join(cache, "org.osbuild.ostree") + os.makedirs(cache, exist_ok=True) + + download(items, cache) + + +def main(): + service = OSTreeSource.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/sources/org.osbuild.skopeo b/sources/org.osbuild.skopeo new file mode 100755 index 0000000..c9f7b23 --- /dev/null +++ b/sources/org.osbuild.skopeo @@ -0,0 +1,134 @@ +#!/usr/bin/python3 +"""Fetch container image from a registry using skopeo + +Buildhost commands used: `skopeo`. +""" + +import errno +import os +import sys +import subprocess +import tempfile +import hashlib + +from osbuild import sources +from osbuild.util import ctx + +SCHEMA = """ +"additionalProperties": false, +"definitions": { + "item": { + "description": "The container image to fetch indexed by the container image id", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "sha256:[0-9a-f]{64}": { + "type": "object", + "additionalProperties": false, + "required": ["image"], + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "required": ["name", "digest"], + "properties": { + "name": { + "type": "string", + "description": "Name of the image (including registry)." + }, + "digest": { + "type": "string", + "description": "Digest of image in registry.", + "pattern": "sha256:[0-9a-f]{64}" + }, + "tls-verify": { + "type": "boolean", + "description": "Require https (default true)." + } + } + } + } + } + } + } +}, +"properties": { + "items": {"$ref": "#/definitions/item"}, + "digests": {"$ref": "#/definitions/item"} +}, +"oneOf": [{ + "required": ["items"] +}, { + "required": ["digests"] +}] +""" + + +def download(items, cache): + for image_id, item in items.items(): + image = item["image"] + imagename = image["name"] + digest = image["digest"] + tls_verify = image.get("tls-verify", True) + + path = f"{cache}/{image_id}" + if os.path.isdir(path): + continue + + with tempfile.TemporaryDirectory(prefix="tmp-download-", dir=cache) as tmpdir: + archive_path = os.path.join(tmpdir, "container-image.tar") + + source = f"docker://{imagename}@{digest}" + + # We use the docker format, not oci, because that is the + # default return image type of real world registries, + # allowing the image to get the same image id as if you + # did "podman pull" (rather than converting the image to + # oci format, changing the id) + destination = f"docker-archive:{archive_path}" + + extra_args = [] + + # The archive format can't store signatures, but we still verify them during download + extra_args.append("--remove-signatures") + + if not tls_verify: + extra_args.append("--src-tls-verify=false") + + subprocess.run(["skopeo", "copy"] + extra_args + [source, destination], + encoding="utf-8", + check=True) + + # Verify that the digest supplied downloaded the correct container image id. + # The image id is the digest of the config, but skopeo can' currently + # get the config id, only the full config, so we checksum it ourselves. + res = subprocess.run(["skopeo", "inspect", "--raw", "--config", destination], + capture_output=True, + check=True) + downloaded_id = "sha256:" + hashlib.sha256(res.stdout).hexdigest() + if downloaded_id != image_id: + raise RuntimeError( + f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}") + + # Atomically move download dir into place on successful download + os.chmod(tmpdir, 0o755) + with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST): + os.rename(tmpdir, path) + + +class SkopeoSource(sources.SourceService): + + def download(self, items, cache, _options): + cache = os.path.join(cache, "org.osbuild.containers") + os.makedirs(cache, exist_ok=True) + + download(items, cache) + + +def main(): + service = SkopeoSource.from_args(sys.argv[1:]) + service.main() + + +if __name__ == '__main__': + main() diff --git a/stages/org.osbuild.anaconda b/stages/org.osbuild.anaconda new file mode 100755 index 0000000..7ccebfe --- /dev/null +++ b/stages/org.osbuild.anaconda @@ -0,0 +1,58 @@ +#!/usr/bin/python3 +""" +Configure basic aspects of the anaconda installer + +Create an anaconda configuration file `90-osbuild.conf` in +the folder `/etc/anaconda/conf.d` to configure anaconda. + +Currently only the list of enabled kickstart modules is +configurable via the `kickstart-modules` option. +""" + +import os +import sys + + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": true, +"required": ["kickstart-modules"], +"properties": { + "kickstart-modules": { + "type": "array", + "description": "Kick start modules to enable", + "items": { + "type": "string" + }, + "minItems": 1 + } +} +""" + +CONFIG = """ +# osbuild customizations + +[Anaconda] +# List of enabled Anaconda DBus modules +kickstart_modules = +""" + + +def main(tree, options): + modules = options["kickstart-modules"] + product_dir = os.path.join(tree, "etc/anaconda/conf.d") + os.makedirs(product_dir, exist_ok=True) + + with open(os.path.join(product_dir, "90-osbuild.conf"), "w") as f: + f.write(CONFIG) + for m in modules: + f.write(f" {m}\n") + + +if __name__ == '__main__': + stage_args = osbuild.api.arguments() + r = main(stage_args["tree"], + stage_args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.authconfig b/stages/org.osbuild.authconfig new file mode 100755 index 0000000..e957d98 --- /dev/null +++ b/stages/org.osbuild.authconfig @@ -0,0 +1,42 @@ +#!/usr/bin/python3 +""" +Configure authentication sources using authconfig. + +Applies the default settings. Backups are cleared. + +Notes: + - Requires 'chroot' in the buildroot. + - Runs the 'authconfig' binary from the image in the chroot. +""" + + +import shutil +import subprocess +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"description": "Configure authentication sources." +""" + + +def main(tree): + cmd = [ + "/usr/sbin/chroot", tree, + "/usr/sbin/authconfig", "--nostart", "--updateall" + ] + + subprocess.run(cmd, check=True) + + shutil.rmtree(f"{tree}/var/lib/authselect/backups", ignore_errors=True) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"]) + sys.exit(r) diff --git a/stages/org.osbuild.authselect b/stages/org.osbuild.authselect new file mode 100755 index 0000000..ef46471 --- /dev/null +++ b/stages/org.osbuild.authselect @@ -0,0 +1,64 @@ +#!/usr/bin/python3 +""" +Select system identity and authentication sources with authselect. + +Sets system identity and authentication sources. + +The stage calls `authselect select` to set authselect profile to 'profile'. +Optionally a list of profile features to enable may be provided using 'features' +option. The list of available profile features can be obtained by running +`authselect list-features `. + +Notes: + - Requires 'chroot' in the buildroot. + - Runs the 'authselect' binary from the image in the chroot. +""" + + +import subprocess +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"required": ["profile"], +"description": "Select system identity and authentication sources.", +"properties": { + "profile": { + "type": "string", + "description": "Desired authselect profile to activate." + }, + "features": { + "type": "array", + "description": "Features of the selected profile to activate.", + "minItems": 1, + "items": { + "type": "string" + } + } +} +""" + + +def main(tree, options): + profile = options["profile"] + features = options.get("features", []) + + cmd = [ + "/usr/sbin/chroot", tree, + # force authselect to overwrite existing files without making a backup + "/usr/bin/authselect", "select", "--force", "--nobackup", profile + ] + cmd.extend(features) + + subprocess.run(cmd, check=True) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.bootiso.mono b/stages/org.osbuild.bootiso.mono new file mode 100755 index 0000000..f1fdd39 --- /dev/null +++ b/stages/org.osbuild.bootiso.mono @@ -0,0 +1,440 @@ +#!/usr/bin/python3 +""" +Assemble a file system tree for a bootable iso + +This stage prepares a file system tree for a bootable ISO, like the +Anaconda installer. It follows the convention used by Lorax to +create the boot isos: It takes an input `rootfs`, which will serve +as the root file system. This is copied into a file with a `ext4` +file system which in turn will be made into a squashfs file system. +Options for controlling the root file-system creation can be given +via `rootfs`, like it size and the compression to be used. + +The boot loader is configured via the `isolinux` and `efi` options. +Which combination makes sense depends on the targeted platform and +architecture. +The kernel and initrd are taken from the tree given via the `kernel` +input, or if that was not specified, from `rootfs`. In either case +it will look for the specified kernel in the `/boot` directory. +Additionally kernel command line flags can passed via `kernel_opts`. + +This stage has the `.mono` suffix to indicate that is a monolithic +stage that could, and in the future will be, broken up into smaller +pieces. +""" + +import contextlib +import os +import re +import shutil +import subprocess +import sys +import tempfile +import osbuild.remoteloop as remoteloop + +import osbuild.api + + +SCHEMA_2 = """ +"options": { + "additionalProperties": false, + "required": ["product", "kernel", "isolabel"], + "properties": { + "product": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version"], + "properties": { + "name": {"type": "string"}, + "version": {"type": "string"} + } + }, + "kernel": { + "type": "string" + }, + "isolabel": { + "type": "string" + }, + "efi": { + "type": "object", + "additionalProperties": false, + "required": ["architectures", "vendor"], + "properties": { + "architectures": { + "type": "array", + "items": { + "type": "string" + } + }, + "vendor": { + "type": "string" + } + } + }, + "isolinux": { + "type": "object", + "additionalProperties": false, + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean" + }, + "debug": { + "type": "boolean" + } + } + }, + "kernel_opts": { + "description": "Additional kernel boot options", + "type": "string" + }, + "templates": { + "type": "string", + "default": "99-generic" + }, + "rootfs": { + "type": "object", + "additionalProperties": false, + "properties": { + "compression": { + "type": "object", + "additionalProperties": false, + "required": ["method"], + "properties": { + "method": { + "enum": ["gzip", "xz", "lz4"] + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "bcj": { + "enum": [ + "x86", + "arm", + "armthumb", + "powerpc", + "sparc", + "ia64" + ] + } + } + } + } + }, + "size": { + "type": "integer", + "description": "size in MB", + "default": 3072 + } + } + } + } +}, +"inputs": { + "type": "object", + "additionalProperties": false, + "required": ["rootfs"], + "properties": { + "rootfs": { + "type": "object", + "additionalProperties": true + }, + "kernel": { + "type": "object", + "additionalProperties": true + } + } +} +""" + + +LORAX_TEMPLATES = "/usr/share/lorax/templates.d" + + +@contextlib.contextmanager +def mount(source, dest): + subprocess.run(["mount", source, dest], check=True) + try: + yield dest + finally: + subprocess.run(["umount", "-R", dest], check=True) + + +def install(src, dst, mode=None): + shutil.copyfile(src, dst) + if mode: + os.chmod(dst, mode) + + +def replace(target, patterns): + finder = [(re.compile(p), s) for p, s in patterns] + newfile = target + ".replace" + + with open(target, "r") as i, open(newfile, "w") as o: + for line in i: + for p, s in finder: + line = p.sub(s, line) + o.write(line) + os.rename(newfile, target) + + +def make_rootfs(tree, image, size, workdir, loop_client): + with open(image, "w") as f: + os.ftruncate(f.fileno(), size) + + root = os.path.join(workdir, "rootfs") + os.makedirs(root) + + with loop_client.device(image, 0, size) as dev: + subprocess.run(["mkfs.ext4", + "-L", "Anaconda", + "-b", "4096", + "-m", "0", + dev], + input="y", encoding='utf-8', check=True) + + with mount(dev, root): + print("copying tree") + subprocess.run(["cp", "-a", f"{tree}/.", root], + check=True) + print("done") + + +def make_efi(efi, info, root, loop_client): + arches = efi["architectures"] + vendor = efi["vendor"] + + efidir = os.path.join(root, "EFI", "BOOT") + os.makedirs(efidir) + + #arch related data + for arch in arches: + arch = arch.lower() + targets = [ + (f"shim{arch}.efi", f"BOOT{arch}.EFI".upper()), + (f"mm{arch}.efi", f"mm{arch}.efi"), + (f"gcd{arch}.efi", f"grub{arch}.efi") + ] + + for src, dst in targets: + shutil.copy2(os.path.join("/boot/efi/EFI/", vendor, src), + os.path.join(efidir, dst)) + + # the font + fontdir = os.path.join(efidir, "fonts") + os.makedirs(fontdir, exist_ok=True) + shutil.copy2("/usr/share/grub/unicode.pf2", fontdir) + + # the config + configdir = info["configdir"] + version = info["version"] + name = info["name"] + isolabel = info["isolabel"] + cmdline = info["cmdline"] + + kdir = "/" + os.path.relpath(info["kerneldir"], start=root) + print(f"kernel dir at {kdir}") + + config = os.path.join(efidir, "grub.cfg") + shutil.copy2(os.path.join(configdir, "grub2-efi.cfg"), config) + + replace(config, [ + ("@VERSION@", version), + ("@PRODUCT@", name), + ("@KERNELNAME@", "vmlinuz"), + ("@KERNELPATH@", os.path.join(kdir, "vmlinuz")), + ("@INITRDPATH@", os.path.join(kdir, "initrd.img")), + ("@ISOLABEL@", isolabel), + ("@ROOT@", cmdline) + ]) + + if "IA32" in arches: + shutil.copy2(config, os.path.join(efidir, "BOOT.cfg")) + + # estimate the size + blocksize = 2048 + size = blocksize * 256 # blocksize * overhead + for parent, dirs, files in os.walk(efidir): + for name in files + dirs: + t = os.path.join(parent, name) + s = os.stat(t).st_size + d = s % blocksize + if not s or d: + s += blocksize - d + size += s + print(f"Estimates efiboot size to be {size}") + + # create the image + image = os.path.join(info["imgdir"], "efiboot.img") + with open(image, "w") as f: + os.ftruncate(f.fileno(), size) + + root = os.path.join(info["workdir"], "mnt") + os.makedirs(root) + + with loop_client.device(image, 0, size) as dev: + subprocess.run(["mkfs.fat", + "-n", "ANACONDA", + dev], + input="y", encoding='utf-8', check=True) + + with mount(dev, root): + target = os.path.join(root, "EFI", "BOOT") + shutil.copytree(efidir, target) + subprocess.run(["ls", root], check=True) + + +def make_isolinux(cfg, root, info, tree): + # the config + configdir = info["configdir"] + version = info["version"] + name = info["name"] + cmdline = info["cmdline"] + kerneldir = info["kerneldir"] + + # boot loader + isolinux = os.path.join(root, "isolinux") + os.makedirs(isolinux) + + isolinuxfiles = [("isolinux.bin", 0o755), + ("ldlinux.c32", 0o755), + ("libcom32.c32", 0o755), + ("libutil.c32", 0o755), + ("vesamenu.c32", 0o755)] + for target, mode in isolinuxfiles: + src = os.path.join("/usr/share/syslinux/", target) + dst = os.path.join(isolinux, target) + install(src, dst, mode) + + if cfg.get("debug"): + src = "/usr/share/syslinux/isolinux-debug.bin" + dst = os.path.join(isolinux, "isolinux.bin") + install(src, dst, 0o755) + + for target in ["isolinux.cfg", "boot.msg", "grub.conf"]: + src = os.path.join(configdir, target) + dst = os.path.join(isolinux, target) + install(src, dst) + + replace(dst, [ + ("@VERSION@", version), + ("@PRODUCT@", name), + ("@ROOT@", cmdline) + ]) + + src = os.path.join(tree, "usr/share/anaconda/boot/syslinux-splash.png") + dst = os.path.join(isolinux, "splash.png") + install(src, dst) + + # link the kernel + os.link(os.path.join(kerneldir, "vmlinuz"), + os.path.join(isolinux, "vmlinuz")) + os.link(os.path.join(kerneldir, "initrd.img"), + os.path.join(isolinux, "initrd.img")) + + +# pylint: disable=too-many-statements +def main(inputs, root, options, workdir, loop_client): + tree = inputs["rootfs"]["path"] + name = options["product"]["name"] + version = options["product"]["version"] + kernel = options["kernel"] + isolabel = options["isolabel"] + templates = options["templates"] + efi = options.get("efi") + isolinux = options.get("isolinux", {}) + kopts = options.get("kernel_opts") + rootfs = options.get("rootfs", {}) + + # input directories + templatedir = os.path.join(LORAX_TEMPLATES, templates) + + # select the template based on the architecture, where + # we reuse the efi setting, since we only support efi + # on aarch64 this is good enough for now + if efi and "AA64" in efi["architectures"]: + arch = "aarch64" + else: + arch = "x86" + + configdir = os.path.join(templatedir, "config_files", arch) + + # output directories + imgdir = os.path.join(root, "images") + pxedir = os.path.join(imgdir, "pxeboot") + + os.makedirs(imgdir) + + # boot configuration + cmdline = f"inst.stage2=hd:LABEL={isolabel}" + if kopts: + cmdline += " " + kopts + + info = { + "version": version, + "name": name, + "isolabel": isolabel, + "workdir": workdir, + "configdir": configdir, + "kerneldir": pxedir, + "imgdir": imgdir, + "cmdline": cmdline + } + + #install the kernel + kerneldir = pxedir + kernel_input = inputs.get("kernel", inputs["rootfs"]) + kernel_tree = kernel_input["path"] + bootdir = os.path.join(kernel_tree, "boot") + + os.makedirs(kerneldir) + install(os.path.join(bootdir, f"vmlinuz-{kernel}"), + os.path.join(kerneldir, "vmlinuz")) + + install(os.path.join(bootdir, f"initramfs-{kernel}.img"), + os.path.join(kerneldir, "initrd.img")) + + # iso linux boot + if isolinux.get("enabled"): + make_isolinux(isolinux, root, info, tree) + + # efi boot + if efi: + make_efi(efi, info, root, loop_client) + + # install.img + # rootfs.img + liveos_work = os.path.join(workdir, "liveos") + liveos = os.path.join(liveos_work, "LiveOS") + os.makedirs(liveos) + + rootfs_size = rootfs.get("size", 3072) * 1024*1024 + compression = rootfs.get("compression", {}) + + rootfs = os.path.join(liveos, "rootfs.img") + make_rootfs(tree, rootfs, rootfs_size, workdir, loop_client) + + installimg = os.path.join(imgdir, "install.img") + cmd = ["mksquashfs", liveos_work, installimg] + + if compression: + method = compression["method"] + opts = compression.get("options", {}) + cmd += ["-comp", method] + for opt, val in opts.items(): + cmd += [f"-X{opt}", val] + + subprocess.run(cmd, check=True) + + +if __name__ == '__main__': + args = osbuild.api.arguments() + _output_dir = args["tree"] + with tempfile.TemporaryDirectory(dir=_output_dir) as _workdir: + ret = main(args["inputs"], + _output_dir, + args["options"], + _workdir, + remoteloop.LoopClient("/run/osbuild/api/remoteloop")) + sys.exit(ret) diff --git a/stages/org.osbuild.buildstamp b/stages/org.osbuild.buildstamp new file mode 100755 index 0000000..d1c5523 --- /dev/null +++ b/stages/org.osbuild.buildstamp @@ -0,0 +1,86 @@ +#!/usr/bin/python3 +""" +Create a /.buildstamp file describing the system + +This will create a './buildstamp' with the specified parameters. +""" + +import configparser +import datetime +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": true, +"required": ["arch", "product", "version", "final"], +"properties": { + "arch": { + "description": "Build architecture.", + "type": "string" + }, + "product": { + "description": "The product name.", + "type": "string" + }, + "version": { + "description": "The version .", + "type": "string" + }, + "final": { + "description": "The product.", + "type": "boolean" + }, + "variant": { + "description": "The variant of the product.", + "type": "string" + }, + "bugurl": { + "description": "The bugurl of the product.", + "type": "string" + } +} +""" + + +def main(tree, options): + buildarch = options["arch"] + product = options["product"] + version = options["version"] + isfinal = options["final"] + variant = options.get("variant") + bugurl = options.get("bugurl") + + now = datetime.datetime.now() + datestr = now.strftime("%Y%m%d%H%M") + uid = f"{datestr}.{buildarch}" + + stamp = configparser.ConfigParser() + stamp['Main'] = { + "Product": product, + "Version": version, + "IsFinal": isfinal, + "UUID": uid, + } + + if bugurl: + stamp.set("Main", "BugURL", bugurl) + + if variant: + stamp.set("Main", "Variant", variant) + + stamp["Compose"] = { + "osbuild": "devel", + } + + with open(f"{tree}/.buildstamp", "w") as f: + stamp.write(f) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.chmod b/stages/org.osbuild.chmod new file mode 100755 index 0000000..0453679 --- /dev/null +++ b/stages/org.osbuild.chmod @@ -0,0 +1,69 @@ +#!/usr/bin/python3 +""" +Change file mode bits + +Change the file mode bits of one or more files or directories inside the tree. +""" + +import os +import subprocess +import sys + +import osbuild.api +from osbuild.util.path import in_tree + +SCHEMA_2 = r""" +"options": { + "additionalProperties": false, + "properties": { + "items": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^\\/(?!\\.\\.)((?!\\/\\.\\.\\/).)+$": { + "type": "object", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "description": "Symbolic or numeric octal mode" + }, + "recursive": { + "type": "boolean", + "description": "Change modes recursively", + "default": false + } + } + } + } + } + } +} +""" + + +def chmod(path: str, mode: str, recursive: bool): + arguments = [mode] + if recursive: + arguments.append("--recursive") + arguments.append("--") + arguments.append(path) + + subprocess.run(["chmod", *arguments], check=True) + + +def main(tree, options): + for path, cmdargs in options["items"].items(): + mode = cmdargs["mode"] + recursive = cmdargs.get("recursive", False) + realpath = os.path.join(tree, path.lstrip("/")) + if not in_tree(realpath, tree, must_exist=True): + raise ValueError(f"path {path} not in tree") + chmod(realpath, mode, recursive) + + return 0 + + +if __name__ == "__main__": + args = osbuild.api.arguments() + sys.exit(main(args["tree"], args["options"])) diff --git a/stages/org.osbuild.chrony b/stages/org.osbuild.chrony new file mode 100755 index 0000000..5a43d56 --- /dev/null +++ b/stages/org.osbuild.chrony @@ -0,0 +1,184 @@ +#!/usr/bin/python3 +""" +Configure chrony to set system time from the network. + +Configures `chrony` by modifying `/etc/chrony.conf`. + +Before new values are added to the chrony configuration, all lines starting with +"server", "pool" or "peer" are removed. + +The 'timeservers' option provides a very high-level way of configuring chronyd +with specific timeservers. Its value is a list of strings representing the +hostname or IP address of the timeserver. For each list item, the following +line will be added to the configuration: +`server iburst` + +The 'servers' option provides a direct mapping to the `server` directive from +chrony configuration. Its value is a list of dictionaries representing each +timeserver which should be added to the configuration. For each list item, +a `server` directive will be added the configuration. Currently supported +subset of options which can be specified for each timeserver item: + - 'hostname' (REQUIRED) + - 'minpoll' + - 'maxpoll' + - 'iburst' (defaults to true) + - 'prefer' (defaults to false) + +The 'leapsectz' option configures chrony behavior related to automatic checking +of the next occurrence of the leap second, using the provided timezone. Its +value is a string representing a timezone from the system tz database (e.g. +'right/UTC'). If an empty string is provided, then all occurrences of +'leapsectz' directive are removed from the configuration. + +Constraints: + - Exactly one of 'timeservers' or 'servers' options must be provided. +""" + + +import sys +import re + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"oneOf": [ + {"required": ["timeservers"]}, + {"required": ["servers"]} +], +"properties": { + "timeservers": { + "type": "array", + "items": { "type": "string" }, + "description": "Array of NTP server addresses." + }, + "servers": { + "type": "array", + "items": { + "additionalProperties": false, + "type": "object", + "required": ["hostname"], + "properties": { + "hostname": { + "type": "string", + "description": "Hostname or IP address of a NTP server." + }, + "minpoll": { + "type": "integer", + "description": "Specifies the minimum interval between requests sent to the server as a power of 2 in seconds.", + "minimum": -6, + "maximum": 24 + }, + "maxpoll": { + "type": "integer", + "description": "Specifies the maximum interval between requests sent to the server as a power of 2 in seconds.", + "minimum": -6, + "maximum": 24 + }, + "iburst": { + "type": "boolean", + "default": true, + "description": "Configures chronyd behavior related to burst requests on startup." + }, + "prefer": { + "type": "boolean", + "default": false, + "description": "Prefer this source over sources without the prefer option." + } + } + } + }, + "leapsectz": { + "type": "string", + "description": "Timezone used by chronyd to determine when will the next leap second occur. Empty value will remove the option." + } +} +""" + + +DELETE_TIME_SOURCE_LINE_REGEX = re.compile(r"(server|pool|peer) ") +DELETE_LEAPSECTZ_LINE_REGEX = re.compile(r"leapsectz ") + + +# In-place modify the passed 'chrony_conf_lines' by removing lines which +# match the provided regular expression. +def delete_config_lines(chrony_conf_lines, compiled_re): + chrony_conf_lines[:] = [line for line in chrony_conf_lines if not compiled_re.match(line)] + + +# Modifies the passed 'chrony_conf_lines' in-place. +def handle_timeservers(chrony_conf_lines, timeservers): + # prepend new server lines + new_lines = [f"server {server} iburst" for server in timeservers] + chrony_conf_lines[:] = new_lines + chrony_conf_lines + + +# Modifies the passed 'chrony_conf_lines' in-place. +def handle_servers(chrony_conf_lines, servers): + new_lines = [] + + for server in servers: + new_line = f"server {server['hostname']}" + if server.get("prefer", False): + new_line += " prefer" + if server.get("iburst", True): + new_line += " iburst" + # Default to 'None', because the value can be zero. + minpoll_value = server.get("minpoll", None) + if minpoll_value is not None: + new_line += f" minpoll {minpoll_value}" + # Default to 'None', because the value can be zero. + maxpoll_value = server.get("maxpoll", None) + if maxpoll_value is not None: + new_line += f" maxpoll {maxpoll_value}" + new_lines.append(new_line) + + chrony_conf_lines[:] = new_lines + chrony_conf_lines + + +# Modifies the passed 'chrony_conf_lines' in-place. +def handle_leapsectz(chrony_conf_lines, timezone): + # Delete the directive as the first step, to prevent the situation of + # having it defined multiple times in the configuration. + delete_config_lines(chrony_conf_lines, DELETE_LEAPSECTZ_LINE_REGEX) + + if timezone: + chrony_conf_lines[:] = [f"leapsectz {timezone}"] + chrony_conf_lines + + +def main(tree, options): + timeservers = options.get("timeservers", []) + servers = options.get("servers", []) + # Empty string value will remove the option from the configuration, + # therefore default to 'None' to distinguish these two cases. + leapsectz = options.get("leapsectz", None) + + with open(f"{tree}/etc/chrony.conf") as f: + chrony_conf = f.read() + + # Split to lines and remove ones starting with server, pool or peer. + # At least one option configuring NTP servers is required, therefore + # we do it before applying the configuration. + lines = chrony_conf.split('\n') + delete_config_lines(lines, DELETE_TIME_SOURCE_LINE_REGEX) + + if timeservers: + handle_timeservers(lines, timeservers) + if servers: + handle_servers(lines, servers) + if leapsectz is not None: + handle_leapsectz(lines, leapsectz) + + new_chrony_conf = "\n".join(lines) + + with open(f"{tree}/etc/chrony.conf", "w") as f: + f.write(new_chrony_conf) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.clevis.luks-bind b/stages/org.osbuild.clevis.luks-bind new file mode 100755 index 0000000..bb74a7a --- /dev/null +++ b/stages/org.osbuild.clevis.luks-bind @@ -0,0 +1,78 @@ +#!/usr/bin/python3 +""" +Bind a LUKS device using the specified policy. + +Buildhost commands used: `clevis`, `clevis-luks`, `clevis-pin-*`. +""" + + +import os +import subprocess +import sys + + +import osbuild.api + + +SCHEMA_2 = r""" +"devices": { + "type": "object", + "additionalProperties": true, + "required": ["device"], + "properties": { + "device": { + "type": "object", + "additionalProperties": true + } + } +}, +"options": { + "additionalProperties": false, + "required": ["passphrase", "pin", "policy"], + "properties": { + "passphrase": { + "description": "Passphrase to unlock the container", + "type": "string" + }, + "pin": { + "description": "The pin to use", + "type": "string" + }, + "policy": { + "description": "Policy to use with the given pin", + "type": "string" + } + } +} +""" + + +def main(devices, options): + device = devices["device"] + passphrase = options["passphrase"] + path = os.path.join("/dev", device["path"]) + policy = options["policy"] + pin = options["pin"] + + command = [ + "clevis", + "luks", + "bind", + "-k-", + "-y", + "-f", + "-d", path, pin, policy + ] + + # The null|sss pin need this + os.symlink("/proc/self/fd", "/dev/fd") + + subprocess.run(command, + encoding='utf-8', check=True, + input=passphrase) + + +if __name__ == '__main__': + args = osbuild.api.arguments() + ret = main(args["devices"], args["options"]) + sys.exit(ret) diff --git a/stages/org.osbuild.cloud-init b/stages/org.osbuild.cloud-init new file mode 100755 index 0000000..ca1aff4 --- /dev/null +++ b/stages/org.osbuild.cloud-init @@ -0,0 +1,163 @@ +#!/usr/bin/python3 +""" +Configure cloud-init + +The 'config' option allows to configure cloud-init by creating a +configuration file under `/etc/cloud/cloud.cfg.d` with the name +specified by `filename`. + +Constrains: + - Each configuration file definition must contain at least one configuration + +Currently supported subset of cloud-init configuration: + - 'system_info' section + - 'default_user' section + - 'name' option +""" + + +import sys +import yaml + + +import osbuild.api + + +SCHEMA = r""" +"definitions": { + "reporting_handlers": { + "type": "string", + "enum": ["log", "print", "webhook", "hyperv"] + } +}, +"additionalProperties": false, +"required": ["config", "filename"], +"properties": { + "filename": { + "type": "string", + "description": "Name of the cloud-init configuration file.", + "pattern": "^[\\w.-]{1,251}\\.cfg$" + }, + "config": { + "additionalProperties": false, + "type": "object", + "description": "cloud-init configuration", + "minProperties": 1, + "properties": { + "system_info": { + "additionalProperties": false, + "type": "object", + "description": "'system_info' configuration section.", + "minProperties": 1, + "properties": { + "default_user": { + "additionalProperties": false, + "type": "object", + "description": "Configuration of the 'default' user created by cloud-init.", + "minProperties": 1, + "properties": { + "name": { + "type": "string", + "description": "username of the 'default' user." + } + } + } + } + }, + "reporting": { + "type": "object", + "additionalProperties": false, + "description": "Define reporting endpoints.", + "minProperties": 1, + "properties": { + "logging": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "$ref": "#/definitions/reporting_handlers" + } + } + }, + "telemetry": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "$ref": "#/definitions/reporting_handlers" + } + } + } + } + }, + "datasource_list": { + "type": "array", + "items": { + "type": "string", + "enum": ["Azure"] + } + }, + "datasource": { + "type": "object", + "description": "Sources of configuration data for cloud-init.", + "minProperties": 1, + "properties": { + "Azure": { + "type": "object", + "minProperties": 1, + "properties": { + "apply_network_config": { + "type": "boolean", + "description": "Whether to use network configuration described by Azure’s IMDS endpoint", + "default": true + } + } + } + } + }, + "output": { + "type": "object", + "minProperties": 1, + "properties": { + "init": { + "description": "Redirect the output of the init stage", + "type": "string" + }, + "config": { + "description": "Redirect the output of the config stage", + "type": "string" + }, + "final": { + "description": "Redirect the output of the final stage", + "type": "string" + }, + "all": { + "description": "Redirect the output of all stages", + "type": "string" + } + } + } + } + } +} +""" + + +# Writes the passed `config` object as is into the configuration file in YAML format. +# The validity of the `config` content is assured by the SCHEMA. +def main(tree, options): + filename = options.get("filename") + config = options.get("config", {}) + + config_files_dir = f"{tree}/etc/cloud/cloud.cfg.d" + + with open(f"{config_files_dir}/{filename}", "w") as f: + yaml.dump(config, f, default_flow_style=False) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.copy b/stages/org.osbuild.copy new file mode 100755 index 0000000..1933096 --- /dev/null +++ b/stages/org.osbuild.copy @@ -0,0 +1,141 @@ +#!/usr/bin/python3 +""" +Copy items + +Stage to copy items, that is files or trees, from inputs to mount +points or the tree. Multiple items can be copied. The source and +destination is an url. Supported locations ('schemes') are `tree`, +`mount` and `input`. +The path format follows the rsync convention that if the paths +ends with a slash `/` the content of that directory is copied not +the directory itself. +""" + +import os +import subprocess +import sys + +from typing import Dict +from urllib.parse import urlparse, ParseResult + +import osbuild.api + + +SCHEMA_2 = r""" +"options": { + "additionalProperties": false, + "required": ["paths"], + "properties": { + "paths": { + "description": "Array of items to copy", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["from", "to"], + "properties": { + "from": { + "type": "string", + "description": "The source", + "pattern": "^input:\/\/[^\/]+\/" + }, + "to": { + "oneOf": [ + { + "type": "string", + "description": "The destination, if a mount", + "pattern": "^mount:\/\/[^\/]+\/" + }, + { + "type": "string", + "description": "The destination, if a tree", + "pattern": "^tree:\/\/\/" + } + ] + } + } + } + } + } +}, +"devices": { + "type": "object", + "additionalProperties": true +}, +"mounts": { + "type": "array" +}, +"inputs": { + "type": "object", + "additionalProperties": true +} +""" + + +def parse_mount(url: ParseResult, args: Dict): + name = url.netloc + if name: + root = args["mounts"].get(name, {}).get("path") + if not root: + raise ValueError(f"Unknown mount '{root}'") + else: + root = args["paths"]["mounts"] + + return root + + +def parse_input(url: ParseResult, args: Dict): + name = url.netloc + root = args["inputs"].get(name, {}).get("path") + if root is None: + raise ValueError(f"Unknown input '{root}'") + + return root + + +def parse_location(location, args): + url = urlparse(location) + + scheme = url.scheme + if scheme == "tree": + root = args["tree"] + elif scheme == "mount": + root = parse_mount(url, args) + elif scheme == "input": + root = parse_input(url, args) + else: + raise ValueError(f"Unsupported scheme '{scheme}'") + + assert url.path.startswith("/") + + path = os.path.relpath(url.path, "/") + path = os.path.join(root, path) + path = os.path.normpath(path) + + if url.path.endswith("/"): + path = os.path.join(path, ".") + + return path + + +def main(args, options): + items = options["paths"] + + for path in items: + src = parse_location(path["from"], args) + dst = parse_location(path["to"], args) + + print(f"copying '{src}' -> '{dst}'") + + subprocess.run(["cp", "-a", "--reflink=auto", + src, dst], + check=True) + + return 0 + + +if __name__ == '__main__': + _args = osbuild.api.arguments() + r = main(_args, _args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.cron.script b/stages/org.osbuild.cron.script new file mode 100755 index 0000000..d18ef20 --- /dev/null +++ b/stages/org.osbuild.cron.script @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +""" +Run a script at regular intervals. + +Execute a script at regular intervals. This uses the cron drop-in +directories in etc, which correspond to the supported intervals: + `cron.hourly/`, `cron.daily/`, `cron.weekly/`, `cron.monthly/` + +NB: Does itself not create the directories so they must be created +via the package that provides the facility, like `cronie` or on +older systems `crontabs`. +""" + + +import os +import sys + +import osbuild.api + + +SCHEMA_2 = r""" +"options": { + "additionalProperties": false, + "required": ["interval", "filename"], + "oneOf": [ + {"required": ["simple"]} + ], + "properties": { + "interval": { + "type": "string", + "enum": ["hourly", "daily", "weekly", "monthly"] + }, + "filename": { + "type": "string", + "description": "Name of the cron script", + "pattern": "^[\\w.-]{1,255}$" + }, + "simple": { + "type": "object", + "description": "A simple command to run.", + "required": ["command"], + "properties": { + "comment": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + } + } + } + } +} +""" + + +def format_comment(comment): + lines = comment.split("\n") + return "\n".join(map(lambda c: f"# {c}", lines)) + + +def main(tree, options): + interval = options["interval"] + filename = options["filename"] + filepath = os.path.join(tree, "etc", f"cron.{interval}", filename) + + with open(filepath, "w", encoding="utf-8") as f: + cmd = options["simple"] + cmdline = cmd["command"] + comment = cmd.get("comment") + f.write("!/bin/bash\n") + if comment: + comment = "\n".join(map(lambda c: f"# {c}", comment)) + f.write(f"{comment}\n") + f.write(f"{cmdline}\n") + os.fchmod(f.fileno(), 0o755) + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args.get("options", {})) + sys.exit(r) diff --git a/stages/org.osbuild.crypttab b/stages/org.osbuild.crypttab new file mode 100755 index 0000000..f12beec --- /dev/null +++ b/stages/org.osbuild.crypttab @@ -0,0 +1,91 @@ +#!/usr/bin/python3 +""" +Create /etc/crypttab entries for encrypted block devices + +See crypttab(5) for a detailed description of the format but in brief: +each item in the list of `volumes` describes an encrypted block device +and how it should it should be setup. The block device is identified +either by `uuid` or by `path` (device node path). The volume will be +named as `volume`, i.e. made available as `/dev/mapper/$volume`. +Additionally, a keyfile can (optionally) be specified via `keyfile`. +Specific device options can be specified via `options`. + +This stage replaces /etc/crypttab, removing any existing entries. +""" + + +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"required": ["volumes"], +"properties": { + "volumes": { + "type": "array", + "description": "array of volume objects", + "items": { + "type": "object", + "oneOf": [{ + "required": ["uuid", "volume"] + }, { + "required": ["path", "volume"] + }], + "properties": { + "volume": { + "description": "volume mountpoint", + "type": "string" + }, + "uuid": { + "description": "device UUID", + "type": "string" + }, + "path": { + "description": "device path", + "type": "string" + }, + "keyfile": { + "description": "", + "type": "string", + "default": "none" + }, + "options": { + "description": "options (comma-separated)", + "type": "string", + "default": "" + } + } + } + } +} +""" + + +def main(tree, options): + volumes = options["volumes"] + + with open(f"{tree}/etc/crypttab", "w") as f: + for volume in volumes: + name = volume["volume"] + uuid = volume.get("uuid") + path = volume.get("path") + options = volume.get("options", "") + keyfile = volume.get("keyfile", "none") + + if uuid: + device = f"UUID={uuid}" + elif path: + device = path + else: + raise ValueError("Need 'uuid' or 'label'") + + f.write( + f"{name}\t{device}\t{keyfile}\t{options}\n") + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.debug-shell b/stages/org.osbuild.debug-shell new file mode 100755 index 0000000..8f944c1 --- /dev/null +++ b/stages/org.osbuild.debug-shell @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +""" +Set up an early root shell on a certain tty + +Creates a systemd unit file at /etc/systemd/system/osbuild-debug-shell.service +which starts an early-boot root shell on the given `tty`. + +Also symlinks the service file into /etc/systemd/system/sysinit.target.wants/. +""" + + +import os +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"required": ["tty"], +"properties": { + "tty": { + "type": "string", + "description": "Absolute path of the tty device to start a root shell on." + } +} +""" + + +def main(tree, options): + tty = options["tty"] + + unit = f""" +[Unit] +Description=Early root shell on {tty} FOR DEBUGGING ONLY +DefaultDependencies=no +IgnoreOnIsolate=yes +ConditionPathExists={tty} + +[Service] +Environment=TERM=linux +ExecStart=/bin/sh +Restart=always +RestartSec=0 +StandardInput=tty +TTYPath={tty} +TTYReset=yes +TTYVHangup=yes +KillMode=process +IgnoreSIGPIPE=no +# bash ignores SIGTERM +KillSignal=SIGHUP + +# Unset locale for the console getty since the console has problems +# displaying some internationalized messages. +UnsetEnvironment=LANG LANGUAGE LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT LC_IDENTIFICATION +""" + + with open(f"{tree}/etc/systemd/system/osbuild-debug-shell.service", "w") as f: + f.write(unit) + + os.symlink("../osbuild-debug-shell.service", + f"{tree}/etc/systemd/system/sysinit.target.wants/osbuild-debug-shell.service") + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.discinfo b/stages/org.osbuild.discinfo new file mode 100755 index 0000000..3fc8660 --- /dev/null +++ b/stages/org.osbuild.discinfo @@ -0,0 +1,49 @@ +#!/usr/bin/python3 +""" +Create a `.discinfo` file describing disk + +This will create a `.discinfo` file with the specified parameters. +""" + +import os +import time +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": true, +"required": ["basearch", "release"], +"properties": { + "basearch": { + "description": "Build architecture.", + "type": "string" + }, + "release": { + "description": "The product name.", + "type": "string" + } +} +""" + + +def main(tree, options): + basearch = options["basearch"] + release = options["release"] + + # Based on `pylorax/discinfo.py` + + timestamp = time.time() + with open(os.path.join(tree, ".discinfo"), "w") as f: + f.write(f"{timestamp}\n") + f.write(f"{release}\n") + f.write(f"{basearch}\n") + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.dnf-automatic.config b/stages/org.osbuild.dnf-automatic.config new file mode 100755 index 0000000..a797586 --- /dev/null +++ b/stages/org.osbuild.dnf-automatic.config @@ -0,0 +1,92 @@ +#!/usr/bin/python3 +""" +Change DNF Automatic configuration. + +The stage changes persistent DNF Automatic configuration. Currently, only +a subset of options can be set: + - 'commands' section + - apply_updates + - upgrade_type +""" + + +import sys +import iniparse + +import osbuild.api + + +SCHEMA = r""" +"definitions": { + "commands": { + "type": "object", + "additionalProperties": false, + "description": "'commands' configuration section.", + "properties": { + "apply_updates": { + "type": "boolean", + "description": "Whether packages comprising the available updates should be installed." + }, + "upgrade_type": { + "type": "string", + "description": "What kind of upgrades to look at.", + "enum": ["default", "security"] + } + } + } +}, +"additionalProperties": false, +"description": "DNF Automatic configuration.", +"properties": { + "config": { + "type": "object", + "additionalProperties": false, + "description": "configuration definition.", + "properties": { + "commands": { + "$ref": "#/definitions/commands" + } + } + } +} +""" + + +def bool_to_yes_no(b): + if b: + return "yes" + return "no" + + +def main(tree, options): + config = options.get("config") + dnf_automatic_config_path = f"{tree}/etc/dnf/automatic.conf" + dnf_automatic_conf = iniparse.SafeConfigParser() + + # do not touch the config file if not needed + if config is None: + return 0 + + try: + with open(dnf_automatic_config_path, "r") as f: + dnf_automatic_conf.readfp(f) + except FileNotFoundError: + print(f"Error: DNF automatic configuration file '{dnf_automatic_config_path}' does not exist.") + return 1 + + for config_section, config_options in config.items(): + for option, value in config_options.items(): + if isinstance(value, bool): + value = bool_to_yes_no(value) + dnf_automatic_conf.set(config_section, option, value) + + with open(dnf_automatic_config_path, "w") as f: + dnf_automatic_conf.write(f) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.dnf.config b/stages/org.osbuild.dnf.config new file mode 100755 index 0000000..a9a160e --- /dev/null +++ b/stages/org.osbuild.dnf.config @@ -0,0 +1,135 @@ +#!/usr/bin/python3 +""" +Change DNF configuration. + +The stage changes persistent DNF configuration on the filesystem. Currently, +only DNF variables can be defined. +""" + +import os +import sys +import iniparse + +import osbuild.api + + +SCHEMA = r""" +"definitions": { + "variable": { + "type": "object", + "additionalProperties": false, + "required": ["name", "value"], + "description": "DNF variable to configure persistently.", + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z0-9_]+$", + "description": "Name of the variable." + }, + "value": { + "type": "string", + "description": "Value of the variable." + } + } + }, + "config_main": { + "ip_resolve": { + "type": "string", + "enum": ["4", "IPv4", "6", "IPv6"], + "description": "Determines how DNF resolves host names." + } + } +}, +"additionalProperties": false, +"description": "DNF configuration.", +"properties": { + "variables": { + "type": "array", + "description": "DNF variables to configure persistently.", + "items": { + "$ref": "#/definitions/variable" + } + }, + "config": { + "additionalProperties": false, + "type": "object", + "description": "DNF global configuration.", + "properties": { + "main": { + "$ref": "#/definitions/config_main" + } + } + } +} +""" + + +def configure_variable(tree, name, value): + """ + Creates a persistent DNF configuration for the given variable name and + value. + + From DNF.CONF(5): + Filenames may contain only alphanumeric characters and underscores and be + in lowercase. + """ + vars_directory = "/etc/dnf/vars" + + with open(f"{tree}{vars_directory}/{name}", "w") as f: + f.write(value + "\n") + + +def make_value(value): + val = str(value) + return val + + +def make_section(cfg, name, settings): + if not cfg.has_section(name): + cfg.add_section(name) + + for key, value in settings.items(): + val = make_value(value) + cfg.set(name, key, val) + + +def make_dnf_config(tree, config_options): + """ + Merges the given config object into /etc/dnf/dnf.conf, overwriting existing + values. + """ + dnf_config_path = f"{tree}/etc/dnf/dnf.conf" + dnf_config = iniparse.SafeConfigParser() + + try: + with open(dnf_config_path, "r") as f: + dnf_config.readfp(f) + except FileNotFoundError: + print(f"Warning: DNF configuration file '{dnf_config_path}' does not exist, will create it.") + os.makedirs(f"{tree}/etc/dnf", exist_ok=True) + + for section, items in config_options.items(): + make_section(dnf_config, section, items) + + with open(dnf_config_path, "w") as f: + os.fchmod(f.fileno(), 0o644) + dnf_config.write(f) + + +def main(tree, options): + variables = options.get("variables", []) + + for variable in variables: + configure_variable(tree, variable["name"], variable["value"]) + + config_options = options.get("config") + if config_options: + make_dnf_config(tree, config_options) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.dracut b/stages/org.osbuild.dracut new file mode 100755 index 0000000..ef4c3e7 --- /dev/null +++ b/stages/org.osbuild.dracut @@ -0,0 +1,209 @@ +#!/usr/bin/python3 +""" +Create (re-create) the initial RAM file-system + +Uses `dracut` to re-create the initial RAM filesystem, see man dracut(8). +The kernels for which the initramfs should be generated need to be provided +via `kernel` matching their name on the disk, like "5.6.6-300.fc32.x86_64". + +Supports most options also found in `dracut`(8). See the respective man +page and schema for this stage. + +NB: needs chroot for now as well as `strip` for stripping the initrfams. +""" + +import subprocess +import sys + +import osbuild.api + + +SCHEMA = """ +"required": ["kernel"], +"properties": { + "kernel": { + "description": "List of target kernel versions", + "type": "array", + "items": { + "type": "string", + "description": "A kernel version" + } + }, + "compress": { + "description": "Compress the initramfs, passed via `--compress`", + "type": "string" + }, + "modules": { + "description": "Exact list of dracut modules to use.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "add_modules": { + "description": "Additional dracut modules to include.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "omit_modules": { + "description": "Dracut modules to not include.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "drivers": { + "description": "Kernel modules to exclusively include.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension" + } + }, + "add_drivers": { + "description": "Add a specific kernel modules.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension" + } + }, + "force_drivers": { + "description": "Add driver and ensure that they are tried to be loaded.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension" + } + }, + "filesystems": { + "description": "Kernel filesystem modules to exclusively include.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension" + } + }, + "include": { + "description": "Add custom files to the initramfs.", + "type": "array", + "items": { + "type": "object", + "description": "What (keys) to include where (values)" + } + }, + "install": { + "description": "Install the specified files.", + "type": "array", + "items": { + "type": "string" + } + }, + "early_microcode": { + "description": "Combine early microcode with the initramfs.", + "type": "boolean", + "default": false + }, + "reproducible": { + "description": "Create reproducible images.", + "type": "boolean" + }, + "extra": { + "description": "Extra arguments to directly pass to dracut", + "type": "array", + "items": { + "type": "string", + "description": "Individual extra arguments" + } + } +} +""" + + +def yesno(name: str, value: bool) -> str: + prefix = "" if value else "no-" + return f"--{prefix}{name}" + + +#pylint: disable=too-many-branches +def main(tree, options): + kernels = options["kernel"] + compress = options.get("compress") + modules = options.get("modules", []) # dracut modules + add_modules = options.get("add_modules", []) + omit_modules = options.get("omit_modules", []) + drivers = options.get("drivers", []) # kernel modules + add_drivers = options.get("add_drivers", []) + force_drivers = options.get("force_drivers", []) + filesystems = options.get("filesystems", []) + include = options.get("include", []) + install = options.get("install", []) + early_microcode = options.get("early_microcode", False) + reproducible = options.get("reproducible", True) + extra = options.get("extra", []) + + # initrds may have already been created, force the recreation + opts = ["--force", "-v", "--show-modules"] + + opts += [ + yesno("early-microcode", early_microcode), + yesno("reproducible", reproducible) + ] + + if compress: + opts += [f"--compress={compress}"] + + if modules: + opts += ["--modules", " ".join(modules)] + + if add_modules: + opts += ["--add", " ".join(add_modules)] + + if omit_modules: + opts += ["--omit", " ".join(omit_modules)] + + if drivers: + opts += ["--drivers", " ".join(drivers)] + + if add_drivers: + opts += ["--add-drivers", " ".join(add_drivers)] + + if force_drivers: + opts += ["--force-drivers", " ".join(force_drivers)] + + if filesystems: + opts += ["--filesystems", " ".join(filesystems)] + + if include: + for l in include: + for k, v in l.items(): + opts += ["--include", k, v] + + if install: + for i in install: + opts += ["--install", i] + + opts += extra + + for kver in kernels: + print(f"Building initramfs for {kver}", file=sys.stderr) + + subprocess.run(["/usr/sbin/chroot", tree, + "/usr/bin/dracut", + "--no-hostonly", + "--kver", kver] + + opts, + check=True) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.dracut.conf b/stages/org.osbuild.dracut.conf new file mode 100755 index 0000000..4cd6010 --- /dev/null +++ b/stages/org.osbuild.dracut.conf @@ -0,0 +1,187 @@ +#!/usr/bin/python3 +""" +Configure dracut. + +The 'config' option allows to create a dracut configuration file under +`/usr/lib/dracut/dracut.conf.d/` with the name `filename`. Only a subset +of configuration options is supported, with the intention to provide +functional parity with `org.osbuild.dracut` stage. + +Constrains: + - At least one configuration option must be specified for each configuration + +Supported configuration options: + - compress + - dracutmodules + - add_dracutmodules + - omit_dracutmodules + - drivers + - add_drivers + - force_drivers + - filesystems + - install_items + - early_microcode + - reproducible +""" + +import sys + +import osbuild.api + + +SCHEMA = r""" +"additionalProperties": false, +"required": ["config", "filename"], +"properties": { + "filename": { + "type": "string", + "description": "Name of the dracut configuration file.", + "pattern": "^[\\w.-]{1,250}\\.conf$" + }, + "config": { + "additionalProperties": false, + "type": "object", + "description": "dracut configuration.", + "minProperties": 1, + "properties": { + "compress": { + "description": "Compress the generated initramfs using the passed compression program.", + "type": "string" + }, + "dracutmodules": { + "description": "Exact list of dracut modules to use.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "add_dracutmodules": { + "description": "Additional dracut modules to include.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "omit_dracutmodules": { + "description": "Dracut modules to not include.", + "type": "array", + "items": { + "type": "string", + "description": "A dracut module, e.g. base, nfs, network ..." + } + }, + "drivers": { + "description": "Kernel modules to exclusively include.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension." + } + }, + "add_drivers": { + "description": "Add a specific kernel modules.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension." + } + }, + "force_drivers": { + "description": "Add driver and ensure that they are tried to be loaded.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension." + } + }, + "filesystems": { + "description": "Kernel filesystem modules to exclusively include.", + "type": "array", + "items": { + "type": "string", + "description": "A kernel module without the .ko extension." + } + }, + "install_items": { + "description": "Install the specified files.", + "type": "array", + "items": { + "type": "string", + "description": "Specify additional files to include in the initramfs." + } + }, + "early_microcode": { + "description": "Combine early microcode with the initramfs.", + "type": "boolean" + }, + "reproducible": { + "description": "Create reproducible images.", + "type": "boolean" + } + } + } +} +""" + + +def bool_to_string(value): + return "yes" if value else "no" + + +# Writes to a given file option with the following format: +# persistent_policy="" +def string_option_writer(f, option, value): + f.write(f'{option}="{value}"\n') + + +# Writes to a given file option with the following format: +# add_dracutmodules+=" " +def list_option_writer(f, option, value): + value_str = " ".join(value) + f.write(f'{option}+=" {value_str} "\n') + + +# Writes to a given file option with the following format: +# reproducible="{yes|no}" +def bool_option_writer(f, option, value): + f.write(f'{option}="{bool_to_string(value)}"\n') + + +def main(tree, options): + config = options["config"] + filename = options["filename"] + + config_files_dir = f"{tree}/usr/lib/dracut/dracut.conf.d" + + SUPPORTED_OPTIONS = { + # simple string options + "compress": string_option_writer, + # list options + "add_dracutmodules": list_option_writer, + "dracutmodules": list_option_writer, + "omit_dracutmodules": list_option_writer, + "drivers": list_option_writer, + "add_drivers": list_option_writer, + "force_drivers": list_option_writer, + "filesystems": list_option_writer, + "install_items": list_option_writer, + # bool options + "early_microcode": bool_option_writer, + "reproducible": bool_option_writer + } + + with open(f"{config_files_dir}/{filename}", "w") as f: + for option, value in config.items(): + try: + writter_func = SUPPORTED_OPTIONS[option] + writter_func(f, option, value) + except KeyError as e: + raise ValueError(f"unsupported configuration option '{option}'") from e + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.error b/stages/org.osbuild.error new file mode 100755 index 0000000..dd5b3f7 --- /dev/null +++ b/stages/org.osbuild.error @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +""" +Return an error + +Error stage. Return the given error. Useful for testing, debugging, and +wasting time. +""" + + +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"properties": { + "returncode": { + "description": "What to return code to use", + "type": "number", + "default": 255 + } +} +""" + + +def main(_tree, options): + errno = options.get("returncode", 255) + print(f"Error stage will now return error: {errno}") + return errno + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args.get("options", {})) + sys.exit(r) diff --git a/stages/org.osbuild.fdo b/stages/org.osbuild.fdo new file mode 100755 index 0000000..35cf505 --- /dev/null +++ b/stages/org.osbuild.fdo @@ -0,0 +1,55 @@ +#!/usr/bin/python3 +""" +FDO stage to write down the initial DIUN pub key root certificates +to be read by the manufacturer client + +This will create a '/fdo_diun_root_certs.pem' with content +specified via the `rootcerts` input. +""" + +import shutil +import os +import sys + +import osbuild.api + + +SCHEMA_2 = r""" +"inputs": { + "type": "object", + "additionalProperties": false, + "required": ["rootcerts"], + "properties": { + "rootcerts": { + "type": "object", + "additionalProperties": true + } + } +}, +"options": { + "additionalProperties": false +} +""" + + +def parse_input(inputs): + image = inputs["rootcerts"] + files = image["data"]["files"] + assert len(files) == 1 + + filename, _ = files.popitem() + filepath = os.path.join(image["path"], filename) + return filepath + + +def main(inputs, tree): + certs = parse_input(inputs) + shutil.copy(certs, f"{tree}/fdo_diun_pub_key_root_certs.pem") + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["inputs"], args["tree"]) + sys.exit(r) diff --git a/stages/org.osbuild.firewall b/stages/org.osbuild.firewall new file mode 100755 index 0000000..544373c --- /dev/null +++ b/stages/org.osbuild.firewall @@ -0,0 +1,102 @@ +#!/usr/bin/python3 +""" +Configure firewall + +Configure firewalld using the `firewall-offline-cmd` from inside the target. + +This stage adds each of the given `ports` and `enabled_services` to the default +firewall zone using the `--port` and `--service` options, then removes the +services listed in `disabled_services` with `--remove-service`. + +Ports should be specified as "portid:protocol" or "portid-portid:protocol", +where "portid" is a number (or a port name from `/etc/services`, like "ssh" or +"echo") and "protocol" is one of "tcp", "udp", "sctp", or "dccp". + +Enabling or disabling a service that is already enabled or disabled will not +cause an error. + +Attempting to enable/disable an unknown service name will cause this stage to +fail. Known service names are determined by the contents of firewalld's +configuration directories, usually `/{lib,etc}/firewalld/services/*.xml`, and +may vary from release to release. + +WARNING: this stage uses `chroot` to run `firewall-offline-cmd` inside the +target tree, which means it may fail unexpectedly when the buildhost and target +are different arches or OSes. +""" + + +import subprocess +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"properties": { + "ports": { + "description": "Ports (or port ranges) to open", + "type": "array", + "items": { + "type": "string", + "description": "A port or port range: 'portid[-portid]:protocol'", + "pattern": ".:(tcp|udp|sctp|dccp)$" + } + }, + "enabled_services": { + "description": "Network services to allow in the default firewall zone", + "type": "array", + "items": { + "type": "string", + "description": "Service name (from /{lib,etc}/firewalld/services/*.xml)" + } + }, + "disabled_services": { + "description": "Network services to remove from the default firewall zone", + "type": "array", + "items": { + "type": "string", + "description": "Service name (from /{lib,etc}/firewalld/services/*.xml)" + } + }, + "default_zone": { + "description": "Set default zone for connections and interfaces where no zone has been selected.", + "type": "string" + } +} +""" + + +def main(tree, options): + # Takes a list of : pairs + ports = options.get("ports", []) + # These must be defined for firewalld. It has a set of pre-defined services here: /usr/lib/firewalld/services/, but + # you can also define you own XML files in /etc/firewalld. + enabled_services = options.get("enabled_services", []) + disabled_services = options.get("disabled_services", []) + + default_zone = options.get("default_zone", "") + + # firewall-offline-cmd does not implement --root option so we must chroot it + if default_zone: + subprocess.run(["chroot", tree, "firewall-offline-cmd", f"--set-default-zone={default_zone}"], check=True) + + # The options below are "lokkit" compatibility options and can not be used + # with other options. + if ports or enabled_services or disabled_services: + subprocess.run(["chroot", + tree, + "firewall-offline-cmd"] + + list(map(lambda x: f"--port={x}", ports)) + + list(map(lambda x: f"--service={x}", enabled_services)) + + list(map(lambda x: f"--remove-service={x}", disabled_services)), + check=True) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.first-boot b/stages/org.osbuild.first-boot new file mode 100755 index 0000000..005ded2 --- /dev/null +++ b/stages/org.osbuild.first-boot @@ -0,0 +1,89 @@ +#!/usr/bin/python3 +""" +Execute commands on first-boot + +Sequentially execute a list of commands on first-boot / instantiation. + +This stage uses a logic similar to systemd's first-boot to execute a given +script only the first time the image is booted. + +An empty flag file /etc/osbuild-first-boot is written to /etc and a systemd +service is enabled that is only run when the file exits, and will remove it +before executing the given commands. + +If the flag-file cannot be removed, the service fails without executing +any further first-boot commands. +""" + + +import os +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"required": ["commands"], +"properties": { + "commands": { + "type": "array", + "description": "The command lines to execute", + "items": { + "type": "string" + } + }, + "wait_for_network": { + "type": "boolean", + "description": "Wait for the network to be up before executing", + "default": false + } +} +""" + + +def add_first_boot(tree, commands, wait_for_network): + if wait_for_network: + network = """Wants=network-online.target +After=network-online.target""" + else: + network = "" + + execs = "\n" + for command in commands: + execs += f"ExecStart={command}\n" + + service = f"""[Unit] +Description=OSBuild First Boot Service +ConditionPathExists=/etc/osbuild-first-boot +{network} + +[Service] +Type=oneshot +{execs}""" + + os.makedirs(f"{tree}/usr/lib/systemd/system/default.target.wants", exist_ok=True) + with open(f"{tree}/usr/lib/systemd/system/osbuild-first-boot.service", "w") as f: + f.write(service) + os.symlink("../osbuild-first-boot.service", + f"{tree}/usr/lib/systemd/system/default.target.wants/osbuild-first-boot.service") + + os.makedirs(f"{tree}/etc", exist_ok=True) + open(f"{tree}/etc/osbuild-first-boot", 'a').close() + + +def main(tree, options): + commands = options["commands"] + wait_for_network = options.get("wait_for_network", False) + + commands = ["/usr/bin/rm /etc/osbuild-first-boot"] + commands + + add_first_boot(tree, commands, wait_for_network) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.fix-bls b/stages/org.osbuild.fix-bls new file mode 100755 index 0000000..23380e9 --- /dev/null +++ b/stages/org.osbuild.fix-bls @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +""" +Fix paths in /boot/loader/entries + +Fixes paths in /boot/loader/entries that have incorrect paths for /boot. + +This happens because some boot loader config tools (e.g. grub2-mkrelpath) +examine /proc/self/mountinfo to find the "real" path to /boot, and find the +path to the osbuild tree - which won't be valid at boot time for this image. + +The paths in the Bootloader Specification are relative to the partition +they are located on, i.e. `/boot/loader/...` if `/boot` is on the root +file-system partition. If `/boot` is on a separate partition, the correct +path would be `/loader/.../` The `prefix` can be used to adjust for that. +By default it is `/boot`, i.e. assumes `/boot` is on the root file-system. + +This stage reads and (re)writes all .conf files in /boot/loader/entries. +""" + + +import glob +import re +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"properties": { + "prefix": { + "description": "Prefix to use, normally `/boot`", + "type": "string", + "default": "/boot" + } +} +""" + + +def main(tree, options): + """Fix broken paths in /boot/loader/entries. + + grub2-mkrelpath uses /proc/self/mountinfo to find the source of the file + system it is installed to. This breaks in a container, because we + bind-mount the tree from the host. + """ + prefix = options.get("prefix", "/boot") + + path_re = re.compile(r"(/.*)+/boot") + + for name in glob.glob(f"{tree}/boot/loader/entries/*.conf"): + with open(name) as f: + entry = f.read().splitlines(keepends=True) + + with open(name, "w") as f: + for line in entry: + f.write(path_re.sub(prefix, line)) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.fstab b/stages/org.osbuild.fstab new file mode 100755 index 0000000..f95214f --- /dev/null +++ b/stages/org.osbuild.fstab @@ -0,0 +1,161 @@ +#!/usr/bin/python3 +""" +Create /etc/fstab entries for filesystems + +Each filesystem item must have at least the fs_spec, i.e `uuid`, +`label`, `partlabel` or `device` and a `path` (mount point). + +This stage replaces /etc/fstab, removing any existing entries. + +NB: The ostree configuration options are experimental and might +be replaced with a different mechanism in the near future. +""" + + +import sys + +import osbuild.api +from osbuild.util import ostree + + +SCHEMA = """ +"additionalProperties": false, +"required": ["filesystems"], +"properties": { + "ostree": { + "type": "object", + "additionalProperties": false, + "required": ["deployment"], + "properties": { + "deployment": { + "type": "object", + "additionalProperties": false, + "required": ["osname","ref"], + "properties": { + "osname": { + "description": "Name of the stateroot to be used in the deployment", + "type": "string" + }, + "ref": { + "description": "OStree ref to create and use for deployment", + "type": "string" + }, + "serial": { + "description": "The deployment serial (usually '0')", + "type": "number", + "default": 0 + } + } + } + } + }, + "filesystems": { + "type": "array", + "description": "array of filesystem objects", + "items": { + "type": "object", + "oneOf": [{ + "required": ["device", "path"] + }, { + "required": ["uuid", "path"] + }, { + "required": ["label", "path"] + }, { + "required": ["partlabel", "path"] + }], + "properties": { + "device": { + "description": "Device node", + "type": "string" + }, + "uuid": { + "description": "Filesystem UUID", + "type": "string" + }, + "label": { + "description": "Filesystem label", + "type": "string" + }, + "partlabel": { + "description": "Partition label.", + "type": "string" + }, + "path": { + "description": "Filesystem mountpoint", + "type": "string" + }, + "vfs_type": { + "description": "Filesystem type", + "type": "string", + "default": "none" + }, + "options": { + "description": "Filesystem options (comma-separated)", + "type": "string", + "default": "defaults" + }, + "freq": { + "description": "dump(8) period in days", + "type": "number", + "default": 0 + }, + "passno": { + "description": "pass number on parallel fsck(8)", + "type": "number", + "default": 0 + } + } + } + } +} +""" + + +def main(tree, options): + filesystems = options["filesystems"] + ostree_options = options.get("ostree") + + path = f"{tree}/etc/fstab" + + if ostree_options: + deployment = ostree_options["deployment"] + osname = deployment["osname"] + ref = deployment["ref"] + serial = deployment.get("serial", 0) + + root = ostree.deployment_path(tree, osname, ref, serial) + + print(f"ostree support active: {root}") + + path = f"{root}/etc/fstab" + + with open(path, "w") as f: + for filesystem in filesystems: + uuid = filesystem.get("uuid") + path = filesystem["path"] + label = filesystem.get("label") + partlabel = filesystem.get("partlabel") + device = filesystem.get("device") + vfs_type = filesystem.get("vfs_type", "none") + options = filesystem.get("options", "defaults") + freq = filesystem.get("freq", 0) + passno = filesystem.get("passno", 0) + + if uuid: + fs_spec = f"UUID={uuid}" + elif label: + fs_spec = f"LABEL={label}" + elif partlabel: + fs_spec = f"PARTLABEL={partlabel}" + elif device: + fs_spec = device + else: + raise ValueError("Need 'uuid' or 'label'") + + f.write(f"{fs_spec}\t{path}\t{vfs_type}\t{options}\t{freq}\t{passno}\n") + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.groups b/stages/org.osbuild.groups new file mode 100755 index 0000000..273c3a4 --- /dev/null +++ b/stages/org.osbuild.groups @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +""" +Create group accounts + +Create group accounts, optionally assigning them static GIDs. + +Runs `groupadd` from the buildhost to create the groups listed in `groups`. +If no `gid` is given, `groupadd` will choose one. + +If the specified group name or GID is already in use, this stage will fail. +""" + + +import subprocess +import sys + +import osbuild.api + + +SCHEMA = """ +"additionalProperties": false, +"properties": { + "groups": { + "type": "object", + "additionalProperties": false, + "description": "Keys are group names, values are objects with group info", + "patternProperties": { + "^[A-Za-z0-9_][A-Za-z0-9_-]{0,31}$": { + "type": "object", + "properties": { + "gid": { + "type": "number", + "description": "GID for this group" + } + } + } + } + } +} +""" + + +def groupadd(root, name, gid=None): + arguments = [] + if gid: + arguments += ["--gid", str(gid)] + + subprocess.run(["groupadd", "--root", root, *arguments, name], check=True) + + +def main(tree, options): + groups = options["groups"] + + for name, group_options in groups.items(): + gid = group_options.get("gid") + + groupadd(tree, name, gid) + + return 0 + + +if __name__ == '__main__': + args = osbuild.api.arguments() + r = main(args["tree"], args["options"]) + sys.exit(r) diff --git a/stages/org.osbuild.grub2 b/stages/org.osbuild.grub2 new file mode 100755 index 0000000..0b4c11d --- /dev/null +++ b/stages/org.osbuild.grub2 @@ -0,0 +1,604 @@ +#!/usr/bin/python3 +""" +Configure GRUB2 bootloader and set boot options + +Configure the system to use GRUB2 as the bootloader, and set boot options. + +Sets the GRUB2 boot/root filesystem to `rootfs`. If a separated boot +partition is used it can be specified via `bootfs`. The file-systems +can be identified either via uuid (`{"uuid": ""}`) or label +(`{"label": "