From 6b65a7ae6b3c61c82981995b6d18b5526a344103 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 07 2019 14:19:43 +0000 Subject: Add "pungi_compose" source type. When using this source_type, the "source" should contain URL to variant repository of external compose generated by the Pungi. For example https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Server/. The generated compose will contain the same set of RPMs as the given external compose variant. The packages will be taken from the configured Koji instance. The use-case here is to be able to mirror existing releng compose in a reproducible way. While typical releng composes are removed after few days/weeks, the ODCS keeps the list of builds/packages in the database and therefore can regenerate the same compose later when needed. --- diff --git a/README.md b/README.md index 6983ea8..e5d9aa4 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ Both `sources` and `source_type` are strings. Depending on `source_type` value, | pulp | White-space separated list of content-sets. Repositories defined by these content-sets will be included in a compose. | | raw_config | String in `name#commit` hash format. The `name` must match one of the raw config locations defined in ODCS server config as `raw_config_urls`. The `commit` is commit hash defining the version of raw config to use. This config is then used as input config for Pungi. | | build | Source should be omitted in the request. The list of Koji builds included in a compose is defined by `builds` attribute. | +| pungi_compose | URL to variant repository of external compose generated by the Pungi. For example https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Server/. The generated compose will contain the same set of RPMs as the given external compose variant. The packages will be taken from the configured Koji instance. | There are also additional optional attributes you can pass to `new_compose(...)` method: diff --git a/common/odcs/common/types.py b/common/odcs/common/types.py index 6db313d..a4920b4 100644 --- a/common/odcs/common/types.py +++ b/common/odcs/common/types.py @@ -29,6 +29,7 @@ class PungiSourceType: PULP = 4 RAW_CONFIG = 5 BUILD = 6 + PUNGI_COMPOSE = 7 PUNGI_SOURCE_TYPE_NAMES = { @@ -45,6 +46,9 @@ PUNGI_SOURCE_TYPE_NAMES = { # Generates compose using exactly defined set of Koji builds without # pulling in RPMs from any Koji tag. "build": PungiSourceType.BUILD, + # Generates compose using the same set of RPMs as in existing 3rd party + # Pungi compose. + "pungi_compose": PungiSourceType.PUNGI_COMPOSE, } INVERSE_PUNGI_SOURCE_TYPE_NAMES = { diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index a455588..d10a747 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -34,6 +34,7 @@ from odcs.server.models import Compose, COMPOSE_STATES, COMPOSE_FLAGS from odcs.server.pungi import Pungi, PungiConfig, PungiSourceType, PungiLogs, RawPungiConfig from odcs.server.pulp import Pulp from odcs.server.cache import KojiTagCache +from odcs.server.pungi_compose import PungiCompose from concurrent.futures import ThreadPoolExecutor import glob import odcs.server.utils @@ -351,6 +352,35 @@ def resolve_compose(compose): for m in new_mbs_modules if m['name'] not in conf.base_module_names) compose.source = ' '.join(uids) + elif compose.source_type == PungiSourceType.PUNGI_COMPOSE: + external_compose = PungiCompose(compose.source) + rpms_data = external_compose.get_rpms_data() + + # If there is None in the sigkeys, it means unsigned packages are + # allowed. The sigkeys in the `compose.sigkeys` are sorted by + # preference and unsigned packages should be tried as last. + # Therefore we need to remove None from `sigkeys` and handle + # it as last element in `compose.sigkeys`. + if None in rpms_data["sigkeys"]: + allow_unsigned = True + # Remove None from sigkeys. + rpms_data["sigkeys"].remove(None) + else: + allow_unsigned = False + compose.sigkeys = " ".join(rpms_data["sigkeys"]) + if allow_unsigned: + # Unsigned packages are allowed by white-space in the end of + # `compose.sigkeys`. + compose.sigkeys += " " + + compose.arches = " ".join(rpms_data["arches"]) + compose.builds = " ".join(rpms_data["builds"].keys()) + + packages = set() + for rpms in rpms_data["builds"].values(): + for rpm_nevra in rpms: + packages.add(productmd.common.parse_nvra(rpm_nevra)['name']) + compose.packages = " ".join(packages) def get_reusable_compose(compose): @@ -579,6 +609,9 @@ def generate_pungi_compose(compose): """ koji_tag_cache = KojiTagCache() + # Resolve the general data in the compose. + resolve_compose(compose) + # Reformat the data from database packages = compose.packages if packages: @@ -587,9 +620,6 @@ def generate_pungi_compose(compose): if builds: builds = builds.split(" ") - # Resolve the general data in the compose. - resolve_compose(compose) - # Check if we can reuse some existing compose instead of # generating new one. compose_to_reuse = get_reusable_compose(compose) diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index b515d4c..58b5027 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -157,11 +157,9 @@ class PungiConfig(BasePungiConfig): if self.packages: raise ValueError("Exact packages cannot be set for MODULE " "source type.") - elif source_type == PungiSourceType.REPO: - self.gather_source = "comps" - self.gather_method = "deps" - self.koji_tag = None - elif source_type == PungiSourceType.BUILD: + elif source_type in [PungiSourceType.BUILD, + PungiSourceType.PUNGI_COMPOSE, + PungiSourceType.REPO]: self.gather_source = "comps" self.gather_method = "deps" self.koji_tag = None diff --git a/server/odcs/server/pungi_compose.py b/server/odcs/server/pungi_compose.py new file mode 100644 index 0000000..524992a --- /dev/null +++ b/server/odcs/server/pungi_compose.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import os +import requests +import productmd.common + + +class PungiCompose(object): + """Represents 3rd party Pungi Compose""" + + def __init__(self, variant_url): + """ + Creates new PungiCompose. + + :param str metadata_url: URL to Pungi variant repository directory. + """ + # The `variant_url` is for example "http://localhost/foo/compose/Server". + self.variant_url = variant_url.rstrip("/") + # The `variant_name` is for example `Server` + self.variant_name = os.path.basename(self.variant_url) + # The `metadata_url` is for example "http://localhost/foo/compose/metadata". + self.metadata_url = os.path.join( + os.path.dirname(self.variant_url), "metadata") + + def _fetch_json(self, url): + """ + Fetches the json file represented by `url`. + """ + r = requests.get(url) + r.raise_for_status() + return r.json() + + def get_rpms_data(self): + """ + Returns the data describing the RPMs in the pungi compose. + :rtype: dict. + :return: Dictionary with RPMs data in following format: + { + "sigkeys": set() with sigkeys used in the compose. + "arches": set() with all the arches used in the compose. + "builds": { + koji-build-nvr1: set() with the RPMs NEVRAs, + koji-build-nvr2: ..., + ... + } + } + """ + ret = {} + ret["sigkeys"] = set() + ret["arches"] = set() + ret["builds"] = {} + + # Fetch the rpms.json and get the part containing SRPMs + # for the right variant. + url = os.path.join(self.metadata_url, "rpms.json") + data = self._fetch_json(url) + srpms_per_arch = data.get("payload", {}).get("rpms", {}).get( + self.variant_name) + if not srpms_per_arch: + raise ValueError("The %s does not contain payload -> rpms -> %s " + "section" % (url, self.variant_name)) + + # Go through the data and fill in the dict to return. + for arch, srpms in srpms_per_arch.items(): + ret["arches"].add(arch) + for srpm_nevra, rpms in srpms.items(): + packages = set() + for rpm_nevra, rpm_data in rpms.items(): + packages.add(rpm_nevra) + ret["sigkeys"].add(rpm_data["sigkey"]) + + srpm_nvr = "{name}-{version}-{release}".format( + **productmd.common.parse_nvra(srpm_nevra)) + ret["builds"][srpm_nvr] = packages + + return ret diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index bd49d04..c42fcea 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -666,6 +666,39 @@ gpgcheck=0 resolve_compose(c) self.assertEqual(c.source, "bar:0:1:y foo:0:1:x platform:0:1:z") + @patch('odcs.server.pungi_compose.PungiCompose.get_rpms_data') + def test_resolve_compose_pungi_compose_source_type(self, get_rpms_data): + get_rpms_data.return_value = { + 'sigkeys': set(['sigkey1', None]), + 'arches': set(['x86_64']), + 'builds': { + 'flatpak-rpm-macros-29-6.module+125+c4f5c7f2': set([ + 'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src', + 'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64']), + 'flatpak-runtime-config-29-4.module+125+c4f5c7f2': set([ + 'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src', + 'flatpak-runtime-config2-0:29-4.module+125+c4f5c7f2.x86_64']) + } + } + + c = Compose.create( + db.session, "me", PungiSourceType.PUNGI_COMPOSE, + "http://localhost/compose/Temporary", + COMPOSE_RESULTS["repository"], 3600) + db.session.add(c) + db.session.commit() + + resolve_compose(c) + self.assertEqual(c.sigkeys.split(" "), ["sigkey1", ""]) + self.assertEqual(c.arches.split(" "), ["x86_64"]) + self.assertEqual(set(c.builds.split(" ")), set([ + 'flatpak-rpm-macros-29-6.module+125+c4f5c7f2', + 'flatpak-runtime-config-29-4.module+125+c4f5c7f2'])) + self.assertEqual(set(c.packages.split(" ")), set([ + 'flatpak-rpm-macros', + 'flatpak-runtime-config', + 'flatpak-runtime-config2'])) + class TestGeneratePungiCompose(ModelsBaseTest): diff --git a/server/tests/test_pungi_compose.py b/server/tests/test_pungi_compose.py new file mode 100644 index 0000000..272df42 --- /dev/null +++ b/server/tests/test_pungi_compose.py @@ -0,0 +1,105 @@ +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import six +import unittest +from mock import patch + +from odcs.server.pungi_compose import PungiCompose + + +RPMS_JSON = { + "header": { + "type": "productmd.rpms", + "version": "1.2" + }, + "payload": { + "compose": { + "date": "20181210", + "id": "odcs-691-1-20181210.n.0", + "respin": 0, + "type": "nightly" + }, + "rpms": { + "Temporary": { + "x86_64": { + "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src": { + "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/f/flatpak-rpm-macros-29-6.module+125+c4f5c7f2.src.rpm", + "sigkey": None + }, + "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/f/flatpak-rpm-macros-29-6.module+125+c4f5c7f2.x86_64.rpm", + "sigkey": None + } + }, + "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src": { + "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src": { + "category": "source", + "path": "Temporary/source/tree/Packages/f/flatpak-runtime-config-29-4.module+125+c4f5c7f2.src.rpm", + "sigkey": "sigkey1" + }, + "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.x86_64": { + "category": "binary", + "path": "Temporary/x86_64/os/Packages/f/flatpak-runtime-config-29-4.module+125+c4f5c7f2.x86_64.rpm", + "sigkey": "sigkey1" + } + } + } + } + } + } +} + + +@patch("odcs.server.pungi_compose.PungiCompose._fetch_json") +class TestPungiCompose(unittest.TestCase): + + def test_get_rpms_data(self, fetch_json): + fetch_json.return_value = RPMS_JSON + compose = PungiCompose("http://localhost/compose/Temporary") + data = compose.get_rpms_data() + + expected = { + 'sigkeys': set(['sigkey1', None]), + 'arches': set(['x86_64']), + 'builds': { + 'flatpak-rpm-macros-29-6.module+125+c4f5c7f2': set([ + 'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src', + 'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64']), + 'flatpak-runtime-config-29-4.module+125+c4f5c7f2': set([ + 'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src', + 'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.x86_64']) + } + } + + self.assertEqual(data, expected) + + def test_get_rpms_data_unknown_variant(self, fetch_json): + fetch_json.return_value = RPMS_JSON + msg = ("The http://localhost/compose/metadata/rpms.json does not " + "contain payload -> rpms -> Workstation section") + with six.assertRaisesRegex(self, ValueError, msg): + compose = PungiCompose("http://localhost/compose/Workstation") + compose.get_rpms_data()