#248 Add "pungi_compose" source type.
Merged 3 months ago by jkaluza. Opened 4 months ago by jkaluza.

file modified
+1

@@ -126,6 +126,7 @@ 

  | pulp          | White-space separated list of content-sets. Repositories defined by these content-sets will be included in a compose. |

  | raw_config    | String in `name#commit` hash format. The `name` must match one of the raw config locations defined in ODCS server config as `raw_config_urls`. The `commit` is commit hash defining the version of raw config to use. This config is then used as input config for Pungi. |

  | build         | Source should be omitted in the request. The list of Koji builds included in a compose is defined by `builds` attribute. |

+ | pungi_compose | URL to variant repository of external compose generated by the Pungi. For example https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Server/. The generated compose will contain the same set of RPMs as the given external compose variant. The packages will be taken from the configured Koji instance. |

  

  There are also additional optional attributes you can pass to `new_compose(...)` method:

  

@@ -29,6 +29,7 @@ 

      PULP = 4

      RAW_CONFIG = 5

      BUILD = 6

+     PUNGI_COMPOSE = 7

  

  

  PUNGI_SOURCE_TYPE_NAMES = {

@@ -45,6 +46,9 @@ 

      # Generates compose using exactly defined set of Koji builds without

      # pulling in RPMs from any Koji tag.

      "build": PungiSourceType.BUILD,

+     # Generates compose using the same set of RPMs as in existing 3rd party

+     # Pungi compose.

+     "pungi_compose": PungiSourceType.PUNGI_COMPOSE,

  }

  

  INVERSE_PUNGI_SOURCE_TYPE_NAMES = {

file modified
+33 -3

@@ -34,6 +34,7 @@ 

  from odcs.server.pungi import Pungi, PungiConfig, PungiSourceType, PungiLogs, RawPungiConfig

  from odcs.server.pulp import Pulp

  from odcs.server.cache import KojiTagCache

+ from odcs.server.pungi_compose import PungiCompose

  from concurrent.futures import ThreadPoolExecutor

  import glob

  import odcs.server.utils

@@ -351,6 +352,35 @@ 

              for m in new_mbs_modules

              if m['name'] not in conf.base_module_names)

          compose.source = ' '.join(uids)

+     elif compose.source_type == PungiSourceType.PUNGI_COMPOSE:

+         external_compose = PungiCompose(compose.source)

+         rpms_data = external_compose.get_rpms_data()

+ 

+         # If there is None in the sigkeys, it means unsigned packages are

+         # allowed. The sigkeys in the `compose.sigkeys` are sorted by

+         # preference and unsigned packages should be tried as last.

+         # Therefore we need to remove None from `sigkeys` and handle

+         # it as last element in `compose.sigkeys`.

+         if None in rpms_data["sigkeys"]:

+             allow_unsigned = True

+             # Remove None from sigkeys.

+             rpms_data["sigkeys"].remove(None)

+         else:

+             allow_unsigned = False

+         compose.sigkeys = " ".join(rpms_data["sigkeys"])

+         if allow_unsigned:

+             # Unsigned packages are allowed by white-space in the end of

+             # `compose.sigkeys`.

+             compose.sigkeys += " "

+ 

+         compose.arches = " ".join(rpms_data["arches"])

+         compose.builds = " ".join(rpms_data["builds"].keys())

+ 

+         packages = set()

+         for rpms in rpms_data["builds"].values():

+             for rpm_nevra in rpms:

+                 packages.add(productmd.common.parse_nvra(rpm_nevra)['name'])

+         compose.packages = " ".join(packages)

  

  

  def get_reusable_compose(compose):

@@ -579,6 +609,9 @@ 

      """

      koji_tag_cache = KojiTagCache()

  

+     # Resolve the general data in the compose.

+     resolve_compose(compose)

+ 

      # Reformat the data from database

      packages = compose.packages

      if packages:

@@ -587,9 +620,6 @@ 

      if builds:

          builds = builds.split(" ")

  

-     # Resolve the general data in the compose.

-     resolve_compose(compose)

- 

      # Check if we can reuse some existing compose instead of

      # generating new one.

      compose_to_reuse = get_reusable_compose(compose)

file modified
+3 -5

@@ -157,11 +157,9 @@ 

              if self.packages:

                  raise ValueError("Exact packages cannot be set for MODULE "

                                   "source type.")

-         elif source_type == PungiSourceType.REPO:

-             self.gather_source = "comps"

-             self.gather_method = "deps"

-             self.koji_tag = None

-         elif source_type == PungiSourceType.BUILD:

+         elif source_type in [PungiSourceType.BUILD,

+                              PungiSourceType.PUNGI_COMPOSE,

+                              PungiSourceType.REPO]:

              self.gather_source = "comps"

              self.gather_method = "deps"

              self.koji_tag = None

@@ -0,0 +1,97 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2018  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in

+ # all copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import os

+ import requests

+ import productmd.common

+ 

+ 

+ class PungiCompose(object):

+     """Represents 3rd party Pungi Compose"""

+ 

+     def __init__(self, variant_url):

+         """

+         Creates new PungiCompose.

+ 

+         :param str metadata_url: URL to Pungi variant repository directory.

+         """

+         # The `variant_url` is for example "http://localhost/foo/compose/Server".

+         self.variant_url = variant_url.rstrip("/")

Can we add some validation that variant_url is indeed a URL? We can then provide a meaningful error message to user otherwise.

+         # The `variant_name` is for example `Server`

+         self.variant_name = os.path.basename(self.variant_url)

+         # The `metadata_url` is for example "http://localhost/foo/compose/metadata".

+         self.metadata_url = os.path.join(

+             os.path.dirname(self.variant_url), "metadata")

+ 

+     def _fetch_json(self, url):

+         """

+         Fetches the json file represented by `url`.

+         """

+         r = requests.get(url)

+         r.raise_for_status()

+         return r.json()

+ 

+     def get_rpms_data(self):

+         """

+         Returns the data describing the RPMs in the pungi compose.

+         :rtype: dict.

+         :return: Dictionary with RPMs data in following format:

+             {

+                 "sigkeys": set() with sigkeys used in the compose.

+                 "arches": set() with all the arches used in the compose.

+                 "builds": {

+                     koji-build-nvr1: set() with the RPMs NEVRAs,

+                     koji-build-nvr2: ...,

+                     ...

+                 }

+             }

+         """

+         ret = {}

+         ret["sigkeys"] = set()

+         ret["arches"] = set()

+         ret["builds"] = {}

+ 

+         # Fetch the rpms.json and get the part containing SRPMs

+         # for the right variant.

+         url = os.path.join(self.metadata_url, "rpms.json")

+         data = self._fetch_json(url)

+         srpms_per_arch = data.get("payload", {}).get("rpms", {}).get(

+             self.variant_name)

+         if not srpms_per_arch:

+             raise ValueError("The %s does not contain payload -> rpms -> %s "

+                              "section" % (url, self.variant_name))

+ 

+         # Go through the data and fill in the dict to return.

+         for arch, srpms in srpms_per_arch.items():

+             ret["arches"].add(arch)

+             for srpm_nevra, rpms in srpms.items():

+                 packages = set()

+                 for rpm_nevra, rpm_data in rpms.items():

+                     packages.add(rpm_nevra)

+                     ret["sigkeys"].add(rpm_data["sigkey"])

+ 

+                 srpm_nvr = "{name}-{version}-{release}".format(

+                     **productmd.common.parse_nvra(srpm_nevra))

+                 ret["builds"][srpm_nvr] = packages

+ 

+         return ret

@@ -666,6 +666,39 @@ 

              resolve_compose(c)

              self.assertEqual(c.source, "bar:0:1:y foo:0:1:x platform:0:1:z")

  

+     @patch('odcs.server.pungi_compose.PungiCompose.get_rpms_data')

It would be great to add some negative tests as well.

+     def test_resolve_compose_pungi_compose_source_type(self, get_rpms_data):

+         get_rpms_data.return_value = {

+             'sigkeys': set(['sigkey1', None]),

+             'arches': set(['x86_64']),

+             'builds': {

+                 'flatpak-rpm-macros-29-6.module+125+c4f5c7f2': set([

+                     'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src',

+                     'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64']),

+                 'flatpak-runtime-config-29-4.module+125+c4f5c7f2': set([

+                     'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src',

+                     'flatpak-runtime-config2-0:29-4.module+125+c4f5c7f2.x86_64'])

+             }

+         }

+ 

+         c = Compose.create(

+             db.session, "me", PungiSourceType.PUNGI_COMPOSE,

+             "http://localhost/compose/Temporary",

+             COMPOSE_RESULTS["repository"], 3600)

+         db.session.add(c)

+         db.session.commit()

+ 

+         resolve_compose(c)

+         self.assertEqual(c.sigkeys.split(" "), ["sigkey1", ""])

+         self.assertEqual(c.arches.split(" "), ["x86_64"])

+         self.assertEqual(set(c.builds.split(" ")), set([

+             'flatpak-rpm-macros-29-6.module+125+c4f5c7f2',

+             'flatpak-runtime-config-29-4.module+125+c4f5c7f2']))

+         self.assertEqual(set(c.packages.split(" ")), set([

+             'flatpak-rpm-macros',

+             'flatpak-runtime-config',

+             'flatpak-runtime-config2']))

+ 

  

  class TestGeneratePungiCompose(ModelsBaseTest):

  

@@ -0,0 +1,105 @@ 

+ # Copyright (c) 2018  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import six

+ import unittest

+ from mock import patch

+ 

+ from odcs.server.pungi_compose import PungiCompose

+ 

+ 

+ RPMS_JSON = {

+     "header": {

+         "type": "productmd.rpms",

+         "version": "1.2"

+     },

+     "payload": {

+         "compose": {

+             "date": "20181210",

+             "id": "odcs-691-1-20181210.n.0",

+             "respin": 0,

+             "type": "nightly"

+         },

+         "rpms": {

+             "Temporary": {

+                 "x86_64": {

+                     "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src": {

+                         "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src": {

+                             "category": "source",

+                             "path": "Temporary/source/tree/Packages/f/flatpak-rpm-macros-29-6.module+125+c4f5c7f2.src.rpm",

+                             "sigkey": None

+                         },

+                         "flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64": {

+                             "category": "binary",

+                             "path": "Temporary/x86_64/os/Packages/f/flatpak-rpm-macros-29-6.module+125+c4f5c7f2.x86_64.rpm",

+                             "sigkey": None

+                         }

+                     },

+                     "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src": {

+                         "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src": {

+                             "category": "source",

+                             "path": "Temporary/source/tree/Packages/f/flatpak-runtime-config-29-4.module+125+c4f5c7f2.src.rpm",

+                             "sigkey": "sigkey1"

+                         },

+                         "flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.x86_64": {

+                             "category": "binary",

+                             "path": "Temporary/x86_64/os/Packages/f/flatpak-runtime-config-29-4.module+125+c4f5c7f2.x86_64.rpm",

+                             "sigkey": "sigkey1"

+                         }

+                     }

+                 }

+             }

+         }

+     }

+ }

+ 

+ 

+ @patch("odcs.server.pungi_compose.PungiCompose._fetch_json")

+ class TestPungiCompose(unittest.TestCase):

+ 

+     def test_get_rpms_data(self, fetch_json):

+         fetch_json.return_value = RPMS_JSON

+         compose = PungiCompose("http://localhost/compose/Temporary")

+         data = compose.get_rpms_data()

+ 

+         expected = {

+             'sigkeys': set(['sigkey1', None]),

+             'arches': set(['x86_64']),

+             'builds': {

+                 'flatpak-rpm-macros-29-6.module+125+c4f5c7f2': set([

+                     'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.src',

+                     'flatpak-rpm-macros-0:29-6.module+125+c4f5c7f2.x86_64']),

+                 'flatpak-runtime-config-29-4.module+125+c4f5c7f2': set([

+                     'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.src',

+                     'flatpak-runtime-config-0:29-4.module+125+c4f5c7f2.x86_64'])

+             }

+         }

+ 

+         self.assertEqual(data, expected)

+ 

+     def test_get_rpms_data_unknown_variant(self, fetch_json):

+         fetch_json.return_value = RPMS_JSON

+         msg = ("The http://localhost/compose/metadata/rpms.json does not "

+                "contain payload -> rpms -> Workstation section")

+         with six.assertRaisesRegex(self, ValueError, msg):

+             compose = PungiCompose("http://localhost/compose/Workstation")

+             compose.get_rpms_data()

When using this source_type, the "source" should contain URL to variant
repository of external compose generated by the Pungi. For example
https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/compose/Server/.

The generated compose will contain the same set of RPMs as the given
external compose variant. The packages will be taken from the configured
Koji instance.

The use-case here is to be able to mirror existing releng compose in
a reproducible way. While typical releng composes are removed after few
days/weeks, the ODCS keeps the list of builds/packages in the database
and therefore can regenerate the same compose later when needed.

Probably rpms_data["sigkeys"].remove(None) is a bit more efficient and easier to understand; it wouldn't require comment :)

For readability, consider:

for rpm_nevra in rpms:
  packages.add(productmd.common.parse_nvra(rpm_nevra)['name'])

Maybe also combine PungiSourceType.REPO here?

Can we add some validation that variant_url is indeed a URL? We can then provide a meaningful error message to user otherwise.

It would be great to add some negative tests as well.

rebased onto 6b65a7a

3 months ago

I've just found out that there was missing test_pungi_compose.py file which was never committed. I've done that now and also fixed issues @lucarval pointed out.

Pull-Request has been merged by jkaluza

3 months ago