#368 Add new "/verify-image" and "/verify-image-repository" API endpoints.
Merged 4 months ago by jkaluza. Opened 4 months ago by jkaluza.
jkaluza/freshmaker verify-api  into  master

@@ -0,0 +1,181 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in

+ # all copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ from freshmaker import conf

+ from freshmaker.lightblue import LightBlue

+ 

+ 

+ class ImageVerifier(object):

+ 

+     def __init__(self, lb=None):

+         """

+         Creates new ImageVerifier.

+ 

+         :param LightBlue lb: Lightblue instance to use to verify images.

+             When None, new default Lightblue class is created.

+         """

+         self.lb = lb if lb else LightBlue(

+             server_url=conf.lightblue_server_url,

+             cert=conf.lightblue_certificate,

+             private_key=conf.lightblue_private_key)

+ 

+     def _verify_repository_data(self, repo):

+         """

+         Verifies the Lightblue ContainerRepository data.

+         Raises ValueError in case of error.

+         """

+         if "Generally Available" not in repo["release_categories"]:

+             raise ValueError(

+                 "Only repositories with \"Generally Available\" release_categories can be "

+                 "rebuilt, but found %r." % repo["release_categories"])

+ 

+         if not repo["published"]:

+             raise ValueError(

+                 "Only published repositories can be rebuilt, but this repository is not "

+                 "published.")

+ 

+         if "auto_rebuild_tags" in repo and repo["auto_rebuild_tags"] == []:

+             raise ValueError(

+                 "The \"auto_rebuild_tags\" in COMET is set to an empty list, this means "

+                 "rebuilds of images in this repository are disabled.")

+ 

+     def _verify_image_data(self, image):

+         """

+         Verifies the Lightblue ContainerImage data.

+         Raises ValueError in case of error.

+         """

+         if not image["content_sets"]:

+             raise ValueError(

+                 "Found image \"%s\" in this repository, but it cannot be rebuilt, because "

+                 "the \"content_sets\" are not set for this image." % image["brew"]["build"])

+ 

+     def _get_repository_from_name(self, repo_name):

+         """

+         Returns the ContainerRepository object based on the Repository name.

+         """

+         query = {

+             "objectType": "containerRepository",

+             "query": {

+                 "$and": [

+                     {

+                         "field": "repository",

+                         "op": "=",

+                         "rvalue": repo_name

+                     },

+ 

+                 ]

+             },

+             "projection": [

+                 {"field": "*", "include": True, "recursive": True}

+             ]

+         }

+ 

+         repos = self.lb.find_container_repositories(query)

+         if not repos:

+             raise ValueError("Cannot get repository %s from Lightblue." % repo_name)

+         if len(repos) != 1:

+             raise ValueError("Multiple records found in Lightblue for repository %s." % repo_name)

+ 

+         return repos[0]

+ 

+     def _get_repository_from_image(self, nvr):

+         """

+         Returns the ContainerRepository object based on the image NVR.

+         """

+         query = {

+             "objectType": "containerRepository",

+             "query": {

+                 "$and": [

+                     {

+                         "field": "images.*.brew.build",

+                         "op": "=",

+                         "rvalue": nvr

+                     },

+ 

+                 ]

+             },

+             "projection": [

+                 {"field": "*", "include": True, "recursive": True}

+             ]

+         }

+ 

+         repos = self.lb.find_container_repositories(query)

+         if not repos:

+             raise ValueError("Cannot get repository for image %s from Lightblue." % nvr)

+         if len(repos) != 1:

+             raise ValueError(

+                 "Image %s found in multiple repositories in Lightblue." % nvr)

+ 

+         return repos[0]

+ 

+     def verify_image(self, image_nvr):

+         """

+         Verifies the image defined by `image_nvr`.

+         Raises ValueError in case of error.

+ 

+         :param str image_nvr: NVR of image to verify.

+         :rtype: dict

+         :return: Dict with image NVR as key and list of content_sets as values.

+         """

+         repo = self._get_repository_from_image(image_nvr)

+         self._verify_repository_data(repo)

+ 

+         images = self.lb.get_images_by_nvrs([image_nvr], include_rpms=False)

+         if not images:

+             raise ValueError(

+                 "No published images tagged by %r found in repository" % (

+                     repo["auto_rebuild_tags"]))

+ 

+         image = images[0]

+         self._verify_image_data(image)

+ 

+         return {

+             image["brew"]["build"]: image["content_sets"]

+         }

+ 

+     def verify_repository(self, repo_name):

+         """

+         Verifies the images in repository defined by `repo_name`.

+         Raises ValueError in case of error.

+ 

+         :param str repo_name: Name of repository to verify.

+         :rtype: dict

+         :return: Dict with image NVR as key and list of content_sets as values.

+         """

+         repo = self._get_repository_from_name(repo_name)

+         self._verify_repository_data(repo)

+ 

+         rebuildable_images = {}

+         images = self.lb.find_images_with_included_srpms(

+             [], [], {repo["repository"]: repo}, include_rpms=False)

+         for image in images:

+             nvr = image["brew"]["build"]

+             self._verify_image_data(image)

+             rebuildable_images[nvr] = image["content_sets"]

+ 

+         if not rebuildable_images:

+             raise ValueError(

+                 "No published images tagged by %r found in repository" % (

+                     repo["auto_rebuild_tags"]))

+ 

+         return rebuildable_images

file modified
+25 -16

@@ -843,7 +843,8 @@ 

          return ret

  

      def find_images_with_included_srpms(

-             self, content_sets, srpm_nvrs, repositories, published=True):

+             self, content_sets, srpm_nvrs, repositories, published=True,

+             include_rpms=True):

          """Query lightblue and find containerImages in given

          containerRepositories. By default limit only to images which have been

          published to at least one repository and images which have latest tag.

@@ -854,6 +855,7 @@ 

          :param list repositories: List of repository names to look for.

          :param bool published: whether to limit queries to published

              repositories

+         :param bool include_rpms: whether to include the RPMs in the result.

          """

          auto_rebuild_tags = set()

          for repo in repositories.values():

@@ -871,26 +873,12 @@ 

                  "$and": [

                      {

                          "$or": [{

-                             "field": "content_sets.*",

-                             "op": "=",

-                             "rvalue": r

-                         } for r in content_sets]

-                     },

-                     {

-                         "$or": [{

                              "field": "repositories.*.tags.*.name",

                              "op": "=",

                              "rvalue": tag

                          } for tag in auto_rebuild_tags]

                      },

                      {

-                         "$or": [{

-                             "field": "rpm_manifest.*.rpms.*.srpm_name",

-                             "op": "=",

-                             "rvalue": srpm_name

-                         } for srpm_name in srpm_name_to_nvr.keys()]

-                     },

-                     {

                          "field": "parsed_data.files.*.key",

                          "op": "=",

                          "rvalue": "buildfile"

@@ -898,9 +886,30 @@ 

                  ]

              },

              "projection": self._get_default_projection(

-                 srpm_names=srpm_name_to_nvr.keys())

+                 srpm_names=srpm_name_to_nvr.keys(),

+                 include_rpms=include_rpms)

          }

  

+         if content_sets:

+             image_request["query"]["$and"].append(

+                 {

+                     "$or": [{

+                         "field": "content_sets.*",

+                         "op": "=",

+                         "rvalue": r

+                     } for r in content_sets]

+                 })

+ 

+         if srpm_nvrs:

+             image_request["query"]["$and"].append(

+                 {

+                     "$or": [{

+                         "field": "rpm_manifest.*.rpms.*.srpm_name",

+                         "op": "=",

+                         "rvalue": srpm_name

+                     } for srpm_name in srpm_name_to_nvr.keys()]

+                 })

+ 

          if published is not None:

              image_request["query"]["$and"].append(

                  {

file modified
+47

@@ -41,6 +41,7 @@ 

  from freshmaker.parsers.internal.manual_rebuild import FreshmakerManualRebuildParser

  from freshmaker.monitor import (

      monitor_api, freshmaker_build_api_latency, freshmaker_event_api_latency)

+ from freshmaker.image_verifier import ImageVerifier

  

  api_v1 = {

      'event_types': {

@@ -131,6 +132,22 @@ 

                  'methods': ['GET'],

              }

          },

+     },

+     'verify_image': {

+         'verify_image': {

+             'url': '/api/1/verify-image/<image>',

+             'options': {

+                 'methods': ['GET'],

+             }

+         },

+     },

+     'verify_image_repository': {

+         'verify_image_repository': {

+             'url': '/api/1/verify-image-repository/<project>/<repo>',

+             'options': {

+                 'methods': ['GET'],

+             }

+         },

      }

  }

  

@@ -294,6 +311,34 @@ 

          return jsonify(json), 200

  

  

+ class VerifyImageAPI(MethodView):

+     def get(self, image):

+         if not image:

+             raise ValueError("No image name provided")

+ 

+         verifier = ImageVerifier()

+         images = verifier.verify_image(image)

+         ret = {}

+         ret["msg"] = ("Found %d images which are handled by Freshmaker for defined "

+                       "content_sets." % len(images))

+         ret["images"] = images

+         return jsonify(ret), 200

+ 

+ 

+ class VerifyImageRepositoryAPI(MethodView):

+     def get(self, project, repo):

+         if not project and not repo:

+             raise ValueError("No image repository name provided")

+ 

+         verifier = ImageVerifier()

+         images = verifier.verify_repository("%s/%s" % (project, repo))

+         ret = {}

+         ret["msg"] = ("Found %d images which are handled by Freshmaker for defined "

+                       "content_sets." % len(images))

+         ret["images"] = images

+         return jsonify(ret), 200

+ 

+ 

  API_V1_MAPPING = {

      'events': EventAPI,

      'builds': BuildAPI,

@@ -301,6 +346,8 @@ 

      'build_types': BuildTypeAPI,

      'build_states': BuildStateAPI,

      'about': AboutAPI,

+     'verify_image': VerifyImageAPI,

+     'verify_image_repository': VerifyImageRepositoryAPI,

  }

  

  

@@ -0,0 +1,146 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in

+ # all copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import six

+ from mock import MagicMock

+ 

+ from freshmaker.image_verifier import ImageVerifier

+ from tests import helpers

+ 

+ 

+ class TestImageVerifier(helpers.FreshmakerTestCase):

+ 

+     def setUp(self):

+         super(TestImageVerifier, self).setUp()

+         self.lb = MagicMock()

+         self.verifier = ImageVerifier(self.lb)

+ 

+     def test_verify_repository_no_repo(self):

+         self.lb.find_container_repositories.return_value = None

+         six.assertRaisesRegex(

+             self, ValueError, r'Cannot get repository.*',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_get_verify_repository_multiple_repos(self):

+         self.lb.find_container_repositories.return_value = ["foo", "bar"]

+         six.assertRaisesRegex(

+             self, ValueError, r'Multiple records found.*',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository_deprecated(self):

+         self.lb.find_container_repositories.return_value = [{

+             "release_categories": "Deprecated",

+             "published": True,

+             "auto_rebuild_tags": "latest"}]

+         six.assertRaisesRegex(

+             self, ValueError, r'.*but found \'Deprecated\'.',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository_not_published(self):

+         self.lb.find_container_repositories.return_value = [{

+             "release_categories": "Generally Available",

+             "published": False,

+             "auto_rebuild_tags": "latest"}]

+         six.assertRaisesRegex(

+             self, ValueError, r'.*is not published.',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository_no_auto_rebuild_tags(self):

+         self.lb.find_container_repositories.return_value = [{

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": []}]

+         six.assertRaisesRegex(

+             self, ValueError, r'.*this repository are disabled.',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository_no_images(self):

+         self.lb.find_container_repositories.return_value = [{

+             "repository": "foo/bar",

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": ["latest"]}]

+         self.lb.get_images_by_nvrs.return_value = []

+         six.assertRaisesRegex(

+             self, ValueError, r'No published images tagged by.*',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository_no_content_sets(self):

+         self.lb.find_container_repositories.return_value = [{

+             "repository": "foo/bar",

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": ["latest"]}]

+         self.lb.find_images_with_included_srpms.return_value = [{

+             "brew": {"build": "foo-1-1"},

+             "content_sets": []}]

+         six.assertRaisesRegex(

+             self, ValueError, r'.*are not set for this image.',

+             self.verifier.verify_repository, "foo/bar")

+ 

+     def test_verify_repository(self):

+         self.lb.find_container_repositories.return_value = [{

+             "repository": "foo/bar",

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": ["latest"]}]

+         self.lb.find_images_with_included_srpms.return_value = [{

+             "brew": {"build": "foo-1-1"},

+             "content_sets": ["content-set"]}]

+         ret = self.verifier.verify_repository("foo/bar")

+         self.assertEqual(ret, {"foo-1-1": ["content-set"]})

+ 

+     def test_get_verify_image(self):

+         self.lb.find_container_repositories.return_value = [{

+             "repository": "foo/bar",

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": ["latest"]}]

+         self.lb.get_images_by_nvrs.return_value = [{

+             "brew": {"build": "foo-1-1"},

+             "content_sets": ["content-set"]}]

+         ret = self.verifier.verify_image("foo-1-1")

+         self.assertEqual(ret, {"foo-1-1": ["content-set"]})

+ 

+     def test_get_verify_image_no_repo(self):

+         self.lb.find_container_repositories.return_value = []

+         six.assertRaisesRegex(

+             self, ValueError, r'Cannot get repository.*',

+             self.verifier.verify_image, "foo/bar")

+ 

+     def test_get_verify_image_multiple_repos(self):

+         self.lb.find_container_repositories.return_value = ["foo", "bar"]

+         six.assertRaisesRegex(

+             self, ValueError, r'.*found in multiple repositories in Lightblue.',

+             self.verifier.verify_image, "foo/bar")

+ 

+     def test_verify_image_no_images(self):

+         self.lb.find_container_repositories.return_value = [{

+             "repository": "foo/bar",

+             "release_categories": "Generally Available",

+             "published": True,

+             "auto_rebuild_tags": ["latest"]}]

+         self.lb.get_images_by_nvrs.return_value = []

+         six.assertRaisesRegex(

+             self, ValueError, r'No published images tagged by.*',

+             self.verifier.verify_image, "foo/bar")

file modified
+20 -20

@@ -969,33 +969,38 @@ 

                      {

                          "$or": [

                              {

-                                 "field": "content_sets.*",

+                                 "field": "repositories.*.tags.*.name",

                                  "op": "=",

-                                 "rvalue": "content-set-1"

+                                 "rvalue": "latest"

                              },

                              {

-                                 "field": "content_sets.*",

+                                 "field": "repositories.*.tags.*.name",

                                  "op": "=",

-                                 "rvalue": "content-set-2"

+                                 "rvalue": "tag1"

                              },

-                         ],

-                     },

-                     {

-                         "$or": [

                              {

                                  "field": "repositories.*.tags.*.name",

                                  "op": "=",

-                                 "rvalue": "latest"

+                                 "rvalue": "tag2"

                              },

+                         ],

+                     },

+                     {

+                         "field": "parsed_data.files.*.key",

+                         "op": "=",

+                         "rvalue": "buildfile"

+                     },

+                     {

+                         "$or": [

                              {

-                                 "field": "repositories.*.tags.*.name",

+                                 "field": "content_sets.*",

                                  "op": "=",

-                                 "rvalue": "tag1"

+                                 "rvalue": "content-set-1"

                              },

                              {

-                                 "field": "repositories.*.tags.*.name",

+                                 "field": "content_sets.*",

                                  "op": "=",

-                                 "rvalue": "tag2"

+                                 "rvalue": "content-set-2"

                              },

                          ],

                      },

@@ -1009,11 +1014,6 @@ 

                          ],

                      },

                      {

-                         "field": "parsed_data.files.*.key",

-                         "op": "=",

-                         "rvalue": "buildfile"

-                     },

-                     {

                          "field": "repositories.*.published",

                          "op": "=",

                          "rvalue": True

@@ -1028,8 +1028,8 @@ 

          # the tags criteria in order to assert with expected value.

          args, _ = cont_images.call_args

          request_arg = args[0]

-         tags_criteira = request_arg['query']['$and'][1]['$or']

-         request_arg['query']['$and'][1]['$or'] = sorted(

+         tags_criteira = request_arg['query']['$and'][0]['$or']

+         request_arg['query']['$and'][0]['$or'] = sorted(

              tags_criteira, key=lambda item: item['rvalue'])

  

          self.assertEqual(expected_image_request, request_arg)

file modified
+19

@@ -413,6 +413,25 @@ 

          # version is 'unknown' in case of skip_install=True in tox.ini

          self.assertEqual(data['version'], 'unknown')

  

+     @patch("freshmaker.views.ImageVerifier")

+     def test_verify_image(self, verifier):

+         verifier.return_value.verify_image.return_value = {"foo-1-1": ["content-set"]}

+         resp = self.client.get('/api/1/verify-image/foo-1-1')

+         data = json.loads(resp.get_data(as_text=True))

+         self.assertEqual(data, {

+             'images': {'foo-1-1': ['content-set']},

+             'msg': 'Found 1 images which are handled by Freshmaker for defined content_sets.'})

+ 

+     @patch("freshmaker.views.ImageVerifier")

+     def test_verify_image_repository(self, verifier):

+         verifier.return_value.verify_repository.return_value = {

+             "foo-1-1": ["content-set"]}

+         resp = self.client.get('/api/1/verify-image-repository/foo/bar')

+         data = json.loads(resp.get_data(as_text=True))

+         self.assertEqual(data, {

+             'images': {'foo-1-1': ['content-set']},

+             'msg': 'Found 1 images which are handled by Freshmaker for defined content_sets.'})

+ 

  

  class TestViewsMultipleFilterValues(helpers.ModelsTestCase):

      def setUp(self):

These are used to verify whether all conditions are satisfied for particular
container image repository or container image. This will allow container
image maintainers to check if their images are set up properly to be considered
for a rebuild by Freshmaker.

This saves me lot of time when debugging these issues with maintainers.

Instead of returning a list of repository objects, suggest to just return one. This could make it easier to understand the method which calls this one.

rebased onto 52893be

4 months ago

Updated based on the @cqi comment.

like return type? you alredy specified that in the return clause.

@jkaluza generally looks good, only minor nitpick with regards to the double documentation of return types.

That comes from docstring, for example: https://thomas-cokelaer.info/tutorials/sphinx/docstring_python.html.

But I admit I/we use ":param type name:" which is not mentioned there... :( Probably it does not make sense to use :rtype: either.

Pull-Request has been merged by jkaluza

4 months ago