From 52893be4b4f76a28a9c116777b4c3e82f4180a40 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Feb 21 2019 12:06:46 +0000 Subject: Add new "/verify-image" and "/verify-image-repository" API endpoints. These are used to verify whether all conditions are satisfied for particular container image repository or container image. This will allow container image maintainers to check if their images are set up properly to be considered for a rebuild by Freshmaker. This saves me lot of time when debugging these issues with maintainers. --- diff --git a/freshmaker/image_verifier.py b/freshmaker/image_verifier.py new file mode 100644 index 0000000..ba7ff76 --- /dev/null +++ b/freshmaker/image_verifier.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from freshmaker import conf +from freshmaker.lightblue import LightBlue + + +class ImageVerifier(object): + + def __init__(self, lb=None): + """ + Creates new ImageVerifier. + + :param LightBlue lb: Lightblue instance to use to verify images. + When None, new default Lightblue class is created. + """ + self.lb = lb if lb else LightBlue( + server_url=conf.lightblue_server_url, + cert=conf.lightblue_certificate, + private_key=conf.lightblue_private_key) + + def _verify_repository_data(self, repo): + """ + Verifies the Lightblue ContainerRepository data. + Raises ValueError in case of error. + """ + if "Generally Available" not in repo["release_categories"]: + raise ValueError( + "Only repositories with \"Generally Available\" release_categories can be " + "rebuilt, but found %r." % repo["release_categories"]) + + if not repo["published"]: + raise ValueError( + "Only published repositories can be rebuilt, but this repository is not " + "published.") + + if "auto_rebuild_tags" in repo and repo["auto_rebuild_tags"] == []: + raise ValueError( + "The \"auto_rebuild_tags\" in COMET is set to an empty list, this means " + "rebuilds of images in this repository are disabled.") + + def _verify_image_data(self, image): + """ + Verifies the Lightblue ContainerImage data. + Raises ValueError in case of error. + """ + if not image["content_sets"]: + raise ValueError( + "Found image \"%s\" in this repository, but it cannot be rebuilt, because " + "the \"content_sets\" are not set for this image." % image["brew"]["build"]) + + def _get_repository_from_name(self, repo_name): + """ + Returns the ContainerRepository object based on the Repository name. + """ + query = { + "objectType": "containerRepository", + "query": { + "$and": [ + { + "field": "repository", + "op": "=", + "rvalue": repo_name + }, + + ] + }, + "projection": [ + {"field": "*", "include": True, "recursive": True} + ] + } + + repos = self.lb.find_container_repositories(query) + if not repos: + raise ValueError("Cannot get repository %s from Lightblue." % repo_name) + if len(repos) != 1: + raise ValueError("Multiple records found in Lightblue for repository %s." % repo_name) + + return repos[0] + + def _get_repository_from_image(self, nvr): + """ + Returns the ContainerRepository object based on the image NVR. + """ + query = { + "objectType": "containerRepository", + "query": { + "$and": [ + { + "field": "images.*.brew.build", + "op": "=", + "rvalue": nvr + }, + + ] + }, + "projection": [ + {"field": "*", "include": True, "recursive": True} + ] + } + + repos = self.lb.find_container_repositories(query) + if not repos: + raise ValueError("Cannot get repository for image %s from Lightblue." % nvr) + if len(repos) != 1: + raise ValueError( + "Image %s found in multiple repositories in Lightblue." % nvr) + + return repos[0] + + def verify_image(self, image_nvr): + """ + Verifies the image defined by `image_nvr`. + Raises ValueError in case of error. + + :param str image_nvr: NVR of image to verify. + :rtype: dict + :return: Dict with image NVR as key and list of content_sets as values. + """ + repo = self._get_repository_from_image(image_nvr) + self._verify_repository_data(repo) + + images = self.lb.get_images_by_nvrs([image_nvr], include_rpms=False) + if not images: + raise ValueError( + "No published images tagged by %r found in repository" % ( + repo["auto_rebuild_tags"])) + + image = images[0] + self._verify_image_data(image) + + return { + image["brew"]["build"]: image["content_sets"] + } + + def verify_repository(self, repo_name): + """ + Verifies the images in repository defined by `repo_name`. + Raises ValueError in case of error. + + :param str repo_name: Name of repository to verify. + :rtype: dict + :return: Dict with image NVR as key and list of content_sets as values. + """ + repo = self._get_repository_from_name(repo_name) + self._verify_repository_data(repo) + + rebuildable_images = {} + images = self.lb.find_images_with_included_srpms( + [], [], {repo["repository"]: repo}, include_rpms=False) + for image in images: + nvr = image["brew"]["build"] + self._verify_image_data(image) + rebuildable_images[nvr] = image["content_sets"] + + if not rebuildable_images: + raise ValueError( + "No published images tagged by %r found in repository" % ( + repo["auto_rebuild_tags"])) + + return rebuildable_images diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 19d65bd..a9ab65e 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -843,7 +843,8 @@ class LightBlue(object): return ret def find_images_with_included_srpms( - self, content_sets, srpm_nvrs, repositories, published=True): + self, content_sets, srpm_nvrs, repositories, published=True, + include_rpms=True): """Query lightblue and find containerImages in given containerRepositories. By default limit only to images which have been published to at least one repository and images which have latest tag. @@ -854,6 +855,7 @@ class LightBlue(object): :param list repositories: List of repository names to look for. :param bool published: whether to limit queries to published repositories + :param bool include_rpms: whether to include the RPMs in the result. """ auto_rebuild_tags = set() for repo in repositories.values(): @@ -871,26 +873,12 @@ class LightBlue(object): "$and": [ { "$or": [{ - "field": "content_sets.*", - "op": "=", - "rvalue": r - } for r in content_sets] - }, - { - "$or": [{ "field": "repositories.*.tags.*.name", "op": "=", "rvalue": tag } for tag in auto_rebuild_tags] }, { - "$or": [{ - "field": "rpm_manifest.*.rpms.*.srpm_name", - "op": "=", - "rvalue": srpm_name - } for srpm_name in srpm_name_to_nvr.keys()] - }, - { "field": "parsed_data.files.*.key", "op": "=", "rvalue": "buildfile" @@ -898,9 +886,30 @@ class LightBlue(object): ] }, "projection": self._get_default_projection( - srpm_names=srpm_name_to_nvr.keys()) + srpm_names=srpm_name_to_nvr.keys(), + include_rpms=include_rpms) } + if content_sets: + image_request["query"]["$and"].append( + { + "$or": [{ + "field": "content_sets.*", + "op": "=", + "rvalue": r + } for r in content_sets] + }) + + if srpm_nvrs: + image_request["query"]["$and"].append( + { + "$or": [{ + "field": "rpm_manifest.*.rpms.*.srpm_name", + "op": "=", + "rvalue": srpm_name + } for srpm_name in srpm_name_to_nvr.keys()] + }) + if published is not None: image_request["query"]["$and"].append( { diff --git a/freshmaker/views.py b/freshmaker/views.py index bfdefdb..3b9cc83 100644 --- a/freshmaker/views.py +++ b/freshmaker/views.py @@ -41,6 +41,7 @@ from freshmaker.auth import login_required, requires_role, require_scopes from freshmaker.parsers.internal.manual_rebuild import FreshmakerManualRebuildParser from freshmaker.monitor import ( monitor_api, freshmaker_build_api_latency, freshmaker_event_api_latency) +from freshmaker.image_verifier import ImageVerifier api_v1 = { 'event_types': { @@ -131,6 +132,22 @@ api_v1 = { 'methods': ['GET'], } }, + }, + 'verify_image': { + 'verify_image': { + 'url': '/api/1/verify-image/', + 'options': { + 'methods': ['GET'], + } + }, + }, + 'verify_image_repository': { + 'verify_image_repository': { + 'url': '/api/1/verify-image-repository//', + 'options': { + 'methods': ['GET'], + } + }, } } @@ -294,6 +311,34 @@ class AboutAPI(MethodView): return jsonify(json), 200 +class VerifyImageAPI(MethodView): + def get(self, image): + if not image: + raise ValueError("No image name provided") + + verifier = ImageVerifier() + images = verifier.verify_image(image) + ret = {} + ret["msg"] = ("Found %d images which are handled by Freshmaker for defined " + "content_sets." % len(images)) + ret["images"] = images + return jsonify(ret), 200 + + +class VerifyImageRepositoryAPI(MethodView): + def get(self, project, repo): + if not project and not repo: + raise ValueError("No image repository name provided") + + verifier = ImageVerifier() + images = verifier.verify_repository("%s/%s" % (project, repo)) + ret = {} + ret["msg"] = ("Found %d images which are handled by Freshmaker for defined " + "content_sets." % len(images)) + ret["images"] = images + return jsonify(ret), 200 + + API_V1_MAPPING = { 'events': EventAPI, 'builds': BuildAPI, @@ -301,6 +346,8 @@ API_V1_MAPPING = { 'build_types': BuildTypeAPI, 'build_states': BuildStateAPI, 'about': AboutAPI, + 'verify_image': VerifyImageAPI, + 'verify_image_repository': VerifyImageRepositoryAPI, } diff --git a/tests/test_image_verifier.py b/tests/test_image_verifier.py new file mode 100644 index 0000000..c53dfbf --- /dev/null +++ b/tests/test_image_verifier.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import six +from mock import MagicMock + +from freshmaker.image_verifier import ImageVerifier +from tests import helpers + + +class TestImageVerifier(helpers.FreshmakerTestCase): + + def setUp(self): + super(TestImageVerifier, self).setUp() + self.lb = MagicMock() + self.verifier = ImageVerifier(self.lb) + + def test_verify_repository_no_repo(self): + self.lb.find_container_repositories.return_value = None + six.assertRaisesRegex( + self, ValueError, r'Cannot get repository.*', + self.verifier.verify_repository, "foo/bar") + + def test_get_verify_repository_multiple_repos(self): + self.lb.find_container_repositories.return_value = ["foo", "bar"] + six.assertRaisesRegex( + self, ValueError, r'Multiple records found.*', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository_deprecated(self): + self.lb.find_container_repositories.return_value = [{ + "release_categories": "Deprecated", + "published": True, + "auto_rebuild_tags": "latest"}] + six.assertRaisesRegex( + self, ValueError, r'.*but found \'Deprecated\'.', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository_not_published(self): + self.lb.find_container_repositories.return_value = [{ + "release_categories": "Generally Available", + "published": False, + "auto_rebuild_tags": "latest"}] + six.assertRaisesRegex( + self, ValueError, r'.*is not published.', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository_no_auto_rebuild_tags(self): + self.lb.find_container_repositories.return_value = [{ + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": []}] + six.assertRaisesRegex( + self, ValueError, r'.*this repository are disabled.', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository_no_images(self): + self.lb.find_container_repositories.return_value = [{ + "repository": "foo/bar", + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": ["latest"]}] + self.lb.get_images_by_nvrs.return_value = [] + six.assertRaisesRegex( + self, ValueError, r'No published images tagged by.*', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository_no_content_sets(self): + self.lb.find_container_repositories.return_value = [{ + "repository": "foo/bar", + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": ["latest"]}] + self.lb.find_images_with_included_srpms.return_value = [{ + "brew": {"build": "foo-1-1"}, + "content_sets": []}] + six.assertRaisesRegex( + self, ValueError, r'.*are not set for this image.', + self.verifier.verify_repository, "foo/bar") + + def test_verify_repository(self): + self.lb.find_container_repositories.return_value = [{ + "repository": "foo/bar", + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": ["latest"]}] + self.lb.find_images_with_included_srpms.return_value = [{ + "brew": {"build": "foo-1-1"}, + "content_sets": ["content-set"]}] + ret = self.verifier.verify_repository("foo/bar") + self.assertEqual(ret, {"foo-1-1": ["content-set"]}) + + def test_get_verify_image(self): + self.lb.find_container_repositories.return_value = [{ + "repository": "foo/bar", + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": ["latest"]}] + self.lb.get_images_by_nvrs.return_value = [{ + "brew": {"build": "foo-1-1"}, + "content_sets": ["content-set"]}] + ret = self.verifier.verify_image("foo-1-1") + self.assertEqual(ret, {"foo-1-1": ["content-set"]}) + + def test_get_verify_image_no_repo(self): + self.lb.find_container_repositories.return_value = [] + six.assertRaisesRegex( + self, ValueError, r'Cannot get repository.*', + self.verifier.verify_image, "foo/bar") + + def test_get_verify_image_multiple_repos(self): + self.lb.find_container_repositories.return_value = ["foo", "bar"] + six.assertRaisesRegex( + self, ValueError, r'.*found in multiple repositories in Lightblue.', + self.verifier.verify_image, "foo/bar") + + def test_verify_image_no_images(self): + self.lb.find_container_repositories.return_value = [{ + "repository": "foo/bar", + "release_categories": "Generally Available", + "published": True, + "auto_rebuild_tags": ["latest"]}] + self.lb.get_images_by_nvrs.return_value = [] + six.assertRaisesRegex( + self, ValueError, r'No published images tagged by.*', + self.verifier.verify_image, "foo/bar") diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index 1f2a5db..5124152 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -969,33 +969,38 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): { "$or": [ { - "field": "content_sets.*", + "field": "repositories.*.tags.*.name", "op": "=", - "rvalue": "content-set-1" + "rvalue": "latest" }, { - "field": "content_sets.*", + "field": "repositories.*.tags.*.name", "op": "=", - "rvalue": "content-set-2" + "rvalue": "tag1" }, - ], - }, - { - "$or": [ { "field": "repositories.*.tags.*.name", "op": "=", - "rvalue": "latest" + "rvalue": "tag2" }, + ], + }, + { + "field": "parsed_data.files.*.key", + "op": "=", + "rvalue": "buildfile" + }, + { + "$or": [ { - "field": "repositories.*.tags.*.name", + "field": "content_sets.*", "op": "=", - "rvalue": "tag1" + "rvalue": "content-set-1" }, { - "field": "repositories.*.tags.*.name", + "field": "content_sets.*", "op": "=", - "rvalue": "tag2" + "rvalue": "content-set-2" }, ], }, @@ -1009,11 +1014,6 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): ], }, { - "field": "parsed_data.files.*.key", - "op": "=", - "rvalue": "buildfile" - }, - { "field": "repositories.*.published", "op": "=", "rvalue": True @@ -1028,8 +1028,8 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): # the tags criteria in order to assert with expected value. args, _ = cont_images.call_args request_arg = args[0] - tags_criteira = request_arg['query']['$and'][1]['$or'] - request_arg['query']['$and'][1]['$or'] = sorted( + tags_criteira = request_arg['query']['$and'][0]['$or'] + request_arg['query']['$and'][0]['$or'] = sorted( tags_criteira, key=lambda item: item['rvalue']) self.assertEqual(expected_image_request, request_arg) diff --git a/tests/test_views.py b/tests/test_views.py index 68d718a..b4bc55b 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -413,6 +413,25 @@ class TestViews(helpers.ModelsTestCase): # version is 'unknown' in case of skip_install=True in tox.ini self.assertEqual(data['version'], 'unknown') + @patch("freshmaker.views.ImageVerifier") + def test_verify_image(self, verifier): + verifier.return_value.verify_image.return_value = {"foo-1-1": ["content-set"]} + resp = self.client.get('/api/1/verify-image/foo-1-1') + data = json.loads(resp.get_data(as_text=True)) + self.assertEqual(data, { + 'images': {'foo-1-1': ['content-set']}, + 'msg': 'Found 1 images which are handled by Freshmaker for defined content_sets.'}) + + @patch("freshmaker.views.ImageVerifier") + def test_verify_image_repository(self, verifier): + verifier.return_value.verify_repository.return_value = { + "foo-1-1": ["content-set"]} + resp = self.client.get('/api/1/verify-image-repository/foo/bar') + data = json.loads(resp.get_data(as_text=True)) + self.assertEqual(data, { + 'images': {'foo-1-1': ['content-set']}, + 'msg': 'Found 1 images which are handled by Freshmaker for defined content_sets.'}) + class TestViewsMultipleFilterValues(helpers.ModelsTestCase): def setUp(self):