#338 Initial code to send rebuild requests to Bob on Errata advisory state change.
Merged 3 months ago by jkaluza. Opened 3 months ago by jkaluza.
jkaluza/freshmaker bob  into  master

file modified
+1

@@ -320,6 +320,7 @@ 

      }

  

      KRB_AUTH_CCACHE_FILE = "freshmaker_cc_$pid_$tid"

+     ERRATA_TOOL_SERVER_URL = "http://localhost/"  # fake URL just for tests.

  

  

  class ProdConfiguration(BaseConfiguration):

file modified
+20

@@ -294,6 +294,26 @@ 

              'type': str,

              'default': '',

              'desc': 'Password to login Pulp.'},

+         'pulp_docker_server_url': {

+             'type': str,

+             'default': '',

+             'desc': 'Server URL of Pulp Docker.'},

+         'pulp_docker_username': {

+             'type': str,

+             'default': '',

+             'desc': 'Username to login Pulp Docker.'},

+         'pulp_docker_password': {

+             'type': str,

+             'default': '',

+             'desc': 'Password to login Pulp Docker.'},

+         'bob_server_url': {

+             'type': str,

+             'default': '',

+             'desc': 'Server URL of Bob container images rebuild service.'},

+         'bob_auth_token': {

+             'type': str,

+             'default': '',

+             'desc': 'Auth token for Bob container images rebuild service.'},

          'odcs_server_url': {

              'type': str,

              'default': '',

file modified
+45

@@ -21,10 +21,12 @@ 

  #

  # Written by Jan Kaluza <jkaluza@redhat.com>

  

+ import xmlrpclib

  import os

  import requests

  import dogpile.cache

  from requests_kerberos import HTTPKerberosAuth

+ from kobo.xmlrpc import SafeCookieTransport

  

  from freshmaker.events import (

      BrewSignRPMEvent, ErrataBaseEvent,

@@ -117,6 +119,10 @@ 

          else:

              self.server_url = conf.errata_tool_server_url.rstrip('/')

  

+         xmlrpc_url = self.server_url + '/errata/xmlrpc.cgi'

+         self.xmlrpc = xmlrpclib.ServerProxy(

+             xmlrpc_url, transport=SafeCookieTransport())

+ 

      def _errata_authorized_get(self, *args, **kwargs):

          r = requests.get(

              *args,

@@ -175,6 +181,45 @@ 

  

          return advisories

  

+     def get_docker_repo_tags(self, errata_id):

+         """

+         Get ET repo/tag configuration using XML-RPC call

+         get_advisory_cdn_docker_file_list

+         :param int errata_id: Errata advisory ID.

+         :rtype: dict

+         :return: Dict of advisory builds with repo and tag config:

+             {

+                 'build_NVR': {

+                     'cdn_repo1': [

+                         'tag1',

+                         'tag2'

+                     ],

+                     ...

+                 },

+                 ...

+             }

+         """

+         try:

+             response = self.xmlrpc.get_advisory_cdn_docker_file_list(

+                 errata_id)

+         except Exception:

+             log.exception("Canot call XMLRPC get_advisory_cdn_docker_file_list call.")

+             return None

+         if response is None:

+             log.warning("The get_advisory_cdn_docker_file_list XMLRPC call "

+                         "returned None.")

+             return None

+ 

+         repo_tags = dict()

+         for build_nvr in response:

+             if build_nvr not in repo_tags:

+                 repo_tags[build_nvr] = dict()

+             repos = response[build_nvr]['docker']['target']['repos']

+             for repo in repos:

+                 tags = repos[repo]['tags']

+                 repo_tags[build_nvr][repo] = tags

+         return repo_tags

+ 

      def advisories_from_event(self, event):

          """

          Returns list of ErrataAdvisory instances associated with

@@ -0,0 +1,22 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2017  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ 

+ from .rebuild_images_on_image_advisory_change import RebuildImagesOnImageAdvisoryChange  # noqa

@@ -0,0 +1,113 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import requests

+ 

+ from freshmaker import conf, db

+ from freshmaker.models import Event

+ from freshmaker.errata import Errata

+ from freshmaker.pulp import Pulp

+ from freshmaker.events import ErrataAdvisoryStateChangedEvent

+ from freshmaker.handlers import ContainerBuildHandler, fail_event_on_handler_exception

+ from freshmaker.types import EventState

+ 

+ 

+ class RebuildImagesOnImageAdvisoryChange(ContainerBuildHandler):

+     name = 'RebuildImagesOnImageAdvisoryChange'

+ 

+     def can_handle(self, event):

+         return isinstance(event, ErrataAdvisoryStateChangedEvent)

+ 

+     @fail_event_on_handler_exception

+     def handle(self, event):

+         if event.dry_run:

+             self.force_dry_run()

+ 

+         db_event = Event.get_or_create_from_event(db.session, event)

+         self.set_context(db_event)

+ 

+         # Check if we are allowed to build this advisory.

+         if not event.is_allowed(self):

+             msg = ("Errata advisory {0} is not allowed by internal policy "

+                    "to trigger Bob rebuilds.".format(event.advisory.errata_id))

+             db_event.transition(EventState.SKIPPED, msg)

+             db.session.commit()

+             self.log_info(msg)

+             return []

+ 

+         self.rebuild_images_depending_on_advisory(

+             db_event, event.advisory.errata_id)

+ 

+     def rebuild_images_depending_on_advisory(self, db_event, errata_id):

+         """

+         Submits requests to Bob to rebuild the images depending on the

+         images updated in the advisory with ID `errata_id`.

+         """

+         # Get the list of CDN repository names for each build in the advisory

+         # as well as the name of tags used for the images.

+         errata = Errata()

+         repo_tags = errata.get_docker_repo_tags(errata_id)

+         if not repo_tags:

+             msg = "No CDN repo found for advisory %r" % errata_id

+             self.log_info(msg)

+             db_event.transition(EventState.FAILED, msg)

+             db.session.commit()

+             return

+ 

+         # Use the Pulp to get the Docker repository name from the CDN repository

+         # name and store it into `docker_repos` dict.

+         pulp = Pulp(conf.pulp_docker_server_url, conf.pulp_docker_username,

+                     conf.pulp_docker_password)

+         # {docker_repository_name: [list, of, docker, tags], ...}

+         docker_repos = {}

+         for per_build_repo_tags in repo_tags.values():

+             for cdn_repo, docker_repo_tags in per_build_repo_tags.items():

+                 docker_repo = pulp.get_docker_repository_name(cdn_repo)

+                 if not docker_repo:

+                     self.log_error("No Docker repo found for CDN repo %r", cdn_repo)

+                     continue

+                 docker_repos[docker_repo] = docker_repo_tags

+ 

+         self.log_info("Found following Docker repositories updated by the advisory: %r",

+                       docker_repos.keys())

+ 

+         # Submit rebuild request to Bob :).

+         for repo_name in docker_repos.keys():

+             self.log_info("Requesting Bob rebuild of %s", repo_name)

+             bob_url = "%s/update_children/%s" % (

+                 conf.bob_server_url.rstrip('/'), repo_name)

+             headers = {"Authorization": "Bearer %s" % conf.bob_auth_token}

+             if self.dry_run:

+                 self.log_info("DRY RUN: Skipping request to Bob.")

+                 continue

+ 

+             r = requests.get(bob_url, headers=headers)

+             r.raise_for_status()

+             # TODO: Once the Bob API is clear here, we can handle the response,

+             # but for now just log it. This should also be changed to log_debug

+             # once we are in production, but for now log_info makes debugging

+             # this new code easier.

+             self.log_info("Response: %r", r.json())

+ 

+         db_event.transition(EventState.COMPLETE)

+         db.session.commit()

file modified
+27

@@ -42,6 +42,14 @@ 

          r.raise_for_status()

          return r.json()

  

+     def _rest_get(self, endpoint, **kwargs):

+         r = requests.get(

+             '{0}{1}'.format(self.rest_api_root, endpoint.lstrip('/')),

+             params=kwargs,

+             auth=(self.username, self.password))

+         r.raise_for_status()

+         return r.json()

+ 

      def get_content_set_by_repo_ids(self, repo_ids):

          """Get content_sets by repository IDs

  

@@ -60,3 +68,22 @@ 

          repos = self._rest_post('repositories/search/', json.dumps(query_data))

          return [repo['notes']['content_set'] for repo in repos

                  if 'content_set' in repo['notes']]

+ 

+     def get_docker_repository_name(self, cdn_repo):

+         """

+         Getting docker repository name from pulp using cdn repo name.

+ 

+         :param str cdn_repo: The CDN repo name from Errata Tool.

+         :rtype: str

+         :return: Docker repository name.

+         """

+         response = self._rest_get(

+             'repositories/%s/' % cdn_repo, distributors=True)

+ 

+         docker_repository_name = None

+         for distributor in response['distributors']:

+             if distributor['distributor_type_id'] == 'docker_distributor_web':

+                 docker_repository_name = \

+                     distributor['config']['repo-registry-id']

+                 break

+         return docker_repository_name

@@ -0,0 +1,22 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2016  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

@@ -0,0 +1,120 @@ 

+ # Copyright (c) 2019  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ from mock import patch, MagicMock, call

+ 

+ import freshmaker

+ from freshmaker.errata import ErrataAdvisory

+ from freshmaker.events import ErrataAdvisoryStateChangedEvent

+ from freshmaker.handlers.bob import RebuildImagesOnImageAdvisoryChange

+ from tests import helpers

+ 

+ 

+ class RebuildImagesOnImageAdvisoryChangeTest(helpers.ModelsTestCase):

+ 

+     def setUp(self):

+         super(RebuildImagesOnImageAdvisoryChangeTest, self).setUp()

+ 

+         self.event = ErrataAdvisoryStateChangedEvent(

+             "123",

+             ErrataAdvisory(123, "RHBA-2017", "SHIPPED_LIVE", [],

+                            security_impact="",

+                            product_short_name="product"))

+         self.handler = RebuildImagesOnImageAdvisoryChange()

+         self.db_event = MagicMock()

+ 

+     @patch.object(freshmaker.conf, 'handler_build_whitelist', new={

+         'RebuildImagesOnImageAdvisoryChange': {

+             "image": {"advisory_state": "SHIPPED_LIVE"}

+         }

+     })

+     @patch("freshmaker.handlers.bob.RebuildImagesOnImageAdvisoryChange."

+            "rebuild_images_depending_on_advisory")

+     def test_handler_allowed(self, rebuild_images):

+         self.event.advisory.state = "NEW_FILES"

+         self.handler.handle(self.event)

+         rebuild_images.assert_not_called()

+ 

+         self.event.advisory.state = "SHIPPED_LIVE"

+         self.handler.handle(self.event)

+         rebuild_images.assert_called_once()

+ 

+     @patch("freshmaker.errata.Errata.get_docker_repo_tags")

+     @patch("freshmaker.pulp.Pulp.get_docker_repository_name")

+     @patch("freshmaker.handlers.bob."

+            "rebuild_images_on_image_advisory_change.requests.get")

+     @patch.object(freshmaker.conf, 'bob_auth_token', new="x")

+     @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/")

+     def test_rebuild_images_depending_on_advisory(

+             self, requests_get, get_docker_repository_name,

+             get_docker_repo_tags):

+         get_docker_repo_tags.return_value = {

+             'foo-container-1-1': {'foo-526': ['5.26', 'latest']},

+             'bar-container-1-1': {'bar-526': ['5.26', 'latest']}}

+         get_docker_repository_name.side_effect = [

+             "scl/foo-526", "scl/bar-526"]

+         self.handler.rebuild_images_depending_on_advisory(self.db_event, 123)

+ 

+         get_docker_repo_tags.assert_called_once_with(123)

+         get_docker_repository_name.assert_has_calls([

+             call("bar-526"), call("foo-526")])

+         requests_get.assert_any_call(

+             'http://localhost/update_children/scl/foo-526',

+             headers={'Authorization': 'Bearer x'})

+         requests_get.assert_any_call(

+             'http://localhost/update_children/scl/bar-526',

+             headers={'Authorization': 'Bearer x'})

+ 

+     @patch("freshmaker.errata.Errata.get_docker_repo_tags")

+     @patch("freshmaker.pulp.Pulp.get_docker_repository_name")

+     @patch("freshmaker.handlers.bob."

+            "rebuild_images_on_image_advisory_change.requests.get")

+     @patch.object(freshmaker.conf, 'bob_auth_token', new="x")

+     @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/")

+     def test_rebuild_images_depending_on_advisory_unknown_advisory(

+             self, requests_get, get_docker_repository_name,

+             get_docker_repo_tags):

+         get_docker_repo_tags.return_value = None

+         self.handler.rebuild_images_depending_on_advisory(self.db_event, 123)

+ 

+         get_docker_repo_tags.assert_called_once_with(123)

+         get_docker_repository_name.assert_not_called()

+         requests_get.assert_not_called()

+ 

+     @patch("freshmaker.errata.Errata.get_docker_repo_tags")

+     @patch("freshmaker.pulp.Pulp.get_docker_repository_name")

+     @patch("freshmaker.handlers.bob."

+            "rebuild_images_on_image_advisory_change.requests.get")

+     @patch.object(freshmaker.conf, 'bob_auth_token', new="x")

+     @patch.object(freshmaker.conf, 'bob_server_url', new="http://localhost/")

+     def test_rebuild_images_depending_on_advisory_dry_run(

+             self, requests_get, get_docker_repository_name,

+             get_docker_repo_tags):

+         get_docker_repo_tags.return_value = {

+             'foo-container-1-1': {'foo-526': ['5.26', 'latest']}}

+         get_docker_repository_name.return_value = "scl/foo-526"

+         self.handler.force_dry_run()

+         self.handler.rebuild_images_depending_on_advisory(self.db_event, 123)

+ 

+         get_docker_repo_tags.assert_called_once_with(123)

+         get_docker_repository_name.assert_called_once_with("foo-526")

+         requests_get.assert_not_called()

file modified
+31

@@ -321,3 +321,34 @@ 

          MockedErrataAPI(errata_rest_get, errata_http_get)

          ret = self.errata.get_builds(28484, "RHEL-7")

          self.assertEqual(ret, set(['libntirpc-1.4.3-4.el7rhgs']))

+ 

+     def test_get_docker_repo_tags(self):

+         with patch.object(self.errata, "xmlrpc") as xmlrpc:

+             xmlrpc.get_advisory_cdn_docker_file_list.return_value = {

+                 'foo-container-1-1': {

+                     'docker': {

+                         'target': {

+                             'repos': {

+                                 'foo-526': {'tags': ['5.26', 'latest']}

+                             }

+                         }

+                     }

+                 }

+             }

+             repo_tags = self.errata.get_docker_repo_tags(28484)

+ 

+             expected = {'foo-container-1-1': {'foo-526': ['5.26', 'latest']}}

+             self.assertEqual(repo_tags, expected)

+ 

+     def test_get_docker_repo_tags_xmlrpc_exception(self):

+         with patch.object(self.errata, "xmlrpc") as xmlrpc:

+             xmlrpc.get_advisory_cdn_docker_file_list.side_effect = ValueError(

+                 "Expected XMLRPC test exception")

+             repo_tags = self.errata.get_docker_repo_tags(28484)

+             self.assertEqual(repo_tags, None)

+ 

+     def test_get_docker_repo_tags_xmlrpc_non_returned(self):

+         with patch.object(self.errata, "xmlrpc") as xmlrpc:

+             xmlrpc.get_advisory_cdn_docker_file_list.return_value = None

+             repo_tags = self.errata.get_docker_repo_tags(28484)

+             self.assertEqual(repo_tags, None)

file modified
+22

@@ -158,3 +158,25 @@ 

  

          self.assertEqual(['rhel-7-workstation-rpms', 'rhel-7-desktop-rpms'],

                           content_sets)

+ 

+     @patch('freshmaker.pulp.requests.get')

+     def test_get_docker_repository_name(self, get):

+         get.return_value.json.return_value = {

+             'display_name': 'foo-526',

+             'description': 'Foo',

+             'distributors': [

+                 {'repo_id': 'foo-526',

+                  'distributor_type_id': 'docker_distributor_web',

+                  'config': {'repo-registry-id': 'scl/foo-526'}}

+             ]

+         }

+ 

+         pulp = Pulp(self.server_url, username=self.username, password=self.password)

+         repo_name = pulp.get_docker_repository_name("foo-526")

+ 

+         get.assert_called_once_with(

+             '{}pulp/api/v2/repositories/foo-526/'.format(self.server_url),

+             params={"distributors": True},

+             auth=(self.username, self.password))

+ 

+         self.assertEqual(repo_name, "scl/foo-526")

This adds new handlers.bob.RebuildImagesOnImageAdvisoryChange handler which
handles the ErrataAdvisoryStateChangedEvent.

This handler uses Errata Tool and Pulp to get the list of Docker repositories
in which the images attached to advisory will end up (ended up). It then
requests Bob to rebuild images depending on them.

This also adds new pulp_docker_* config variables, because Docker images are
stored in different Pulp instance than RPMs.

Yeah, this should most likely be repo_name. Good catch. I'm going to fix this and extend the tests to catch this issue.

rebased onto 115c281

3 months ago

Commit 484ffba fixes this pull-request

Pull-Request has been merged by jkaluza

3 months ago

Pull-Request has been merged by jkaluza

3 months ago