#38 Provide method for getting repo/hash of images to rebuild
Merged 8 years ago by jkaluza. Opened 8 years ago by sochotni.
sochotni/freshmaker content_set_usage  into  master

file modified
+158
@@ -204,3 +204,161 @@ 

              image.update(image_data)

              images.append(image)

          return images

+ 

+     def find_repositories_with_content_sets(self,

+                                             content_sets,

+                                             published=True,

+                                             deprecated=False,

+                                             release_category="Generally Available"):

+         """Query lightblue and find containerRepositories which have content

+         from at least one of the content_sets. By default ignore unpublished,

+         deprecated repos or non-GA repositories

+ 

+         :param list content_sets: list of strings (content sets) to consider

+             when looking for the packages

+         :param bool published: whether to limit queries to published

+             repositories

+         :param bool deprecated: set to True to limit results to deprecated

+             repositories

+         :param str release_category: filter only repositories with specific

+             release category (options: Deprecated, Generally Available, Beta, Tech Preview)

+         """

+         repo_request = {

+             "objectType": "containerRepository",

+             "query": {

+                 "$and": [

+                     {

+                         "$or": [{

+                             "field": "content_sets.*",

+                             "op": "=",

+                             "rvalue": c

+                         } for c in content_sets]

+                     },

+                     {

+                         "field": "published",

+                         "op": "=",

+                         "rvalue": published

+                     },

+                     {

+                         "field": "deprecated",

+                         "op": "=",

+                         "rvalue": deprecated

+                     },

+                     {

+                         "field": "release_categories.*",

+                         "op": "=",

+                         "rvalue": release_category

+                     }

+                 ]

+             },

+             "projection": [

+                 {"field": "repository", "include": True},

+                 {"field": "content_sets", "include": True, "recursive": True}

+             ]

+         }

+         return self.find_container_repositories(repo_request)

+ 

+ 

+     def find_images_with_included_srpm(self, repositories, srpm_name,

+                                        published=True):

+ 

+         """Query lightblue and find containerImages in given

+         containerRepositories. By default limit only to images which have been

+         published to at least one repository and images which have latest tag.

+ 

+         :param dict repositories: dictionary with repository names to look inside

+         :param str srpm_name: srpm_name (source rpm name) to look for

+         :param bool published: whether to limit queries to images with at least

+             one published repository

+         """

+         image_request = {

+             "objectType": "containerImage",

+             "query": {

+                 "$and": [

+                     {

+                         "$or": [{

+                             "field": "repositories.*.repository",

+                             "op": "=",

+                             "rvalue": r['repository']

+                         } for r in repositories]

+                     },

+                     {

+                         "field": "repositories.*.published",

+                         "op": "=",

+                         "rvalue": published

+                     },

+                     {

+                         "field": "repositories.*.tags.*.name",

+                         "op": "=",

+                         "rvalue": "latest"

+                     },

+                     {

+                         "field": "parsed_data.rpm_manifest.*.srpm_name",

+                         "op": "=",

+                         "rvalue": srpm_name

+                     },

+                     {

+                         "field": "parsed_data.files.*.key",

+                         "op": "=",

+                         "rvalue": "buildfile"

+                     }

+                 ]

+             },

+             "projection": [

+                 {"field": "brew", "include": True, "recursive": True},

+                 {"field": "parsed_data.files", "include": True, "recursive": True},

+                 {"field": "parsed_data.rpm_manifest.*.srpm_nevra", "include": True, "recursive": True},

+                 {"field": "parsed_data.rpm_manifest.*.srpm_name", "include": True, "recursive": True}

+             ]

+         }

+         return self.find_container_images(image_request)

+ 

+ 

+     def find_images_with_package_from_content_set(self, srpm_name,

+                     content_sets,

+                     published=True,

+                     deprecated=False,

+                     release_category="Generally Available"):

+         """Query lightblue and find containers which contain given

+         package from one of content sets

+ 

+         :param str srpm_name: srpm_name (source rpm name) to look for

+         :param list content_sets: list of strings (content sets) to consider

+             when looking for the packages

+ 

+         :return: a list of dictionaries with three keys - repository, commit and

+             srpm_nevra. Repository is a name git repository including the

+             namespace. Commit is a git ref - usually a git commit

+             hash. srpm_nevra is whole NEVRA of source rpm that is included in

+             the given image - can be used for comparisons if needed

+         :rtype: list

+         """

+         repos = self.find_repositories_with_content_sets(content_sets,

+                                                          published=published,

+                                                          deprecated=deprecated,

+                                                          release_category=release_category)

+         if not repos:

+             return []

+         images = self.find_images_with_included_srpm(repos,

+                                                      srpm_name,

+                                                      published=published)

+         commits = []

+         for image in images:

+             for f in image["parsed_data"]["files"]:

+                 if f['key'] == 'buildfile':

+                     dockerfile_url = f['content_url']

+                     break

+ 

+             for rpm in image["parsed_data"]["rpm_manifest"]:

+                 print(rpm)

+                 if rpm["srpm_name"] == srpm_name:

+                     srpm_nevra = rpm['srpm_nevra']

+                     break

+ 

+             dockerfile, _, commit = dockerfile_url.partition("?id=")

+             _, _, reponame = dockerfile.partition("/cgit/")

+             reponame = reponame.replace("/plain/Dockerfile","")

+             commits.append({"repository": reponame,

+                             "commit": commit,

+                             "srpm_nevra": srpm_nevra})

+         return commits

file modified
+249 -2
@@ -23,8 +23,7 @@ 

  import json

  import unittest

  

- from mock import call

- from mock import patch

+ from mock import call, patch, Mock

  from six.moves import http_client

  

  from freshmaker.lightblue import ContainerImage
@@ -110,6 +109,77 @@ 

          self.fake_server_url = 'lightblue.localhost'

          self.fake_cert_file = 'path/to/cert'

          self.fake_private_key = 'path/to/private-key'

+         self.fake_repositories_with_content_sets = [

+             {

+                 "repository": "product/repo1",

+                 "content_sets": ["dummy-content-set-1",

+                                  "dummy-content-set-2"]

+             },

+             {

+                 "repository": "product2/repo2",

+                 "content_sets": ["dummy-content-set-1"]

+             }

+         ]

+ 

+         self.fake_images_with_parsed_data = [

+             {

+                 'brew': {

+                     'completion_date': u'20170421T04:27:51.000-0400',

+                     'build': 'package-name-1-4-12.10',

+                     'package': 'package-name-1'

+                 },

+                 'parsed_data': {

+                     'files': [

+                         {

+                             'key': 'buildfile',

+                             'content_url': 'http://git.repo.com/cgit/rpms/repo-1/plain/Dockerfile?id=commit_hash1',

+                             'filename': u'Dockerfile'

+                         }

+                     ],

+                     'rpm_manifest': [

+                         {

+                             "srpm_name": "openssl",

+                             "srpm_nevra": "openssl-0:1.2.3-1.src"

+                         },

+                         {

+                             "srpm_name": "tespackage",

+                             "srpm_nevra": "testpackage-10:1.2.3-1.src"

+                         }

+                     ]

+                 }

+             },

+             {

+                 'brew': {

+                     'completion_date': u'20170421T04:27:51.000-0400',

+                     'build': 'package-name-2-4-12.10',

+                     'package': 'package-name-2'

+                 },

+                 'parsed_data': {

+                     'files': [

+                         {

+                             'key': 'buildfile',

+                             'content_url': 'http://git.repo.com/cgit/ns/repo-2/plain/Dockerfile?id=commit_hash2',

+                             'filename': 'Dockerfile'

+                         },

+                         {

+                             'key': 'bogusfile',

+                             'content_url': 'bogus_test_url',

+                             'filename': 'bogus.file'

+                         }

+                     ],

+                     'rpm_manifest': [

+                         {

+                             "srpm_name": "openssl",

+                             "srpm_nevra": "openssl-1:1.2.3-1.src"

+                         },

+                         {

+                             "srpm_name": "tespackage2",

+                             "srpm_nevra": "testpackage2-10:1.2.3-1.src"

+                         }

+                     ]

+                 }

+             },

+         ]

  

      @patch('freshmaker.lightblue.requests.post')

      def test_find_container_images(self, post):
@@ -266,6 +336,183 @@ 

              self.assertRaises(LightBlueRequestFailure,

                                lb._make_request, 'find/containerRepository/', fake_request)

  

+     @patch('freshmaker.lightblue.LightBlue.find_container_repositories')

+     @patch('os.path.exists')

+     def test_find_repositories_with_content_sets(self, exists,

+                                                  cont_repos):

+         exists.return_value = True

+         queried_content_set = "rhel-7-server-rpms"

+         cont_repos.return_value = self.fake_repositories_with_content_sets

+         lb = LightBlue(server_url=self.fake_server_url,

+                        cert=self.fake_cert_file,

+                        private_key=self.fake_private_key)

+         ret = lb.find_repositories_with_content_sets([queried_content_set])

+         expected_repo_request = {

+             "objectType": "containerRepository",

+             "query": {

+                 "$and": [

+                     {

+                         "$or": [

+                             {

+                                 "field": "content_sets.*",

+                                 "op": "=",

+                                 "rvalue": queried_content_set

+                             }

+                         ],

+                     },

+                     {

+                         "field": "published",

+                         "op": "=",

+                         "rvalue": True

+                     },

+                     {

+                         "field": "deprecated",

+                         "op": "=",

+                         "rvalue": False

+                     },

+                     {

+                         "field": "release_categories.*",

+                         "op": "=",

+                         "rvalue": "Generally Available"

+                     }

+                 ]

+             },

+             "projection": [

+                 {"field": "repository", "include": True},

+                 {"field": "content_sets", "include": True, "recursive": True}

+ 

+             ]

+         }

+         cont_repos.assert_called_with(expected_repo_request)

+         self.assertEqual(ret, cont_repos.return_value)

+ 

+     @patch('freshmaker.lightblue.LightBlue.find_container_images')

+     @patch('os.path.exists')

+     def test_images_with_included_srpm(self, exists,

+                                        cont_images):

+ 

+         exists.return_value = True

+         lb = LightBlue(server_url=self.fake_server_url,

+                        cert=self.fake_cert_file,

+                        private_key=self.fake_private_key)

+         repositories = self.fake_repositories_with_content_sets

+         cont_images.return_value = self.fake_images_with_parsed_data

+         ret = lb.find_images_with_included_srpm(repositories,

+                                                 "openssl")

+ 

+         expected_image_request = {

+             "objectType": "containerImage",

+             "query": {

+                 "$and": [

+                     {

+                         "$or": [

+                             {

+                                 "field": "repositories.*.repository",

+                                 "op": "=",

+                                 "rvalue": "product/repo1"

+                             },

+                             {

+                                 "field": "repositories.*.repository",

+                                 "op": "=",

+                                 "rvalue": "product2/repo2"

+                             },

+                         ],

+                     },

+                     {

+                         "field": "repositories.*.published",

+                         "op": "=",

+                         "rvalue": True

+                     },

+                     {

+                         "field": "repositories.*.tags.*.name",

+                         "op": "=",

+                         "rvalue": "latest"

+                     },

+                     {

+                         "field": "parsed_data.rpm_manifest.*.srpm_name",

+                         "op": "=",

+                         "rvalue": "openssl"

+                     },

+                     {

+                         "field": "parsed_data.files.*.key",

+                         "op": "=",

+                         "rvalue": "buildfile"

+                     }

+                 ]

+             },

+             "projection": [

+                 {"field": "brew", "include": True, "recursive": True},

+                 {"field": "parsed_data.files", "include": True, "recursive": True},

+                 {"field": "parsed_data.rpm_manifest.*.srpm_nevra", "include": True, "recursive": True},

+                 {"field": "parsed_data.rpm_manifest.*.srpm_name", "include": True, "recursive": True}

+             ]

+         }

+         cont_images.assert_called_with(expected_image_request)

+         self.assertEqual(ret, cont_images.return_value)

+ 

+ 

+     @patch('freshmaker.lightblue.LightBlue.find_container_repositories')

+     @patch('freshmaker.lightblue.LightBlue.find_container_images')

+     @patch('os.path.exists')

+     def test_images_with_content_set_packages(self, exists,

+                                               cont_images,

+                                               cont_repos):

+ 

+         exists.return_value = True

+         cont_repos.return_value = self.fake_repositories_with_content_sets

+         cont_images.return_value = self.fake_images_with_parsed_data

+ 

+         lb = LightBlue(server_url=self.fake_server_url,

+                        cert=self.fake_cert_file,

+                        private_key=self.fake_private_key)

+         ret = lb.find_images_with_package_from_content_set("openssl",

+                                                            ["dummy-content-set-1"])

+ 

+         self.assertEqual(2,len(ret))

+         self.assertEqual(ret,

+                          [

+                              {

+                                  "repository": "rpms/repo-1",

+                                  "commit": "commit_hash1",

+                                  "srpm_nevra": "openssl-0:1.2.3-1.src"

+                              },

+                              {

+                                  "repository": "ns/repo-2",

+                                  "commit": "commit_hash2",

+                                  "srpm_nevra": "openssl-1:1.2.3-1.src"

+                              }

+                          ])

+ 

+     @patch('freshmaker.lightblue.LightBlue.find_container_repositories')

+     @patch('freshmaker.lightblue.LightBlue.find_container_images')

+     @patch('os.path.exists')

+     def test_images_with_content_set_packages_exception(self, exists,

+                                                         cont_images,

+                                                         cont_repos):

+ 

+         exists.return_value = True

+         cont_repos.side_effect = LightBlueRequestFailure(

+             {"errors": [{"msg": "dummy error"}]}, http_client.REQUEST_TIMEOUT)

+         cont_images.return_value = self.fake_images_with_parsed_data

+ 

+         lb = LightBlue(server_url=self.fake_server_url,

+                        cert=self.fake_cert_file,

+                        private_key=self.fake_private_key)

+         with self.assertRaises(LightBlueRequestFailure):

+             lb.find_images_with_package_from_content_set(

+                 "openssl",

+                 ["dummy-content-set-1"])

+ 

+         cont_repos.return_value = self.fake_repositories_with_content_sets

+         cont_images.side_effect = LightBlueRequestFailure(

+             {"errors": [{"msg": "dummy error"}]}, http_client.REQUEST_TIMEOUT)

+ 

+         with self.assertRaises(LightBlueRequestFailure):

+             lb.find_images_with_package_from_content_set(

+                 "openssl",

+                 ["dummy-content-set-1"])

+ 

+ 

  

  class TestEntityVersion(unittest.TestCase):

      """Test case for ensuring correct entity version in request"""

Given a source rpm package name and content sets where it will be
shipped - this new method will figure out which supported images contain
content from this source rpm it will return an array of tuples -
repository namespace/name and a git commit hash. One tuple is returned
for each container image which needs to be rebuilt to include the new
rpm content

Looks OK to me so far :).

rebased

8 years ago

rebased

8 years ago

+1

I think I will need these methods while writing code to implement image rebuild for internal. :)

@cqi: Yes, we now have cards in Jira for each step of the image rebuild plan and Stano took one of these in free time :).

@sochotni Do I need continue to do something based on ur patch? Feel free ping me if you don't have enough time to continue on this pr.

I should have time to finish this off. Specifically I'd like to do at least:
- split methods into separate commits
- add exception handling tests

Then I'll remove WIP

rebased

8 years ago

Pull-Request has been merged by jkaluza

8 years ago