PR#531: Get content_sets for all architectures - freshmaker

freshmaker

#531 Get content_sets for all architectures

Merged 4 years ago by gnaponie. Opened 4 years ago by gnaponie.

gnaponie/freshmaker CLOUDWF-1561 into master

Get content_sets for all architectures

Giulia Naponiello • 4 years ago

370532e

freshmaker/lightblue.py

file modified

+20 -19

		`@@ -594,6 +594,26 @@`
		`arch_image.update_multi_arch(image)`
		`image.update_multi_arch(arch_image)`

		`+ # There can be multi-arch images which share the same`
		`+ # image['brew']['build']. Freshmaker is not interested in the image`
		`+ # architecture, it is only interested in NVR, so group the images`
		`+ # by the same image['brew']['build'] and include just first one in the`
		`+ # image list.`
		`+ sorted_images = sorted_by_nvr(images, reverse=True)`
		`+ images = []`
		`+`
		`+ # We must combine content_sets with same image NVR`
		`+ # but different architectures into one content_sets field`
		`+ for k, temp_images in groupby(sorted_images, key=lambda item: item.nvr):`
		`+ temp_images = list(temp_images)`
		`+ img = temp_images[0]`
		`+ if 'content_sets' in img and len(temp_images) > 1:`
		`+ new_content_sets = set(img.get('content_sets'))`
		`+ for i in temp_images[1:]:`
		`+ new_content_sets.update(i.get('content_sets', []))`
		`+ img["content_sets"] = list(new_content_sets)`
		`+ images.append(img)`
		`+`
		`return images`

		`def _set_container_repository_filters(`
		`@@ -1161,25 +1181,6 @@`
		# therefore set `published` to None.
		`images = self.get_images_by_nvrs(`
		`leaf_container_images, None, content_sets, srpm_nvrs)`
		`- # There can be multi-arch images which share the same`
		`- # image['brew']['build']. Freshmaker is not interested in the image`
		`- # architecture, it is only interested in NVR, so group the images`
		`- # by the same image['brew']['build'] and include just first one in the`
		`- # image list.`
		`- sorted_images = sorted_by_nvr(images, reverse=True)`
		`- images = []`
		`-`
		`- # We must combine content_sets with same image NVR`
		`- # but different architectures into one content_sets field`
		`- for k, temp_images in groupby(sorted_images, key=lambda item: item.nvr):`
		`- temp_images = list(temp_images)`
		`- img = temp_images[0]`
		`- if 'content_sets' in img and len(temp_images) > 1:`
		`- new_content_sets = set(img.get('content_sets'))`
		`- for i in temp_images[1:]:`
		`- new_content_sets.update(i.get('content_sets', []))`
		`- img["content_sets"] = list(new_content_sets)`
		`- images.append(img)`

		`# In case we query for unpublished images, we need to return just`
		`# the latest NVR for given name-version, otherwise images would`

tests/test_lightblue.py

file modified

+12 -5

		`@@ -834,6 +834,10 @@`
		`self.assertEqual(2, len(images))`

		`image = images[0]`
		`+ self.assertEqual('57ea8d289c624c035f96f4db', image['_id'])`
		`+ self.assertEqual('sadc-docker',`
		`+ image['brew']['package'])`
		`+ image = images[1]`
		`self.assertEqual('57ea8d1f9c624c035f96f4b0', image['_id'])`
		`self.assertEqual('jboss-webserver-3-webserver30-tomcat7-openshift-docker',`
		`image['brew']['package'])`
		`@@ -858,6 +862,7 @@`
		`'build': 'sadc-container-1.1-6',`
		`'package': 'sadc-container',`
		`},`
		`+ 'content_sets': ['dummy-content-set-1'],`
		`},`
		`{`
		`'_id': '57ea8d289c624c035f96f4db',`
		`@@ -868,6 +873,7 @@`
		`'build': 'sadc-container-1.1-6',`
		`'package': 'sadc-container',`
		`},`
		`+ 'content_sets': ['dummy-content-set-2'],`
		`}`
		`],`
		`'status': 'COMPLETE',`
		`@@ -897,7 +903,7 @@`
		`cert=(self.fake_cert_file, self.fake_private_key),`
		`headers={'Content-Type': 'application/json'}`
		`)`
		`- self.assertEqual(2, len(images))`
		`+ self.assertEqual(1, len(images))`
		`# Verify update_multi_arch is first called with the second image,`
		`# then with the first image. This is to ensure all ContainerImage`
		`# objects for the same Brew build have the same multi arch data.`
		`@@ -2032,9 +2038,10 @@`
		`new_images = [ContainerImage.create(i) for i in new_images]`
		`find_repos.return_value = self.fake_repositories_with_content_sets`
		`find_images.return_value = self.fake_container_images + new_images`
		`- right_content_sets = [["dummy-content-set-1"],`
		`- ["dummy-content-set-1", "dummy-content-set-2"],`
		`- ["content-set-1", "content-set-2", "content-set-3"]]`
		`+ right_content_sets = [["dummy-content-set-1", "dummy-content-set-2"],`
		`+ ["dummy-content-set-1"],`
		`+ ["content-set-1", "content-set-2"],`
		`+ ["content-set-2", "content-set-3"]]`
		`with patch('os.path.exists'):`
		`lb = LightBlue(server_url=self.fake_server_url,`
		`cert=self.fake_cert_file,`
		`@@ -2044,7 +2051,7 @@`
		`["content-set-1", "content-set-2", "content-set-3"],`
		`leaf_container_images=['placeholder'])`

		`- self.assertEqual(3, len(ret))`
		`+ self.assertEqual(4, len(ret))`
		`images_content_sets = [sorted(i.get('content_sets', ['!'])) for i in ret]`
		`self.assertEqual(images_content_sets, right_content_sets)`

gnaponie commented 4 years ago

When setting generate_pulp_repos to True, we should ensure that the
content_sets for the one image contains the content sets for all the
architectures of that image. Let's do this for every container image
returned from LightBlue, and not that late in the process instead.

ref: CLOUDWF-1561

Signed-off-by: Giulia Naponiello gnaponie@redhat.com

gnaponie commented 4 years ago

@jkaluza could you review? I'm not so sure about the tests... but it makes sense based on the new code.