PR#299: Renew ODCS composes which have been used in the original image build. - freshmaker

freshmaker

#299 Renew ODCS composes which have been used in the original image build.

Merged 5 years ago by jkaluza. Opened 5 years ago by jkaluza.

jkaluza/freshmaker renew-older-composes into master

Renew ODCS composes which have been used in the original image build.

Jan Kaluza • 5 years ago

3c7cf21

freshmaker/handlers/__init__.py

file modified

+10 -6

		`@@ -466,19 +466,23 @@`

		`release = parse_NVR(build.rebuilt_nvr)["release"]`

		`+ # Get the list of ODCS compose IDs which should be used to build`
		`+ # the image.`
		`+ compose_ids = []`
		`+ for relation in build.composes:`
		`+ compose_ids.append(relation.compose.odcs_compose_id)`
		`+ if args["renewed_odcs_compose_ids"]:`
		`+ compose_ids += args["renewed_odcs_compose_ids"]`
		`+`
		`# OSBS cannot handle both repo_urls and compose_ids in the same time.`
		`# We use repo_urls only in special cases to build base images. In this`
		`# cases, we want to convert compose_ids to repository URLs. Otherwise,`
		`# just pass compose_ids to OSBS via Koji.`
		`- compose_ids = []`
		`if repo_urls:`
		`repo_urls += [self.odcs_get_compose(`
		`- rel.compose.odcs_compose_id)['result_repofile']`
		`- for rel in build.composes]`
		`- else:`
		`+ compose_id)['result_repofile']`
		`+ for compose_id in compose_ids]`
		`compose_ids = []`
		`- for relation in build.composes:`
		`- compose_ids.append(relation.compose.odcs_compose_id)`

		`return self.build_container(`
		`scm_url, branch, target, repo_urls=repo_urls,`

freshmaker/handlers/errata/errata_advisory_rpms_signed.py

file modified

		`@@ -553,6 +553,7 @@`
		`build_args["target"] = image["target"]`
		`build_args["branch"] = image["git_branch"]`
		`build_args["arches"] = image["arches"]`
		`+ build_args["renewed_odcs_compose_ids"] = image["odcs_compose_ids"]`
		`build.build_args = json.dumps(build_args)`

		`db.session.commit()`

freshmaker/lightblue.py

file modified

		`@@ -180,6 +180,7 @@`
		`"git_branch": None,`
		`"error": None,`
		`"arches": None,`
		`+ "odcs_compose_ids": None,`
		`}`

		`@region.cache_on_arguments()`
		`@@ -210,6 +211,14 @@`
		`"Cannot find task_id or container_koji_task_id "`
		`"in the Koji build %r" % build)`

		`+ # Get the list of ODCS composes used to build the image.`
		`+ if ("extra" in build and`
		`+ "image" in build["extra"] and`
		`+ "odcs" in build["extra"]["image"] and`
		`+ "compose_ids" in build["extra"]["image"]["odcs"]):`
		`+ data["odcs_compose_ids"] = \`
		`+ build["extra"]["image"]["odcs"]["compose_ids"]`
		`+`
		`brew_task = session.get_task_request(`
		`build['task_id'])`
		`source = brew_task[0]`

tests/test_errata_advisory_rpms_signed_handler.py

file modified

		`@@ -92,6 +92,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`self.image_b = ContainerImage({`
		`'repository': 'repo_2',`
		`@@ -111,6 +112,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`self.image_c = ContainerImage({`
		`'repository': 'repo_2',`
		`@@ -131,6 +133,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`self.image_d = ContainerImage({`
		`'repository': 'repo_2',`
		`@@ -151,6 +154,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`self.image_e = ContainerImage({`
		`'repository': 'repo_2',`
		`@@ -171,6 +175,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`self.image_f = ContainerImage({`
		`'repository': 'repo_2',`
		`@@ -191,6 +196,7 @@`
		`]`
		`},`
		`"generate_pulp_repos": True,`
		`+ "odcs_compose_ids": None,`
		`})`
		`# For simplicify, mocking _find_images_to_rebuild to just return one`
		`# batch, which contains images found for rebuild from parent to`

tests/test_errata_advisory_state_changed.py

file modified

+11

		`@@ -273,6 +273,7 @@`
		`"content_sets": ["first-content-set"],`
		`"generate_pulp_repos": True,`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": [10, 11],`
		`})`

		`@patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.create_odcs_client')`
		`@@ -343,6 +344,8 @@`
		`self.assertEqual(args["commit"], build.name + "_123")`
		`self.assertEqual(args["parent"],`
		`build.dep_on.rebuilt_nvr if build.dep_on else None)`
		`+ self.assertEqual(args["renewed_odcs_compose_ids"],`
		`+ [10, 11])`


		`class TestCheckImagesToRebuild(helpers.ModelsTestCase):`
		`@@ -834,6 +837,7 @@`
		`"error": None,`
		`"generate_pulp_repos": True,`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})],`
		`[ContainerImage({`
		`"brew": {`
		`@@ -876,6 +880,7 @@`
		`"error": None,`
		`"generate_pulp_repos": True,`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})]`
		`]`

		`@@ -921,6 +926,7 @@`
		`"error": None,`
		`"generate_pulp_repos": False,`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})]`
		`]`

		`@@ -1049,6 +1055,7 @@`
		`"git_branch": "rhel-7",`
		`"error": "Some error occurs while getting this image.",`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})]`
		`]`

		`@@ -1084,6 +1091,7 @@`
		`"git_branch": "rhel-7",`
		`"error": "Some error occurs while getting this image.",`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})]`
		`]`

		`@@ -1119,6 +1127,7 @@`
		`"git_branch": "rhel-7",`
		`"error": "Some error occured.",`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})],`
		`[ContainerImage({`
		`"brew": {`
		`@@ -1160,6 +1169,7 @@`
		`"git_branch": "rhel-7",`
		`"error": "Some error occured too.",`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})]`
		`]`

		`@@ -1199,6 +1209,7 @@`
		`"git_branch": "rhel-7",`
		`"error": "Some error occured.",`
		`"arches": "x86_64",`
		`+ "odcs_compose_ids": None,`
		`})],`
		`]`

tests/test_handler.py

file modified

+16

		`@@ -119,6 +119,7 @@`
		`build_args["target"] = "target"`
		`build_args["branch"] = "branch"`
		`build_args["arches"] = "x86_64"`
		`+ build_args["renewed_odcs_compose_ids"] = None`

		`self.build_1 = ArtifactBuild.create(`
		`db.session, self.event, 'build-1', ArtifactType.IMAGE,`
		`@@ -195,6 +196,21 @@`
		`koji_parent_build=None, release='2', repo_urls=[])`

		`@patch("freshmaker.handlers.ContainerBuildHandler.build_container")`
		`+ def test_build_image_artifact_build_renewed_odcs_composes(`
		`+ self, build_container):`
		`+ build_args = json.loads(self.build_1.build_args)`
		`+ build_args["renewed_odcs_compose_ids"] = [7300, 7301]`
		`+ self.build_1.build_args = json.dumps(build_args)`
		`+ db.session.commit()`
		`+`
		`+ handler = MyHandler()`
		`+ handler.build_image_artifact_build(self.build_1)`
		`+ build_container.assert_called_once_with(`
		`+ 'git://pkgs.fedoraproject.org/repo#hash', 'branch', 'target',`
		`+ arch_override='x86_64', compose_ids=[5, 6, 7, 8, 7300, 7301],`
		`+ isolated=True, koji_parent_build=None, release='2', repo_urls=[])`
		`+`
		`+ @patch("freshmaker.handlers.ContainerBuildHandler.build_container")`
		`def test_build_image_artifact_build_repo_urls(`
		`self, build_container):`
		`handler = MyHandler()`

tests/test_lightblue.py

file modified

+27

		`@@ -295,6 +295,31 @@`

		`@patch('freshmaker.kojiservice.KojiService.get_build')`
		`@patch('freshmaker.kojiservice.KojiService.get_task_request')`
		`+ def test_resolve_commit_odcs_compose_ids(`
		`+ self, get_task_request, get_build):`
		`+ get_build.return_value = {`
		`+ "task_id": 123456,`
		`+ 'extra': {`
		`+ 'image': {`
		`+ 'odcs': {`
		`+ 'compose_ids': [7300, 7301],`
		`+ 'signing_intent': 'release',`
		`+ 'signing_intent_overridden': False`
		`+ }`
		`+ }`
		`+ }`
		`+ }`
		`+ get_task_request.return_value = [`
		`+ "git://example.com/rpms/repo-1?#commit_hash1", "target1", {}]`
		`+`
		`+ self.dummy_image.resolve_commit()`
		`+ self.assertEqual(self.dummy_image["repository"], "rpms/repo-1")`
		`+ self.assertEqual(self.dummy_image["commit"], "commit_hash1")`
		`+ self.assertEqual(self.dummy_image["target"], "target1")`
		`+ self.assertEqual(self.dummy_image["odcs_compose_ids"], [7300, 7301])`
		`+`
		`+ @patch('freshmaker.kojiservice.KojiService.get_build')`
		`+ @patch('freshmaker.kojiservice.KojiService.get_task_request')`
		`def test_resolve_commit_koji_fallback(self, get_task_request, get_build):`
		`get_build.return_value = {"task_id": 123456}`
		`get_task_request.return_value = [`
		`@@ -304,6 +329,7 @@`
		`self.assertEqual(self.dummy_image["repository"], "rpms/repo-1")`
		`self.assertEqual(self.dummy_image["commit"], "commit_hash1")`
		`self.assertEqual(self.dummy_image["target"], "target1")`
		`+ self.assertEqual(self.dummy_image["odcs_compose_ids"], None)`

		`@patch('freshmaker.kojiservice.KojiService.get_build')`
		`@patch('freshmaker.kojiservice.KojiService.get_task_request')`
		`@@ -968,6 +994,7 @@`
		`"git_branch": "mybranch",`
		`"error": None,`
		`"arches": None,`
		`+ "odcs_compose_ids": None,`
		`"brew": {`
		`"completion_date": u"20170421T04:27:51.000-0400",`
		`"build": "package-name-2-4-12.10",`

jkaluza commented 5 years ago

Some images contain RPMs which are internal only and are used only as a build-time dependency of the image. Those RPMs are therefore never shipped and don't exist in any Pulp repository. But Freshmaker currently includes only repositories from Pulp - that means that build of such images would fail, because the internal only package cannot be installed from public Pulp repo.

In the original build, the internal only package is part of ODCS compose as requested by OSBS build system.

This PR gets the list of ODCS compose IDs which have been used to build the original image and passes them to OSBS when rebuildiong the image. This means that Freshmaker now submits image with Pulp ODCS compose IDs together with original Koji tag ODCS compose IDs.

The idea is that the Koji tag compose repositories will contain all the packages (including internal-only packages) which are needed to build the image in the same versions as in the time when building it originally. The Pulp repos will provide the latest versions of those packages, so when the "yum install" phase of image build is running, latest RPMs from Pulp repos will be used and internal-only packages will be taken from original renewed compose(s).