From 8133676270eb0c37c3e14a873b9f490d6541e5ef Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Oct 20 2021 06:22:19 +0000 Subject: osbs: Reuse images from old compose JIRA: RHELCMP-5972 Signed-off-by: Haibo Lin --- diff --git a/pungi/checks.py b/pungi/checks.py index c79f2bb..e5c2f22 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1202,6 +1202,7 @@ def make_schema(): "anyOf": [{"type": "string"}, {"type": "number"}], "default": 10 * 1024 * 1024, }, + "osbs_allow_reuse": {"type": "boolean", "default": False}, "osbs": { "type": "object", "patternProperties": { diff --git a/pungi/phases/osbs.py b/pungi/phases/osbs.py index 8a39523..9db4371 100644 --- a/pungi/phases/osbs.py +++ b/pungi/phases/osbs.py @@ -1,23 +1,29 @@ # -*- coding: utf-8 -*- +import copy import fnmatch import json import os from kobo.threads import ThreadPool, WorkerThread from kobo import shortcuts +from productmd.rpms import Rpms +from six.moves import configparser from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util from ..wrappers import kojiwrapper +from ..wrappers.scm import get_file_from_scm class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase): name = "osbs" - def __init__(self, compose): + def __init__(self, compose, pkgset_phase, buildinstall_phase): super(OSBSPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) self.pool.registries = {} + self.pool.pkgset_phase = pkgset_phase + self.pool.buildinstall_phase = buildinstall_phase def run(self): for variant in self.compose.get_variants(): @@ -77,8 +83,8 @@ class OSBSThread(WorkerThread): def worker(self, compose, variant, config): msg = "OSBS task for variant %s" % variant.uid self.pool.log_info("[BEGIN] %s" % msg) - koji = kojiwrapper.KojiWrapper(compose) - koji.login() + + original_config = copy.deepcopy(config) # Start task source = config.pop("url") @@ -94,33 +100,99 @@ class OSBSThread(WorkerThread): config["yum_repourls"] = repos - task_id = koji.koji_proxy.buildContainer( - source, target, config, priority=priority + log_dir = os.path.join(compose.paths.log.topdir(), "osbs") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + reuse_file = log_file[:-4] + ".reuse.json" + + try: + image_conf = self._get_image_conf(compose, original_config) + except Exception as e: + image_conf = None + self.pool.log_info( + "Can't get image-build.conf for variant: %s source: %s - %s" + % (variant.uid, source, str(e)) + ) + + koji = kojiwrapper.KojiWrapper(compose) + koji.login() + + task_id = self._try_to_reuse( + compose, variant, original_config, image_conf, reuse_file ) + if not task_id: + task_id = koji.koji_proxy.buildContainer( + source, target, config, priority=priority + ) + koji.save_task_id(task_id) # Wait for it to finish and capture the output into log file (even # though there is not much there). - log_dir = os.path.join(compose.paths.log.topdir(), "osbs") - util.makedirs(log_dir) - log_file = os.path.join( - log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) - ) if koji.watch_task(task_id, log_file) != 0: raise RuntimeError( "OSBS: task %s failed: see %s for details" % (task_id, log_file) ) scratch = config.get("scratch", False) - nvr = add_metadata(variant, task_id, compose, scratch) + nvr, archive_ids = add_metadata(variant, task_id, compose, scratch) if nvr: registry = get_registry(compose, nvr, registry) if registry: self.pool.registries[nvr] = registry + self._write_reuse_metadata( + compose, + variant, + original_config, + image_conf, + task_id, + archive_ids, + reuse_file, + ) + self.pool.log_info("[DONE ] %s" % msg) + def _get_image_conf(self, compose, config): + """Get image-build.conf from git repo. + + :param Compose compose: Current compose. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + """ + tmp_dir = compose.mkdtemp(prefix="osbs_") + + url = config["url"].split("#") + if len(url) == 1: + url.append(config["git_branch"]) + + filename = "image-build.conf" + get_file_from_scm( + { + "scm": "git", + "repo": url[0], + "branch": url[1], + "file": [filename], + }, + tmp_dir, + ) + + c = configparser.ConfigParser() + c.read(os.path.join(tmp_dir, filename)) + return c + + def _get_ksurl(self, image_conf): + """Get ksurl from image-build.conf""" + ksurl = image_conf.get("image-build", "ksurl") + + if ksurl: + resolver = util.GitUrlResolver(offline=False) + return resolver(ksurl) + else: + return None + def _get_repo(self, compose, repo, gpgkey=None): """ Return repo file URL of repo, if repo contains "://", it's already a @@ -177,6 +249,151 @@ class OSBSThread(WorkerThread): return util.translate_path(compose, repo_file) + def _try_to_reuse(self, compose, variant, config, image_conf, reuse_file): + """Try to reuse results of old compose. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + :param ConfigParser image_conf: ConfigParser obj of image-build.conf. + :param str reuse_file: Path to reuse metadata file + """ + log_msg = "Cannot reuse old osbs phase results - %s" + + if not compose.conf["osbs_allow_reuse"]: + self.pool.log_info(log_msg % "reuse of old osbs results is disabled.") + return False + + old_reuse_file = compose.paths.old_compose_path(reuse_file) + if not old_reuse_file: + self.pool.log_info(log_msg % "Can't find old reuse metadata file") + return False + + try: + with open(old_reuse_file) as f: + old_reuse_metadata = json.load(f) + except Exception as e: + self.pool.log_info( + log_msg % "Can't load old reuse metadata file: %s" % str(e) + ) + return False + + if old_reuse_metadata["config"] != config: + self.pool.log_info(log_msg % "osbs config changed") + return False + + if not image_conf: + self.pool.log_info(log_msg % "Can't get image-build.conf") + return False + + # Make sure ksurl not change + try: + ksurl = self._get_ksurl(image_conf) + except Exception as e: + self.pool.log_info( + log_msg % "Can't get ksurl from image-build.conf - %s" % str(e) + ) + return False + + if not old_reuse_metadata["ksurl"]: + self.pool.log_info( + log_msg % "Can't get ksurl from old compose reuse metadata." + ) + return False + + if ksurl != old_reuse_metadata["ksurl"]: + self.pool.log_info(log_msg % "ksurl changed") + return False + + # Make sure buildinstall phase is reused + try: + arches = image_conf.get("image-build", "arches").split(",") + except Exception as e: + self.pool.log_info( + log_msg % "Can't get arches from image-build.conf - %s" % str(e) + ) + for arch in arches: + if not self.pool.buildinstall_phase.reused(variant, arch): + self.pool.log_info( + log_msg % "buildinstall phase changed %s.%s" % (variant, arch) + ) + return False + + # Make sure rpms installed in image exists in current compose + rpm_manifest_file = compose.paths.compose.metadata("rpms.json") + rpm_manifest = Rpms() + rpm_manifest.load(rpm_manifest_file) + rpms = set() + for variant in rpm_manifest.rpms: + for arch in rpm_manifest.rpms[variant]: + for src in rpm_manifest.rpms[variant][arch]: + for nevra in rpm_manifest.rpms[variant][arch][src]: + rpms.add(nevra) + + for nevra in old_reuse_metadata["rpmlist"]: + if nevra not in rpms: + self.pool.log_info( + log_msg % "%s does not exist in current compose" % nevra + ) + return False + + self.pool.log_info( + "Reusing old OSBS task %d result" % old_reuse_file["task_id"] + ) + return old_reuse_file["task_id"] + + def _write_reuse_metadata( + self, compose, variant, config, image_conf, task_id, archive_ids, reuse_file + ): + """Write metadata to file for reusing. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param dict config: One osbs config item of compose.conf["osbs"][$variant] + :param ConfigParser image_conf: ConfigParser obj of image-build.conf. + :param int task_id: Koji task id of osbs task. + :param list archive_ids: List of koji archive id + :param str reuse_file: Path to reuse metadata file. + """ + msg = "Writing reuse metadata file %s" % reuse_file + compose.log_info(msg) + + rpmlist = set() + koji = kojiwrapper.KojiWrapper(compose) + for archive_id in archive_ids: + rpms = koji.koji_proxy.listRPMs(imageID=archive_id) + for item in rpms: + if item["epoch"]: + rpmlist.add( + "%s:%s-%s-%s.%s" + % ( + item["name"], + item["epoch"], + item["version"], + item["release"], + item["arch"], + ) + ) + else: + rpmlist.add("%s.%s" % (item["nvr"], item["arch"])) + + try: + ksurl = self._get_ksurl(image_conf) + except Exception: + ksurl = None + + data = { + "config": config, + "ksurl": ksurl, + "rpmlist": sorted(rpmlist), + "task_id": task_id, + } + try: + with open(reuse_file, "w") as f: + json.dump(data, f, indent=4) + except Exception as e: + compose.log_info(msg + " failed - %s" % str(e)) + def add_metadata(variant, task_id, compose, is_scratch): """Given a task ID, find details about the container and add it to global @@ -200,7 +417,7 @@ def add_metadata(variant, task_id, compose, is_scratch): compose.containers_metadata.setdefault(variant.uid, {}).setdefault( "scratch", [] ).append(metadata) - return None + return None, [] else: build_id = int(result["koji_builds"][0]) @@ -218,6 +435,7 @@ def add_metadata(variant, task_id, compose, is_scratch): "creation_time": buildinfo["creation_time"], } ) + archive_ids = [] for archive in archives: data = { "filename": archive["filename"], @@ -234,4 +452,5 @@ def add_metadata(variant, task_id, compose, is_scratch): compose.containers_metadata.setdefault(variant.uid, {}).setdefault( arch, [] ).append(data) - return nvr + archive_ids.append(archive["id"]) + return nvr, archive_ids diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index ed3b081..9570e3c 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -408,7 +408,7 @@ def run_compose( livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) osbuild_phase = pungi.phases.OSBuildPhase(compose) - osbs_phase = pungi.phases.OSBSPhase(compose) + osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase) image_container_phase = pungi.phases.ImageContainerPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) diff --git a/tests/test_osbs_phase.py b/tests/test_osbs_phase.py index fde5753..456fe1f 100644 --- a/tests/test_osbs_phase.py +++ b/tests/test_osbs_phase.py @@ -19,7 +19,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): pool = ThreadPool.return_value - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) phase.run() self.assertEqual(len(pool.add.call_args_list), 1) @@ -33,7 +33,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): compose = helpers.DummyCompose(self.topdir, {}) compose.just_phases = None compose.skip_phases = [] - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) self.assertTrue(phase.skip()) @mock.patch("pungi.phases.osbs.ThreadPool") @@ -42,7 +42,7 @@ class OSBSPhaseTest(helpers.PungiTestCase): compose.just_phases = None compose.skip_phases = [] compose.notifier = mock.Mock() - phase = osbs.OSBSPhase(compose) + phase = osbs.OSBSPhase(compose, None, None) phase.start() phase.stop() phase.pool.registries = {"foo": "bar"} @@ -139,6 +139,8 @@ METADATA = { } } +RPMS = [] + SCRATCH_TASK_RESULT = { "koji_builds": [], "repositories": [ @@ -182,6 +184,7 @@ class OSBSThreadTest(helpers.PungiTestCase): self.wrapper.koji_proxy.getTaskResult.return_value = TASK_RESULT self.wrapper.koji_proxy.getBuild.return_value = BUILD_INFO self.wrapper.koji_proxy.listArchives.return_value = ARCHIVES + self.wrapper.koji_proxy.listRPMs.return_value = RPMS self.wrapper.koji_proxy.getLatestBuilds.return_value = [ mock.Mock(), mock.Mock(), @@ -233,6 +236,7 @@ class OSBSThreadTest(helpers.PungiTestCase): [ mock.call.koji_proxy.getBuild(54321), mock.call.koji_proxy.listArchives(54321), + mock.call.koji_proxy.listRPMs(imageID=1436049), ] ) self.assertEqual(self.wrapper.mock_calls, expect_calls) @@ -269,8 +273,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertIn(" Possible reason: %r is a required property" % key, errors) self.assertEqual([], warnings) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_minimal_run(self, KojiWrapper): + def test_minimal_run(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -285,8 +290,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_failable(self, KojiWrapper): + def test_run_failable(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -302,8 +308,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_more_args(self, KojiWrapper): + def test_run_with_more_args(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -322,8 +329,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_extra_repos(self, KojiWrapper): + def test_run_with_extra_repos(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -395,8 +403,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectCalls(options) self._assertCorrectMetadata() + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_deprecated_registry(self, KojiWrapper): + def test_run_with_deprecated_registry(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -426,8 +435,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertRepoFile(["Server", "Everything"]) self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}}) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_registry(self, KojiWrapper): + def test_run_with_registry(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -457,8 +467,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertRepoFile(["Server", "Everything"]) self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": [{"foo": "bar"}]}) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_extra_repos_in_list(self, KojiWrapper): + def test_run_with_extra_repos_in_list(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate", @@ -487,8 +498,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self._assertCorrectMetadata() self._assertRepoFile(["Server", "Everything", "Client"]) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_run_with_gpgkey_enabled(self, KojiWrapper): + def test_run_with_gpgkey_enabled(self, KojiWrapper, get_file_from_scm): gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release" cfg = { "url": "git://example.com/repo?#BEEFCAFE", @@ -547,8 +559,9 @@ class OSBSThreadTest(helpers.PungiTestCase): } self._assertConfigMissing(cfg, "git_branch") + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_failing_task(self, KojiWrapper): + def test_failing_task(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "fedora-24-docker-candidate", @@ -563,8 +576,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details") + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_failing_task_with_failable(self, KojiWrapper): + def test_failing_task_with_failable(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "fedora-24-docker-candidate", @@ -577,8 +591,9 @@ class OSBSThreadTest(helpers.PungiTestCase): self.t.process((self.compose, self.compose.variants["Server"], cfg), 1) + @mock.patch("pungi.phases.osbs.get_file_from_scm") @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper") - def test_scratch_metadata(self, KojiWrapper): + def test_scratch_metadata(self, KojiWrapper, get_file_from_scm): cfg = { "url": "git://example.com/repo?#BEEFCAFE", "target": "f24-docker-candidate",