From c625b253c57a6fdc82109e9267c0be0e30d395c1 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Oct 19 2017 07:27:35 +0000 Subject: Check requested packages are present in generated pungi compose There have been a bug in pungi and it did not generate compose with all the requested RPMs, but still returned zero error code. We should prevent that by cheking the compose/metadata/rpms.json in resulting compose to find out if all the packages we requested are there and if not, fail the compose. FIXES: #106 --- diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 848314a..63a7e2d 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -25,6 +25,9 @@ import koji import os import threading import shutil +import six +import productmd.compose +import productmd.common from datetime import datetime from odcs.server import log, conf, app, db from odcs.server.models import Compose, COMPOSE_STATES, COMPOSE_FLAGS @@ -434,6 +437,35 @@ def generate_pungi_compose(compose): db.session.commit() +def validate_pungi_compose(compose): + """ + Validate the compose is generated by pungi as expected. + """ + # the requested packages should be present in the generated compose + if compose.packages: + packages = compose.packages.split() + pungi_compose = productmd.compose.Compose(compose.toplevel_dir) + rm = pungi_compose.rpms.rpms + srpm_nevras = [] + for variant in rm: + for arch in rm[variant]: + for srpm_nevra, data in six.iteritems(rm[variant][arch]): + for rpm_nevra, data in six.iteritems(rm[variant][arch][srpm_nevra]): + if data['category'] != 'source': + continue + srpm_nevras.append(srpm_nevra) + srpms = [productmd.common.parse_nvra(n)['name'] for n in srpm_nevras] + not_found = [] + for pkg in packages: + if pkg not in srpms: + not_found.append(pkg) + if not_found: + msg = "The following requested packages are not present in the generated compose: %s." % \ + " ".join(not_found) + log.error(msg) + raise RuntimeError(msg) + + def generate_compose(compose_id): """ Generates the compose defined by its `compose_id`. It is run by @@ -453,6 +485,7 @@ def generate_compose(compose_id): generate_pulp_compose(compose) else: generate_pungi_compose(compose) + validate_pungi_compose(compose) except: # Something went wrong, log the exception and update the compose # state in database. diff --git a/server/requirements.txt b/server/requirements.txt index f003157..f5cbd97 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -21,3 +21,4 @@ requests jinja2 pyldap defusedxml +productmd diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 986b6ab..afbf347 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -22,6 +22,7 @@ import os from mock import patch, MagicMock +from productmd.rpms import Rpms from odcs.server import db from odcs.server.models import Compose @@ -29,7 +30,8 @@ from odcs.common.types import COMPOSE_FLAGS, COMPOSE_RESULTS, COMPOSE_STATES from odcs.server.pdc import ModuleLookupError from odcs.server.pungi import PungiSourceType from odcs.server.backend import (resolve_compose, get_reusable_compose, - generate_pulp_compose) + generate_pulp_compose, validate_pungi_compose) +from odcs.server.utils import makedirs import odcs.server.backend from utils import ModelsBaseTest @@ -262,3 +264,43 @@ gpgcheck=0 pulp_rest_post.assert_called_once_with('repositories/search/', expected_query) _write_repo_file.assert_not_called() + + def test_validate_generated_pungi_compose_missing_packages(self): + c = Compose.create( + db.session, "me", PungiSourceType.KOJI_TAG, "f26", + COMPOSE_RESULTS["repository"], 60, packages='pkg1 pkg2') + db.session.commit() + compose_dir = os.path.join(c.toplevel_dir, 'compose') + metadata_dir = os.path.join(compose_dir, 'metadata') + rpms_metadata = os.path.join(metadata_dir, 'rpms.json') + makedirs(metadata_dir) + + rm = Rpms() + rm.header.version = "1.0" + rm.compose.id = "Me-26-20161212.0" + rm.compose.type = "production" + rm.compose.date = "20161212" + rm.compose.respin = 0 + + rm.add( + "Temporary", + "x86_64", + "pkg1-0:2.18-11.fc26.x86_64.rpm", + path="Temporary/x86_64/os/Packages/p/pkg1-2.18-11.fc26.x86_64.rpm", + sigkey="246110c1", + category="binary", + srpm_nevra="pkg1-0:2.18-11.fc26.src.rpm", + ) + rm.add( + "Temporary", + "x86_64", + "pkg1-0:2.18-11.fc20.src.rpm", + path="Temporary/source/SRPMS/p/pkg1-2.18-11.fc26.x86_64.rpm", + sigkey="246110c1", + category="source", + ) + rm.dump(rpms_metadata) + with self.assertRaises(RuntimeError) as ctx: + validate_pungi_compose(c) + self.assertIn('The following requested packages are not present in the generated compose: pkg2.', + str(ctx.exception)) diff --git a/server/tests/test_composerthread.py b/server/tests/test_composerthread.py index da88406..5de41fd 100644 --- a/server/tests/test_composerthread.py +++ b/server/tests/test_composerthread.py @@ -127,7 +127,8 @@ class TestComposerThread(ModelsBaseTest): self.assertEqual(c.state, COMPOSE_STATES["failed"]) @mock_pdc - def test_submit_build_reuse_repo(self): + @patch("odcs.server.backend.validate_pungi_compose") + def test_submit_build_reuse_repo(self, mock_validate_pungi_compose): self._add_repo_composes() c = db.session.query(Compose).filter(Compose.id == 2).one() @@ -137,6 +138,7 @@ class TestComposerThread(ModelsBaseTest): self.assertEqual(c.state, COMPOSE_STATES["done"]) self.assertEqual(c.result_repo_dir, "./latest-odcs-1-1/compose/Temporary") self.assertEqual(c.result_repo_url, "http://localhost/odcs/latest-odcs-1-1/compose/Temporary") + mock_validate_pungi_compose.assert_called_once() @mock_pdc def test_submit_build_reuse_module(self):