From ee4144b8697c0911f779fc17422beacd5d7e9172 Mon Sep 17 00:00:00 2001 From: Jan Kaluža Date: Mar 08 2018 12:58:16 +0000 Subject: Merge #173 `[do not merge] Use libmodulemd instead of old modulemd module.` --- diff --git a/server/odcs/server/pdc.py b/server/odcs/server/pdc.py index f27cd75..9eba867 100644 --- a/server/odcs/server/pdc.py +++ b/server/odcs/server/pdc.py @@ -23,13 +23,17 @@ import inspect import requests import re -import modulemd + from pdc_client import PDCClient from beanbag.bbexcept import BeanBagException import odcs.server.utils from odcs.server import log +import gi +gi.require_version('Modulemd', '1.0') # noqa +from gi.repository import Modulemd + class ModuleLookupError(Exception): pass @@ -170,20 +174,22 @@ class PDC(object): new_modules = [] for module in modules: - mmd = modulemd.ModuleMetadata() - mmd.loads(module['modulemd']) + mmd = Modulemd.Module.new_from_string(module['modulemd']) + mmd.upgrade() # Check runtime dependency (name:stream) of a module and if this # dependency is already in module_map/new_modules, do nothing. # But otherwise get the latest module in this name:stream from PDC # and add it to new_modules/module_map. - for name, stream in mmd.requires.items(): - key = "%s:%s" % (name, stream) - if key not in module_map: - new_module = self.get_latest_module( - variant_id=name, variant_version=stream) - new_modules.append(new_module) - module_map[key] = new_module + for deps in mmd.get_dependencies(): + for name, streams in deps.get_requires().items(): + for stream in streams.get(): + key = "%s:%s" % (name, stream) + if key not in module_map: + new_module = self.get_latest_module( + variant_id=name, variant_version=stream) + new_modules.append(new_module) + module_map[key] = new_module return new_modules diff --git a/server/requirements.txt b/server/requirements.txt index 08d7249..d266f5d 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,7 +1,6 @@ funcsigs # Python2 only httplib2 mock -modulemd munch pdc-client pyOpenSSL diff --git a/server/tests/pdc.py b/server/tests/pdc.py index e48840b..32779f4 100644 --- a/server/tests/pdc.py +++ b/server/tests/pdc.py @@ -23,19 +23,35 @@ from functools import wraps import json -import modulemd import responses from six.moves.urllib.parse import urlparse, parse_qs from odcs.server import conf - -def make_module(name, stream, version, requires={}): - mmd = modulemd.ModuleMetadata() - mmd.name = name - mmd.stream = stream - mmd.version = version - mmd.requires.update(requires) +import gi +gi.require_version('Modulemd', '1.0') # noqa +from gi.repository import Modulemd + + +def make_module(name, stream, version, requires={}, mdversion=1): + mmd = Modulemd.Module() + mmd.set_mdversion(mdversion) + mmd.set_name(name) + mmd.set_stream(stream) + mmd.set_version(version) + mmd.set_summary("foo") + mmd.set_description("foo") + licenses = Modulemd.SimpleSet() + licenses.add("GPL") + mmd.set_module_licenses(licenses) + + if mdversion == 1: + mmd.set_requires(requires) + else: + deps = Modulemd.Dependencies() + for req_name, req_stream in requires.items(): + deps.add_requires_single(req_name, req_stream) + mmd.set_dependencies((deps, )) return { 'variant_id': name, @@ -46,7 +62,7 @@ def make_module(name, stream, version, requires={}): } -TEST_PDC_MODULES = [ +TEST_PDC_MODULES_MMDv1 = [ # test_backend.py make_module('moduleA', 'f26', 20170809000000, {'moduleB': 'f26'}), @@ -69,46 +85,76 @@ TEST_PDC_MODULES = [ ] -def mock_pdc(f): +TEST_PDC_MODULES_MMDv2 = [ + # test_backend.py + make_module('moduleA', 'f26', 20170809000000, + {'moduleB': 'f26'}, 2), + make_module('moduleA', 'f26', 20170805000000, + {'moduleB': 'f26'}, 2), + + make_module('moduleB', 'f26', 20170808000000, + {'moduleC': 'f26', 'moduleD': 'f26'}, 2), + make_module('moduleB', 'f27', 2017081000000, + {'moduleC': 'f27'}, 2), + + make_module('moduleC', 'f26', 20170807000000, + {'moduleD': 'f26'}, 2), + + make_module('moduleD', 'f26', 20170806000000, {}, 2), + + # test_composerthread.py + make_module('testmodule', 'master', 20170515074418, {}, 2), + make_module('testmodule', 'master', 20170515074419, {}, 2) +] + + +def mock_pdc(mdversion=2): """ Decorator that sets up a test environment so that calls to the PDC to look up modules are redirected to return results from the TEST_MODULES array above. """ + def wrapper(f): + @wraps(f) + def wrapped(*args, **kwargs): + def handle_unreleasedvariants(request): + query = parse_qs(urlparse(request.url).query) + variant_id = query['variant_id'] + variant_version = query['variant_version'] + variant_release = query.get('variant_release', None) + + if mdversion == 1: + modules = TEST_PDC_MODULES_MMDv1 + else: + modules = TEST_PDC_MODULES_MMDv2 - @wraps(f) - def wrapped(*args, **kwargs): - def handle_unreleasedvariants(request): - query = parse_qs(urlparse(request.url).query) - variant_id = query['variant_id'] - variant_version = query['variant_version'] - variant_release = query.get('variant_release', None) - - body = [] - for module in TEST_PDC_MODULES: - if module['variant_id'] not in variant_id: - continue - if module['variant_version'] not in variant_version: - continue - if variant_release is not None: - if module['variant_release'] not in variant_release: + body = [] + for module in modules: + if module['variant_id'] not in variant_id: continue + if module['variant_version'] not in variant_version: + continue + if variant_release is not None: + if module['variant_release'] not in variant_release: + continue - fields = query.get('fields', None) - if fields is not None: - return_module = {} - for field in fields: - return_module[field] = module[field] - else: - return_module = module + fields = query.get('fields', None) + if fields is not None: + return_module = {} + for field in fields: + return_module[field] = module[field] + else: + return_module = module - body.append(return_module) + body.append(return_module) - return (200, {}, json.dumps(body)) + return (200, {}, json.dumps(body)) - responses.add_callback(responses.GET, conf.pdc_url + '/unreleasedvariants/', - content_type='application/json', - callback=handle_unreleasedvariants) + responses.add_callback( + responses.GET, conf.pdc_url + '/unreleasedvariants/', + content_type='application/json', + callback=handle_unreleasedvariants) - return f(*args, **kwargs) + return f(*args, **kwargs) - return responses.activate(wrapped) + return responses.activate(wrapped) + return wrapper diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 701ab9e..2214b6d 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -57,7 +57,7 @@ class TestBackend(ModelsBaseTest): c = db.session.query(Compose).filter(Compose.id == 1).one() self.assertEqual(c.koji_event, 1496834159) - @mock_pdc + @mock_pdc() def test_resolve_compose_module(self): c = Compose.create( db.session, "me", PungiSourceType.MODULE, @@ -75,7 +75,7 @@ class TestBackend(ModelsBaseTest): "moduleC:f26:20170807000000", "moduleD:f26:20170806000000"])) - @mock_pdc + @mock_pdc() def test_resolve_compose_module_no_deps(self): c = Compose.create( db.session, "me", PungiSourceType.MODULE, @@ -90,7 +90,7 @@ class TestBackend(ModelsBaseTest): c = db.session.query(Compose).filter(Compose.id == 1).one() self.assertEqual(c.source, "moduleA:f26:20170809000000") - @mock_pdc + @mock_pdc() def expect_module_lookup_error(self, source, match, flags=0): c = Compose.create( db.session, "me", PungiSourceType.MODULE, @@ -102,6 +102,51 @@ class TestBackend(ModelsBaseTest): with self.assertRaisesRegexp(ModuleLookupError, match): resolve_compose(c) + @mock_pdc(1) + def test_resolve_compose_module_mmdv1(self): + c = Compose.create( + db.session, "me", PungiSourceType.MODULE, + "moduleA-f26", + COMPOSE_RESULTS["repository"], 3600) + db.session.commit() + + resolve_compose(c) + db.session.commit() + + c = db.session.query(Compose).filter(Compose.id == 1).one() + self.assertEqual(c.source, + ' '.join(["moduleA:f26:20170809000000", + "moduleB:f26:20170808000000", + "moduleC:f26:20170807000000", + "moduleD:f26:20170806000000"])) + + @mock_pdc(1) + def test_resolve_compose_module_no_deps_mmdv1(self): + c = Compose.create( + db.session, "me", PungiSourceType.MODULE, + "moduleA-f26 moduleA-f26", + COMPOSE_RESULTS["repository"], 3600, + flags=COMPOSE_FLAGS["no_deps"]) + db.session.commit() + + resolve_compose(c) + db.session.commit() + + c = db.session.query(Compose).filter(Compose.id == 1).one() + self.assertEqual(c.source, "moduleA:f26:20170809000000") + + @mock_pdc(1) + def expect_module_lookup_error_mmdv1(self, source, match, flags=0): + c = Compose.create( + db.session, "me", PungiSourceType.MODULE, + source, + COMPOSE_RESULTS["repository"], 3600, + flags=flags) + db.session.commit() + + with self.assertRaisesRegexp(ModuleLookupError, match): + resolve_compose(c) + def test_resolve_compose_module_not_found(self): self.expect_module_lookup_error("moduleA-f30", "Failed to find") @@ -115,7 +160,7 @@ class TestBackend(ModelsBaseTest): "moduleA-f26-20170809000000 moduleA-f26-20170805000000", "conflicts with") - @mock_pdc + @mock_pdc() def test_resolve_compose_module_not_conflict(self): c = Compose.create( db.session, "me", PungiSourceType.MODULE, @@ -126,6 +171,17 @@ class TestBackend(ModelsBaseTest): resolve_compose(c) + @mock_pdc(1) + def test_resolve_compose_module_not_conflict_mmdv1(self): + c = Compose.create( + db.session, "me", PungiSourceType.MODULE, + "moduleB-f26 moduleB-f27", + COMPOSE_RESULTS["repository"], 3600, + flags=COMPOSE_FLAGS["no_deps"]) + db.session.commit() + + resolve_compose(c) + def test_resolve_compose_module_dep_not_found(self): self.expect_module_lookup_error( "moduleB-f26 moduleB-f27",