From 7b8947f660c9fdb832e988b67ba259047635ae2c Mon Sep 17 00:00:00 2001 From: mprahl Date: Apr 25 2019 17:15:27 +0000 Subject: Allow buildrequring a virtual stream of a base module --- diff --git a/module_build_service/models.py b/module_build_service/models.py index 5a27fde..63d1526 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -468,6 +468,17 @@ class ModuleBuild(MBSBase): return ret @staticmethod + def get_module_count(session, **kwargs): + """ + Determine the number of modules that match the provided filter. + + :param session: SQLAlchemy session + :return: the number of modules that match the provided filter + :rtype: int + """ + return session.query(func.count(ModuleBuild.id)).filter_by(**kwargs).scalar() + + @staticmethod def get_build_by_koji_tag(session, tag): """Get build by its koji_tag""" return session.query(ModuleBuild).filter_by(koji_tag=tag).first() diff --git a/module_build_service/resolver/DBResolver.py b/module_build_service/resolver/DBResolver.py index 074b94b..0b9080d 100644 --- a/module_build_service/resolver/DBResolver.py +++ b/module_build_service/resolver/DBResolver.py @@ -28,6 +28,7 @@ from module_build_service import log, db from module_build_service.resolver.base import GenericResolver from module_build_service import models from module_build_service.errors import UnprocessableEntity +from module_build_service.utils.submit import load_mmd import sqlalchemy @@ -53,6 +54,37 @@ class DBResolver(GenericResolver): raise UnprocessableEntity( 'Cannot find any module builds for %s:%s' % (name, stream)) + def get_module_count(self, **kwargs): + """ + Determine the number of modules that match the provided filter. + + :return: the number of modules that match the provided filter + :rtype: int + """ + with models.make_session(self.config) as session: + return models.ModuleBuild.get_module_count(session, **kwargs) + + def get_latest_with_virtual_stream(self, name, virtual_stream): + """ + Get the latest module with the input virtual stream based on the stream version and version. + + :param str name: the module name to search for + :param str virtual_stream: the module virtual stream to search for + :return: the module's modulemd or None + :rtype: Modulemd.Module or None + """ + with models.make_session(self.config) as session: + query = session.query(models.ModuleBuild).filter_by(name=name) + query = models.ModuleBuild._add_virtual_streams_filter(session, query, [virtual_stream]) + # Cast the version as an integer so that we get proper ordering + module = query.order_by( + models.ModuleBuild.stream_version.desc(), + sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc() + ).first() + + if module: + return load_mmd(module.modulemd) + def get_module_modulemds(self, name, stream, version=None, context=None, strict=False, stream_version_lte=False, virtual_streams=None): """ @@ -72,7 +104,6 @@ class DBResolver(GenericResolver): logic. When falsy, no filtering occurs. :return: List of Modulemd metadata instances matching the query """ - from module_build_service.utils import load_mmd if version and context: mmd = self._get_module(name, stream, version, context, strict=strict) if mmd is None: diff --git a/module_build_service/resolver/MBSResolver.py b/module_build_service/resolver/MBSResolver.py index 7d775ee..d7cc24c 100644 --- a/module_build_service/resolver/MBSResolver.py +++ b/module_build_service/resolver/MBSResolver.py @@ -126,6 +126,51 @@ class MBSResolver(GenericResolver): if rv: return rv[0] + def get_module_count(self, **kwargs): + """ + Determine the number of modules that match the provided filter. + + :return: the number of modules that match the provided filter + :rtype: int + """ + query = { + "page": 1, + "per_page": 1, + "short": True, + } + query.update(kwargs) + res = self.session.get(self.mbs_prod_url, params=query) + if not res.ok: + raise RuntimeError(self._generic_error % (query, res.status_code)) + + data = res.json() + return data["meta"]["total"] + + def get_latest_with_virtual_stream(self, name, virtual_stream): + """ + Get the latest module with the input virtual stream based on the stream version and version. + + :param str name: the module name to search for + :param str virtual_stream: the module virtual stream to search for + :return: the module's modulemd or None + :rtype: Modulemd.Module or None + """ + query = { + "name": name, + "order_desc_by": ["stream_version", "version"], + "page": 1, + "per_page": 1, + "verbose": True, + "virtual_stream": virtual_stream, + } + res = self.session.get(self.mbs_prod_url, params=query) + if not res.ok: + raise RuntimeError(self._generic_error % (query, res.status_code)) + + data = res.json() + if data["items"]: + return load_mmd(data["items"][0]["modulemd"]) + def get_module_modulemds(self, name, stream, version=None, context=None, strict=False, stream_version_lte=False, virtual_streams=None): """ diff --git a/module_build_service/resolver/base.py b/module_build_service/resolver/base.py index c9ca54c..149f4ad 100644 --- a/module_build_service/resolver/base.py +++ b/module_build_service/resolver/base.py @@ -106,6 +106,14 @@ class GenericResolver(six.with_metaclass(ABCMeta)): return load_mmd(yaml) @abstractmethod + def get_module_count(self, **kwargs): + raise NotImplementedError() + + @abstractmethod + def get_latest_with_virtual_stream(self, name, virtual_stream): + raise NotImplementedError() + + @abstractmethod def get_module_modulemds(self, name, stream, version=None, context=None, strict=False, stream_version_lte=None, virtual_streams=None): raise NotImplementedError() diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 17d9a67..7617520 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -33,18 +33,17 @@ from functools import partial from multiprocessing.dummy import Pool as ThreadPool from datetime import datetime import copy -from module_build_service.utils import to_text_type import kobo.rpmlib import requests from gi.repository import GLib import module_build_service.scm - from module_build_service import conf, db, log, models, Modulemd from module_build_service.errors import ( ValidationError, UnprocessableEntity, Forbidden, Conflict) from module_build_service import glib +from module_build_service.utils import to_text_type def record_filtered_rpms(mmd): @@ -603,6 +602,78 @@ def _apply_dep_overrides(mmd, params): mmd.set_dependencies(deps) +def _handle_base_module_virtual_stream_br(mmd): + """ + Translate a base module virtual stream buildrequire to an actual stream on the input modulemd. + + :param Modulemd.Module mmd: the modulemd to apply the overrides on + """ + from module_build_service.resolver import system_resolver + + overridden = False + deps = mmd.get_dependencies() + for dep in deps: + brs = dep.get_buildrequires() + + for base_module in conf.base_module_names: + if base_module not in brs: + continue + + streams = list(brs[base_module].get()) + new_streams = copy.copy(streams) + for i, stream in enumerate(streams): + # Ignore streams that start with a minus sign, since those are handled in the + # MSE code + if stream.startswith('-'): + continue + + # Check if the base module stream is available + log.debug( + 'Checking to see if the base module "%s:%s" is available', base_module, stream) + if system_resolver.get_module_count(name=base_module, stream=stream) > 0: + continue + + # If the base module stream is not available, check if there's a virtual stream + log.debug( + 'Checking to see if there is a base module "%s" with the virtual stream "%s"', + base_module, + stream + ) + base_module_mmd = system_resolver.get_latest_with_virtual_stream( + name=base_module, virtual_stream=stream) + if not base_module_mmd: + # If there isn't this base module stream or virtual stream available, skip it, + # and let the dep solving code deal with it like it normally would + log.warning( + 'There is no base module "%s" with stream/virtual stream "%s"', + base_module, + stream + ) + continue + + latest_stream = base_module_mmd.get_stream() + log.info( + ('Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual ' + 'stream'), + base_module, + stream, + base_module, + latest_stream, + stream + ) + new_streams[i] = latest_stream + overridden = True + + if streams != new_streams: + brs[base_module].set(new_streams) + + if overridden: + dep.set_buildrequires(brs) + + if overridden: + mmd.set_dependencies(deps) + + def submit_module_build(username, mmd, params): """ Submits new module build. @@ -631,6 +702,7 @@ def submit_module_build(username, mmd, params): if "default_streams" in params: default_streams = params["default_streams"] _apply_dep_overrides(mmd, params) + _handle_base_module_virtual_stream_br(mmd) mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams) if not mmds: diff --git a/tests/staged_data/testmodule_el8.yaml b/tests/staged_data/testmodule_el8.yaml new file mode 100644 index 0000000..e9c1515 --- /dev/null +++ b/tests/staged_data/testmodule_el8.yaml @@ -0,0 +1,37 @@ +document: modulemd +version: 1 +data: + summary: A test module in all its beautiful beauty + description: >- + This module demonstrates how to write simple modulemd files And + can be used for testing the build and release pipeline. ’ + license: + module: [ MIT ] + dependencies: + buildrequires: + platform: el8 + requires: + platform: el8 + references: + community: https://docs.pagure.org/modularity/ + documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules + profiles: + default: + rpms: + - tangerine + api: + rpms: + - perl-Tangerine + - tangerine + components: + rpms: + perl-List-Compare: + rationale: A dependency of tangerine. + ref: master + perl-Tangerine: + rationale: Provides API for this module and is a dependency of tangerine. + ref: master + tangerine: + rationale: Provides API for this module. + buildorder: 10 + ref: master diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index ac1e319..1391b77 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -173,6 +173,14 @@ class TestModelsGetStreamsContexts: assert builds == set(['platform:f29.1.0:15:c11', 'platform:f29.1.0:15:c11.another', 'platform:f29.2.0:1:c11']) + def test_get_module_count(self): + clean_database(False) + make_module("platform:f29.1.0:10:c11", {}, {}) + make_module("platform:f29.1.0:10:c12", {}, {}) + with make_session(conf) as session: + count = ModuleBuild.get_module_count(session, name="platform") + assert count == 2 + def test_add_virtual_streams_filter(self): clean_database(False) make_module("platform:f29.1.0:10:c1", {}, {}, virtual_streams=["f29"]) diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 6ac5baf..ade8a41 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -227,3 +227,20 @@ class TestDBModule: set(['bar']) } assert result == expected + + def test_get_latest_with_virtual_stream(self): + tests.init_data(1, multiple_stream_versions=True) + resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db') + mmd = resolver.get_latest_with_virtual_stream('platform', 'f29') + assert mmd + assert mmd.get_stream() == 'f29.2.0' + + def test_get_latest_with_virtual_stream_none(self): + resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db') + mmd = resolver.get_latest_with_virtual_stream('platform', 'doesnotexist') + assert not mmd + + def test_get_module_count(self): + resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db') + count = resolver.get_module_count(name='platform', stream='f28') + assert count == 1 diff --git a/tests/test_resolver/test_mbs.py b/tests/test_resolver/test_mbs.py index 5d4a2ac..de7dc6a 100644 --- a/tests/test_resolver/test_mbs.py +++ b/tests/test_resolver/test_mbs.py @@ -367,3 +367,73 @@ class TestMBSModule: assert '10' == mmd.get_stream() assert 1 == mmd.get_version() assert 'c1' == mmd.get_context() + + @patch("requests.Session") + def test_get_module_count(self, mock_session): + mock_res = Mock() + mock_res.ok.return_value = True + mock_res.json.return_value = { + "items": [ + { + "name": "platform", + "stream": "f28", + "version": "3", + "context": "00000000", + } + ], + "meta": { + "total": 5 + } + } + mock_session.return_value.get.return_value = mock_res + + resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs") + count = resolver.get_module_count(name="platform", stream="f28") + + assert count == 5 + mock_session.return_value.get.assert_called_once_with( + "https://mbs.fedoraproject.org/module-build-service/1/module-builds/", + params={ + "name": "platform", + "page": 1, + "per_page": 1, + "short": True, + "stream": "f28", + } + ) + + @patch("requests.Session") + def test_get_latest_with_virtual_stream(self, mock_session, platform_mmd): + mock_res = Mock() + mock_res.ok.return_value = True + mock_res.json.return_value = { + "items": [ + { + "context": "00000000", + "modulemd": platform_mmd, + "name": "platform", + "stream": "f28", + "version": "3", + } + ], + "meta": { + "total": 5 + } + } + mock_session.return_value.get.return_value = mock_res + + resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs") + mmd = resolver.get_latest_with_virtual_stream("platform", "virtualf28") + + assert mmd.get_name() == "platform" + mock_session.return_value.get.assert_called_once_with( + "https://mbs.fedoraproject.org/module-build-service/1/module-builds/", + params={ + "name": "platform", + "order_desc_by": ["stream_version", "version"], + "page": 1, + "per_page": 1, + "verbose": True, + "virtual_stream": "virtualf28", + } + ) diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 62bdd2c..e373684 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -32,7 +32,7 @@ from os.path import basename, dirname, splitext from requests.utils import quote import hashlib import pytest -from module_build_service.utils import to_text_type, load_mmd_file +from module_build_service.utils import to_text_type, load_mmd_file, load_mmd import re from tests import app, init_data, clean_database, reuse_component_init_data @@ -2111,3 +2111,31 @@ class TestViews: data = json.loads(rv.data)[0] mmd = module_build_service.utils.load_mmd(data['modulemd']) assert mmd.get_xmd()['mbs']['disttag_marking'] == 'product12' + + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_request_platform_virtual_stream(self, mocked_scm, mocked_get_user): + # Create a platform with el8.25.0 but with the virtual stream el8 + mmd = load_mmd_file(path.join(base_dir, 'staged_data', 'platform.yaml')) + mmd.set_stream('el8.25.0') + xmd = from_variant_dict(mmd.get_xmd()) + xmd['mbs']['virtual_streams'] = ['el8'] + mmd.set_xmd(dict_values(xmd)) + import_mmd(db.session, mmd) + + # Use a testmodule that buildrequires platform:el8 + FakeSCM(mocked_scm, 'testmodule', 'testmodule_el8.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + + post_url = '/module-build-service/2/module-builds/' + scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe' + 'efbd35246a81b6cb8d49') + rv = self.client.post(post_url, data=json.dumps({'branch': 'master', 'scmurl': scm_url})) + data = json.loads(rv.data) + print(data) + + mmd = load_mmd(data[0]['modulemd']) + assert len(mmd.get_dependencies()) == 1 + dep = mmd.get_dependencies()[0] + assert set(dep.get_buildrequires()['platform'].get()) == set(['el8.25.0']) + assert set(dep.get_requires()['platform'].get()) == set(['el8'])