PR#1204: Allow getting the latest stream of base module. - fm-orchestrator

fm-orchestrator

#1204 Allow getting the latest stream of base module.

Merged 5 years ago by mprahl. Opened 5 years ago by jkaluza.

jkaluza/fm-orchestrator latest-stream into master

Allow getting the latest stream of base module.

Jan Kaluza • 5 years ago

bc4a019

module_build_service/utils/mse.py

file modified

+45 -2

		`@@ -188,6 +188,39 @@`
		`return mmds`


		`+ def _get_latest_base_module_mmd_with_virtual_stream(name, virtual_stream):`
		`+ """`
		`+ Returns the Modulemd instance of Module build with highest stream_version,`
		+ with name `name`, and containing the virtual stream `virtual_stream` in its
		`+ xmd["mbs"]["virtual_streams"] section`
		`+`
		`+ :param str name: Name of a module to return.`
		`+ :param str virtual_stream: Name of virtual stream which must be included in`
		`+ the virtual_streams xmd section.`
		`+ :return Modulemd: Modulemd instance of such module with highest stream_version.`
		`+ """`
		`+ with models.make_session(conf) as session:`
		`+ builds = models.ModuleBuild.get_last_build_in_all_streams(session, name)`
		`+ if not builds:`
		`+ return None`
		`+`
		`+ build_to_return = None`
		`+ for build in builds:`
		`+ mmd = build.mmd()`
		`+ xmd = mmd.get_xmd()`
		`+ if "mbs" not in xmd.keys() or "virtual_streams" not in xmd["mbs"].keys():`
		`+ continue`
		`+`
		`+ virtual_streams = xmd["mbs"]["virtual_streams"]`
		`+ if virtual_stream not in virtual_streams:`
		`+ continue`
		`+`
		`+ if not build_to_return or build_to_return.stream_version < build.stream_version:`
		`+ build_to_return = build`
		`+`
		`+ return build_to_return.mmd() if build_to_return else None`
		`+`
		`+`
		`def _get_base_module_mmds(mmd):`
		`"""`
		Returns list of MMDs of base modules buildrequired by `mmd` including the compatible
		`@@ -226,8 +259,18 @@`
		`# zero or one module build.`
		`mmds = resolver.get_module_modulemds(name, stream)`
		`if not mmds:`
		`- continue`
		`- stream_mmd = mmds[0]`
		`+ # The given name:stream base module does not exist. It might be the case`
		`+ # when user requests name:virtual_stream (like platform:f29). In this case,`
		`+ # we need to find the latest (the one with highest stream_version) base`
		`+ # module.`
		`+ stream_mmd = _get_latest_base_module_mmd_with_virtual_stream(name, stream)`
		`+ if not stream_mmd:`
		`+ continue`
		+ # The `stream` does not exist and we are going to use the stream name of
		`+ # the latest base module with given virtual stream, so override it here.`
		`+ stream = stream_mmd.get_stream()`
		`+ else:`
		`+ stream_mmd = mmds[0]`

		`# In case there are no virtual_streams in the buildrequired name:stream,`
		`# it is clear that there are no compatible streams, so return just this`

tests/test_utils/test_utils_mse.py

file modified

+25

		`@@ -403,3 +403,28 @@`
		`for mmd_ in mmds:`
		`actual.add('{}:{}'.format(mmd_.get_name(), mmd_.get_stream()))`
		`assert actual == expected`
		`+`
		`+ def test__get_base_module_mmds_virtual_streams_latest_stream_version(self):`
		`+ init_data(data_size=1, multiple_stream_versions=True)`
		`+ mmd = module_build_service.utils.load_mmd(`
		`+ os.path.join(base_dir, 'staged_data', 'testmodule_v2.yaml'), True)`
		`+ deps = mmd.get_dependencies()`
		`+ brs = deps[0].get_buildrequires()`
		`+ brs['platform'].set(['f35'])`
		`+ deps[0].set_buildrequires(brs)`
		`+ mmd.set_dependencies(deps)`
		`+`
		`+ make_module("platform:f35.1.0:1:1", {}, {}, virtual_streams=["f35"])`
		`+ make_module("platform:f35.2.0:1:1", {}, {}, virtual_streams=["f35"])`
		`+ make_module("platform:f35.0.0:1:1", {}, {}, virtual_streams=["f35"])`
		`+`
		`+ mmds = module_build_service.utils.mse._get_base_module_mmds(mmd)`
		`+ expected = set(['platform:f35.0.0', 'platform:f35.1.0', 'platform:f35.2.0'])`
		`+`
		`+ # Verify no duplicates were returned before doing set operations`
		`+ assert len(mmds) == len(expected)`
		`+ # Verify the expected ones were returned`
		`+ actual = set()`
		`+ for mmd_ in mmds:`
		`+ actual.add('{}:{}'.format(mmd_.get_name(), mmd_.get_stream()))`
		`+ assert actual == expected`

jkaluza commented 5 years ago

If base module name:stream is not found, consider stream as virtual stream
and return the latest (the one with highest stream_version) base module with
that virtual stream.

rebased onto 8821ae96c6070f91bba7da6044f1ad3e18c5fddb

5 years ago

mprahl commented 5 years ago

My concern is that performance will suffer every time more platforms are added. I think the better approach is to store the virtual streams in the database so that one SQL query could give the module with that virtual stream and with the latest stream version.

Otherwise it looks good.

rebased onto bc4a019

5 years ago

mprahl commented 5 years ago

We can convert this to use the database in a future PR. I'll merge this now.