PR#290: Add new `include_done_modules` flag to generate compose with modules in `done` state. - odcs

odcs

#290 Add new `include_done_modules` flag to generate compose with modules in `done` state.

Merged 4 years ago by jkaluza. Opened 4 years ago by jkaluza.

jkaluza/odcs done-state into master

Add new `include_done_modules` flag to generate compose with modules in `done` state.

Jan Kaluza • 4 years ago

c770442

common/odcs/common/types.py

file modified

		`@@ -101,6 +101,8 @@`
		`"include_unpublished_pulp_repos": 4,`
		`# Abort the compose when some package has broken dependencies.`
		`"check_deps": 8,`
		`+ # Include also module builds in the "done" state.`
		`+ "include_done_modules": 16,`
		`}`

		`INVERSE_COMPOSE_FLAGS = {v: k for k, v in COMPOSE_FLAGS.items()}`

docs/api.rst

file modified

+2 -1

		`@@ -65,6 +65,7 @@`
		`- no_inheritance - Only packages/modules directly tagged in the requested Koji tag will be added to the module. Inherited tags will be ignored.`
		`- include_unpublished_pulp_repos - Even unpublished Pulp repositories will be included in the resulting compose.`
		`- check_deps - Compose will fail if the RPM-level dependencies between packages in the compose are not satisfied.`
		+ - include_done_modules - Compose can include also modules which are in the ``done`` state. By default, only modules in ``ready`` state are allowed to be included in a composes.

		`.. _koji_event:`

		`@@ -139,7 +140,7 @@`
		Number defining the type of ``source`` giving it particular meaning:

		- 1 (``tag``) - The ``source`` is name of Koji tag to take the builds from. Additional Koji builds can be added by when the ``builds`` option is set.
		- - 2 (``module``) - The ``source`` is the list of modules in ``N:S``, ``N:S:V`` or ``N:S:V:C`` format. When using ``N:S`` format, the latest version found in MBS is used.
		+ - 2 (``module``) - The ``source`` is the list of modules in ``N:S``, ``N:S:V`` or ``N:S:V:C`` format. When using ``N:S`` format, the latest version found in MBS is used. When using ``N:S`` or ``N:S:V``, the module needs to be in the ``ready`` state in the MBS. When using ``N:S:V:C``, the module can be even in the ``done`` state in the MBS.
		- 3 (``repo``) - The ``source`` is full path to repository from which the packages are taken. This is often disabled source type by deployed ODCS servers.
		- 4 (``pulp``) - The ``source`` is the list of Pulp content-sets. Repositories defined by these content-sets will be included in the compose.
		- 5 (``raw_config``) - The ``source`` is string in the ``name#commit`` hash format. The ``name`` must match one of the raw config locations defined in ODCS server config as ``raw_config_urls``. The ``commit`` is commit hash defining the version of raw config to use. This config is then used as input config for Pungi.

server/odcs/server/backend.py

file modified

+7 -1

		`@@ -350,9 +350,15 @@`
		`mbs = odcs.server.mbs.MBS(conf)`
		`modules = compose.source.split(" ")`

		`+ include_done = compose.flags & COMPOSE_FLAGS["include_done_modules"]`
		`specified_mbs_modules = []`
		`for module in modules:`
		`- specified_mbs_modules += mbs.get_latest_modules(module)`
		`+ # In case the module is defined by complete NSVC, include it in a compose`
		`+ # even if it is in "done" state, because submitter directly asked for this`
		`+ # NSVC.`
		`+ is_complete_nsvc = module.count(":") == 3`
		`+ specified_mbs_modules += mbs.get_latest_modules(`
		`+ module, include_done or is_complete_nsvc)`

		`expand = not compose.flags & COMPOSE_FLAGS["no_deps"]`
		`new_mbs_modules = mbs.validate_module_list(specified_mbs_modules, expand=expand)`

server/odcs/server/mbs.py

file modified

+6 -3

		`@@ -46,17 +46,19 @@`
		`r.raise_for_status()`
		`return r.json()`

		`- def get_latest_modules(self, nsvc):`
		`+ def get_latest_modules(self, nsvc, include_done=False):`
		`"""`
		`Query MBS and return the latest version of the module specified by nsvc.`

		`:param nsvc: N:S:V[:C] of a module to include in a compose.`
		`+ :param include_done: When True, also module builds in the "done" state`
		`+ are included in a result.`
		`:raises ModuleLookupError: if the module couldn't be found`
		`:return: the latest version of the module.`
		`"""`
		`params = {`
		`"nsvc": nsvc,`
		`- "state": 5, # 5 is "ready".`
		`+ "state": [3, 5] if include_done else [5], # 5 is "ready", 3 is "done".`
		`"verbose": True, # Needed to get modulemd in response.`
		`"order_desc_by": "version",`
lucarval commented 4 years ago You probably want to also sort by state so you prefer builds in ready state over done state.
		`}`
		`@@ -76,8 +78,9 @@`
		`devel_module = True`

		`if not modules["meta"]["total"]:`
		`+ state_msg = "ready or done" if include_done else "ready"`
		`raise ModuleLookupError(`
		`- "Failed to find module %s in the MBS." % nsvc)`
		`+ "Failed to find module %s in %s state in the MBS." % (nsvc, state_msg))`

		`ret = []`
		`# In case the nsvc is just "name:stream", there might be multiple`

server/tests/mbs.py

file modified

+9 -2

		`@@ -34,7 +34,7 @@`


		`def make_module(name, stream, version, requires={}, mdversion=1,`
		`- context=None):`
		`+ context=None, state=5):`
		`mmd = Modulemd.Module()`
		`mmd.set_mdversion(mdversion)`
		`mmd.set_name(name)`
		`@@ -60,7 +60,8 @@`
		`'stream': stream,`
		`'version': str(version),`
		`'context': context or '00000000',`
		`- 'modulemd': mmd.dumps()`
		`+ 'modulemd': mmd.dumps(),`
		`+ 'state': state,`
		`}`


		`@@ -104,6 +105,9 @@`

		`make_module('moduleD', 'f26', 20170806000000, {}, 2),`

		`+ # module builds in "done" state.`
		`+ make_module('testmodule', 'master', 20180515074419, {}, 2, state=3),`
		`+`
		`# test_composerthread.py`
		`make_module('testmodule', 'master', 20170515074418, {}, 2),`
		`make_module('testmodule', 'master', 20170515074419, {}, 2),`
		`@@ -128,6 +132,7 @@`
		`def wrapped(args, *kwargs):`
		`def handle_module_builds(request):`
		`query = parse_qs(urlparse(request.url).query)`
		`+ states = [int(s) for s in query['state']]`
		`nsvc = query['nsvc'][0]`
		`nsvc_parts = nsvc.split(":")`
		`nsvc_keys = ["name", "stream", "version", "context"]`
		`@@ -147,6 +152,8 @@`
		`if key in nsvc_dict and nsvc_dict[key] != module[key]:`
		`skip = True`
		`break`
		`+ if module["state"] not in states:`
		`+ skip = True`
		`if skip:`
		`continue`
		`body["items"].append(module)`

server/tests/test_backend.py

file modified

+30 -1

		`@@ -78,6 +78,35 @@`
		`"moduleD:f26:20170806000000:00000000"]))`

		`@mock_mbs()`
		`+ def test_resolve_compose_module_include_done_modules(self):`
		`+ c = Compose.create(`
		`+ db.session, "me", PungiSourceType.MODULE,`
		`+ "testmodule:master",`
		`+ COMPOSE_RESULTS["repository"], 3600,`
		`+ flags=COMPOSE_FLAGS["include_done_modules"])`
		`+ db.session.commit()`
		`+`
		`+ resolve_compose(c)`
		`+ db.session.commit()`
		`+`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ self.assertEqual(c.source, "testmodule:master:20180515074419:00000000")`
		`+`
		`+ @mock_mbs()`
		`+ def test_resolve_compose_module_include_done_modules_full_nsvc(self):`
		`+ c = Compose.create(`
		`+ db.session, "me", PungiSourceType.MODULE,`
		`+ "testmodule:master:20180515074419:00000000",`
		`+ COMPOSE_RESULTS["repository"], 3600)`
		`+ db.session.commit()`
		`+`
		`+ resolve_compose(c)`
		`+ db.session.commit()`
		`+`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ self.assertEqual(c.source, "testmodule:master:20180515074419:00000000")`
		`+`
		`+ @mock_mbs()`
		`def test_resolve_compose_module_devel(self):`
		`c = Compose.create(`
		`db.session, "me", PungiSourceType.MODULE,`
		`@@ -260,7 +289,7 @@`
		`def test_resolve_compose_module_dep_not_found(self):`
		`self.expect_module_lookup_error(`
		`"moduleB:f26 moduleB:f27",`
		`- "Failed to find module moduleC:f27 in the MBS.")`
		`+ "Failed to find module moduleC:f27 in ready state in the MBS.")`

		`@patch("odcs.server.backend.create_koji_session")`
		`def test_resolve_compose_repo_no_override_koji_event(`

jkaluza commented 4 years ago

Currently, only modules in the ready state can appear in a compose.
This is correct for most cases, but the ODCS is also used by gating
CI which needs to generate compose from just-built module in the done
state in order to test it.

New flag is added in this commit to allow this behaviour.

rebased onto 3c38a9814e6e3a24fb2bba343a20bbca7ad2dfb8

4 years ago