From b31c2c67f038e4eb06ca32057c7b2708f0b27982 Mon Sep 17 00:00:00 2001 From: Jan Kaluža Date: Nov 21 2017 07:44:06 +0000 Subject: Merge #130 `Add 'results' to new_compose API to list all the additional results of a compose. Add 'boot.iso' result type to generate boot.iso needed for base container image rebuilds.` --- diff --git a/README.md b/README.md index 79a8ddf..448cfbb 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ There are also additional optional attributes you can pass to `new_compose(...)` - `flags` - List of flags to further modify the compose output: - `no_deps` - For `tag` `source_type`, do not resolve dependencies between packages and include only packages listed in the `packages` in the compose. For `module` `source_type`, do not resolve dependencies between modules and include only the requested module in the compose. - `sigkeys` - List of signature keys IDs. Only packages signed by one of these keys will be included in a compose. If there is no signed version of a package, compose will fail. It is also possible to pass an empty-string in a list meaning unsigned packages are allowed. For example if you want to prefer packages signed by key with ID `123` and also allow unsigned packages to appear in a compose, you can do it by setting sigkeys to `["123", ""]`. +- `results` - List of additional results which will be generated as part of a compose. Valid keys are: + - `iso` - Generates non-installable ISO files with RPMs from a compose. + - `boot.iso` - Generates `images/boot.iso` file which is needed to build base container images from resulting compose. The `new_compose` method returns `dict` object describing the compose, for example: diff --git a/client/odcs/client/odcs.py b/client/odcs/client/odcs.py index 921ac98..9f2fc81 100644 --- a/client/odcs/client/odcs.py +++ b/client/odcs/client/odcs.py @@ -194,7 +194,7 @@ class ODCS(object): def new_compose(self, source, source_type, seconds_to_live=None, packages=[], flags=[], - sigkeys=None): + sigkeys=None, results=None): """Request a new compose :param str source: from where to grab and make new compose, different @@ -216,6 +216,10 @@ class ODCS(object): listed in ``source`` without their dependencies. :param list sigkeys: List of signature keys by which the packages in compose must be signed. + :param list results: List of additional results which will be generated + by ODCS as part of this compose. Can be "iso" for iso files with + packages or "boot.iso" for images/boot.iso needed to generate + container base images or installable DVDs. :return: the newly created Compose :rtype: dict """ @@ -230,6 +234,8 @@ class ODCS(object): request_data['seconds-to-live'] = seconds_to_live if flags: request_data['flags'] = flags + if results: + request_data['results'] = results r = self._post('composes/', request_data) return r.json() diff --git a/client/tests/test_client_odcs.py b/client/tests/test_client_odcs.py index 6ceb9a3..2ee09b7 100644 --- a/client/tests/test_client_odcs.py +++ b/client/tests/test_client_odcs.py @@ -234,7 +234,8 @@ class TestNewCompose(unittest.TestCase): new_compose = self.odcs.new_compose('cf-1.0-rhel-5', 'tag', packages=['libdnet'], - sigkeys=['123', '456']) + sigkeys=['123', '456'], + results=["boot.iso"]) self.assertEqual(fake_new_compose, new_compose) requests.post.assert_called_once_with( @@ -243,7 +244,8 @@ class TestNewCompose(unittest.TestCase): 'source': {'source': 'cf-1.0-rhel-5', 'type': 'tag', 'packages': ['libdnet'], - 'sigkeys': ['123', '456']} + 'sigkeys': ['123', '456']}, + 'results': ['boot.iso'], }), headers={'Content-Type': 'application/json'} ) diff --git a/common/odcs/common/types.py b/common/odcs/common/types.py index d4cbd10..88d2f48 100644 --- a/common/odcs/common/types.py +++ b/common/odcs/common/types.py @@ -58,6 +58,7 @@ COMPOSE_RESULTS = { "repository": 1, "iso": 2, "ostree": 4, + "boot.iso": 8, } COMPOSE_FLAGS = { diff --git a/server/conf/config.py b/server/conf/config.py index 64ada9b..f0fd61b 100644 --- a/server/conf/config.py +++ b/server/conf/config.py @@ -138,8 +138,8 @@ class TestConfiguration(BaseConfiguration): LOG_LEVEL = 'debug' DEBUG = True - SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format( - path.join(dbdir, 'test_odcs.db')) + # Use in-memory sqlite db to make tests fast. + SQLALCHEMY_DATABASE_URI = 'sqlite://' PUNGI_CONF_PATH = path.join(confdir, 'pungi.conf') # Global network-related values, in seconds diff --git a/server/conf/pungi.conf b/server/conf/pungi.conf index e5f2834..b8d8334 100644 --- a/server/conf/pungi.conf +++ b/server/conf/pungi.conf @@ -67,7 +67,9 @@ skip_phases = [ {%- if "iso" not in config.results %} "createiso", {%- endif %} +{%- if "boot.iso" not in config.results %} "buildinstall", +{%- endif %} "live_media", "live_images", "ostree"] diff --git a/server/odcs/server/models.py b/server/odcs/server/models.py index f055514..3e29669 100644 --- a/server/odcs/server/models.py +++ b/server/odcs/server/models.py @@ -37,7 +37,8 @@ from odcs.server import db from odcs.server.events import cache_composes_if_state_changed from odcs.server.events import start_to_publish_messages from odcs.common.types import ( - COMPOSE_STATES, INVERSE_COMPOSE_STATES, COMPOSE_FLAGS) + COMPOSE_STATES, INVERSE_COMPOSE_STATES, COMPOSE_FLAGS, + COMPOSE_RESULTS) from sqlalchemy import event, or_ from flask_sqlalchemy import SignallingSession @@ -228,6 +229,14 @@ class Compose(ODCSBase): continue if self.flags & value: flags.append(name) + + results = [] + for name, value in COMPOSE_RESULTS.items(): + if value == 0: + continue + if self.results & value: + results.append(name) + return { 'id': self.id, 'owner': self.owner, @@ -242,6 +251,7 @@ class Compose(ODCSBase): 'result_repo': self.result_repo_url, 'result_repofile': self.result_repofile_url, 'flags': flags, + 'results': results, 'sigkeys': self.sigkeys if self.sigkeys else "" } diff --git a/server/odcs/server/views.py b/server/odcs/server/views.py index 40c3baa..0554a11 100644 --- a/server/odcs/server/views.py +++ b/server/odcs/server/views.py @@ -224,9 +224,16 @@ class ODCSAPI(MethodView): raise ValueError("Unknown flag %s", name) flags |= COMPOSE_FLAGS[name] + results = COMPOSE_RESULTS["repository"] + if "results" in data: + for name in data["results"]: + if name not in COMPOSE_RESULTS: + raise ValueError("Unknown result %s", name) + results |= COMPOSE_RESULTS[name] + compose = Compose.create( db.session, self._get_compose_owner(), source_type, source, - COMPOSE_RESULTS["repository"], seconds_to_live, + results, seconds_to_live, packages, flags, sigkeys) db.session.add(compose) db.session.commit() diff --git a/server/tests/test_models.py b/server/tests/test_models.py index a160435..b33dc40 100644 --- a/server/tests/test_models.py +++ b/server/tests/test_models.py @@ -58,6 +58,7 @@ class TestModels(ModelsBaseTest): 'time_removed': None, 'time_to_expire': c.json()["time_to_expire"], 'flags': [], + 'results': ['repository'], 'sigkeys': ''} self.assertEqual(c.json(), expected_json) diff --git a/server/tests/test_pungi.py b/server/tests/test_pungi.py index 26ea7f2..a657340 100644 --- a/server/tests/test_pungi.py +++ b/server/tests/test_pungi.py @@ -83,6 +83,7 @@ class TestPungiConfig(unittest.TestCase): self.assertEqual(cfg["release_short"], "MBS-512") self.assertEqual(cfg["release_version"], "1") self.assertTrue("createiso" in cfg["skip_phases"]) + self.assertTrue("buildinstall" in cfg["skip_phases"]) @patch("odcs.server.pungi.log") def test_get_pungi_conf_exception(self, log): @@ -111,6 +112,20 @@ class TestPungiConfig(unittest.TestCase): cfg = self._load_pungi_cfg(template) self.assertTrue("createiso" not in cfg["skip_phases"]) + def test_get_pungi_conf_boot_iso(self): + _, mock_path = tempfile.mkstemp() + template_path = os.path.abspath(os.path.join(test_dir, + "../conf/pungi.conf")) + shutil.copy2(template_path, mock_path) + + with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path): + pungi_cfg = PungiConfig("MBS-512", "1", PungiSourceType.MODULE, + "testmodule-master", + results=COMPOSE_RESULTS["boot.iso"]) + template = pungi_cfg.get_pungi_config() + cfg = self._load_pungi_cfg(template) + self.assertTrue("buildinstall" not in cfg["skip_phases"]) + def test_get_pungi_conf_koji_inherit(self): _, mock_path = tempfile.mkstemp() template_path = os.path.abspath(os.path.join(test_dir, diff --git a/server/tests/test_views.py b/server/tests/test_views.py index 191cc0a..e62b5a4 100644 --- a/server/tests/test_views.py +++ b/server/tests/test_views.py @@ -222,6 +222,7 @@ class TestViews(ViewBaseTest): 'time_removed': None, 'time_to_expire': data["time_to_expire"], 'flags': [], + 'results': ['repository'], 'sigkeys': ''} self.assertEqual(data, expected_json) @@ -265,6 +266,26 @@ class TestViews(ViewBaseTest): self.assertEqual(c.state, COMPOSE_STATES["wait"]) self.assertEqual(c.flags, COMPOSE_FLAGS["no_inheritance"]) + def test_submit_build_boot_iso(self): + with self.test_request_context(user='dev'): + flask.g.oidc_scopes = [ + '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose') + ] + + rv = self.client.post('/api/1/composes/', data=json.dumps( + {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']}, + 'results': ['boot.iso']})) + data = json.loads(rv.data.decode('utf8')) + + self.assertEqual(data['results'], ['repository', 'boot.iso']) + + db.session.expire_all() + c = db.session.query(Compose).filter(Compose.id == 3).one() + self.assertEqual(c.state, COMPOSE_STATES["wait"]) + self.assertEqual( + c.results, + COMPOSE_RESULTS["boot.iso"] | COMPOSE_RESULTS["repository"]) + def test_submit_build_sigkeys(self): with self.test_request_context(user='dev'): flask.g.oidc_scopes = [ @@ -652,6 +673,7 @@ class TestViews(ViewBaseTest): 'time_removed': None, 'time_to_expire': data["time_to_expire"], 'flags': [], + 'results': ['repository'], 'sigkeys': ''} self.assertEqual(data, expected_json)