From 6525a29f9f471705d1592a96bb991d7cd7489e7b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Jan 09 2020 15:05:58 +0000 Subject: Generate ComposeInfo which is valid also for raw_configs. Signed-off-by: Jan Kaluza --- diff --git a/client/contrib/odcs b/client/contrib/odcs index 4c8713e..42a79b7 100755 --- a/client/contrib/odcs +++ b/client/contrib/odcs @@ -124,6 +124,12 @@ create_parser.add_argument( '--lookaside-repo', default=[], action='append', metavar="lookaside_repos", help="Koji tag with module builds.") +create_parser.add_argument( + '--label', default=None, + help="Label for raw_config compose.") +create_parser.add_argument( + '--compose-type', default=None, + help="Compose type for raw_config compose.") wait_parser = subparsers.add_parser( 'wait', help='wait for a compose to finish') @@ -218,6 +224,8 @@ try: module_defaults_url=args.module_defaults_url, module_defaults_commit=args.module_defaults_commit, lookaside_repos=args.lookaside_repo, + label=args.label, + compose_type=args.compose_type ) elif args.command == "wait": result = {"id": int(args.compose_id)} diff --git a/client/odcs/client/odcs.py b/client/odcs/client/odcs.py index b2e29ca..d0a70e7 100644 --- a/client/odcs/client/odcs.py +++ b/client/odcs/client/odcs.py @@ -214,7 +214,7 @@ class ODCS(object): sigkeys=None, koji_event=None, results=None, arches=None, builds=None, modular_koji_tags=None, module_defaults_url=None, module_defaults_commit=None, - lookaside_repos=None): + lookaside_repos=None, label=None, compose_type=None): """Request a new compose :param str source: from where to grab and make new compose, different @@ -263,6 +263,10 @@ class ODCS(object): request_data['source']['module_defaults_commit'] = module_defaults_commit if lookaside_repos: request_data['lookaside_repos'] = lookaside_repos + if label: + request_data['label'] = label + if compose_type: + request_data['compose_type'] = compose_type if seconds_to_live is not None: request_data['seconds-to-live'] = seconds_to_live if flags: diff --git a/docs/api.rst b/docs/api.rst index db2d959..6776b56 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -46,6 +46,11 @@ The fields used in the ODCS compose JSON have following meaning: *arches* - ``(white-space separated list of strings)`` List of architectures the compose is generated for. The strings to express particular architecture are the same as the ones used by Koji build systemd. +.. _compose_type: + +*compose_type* - ``(string)`` + Type of the compose when generating raw_config compose. Can be "test", "nightly", "ci", "production". + .. _builds: *builds* - ``(white-space separated list of strings or null)`` @@ -72,6 +77,11 @@ The fields used in the ODCS compose JSON have following meaning: *koji_event* - ``(number or null)`` The Koji event defining the point in Koji history when the compose was generated. It can be ``null`` if source type does not relate to Koji tag. +.. _label: + +*label* - ``(string)`` + Compose label when generating raw_config compose. + .. _lookaside_repos: *lookaside_repos* - ``(white-space separated list of strings or null)`` diff --git a/server/odcs/server/api_utils.py b/server/odcs/server/api_utils.py index 01a5250..37221db 100644 --- a/server/odcs/server/api_utils.py +++ b/server/odcs/server/api_utils.py @@ -270,7 +270,8 @@ def filter_composes(flask_request): """ search_query = dict() - for key in ['owner', 'source_type', 'source', 'state', 'koji_task_id']: + for key in ['owner', 'source_type', 'source', 'state', 'koji_task_id', + 'pungi_compose_id', 'compose_type', 'label']: if flask_request.args.get(key, None): search_query[key] = flask_request.args[key] diff --git a/server/odcs/server/migrations/versions/cd0781bbdab1_.py b/server/odcs/server/migrations/versions/cd0781bbdab1_.py new file mode 100644 index 0000000..d2e862d --- /dev/null +++ b/server/odcs/server/migrations/versions/cd0781bbdab1_.py @@ -0,0 +1,26 @@ +"""Add compose_type, label and pungi_compose_id columns. + +Revision ID: cd0781bbdab1 +Revises: de0a86d7de49 +Create Date: 2020-01-09 11:14:30.378081 + +""" + +# revision identifiers, used by Alembic. +revision = 'cd0781bbdab1' +down_revision = 'de0a86d7de49' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('composes', sa.Column('compose_type', sa.String(), nullable=True)) + op.add_column('composes', sa.Column('label', sa.String(), nullable=True)) + op.add_column('composes', sa.Column('pungi_compose_id', sa.String(), nullable=True)) + + +def downgrade(): + op.drop_column('composes', 'pungi_compose_id') + op.drop_column('composes', 'label') + op.drop_column('composes', 'compose_type') diff --git a/server/odcs/server/models.py b/server/odcs/server/models.py index 9c44fa1..61d2799 100644 --- a/server/odcs/server/models.py +++ b/server/odcs/server/models.py @@ -149,13 +149,20 @@ class Compose(ODCSBase): multilib_method = db.Column(db.Integer) # White-space separated lookaside repository URLs. lookaside_repos = db.Column(db.String, nullable=True) + # Compose label stored to ComposeInfo for Raw config composes. + label = db.Column(db.String, nullable=True) + # Compose type stored to ComposeInfo for Raw config composes. + compose_type = db.Column(db.String, nullable=True) + # Compose id as generated by Pungi for its ComposeInfo metadata. + pungi_compose_id = db.Column(db.String, nullable=True) @classmethod def create(cls, session, owner, source_type, source, results, seconds_to_live, packages=None, flags=0, sigkeys=None, koji_event=None, arches=None, multilib_arches=None, multilib_method=None, builds=None, lookaside_repos=None, - modular_koji_tags=None, module_defaults_url=None): + modular_koji_tags=None, module_defaults_url=None, + label=None, compose_type=None): now = datetime.utcnow() compose = cls( owner=owner, @@ -176,6 +183,8 @@ class Compose(ODCSBase): lookaside_repos=lookaside_repos, modular_koji_tags=modular_koji_tags, module_defaults_url=module_defaults_url, + label=label, + compose_type=compose_type, ) session.add(compose) return compose @@ -210,6 +219,11 @@ class Compose(ODCSBase): lookaside_repos=compose.lookaside_repos, modular_koji_tags=compose.modular_koji_tags, module_defaults_url=compose.module_defaults_url, + label=compose.label, + compose_type=compose.compose_type, + # Set pungi_compose_id to None, because it is regenerated once + # this copied Compose is started. + pungi_compose_id=None, ) session.add(compose) return compose @@ -335,6 +349,9 @@ class Compose(ODCSBase): 'lookaside_repos': self.lookaside_repos, 'modular_koji_tags': self.modular_koji_tags, 'module_defaults_url': self.module_defaults_url, + 'label': self.label, + 'compose_type': self.compose_type, + 'pungi_compose_id': self.pungi_compose_id, } @staticmethod diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index 197f5c6..770d95c 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -28,10 +28,12 @@ import tempfile import jinja2 import time from productmd.composeinfo import ComposeInfo +from kobo.conf import PyConfigParser import odcs.server.utils -from odcs.server import conf, log +from odcs.server import conf, log, db from odcs.server import comps +from odcs.server.models import Compose from odcs.common.types import ( PungiSourceType, COMPOSE_RESULTS, MULTILIB_METHODS, INVERSE_PUNGI_SOURCE_TYPE_NAMES, COMPOSE_FLAGS) @@ -336,7 +338,7 @@ class Pungi(object): pungi_cmd += ["--old-composes", self.old_compose] return pungi_cmd - def _prepare_compose_dir(self, conf_topdir, targetdir): + def _prepare_compose_dir(self, compose, conf_topdir, targetdir): """ Creates the compose directory and returns the full path to it. """ @@ -346,30 +348,45 @@ class Pungi(object): compose_dir = os.path.join(targetdir, compose_id) makedirs(compose_dir) - # Generate ComposeInfo which is needed for Pungi. - # These variables can be hardcoded, because we only generate composes - # like this in ODCS. + conf = PyConfigParser() + conf.load_from_file(os.path.join(conf_topdir, "pungi.conf")) + ci = ComposeInfo() - ci.release.name = "odcs-%s" % self.compose_id - ci.release.short = "odcs-%s" % self.compose_id - ci.release.version = "1" - ci.release.is_layered = False - ci.release.type = "ga" - ci.release.internal = False - ci.compose.id = compose_id - ci.compose.label = None - ci.compose.type = "nightly" + ci.release.name = conf["release_name"] + ci.release.short = conf["release_short"] + ci.release.version = conf["release_version"] + ci.release.is_layered = True if conf.get("base_product_name", "") else False + ci.release.type = conf.get("release_type", "ga").lower() + ci.release.internal = bool(conf.get("release_internal", False)) + if ci.release.is_layered: + ci.base_product.name = conf["base_product_name"] + ci.base_product.short = conf["base_product_short"] + ci.base_product.version = conf["base_product_version"] + ci.base_product.type = conf.get("base_product_type", "ga").lower() + + ci.compose.label = compose.label + ci.compose.type = compose.compose_type or "nightly" ci.compose.date = compose_date ci.compose.respin = 0 + while True: + ci.compose.id = ci.create_compose_id() + existing_compose = Compose.query.filter( + Compose.pungi_compose_id == ci.compose.id).first() + if not existing_compose: + break + ci.compose.respin += 1 + # Dump the compose info to work/global/composeinfo-base.json. work_dir = os.path.join(compose_dir, "work", "global") makedirs(work_dir) ci.dump(os.path.join(work_dir, "composeinfo-base.json")) + compose.pungi_compose_id = ci.compose.id + return compose_dir - def run_locally(self): + def run_locally(self, compose): """ Runs local Pungi compose. """ @@ -377,9 +394,14 @@ class Pungi(object): try: td = tempfile.mkdtemp() self._write_cfgs(td) - compose_dir = self._prepare_compose_dir(td, conf.target_dir) + compose_dir = self._prepare_compose_dir(compose, td, conf.target_dir) pungi_cmd = self.get_pungi_cmd(td, conf.target_dir, compose_dir) + # Commit the session to ensure that all the `compose` changes are + # stored database before executing the compose and are not just + # cached locally in the SQLAlchemy. + db.session.commit() + log_out_path = os.path.join(compose_dir, "pungi-stdout.log") log_err_path = os.path.join(compose_dir, "pungi-stderr.log") @@ -405,7 +427,7 @@ class Pungi(object): :param models.Compose compose: Compose this Pungi process is running for. """ - self.run_locally() + self.run_locally(compose) class PungiLogs(object): diff --git a/server/odcs/server/views.py b/server/odcs/server/views.py index c554bef..3e0672b 100644 --- a/server/odcs/server/views.py +++ b/server/odcs/server/views.py @@ -244,6 +244,8 @@ class ODCSAPI(MethodView): :jsonparam list multilib_arches: List of :ref:`multilib arches`. :jsonparam string multilib_method: List defining the :ref:`multilib method`. :jsonparam list lookaside_repos: List of :ref:`lookaside_repos`. + :jsonparam string label: String defining the :ref:`label