From 80ca92c5fb53ebf0187e5ce214a43bc27ddf3ff7 Mon Sep 17 00:00:00 2001 From: Jan Kaluža Date: Jun 27 2017 08:48:46 +0000 Subject: Merge #18 `Allow regenerating the compose after its expiration` --- diff --git a/odcs/__init__.py b/odcs/__init__.py index 8d7f9ff..c0f9d84 100644 --- a/odcs/__init__.py +++ b/odcs/__init__.py @@ -23,12 +23,13 @@ from logging import getLogger -from flask import Flask +from flask import Flask, jsonify from flask_sqlalchemy import SQLAlchemy from odcs.logger import init_logging from odcs.config import init_config from odcs.proxy import ReverseProxy +from odcs.errors import NotFound app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) @@ -40,3 +41,26 @@ init_logging(conf) log = getLogger(__name__) from odcs import views + +def json_error(status, error, message): + response = jsonify( + {'status': status, + 'error': error, + 'message': message}) + response.status_code = status + return response + +@app.errorhandler(ValueError) +def validationerror_error(e): + """Flask error handler for ValueError exceptions""" + return json_error(400, 'Bad Request', e.args[0]) + +@app.errorhandler(RuntimeError) +def runtimeerror_error(e): + """Flask error handler for RuntimeError exceptions""" + return json_error(500, 'Internal Server Error', e.args[0]) + +@app.errorhandler(NotFound) +def notfound_error(e): + """Flask error handler for NotFound exceptions""" + return json_error(404, 'Not Found', e.args[0]) diff --git a/odcs/backend.py b/odcs/backend.py index 06543bb..af7d875 100644 --- a/odcs/backend.py +++ b/odcs/backend.py @@ -200,8 +200,12 @@ def resolve_compose(compose): revision = e.find("{http://linux.duke.edu/metadata/repo}revision").text compose.koji_event = int(revision) elif compose.source_type == PungiSourceType.KOJI_TAG: - koji_session = create_koji_session() - compose.koji_event = int(koji_session.getLastEvent()['id']) + # If compose.koji_event is set, it means that we are regenerating + # previous compose and we have to respect the previous koji_event to + # get the same results. + if not compose.koji_event: + koji_session = create_koji_session() + compose.koji_event = int(koji_session.getLastEvent()['id']) elif compose.source_type == PungiSourceType.MODULE: # Resolve the latest release of modules which do not have the release # string defined in the compose.source. @@ -236,6 +240,13 @@ def get_reusable_compose(compose): Compose.source_type == compose.source_type).all() for old_compose in composes: + # Skip the old_compose in case it reuses another compose. In that case + # the reused compose is also in composes list, so we won't miss it. We + # don't want chain of composes reusing each other, because it would + # break the time_to_expire handling. + if old_compose.reused_id: + continue + packages = set(compose.packages.split(" ")) \ if compose.packages else set() old_packages = set(old_compose.packages.split(" ")) \ diff --git a/odcs/errors.py b/odcs/errors.py new file mode 100644 index 0000000..f825057 --- /dev/null +++ b/odcs/errors.py @@ -0,0 +1,26 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza +""" Defines custom exceptions and error handling functions """ + + +class NotFound(ValueError): + pass diff --git a/odcs/models.py b/odcs/models.py index d4464af..b342543 100644 --- a/odcs/models.py +++ b/odcs/models.py @@ -108,6 +108,32 @@ class Compose(ODCSBase): session.add(compose) return compose + @classmethod + def create_copy(cls, session, compose, owner=None, seconds_to_live=None): + """ + Creates new compose with all the options influencing the resulting + compose copied from the `compose`. The `owner` and `seconds_to_live` + can be set independently. The state of copies compose is "wait". + """ + now = datetime.utcnow() + if not seconds_to_live: + seconds_to_live = conf.seconds_to_live + + compose = cls( + owner=owner or compose.owner, + source_type=compose.source_type, + source=compose.source, + state="wait", + results=compose.results, + time_submitted=now, + time_to_expire=now + timedelta(seconds=seconds_to_live), + packages=compose.packages, + flags=compose.flags, + koji_event=compose.koji_event, + ) + session.add(compose) + return compose + @property def name(self): if self.reused_id: diff --git a/odcs/views.py b/odcs/views.py index 3f962b5..51d358f 100644 --- a/odcs/views.py +++ b/odcs/views.py @@ -27,7 +27,8 @@ from flask.views import MethodView from flask import request, jsonify from odcs import app, db, log, conf -from odcs.models import Compose, COMPOSE_RESULTS, COMPOSE_FLAGS +from odcs.errors import NotFound +from odcs.models import Compose, COMPOSE_RESULTS, COMPOSE_FLAGS, COMPOSE_STATES from odcs.pungi import PungiSourceType from odcs.api_utils import pagination_metadata, filter_composes @@ -72,7 +73,7 @@ class ODCSAPI(MethodView): if compose: return jsonify(compose.json()), 200 else: - raise ValueError('No such compose found.') + raise NotFound('No such compose found.') def post(self): owner = "Unknown" # TODO @@ -83,6 +84,32 @@ class ODCSAPI(MethodView): log.exception('Invalid JSON submitted') raise ValueError('Invalid JSON submitted') + # If "id" is in data, it means client wants to regenerate an expired + # compose. + if "id" in data: + old_compose = Compose.query.filter( + Compose.id == data["id"], + Compose.state.in_( + [COMPOSE_STATES["removed"], + COMPOSE_STATES["failed"]])).first() + if not old_compose: + err = "No expired or failed compose with id %s" % data["id"] + log.error(err) + raise ValueError(err) + + log.info("%r: Going to regenerate the compose", old_compose) + + seconds_to_live = conf.seconds_to_live + if "seconds-to-live" in data: + seconds_to_live = max(int(seconds_to_live), + conf.max_seconds_to_live) + + compose = Compose.create_copy(db.session, old_compose, owner, + seconds_to_live) + db.session.add(compose) + db.session.commit() + return jsonify(compose.json()), 200 + needed_keys = ["source_type", "source"] for key in needed_keys: if key not in data: diff --git a/tests/test_backend.py b/tests/test_backend.py index 66173a6..b80245a 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -22,6 +22,7 @@ import unittest import os +from mock import patch, MagicMock from odcs import db from odcs.models import Compose, COMPOSE_RESULTS, COMPOSE_STATES @@ -55,6 +56,25 @@ class TestBackend(unittest.TestCase): c = db.session.query(Compose).filter(Compose.id == 1).one() self.assertEqual(c.koji_event, 1496834159) + @patch("odcs.backend.create_koji_session") + def test_resolve_compose_repo_no_override_koji_event( + self, create_koji_session): + koji_session = MagicMock() + create_koji_session.return_value = koji_session + koji_session.getLastEvent.return_value = {"id": 123} + + c = Compose.create( + db.session, "me", PungiSourceType.KOJI_TAG, "f26", + COMPOSE_RESULTS["repository"], 3600, packages="ed") + c.koji_event = 1 + db.session.commit() + + resolve_compose(c) + db.session.commit() + db.session.expire_all() + c = db.session.query(Compose).filter(Compose.id == 1).one() + self.assertEqual(c.koji_event, 1) + def test_get_reusable_compose(self): old_c = Compose.create( db.session, "me", PungiSourceType.REPO, os.path.join(thisdir, "repo"), diff --git a/tests/test_views.py b/tests/test_views.py index ec26afb..23f8432 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -84,6 +84,50 @@ class TestViews(unittest.TestCase): c = db.session.query(Compose).filter(Compose.id == 1).one() self.assertEqual(c.state, COMPOSE_STATES["wait"]) + def test_submit_build_resurrection_removed(self): + self.c1.state = COMPOSE_STATES["removed"] + self.c1.reused_id = 1 + db.session.commit() + rv = self.client.post('/odcs/1/composes/', data=json.dumps({'id': 1})) + data = json.loads(rv.data.decode('utf8')) + + self.assertEqual(data['id'], 3) + self.assertEqual(data['state_name'], 'wait') + self.assertEqual(data['source'], 'testmodule-master') + self.assertEqual(data['time_removed'], None) + + c = db.session.query(Compose).filter(Compose.id == 3).one() + self.assertEqual(c.reused_id, None) + + def test_submit_build_resurrection_failed(self): + self.c1.state = COMPOSE_STATES["failed"] + self.c1.reused_id = 1 + db.session.commit() + rv = self.client.post('/odcs/1/composes/', data=json.dumps({'id': 1})) + data = json.loads(rv.data.decode('utf8')) + + self.assertEqual(data['id'], 3) + self.assertEqual(data['state_name'], 'wait') + self.assertEqual(data['source'], 'testmodule-master') + self.assertEqual(data['time_removed'], None) + + c = db.session.query(Compose).filter(Compose.id == 3).one() + self.assertEqual(c.reused_id, None) + + def test_submit_build_resurrection_no_removed(self): + db.session.commit() + rv = self.client.post('/odcs/1/composes/', data=json.dumps({'id': 1})) + data = json.loads(rv.data.decode('utf8')) + + self.assertEqual(data['message'], 'No expired or failed compose with id 1') + + def test_submit_build_resurrection_not_found(self): + db.session.commit() + rv = self.client.post('/odcs/1/composes/', data=json.dumps({'id': 100})) + data = json.loads(rv.data.decode('utf8')) + + self.assertEqual(data['message'], 'No expired or failed compose with id 100') + def test_query_compose(self): resp = self.client.get('/odcs/1/composes/1') data = json.loads(resp.data.decode('utf8'))