From 9ced265c41e1989f5df4317969c48b2f5a5d5101 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 04 2015 02:42:02 +0000 Subject: [PATCH 1/4] Fix database upgrades from partially initialized schema 1 databases Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 835924d..fcf058d 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -342,7 +342,8 @@ class Store(Log): fallback_version = self.load_options('dbinfo').get('scheme', {}) if 'version' in fallback_version: - return int(fallback_version['version']) + # Explanation for this is in def upgrade_database(self) + return -1 else: return None @@ -388,6 +389,14 @@ class Store(Log): # Just initialize a new schema self._initialize_schema() self._store_new_schema_version(self._code_schema_version()) + elif old_schema_version == -1: + # This is a special-case from 1.0: we only created tables at the + # first time they were actually used, but the upgrade code assumes + # that the tables exist. So let's fix this. + self._initialize_schema() + # The old version was schema version 1 + self._store_new_schema_version(1) + self.upgrade_database() elif old_schema_version != self._code_schema_version(): # Upgrade from old_schema_version to code_schema_version self.debug('Upgrading from schema version %i' % old_schema_version) From f958c5e78a7d5665073483d79926e91709a355fe Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 04 2015 02:42:12 +0000 Subject: [PATCH 2/4] Add openid_extensions table to be created Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index 40e8b8e..7c637d4 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -1,6 +1,6 @@ # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING -from ipsilon.util.data import Store, UNIQUE_DATA_TABLE +from ipsilon.util.data import Store, UNIQUE_DATA_TABLE, OPTIONS_TABLE from openid import oidutil from openid.association import Association @@ -82,6 +82,9 @@ class OpenIDStore(Store, OpenIDStoreInterface): q = self._query(self._db, 'association', UNIQUE_DATA_TABLE, trans=False) q.create() + q = self._query(self._db, 'openid_extensions', OPTIONS_TABLE, + trans=False) + q.create() def _upgrade_schema(self, old_version): if old_version == 1: @@ -92,6 +95,11 @@ class OpenIDStore(Store, OpenIDStoreInterface): self._db.add_constraint(table.primary_key) for index in table.indexes: self._db.add_index(index) + table = self._query(self._db, 'openid_extensions', OPTIONS_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) return 2 else: raise NotImplementedError() diff --git a/tests/blobs/old_dbs/v1/openid.sqlite.dump b/tests/blobs/old_dbs/v1/openid.sqlite.dump index 4618785..2ffef8e 100644 --- a/tests/blobs/old_dbs/v1/openid.sqlite.dump +++ b/tests/blobs/old_dbs/v1/openid.sqlite.dump @@ -11,4 +11,9 @@ CREATE TABLE association ( name TEXT NOT NULL, value TEXT ); +CREATE TABLE openid_extensions ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); COMMIT; From 3c242c8b745582acf3d68d7b965a5570dff35d99 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 04 2015 02:42:15 +0000 Subject: [PATCH 3/4] Fix the database upgrade for readonly databases (file-conf) Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index fcf058d..e483562 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -627,10 +627,11 @@ class AdminStore(Store): raise NotImplementedError() def create_plugin_data_table(self, plugin_name): - table = plugin_name+'_data' - q = self._query(self._db, table, UNIQUE_DATA_TABLE, - trans=False) - q.create() + if not self.is_readonly: + table = plugin_name+'_data' + q = self._query(self._db, table, UNIQUE_DATA_TABLE, + trans=False) + q.create() class UserStore(Store): From 2ff2f766737abf1615bca802677cb2386b32213d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 04 2015 02:42:38 +0000 Subject: [PATCH 4/4] Close connections after creating the tables This needs to be done manually in the case of upgrades, because there is no cherrypy end_request to help us close sessions. Everytime we do a self._query a new connection is allocated, so we need to make sure to terminate them all. Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index 7c637d4..0eaee0a 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -82,9 +82,11 @@ class OpenIDStore(Store, OpenIDStoreInterface): q = self._query(self._db, 'association', UNIQUE_DATA_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access q = self._query(self._db, 'openid_extensions', OPTIONS_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index e483562..8d2a1d5 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -331,6 +331,7 @@ class Store(Log): # the main codebase, and even in the same database. q = self._query(self._db, 'dbinfo', OPTIONS_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access cls_name = self.__class__.__name__ current_version = self.load_options('dbinfo').get('%s_schema' % cls_name, {}) @@ -608,6 +609,7 @@ class AdminStore(Store): 'provider_config']: q = self._query(self._db, table, OPTIONS_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: @@ -632,6 +634,7 @@ class AdminStore(Store): q = self._query(self._db, table, UNIQUE_DATA_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access class UserStore(Store): @@ -654,6 +657,7 @@ class UserStore(Store): def _initialize_schema(self): q = self._query(self._db, 'users', OPTIONS_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: @@ -678,6 +682,7 @@ class TranStore(Store): q = self._query(self._db, 'transactions', UNIQUE_DATA_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: @@ -779,6 +784,7 @@ class SAML2SessionStore(Store): q = self._query(self._db, self.table, UNIQUE_DATA_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index 8df3b4d..86113a1 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -21,6 +21,7 @@ class SessionStore(Store): q = self._query(self._db, 'sessions', SESSION_TABLE, trans=False) q.create() + q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: