From 61de77fc7eae1f844944bd692d13bf27a1fda6fe Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Sep 02 2015 15:17:52 +0000
Subject: [PATCH 1/3] Add SQL primary key and indexes


Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
Reviewed-by: Rob Crittenden <rcritten@redhat.com>

---

diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py
index e759bca..9b2bc4f 100644
--- a/ipsilon/providers/openid/store.py
+++ b/ipsilon/providers/openid/store.py
@@ -1,6 +1,6 @@
 # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING
 
-from ipsilon.util.data import Store, UNIQUE_DATA_COLUMNS
+from ipsilon.util.data import Store, UNIQUE_DATA_TABLE
 
 from openid import oidutil
 from openid.association import Association
@@ -79,7 +79,7 @@ class OpenIDStore(Store, OpenIDStoreInterface):
                 self.del_unique_data('association', iden)
 
     def _initialize_schema(self):
-        q = self._query(self._db, 'association', UNIQUE_DATA_COLUMNS,
+        q = self._query(self._db, 'association', UNIQUE_DATA_TABLE,
                         trans=False)
         q.create()
 
diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py
index c0fe4ab..e92aae4 100644
--- a/ipsilon/util/data.py
+++ b/ipsilon/util/data.py
@@ -6,6 +6,7 @@ from ipsilon.util.log import Log
 from sqlalchemy import create_engine
 from sqlalchemy import MetaData, Table, Column, Text
 from sqlalchemy.pool import QueuePool, SingletonThreadPool
+from sqlalchemy.schema import PrimaryKeyConstraint, Index
 from sqlalchemy.sql import select, and_
 import ConfigParser
 import os
@@ -13,9 +14,15 @@ import uuid
 import logging
 
 
-CURRENT_SCHEMA_VERSION = 1
-OPTIONS_COLUMNS = ['name', 'option', 'value']
-UNIQUE_DATA_COLUMNS = ['uuid', 'name', 'value']
+CURRENT_SCHEMA_VERSION = 2
+OPTIONS_TABLE = {'columns': ['name', 'option', 'value'],
+                 'primary_key': ('name', 'option'),
+                 'indexes': [('name',)]
+                 }
+UNIQUE_DATA_TABLE = {'columns': ['uuid', 'name', 'value'],
+                     'primary_key': ('uuid', 'name'),
+                     'indexes': [('uuid',)]
+                     }
 
 
 class DatabaseError(Exception):
@@ -74,16 +81,27 @@ class SqlStore(Log):
 
 class SqlQuery(Log):
 
-    def __init__(self, db_obj, table, columns, trans=True):
+    def __init__(self, db_obj, table, table_def, trans=True):
         self._db = db_obj
         self._con = self._db.connection()
         self._trans = self._con.begin() if trans else None
-        self._table = self._get_table(table, columns)
-
-    def _get_table(self, name, columns):
-        table = Table(name, MetaData(self._db.engine()))
-        for c in columns:
-            table.append_column(Column(c, Text()))
+        self._table = self._get_table(table, table_def)
+
+    def _get_table(self, name, table_def):
+        if isinstance(table_def, list):
+            table_def = {'columns': table_def,
+                         'indexes': [],
+                         'primary_key': None}
+        table_creation = []
+        for col_name in table_def['columns']:
+            table_creation.append(Column(col_name, Text()))
+        if table_def['primary_key']:
+            table_creation.append(PrimaryKeyConstraint(
+                *table_def['primary_key']))
+        for index in table_def['indexes']:
+            idx_name = 'idx_%s_%s' % (name, '_'.join(index))
+            table_creation.append(Index(idx_name, *index))
+        table = Table(name, MetaData(self._db.engine()), *table_creation)
         return table
 
     def _where(self, kvfilter):
@@ -159,7 +177,12 @@ class FileStore(Log):
 
 class FileQuery(Log):
 
-    def __init__(self, fstore, table, columns, trans=True):
+    def __init__(self, fstore, table, table_def, trans=True):
+        # We don't need indexes in a FileQuery, so drop that info
+        if isinstance(table_def, dict):
+            columns = table_def['columns']
+        else:
+            columns = table_def
         self._fstore = fstore
         self._config = fstore.get_config()
         self._section = table
@@ -278,7 +301,7 @@ class Store(Log):
         # We are storing multiple versions: one per class
         # That way, we can support plugins with differing schema versions from
         #  the main codebase, and even in the same database.
-        q = self._query(self._db, 'dbinfo', OPTIONS_COLUMNS, trans=False)
+        q = self._query(self._db, 'dbinfo', OPTIONS_TABLE, trans=False)
         q.create()
         cls_name = self.__class__.__name__
         current_version = self.load_options('dbinfo').get('%s_schema'
@@ -387,7 +410,7 @@ class Store(Log):
         kvfilter = dict()
         if name:
             kvfilter['name'] = name
-        options = self._load_data(table, OPTIONS_COLUMNS, kvfilter)
+        options = self._load_data(table, OPTIONS_TABLE, kvfilter)
         if name and name in options:
             return options[name]
         return options
@@ -396,7 +419,7 @@ class Store(Log):
         curvals = dict()
         q = None
         try:
-            q = self._query(self._db, table, OPTIONS_COLUMNS)
+            q = self._query(self._db, table, OPTIONS_TABLE)
             rows = q.select({'name': name}, ['option', 'value'])
             for row in rows:
                 curvals[row[0]] = row[1]
@@ -419,7 +442,7 @@ class Store(Log):
         kvfilter = {'name': name}
         q = None
         try:
-            q = self._query(self._db, table, OPTIONS_COLUMNS)
+            q = self._query(self._db, table, OPTIONS_TABLE)
             if options is None:
                 q.delete(kvfilter)
             else:
@@ -437,7 +460,7 @@ class Store(Log):
         newid = str(uuid.uuid4())
         q = None
         try:
-            q = self._query(self._db, table, UNIQUE_DATA_COLUMNS)
+            q = self._query(self._db, table, UNIQUE_DATA_TABLE)
             for name in data:
                 q.insert((newid, name, data[name]))
             q.commit()
@@ -456,12 +479,12 @@ class Store(Log):
             kvfilter['name'] = name
         if value:
             kvfilter['value'] = value
-        return self._load_data(table, UNIQUE_DATA_COLUMNS, kvfilter)
+        return self._load_data(table, UNIQUE_DATA_TABLE, kvfilter)
 
     def save_unique_data(self, table, data):
         q = None
         try:
-            q = self._query(self._db, table, UNIQUE_DATA_COLUMNS)
+            q = self._query(self._db, table, UNIQUE_DATA_TABLE)
             for uid in data:
                 curvals = dict()
                 rows = q.select({'uuid': uid}, ['name', 'value'])
@@ -490,7 +513,7 @@ class Store(Log):
     def del_unique_data(self, table, uuidval):
         kvfilter = {'uuid': uuidval}
         try:
-            q = self._query(self._db, table, UNIQUE_DATA_COLUMNS, trans=False)
+            q = self._query(self._db, table, UNIQUE_DATA_TABLE, trans=False)
             q.delete(kvfilter)
         except Exception, e:  # pylint: disable=broad-except
             self.error("Failed to delete data from %s: [%s]" % (table, e))
@@ -498,7 +521,7 @@ class Store(Log):
     def _reset_data(self, table):
         q = None
         try:
-            q = self._query(self._db, table, UNIQUE_DATA_COLUMNS)
+            q = self._query(self._db, table, UNIQUE_DATA_TABLE)
             q.drop()
             q.create()
             q.commit()
@@ -536,7 +559,7 @@ class AdminStore(Store):
                       'info_config',
                       'login_config',
                       'provider_config']:
-            q = self._query(self._db, table, OPTIONS_COLUMNS, trans=False)
+            q = self._query(self._db, table, OPTIONS_TABLE, trans=False)
             q.create()
 
     def _upgrade_schema(self, old_version):
@@ -561,7 +584,7 @@ class UserStore(Store):
         return self.load_options(plugin+"_data", user)
 
     def _initialize_schema(self):
-        q = self._query(self._db, 'users', OPTIONS_COLUMNS, trans=False)
+        q = self._query(self._db, 'users', OPTIONS_TABLE, trans=False)
         q.create()
 
     def _upgrade_schema(self, old_version):
@@ -574,7 +597,7 @@ class TranStore(Store):
         super(TranStore, self).__init__('transactions.db')
 
     def _initialize_schema(self):
-        q = self._query(self._db, 'transactions', UNIQUE_DATA_COLUMNS,
+        q = self._query(self._db, 'transactions', UNIQUE_DATA_TABLE,
                         trans=False)
         q.create()
 
@@ -588,7 +611,7 @@ class SAML2SessionStore(Store):
         super(SAML2SessionStore, self).__init__(database_url=database_url)
         self.table = 'saml2_sessions'
         # pylint: disable=protected-access
-        table = SqlQuery(self._db, self.table, UNIQUE_DATA_COLUMNS)._table
+        table = SqlQuery(self._db, self.table, UNIQUE_DATA_TABLE)._table
         table.create(checkfirst=True)
 
     def _get_unique_id_from_column(self, name, value):
@@ -609,7 +632,7 @@ class SAML2SessionStore(Store):
 
     def remove_expired_sessions(self):
         # pylint: disable=protected-access
-        table = SqlQuery(self._db, self.table, UNIQUE_DATA_COLUMNS)._table
+        table = SqlQuery(self._db, self.table, UNIQUE_DATA_TABLE)._table
         sel = select([table.columns.uuid]). \
             where(and_(table.c.name == 'expiration_time',
                        table.c.value <= datetime.datetime.now()))
@@ -665,7 +688,7 @@ class SAML2SessionStore(Store):
         self._reset_data(self.table)
 
     def _initialize_schema(self):
-        q = self._query(self._db, self.table, UNIQUE_DATA_COLUMNS,
+        q = self._query(self._db, self.table, UNIQUE_DATA_TABLE,
                         trans=False)
         q.create()
 
diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py
index b870319..ef059d1 100644
--- a/ipsilon/util/sessions.py
+++ b/ipsilon/util/sessions.py
@@ -10,12 +10,15 @@ except ImportError:
     import pickle
 
 
-SESSION_COLUMNS = ['id', 'data', 'expiration_time']
+SESSION_TABLE = {'columns': ['id', 'data', 'expiration_time'],
+                 'primary_key': ('id', ),
+                 'indexes': [('expiration_time',)]
+                 }
 
 
 class SessionStore(Store):
     def _initialize_schema(self):
-        q = self._query(self._db, 'sessions', SESSION_COLUMNS,
+        q = self._query(self._db, 'sessions', SESSION_TABLE,
                         trans=False)
         q.create()
 
@@ -44,12 +47,12 @@ class SqlSession(Session):
         cls._db = cls._store._db
 
     def _exists(self):
-        q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS)
+        q = SqlQuery(self._db, 'sessions', SESSION_TABLE)
         result = q.select({'id': self.id})
         return True if result.fetchone() else False
 
     def _load(self):
-        q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS)
+        q = SqlQuery(self._db, 'sessions', SESSION_TABLE)
         result = q.select({'id': self.id})
         r = result.fetchone()
         if r:
@@ -59,7 +62,7 @@ class SqlSession(Session):
     def _save(self, expiration_time):
         q = None
         try:
-            q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS, trans=True)
+            q = SqlQuery(self._db, 'sessions', SESSION_TABLE, trans=True)
             q.delete({'id': self.id})
             data = pickle.dumps((self._data, expiration_time), self._proto)
             q.insert((self.id, base64.b64encode(data), expiration_time))
@@ -70,7 +73,7 @@ class SqlSession(Session):
             raise
 
     def _delete(self):
-        q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS)
+        q = SqlQuery(self._db, 'sessions', SESSION_TABLE)
         q.delete({'id': self.id})
 
     # copy what RamSession does for now

From cff71af03913b5b0987171205ef0c460b2f6fff8 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Sep 02 2015 15:17:55 +0000
Subject: [PATCH 2/3] Implement database upgrade for indexes


This implements upgrading from schema version 1 to 2.

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
Reviewed-by: Rob Crittenden <rcritten@redhat.com>

---

diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py
index 9b2bc4f..40e8b8e 100644
--- a/ipsilon/providers/openid/store.py
+++ b/ipsilon/providers/openid/store.py
@@ -84,4 +84,14 @@ class OpenIDStore(Store, OpenIDStoreInterface):
         q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            # pylint: disable=protected-access
+            table = self._query(self._db, 'association', UNIQUE_DATA_TABLE,
+                                trans=False)._table
+            self._db.add_constraint(table.primary_key)
+            for index in table.indexes:
+                self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py
index e92aae4..79b7156 100644
--- a/ipsilon/util/data.py
+++ b/ipsilon/util/data.py
@@ -6,7 +6,8 @@ from ipsilon.util.log import Log
 from sqlalchemy import create_engine
 from sqlalchemy import MetaData, Table, Column, Text
 from sqlalchemy.pool import QueuePool, SingletonThreadPool
-from sqlalchemy.schema import PrimaryKeyConstraint, Index
+from sqlalchemy.schema import (PrimaryKeyConstraint, Index, AddConstraint,
+                               CreateIndex)
 from sqlalchemy.sql import select, and_
 import ConfigParser
 import os
@@ -29,7 +30,16 @@ class DatabaseError(Exception):
     pass
 
 
-class SqlStore(Log):
+class BaseStore(Log):
+    # Some helper functions used for upgrades
+    def add_constraint(self, table):
+        raise NotImplementedError()
+
+    def add_index(self, index):
+        raise NotImplementedError()
+
+
+class SqlStore(BaseStore):
     __instances = {}
 
     @classmethod
@@ -61,6 +71,18 @@ class SqlStore(Log):
         self._dbengine = create_engine(engine_name, **pool_args)
         self.is_readonly = False
 
+    def add_constraint(self, constraint):
+        if self._dbengine.dialect.name != 'sqlite':
+            # It is impossible to add constraints to a pre-existing table for
+            #  SQLite
+            # source: http://www.sqlite.org/omitted.html
+            create_constraint = AddConstraint(constraint, bind=self._dbengine)
+            create_constraint.execute()
+
+    def add_index(self, index):
+        add_index = CreateIndex(index, bind=self._dbengine)
+        add_index.execute()
+
     def debug(self, fact):
         if self.db_conn_log:
             super(SqlStore, self).debug(fact)
@@ -151,7 +173,7 @@ class SqlQuery(Log):
         self._con.execute(self._table.delete(self._where(kvfilter)))
 
 
-class FileStore(Log):
+class FileStore(BaseStore):
 
     def __init__(self, name):
         self._filename = name
@@ -174,6 +196,12 @@ class FileStore(Log):
             self._config.read(self._filename)
         return self._config
 
+    def add_constraint(self, table):
+        raise NotImplementedError()
+
+    def add_index(self, index):
+        raise NotImplementedError()
+
 
 class FileQuery(Log):
 
@@ -350,6 +378,7 @@ class Store(Log):
         #  themselves.
         # They might implement downgrading if that's feasible, or just throw
         #  NotImplementedError
+        # Should return the new schema version
         raise NotImplementedError()
 
     def upgrade_database(self):
@@ -361,8 +390,17 @@ class Store(Log):
             self._store_new_schema_version(self._code_schema_version())
         elif old_schema_version != self._code_schema_version():
             # Upgrade from old_schema_version to code_schema_version
-            self._upgrade_schema(old_schema_version)
-            self._store_new_schema_version(self._code_schema_version())
+            self.debug('Upgrading from schema version %i' % old_schema_version)
+            new_version = self._upgrade_schema(old_schema_version)
+            if not new_version:
+                error = ('Schema upgrade error: %s did not provide a ' +
+                         'new schema version number!' %
+                         self.__class__.__name__)
+                self.error(error)
+                raise Exception(error)
+            self._store_new_schema_version(new_version)
+            # Check if we are now up-to-date
+            self.upgrade_database()
 
     @property
     def is_readonly(self):
@@ -563,7 +601,21 @@ class AdminStore(Store):
             q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            for table in ['config',
+                          'info_config',
+                          'login_config',
+                          'provider_config']:
+                # pylint: disable=protected-access
+                table = self._query(self._db, table, OPTIONS_TABLE,
+                                    trans=False)._table
+                self._db.add_constraint(table.primary_key)
+                for index in table.indexes:
+                    self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
 
 
 class UserStore(Store):
@@ -588,7 +640,17 @@ class UserStore(Store):
         q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            # pylint: disable=protected-access
+            table = self._query(self._db, 'users', OPTIONS_TABLE,
+                                trans=False)._table
+            self._db.add_constraint(table.primary_key)
+            for index in table.indexes:
+                self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
 
 
 class TranStore(Store):
@@ -602,7 +664,17 @@ class TranStore(Store):
         q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            # pylint: disable=protected-access
+            table = self._query(self._db, 'transactions', UNIQUE_DATA_TABLE,
+                                trans=False)._table
+            self._db.add_constraint(table.primary_key)
+            for index in table.indexes:
+                self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
 
 
 class SAML2SessionStore(Store):
@@ -693,4 +765,14 @@ class SAML2SessionStore(Store):
         q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            # pylint: disable=protected-access
+            table = self._query(self._db, self.table, UNIQUE_DATA_TABLE,
+                                trans=False)._table
+            self._db.add_constraint(table.primary_key)
+            for index in table.indexes:
+                self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py
index ef059d1..8df3b4d 100644
--- a/ipsilon/util/sessions.py
+++ b/ipsilon/util/sessions.py
@@ -23,7 +23,17 @@ class SessionStore(Store):
         q.create()
 
     def _upgrade_schema(self, old_version):
-        raise NotImplementedError()
+        if old_version == 1:
+            # In schema version 2, we added indexes and primary keys
+            # pylint: disable=protected-access
+            table = self._query(self._db, 'sessions', SESSION_TABLE,
+                                trans=False)._table
+            self._db.add_constraint(table.primary_key)
+            for index in table.indexes:
+                self._db.add_index(index)
+            return 2
+        else:
+            raise NotImplementedError()
 
 
 class SqlSession(Session):

From ee2cd3b48ed3903dbb35726eb9992dd13f826f48 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Sep 02 2015 15:19:23 +0000
Subject: [PATCH 3/3] Add test suite for database upgrades


Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
Reviewed-by: Rob Crittenden <rcritten@redhat.com>

---

diff --git a/Makefile b/Makefile
index d15ca4d..ddfdf14 100644
--- a/Makefile
+++ b/Makefile
@@ -104,6 +104,7 @@ tests: wrappers
 	PYTHONPATH=./ ./tests/tests.py --test=ldap
 	PYTHONPATH=./ ./tests/tests.py --test=ldapdown
 	PYTHONPATH=./ ./tests/tests.py --test=openid
+	PYTHONPATH=./ ./tests/tests.py --test=dbupgrades
 
 test: lp-test unittests tests
 
diff --git a/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump b/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump
new file mode 100644
index 0000000..d8e4d0a
--- /dev/null
+++ b/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump
@@ -0,0 +1,65 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE dbinfo (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "dbinfo" VALUES('AdminStore_schema','version','1');
+CREATE TABLE config (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE info_config (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE login_config (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "login_config" VALUES('global','enabled','testauth');
+CREATE TABLE provider_config (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "provider_config" VALUES('openid','endpoint url','http://127.0.0.11:45081/idp_v1/openid/');
+INSERT INTO "provider_config" VALUES('openid','database url','openid.sqlite');
+INSERT INTO "provider_config" VALUES('openid','identity url template','http://127.0.0.11:45081/idp_v1/openid/id/%(username)s');
+INSERT INTO "provider_config" VALUES('openid','enabled extensions','');
+INSERT INTO "provider_config" VALUES('global','enabled','openid,persona,saml2');
+INSERT INTO "provider_config" VALUES('persona','allowed domains','127.0.0.11:45081');
+INSERT INTO "provider_config" VALUES('persona','issuer domain','127.0.0.11:45081');
+INSERT INTO "provider_config" VALUES('persona','idp key file','persona/persona.key');
+INSERT INTO "provider_config" VALUES('saml2','idp nameid salt','6c78ae3b33db4fe4886edb1679490821');
+INSERT INTO "provider_config" VALUES('saml2','idp metadata validity','1825');
+INSERT INTO "provider_config" VALUES('saml2','idp certificate file','saml2/idp.pem');
+INSERT INTO "provider_config" VALUES('saml2','idp key file','saml2/idp.key');
+INSERT INTO "provider_config" VALUES('saml2','session database url','saml2.sessions.db.sqlite');
+INSERT INTO "provider_config" VALUES('saml2','idp metadata file','metadata.xml');
+INSERT INTO "provider_config" VALUES('saml2','idp storage path','saml2');
+CREATE TABLE testauth_data (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE openid_data (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE persona_data (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE saml2_data (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+COMMIT;
diff --git a/tests/blobs/old_dbs/v1/openid.sqlite.dump b/tests/blobs/old_dbs/v1/openid.sqlite.dump
new file mode 100644
index 0000000..4618785
--- /dev/null
+++ b/tests/blobs/old_dbs/v1/openid.sqlite.dump
@@ -0,0 +1,14 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE dbinfo (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "dbinfo" VALUES('OpenIDStore_schema','version','1');
+CREATE TABLE association (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+COMMIT;
diff --git a/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump b/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump
new file mode 100644
index 0000000..1e8a694
--- /dev/null
+++ b/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump
@@ -0,0 +1,14 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE saml2_sessions (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+CREATE TABLE dbinfo (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "dbinfo" VALUES('SAML2SessionStore_schema','version','1');
+COMMIT;
diff --git a/tests/blobs/old_dbs/v1/transactions.sqlite.dump b/tests/blobs/old_dbs/v1/transactions.sqlite.dump
new file mode 100644
index 0000000..b803e78
--- /dev/null
+++ b/tests/blobs/old_dbs/v1/transactions.sqlite.dump
@@ -0,0 +1,14 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE dbinfo (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "dbinfo" VALUES('TranStore_schema','version','1');
+CREATE TABLE transactions (
+	uuid TEXT NOT NULL, 
+	name TEXT NOT NULL, 
+	value TEXT
+);
+COMMIT;
diff --git a/tests/blobs/old_dbs/v1/userprefs.sqlite.dump b/tests/blobs/old_dbs/v1/userprefs.sqlite.dump
new file mode 100644
index 0000000..a20af5e
--- /dev/null
+++ b/tests/blobs/old_dbs/v1/userprefs.sqlite.dump
@@ -0,0 +1,14 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE dbinfo (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+INSERT INTO "dbinfo" VALUES('UserStore_schema','version','1');
+CREATE TABLE users (
+	name TEXT NOT NULL, 
+	option TEXT NOT NULL, 
+	value TEXT
+);
+COMMIT;
diff --git a/tests/dbupgrades.py b/tests/dbupgrades.py
new file mode 100755
index 0000000..73d04fb
--- /dev/null
+++ b/tests/dbupgrades.py
@@ -0,0 +1,149 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING
+
+from helpers.common import IpsilonTestBase  # pylint: disable=relative-import
+from helpers.http import HttpSessions  # pylint: disable=relative-import
+import os
+import pwd
+import sys
+import signal
+import subprocess
+import ipsilon.util.data
+
+idp_g = {'TEMPLATES': '${TESTDIR}/templates/install',
+         'CONFDIR': '${TESTDIR}/etc',
+         'DATADIR': '${TESTDIR}/lib',
+         'HTTPDCONFD': '${TESTDIR}/${NAME}/conf.d',
+         'STATICDIR': '${ROOTDIR}',
+         'BINDIR': '${ROOTDIR}/ipsilon',
+         'WSGI_SOCKET_PREFIX': '${TESTDIR}/${NAME}/logs/wsgi'}
+
+
+idp_a = {'hostname': '${ADDRESS}:${PORT}',
+         'admin_user': '${TEST_USER}',
+         'system_user': '${TEST_USER}',
+         'instance': '${NAME}',
+         'secure': 'no',
+         'testauth': 'yes',
+         'pam': 'no',
+         'gssapi': 'no',
+         'ipa': 'no',
+         'server_debugging': 'True'}
+
+
+class IpsilonTest(IpsilonTestBase):
+
+    def __init__(self):
+        super(IpsilonTest, self).__init__('dbupgrades', __file__)
+
+    def setup_servers(self, env=None):
+        pass
+
+    def test_upgrade_from(self, env, old_version):
+        # Setup IDP Server
+        print "Installing IDP server to test upgrade from %i" % old_version
+        name = 'idp_v%i' % old_version
+        addr = '127.0.0.%i' % (10 + old_version)
+        port = str(45080 + old_version)
+        idp = self.generate_profile(idp_g, idp_a, name, addr, port)
+        conf = self.setup_idp_server(idp, name, addr, port, env)
+
+        # Move database of old_version into place
+        cfgfile = os.path.join(self.testdir, 'etc', name, 'ipsilon.conf')
+        db_indir = os.path.join(self.rootdir, 'tests', 'blobs', 'old_dbs',
+                                'v%i' % old_version)
+        db_outdir = os.path.join(self.testdir, 'lib', name)
+
+        if old_version > 0:
+            for database in ['adminconfig',
+                             'openid',
+                             'saml2.sessions.db',
+                             'transactions',
+                             'userprefs']:
+                db_in = os.path.join(db_indir, '%s.sqlite.dump' % database)
+                db_out = os.path.join(db_outdir, '%s.sqlite' % database)
+                os.unlink(db_out)
+                cmd = ['/bin/sqlite3', db_out, '.read %s' % db_in]
+                subprocess.check_call(cmd)
+
+            # Upgrade that database
+            cmd = [os.path.join(self.rootdir,
+                                'ipsilon/install/ipsilon-upgrade-database'),
+                   cfgfile]
+            subprocess.check_call(cmd,
+                                  cwd=os.path.join(self.testdir, 'lib', name),
+                                  env=env)
+
+        # Check some version-specific changes, to see if the upgrade went OK
+        if old_version == 0:
+            # Check all features in a newly created database
+            # Let's verify if at least one index was created
+            test_db = os.path.join(db_outdir, 'adminconfig.sqlite')
+            p = subprocess.Popen(['/bin/sqlite3', test_db, '.dump'],
+                                 stdout=subprocess.PIPE)
+            output, _ = p.communicate()
+            if p.returncode:
+                print 'Sqlite dump failed'
+                sys.exit(1)
+            if 'CREATE INDEX' not in output:
+                raise Exception('Database upgrade did not introduce index')
+            if 'PRIMARY KEY' not in output:
+                raise Exception('Database upgrade did not introduce primary ' +
+                                'key')
+        elif old_version == 1:
+            # In 1 -> 2, we added indexes and primary keys
+            # Let's verify if at least one index was created
+            test_db = os.path.join(db_outdir, 'adminconfig.sqlite')
+            p = subprocess.Popen(['/bin/sqlite3', test_db, '.dump'],
+                                 stdout=subprocess.PIPE)
+            output, _ = p.communicate()
+            if p.returncode:
+                print 'Sqlite dump failed'
+                sys.exit(1)
+            if 'CREATE INDEX' not in output:
+                raise Exception('Database upgrade did not introduce index')
+            # SQLite did not support creating primary keys, so we can't test
+
+        # Start the httpd server
+        http_server = self.start_http_server(conf, env)
+
+        # Now attempt to use the upgraded database
+        exe = self.execname
+        if exe.endswith('c'):
+            exe = exe[:-1]
+        exe = [exe]
+        exe.append(str(old_version))
+        exe.append(name)
+        exe.append('%s:%s' % (addr, port))
+        exit_code = subprocess.call(exe, env=env)
+        if exit_code:
+            sys.exit(exit_code)
+
+        # Now kill the last http server
+        os.killpg(http_server.pid, signal.SIGTERM)
+        self.processes.remove(http_server)
+
+    def run(self, env):
+        for version in range(ipsilon.util.data.CURRENT_SCHEMA_VERSION):
+            self.test_upgrade_from(env, version)
+
+
+if __name__ == '__main__':
+    from_version = sys.argv[1]
+    idpname = sys.argv[2]
+    url = sys.argv[3]
+
+    user = pwd.getpwuid(os.getuid())[0]
+
+    sess = HttpSessions()
+    sess.add_server(idpname, 'http://%s' % url, user,
+                    'ipsilon')
+
+    print "dbupgrades: From v%s: Authenticate to IDP ..." % from_version,
+    try:
+        sess.auth_to_idp(idpname)
+    except Exception, e:  # pylint: disable=broad-except
+        print >> sys.stderr, " ERROR: %s" % repr(e)
+        sys.exit(1)
+    print " SUCCESS"
diff --git a/tests/helpers/common.py b/tests/helpers/common.py
index eadfdc3..a7021e2 100755
--- a/tests/helpers/common.py
+++ b/tests/helpers/common.py
@@ -177,6 +177,7 @@ class IpsilonTestBase(object):
         p = subprocess.Popen(['/usr/sbin/httpd', '-DFOREGROUND', '-f', conf],
                              env=env, preexec_fn=os.setsid)
         self.processes.append(p)
+        return p
 
     def start_pgdb_server(self, datadir, rundir, log, addr, port, env):
         p = subprocess.Popen(['/usr/bin/pg_ctl', 'start', '-D', datadir, '-o',

