#274 Fix database upgrade problems with readonly plugin databases
Merged 4 years ago by puiterwijk. Opened 4 years ago by puiterwijk.
puiterwijk/ipsilon readonlydb  into  master

file modified
+6
@@ -860,6 +860,12 @@ 

          raise NotImplementedError()

  

      def upgrade_database(self):

+         if self.is_readonly:

+             # If the database is readonly, we cannot do anything to the

+             #  schema. Let's just return, and assume people checked the

+             #  upgrade notes

+             return

+ 

          # Do whatever is needed to get schema to current version

          old_schema_version = self._get_schema_version()

          if old_schema_version is None:

@@ -0,0 +1,20 @@ 

+ [config]

+ [info_config]

+ [login_config]

+ global enabled=testauth

+ [provider_config]

+ global enabled=openid,persona,saml2

+ openid endpoint url=http://127.0.0.11:45081/idp_v1/openid/

+ openid database url=configfile://openid.cfg

+ openid identity url template=http://127.0.0.11:45081/idp_v1/openid/id/%(username)s

+ openid enabled extensions=

+ persona allowed domains=127.0.0.11:45081

+ persona issuer domain=127.0.0.11:45081

+ persona idp key file=persona/persona.key

+ saml2 idp nameid salt=6c78ae3b33db4fe4886edb1679490821

+ saml2 idp metadata validity=1825

+ saml2 idp certificate file=saml2/idp.pem

+ saml2 idp key file=saml2/idp.key

+ saml2 session database url=saml2.sessions.db.sqlite

+ saml2 idp metadata file=metadata.xml

+ saml2 idp storage path=saml2

file modified
+48 -13
@@ -40,8 +40,12 @@ 

      def setup_servers(self, env=None):

          pass

  

-     def dump_admin_config_db(self, db_outdir):

-         test_db = os.path.join(db_outdir, 'adminconfig.sqlite')

+     def dump_db(self, db_outdir, readonly):

+         if readonly:

+             db_name = 'userprefs'

+         else:

+             db_name = 'adminconfig'

+         test_db = os.path.join(db_outdir, '%s.sqlite' % db_name)

          p = subprocess.Popen(['/usr/bin/sqlite3', test_db, '.dump'],

                               stdout=subprocess.PIPE)

          output, _ = p.communicate()
@@ -50,10 +54,29 @@ 

              sys.exit(1)

          return output

  

-     def test_upgrade_from(self, env, old_version):

+     def use_readonly_adminconfig(self, name):

+         admincfg = os.path.join(self.rootdir, 'tests', 'blobs', 'old_dbs',

+                                 'adminconfig.cfg')

+         cfgfile = os.path.join(self.testdir, 'lib', name, 'ipsilon.conf')

+         with open(cfgfile, 'r') as f:

+             cfg = f.readlines()

+         with open(cfgfile, 'w') as f:

+             for line in cfg:

+                 if line.startswith('admin.config.db'):

+                     line = 'admin.config.db = "configfile://%s"\n' % admincfg

+                 f.write(line)

+         with open(os.path.join(self.testdir, 'lib', name, 'openid.cfg'), 'w'):

+             # Just make the file exist. We don't actually use the OpenID plugin

+             # during this test, but it serves as a test for upgrading with

+             # readonly plugin databases.

+             pass

+ 

+     def test_upgrade_from(self, env, old_version, with_readonly):

          # Setup IDP Server

          print "Installing IDP server to test upgrade from %i" % old_version

          name = 'idp_v%i' % old_version

+         if with_readonly:

+             name = name + '_readonly'

          addr = '127.0.0.%i' % (10 + old_version)

          port = str(45080 + old_version)

          idp = self.generate_profile(idp_g, idp_a, name, addr, port)
@@ -65,6 +88,9 @@ 

                                  'v%i' % old_version)

          db_outdir = os.path.join(self.testdir, 'lib', name)

  

+         if with_readonly:

+             self.use_readonly_adminconfig(name)

+ 

          if old_version > 0:

              for database in ['adminconfig',

                               'openid',
@@ -74,8 +100,10 @@ 

                  db_in = os.path.join(db_indir, '%s.sqlite.dump' % database)

                  db_out = os.path.join(db_outdir, '%s.sqlite' % database)

                  os.unlink(db_out)

-                 cmd = ['/usr/bin/sqlite3', db_out, '.read %s' % db_in]

-                 subprocess.check_call(cmd)

+                 if database not in ['adminconfig',

+                                     'openid'] or not with_readonly:

+                     cmd = ['/usr/bin/sqlite3', db_out, '.read %s' % db_in]

+                     subprocess.check_call(cmd)

  

              # Upgrade that database

              cmd = [os.path.join(self.rootdir,
@@ -89,7 +117,7 @@ 

          if old_version == 0:

              # Check all features in a newly created database

              # Let's verify if at least one index was created

-             output = self.dump_admin_config_db(db_outdir)

+             output = self.dump_db(db_outdir, with_readonly)

              if 'CREATE INDEX' not in output:

                  raise Exception('Database upgrade did not introduce index')

              if 'PRIMARY KEY' not in output:
@@ -98,15 +126,15 @@ 

          elif old_version == 1:

              # In 1 -> 2, we added indexes and primary keys

              # Let's verify if at least one index was created

-             output = self.dump_admin_config_db(db_outdir)

+             output = self.dump_db(db_outdir, with_readonly)

              if 'CREATE INDEX' not in output:

                  raise Exception('Database upgrade did not introduce index')

              # SQLite did not support creating primary keys, so we can't test

  

-         elif old_version == 2:

+         elif old_version == 2 and not with_readonly:

              # Version 3 added the authz_config table

              # Make sure it exists

-             output = self.dump_admin_config_db(db_outdir)

+             output = self.dump_db(db_outdir, with_readonly)

              if 'TABLE authz_config' not in output:

                  raise Exception('Database upgrade did not introduce ' +

                                  'authz_config table')
@@ -120,6 +148,10 @@ 

              exe = exe[:-1]

          exe = [exe]

          exe.append(str(old_version))

+         if with_readonly:

+             exe.append('readonly')

+         else:

+             exe.append('no-readonly')

          exe.append(name)

          exe.append('%s:%s' % (addr, port))

          exit_code = subprocess.call(exe, env=env)
@@ -132,13 +164,15 @@ 

  

      def run(self, env):

          for version in range(ipsilon.util.data.CURRENT_SCHEMA_VERSION):

-             self.test_upgrade_from(env, version)

+             for with_readonly in [True, False]:

+                 self.test_upgrade_from(env, version, with_readonly)

  

  

  if __name__ == '__main__':

      from_version = sys.argv[1]

-     idpname = sys.argv[2]

-     url = sys.argv[3]

+     with_ro = sys.argv[2]

+     idpname = sys.argv[3]

+     url = sys.argv[4]

  

      user = pwd.getpwuid(os.getuid())[0]

  
@@ -146,7 +180,8 @@ 

      sess.add_server(idpname, 'https://%s' % url, user,

                      'ipsilon')

  

-     print "dbupgrades: From v%s: Authenticate to IDP ..." % from_version,

+     print "dbupgrades: From v%s %s: Authenticate to IDP ..." % (from_version,

+                                                                 with_ro),

      try:

          sess.auth_to_idp(idpname)

      except Exception, e:  # pylint: disable=broad-except