From 588891867d0e57b4aa0d25078e18089cd6544b72 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Nov 28 2016 19:02:35 +0000 Subject: WIP: Fixes Signed-off-by: Patrick Uiterwijk --- diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index cfb72dc..6ccbe46 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -12,6 +12,7 @@ from sqlalchemy.sql import select, and_ import ConfigParser import etcd import os +import json import uuid import logging import time @@ -362,7 +363,7 @@ class EtcdQuery(Log): columns = table_def['columns'] if isinstance(columns[0], tuple): columns = [column[0] for column in columns] - self._primary_key = list(table_def['primary_key']) + self._primary_key = tuple(table_def['primary_key']) else: columns = table_def raise Exception('Etcd requires primary key') @@ -383,20 +384,25 @@ class EtcdQuery(Log): Return a tuple with path and then the number of path levels not used. """ path = self._table_dir + self.debug('Getting specific dir for %s' % kvfilter) if kvfilter is None: kvfilter = {} pkeys_used = 0 for pkey in self._primary_key: + self.debug('Checking pkey %s' % pkey) if pkey in kvfilter: + self.debug('Part') pkeys_used += 1 path = os.path.join(path, kvfilter[pkey].replace(' ', '_')) if del_kv: del kvfilter[pkey] else: + self.debug('Not part') break + self.debug('Path: %s' % path) return path, len(self._primary_key) - pkeys_used def rollback(self): @@ -425,42 +431,49 @@ class EtcdQuery(Log): path, levels_unused = self._get_most_specific_dir(kvfilter) try: - res = self._store.client.read(path) + res = self._store.client.read(path, recursive=levels_unused != 0) except etcd.EtcdKeyNotFound: return [] - intermediate = [res] - # Get all children - for _ in range(levels_unused): - int_old = intermediate - intermediate = [] + self.debug('We are going into select with %s' % res.key) - for old in int_old: - for cld in old.children: - if cld.key != old.key: - intermediate.append(self._store.client.read(cld.key)) + if levels_unused == 0: + # This was a fully qualified object, let's use the object + if res.dir: + res = [] + else: + res = [res] + else: + # This was not fully qualified. Given we used recursive=True, we + # know that "children" is the final objects. + res = list([cld for cld in res.children if not cld.dir]) + + self.debug('After going all levels, we ended up with %s' % [i.key for i in res]) results = [] - for obj in intermediate: - for cld in obj.children: - key = os.path.basename(cld.key) - if key in kvfilter: - if cld.value != kvfilter[key]: - # This object had a value that did not match - continue - results.append(obj) + for obj in res: + result = json.loads(obj.value) + + pick_object = True + for key in kvfilter: + if key not in result: + self.debug('Key %s not in object' % key) + pick_object = False + break + if result[key] != kvfilter[key]: + self.debug('key %s value %s != %s' % (key, result[key], kvfilter[key])) + pick_object = False + break + if pick_object: + results.append(result) + + self.debug('Results: %s' % results) rows = [] for obj in results: - if obj.value is None: - continue - values = {} - for cld in obj.children: - key = os.path.basename(cld.key) - values[key] = cld.value row = [] for column in columns: - row.append(values[column]) + row.append(obj[column]) rows.append(tuple(row)) self.debug('Result: %s' % rows) @@ -476,11 +489,10 @@ class EtcdQuery(Log): values[column] = value_row.pop(0) path, levels_unused = self._get_most_specific_dir(values, False) + self.debug('Path: %s' % path) if levels_unused != 0: raise Exception('Unable to insert non-complete object') - for key in values: - self._store.client.write(os.path.join(path, key), values[key], - ttl=ttl) + self._store.client.write(path, json.dumps(values), ttl=ttl) def update(self, values, kvfilter): self.debug('Updating %s, values %s, kvfilter %s' % @@ -491,18 +503,26 @@ class EtcdQuery(Log): raise Exception('Unable to update non-complete object') for key in values: - self._store.client.write(os.path.join(path, key), values[key]) + if key in self._primary_key: + raise Exception('Unable to update primary key values') + + current = json.loads(self._store.client.read(path).value) + for key in values: + current[key] = values[key] + self._store.client.write(path, json.dumps(current)) def delete(self, kvfilter): self.debug('Deleting %s, kvfilter %s' % (self._table_dir, kvfilter)) path, levels_unused = self._get_most_specific_dir(kvfilter) if levels_unused == 0 or len(kvfilter) == 0: + try: + current = json.loads(self._store.client.read(path).value) + except etcd.EtcdKeyNotFound: + self.debug('Nothing to delete') + return for key in kvfilter: - if not self._store.client.read( - os.path.join(path), key) == kvfilter[key]: - # This object matched the primary key but not the rest of - # the filters + if current[key] != kvfilter[key]: return try: self._store.client.delete(path, recursive=True, dir=True) @@ -528,6 +548,7 @@ class Store(Log): _auto_cleanup_tables = [] def __init__(self, config_name=None, database_url=None): + self.error('*****%s config name: %s database: %s' % (self.__class__.__name__, config_name, database_url)) if config_name is None and database_url is None: raise ValueError('config_name or database_url must be provided') if config_name: @@ -536,6 +557,7 @@ class Store(Log): name = cherrypy.config[config_name] else: name = database_url + self.error('Name: %s' % name) if name.startswith('configfile://'): _, filename = name.split('://') self._db = FileStore(filename) @@ -756,8 +778,10 @@ class Store(Log): q = self._query(self._db, table, columns, trans=False) rows = q.select(kvfilter) except Exception, e: # pylint: disable=broad-except + import traceback self.error("Failed to load data for table %s for store %s: [%s]" % (table, self.__class__.__name__, e)) + self.error(traceback.format_exc()) return self._rows_to_dict_tree(rows) def load_config(self): diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index ce8f62b..2f4f974 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING +# Copyright (C) 2014,2016 Ipsilon project Contributors, for license see COPYING import base64 from cherrypy.lib.sessions import Session @@ -13,7 +13,6 @@ except ImportError: import pickle import json - SESSION_TABLE = {'columns': ['id', 'data', 'expiration_time'], 'primary_key': ('id', ), 'indexes': [('expiration_time',)] @@ -164,7 +163,10 @@ class EtcdSession(Session): def _load(self): try: data = self._client.read(self._session_path) - return json.loads(data.value) # pylint: disable=no-member + # pylint: disable=no-member + value, exp_time = json.loads(data.value) + exp_dt = datetime.datetime.utcfromtimestamp(exp_time) + return value, exp_dt except etcd.EtcdKeyNotFound: return None @@ -172,7 +174,7 @@ class EtcdSession(Session): expiration_time = int(time.mktime(expiration_time.timetuple())) ttl = expiration_time - int(time.time()) self._client.write(self._session_path, - json.dumps(self._data), + json.dumps((self._data, expiration_time)), ttl=ttl) def _delete(self): diff --git a/tests/helpers/common.py b/tests/helpers/common.py index 5156171..8fc9639 100755 --- a/tests/helpers/common.py +++ b/tests/helpers/common.py @@ -249,7 +249,7 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir, env['ETCD_ADVERTISE_CLIENT_URLS'] = 'http://%s:%s' % (addr, clientport) env['ETCD_INITIAL_ADVERTISE_PEER_URLS'] = 'http://%s:%s' % (addr, srvport) - p = subprocess.Popen(['/usr/bin/etcd', '--debug'], + p = subprocess.Popen(['/usr/bin/etcd'], env=env, preexec_fn=os.setsid) self.processes.append(p) return p