From 6a81e952ef400e01c131ff0773c92fc040d888f0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 05 2015 20:49:18 +0000 Subject: [PATCH 1/4] Make the database upgrade system use logger Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/install/ipsilon-upgrade-database b/ipsilon/install/ipsilon-upgrade-database index 7ee18fb..cb1d4fa 100755 --- a/ipsilon/install/ipsilon-upgrade-database +++ b/ipsilon/install/ipsilon-upgrade-database @@ -5,10 +5,19 @@ import sys from ipsilon import find_config from ipsilon.tools import dbupgrade +import logging +logger = logging.getLogger(__name__) -try: - dbupgrade.execute_upgrade(find_config()) -except Exception as ex: - print ex - sys.exit(1) + +if __name__ == '__main__': + def_logger = logging.getLogger() + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.ERROR) + def_logger.addHandler(ch) + + try: + dbupgrade.execute_upgrade(find_config()) + except Exception as ex: + logger.error(ex) + sys.exit(1) diff --git a/ipsilon/tools/dbupgrade.py b/ipsilon/tools/dbupgrade.py index e6d2b16..7e008f2 100644 --- a/ipsilon/tools/dbupgrade.py +++ b/ipsilon/tools/dbupgrade.py @@ -10,9 +10,13 @@ from ipsilon.util.data import AdminStore, Store, UserStore, TranStore from ipsilon.util.sessions import SqlSession from ipsilon.root import Root +import logging + +logger = logging.getLogger(__name__) + def _upgrade_database(datastore): - print 'Considering datastore %s' % datastore.__class__.__name__ + logger.debug('Considering datastore %s', datastore.__class__.__name__) # pylint: disable=protected-access current_version = datastore._get_schema_version() # pylint: disable=protected-access @@ -20,22 +24,24 @@ def _upgrade_database(datastore): upgrade_required = False if current_version is None: # Initialize schema - print 'Initializing schema for %s' % datastore.__class__.__name__ + logger.debug('Initializing schema for %s', + datastore.__class__.__name__) upgrade_required = True elif current_version != code_schema_version: - print 'Upgrading schema for %s' % datastore.__class__.__name__ + logger.debug('Upgrading schema for %s', datastore.__class__.__name__) upgrade_required = True else: - print 'Schema for %s is up-to-date' % datastore.__class__.__name__ + logger.debug('Schema for %s is up-to-date', + datastore.__class__.__name__) if upgrade_required: if datastore.is_readonly: - print 'Datastore is readonly. Please fix manually!' + logger.warning('Datastore is readonly. Please fix manually!') return False try: datastore.upgrade_database() except Exception as ex: # pylint: disable=broad-except # Error upgrading database - print 'Error upgrading datastore: %s' % ex + logger.error('Error upgrading datastore: %s', ex) return False else: # Upgrade went OK @@ -45,7 +51,7 @@ def _upgrade_database(datastore): def upgrade_failed(): - print 'Upgrade failed. Please fix errors above and retry' + logger.error('Upgrade failed. Please fix errors above and retry') raise Exception('Upgrading failed') @@ -72,9 +78,9 @@ def execute_upgrade(cfgfile): root = Root('default', template_env) # Handle the session store if that is Sql - print 'Handling sessions datastore' + logger.debug('Handling sessions datastore') if cherrypy.config['tools.sessions.storage_type'] != 'sql': - print 'Not SQL-based, skipping' + logger.debug('Not SQL-based, skipping') else: dburi = cherrypy.config['tools.sessions.storage_dburi'] SqlSession.setup(storage_dburi=dburi) @@ -84,7 +90,8 @@ def execute_upgrade(cfgfile): # Now handle the rest of the default datastores for store in [UserStore, TranStore]: store = store() - print 'Handling default datastore %s' % store.__class__.__name__ + logger.debug('Handling default datastore %s', + store.__class__.__name__) if not _upgrade_database(store): return upgrade_failed() @@ -93,12 +100,13 @@ def execute_upgrade(cfgfile): 'login_config', 'info_config']: for plugin in root._site[facility].enabled: - print 'Handling plugin %s' % plugin + logger.debug('Handling plugin %s', plugin) plugin = root._site[facility].available[plugin] - print 'Creating plugin AdminStore table' + logger.debug('Creating plugin AdminStore table') adminstore.create_plugin_data_table(plugin.name) for store in plugin.used_datastores(): - print 'Handling plugin datastore %s' % store.__class__.__name__ + logger.debug('Handling plugin datastore %s', + store.__class__.__name__) if not _upgrade_database(store): return upgrade_failed() From c9ffd51879f375f1e9e326747130835e117c86e9 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 05 2015 20:49:22 +0000 Subject: [PATCH 2/4] Also add the store name when reporting data load error Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 0981c52..7f387b7 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -510,7 +510,8 @@ class Store(Log): q = self._query(self._db, table, columns, trans=False) rows = q.select(kvfilter) except Exception, e: # pylint: disable=broad-except - self.error("Failed to load data for table %s: [%s]" % (table, e)) + self.error("Failed to load data for table %s for store %s: [%s]" + % (table, self.__class__.__name__, e)) return self._rows_to_dict_tree(rows) def load_config(self): From 4d25f65e78ed2343cde4948172d876e084764731 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 05 2015 20:49:22 +0000 Subject: [PATCH 3/4] Also create plugin UserStore data tables Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/tools/dbupgrade.py b/ipsilon/tools/dbupgrade.py index 7e008f2..021acf3 100644 --- a/ipsilon/tools/dbupgrade.py +++ b/ipsilon/tools/dbupgrade.py @@ -96,6 +96,7 @@ def execute_upgrade(cfgfile): return upgrade_failed() # And now datastores for any of the plugins + userstore = UserStore() for facility in ['provider_config', 'login_config', 'info_config']: @@ -104,6 +105,8 @@ def execute_upgrade(cfgfile): plugin = root._site[facility].available[plugin] logger.debug('Creating plugin AdminStore table') adminstore.create_plugin_data_table(plugin.name) + logger.debug('Creating plugin UserStore table') + userstore.create_plugin_data_table(plugin.name) for store in plugin.used_datastores(): logger.debug('Handling plugin datastore %s', store.__class__.__name__) diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 7f387b7..65bf4b5 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -739,6 +739,14 @@ class UserStore(Store): else: raise NotImplementedError() + def create_plugin_data_table(self, plugin_name): + if not self.is_readonly: + table = plugin_name+'_data' + q = self._query(self._db, table, OPTIONS_TABLE, + trans=False) + q.create() + q._con.close() # pylint: disable=protected-access + class TranStore(Store): From effa209e386930ad705f255e61c24a5dfb057987 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 05 2015 20:49:22 +0000 Subject: [PATCH 4/4] Make it possible to use PluginLoader without store In the case of OpenID extensions, a backend store is not needed for the PluginLoader, since the IDP Plugin has its own configuration for enabled extensions. Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/providers/openid/extensions/common.py b/ipsilon/providers/openid/extensions/common.py index 55809c1..247584c 100644 --- a/ipsilon/providers/openid/extensions/common.py +++ b/ipsilon/providers/openid/extensions/common.py @@ -49,7 +49,7 @@ class LoadExtensions(Log): def __init__(self): self.plugins = PluginLoader(LoadExtensions, - FACILITY, 'OpenidExtension') + FACILITY, 'OpenidExtension', False) self.plugins.get_plugin_data() available = self.plugins.available.keys() diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index 3a45f19..bf2898c 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -94,10 +94,6 @@ class OpenIDStore(Store, OpenIDStoreInterface): trans=False) q.create() q._con.close() # pylint: disable=protected-access - q = self._query(self._db, 'openid_extensions', OPTIONS_TABLE, - trans=False) - q.create() - q._con.close() # pylint: disable=protected-access def _upgrade_schema(self, old_version): if old_version == 1: diff --git a/ipsilon/util/plugin.py b/ipsilon/util/plugin.py index 6aecbf6..87ab1ae 100644 --- a/ipsilon/util/plugin.py +++ b/ipsilon/util/plugin.py @@ -58,19 +58,23 @@ class Plugins(object): class PluginLoader(Log): - def __init__(self, baseobj, facility, plugin_type): + def __init__(self, baseobj, facility, plugin_type, uses_store=True): self._pathname, _ = os.path.split(inspect.getfile(baseobj)) self.facility = facility self._plugin_type = plugin_type self.available = dict() self.enabled = list() - self.__data = None + self.__data = False + self.uses_store = uses_store # Defer initialization or instantiating the store will fail at load # time when used with Installer plugins as the cherrypy config context # is created after all Installer plugins are loaded. @property def _data(self): + if not self.uses_store: + raise Exception('Tried to get plugin data while ' + + 'uses_store=False (%s)' % self.facility) if not self.__data: self.__data = AdminStore() return self.__data @@ -92,7 +96,8 @@ class PluginLoader(Log): def get_plugin_data(self): self.available = self.get_plugins() - self.refresh_enabled() + if self.uses_store: + self.refresh_enabled() def save_enabled(self, enabled): if enabled: