X-Git-Url: http://git.inspyration.org/?a=blobdiff_plain;f=openerp%2Fsql_db.py;h=f18414bdff8b47db7301c57853e00332cf192183;hb=4c68620d563b625532a476f127aca18ce088089b;hp=5d3ad49259f3009cd26850c38a2d7056c2bb7297;hpb=588552b52dbf41381ff9ca80fe414416367acc16;p=odoo%2Fodoo.git diff --git a/openerp/sql_db.py b/openerp/sql_db.py index 5d3ad49..f18414b 100644 --- a/openerp/sql_db.py +++ b/openerp/sql_db.py @@ -20,6 +20,8 @@ # ############################################################################## +#.apidoc title: PostgreSQL interface + """ The PostgreSQL connector is a connectivity layer between the OpenERP code and the database, *not* a database abstraction toolkit. Database abstraction is what @@ -28,19 +30,23 @@ the ORM does, in fact. See also: the `pooler` module """ +#.apidoc add-functions: print_stats +#.apidoc add-classes: Cursor Connection ConnectionPool + __all__ = ['db_connect', 'close_db'] -from threading import currentThread +from functools import wraps import logging -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE -from psycopg2.psycopg1 import cursor as psycopg1cursor -from psycopg2.pool import PoolError - import psycopg2.extensions -import warnings +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_REPEATABLE_READ +from psycopg2.pool import PoolError +from psycopg2.psycopg1 import cursor as psycopg1cursor +from threading import currentThread psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) +_logger = logging.getLogger(__name__) + types_mapping = { 'date': (1082,), 'time': (1083,), @@ -61,7 +67,7 @@ psycopg2.extensions.register_type(psycopg2.extensions.new_type((700, 701, 1700,) import tools -from tools.func import wraps, frame_codeinfo +from tools.func import frame_codeinfo from datetime import datetime as mdt from datetime import timedelta import threading @@ -74,15 +80,76 @@ re_into = re.compile('.* into "?([a-zA-Z_0-9]+)"? .*$'); sql_counter = 0 class Cursor(object): - """ Cursor is an open transaction to Postgres, utilizing a TCP connection - - A lightweight wrapper around psycopg2's `psycopg1cursor` objects - - This is the object behind the `cr` variable used all over the OpenERP - code. + """Represents an open transaction to the PostgreSQL DB backend, + acting as a lightweight wrapper around psycopg2's + ``psycopg1cursor`` objects. + + ``Cursor`` is the object behind the ``cr`` variable used all + over the OpenERP code. + + .. rubric:: Transaction Isolation + + One very important property of database transactions is the + level of isolation between concurrent transactions. + The SQL standard defines four levels of transaction isolation, + ranging from the most strict *Serializable* level, to the least + strict *Read Uncommitted* level. These levels are defined in + terms of the phenomena that must not occur between concurrent + transactions, such as *dirty read*, etc. + In the context of a generic business data management software + such as OpenERP, we need the best guarantees that no data + corruption can ever be cause by simply running multiple + transactions in parallel. Therefore, the preferred level would + be the *serializable* level, which ensures that a set of + transactions is guaranteed to produce the same effect as + running them one at a time in some order. + + However, most database management systems implement a limited + serializable isolation in the form of + `snapshot isolation `_, + providing most of the same advantages as True Serializability, + with a fraction of the performance cost. + With PostgreSQL up to version 9.0, this snapshot isolation was + the implementation of both the ``REPEATABLE READ`` and + ``SERIALIZABLE`` levels of the SQL standard. + As of PostgreSQL 9.1, the previous snapshot isolation implementation + was kept for ``REPEATABLE READ``, while a new ``SERIALIZABLE`` + level was introduced, providing some additional heuristics to + detect a concurrent update by parallel transactions, and forcing + one of them to rollback. + + OpenERP implements its own level of locking protection + for transactions that are highly likely to provoke concurrent + updates, such as stock reservations or document sequences updates. + Therefore we mostly care about the properties of snapshot isolation, + but we don't really need additional heuristics to trigger transaction + rollbacks, as we are taking care of triggering instant rollbacks + ourselves when it matters (and we can save the additional performance + hit of these heuristics). + + As a result of the above, we have selected ``REPEATABLE READ`` as + the default transaction isolation level for OpenERP cursors, as + it will be mapped to the desired ``snapshot isolation`` level for + all supported PostgreSQL version (8.3 - 9.x). + + Note: up to psycopg2 v.2.4.2, psycopg2 itself remapped the repeatable + read level to serializable before sending it to the database, so it would + actually select the new serializable mode on PostgreSQL 9.1. Make + sure you use psycopg2 v2.4.2 or newer if you use PostgreSQL 9.1 and + the performance hit is a concern for you. + + .. attribute:: cache + + Cache dictionary with a "request" (-ish) lifecycle, only lives as + long as the cursor itself does and proactively cleared when the + cursor is closed. + + This cache should *only* be used to store repeatable reads as it + ignores rollbacks and savepoints, it should not be used to store + *any* data which may be modified during the life of the cursor. + """ IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit - __logger = None def check(f): @wraps(f) @@ -95,34 +162,40 @@ class Cursor(object): return f(self, *args, **kwargs) return wrapper - def __init__(self, pool, dbname, serialized=False): - if self.__class__.__logger is None: - self.__class__.__logger = logging.getLogger('db.cursor') + def __init__(self, pool, dbname, serialized=True): self.sql_from_log = {} self.sql_into_log = {} # default log level determined at cursor creation, could be # overridden later for debugging purposes - self.sql_log = self.__logger.isEnabledFor(logging.DEBUG_SQL) + self.sql_log = _logger.isEnabledFor(logging.DEBUG) self.sql_log_count = 0 self.__closed = True # avoid the call of close() (by __del__) if an exception # is raised by any of the following initialisations self._pool = pool self.dbname = dbname + + # Whether to enable snapshot isolation level for this cursor. + # see also the docstring of Cursor. self._serialized = serialized + self._cnx = pool.borrow(dsn(dbname)) self._obj = self._cnx.cursor(cursor_factory=psycopg1cursor) - self.__closed = False # real initialisation value - self.autocommit(False) if self.sql_log: self.__caller = frame_codeinfo(currentframe(),2) else: self.__caller = False + self.__closed = False # real initialisation value + self.autocommit(False) self.__closer = False + self._default_log_exceptions = True + + self.cache = {} + def __del__(self): - if not self.__closed: + if not self.__closed and not self._cnx.closed: # Oops. 'self' has not been closed explicitly. # The cursor will be deleted by the garbage collector, # but the database connection is not put back into the connection @@ -133,15 +206,18 @@ class Cursor(object): msg += "Cursor was created at %s:%s" % self.__caller else: msg += "Please enable sql debugging to trace the caller." - self.__logger.warn(msg) + _logger.warning(msg) self._close(True) @check - def execute(self, query, params=None, log_exceptions=True): + def execute(self, query, params=None, log_exceptions=None): if '%d' in query or '%f' in query: - self.__logger.warn(query) - self.__logger.warn("SQL queries cannot contain %d or %f anymore. " - "Use only %s") + _logger.warning(query) + _logger.warning("SQL queries cannot contain %d or %f anymore. " + "Use only %s") + if params and not isinstance(params, (tuple, list, dict)): + _logger.error("SQL query parameters should be a tuple, list or dict; got %r", params) + raise ValueError("SQL query parameters should be a tuple, list or dict; got %r" % (params,)) if self.sql_log: now = mdt.now() @@ -150,19 +226,19 @@ class Cursor(object): params = params or None res = self._obj.execute(query, params) except psycopg2.ProgrammingError, pe: - if log_exceptions: - self.__logger.error("Programming error: %s, in query %s", pe, query) + if (self._default_log_exceptions if log_exceptions is None else log_exceptions): + _logger.error("Programming error: %s, in query %s", pe, query) raise except Exception: - if log_exceptions: - self.__logger.exception("bad query: %s", self._obj.query or query) + if (self._default_log_exceptions if log_exceptions is None else log_exceptions): + _logger.exception("bad query: %s", self._obj.query or query) raise if self.sql_log: delay = mdt.now() - now delay = delay.seconds * 1E6 + delay.microseconds - self.__logger.log(logging.DEBUG_SQL, "query: %s", self._obj.query) + _logger.debug("query: %s", self._obj.query) self.sql_log_count+=1 res_from = re_from.match(query.lower()) if res_from: @@ -193,16 +269,16 @@ class Cursor(object): if sqllogs[type]: sqllogitems = sqllogs[type].items() sqllogitems.sort(key=lambda k: k[1][1]) - self.__logger.log(logging.DEBUG_SQL, "SQL LOG %s:", type) + _logger.debug("SQL LOG %s:", type) sqllogitems.sort(lambda x,y: cmp(x[1][0], y[1][0])) for r in sqllogitems: delay = timedelta(microseconds=r[1][1]) - self.__logger.log(logging.DEBUG_SQL, "table: %s: %s/%s", + _logger.debug("table: %s: %s/%s", r[0], delay, r[1][0]) sum+= r[1][1] sqllogs[type].clear() sum = timedelta(microseconds=sum) - self.__logger.log(logging.DEBUG_SQL, "SUM %s:%s/%d [%d]", + _logger.debug("SUM %s:%s/%d [%d]", type, sum, self.sql_log_count, sql_counter) sqllogs[type].clear() process('from') @@ -218,13 +294,12 @@ class Cursor(object): if not self._obj: return + del self.cache + if self.sql_log: self.__closer = frame_codeinfo(currentframe(),3) self.print_log() - if not self._serialized: - self.rollback() # Ensure we close the current transaction. - self._obj.close() # This force the cursor to be freed, and thus, available again. It is @@ -235,16 +310,36 @@ class Cursor(object): del self._obj self.__closed = True + # Clean the underlying connection. + self._cnx.rollback() + if leak: self._cnx.leaked = True else: - keep_in_pool = self.dbname not in ('template1', 'template0', 'postgres') + chosen_template = tools.config['db_template'] + templates_list = tuple(set(['template0', 'template1', 'postgres', chosen_template])) + keep_in_pool = self.dbname not in templates_list self._pool.give_back(self._cnx, keep_in_pool=keep_in_pool) @check def autocommit(self, on): - offlevel = [ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE][bool(self._serialized)] - self._cnx.set_isolation_level([offlevel, ISOLATION_LEVEL_AUTOCOMMIT][bool(on)]) + if on: + isolation_level = ISOLATION_LEVEL_AUTOCOMMIT + else: + # If a serializable cursor was requested, we + # use the appropriate PotsgreSQL isolation level + # that maps to snaphsot isolation. + # For all supported PostgreSQL versions (8.3-9.x), + # this is currently the ISOLATION_REPEATABLE_READ. + # See also the docstring of this class. + # NOTE: up to psycopg 2.4.2, repeatable read + # is remapped to serializable before being + # sent to the database, so it is in fact + # unavailable for use with pg 9.1. + isolation_level = ISOLATION_LEVEL_REPEATABLE_READ \ + if self._serialized \ + else ISOLATION_LEVEL_READ_COMMITTED + self._cnx.set_isolation_level(isolation_level) @check def commit(self): @@ -279,7 +374,6 @@ class ConnectionPool(object): The connections are *not* automatically closed. Only a close_db() can trigger that. """ - __logger = logging.getLogger('db.connection_pool') def locked(fun): @wraps(fun) @@ -303,7 +397,7 @@ class ConnectionPool(object): return "ConnectionPool(used=%d/count=%d/max=%d)" % (used, count, self._maxconn) def _debug(self, msg, *args): - self.__logger.log(logging.DEBUG_SQL, ('%r ' + msg), self, *args) + _logger.debug(('%r ' + msg), self, *args) @locked def borrow(self, dsn): @@ -311,11 +405,15 @@ class ConnectionPool(object): # free leaked connections for i, (cnx, _) in tools.reverse_enumerate(self._connections): + if cnx.closed: + self._connections.pop(i) + self._debug('Removing closed connection at index %d: %r', i, cnx.dsn) + continue if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) - self.__logger.warn('%r: Free leaked connection to %r', self, cnx.dsn) + _logger.warning('%r: Free leaked connection to %r', self, cnx.dsn) for i, (cnx, used) in enumerate(self._connections): if not used and dsn_are_equals(cnx.dsn, dsn): @@ -338,8 +436,8 @@ class ConnectionPool(object): try: result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection) - except psycopg2.Error, e: - self.__logger.exception('Connection to the database failed') + except psycopg2.Error: + _logger.exception('Connection to the database failed') raise self._connections.append((result, True)) self._debug('Create new connection') @@ -356,13 +454,14 @@ class ConnectionPool(object): self._debug('Put connection to %r in pool', cnx.dsn) else: self._debug('Forgot connection to %r', cnx.dsn) + cnx.close() break else: raise PoolError('This connection does not below to the pool') @locked def close_all(self, dsn): - self.__logger.info('%r: Close all connections to %r', self, dsn) + _logger.info('%r: Close all connections to %r', self, dsn) for i, (cnx, used) in tools.reverse_enumerate(self._connections): if dsn_are_equals(cnx.dsn, dsn): cnx.close() @@ -372,25 +471,23 @@ class ConnectionPool(object): class Connection(object): """ A lightweight instance of a connection to postgres """ - __logger = logging.getLogger('db.connection') def __init__(self, pool, dbname): self.dbname = dbname self._pool = pool - def cursor(self, serialized=False): + def cursor(self, serialized=True): cursor_type = serialized and 'serialized ' or '' - self.__logger.log(logging.DEBUG_SQL, 'create %scursor to %r', cursor_type, self.dbname) + _logger.debug('create %scursor to %r', cursor_type, self.dbname) return Cursor(self._pool, self.dbname, serialized=serialized) - def serialized_cursor(self): - return self.cursor(True) + # serialized_cursor is deprecated - cursors are serialized by default + serialized_cursor = cursor def __nonzero__(self): """Check if connection is possible""" try: - warnings.warn("You use an expensive function to test a connection.", - DeprecationWarning, stacklevel=1) + _logger.warning("__nonzero__() is deprecated. (It is too expensive to test a connection.)") cr = self.cursor() cr.close() return True @@ -424,9 +521,10 @@ def db_connect(db_name): return Connection(_Pool, db_name) def close_db(db_name): - """ You might want to call openerp.netsvc.Agent.cancel(db_name) along this function.""" - _Pool.close_all(dsn(db_name)) - tools.cache.clean_caches_for_db(db_name) + global _Pool + """ You might want to call openerp.modules.registry.RegistryManager.delete(db_name) along this function.""" + if _Pool: + _Pool.close_all(dsn(db_name)) ct = currentThread() if hasattr(ct, 'dbname'): delattr(ct, 'dbname')