From 8403abe84f201d522058a8f0c6c9013c6bf5add5 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Wed, 13 Feb 2013 13:04:50 +0100 Subject: [PATCH] [FIX] osv: Automatically retry the typical transaction serialization errors lp bug: https://launchpad.net/bugs/992525 fixed bzr revid: cto@openerp.com-20130213120450-t0w3iqx7ik04ceri --- bin/osv/osv.py | 100 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/bin/osv/osv.py b/bin/osv/osv.py index d9d7b35..efc2e75 100644 --- a/bin/osv/osv.py +++ b/bin/osv/osv.py @@ -28,14 +28,19 @@ import netsvc import pooler import copy import logging -from psycopg2 import IntegrityError, errorcodes +from psycopg2 import IntegrityError, OperationalError, errorcodes from tools.func import wraps from tools.translate import translate +import time +import random module_list = [] module_class_list = {} class_pool = {} +PG_CONCURRENCY_ERRORS_TO_RETRY = (errorcodes.LOCK_NOT_AVAILABLE, errorcodes.SERIALIZATION_FAILURE, errorcodes.DEADLOCK_DETECTED) +MAX_TRIES_ON_CONCURRENCY_FAILURE = 5 + class except_osv(Exception): def __init__(self, name, value, exc_type='warning'): self.name = name @@ -116,47 +121,58 @@ class object_proxy(netsvc.Service): def _(src): return tr(src, 'code') - try: - if not pooler.get_pool(dbname)._ready: - raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.') - return f(self, dbname, *args, **kwargs) - except orm.except_orm, inst: - if inst.name == 'AccessError': - self.logger.debug("AccessError", exc_info=True) - self.abortResponse(1, inst.name, 'warning', inst.value) - except except_osv, inst: - self.abortResponse(1, inst.name, inst.exc_type, inst.value) - except IntegrityError, inst: - osv_pool = pooler.get_pool(dbname) - for key in osv_pool._sql_error.keys(): - if key in inst[0]: - self.abortResponse(1, _('Constraint Error'), 'warning', - tr(osv_pool._sql_error[key], 'sql_constraint') or inst[0]) - if inst.pgcode in (errorcodes.NOT_NULL_VIOLATION, errorcodes.FOREIGN_KEY_VIOLATION, errorcodes.RESTRICT_VIOLATION): - msg = _('The operation cannot be completed, probably due to the following:\n- deletion: you may be trying to delete a record while other records still reference it\n- creation/update: a mandatory field is not correctly set') - self.logger.debug("IntegrityError", exc_info=True) - try: - errortxt = inst.pgerror.replace('«','"').replace('»','"') - if '"public".' in errortxt: - context = errortxt.split('"public".')[1] - model_name = table = context.split('"')[1] - else: - last_quote_end = errortxt.rfind('"') - last_quote_begin = errortxt.rfind('"', 0, last_quote_end) - model_name = table = errortxt[last_quote_begin+1:last_quote_end].strip() - model = table.replace("_",".") - model_obj = osv_pool.get(model) - if model_obj: - model_name = model_obj._description or model_obj._name - msg += _('\n\n[object with reference: %s - %s]') % (model_name, model) - except Exception: - pass - self.abortResponse(1, _('Integrity Error'), 'warning', msg) - else: - self.abortResponse(1, _('Integrity Error'), 'warning', inst[0]) - except Exception: - self.logger.exception("Uncaught exception") - raise + tries = 0 + while True: + try: + if not pooler.get_pool(dbname)._ready: + raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.') + return f(self, dbname, *args, **kwargs) + except OperationalError, e: + # Automatically retry the typical transaction serialization errors + if not e.pgcode in PG_CONCURRENCY_ERRORS_TO_RETRY or tries >= MAX_TRIES_ON_CONCURRENCY_FAILURE: + self.logger.warning("%s, maximum number of tries reached" % errorcodes.lookup(e.pgcode)) + raise + wait_time = random.uniform(0.0, 2 ** tries) + tries += 1 + self.logger.info("%s, retrying %d/%d in %.04f sec..." % (errorcodes.lookup(e.pgcode), tries, MAX_TRIES_ON_CONCURRENCY_FAILURE, wait_time)) + time.sleep(wait_time) + except orm.except_orm, inst: + if inst.name == 'AccessError': + self.logger.debug("AccessError", exc_info=True) + self.abortResponse(1, inst.name, 'warning', inst.value) + except except_osv, inst: + self.abortResponse(1, inst.name, inst.exc_type, inst.value) + except IntegrityError, inst: + osv_pool = pooler.get_pool(dbname) + for key in osv_pool._sql_error.keys(): + if key in inst[0]: + self.abortResponse(1, _('Constraint Error'), 'warning', + tr(osv_pool._sql_error[key], 'sql_constraint') or inst[0]) + if inst.pgcode in (errorcodes.NOT_NULL_VIOLATION, errorcodes.FOREIGN_KEY_VIOLATION, errorcodes.RESTRICT_VIOLATION): + msg = _('The operation cannot be completed, probably due to the following:\n- deletion: you may be trying to delete a record while other records still reference it\n- creation/update: a mandatory field is not correctly set') + self.logger.debug("IntegrityError", exc_info=True) + try: + errortxt = inst.pgerror.replace('«','"').replace('»','"') + if '"public".' in errortxt: + context = errortxt.split('"public".')[1] + model_name = table = context.split('"')[1] + else: + last_quote_end = errortxt.rfind('"') + last_quote_begin = errortxt.rfind('"', 0, last_quote_end) + model_name = table = errortxt[last_quote_begin+1:last_quote_end].strip() + model = table.replace("_",".") + model_obj = osv_pool.get(model) + if model_obj: + model_name = model_obj._description or model_obj._name + msg += _('\n\n[object with reference: %s - %s]') % (model_name, model) + except Exception: + pass + self.abortResponse(1, _('Integrity Error'), 'warning', msg) + else: + self.abortResponse(1, _('Integrity Error'), 'warning', inst[0]) + except Exception: + self.logger.exception("Uncaught exception") + raise return wrapper -- 1.7.10.4