[MERGE] osv: automatically retry transactions that failed due to a transient concurre...

lp bug: https://launchpad.net/bugs/992525 fixed

bzr revid: odo@openerp.com-20130219162123-lho14jog2kky5u73

diff --cc bin/osv/osv.py
index d9d7b35,3f8d679..bf6c5ae
--- 1/bin/osv/osv.py
--- 2/bin/osv/osv.py
+++ b/bin/osv/osv.py
@@@ -116,47 -121,60 +121,60 @@@ class object_proxy(netsvc.Service)
              def _(src):
                  return tr(src, 'code')
  
-             try:
-                 if not pooler.get_pool(dbname)._ready:
-                     raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
-                 return f(self, dbname, *args, **kwargs)
-             except orm.except_orm, inst:
-                 if inst.name == 'AccessError':
-                     self.logger.debug("AccessError", exc_info=True)
-                 self.abortResponse(1, inst.name, 'warning', inst.value)
-             except except_osv, inst:
-                 self.abortResponse(1, inst.name, inst.exc_type, inst.value)
-             except IntegrityError, inst:
-                 osv_pool = pooler.get_pool(dbname)
-                 for key in osv_pool._sql_error.keys():
-                     if key in inst[0]:
-                         self.abortResponse(1, _('Constraint Error'), 'warning',
-                                         tr(osv_pool._sql_error[key], 'sql_constraint') or inst[0])
-                 if inst.pgcode in (errorcodes.NOT_NULL_VIOLATION, errorcodes.FOREIGN_KEY_VIOLATION, errorcodes.RESTRICT_VIOLATION):
-                     msg = _('The operation cannot be completed, probably due to the following:\n- deletion: you may be trying to delete a record while other records still reference it\n- creation/update: a mandatory field is not correctly set')
-                     self.logger.debug("IntegrityError", exc_info=True)
-                     try:
-                         errortxt = inst.pgerror.replace('«','"').replace('»','"')
-                         if '"public".' in errortxt:
-                             context = errortxt.split('"public".')[1]
-                             model_name = table = context.split('"')[1]
-                         else:
-                             last_quote_end = errortxt.rfind('"')
-                             last_quote_begin = errortxt.rfind('"', 0, last_quote_end)
-                             model_name = table = errortxt[last_quote_begin+1:last_quote_end].strip()
-                         model = table.replace("_",".")
-                         model_obj = osv_pool.get(model)
-                         if model_obj:
-                             model_name = model_obj._description or model_obj._name
-                         msg += _('\n\n[object with reference: %s - %s]') % (model_name, model)
-                     except Exception:
-                         pass
-                     self.abortResponse(1, _('Integrity Error'), 'warning', msg)
-                 else:
-                     self.abortResponse(1, _('Integrity Error'), 'warning', inst[0])
-             except Exception:
-                 self.logger.exception("Uncaught exception")
-                 raise
+             tries = 0
+             while True:
+                 try:
+                     if not pooler.get_pool(dbname)._ready:
+                         raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
+                     return f(self, dbname, *args, **kwargs)
+                 except OperationalError, e:
+                     # Automatically retry the typical transaction serialization errors
+                     if e.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY:
+                         raise
+                     if tries >= MAX_TRIES_ON_CONCURRENCY_FAILURE:
 -                        self.logger.warning("%s, maximum number of tries reached" % e.pgcode)
++                        self.logger.warning("Concurrent transaction - OperationalError %s, maximum number of tries reached" % e.pgcode)
+                         raise
+                     wait_time = random.uniform(0.0, 2 ** tries)
+                     tries += 1
 -                    self.logger.info("%s, retrying %d/%d in %.04f sec..." % (e.pgcode, tries, MAX_TRIES_ON_CONCURRENCY_FAILURE, wait_time))
++                    self.logger.info("Concurrent transaction detected (%s), retrying %d/%d in %.04f sec..." % (e.pgcode, tries, MAX_TRIES_ON_CONCURRENCY_FAILURE, wait_time))
+                     time.sleep(wait_time)
+                 except orm.except_orm, inst:
+                     if inst.name == 'AccessError':
+                         self.logger.debug("AccessError", exc_info=True)
+                     self.abortResponse(1, inst.name, 'warning', inst.value)
+                 except except_osv, inst:
+                     self.abortResponse(1, inst.name, inst.exc_type, inst.value)
+                 except IntegrityError, inst:
+                     osv_pool = pooler.get_pool(dbname)
+                     for key in osv_pool._sql_error.keys():
+                         if key in inst[0]:
+                             self.abortResponse(1, _('Constraint Error'), 'warning',
+                                             tr(osv_pool._sql_error[key], 'sql_constraint') or inst[0])
+                     if inst.pgcode in (errorcodes.NOT_NULL_VIOLATION, errorcodes.FOREIGN_KEY_VIOLATION, errorcodes.RESTRICT_VIOLATION):
+                         msg = _('The operation cannot be completed, probably due to the following:\n- deletion: you may be trying to delete a record while other records still reference it\n- creation/update: a mandatory field is not correctly set')
+                         self.logger.debug("IntegrityError", exc_info=True)
+                         try:
+                             errortxt = inst.pgerror.replace('«','"').replace('»','"')
+                             if '"public".' in errortxt:
+                                 context = errortxt.split('"public".')[1]
+                                 model_name = table = context.split('"')[1]
+                             else:
+                                 last_quote_end = errortxt.rfind('"')
+                                 last_quote_begin = errortxt.rfind('"', 0, last_quote_end)
+                                 model_name = table = errortxt[last_quote_begin+1:last_quote_end].strip()
+                             model = table.replace("_",".")
+                             model_obj = osv_pool.get(model)
+                             if model_obj:
+                                 model_name = model_obj._description or model_obj._name
+                             msg += _('\n\n[object with reference: %s - %s]') % (model_name, model)
+                         except Exception:
+                             pass
+                         self.abortResponse(1, _('Integrity Error'), 'warning', msg)
+                     else:
+                         self.abortResponse(1, _('Integrity Error'), 'warning', inst[0])
+                 except Exception:
+                     self.logger.exception("Uncaught exception")
+                     raise
  
          return wrapper