[ADD] utility methods to split large lists of ids for use in IN clauses

bzr revid: odo@openerp.com-20100330172603-ty878ds2i9oa33zq

diff --git a/bin/sql_db.py b/bin/sql_db.py
index 17bfa34..94336cd 100644 (file)
--- a/bin/sql_db.py
+++ b/bin/sql_db.py
@@ -63,7 +63,7 @@ re_into = re.compile('.* into "?([a-zA-Z_0-9]+)"? .*$');
 sql_counter = 0
 
 class Cursor(object):
-    IN_MAX = 1000
+    IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit
     __logger = logging.getLogger('db.cursor')
 
     def check(f):
@@ -142,6 +142,12 @@ class Cursor(object):
                 self.sql_into_log[res_into.group(1)][1] += delay
         return res
 
+
+    def split_for_in_conditions(self, ids):
+        """Split a list of identifiers into one or more smaller tuples
+           safe for IN conditions, after uniquifying them."""
+        return tools.misc.split_every(self.IN_MAX, set(ids))
+
     def print_log(self):
         global sql_counter
         sql_counter += self.sql_log_count

diff --git a/bin/tools/misc.py b/bin/tools/misc.py
index 2843248..ff20966 100644 (file)
--- a/bin/tools/misc.py
+++ b/bin/tools/misc.py
@@ -32,6 +32,7 @@ import zipfile
 import release
 import socket
 import re
+from itertools import islice
 
 if sys.version_info[:2] < (2, 4):
     from threadinglocal import local
@@ -1267,6 +1268,18 @@ def detect_server_timezone():
     return 'UTC'
 
 
+def split_every(n, iterable, piece_maker=tuple):
+    """Splits an iterable into length-n pieces. The last piece will be shorter
+       if ``n`` does not evenly divide the iterable length.
+       @param ``piece_maker``: function to build the pieces
+       from the slices (tuple,list,...)
+    """
+    iterator = iter(iterable)
+    piece = piece_maker(islice(iterator, n))
+    while piece:
+        yield piece
+        piece = piece_maker(islice(iterator, n))
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()