move domain parsing into expression class

bzr revid: christophe@tinyerp.com-20080804153250-9crz6sq4hf3rm81n

diff --git a/bin/osv/expression.py b/bin/osv/expression.py
new file mode 100644 (file)
index 0000000..60512fe
--- /dev/null
+++ b/bin/osv/expression.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+
+from tools import flatten
+
+class expression(object):
+    """
+    parse a domain expression
+    examples:
+
+    >>> e = [('foo', '=', 'bar')]
+    >>> expression(e).parse().to_sql()
+    'foo = bar'
+    >>> e = [('id', 'in', [1,2,3])]
+    >>> expression(e).parse().to_sql()
+    'id in (1, 2, 3)'
+    >>> e = [('field', '=', 'value'), ('field', '<>', 'value')]
+    >>> expression(e).parse().to_sql()
+    '( field = value AND field <> value )'
+    >>> e = [('&', ('field', '<', 'value'), ('field', '>', 'value'))]
+    >>> expression(e).parse().to_sql()
+    '( field < value AND field > value )'
+    >>> e = [('|', ('field', '=', 'value'), ('field', '=', 'value'))]
+    >>> expression(e).parse().to_sql()
+    '( field = value OR field = value )'
+    >>> e = [('&', ('field1', '=', 'value'), ('field2', '=', 'value'), ('|', ('field3', '<>', 'value'), ('field4', '=', 'value')))]
+    >>> expression(e).parse().to_sql()
+    '( field1 = value AND field2 = value AND ( field3 <> value OR field4 = value ) )'
+    >>> e = [('&', ('|', ('a', '=', '1'), ('b', '=', '2')), ('|', ('c', '=', '3'), ('d', '=', '4')))]
+    >>> expression(e).parse().to_sql()
+    '( ( a = 1 OR b = 2 ) AND ( c = 3 OR d = 4 ) )'
+    >>> e = [('|', (('a', '=', '1'), ('b', '=', '2')), (('c', '=', '3'), ('d', '=', '4')))]
+    >>> expression(e).parse().to_sql()
+    '( ( a = 1 AND b = 2 ) OR ( c = 3 AND d = 4 ) )'
+    >>> expression(e).parse().get_tables()
+    []
+    >>> expression('fail').parse().to_sql()
+    Traceback (most recent call last):
+    ...
+    ValueError: Bad expression: 'fail'
+    >>> e = [('fail', 'is', 'True')]
+    >>> expression(e).parse().to_sql()
+    Traceback (most recent call last):
+    ...
+    ValueError: Bad expression: ('&', ('fail', 'is', 'True'))
+    """
+
+    def _is_operator(self, element):
+        return isinstance(element, str) \
+           and element in ['&','|']
+
+    def _is_leaf(self, element):
+        return isinstance(element, tuple) \
+           and len(element) == 3 \
+           and element[1] in ('=', '<>', '<=', '<', '>', '>=', '=like', 'like', 'not like', 'ilike', 'not ilike', 'in', 'not in', 'child_of') 
+
+    def _is_expression(self, element):
+        return isinstance(element, tuple) \
+           and len(element) > 2 \
+           and self._is_operator(element[0])
+
+    
+    def __execute_recursive_in(self, cr, s, f, w, ids):
+        ID_MAX = 1000
+        res = []
+        for i in range(0, len(ids), ID_MAX):
+            subids = ids[i:i+ID_MAX]
+            cr.execute('SELECT "%s"'    \
+                       '  FROM "%s"'    \
+                       ' WHERE "%s" in (%s)' % (s, f, w, ','.join(['%d']*len(subids))),
+                       subids)
+            res.extend([r[0] for r in cr.fetchall()])
+        return res
+
+
+    def __init__(self, exp):
+        if exp and isinstance(exp, tuple):
+            if not self._is_leaf(exp) and not self._is_operator(exp[0]):
+                exp = list(exp)
+        if exp and isinstance(exp, list):
+            if len(exp) == 1 and self._is_leaf(exp[0]):
+                exp = exp[0]
+            else:
+                if not self._is_operator(exp[0][0]):
+                    exp.insert(0, '&')
+                    exp = tuple(exp)
+                else:
+                    exp = exp[0]
+
+        self.__exp = exp
+        self.__iexp = exp
+        self.__operator = '&'
+        self.__children = []
+
+        self.__tables = []
+        self.__joins = []
+        self.__table = None
+
+        self.__left, self.__right = None, None
+        if self._is_leaf(self.__exp):
+            self.__left, self.__operator, self.__right = self.__exp
+            if isinstance(self.__right, list):
+                self.__right = tuple(self.__right)
+        elif exp and not self._is_expression(self.__exp):
+            raise ValueError, 'Bad expression: %r' % (self.__exp,)
+
+    def parse(self, cr, uid, table, context):
+
+        def _rec_get(ids, table, parent):
+            if not ids:
+                return []
+            ids2 = table.search(cr, uid, [(parent, 'in', ids)], context=context)
+            return ids + _rec_get(ids2, table, parent)
+        
+        if not self.__exp:
+            return self
+
+        if self._is_leaf(self.__exp):
+            self.__table = table
+            self.__tables.append(self.__table._table)
+            if self.__left in table._inherit_fields:
+                self.__table = table.pool.get(table._inherit_fields[self.__left][0])
+                if self.__table._table not in self.__tables:
+                    self.__tables.append(self.__table._table)
+                    self.__joins.append('%s.%s' % (table._table, table._inherits[self.__table._name]))
+            fargs = self.__left.split('.', 1)
+            field = self.__table._columns.get(fargs[0], False)
+            if not field:
+                if self.__left == 'id' and self.__operator == 'child_of':
+                    self.__right += _rec_get(self.__right, self.__table, self.__table._parent_name)
+                    self.__operator = 'in'
+                return self
+            if len(fargs) > 1:
+                if field._type == 'many2one':
+                    self.__left = fargs[0]
+                    self.__right = table.pool.get(field._obj).search(cr, uid, [(fargs[1], self.__operator, self.__right)], context=context)
+                    self.__operator = 'in'
+                return self
+            
+            field_obj = table.pool.get(field._obj)
+            if field._properties:
+                # this is a function field
+                if not field._fnct_search and not field.store:
+                    # the function field doesn't provide a search function and doesn't store values in the database, so we must ignore it : we generate a dummy leaf
+                    self.__left, self__operator, self.__right = 1, '=', 1
+                    self.__exp = '' # force to generate an empty sql expression
+                else:
+                    # we need to replace this leaf to a '&' expression
+                    # we clone ourself...
+                    import copy
+                    newexp = copy.copy(self)
+                    self.__table = None
+                    self.__tables, self.__joins = [], []
+                    self.__children = []
+                    
+                    if field._fnct_search:
+                        subexp = field.search(cr, uid, table, self.__left, [self.__exp])
+                        self.__children.append(expression(subexp).parse(cr, uid, table, context))
+                    if field.store:
+                        self.__children.append(newexp)
+
+                    self.__left, self.__right = None, None
+                    self.__operator = '&'
+                    self.__exp = ('&',) + tuple( [tuple(e.__exp) for e in self.__children] )
+
+            elif field._type == 'one2many':
+                if isinstance(self.__right, basestring):
+                    ids2 = [x[0] for x in field_obj.name_search(cr, uid, self.__right, [], self.__operator)]
+                else:
+                    ids2 = self.__right
+                if not ids2:
+                    self.__left, self.__operator, self.__right = 'id', '=', '0'
+                else:
+                    self.__left, self.__operator, self.__right = 'id', 'in', self.__execute_recursive_in(cr, field._fields_id, field_obj._table, 'id', ids2)
+
+            elif field._type == 'many2many':
+                #FIXME
+                if self.__operator == 'child_of':
+                    if isinstance(self.__right, basestring):
+                        ids2 = [x[0] for x in field_obj.name_search(cr, uid, self.__right, [], 'like')]
+                    else:
+                        ids2 = self.__right
+                   
+                    def _rec_convert(ids):
+                        if field_obj == table:
+                            return ids
+                        return self.__execute_recursive_in(cr, field._id1, field._rel, field._id2, ids)
+                    
+                    self.__left, self.__operator, self.__right = 'id', 'in', _rec_convert(ids2 + _rec_get(ids2, field_obj, self.__table._parent_name))
+                else:
+                    if isinstance(self.__right, basestring):
+                        res_ids = [x[0] for x in field_obj.name_search(cr, uid, self.__right, [], self.__operator)]
+                    else:
+                        res_ids = self.__right
+                    self.__left, self.__operator, self.__right = 'id', 'in', self.__execute_recursive_in(cr, field._id1, field._rel, field._id2, res_ids) or [0]
+            elif field._type == 'many2one':
+                if self.__operator == 'child_of':
+                    if isinstance(self.__right, basestring):
+                        ids2 = [x[0] for x in field_obj.search_name(cr, uid, self.__right, [], 'like')]
+                    else:
+                        ids2 = list(self.__right)
+                        
+                    self.__operator = 'in'
+                    if field._obj <> self.__table._name:
+                        self.__right = ids2 + _rec_get(ids2, field_obj, self.__table._parent_name)
+                    else:
+                        self.__right = ids2 + _rec_get(ids2, self.__table, self.__left)
+                        self.__left = 'id'
+                else:
+                    if isinstance(self.__right, basestring):
+                        res_ids = field_obj.name_search(cr, uid, self.__right, [], self.__operator)
+                        self.__operator = 'in'
+                        self.__right = map(lambda x: x[0], res_ids)
+            else: 
+                # other field type
+                if field.translate:
+                    if self.__operator in ('like', 'ilike', 'not like', 'not ilike'):
+                        self.__right = '%%%s%%' % self.__right
+
+                    query1 = '( SELECT res_id'          \
+                             '    FROM ir_translation'  \
+                             '   WHERE name = %s'       \
+                             '     AND lang = %s'       \
+                             '     AND type = %s'       \
+                             '     AND value ' + self.__operator + ' %s'    \
+                             ') UNION ('                \
+                             '  SELECT id'              \
+                             '    FROM "' + self.__table._table + '"'       \
+                             '   WHERE "' + self.__left + '" ' + self.__operator + ' %s' \
+                             ')'
+                    query2 = [self.__table._name + ',' + self.__left,
+                              context.get('lang', False) or 'en_US',
+                              'model',
+                              self.__right,
+                              self.__right,
+                             ]
+
+                    self.__left = 'id'
+                    self.__operator = 'inselect'
+                    self.__right = (query1, query2,)
+
+
+        elif self._is_expression(self.__exp):
+            self.__operator = self.__exp[0]
+
+            for element in self.__exp[1:]:
+                if not self._is_operator(element):
+                    self.__children.append(expression(element).parse(cr, uid, table, context))
+        return self
+
+    def to_sql(self):
+        if not self.__exp:
+            return ('', [])
+        elif self._is_leaf(self.__exp):
+            if self.__operator == 'inselect':
+                query = '(%s.%s in (%s))' % (self.__table._table, self.__left, self.__right[0])
+                params = self.__right[1]
+            elif self.__operator in ['in', 'not in']:
+                params = self.__right[:]
+                len_before = len(params)
+                for i in range(len_before)[::-1]:
+                    if params[i] == False:
+                        del params[i]
+
+                len_after = len(params)
+                check_nulls = len_after <> len_before
+                query = '(1=0)'
+                
+                if len_after:
+                    if self.__left == 'id':
+                        instr = ','.join(['%d'] * len_after)
+                    else:
+                        instr = ','.join([self.__table._columns[self.__left]._symbol_set[0]] * len_after)
+
+                    query = '(%s.%s %s (%s))' % (self.__table._table, self.__left, self.__operator, instr)
+
+                if check_nulls:
+                    query = '(%s OR %s IS NULL)' % (query, self.__left)
+            else:
+                params = []
+                if self.__right is False and self.__operator == '=':
+                    query = '%s IS NULL' % self.__left
+                elif self.__right is False and self.__operator == '<>':
+                    query = '%s IS NOT NULL' % self.__left
+                else:
+                    if self.__left == 'id':
+                        query = '%s.id %s %%s' % (self.__table._table, self.__operator)
+                        params = self.__right
+                    else:
+                        like = self.__operator in ('like', 'ilike', 'not like', 'not ilike')
+
+                        op = self.__operator == '=like' and 'like' or self.__operator
+                        if self.__left in self.__table._columns:
+                            format = like and '%s' or self.__table._columns[self.__left]._symbol_set[0]
+                            query = '(%s.%s %s %s)' % (self.__table._table, self.__left, op, format)
+                        else:
+                            query = "(%s.%s %s '%s')" % (self.__table._table, self.__left, op, self.__right)
+ 
+                        add_null = False
+                        if like:
+                            if isinstance(self.__right, str):
+                                str_utf8 = self.__right
+                            elif isinstance(self.__right, unicode):
+                                str_utf8 = self.__right.encode('utf-8')
+                            else:
+                                str_utf8 = str(self.__right)
+                            params = '%%%s%%' % str_utf8
+                            add_null = not str_utf8
+                        elif self.__left in self.__table._columns:
+                            params = self.__table._columns[self.__left]._symbol_set[1](self.__right)
+
+                        if add_null:
+                            query = '(%s OR %s IS NULL)' % (query, self.__left)
+
+            joins = ' AND '.join(map(lambda j: '%s.id = %s' % (self.__table._table, j), self.__joins))
+            if joins:
+                query = '(%s AND (%s))' % (joins, query)
+            if isinstance(params, basestring):
+                params = [params]
+            #print 'SQL:', repr(self.__iexp), '->', query
+            return (query, params)
+
+        else:
+            children = [child.to_sql() for child in self.__children]
+            params = flatten([child[1] for child in children])
+            query = "( %s )" % (" %s " % {'&' : 'AND', '|' : 'OR' }[self.__operator]).join([child[0] for child in children if child[0]])
+            return (query, params)
+
+    def __get_tables(self):
+        return self.__tables + [child.__get_tables() for child in self.__children]
+    
+    def get_tables(self):
+        return [ '"%s"' % t for t in set(flatten(self.__get_tables()))]
+
+    #def 
+
+if __name__ == '__main__':
+    pass
+    #import doctest
+    #doctest.testmod()
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
+

diff --git a/bin/osv/orm.py b/bin/osv/orm.py
index 9c03a23..9173ecf 100644 (file)
--- a/bin/osv/orm.py
+++ b/bin/osv/orm.py
@@ -2187,10 +2187,23 @@ class orm(orm_template):
         # if the object has a field named 'active', filter out all inactive
         # records unless they were explicitely asked for
         if 'active' in self._columns and (active_test and context.get('active_test', True)):
-            active_found = reduce( lambda x, y: x or y == 'active', args, False )
-            if not active_found:
-                args.append(('active', '=', 1))
+            args = [('&', ('active', '=', 1), tuple(args))]
+            #active_found = reduce( lambda x, y: x or y == 'active', args, False )
+            #if not active_found:
+            #    args.append(('active', '=', 1))
+
+        if args:
+            import expression
+            e = expression.expression(args)
+            e.parse(cr, user, self, context)
+            tables = e.get_tables()
+            qu1, qu2 = e.to_sql()
+            qu1 = qu1 and [qu1] or []
+        else:
+            qu1, qu2, tables = [], [], ['"%s"' % self._table]
 
+        print "expression: %r\n%r\n%r\n%r" % (args, qu1, qu2, tables)
+        """
         tables=['"'+self._table+'"']
         joins=[]
         for i, argument in zip(range(len(args)), args):
@@ -2220,7 +2233,7 @@ class orm(orm_template):
                 if field._type == 'many2one':
                     args[i] = (fargs[0], 'in', self.pool.get(field._obj).search(cr, user, [(fargs[1], argument[1], argument[2])], context=context))
                 continue
-            if field._properties:
+            if field._properties: # = fields function
                 arg = [args.pop(i)]
                 j = i
                 while j<len(args):
@@ -2356,7 +2369,7 @@ class orm(orm_template):
 #FIXME: this replace all (..., '=', False) values with 'is null' and this is
 # not what we want for real boolean fields. The problem is, we can't change it
 # easily because we use False everywhere instead of None
-# NOTE FAB: we can't use None becStay tunes ! ause it is not accepted by XML-RPC, that's why
+# NOTE FAB: we can't use None because it is not accepted by XML-RPC, that's why
 # boolean (0-1), None -> False
 # Ged> boolean fields are not always = 0 or 1
                 if (x[2] is False) and (x[1]=='='):
@@ -2421,6 +2434,7 @@ class orm(orm_template):
                     qu2+=x[2]
                 else:
                     qu1.append(' (1=0)')
+"""
         return (qu1,qu2,tables)
 
     def _check_qorder(self, word):

diff --git a/bin/sql_db.py b/bin/sql_db.py
index 98cd44f..9d3f83e 100644 (file)
--- a/bin/sql_db.py
+++ b/bin/sql_db.py
@@ -69,13 +69,13 @@ class fake_cursor:
             sql = sql.encode('utf-8')
         if self.sql_log:
             now = mdt.now()
+            print "SQL LOG query:", sql
+            print "SQL LOG params:", repr(p)
         if p:
             res = self.obj.execute(sql, p)
         else:
             res = self.obj.execute(sql)
         if self.sql_log:
-            print "SQL LOG query:", sql
-            print "SQL LOG params:", repr(p)
             self.count+=1
             res_from = re_from.match(sql.lower())
             if res_from:

diff --git a/bin/tools/expression.py b/bin/tools/expression.py
deleted file mode 100644 (file)
index 92e3590..0000000
--- a/bin/tools/expression.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-
-class expression(object):
-    """
-    parse a domain expression
-    examples:
-
-    >>> e = [('foo', '=', 'bar')]
-    >>> expression(e).parse().to_sql()
-    'foo = bar'
-    >>> e = [('id', 'in', [1,2,3])]
-    >>> expression(e).parse().to_sql()
-    'id in (1, 2, 3)'
-    >>> e = [('field', '=', 'value'), ('field', '<>', 'value')]
-    >>> expression(e).parse().to_sql()
-    '( field = value AND field <> value )'
-    >>> e = [('&', ('field', '<', 'value'), ('field', '>', 'value'))]
-    >>> expression(e).parse().to_sql()
-    '( field < value AND field > value )'
-    >>> e = [('|', ('field', '=', 'value'), ('field', '=', 'value'))]
-    >>> expression(e).parse().to_sql()
-    '( field = value OR field = value )'
-    >>> e = [('&', ('field1', '=', 'value'), ('field2', '=', 'value'), ('|', ('field3', '<>', 'value'), ('field4', '=', 'value')))]
-    >>> expression(e).parse().to_sql()
-    '( field1 = value AND field2 = value AND ( field3 <> value OR field4 = value ) )'
-    >>> e = [('&', ('|', ('a', '=', '1'), ('b', '=', '2')), ('|', ('c', '=', '3'), ('d', '=', '4')))]
-    >>> expression(e).parse().to_sql()
-    '( ( a = 1 OR b = 2 ) AND ( c = 3 OR d = 4 ) )'
-    >>> e = [('|', (('a', '=', '1'), ('b', '=', '2')), (('c', '=', '3'), ('d', '=', '4')))]
-    >>> expression(e).parse().to_sql()
-    '( ( a = 1 AND b = 2 ) OR ( c = 3 AND d = 4 ) )'
-    >>> expression('fail').parse().to_sql()
-    Traceback (most recent call last):
-    ...
-    ValueError: Bad expression: 'fail'
-    >>> e = [('fail', 'is', 'True')]
-    >>> expression(e).parse().to_sql()
-    Traceback (most recent call last):
-    ...
-    ValueError: Bad expression: ('&', ('fail', 'is', 'True'))
-    """
-
-    def _is_operator(self, element):
-        return isinstance(element, str) \
-           and element in ['&','|']
-
-    def _is_leaf(self, element):
-        return isinstance(element, tuple) \
-           and len(element) == 3 \
-           and element[1] in ('=', '<>', '<=', '<', '>', '>=', 'like', 'not like', 'ilike', 'not ilike', 'in', 'not in', 'child_of') 
-
-    def _is_expression(self, element):
-        return isinstance(element, tuple) \
-           and len(element) > 2 \
-           and self._is_operator(element[0])
-
-    def __init__(self, exp):
-        if isinstance(exp, tuple):
-            if not self._is_leaf(exp) and not self._is_operator(exp[0]):
-                exp = list(exp)
-        if isinstance(exp, list):
-            if len(exp) == 1 and self._is_leaf(exp[0]):
-                exp = exp[0]
-            else:
-                if not self._is_operator(exp[0][0]):
-                    exp.insert(0, '&')
-                    exp = tuple(exp)
-                else:
-                    exp = exp[0]
-
-        self.exp = exp
-        self.operator = '&'
-        self.children = []
-
-        self.left, self.right = None, None
-        if self._is_leaf(self.exp):
-            self.left, self.operator, self.right = self.exp
-            if isinstance(self.right, list):
-                self.right = tuple(self.right)
-        elif not self._is_expression(self.exp):
-            raise ValueError, 'Bad expression: %r' % (self.exp,)
-
-    def parse(self):
-        if self._is_leaf(self.exp):
-            pass
-
-        elif self._is_expression(self.exp):
-            self.operator = self.exp[0]
-
-            for element in self.exp[1:]:
-                if not self._is_operator(element):
-                    self.children.append(expression(element).parse())
-        return self
-
-    def to_sql(self):
-        if self._is_leaf(self.exp):
-            return "%s %s %s" % (self.left, self.operator, self.right)
-        else:
-            return "( %s )" % (" %s " % {'&' : 'AND', '|' : 'OR' }[self.operator]).join([child.to_sql() for child in self.children])
-
-
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod()
-
-# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
-

diff --git a/bin/tools/misc.py b/bin/tools/misc.py
index 4d8966f..71e0f96 100644 (file)
--- a/bin/tools/misc.py
+++ b/bin/tools/misc.py
@@ -35,7 +35,7 @@ import os, time, sys
 import inspect
 
 import psycopg
-import netsvc
+#import netsvc
 from config import config
 #import tools