X-Git-Url: http://git.inspyration.org/?a=blobdiff_plain;f=bin%2Ftools%2Fmisc.py;h=ff20966857eb0c983dfc6be082ed7196dc17a91b;hb=ed2c762bc7b209679fd1c9a3691965127df194b8;hp=af3b74aa95d817f0345bb5d61346f6d983176a01;hpb=ca6e1c3451e3a2aed75f85039834d84dbbb749d3;p=odoo%2Fodoo.git diff --git a/bin/tools/misc.py b/bin/tools/misc.py index af3b74a..ff20966 100644 --- a/bin/tools/misc.py +++ b/bin/tools/misc.py @@ -1,21 +1,20 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- ############################################################################## # # OpenERP, Open Source Management Solution -# Copyright (C) 2004-2009 Tiny SPRL (). All Rights Reserved -# $Id$ +# Copyright (C) 2004-2009 Tiny SPRL (). # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License +# You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # ############################################################################## @@ -32,6 +31,8 @@ from config import config import zipfile import release import socket +import re +from itertools import islice if sys.version_info[:2] < (2, 4): from threadinglocal import local @@ -50,63 +51,63 @@ def init_db(cr): cr.commit() for i in addons.get_modules(): - terp_file = addons.get_module_resource(i, '__terp__.py') mod_path = addons.get_module_path(i) if not mod_path: continue - info = False - if os.path.isfile(terp_file) or os.path.isfile(mod_path+'.zip'): - info = eval(file_open(terp_file).read()) - if info: - categs = info.get('category', 'Uncategorized').split('/') - p_id = None - while categs: - if p_id is not None: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id=%s', (categs[0], p_id)) - else: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id is NULL', (categs[0],)) - c_id = cr.fetchone() - if not c_id: - cr.execute('select nextval(\'ir_module_category_id_seq\')') - c_id = cr.fetchone()[0] - cr.execute('insert into ir_module_category \ - (id, name, parent_id) \ - values (%s, %s, %s)', (c_id, categs[0], p_id)) - else: - c_id = c_id[0] - p_id = c_id - categs = categs[1:] - - active = info.get('active', False) - installable = info.get('installable', True) - if installable: - if active: - state = 'to install' - else: - state = 'uninstalled' + + info = addons.load_information_from_description_file(i) + + if not info: + continue + categs = info.get('category', 'Uncategorized').split('/') + p_id = None + while categs: + if p_id is not None: + cr.execute('select id \ + from ir_module_category \ + where name=%s and parent_id=%s', (categs[0], p_id)) else: - state = 'uninstallable' - cr.execute('select nextval(\'ir_module_module_id_seq\')') - id = cr.fetchone()[0] - cr.execute('insert into ir_module_module \ - (id, author, website, name, shortdesc, description, \ - category_id, state, certificate) \ - values (%s, %s, %s, %s, %s, %s, %s, %s, %s)', ( - id, info.get('author', ''), - info.get('website', ''), i, info.get('name', False), - info.get('description', ''), p_id, state, info.get('certificate'))) - cr.execute('insert into ir_model_data \ - (name,model,module, res_id) values (%s,%s,%s,%s)', ( - 'module_meta_information', 'ir.module.module', i, id)) - dependencies = info.get('depends', []) - for d in dependencies: - cr.execute('insert into ir_module_module_dependency \ - (module_id,name) values (%s, %s)', (id, d)) - cr.commit() + cr.execute('select id \ + from ir_module_category \ + where name=%s and parent_id is NULL', (categs[0],)) + c_id = cr.fetchone() + if not c_id: + cr.execute('select nextval(\'ir_module_category_id_seq\')') + c_id = cr.fetchone()[0] + cr.execute('insert into ir_module_category \ + (id, name, parent_id) \ + values (%s, %s, %s)', (c_id, categs[0], p_id)) + else: + c_id = c_id[0] + p_id = c_id + categs = categs[1:] + + active = info.get('active', False) + installable = info.get('installable', True) + if installable: + if active: + state = 'to install' + else: + state = 'uninstalled' + else: + state = 'uninstallable' + cr.execute('select nextval(\'ir_module_module_id_seq\')') + id = cr.fetchone()[0] + cr.execute('insert into ir_module_module \ + (id, author, website, name, shortdesc, description, \ + category_id, state, certificate) \ + values (%s, %s, %s, %s, %s, %s, %s, %s, %s)', ( + id, info.get('author', ''), + info.get('website', ''), i, info.get('name', False), + info.get('description', ''), p_id, state, info.get('certificate'))) + cr.execute('insert into ir_model_data \ + (name,model,module, res_id, noupdate) values (%s,%s,%s,%s,%s)', ( + 'module_meta_information', 'ir.module.module', i, id, True)) + dependencies = info.get('depends', []) + for d in dependencies: + cr.execute('insert into ir_module_module_dependency \ + (module_id,name) values (%s, %s)', (id, d)) + cr.commit() def find_in_path(name): if os.name == "nt": @@ -174,8 +175,8 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): @return: fileobject if pathinfo is False else (fileobject, filepath) """ - - adp = os.path.normcase(os.path.abspath(config['addons_path'])) + import addons + adps = addons.ad_paths rtp = os.path.normcase(os.path.abspath(config['root_path'])) if name.replace(os.path.sep, '/').startswith('addons/'): @@ -190,7 +191,8 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): subdir2 = (subdir2 != 'addons' or None) and subdir2 - try: + for adp in adps: + try: if subdir2: fn = os.path.join(adp, subdir2, name) else: @@ -200,7 +202,7 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): if pathinfo: return fo, fn return fo - except IOError, e: + except IOError, e: pass if subdir: @@ -302,9 +304,114 @@ def reverse_enumerate(l): #---------------------------------------------------------- # Emails #---------------------------------------------------------- +email_re = re.compile(r""" + ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part + @ # mandatory @ sign + [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then? + \. + [a-z]{2,3} # TLD + ) + """, re.VERBOSE) +res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) +command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) +reference_re = re.compile("<.*-openobject-(\\d+)@(.*)>", re.UNICODE) + +priorities = { + '1': '1 (Highest)', + '2': '2 (High)', + '3': '3 (Normal)', + '4': '4 (Low)', + '5': '5 (Lowest)', + } + +def html2plaintext(html, body_id=None, encoding='utf-8'): + ## (c) Fry-IT, www.fry-it.com, 2007 + ## + ## download here: http://www.peterbe.com/plog/html2plaintext + + + """ from an HTML text, convert the HTML to plain text. + If @body_id is provided then this is the tag where the + body (not necessarily ) starts. + """ + try: + from BeautifulSoup import BeautifulSoup, SoupStrainer, Comment + except: + return html + + urls = [] + if body_id is not None: + strainer = SoupStrainer(id=body_id) + else: + strainer = SoupStrainer('body') + + soup = BeautifulSoup(html, parseOnlyThese=strainer, fromEncoding=encoding) + for link in soup.findAll('a'): + title = link.renderContents() + for url in [x[1] for x in link.attrs if x[0]=='href']: + urls.append(dict(url=url, tag=str(link), title=title)) + + html = soup.__str__() + + url_index = [] + i = 0 + for d in urls: + if d['title'] == d['url'] or 'http://'+d['title'] == d['url']: + html = html.replace(d['tag'], d['url']) + else: + i += 1 + html = html.replace(d['tag'], '%s [%s]' % (d['title'], i)) + url_index.append(d['url']) + + html = html.replace('','*').replace('','*') + html = html.replace('','*').replace('','*') + html = html.replace('

','*').replace('

','*') + html = html.replace('

','**').replace('

','**') + html = html.replace('

','**').replace('

','**') + html = html.replace('','/').replace('','/') + + + # the only line breaks we respect is those of ending tags and + # breaks + + html = html.replace('\n',' ') + html = html.replace('
', '\n') + html = html.replace('', '\n') + html = html.replace('

', '\n\n') + html = re.sub('', '\n', html) + html = html.replace(' ' * 2, ' ') + + + # for all other tags we failed to clean up, just remove then and + # complain about them on the stderr + def desperate_fixer(g): + #print >>sys.stderr, "failed to clean up %s" % str(g.group()) + return ' ' + + html = re.sub('<.*?>', desperate_fixer, html) + + # lstrip all lines + html = '\n'.join([x.lstrip() for x in html.splitlines()]) + + for i, url in enumerate(url_index): + if i == 0: + html += '\n\n' + html += '[%s] %s\n' % (i+1, url) + return html + def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False, - attach=None, tinycrm=False, ssl=False, debug=False, subtype='plain', x_headers=None): - """Send an email.""" + attach=None, openobject_id=False, ssl=False, debug=False, subtype='plain', x_headers=None, priority='3'): + + """Send an email. + + Arguments: + + `email_from`: A string used to fill the `From` header, if falsy, + config['email_from'] is used instead. Also used for + the `Reply-To` header if `reply_to` is not provided + + `email_to`: a sequence of addresses to send the mail to. + """ import smtplib from email.MIMEText import MIMEText from email.MIMEBase import MIMEBase @@ -313,25 +420,33 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non from email.Utils import formatdate, COMMASPACE from email.Utils import formatdate, COMMASPACE from email import Encoders + import netsvc if x_headers is None: x_headers = {} - if not ssl: - ssl = config.get('smtp_ssl', False) + if not ssl: ssl = config.get('smtp_ssl', False) - if not email_from and not config['email_from']: - raise Exception("No Email sender by default, see config file") + if not (email_from or config['email_from']): + raise ValueError("Sending an email requires either providing a sender " + "address or having configured one") - if not email_cc: - email_cc = [] - if not email_bcc: - email_bcc = [] + if not email_from: email_from = config.get('email_from', False) - if not attach: - msg = MIMEText(body or '',_subtype=subtype,_charset='utf-8') - else: - msg = MIMEMultipart() + if not email_cc: email_cc = [] + if not email_bcc: email_bcc = [] + if not body: body = u'' + try: email_body = body.encode('utf-8') + except (UnicodeEncodeError, UnicodeDecodeError): + email_body = body + + try: + email_text = MIMEText(email_body.encode('utf8') or '',_subtype=subtype,_charset='utf-8') + except: + email_text = MIMEText(email_body or '',_subtype=subtype,_charset='utf-8') + + if attach: msg = MIMEMultipart() + else: msg = email_text msg['Subject'] = Header(ustr(subject), 'utf-8') msg['From'] = email_from @@ -352,45 +467,73 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non msg['X-OpenERP-Server-Host'] = socket.gethostname() msg['X-OpenERP-Server-Version'] = release.version + msg['X-Priority'] = priorities.get(priority, '3 (Normal)') + # Add dynamic X Header - for key, value in x_headers.items(): + for key, value in x_headers.iteritems(): msg['X-OpenERP-%s' % key] = str(value) - if tinycrm: - msg['Message-Id'] = "<%s-tinycrm-%s@%s>" % (time.time(), tinycrm, socket.gethostname()) + if openobject_id: + msg['Message-Id'] = "<%s-openobject-%s@%s>" % (time.time(), openobject_id, socket.gethostname()) if attach: - msg.attach( MIMEText(body or '', _charset='utf-8', _subtype=subtype) ) - + msg.attach(email_text) for (fname,fcontent) in attach: part = MIMEBase('application', "octet-stream") part.set_payload( fcontent ) Encoders.encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename="%s"' % (fname,)) msg.attach(part) + + class WriteToLogger(object): + def __init__(self): + self.logger = netsvc.Logger() + + def write(self, s): + self.logger.notifyChannel('email_send', netsvc.LOG_DEBUG, s) + + smtp_server = config['smtp_server'] + if smtp_server.startswith('maildir:/'): + from mailbox import Maildir + maildir_path = smtp_server[8:] + try: + mdir = Maildir(maildir_path,factory=None, create = True) + mdir.add(msg.as_string(True)) + return True + except Exception,e: + netsvc.Logger().notifyChannel('email_send (maildir)', netsvc.LOG_ERROR, e) + return False + try: + oldstderr = smtplib.stderr s = smtplib.SMTP() + try: + # in case of debug, the messages are printed to stderr. + if debug: + smtplib.stderr = WriteToLogger() + + s.set_debuglevel(int(bool(debug))) # 0 or 1 + s.connect(smtp_server, config['smtp_port']) + if ssl: + s.ehlo() + s.starttls() + s.ehlo() + + if config['smtp_user'] or config['smtp_password']: + s.login(config['smtp_user'], config['smtp_password']) + s.sendmail(email_from, + flatten([email_to, email_cc, email_bcc]), + msg.as_string() + ) + finally: + s.quit() + if debug: + smtplib.stderr = oldstderr - if debug: - s.debuglevel = 5 - s.connect(config['smtp_server'], config['smtp_port']) - if ssl: - s.ehlo() - s.starttls() - s.ehlo() - - if config['smtp_user'] or config['smtp_password']: - s.login(config['smtp_user'], config['smtp_password']) - - s.sendmail(email_from, - flatten([email_to, email_cc, email_bcc]), - msg.as_string() - ) - s.quit() except Exception, e: - import netsvc netsvc.Logger().notifyChannel('email_send', netsvc.LOG_ERROR, e) return False + return True #---------------------------------------------------------- @@ -501,9 +644,6 @@ class UpdateableDict(local): def __ge__(self, y): return self.dict.__ge__(y) - def __getitem__(self, y): - return self.dict.__getitem__(y) - def __gt__(self, y): return self.dict.__gt__(y) @@ -555,9 +695,9 @@ class cache(object): Use it as a decorator of the function you plan to cache Timeout: 0 = no timeout, otherwise in seconds """ - + __caches = [] - + def __init__(self, timeout=None, skiparg=2, multi=None): assert skiparg >= 2 # at least self and cr if timeout is None: @@ -568,56 +708,57 @@ class cache(object): self.multi = multi self.lasttime = time.time() self.cache = {} - self.fun = None + self.fun = None cache.__caches.append(self) - + def _generate_keys(self, dbname, kwargs2): """ Generate keys depending of the arguments and the self.mutli value """ - + def to_tuple(d): - i = d.items() - i.sort(key=lambda (x,y): x) - return tuple(i) + pairs = d.items() + pairs.sort(key=lambda (k,v): k) + for i, (k, v) in enumerate(pairs): + if isinstance(v, dict): + pairs[i] = (k, to_tuple(v)) + if isinstance(v, (list, set)): + pairs[i] = (k, tuple(v)) + elif not is_hashable(v): + pairs[i] = (k, repr(v)) + return tuple(pairs) if not self.multi: key = (('dbname', dbname),) + to_tuple(kwargs2) yield key, None else: - multis = kwargs2[self.multi][:] + multis = kwargs2[self.multi][:] for id in multis: kwargs2[self.multi] = (id,) key = (('dbname', dbname),) + to_tuple(kwargs2) yield key, id - + def _unify_args(self, *args, **kwargs): # Update named arguments with positional argument values (without self and cr) kwargs2 = self.fun_default_values.copy() kwargs2.update(kwargs) kwargs2.update(dict(zip(self.fun_arg_names, args[self.skiparg-2:]))) - for k in kwargs2: - if isinstance(kwargs2[k], (list, dict, set)): - kwargs2[k] = tuple(kwargs2[k]) - elif not is_hashable(kwargs2[k]): - kwargs2[k] = repr(kwargs2[k]) - return kwargs2 - + def clear(self, dbname, *args, **kwargs): """clear the cache for database dbname if *args and **kwargs are both empty, clear all the keys related to this database """ if not args and not kwargs: - keys_to_del = [key for key in self.cache if key[0][1] == dbname] + keys_to_del = [key for key in self.cache.keys() if key[0][1] == dbname] else: kwargs2 = self._unify_args(*args, **kwargs) - keys_to_del = [key for key, _ in self._generate_keys(dbname, kwargs2) if key in self.cache] - + keys_to_del = [key for key, _ in self._generate_keys(dbname, kwargs2) if key in self.cache.keys()] + for key in keys_to_del: - del self.cache[key] - + self.cache.pop(key) + @classmethod def clean_caches_for_db(cls, dbname): for c in cls.__caches: @@ -633,14 +774,14 @@ class cache(object): self.fun_default_values = {} if argspec[3]: self.fun_default_values = dict(zip(self.fun_arg_names[-len(argspec[3]):], argspec[3])) - + def cached_result(self2, cr, *args, **kwargs): - if time.time()-self.timeout > self.lasttime: + if time.time()-int(self.timeout) > self.lasttime: self.lasttime = time.time() - t = time.time()-self.timeout - for key in self.cache.keys(): - if self.cache[key][1]>> func_foo(42) This will output on the logger: - + [Wed Dec 25 00:00:00 2008] DEBUG:func_foo:baz = 42 [Wed Dec 25 00:00:00 2008] DEBUG:func_foo:qnx = (42, 42) @@ -964,15 +1136,154 @@ def extract_zip_file(zip_file, outdirectory): fp.close() zf.close() +def detect_ip_addr(): + def _detect_ip_addr(): + from array import array + import socket + from struct import pack, unpack + try: + import fcntl + except ImportError: + fcntl = None + + ip_addr = None + + if not fcntl: # not UNIX: + host = socket.gethostname() + ip_addr = socket.gethostbyname(host) + else: # UNIX: + # get all interfaces: + nbytes = 128 * 32 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + names = array('B', '\0' * nbytes) + #print 'names: ', names + outbytes = unpack('iL', fcntl.ioctl( s.fileno(), 0x8912, pack('iL', nbytes, names.buffer_info()[0])))[0] + namestr = names.tostring() + + # try 64 bit kernel: + for i in range(0, outbytes, 40): + name = namestr[i:i+16].split('\0', 1)[0] + if name != 'lo': + ip_addr = socket.inet_ntoa(namestr[i+20:i+24]) + break + + # try 32 bit kernel: + if ip_addr is None: + ifaces = filter(None, [namestr[i:i+32].split('\0', 1)[0] for i in range(0, outbytes, 32)]) + + for ifname in [iface for iface in ifaces if iface != 'lo']: + ip_addr = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, pack('256s', ifname[:15]))[20:24]) + break + + return ip_addr or 'localhost' + try: + ip_addr = _detect_ip_addr() + except: + ip_addr = 'localhost' + return ip_addr + +# RATIONALE BEHIND TIMESTAMP CALCULATIONS AND TIMEZONE MANAGEMENT: +# The server side never does any timestamp calculation, always +# sends them in a naive (timezone agnostic) format supposed to be +# expressed within the server timezone, and expects the clients to +# provide timestamps in the server timezone as well. +# It stores all timestamps in the database in naive format as well, +# which also expresses the time in the server timezone. +# For this reason the server makes its timezone name available via the +# common/timezone_get() rpc method, which clients need to read +# to know the appropriate time offset to use when reading/writing +# times. +def get_win32_timezone(): + """Attempt to return the "standard name" of the current timezone on a win32 system. + @return: the standard name of the current win32 timezone, or False if it cannot be found. + """ + res = False + if (sys.platform == "win32"): + try: + import _winreg + hklm = _winreg.ConnectRegistry(None,_winreg.HKEY_LOCAL_MACHINE) + current_tz_key = _winreg.OpenKey(hklm, r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation", 0,_winreg.KEY_ALL_ACCESS) + res = str(_winreg.QueryValueEx(current_tz_key,"StandardName")[0]) # [0] is value, [1] is type code + _winreg.CloseKey(current_tz_key) + _winreg.CloseKey(hklm) + except: + pass + return res +def detect_server_timezone(): + """Attempt to detect the timezone to use on the server side. + Defaults to UTC if no working timezone can be found. + @return: the timezone identifier as expected by pytz.timezone. + """ + import time + import netsvc + try: + import pytz + except: + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "Python pytz module is not available. Timezone will be set to UTC by default.") + return 'UTC' + + # Option 1: the configuration option (did not exist before, so no backwards compatibility issue) + # Option 2: to be backwards compatible with 5.0 or earlier, the value from time.tzname[0], but only if it is known to pytz + # Option 3: the environment variable TZ + sources = [ (config['timezone'], 'OpenERP configuration'), + (time.tzname[0], 'time.tzname'), + (os.environ.get('TZ',False),'TZ environment variable'), ] + # Option 4: OS-specific: /etc/timezone on Unix + if (os.path.exists("/etc/timezone")): + tz_value = False + try: + f = open("/etc/timezone") + tz_value = f.read(128).strip() + except: + pass + finally: + f.close() + sources.append((tz_value,"/etc/timezone file")) + # Option 5: timezone info from registry on Win32 + if (sys.platform == "win32"): + # Timezone info is stored in windows registry. + # However this is not likely to work very well as the standard name + # of timezones in windows is rarely something that is known to pytz. + # But that's ok, it is always possible to use a config option to set + # it explicitly. + sources.append((get_win32_timezone(),"Windows Registry")) + + for (value,source) in sources: + if value: + try: + tz = pytz.timezone(value) + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_INFO, + "Using timezone %s obtained from %s." % (tz.zone,source)) + return value + except pytz.UnknownTimeZoneError: + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "The timezone specified in %s (%s) is invalid, ignoring it." % (source,value)) + + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "No valid timezone could be detected, using default UTC timezone. You can specify it explicitly with option 'timezone' in the server configuration.") + return 'UTC' + + +def split_every(n, iterable, piece_maker=tuple): + """Splits an iterable into length-n pieces. The last piece will be shorter + if ``n`` does not evenly divide the iterable length. + @param ``piece_maker``: function to build the pieces + from the slices (tuple,list,...) + """ + iterator = iter(iterable) + piece = piece_maker(islice(iterator, n)) + while piece: + yield piece + piece = piece_maker(islice(iterator, n)) if __name__ == '__main__': import doctest doctest.testmod() - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: