X-Git-Url: http://git.inspyration.org/?a=blobdiff_plain;f=bin%2Ftools%2Fmisc.py;h=ff20966857eb0c983dfc6be082ed7196dc17a91b;hb=ed2c762bc7b209679fd1c9a3691965127df194b8;hp=d1e4f9a3914afb0223022be60661950e700116f8;hpb=e98db0ffcdde175ab609f888e916ab95caf8f2b2;p=odoo%2Fodoo.git diff --git a/bin/tools/misc.py b/bin/tools/misc.py index d1e4f9a..ff20966 100644 --- a/bin/tools/misc.py +++ b/bin/tools/misc.py @@ -31,6 +31,8 @@ from config import config import zipfile import release import socket +import re +from itertools import islice if sys.version_info[:2] < (2, 4): from threadinglocal import local @@ -49,63 +51,63 @@ def init_db(cr): cr.commit() for i in addons.get_modules(): - terp_file = addons.get_module_resource(i, '__terp__.py') mod_path = addons.get_module_path(i) if not mod_path: continue - info = False - if os.path.isfile(terp_file) or os.path.isfile(mod_path+'.zip'): - info = eval(file_open(terp_file).read()) - if info: - categs = info.get('category', 'Uncategorized').split('/') - p_id = None - while categs: - if p_id is not None: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id=%s', (categs[0], p_id)) - else: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id is NULL', (categs[0],)) - c_id = cr.fetchone() - if not c_id: - cr.execute('select nextval(\'ir_module_category_id_seq\')') - c_id = cr.fetchone()[0] - cr.execute('insert into ir_module_category \ - (id, name, parent_id) \ - values (%s, %s, %s)', (c_id, categs[0], p_id)) - else: - c_id = c_id[0] - p_id = c_id - categs = categs[1:] - - active = info.get('active', False) - installable = info.get('installable', True) - if installable: - if active: - state = 'to install' - else: - state = 'uninstalled' + + info = addons.load_information_from_description_file(i) + + if not info: + continue + categs = info.get('category', 'Uncategorized').split('/') + p_id = None + while categs: + if p_id is not None: + cr.execute('select id \ + from ir_module_category \ + where name=%s and parent_id=%s', (categs[0], p_id)) + else: + cr.execute('select id \ + from ir_module_category \ + where name=%s and parent_id is NULL', (categs[0],)) + c_id = cr.fetchone() + if not c_id: + cr.execute('select nextval(\'ir_module_category_id_seq\')') + c_id = cr.fetchone()[0] + cr.execute('insert into ir_module_category \ + (id, name, parent_id) \ + values (%s, %s, %s)', (c_id, categs[0], p_id)) else: - state = 'uninstallable' - cr.execute('select nextval(\'ir_module_module_id_seq\')') - id = cr.fetchone()[0] - cr.execute('insert into ir_module_module \ - (id, author, website, name, shortdesc, description, \ - category_id, state, certificate) \ - values (%s, %s, %s, %s, %s, %s, %s, %s, %s)', ( - id, info.get('author', ''), - info.get('website', ''), i, info.get('name', False), - info.get('description', ''), p_id, state, info.get('certificate'))) - cr.execute('insert into ir_model_data \ - (name,model,module, res_id, noupdate) values (%s,%s,%s,%s,%s)', ( - 'module_meta_information', 'ir.module.module', i, id, True)) - dependencies = info.get('depends', []) - for d in dependencies: - cr.execute('insert into ir_module_module_dependency \ - (module_id,name) values (%s, %s)', (id, d)) - cr.commit() + c_id = c_id[0] + p_id = c_id + categs = categs[1:] + + active = info.get('active', False) + installable = info.get('installable', True) + if installable: + if active: + state = 'to install' + else: + state = 'uninstalled' + else: + state = 'uninstallable' + cr.execute('select nextval(\'ir_module_module_id_seq\')') + id = cr.fetchone()[0] + cr.execute('insert into ir_module_module \ + (id, author, website, name, shortdesc, description, \ + category_id, state, certificate) \ + values (%s, %s, %s, %s, %s, %s, %s, %s, %s)', ( + id, info.get('author', ''), + info.get('website', ''), i, info.get('name', False), + info.get('description', ''), p_id, state, info.get('certificate'))) + cr.execute('insert into ir_model_data \ + (name,model,module, res_id, noupdate) values (%s,%s,%s,%s,%s)', ( + 'module_meta_information', 'ir.module.module', i, id, True)) + dependencies = info.get('depends', []) + for d in dependencies: + cr.execute('insert into ir_module_module_dependency \ + (module_id,name) values (%s, %s)', (id, d)) + cr.commit() def find_in_path(name): if os.name == "nt": @@ -173,8 +175,8 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): @return: fileobject if pathinfo is False else (fileobject, filepath) """ - - adp = os.path.normcase(os.path.abspath(config['addons_path'])) + import addons + adps = addons.ad_paths rtp = os.path.normcase(os.path.abspath(config['root_path'])) if name.replace(os.path.sep, '/').startswith('addons/'): @@ -189,7 +191,8 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): subdir2 = (subdir2 != 'addons' or None) and subdir2 - try: + for adp in adps: + try: if subdir2: fn = os.path.join(adp, subdir2, name) else: @@ -199,7 +202,7 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): if pathinfo: return fo, fn return fo - except IOError, e: + except IOError, e: pass if subdir: @@ -301,10 +304,114 @@ def reverse_enumerate(l): #---------------------------------------------------------- # Emails #---------------------------------------------------------- +email_re = re.compile(r""" + ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part + @ # mandatory @ sign + [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then? + \. + [a-z]{2,3} # TLD + ) + """, re.VERBOSE) +res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) +command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) +reference_re = re.compile("<.*-openobject-(\\d+)@(.*)>", re.UNICODE) + +priorities = { + '1': '1 (Highest)', + '2': '2 (High)', + '3': '3 (Normal)', + '4': '4 (Low)', + '5': '5 (Lowest)', + } + +def html2plaintext(html, body_id=None, encoding='utf-8'): + ## (c) Fry-IT, www.fry-it.com, 2007 + ## + ## download here: http://www.peterbe.com/plog/html2plaintext + + + """ from an HTML text, convert the HTML to plain text. + If @body_id is provided then this is the tag where the + body (not necessarily ) starts. + """ + try: + from BeautifulSoup import BeautifulSoup, SoupStrainer, Comment + except: + return html + + urls = [] + if body_id is not None: + strainer = SoupStrainer(id=body_id) + else: + strainer = SoupStrainer('body') + + soup = BeautifulSoup(html, parseOnlyThese=strainer, fromEncoding=encoding) + for link in soup.findAll('a'): + title = link.renderContents() + for url in [x[1] for x in link.attrs if x[0]=='href']: + urls.append(dict(url=url, tag=str(link), title=title)) + + html = soup.__str__() + + url_index = [] + i = 0 + for d in urls: + if d['title'] == d['url'] or 'http://'+d['title'] == d['url']: + html = html.replace(d['tag'], d['url']) + else: + i += 1 + html = html.replace(d['tag'], '%s [%s]' % (d['title'], i)) + url_index.append(d['url']) + + html = html.replace('','*').replace('','*') + html = html.replace('','*').replace('','*') + html = html.replace('

','*').replace('

','*') + html = html.replace('

','**').replace('

','**') + html = html.replace('

','**').replace('

','**') + html = html.replace('','/').replace('','/') + + + # the only line breaks we respect is those of ending tags and + # breaks + + html = html.replace('\n',' ') + html = html.replace('
', '\n') + html = html.replace('', '\n') + html = html.replace('

', '\n\n') + html = re.sub('', '\n', html) + html = html.replace(' ' * 2, ' ') + + + # for all other tags we failed to clean up, just remove then and + # complain about them on the stderr + def desperate_fixer(g): + #print >>sys.stderr, "failed to clean up %s" % str(g.group()) + return ' ' + + html = re.sub('<.*?>', desperate_fixer, html) + + # lstrip all lines + html = '\n'.join([x.lstrip() for x in html.splitlines()]) + + for i, url in enumerate(url_index): + if i == 0: + html += '\n\n' + html += '[%s] %s\n' % (i+1, url) + return html + def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False, - attach=None, tinycrm=False, ssl=False, debug=False, subtype='plain', x_headers=None): + attach=None, openobject_id=False, ssl=False, debug=False, subtype='plain', x_headers=None, priority='3'): + + """Send an email. - """Send an email.""" + Arguments: + + `email_from`: A string used to fill the `From` header, if falsy, + config['email_from'] is used instead. Also used for + the `Reply-To` header if `reply_to` is not provided + + `email_to`: a sequence of addresses to send the mail to. + """ import smtplib from email.MIMEText import MIMEText from email.MIMEBase import MIMEBase @@ -318,24 +425,28 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non if x_headers is None: x_headers = {} - if not ssl: - ssl = config.get('smtp_ssl', False) + if not ssl: ssl = config.get('smtp_ssl', False) - if not email_from and not config['email_from']: - raise Exception("No Email sender by default, see config file") + if not (email_from or config['email_from']): + raise ValueError("Sending an email requires either providing a sender " + "address or having configured one") - if not email_cc: - email_cc = [] - if not email_bcc: - email_bcc = [] + if not email_from: email_from = config.get('email_from', False) - if not attach: - try: - msg = MIMEText(body.encode('utf8') or '',_subtype=subtype,_charset='utf-8') - except: - msg = MIMEText(body or '',_subtype=subtype,_charset='utf-8') - else: - msg = MIMEMultipart() + if not email_cc: email_cc = [] + if not email_bcc: email_bcc = [] + if not body: body = u'' + try: email_body = body.encode('utf-8') + except (UnicodeEncodeError, UnicodeDecodeError): + email_body = body + + try: + email_text = MIMEText(email_body.encode('utf8') or '',_subtype=subtype,_charset='utf-8') + except: + email_text = MIMEText(email_body or '',_subtype=subtype,_charset='utf-8') + + if attach: msg = MIMEMultipart() + else: msg = email_text msg['Subject'] = Header(ustr(subject), 'utf-8') msg['From'] = email_from @@ -356,18 +467,17 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non msg['X-OpenERP-Server-Host'] = socket.gethostname() msg['X-OpenERP-Server-Version'] = release.version + msg['X-Priority'] = priorities.get(priority, '3 (Normal)') + # Add dynamic X Header - for key, value in x_headers.items(): + for key, value in x_headers.iteritems(): msg['X-OpenERP-%s' % key] = str(value) - if tinycrm: - msg['Message-Id'] = "<%s-tinycrm-%s@%s>" % (time.time(), tinycrm, socket.gethostname()) + if openobject_id: + msg['Message-Id'] = "<%s-openobject-%s@%s>" % (time.time(), openobject_id, socket.gethostname()) if attach: - try: - msg.attach(MIMEText(body.encode('utf8') or '',_subtype=subtype,_charset='utf-8')) - except: - msg.attach(MIMEText(body or '', _charset='utf-8', _subtype=subtype) ) + msg.attach(email_text) for (fname,fcontent) in attach: part = MIMEBase('application', "octet-stream") part.set_payload( fcontent ) @@ -382,18 +492,28 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non def write(self, s): self.logger.notifyChannel('email_send', netsvc.LOG_DEBUG, s) + smtp_server = config['smtp_server'] + if smtp_server.startswith('maildir:/'): + from mailbox import Maildir + maildir_path = smtp_server[8:] + try: + mdir = Maildir(maildir_path,factory=None, create = True) + mdir.add(msg.as_string(True)) + return True + except Exception,e: + netsvc.Logger().notifyChannel('email_send (maildir)', netsvc.LOG_ERROR, e) + return False + try: oldstderr = smtplib.stderr s = smtplib.SMTP() - try: # in case of debug, the messages are printed to stderr. if debug: smtplib.stderr = WriteToLogger() s.set_debuglevel(int(bool(debug))) # 0 or 1 - - s.connect(config['smtp_server'], config['smtp_port']) + s.connect(smtp_server, config['smtp_port']) if ssl: s.ehlo() s.starttls() @@ -401,12 +521,10 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non if config['smtp_user'] or config['smtp_password']: s.login(config['smtp_user'], config['smtp_password']) - s.sendmail(email_from, flatten([email_to, email_cc, email_bcc]), msg.as_string() ) - finally: s.quit() if debug: @@ -526,9 +644,6 @@ class UpdateableDict(local): def __ge__(self, y): return self.dict.__ge__(y) - def __getitem__(self, y): - return self.dict.__getitem__(y) - def __gt__(self, y): return self.dict.__gt__(y) @@ -636,13 +751,13 @@ class cache(object): if *args and **kwargs are both empty, clear all the keys related to this database """ if not args and not kwargs: - keys_to_del = [key for key in self.cache if key[0][1] == dbname] + keys_to_del = [key for key in self.cache.keys() if key[0][1] == dbname] else: kwargs2 = self._unify_args(*args, **kwargs) - keys_to_del = [key for key, _ in self._generate_keys(dbname, kwargs2) if key in self.cache] + keys_to_del = [key for key, _ in self._generate_keys(dbname, kwargs2) if key in self.cache.keys()] for key in keys_to_del: - del self.cache[key] + self.cache.pop(key) @classmethod def clean_caches_for_db(cls, dbname): @@ -664,9 +779,9 @@ class cache(object): if time.time()-int(self.timeout) > self.lasttime: self.lasttime = time.time() t = time.time()-int(self.timeout) - old_keys = [key for key in self.cache if self.cache[key][1] < t] + old_keys = [key for key in self.cache.keys() if self.cache[key][1] < t] for key in old_keys: - del self.cache[key] + self.cache.pop(key) kwargs2 = self._unify_args(*args, **kwargs) @@ -737,14 +852,14 @@ def ustr(value): return unicode(value, getlocale()[1]) def exception_to_unicode(e): - if hasattr(e, 'message'): + if (sys.version_info[:2] < (2,6)) and hasattr(e, 'message'): return ustr(e.message) if hasattr(e, 'args'): return "\n".join((ustr(a) for a in e.args)) try: return ustr(e) except: - return u"Unknow message" + return u"Unknown message" # to be compatible with python 2.4 @@ -769,29 +884,15 @@ if not hasattr(__builtin__, 'any'): __builtin__.any = any del any -get_iso = {'ca_ES':'ca', -'cs_CZ': 'cs', -'et_EE': 'et', -'sv_SE': 'sv', -'sq_AL': 'sq', -'uk_UA': 'uk', -'vi_VN': 'vi', -'af_ZA': 'af', -'be_BY': 'be', -'ja_JP': 'ja', -'ko_KR': 'ko' -} - def get_iso_codes(lang): - if lang in get_iso: - lang = get_iso[lang] - elif lang.find('_') != -1: + if lang.find('_') != -1: if lang.split('_')[0] == lang.split('_')[1].lower(): lang = lang.split('_')[0] return lang def get_languages(): languages={ + 'ab_RU': u'Abkhazian (RU)', 'ar_AR': u'Arabic / الْعَرَبيّة', 'bg_BG': u'Bulgarian / български', 'bs_BS': u'Bosnian / bosanski jezik', @@ -799,36 +900,55 @@ def get_languages(): 'cs_CZ': u'Czech / Čeština', 'da_DK': u'Danish / Dansk', 'de_DE': u'German / Deutsch', - 'el_EL': u'Greek / Ελληνικά', + 'el_GR': u'Greek / Ελληνικά', 'en_CA': u'English (CA)', 'en_GB': u'English (UK)', 'en_US': u'English (US)', 'es_AR': u'Spanish (AR) / Español (AR)', 'es_ES': u'Spanish / Español', 'et_EE': u'Estonian / Eesti keel', + 'fa_IR': u'Persian / Iran, Islamic Republic of', 'fi_FI': u'Finland / Suomi', 'fr_BE': u'French (BE) / Français (BE)', 'fr_CH': u'French (CH) / Français (CH)', 'fr_FR': u'French / Français', + 'gl_ES': u'Galician / Spain', + 'gu_IN': u'Gujarati / India', + 'hi_IN': u'Hindi / India', 'hr_HR': u'Croatian / hrvatski jezik', 'hu_HU': u'Hungarian / Magyar', 'id_ID': u'Indonesian / Bahasa Indonesia', 'it_IT': u'Italian / Italiano', + 'iu_CA': u'Inuktitut / Canada', + 'ja_JP': u'Japanese / Japan', + 'ko_KP': u'Korean / Korea, Democratic Peoples Republic of', + 'ko_KR': u'Korean / Korea, Republic of', 'lt_LT': u'Lithuanian / Lietuvių kalba', + 'lv_LV': u'Latvian / Latvia', + 'ml_IN': u'Malayalam / India', + 'mn_MN': u'Mongolian / Mongolia', + 'nb_NO': u'Norwegian Bokmål / Norway', 'nl_NL': u'Dutch / Nederlands', 'nl_BE': u'Dutch (Belgium) / Nederlands (Belgïe)', + 'oc_FR': u'Occitan (post 1500) / France', 'pl_PL': u'Polish / Język polski', 'pt_BR': u'Portugese (BR) / português (BR)', 'pt_PT': u'Portugese / português', 'ro_RO': u'Romanian / limba română', 'ru_RU': u'Russian / русский язык', + 'si_LK': u'Sinhalese / Sri Lanka', + 'sk_SK': u'Slovak / Slovakia', 'sl_SL': u'Slovenian / slovenščina', 'sq_AL': u'Albanian / Shqipëri', + 'sr_RS': u'Serbian / Serbia', 'sv_SE': u'Swedish / svenska', + 'te_IN': u'Telugu / India', 'tr_TR': u'Turkish / Türkçe', 'vi_VN': u'Vietnam / Cộng hòa xã hội chủ nghĩa Việt Nam', 'uk_UA': u'Ukrainian / украї́нська мо́ва', + 'ur_PK': u'Urdu / Pakistan', 'zh_CN': u'Chinese (CN) / 简体中文', + 'zh_HK': u'Chinese (HK)', 'zh_TW': u'Chinese (TW) / 正體字', 'th_TH': u'Thai / ภาษาไทย', 'tlh_TLH': u'Klingon', @@ -850,11 +970,11 @@ def get_user_companies(cr, user): def _get_company_children(cr, ids): if not ids: return [] - cr.execute('SELECT id FROM res_company WHERE parent_id = any(array[%s])' %(','.join([str(x) for x in ids]),)) + cr.execute('SELECT id FROM res_company WHERE parent_id = ANY (%s)', (ids,)) res=[x[0] for x in cr.fetchall()] res.extend(_get_company_children(cr, res)) return res - cr.execute('SELECT comp.id FROM res_company AS comp, res_users AS u WHERE u.id = %s AND comp.id = u.company_id' % (user,)) + cr.execute('SELECT comp.id FROM res_company AS comp, res_users AS u WHERE u.id = %s AND comp.id = u.company_id', (user,)) compids=[cr.fetchone()[0]] compids.extend(_get_company_children(cr, compids)) return compids @@ -1064,6 +1184,101 @@ def detect_ip_addr(): ip_addr = 'localhost' return ip_addr +# RATIONALE BEHIND TIMESTAMP CALCULATIONS AND TIMEZONE MANAGEMENT: +# The server side never does any timestamp calculation, always +# sends them in a naive (timezone agnostic) format supposed to be +# expressed within the server timezone, and expects the clients to +# provide timestamps in the server timezone as well. +# It stores all timestamps in the database in naive format as well, +# which also expresses the time in the server timezone. +# For this reason the server makes its timezone name available via the +# common/timezone_get() rpc method, which clients need to read +# to know the appropriate time offset to use when reading/writing +# times. +def get_win32_timezone(): + """Attempt to return the "standard name" of the current timezone on a win32 system. + @return: the standard name of the current win32 timezone, or False if it cannot be found. + """ + res = False + if (sys.platform == "win32"): + try: + import _winreg + hklm = _winreg.ConnectRegistry(None,_winreg.HKEY_LOCAL_MACHINE) + current_tz_key = _winreg.OpenKey(hklm, r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation", 0,_winreg.KEY_ALL_ACCESS) + res = str(_winreg.QueryValueEx(current_tz_key,"StandardName")[0]) # [0] is value, [1] is type code + _winreg.CloseKey(current_tz_key) + _winreg.CloseKey(hklm) + except: + pass + return res + +def detect_server_timezone(): + """Attempt to detect the timezone to use on the server side. + Defaults to UTC if no working timezone can be found. + @return: the timezone identifier as expected by pytz.timezone. + """ + import time + import netsvc + try: + import pytz + except: + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "Python pytz module is not available. Timezone will be set to UTC by default.") + return 'UTC' + + # Option 1: the configuration option (did not exist before, so no backwards compatibility issue) + # Option 2: to be backwards compatible with 5.0 or earlier, the value from time.tzname[0], but only if it is known to pytz + # Option 3: the environment variable TZ + sources = [ (config['timezone'], 'OpenERP configuration'), + (time.tzname[0], 'time.tzname'), + (os.environ.get('TZ',False),'TZ environment variable'), ] + # Option 4: OS-specific: /etc/timezone on Unix + if (os.path.exists("/etc/timezone")): + tz_value = False + try: + f = open("/etc/timezone") + tz_value = f.read(128).strip() + except: + pass + finally: + f.close() + sources.append((tz_value,"/etc/timezone file")) + # Option 5: timezone info from registry on Win32 + if (sys.platform == "win32"): + # Timezone info is stored in windows registry. + # However this is not likely to work very well as the standard name + # of timezones in windows is rarely something that is known to pytz. + # But that's ok, it is always possible to use a config option to set + # it explicitly. + sources.append((get_win32_timezone(),"Windows Registry")) + + for (value,source) in sources: + if value: + try: + tz = pytz.timezone(value) + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_INFO, + "Using timezone %s obtained from %s." % (tz.zone,source)) + return value + except pytz.UnknownTimeZoneError: + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "The timezone specified in %s (%s) is invalid, ignoring it." % (source,value)) + + netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING, + "No valid timezone could be detected, using default UTC timezone. You can specify it explicitly with option 'timezone' in the server configuration.") + return 'UTC' + + +def split_every(n, iterable, piece_maker=tuple): + """Splits an iterable into length-n pieces. The last piece will be shorter + if ``n`` does not evenly divide the iterable length. + @param ``piece_maker``: function to build the pieces + from the slices (tuple,list,...) + """ + iterator = iter(iterable) + piece = piece_maker(islice(iterator, n)) + while piece: + yield piece + piece = piece_maker(islice(iterator, n)) if __name__ == '__main__': import doctest