1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Business Applications
5 # Copyright (C) 2012-2013 OpenERP S.A. (<http://openerp.com>).
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as
9 # published by the Free Software Foundation, either version 3 of the
10 # License, or (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 ##############################################################################
22 from lxml import etree
26 import lxml.html.clean as clean
34 from openerp.loglevels import ustr
36 _logger = logging.getLogger(__name__)
39 #----------------------------------------------------------
41 #----------------------------------------------------------
43 tags_to_kill = ["script", "head", "meta", "title", "link", "style", "frame", "iframe", "base", "object", "embed"]
44 tags_to_remove = ['html', 'body', 'font']
46 # allow new semantic HTML5 tags
47 allowed_tags = clean.defs.tags | frozenset('article section header footer hgroup nav aside figure'.split())
48 safe_attrs = clean.defs.safe_attrs | frozenset(['style'])
51 def html_sanitize(src, silent=True):
54 src = ustr(src, errors='replace')
56 logger = logging.getLogger(__name__ + '.html_sanitize')
58 # html encode email tags
59 part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)", re.IGNORECASE | re.DOTALL)
60 src = part.sub(lambda m: cgi.escape(m.group(1)), src)
63 'page_structure': True,
64 'style': False, # do not remove style attributes
65 'forms': True, # remove form tags
66 'remove_unknown_tags': False,
67 'allow_tags': allowed_tags,
69 if etree.LXML_VERSION >= (2, 3, 1):
70 # kill_tags attribute has been added in version 2.3.1
72 'kill_tags': tags_to_kill,
73 'remove_tags': tags_to_remove,
76 kwargs['remove_tags'] = tags_to_kill + tags_to_remove
78 if etree.LXML_VERSION >= (3, 1, 0):
80 'safe_attrs_only': True,
81 'safe_attrs': safe_attrs,
84 # lxml < 3.1.0 does not allow to specify safe_attrs. We keep all attributes in order to keep "style"
85 kwargs['safe_attrs_only'] = False
88 # some corner cases make the parser crash (such as <SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT> in test_mail)
89 cleaner = clean.Cleaner(**kwargs)
90 cleaned = cleaner.clean_html(src)
91 except etree.ParserError:
94 logger.warning('ParserError obtained when sanitizing %r', src, exc_info=True)
95 cleaned = '<p>ParserError when sanitizing</p>'
99 logger.warning('unknown error obtained when sanitizing %r', src, exc_info=True)
100 cleaned = '<p>Unknown error when sanitizing</p>'
104 #----------------------------------------------------------
106 #----------------------------------------------------------
108 def html_email_clean(html, remove=False, shorten=False, max_length=300, expand_options=None,
109 protect_sections=False):
110 """ html_email_clean: clean the html by doing the following steps:
112 - try to strip email quotes, by removing blockquotes or having some client-
114 - try to strip signatures
115 - shorten the html to a maximum number of characters if requested
117 Some specific use case:
119 - MsOffice: ``div.style = border-top:solid;`` delimitates the beginning of
120 a quote; detecting by finding WordSection1 of MsoNormal
121 - Hotmail: ``hr.stopSpelling`` delimitates the beginning of a quote; detect
122 Hotmail by funding ``SkyDrivePlaceholder``
124 :param string html: sanitized html; tags like html or head should not
125 be present in the html string. This method therefore
126 takes as input html code coming from a sanitized source,
128 :param boolean remove: remove the html code that is unwanted; otherwise it
129 is only flagged and tagged
130 :param boolean shorten: shorten the html; every excessing content will
131 be flagged as to remove
132 :param int max_length: if shortening, maximum number of characters before
134 :param dict expand_options: options for the read more link when shortening
135 the content.The used keys are the following:
137 - oe_expand_container_tag: class applied to the
138 container of the whole read more link
139 - oe_expand_container_class: class applied to the
140 link container (default: oe_mail_expand)
141 - oe_expand_container_content: content of the
142 container (default: ...)
143 - oe_expand_separator_node: optional separator, like
144 adding ... <br /><br /> <a ...>read more</a> (default: void)
145 - oe_expand_a_href: href of the read more link itself
147 - oe_expand_a_class: class applied to the <a> containing
148 the link itself (default: oe_mail_expand)
149 - oe_expand_a_content: content of the <a> (default: read more)
151 The formatted read more link is the following:
152 <cont_tag class="oe_expand_container_class">
153 oe_expand_container_content
154 if expand_options.get('oe_expand_separator_node'):
155 <oe_expand_separator_node/>
156 <a href="oe_expand_a_href" class="oe_expand_a_class">
161 def _replace_matching_regex(regex, source, replace=''):
162 """ Replace all matching expressions in source by replace """
167 for item in re.finditer(regex, source):
168 dest += source[idx:item.start()] + replace
173 def _create_node(tag, text, tail=None, attrs={}):
174 new_node = etree.Element(tag)
177 for key, val in attrs.iteritems():
178 new_node.set(key, val)
181 def _insert_new_node(node, index, new_node_tag, new_node_text, new_node_tail=None, new_node_attrs={}):
182 new_node = _create_node(new_node_tag, new_node_text, new_node_tail, new_node_attrs)
183 node.insert(index, new_node)
186 def _tag_matching_regex_in_text(regex, node, new_node_tag='span', new_node_attrs={}):
187 text = node.text or ''
188 if not re.search(regex, text):
193 idx, iteration = 0, 0
194 for item in re.finditer(regex, text):
196 cur_node.text = text[idx:item.start()]
198 _insert_new_node(node, (iteration - 1) * 2 + 1, new_node_tag, text[idx:item.start()])
199 new_node = _insert_new_node(node, iteration * 2, new_node_tag, text[item.start():item.end()], None, new_node_attrs)
204 new_node = _insert_new_node(node, -1, new_node_tag, text[idx:] + (cur_node.tail or ''), None, {})
206 def _truncate_node(node, position, find_first_blank=True):
208 innertext = node.text[0:position]
209 outertext = node.text[position:]
211 stop_idx = outertext.find(' ')
213 stop_idx = len(outertext)
216 node.text = innertext + outertext[0:stop_idx]
217 # create <span> ... <a href="#">read more</a></span> node
218 read_more_node = _create_node(
219 expand_options.get('oe_expand_container_tag', 'span'),
220 expand_options.get('oe_expand_container_content', ' ... '),
222 {'class': expand_options.get('oe_expand_container_class', 'oe_mail_expand')}
224 if expand_options.get('oe_expand_separator_node'):
225 read_more_separator_node = _create_node(
226 expand_options.get('oe_expand_separator_node'),
231 read_more_node.append(read_more_separator_node)
232 read_more_link_node = _create_node(
234 expand_options.get('oe_expand_a_content', 'read more'),
237 'href': expand_options.get('oe_expand_a_href', '#'),
238 'class': expand_options.get('oe_expand_a_class', 'oe_mail_expand'),
241 read_more_node.append(read_more_link_node)
242 # create outertext node
243 overtext_node = _create_node('span', outertext[stop_idx:])
245 overtext_node.set('in_overlength', '1')
246 # add newly created nodes in dom
247 node.append(read_more_node)
248 node.append(overtext_node)
250 if expand_options is None:
253 if not html or not isinstance(html, basestring):
258 # ------------------------------------------------------------
259 # TDE TODO: --- MAIL ORIGINAL ---: '[\-]{4,}([^\-]*)[\-]{4,}'
261 # html: remove encoding attribute inside tags
262 doctype = re.compile(r'(<[^>]*\s)(encoding=(["\'][^"\']*?["\']|[^\s\n\r>]+)(\s[^>]*|/)?>)', re.IGNORECASE | re.DOTALL)
263 html = doctype.sub(r"", html)
265 # html: ClEditor seems to love using <div><br /><div> -> replace with <br />
266 br_div_tags = re.compile(r'(<div>\s*<br\s*\/>\s*<\/div>)', re.IGNORECASE)
267 html = _replace_matching_regex(br_div_tags, html, '<br />')
270 root = lxml.html.fromstring(html)
271 if not len(root) and root.text is None and root.tail is None:
272 html = '<div>%s</div>' % html
273 root = lxml.html.fromstring(html)
275 # remove all tails and replace them by a span element, because managing text and tails can be a pain in the ass
276 for node in root.getiterator():
278 tail_node = _create_node('span', node.tail)
280 node.addnext(tail_node)
282 # form node and tag text-based quotes and signature
283 quote_tags = re.compile(r'(\n(>)+[^\n\r]*)')
284 signature = re.compile(r'([-]{2,}[\s]?[\r\n]{1,2}[^.]+)')
285 for node in root.getiterator():
286 _tag_matching_regex_in_text(quote_tags, node, 'span', {'text_quote': '1'})
287 _tag_matching_regex_in_text(signature, node, 'span', {'text_signature': '1'})
290 # ------------------------------------------------------------
293 # signature_begin = False # try dynamic signature recognition
296 overlength_section_id = None
297 overlength_section_count = 0
299 # for node in root.getiterator():
300 for node in root.iter():
301 # update: add a text argument
302 if node.text is None:
305 # root: try to tag the client used to write the html
306 if 'WordSection1' in node.get('class', '') or 'MsoNormal' in node.get('class', ''):
307 root.set('msoffice', '1')
308 if 'SkyDrivePlaceholder' in node.get('class', '') or 'SkyDrivePlaceholder' in node.get('id', ''):
309 root.set('hotmail', '1')
311 # protect sections by tagging section limits and blocks contained inside sections, using an increasing id to re-find them later
312 if node.tag == 'section':
313 overlength_section_count += 1
314 node.set('section_closure', str(overlength_section_count))
315 if node.getparent() is not None and (node.getparent().get('section_closure') or node.getparent().get('section_inner')):
316 node.set('section_inner', str(overlength_section_count))
318 # state of the parsing: flag quotes and tails to remove
320 node.set('in_quote', '1')
321 node.set('tail_remove', '1')
322 # state of the parsing: flag when being in over-length content, depending on section content if defined (only when having protect_sections)
324 if not overlength_section_id or int(node.get('section_inner', overlength_section_count + 1)) > overlength_section_count:
325 node.set('in_overlength', '1')
326 node.set('tail_remove', '1')
328 # find quote in msoffice / hotmail / blockquote / text quote and signatures
329 if root.get('msoffice') and node.tag == 'div' and 'border-top:solid' in node.get('style', ''):
331 node.set('in_quote', '1')
332 node.set('tail_remove', '1')
333 if root.get('hotmail') and node.tag == 'hr' and ('stopSpelling' in node.get('class', '') or 'stopSpelling' in node.get('id', '')):
335 node.set('in_quote', '1')
336 node.set('tail_remove', '1')
337 if node.tag == 'blockquote' or node.get('text_quote') or node.get('text_signature'):
338 node.set('in_quote', '1')
341 # if protect section:
342 # 1/ find the first parent not being inside a section
343 # 2/ add the read more link
345 # 1/ truncate the text at the next available space
346 # 2/ create a 'read more' node, next to current node
347 # 3/ add the truncated text in a new node, next to 'read more' node
348 node_text = (node.text or '').strip().strip('\n').strip()
349 if shorten and not overlength and cur_char_nbr + len(node_text) > max_length:
352 node_to_truncate = node
353 while node_to_truncate.getparent() is not None and \
354 (node_to_truncate.getparent().get('section_inner') or node_to_truncate.getparent().get('section_closure')):
355 node_to_truncate = node_to_truncate.getparent()
356 overlength_section_id = node_to_truncate.get('section_closure')
357 position = len(node_to_truncate.text)
358 find_first_blank = False
360 node_to_truncate = node
361 position = max_length - cur_char_nbr
362 find_first_blank = True
363 node_to_truncate.set('truncate', '1')
364 node_to_truncate.set('truncate_position', str(position))
365 node_to_truncate.set('truncate_blank', str(find_first_blank))
366 cur_char_nbr += len(node_text)
369 # ------------------------------------------------------------
371 for node in root.iter():
372 if node.get('truncate'):
373 _truncate_node(node, int(node.get('truncate_position', '0')), bool(node.get('truncate_blank', 'True')))
376 # ------------------------------------------------------------
379 for node in root.iter():
380 if node.get('in_quote') or node.get('in_overlength'):
381 # copy the node tail into parent text
382 if node.tail and not node.get('tail_remove'):
383 parent = node.getparent()
384 parent.tail = node.tail + (parent.tail or '')
385 to_remove.append(node)
386 if node.get('tail_remove'):
388 for node in to_remove:
390 node.getparent().remove(node)
392 if not expand_options.get('oe_expand_a_class', 'oe_mail_expand') in node.get('class', ''): # trick: read more link should be displayed even if it's in overlength
393 node_class = node.get('class', '') + ' ' + 'oe_mail_cleaned'
394 node.set('class', node_class)
396 # html: \n that were tail of elements have been encapsulated into <span> -> back to \n
397 html = etree.tostring(root, pretty_print=False)
398 linebreaks = re.compile(r'<span>([\s]*[\r\n]+[\s]*)<\/span>', re.IGNORECASE | re.DOTALL)
399 html = _replace_matching_regex(linebreaks, html, '\n')
404 #----------------------------------------------------------
405 # HTML/Text management
406 #----------------------------------------------------------
408 def html2plaintext(html, body_id=None, encoding='utf-8'):
409 """ From an HTML text, convert the HTML to plain text.
410 If @param body_id is provided then this is the tag where the
411 body (not necessarily <body>) starts.
413 ## (c) Fry-IT, www.fry-it.com, 2007
414 ## <peter@fry-it.com>
415 ## download here: http://www.peterbe.com/plog/html2plaintext
418 tree = etree.fromstring(html, parser=etree.HTMLParser())
420 if body_id is not None:
421 source = tree.xpath('//*[@id=%s]' % (body_id,))
423 source = tree.xpath('//body')
429 for link in tree.findall('.//a'):
430 url = link.get('href')
434 link.text = '%s [%s]' % (link.text, i)
435 url_index.append(url)
437 html = ustr(etree.tostring(tree, encoding=encoding))
438 # \r char is converted into , must remove it
439 html = html.replace(' ', '')
441 html = html.replace('<strong>', '*').replace('</strong>', '*')
442 html = html.replace('<b>', '*').replace('</b>', '*')
443 html = html.replace('<h3>', '*').replace('</h3>', '*')
444 html = html.replace('<h2>', '**').replace('</h2>', '**')
445 html = html.replace('<h1>', '**').replace('</h1>', '**')
446 html = html.replace('<em>', '/').replace('</em>', '/')
447 html = html.replace('<tr>', '\n')
448 html = html.replace('</p>', '\n')
449 html = re.sub('<br\s*/?>', '\n', html)
450 html = re.sub('<.*?>', ' ', html)
451 html = html.replace(' ' * 2, ' ')
454 html = '\n'.join([x.strip() for x in html.splitlines()])
455 html = html.replace('\n' * 2, '\n')
457 for i, url in enumerate(url_index):
460 html += ustr('[%s] %s\n') % (i + 1, url)
464 def plaintext2html(text, container_tag=False):
465 """ Convert plaintext into html. Content of the text is escaped to manage
466 html entities, using cgi.escape().
467 - all \n,\r are replaced by <br />
468 - enclose content into <p>
469 - 2 or more consecutive <br /> are considered as paragraph breaks
471 :param string container_tag: container of the html; by default the
472 content is embedded into a <div>
474 text = cgi.escape(ustr(text))
476 # 1. replace \n and \r
477 text = text.replace('\n', '<br/>')
478 text = text.replace('\r', '<br/>')
480 # 2-3: form paragraphs
483 br_tags = re.compile(r'(([<]\s*[bB][rR]\s*\/?[>]\s*){2,})')
484 for item in re.finditer(br_tags, text):
485 final += text[idx:item.start()] + '</p><p>'
487 final += text[idx:] + '</p>'
491 final = '<%s>%s</%s>' % (container_tag, final, container_tag)
494 def append_content_to_html(html, content, plaintext=True, preserve=False, container_tag=False):
495 """ Append extra content at the end of an HTML snippet, trying
496 to locate the end of the HTML document (</body>, </html>, or
497 EOF), and converting the provided content in html unless ``plaintext``
499 Content conversion can be done in two ways:
500 - wrapping it into a pre (preserve=True)
501 - use plaintext2html (preserve=False, using container_tag to wrap the
503 A side-effect of this method is to coerce all HTML tags to
504 lowercase in ``html``, and strip enclosing <html> or <body> tags in
505 content if ``plaintext`` is False.
507 :param str html: html tagsoup (doesn't have to be XHTML)
508 :param str content: extra content to append
509 :param bool plaintext: whether content is plaintext and should
510 be wrapped in a <pre/> tag.
511 :param bool preserve: if content is plaintext, wrap it into a <pre>
512 instead of converting it into html
515 if plaintext and preserve:
516 content = u'\n<pre>%s</pre>\n' % ustr(content)
518 content = '\n%s\n' % plaintext2html(content, container_tag)
520 content = re.sub(r'(?i)(</?html.*>|</?body.*>|<!\W*DOCTYPE.*>)', '', content)
521 content = u'\n%s\n' % ustr(content)
522 # Force all tags to lowercase
523 html = re.sub(r'(</?)\W*(\w+)([ >])',
524 lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
525 insert_location = html.find('</body>')
526 if insert_location == -1:
527 insert_location = html.find('</html>')
528 if insert_location == -1:
529 return '%s%s' % (html, content)
530 return '%s%s%s' % (html[:insert_location], content, html[insert_location:])
532 #----------------------------------------------------------
534 #----------------------------------------------------------
536 # matches any email in a body of text
537 email_re = re.compile(r"""([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6})""", re.VERBOSE)
539 # matches a string containing only one email
540 single_email_re = re.compile(r"""^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}$""", re.VERBOSE)
542 res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE)
543 command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
545 # Updated in 7.0 to match the model name as well
546 # Typical form of references is <timestamp-openerp-record_id-model_name@domain>
547 # group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
548 reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE)
551 # Typical form of bounce is bounce-128-crm.lead-34@domain
552 # group(1) = the mail ID; group(2) = the model (if any); group(3) = the record ID
553 bounce_re = re.compile("[\w]+-(\d+)-?([\w.]+)?-?(\d+)?", re.UNICODE)
555 def generate_tracking_message_id(res_id):
556 """Returns a string that can be used in the Message-ID RFC822 header field
558 Used to track the replies related to a given object thanks to the "In-Reply-To"
559 or "References" fields that Mail User Agents will set.
562 rnd = random.SystemRandom().random()
563 except NotImplementedError:
564 rnd = random.random()
565 rndstr = ("%.15f" % rnd)[2:]
566 return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname())
568 def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False,
569 attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None,
570 smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None):
571 """Low-level function for sending an email (deprecated).
573 :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead.
574 :param email_from: A string used to fill the `From` header, if falsy,
575 config['email_from'] is used instead. Also used for
576 the `Reply-To` header if `reply_to` is not provided
577 :param email_to: a sequence of addresses to send the mail to.
580 # If not cr, get cr from current thread database
583 db_name = getattr(threading.currentThread(), 'dbname', None)
585 local_cr = cr = openerp.registry(db_name).db.cursor()
587 raise Exception("No database cursor found, please pass one explicitly")
591 mail_server_pool = openerp.registry(cr.dbname)['ir.mail_server']
593 # Pack Message into MIME Object
594 email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to,
595 attachments, message_id, references, openobject_id, subtype, headers=headers)
597 res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None,
598 smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password,
599 smtp_encryption=('ssl' if ssl else None), smtp_debug=debug)
601 _logger.exception("tools.email_send failed to deliver email")
608 def email_split(text):
609 """ Return a list of the email addresses found in ``text`` """
612 return re.findall(r'([^ ,<@]+@[^> ,]+)', text)