2 from __future__ import absolute_import
3 from email.utils import parseaddr
10 from ast import literal_eval
11 from openerp.tools import mute_logger
13 # Validation Library https://pypi.python.org/pypi/validate_email/1.1
14 from .validate_email import validate_email
17 from openerp.osv import osv, orm
18 from openerp.osv import fields
19 from openerp.osv.orm import browse_record
20 from openerp.tools.translate import _
22 pattern = re.compile("&(\w+?);")
24 _logger = logging.getLogger('base.partner.merge')
27 # http://www.php2python.com/wiki/function.html-entity-decode/
28 def html_entity_decode_char(m, defs=htmlentitydefs.entitydefs):
30 return defs[m.group(1)]
35 def html_entity_decode(string):
36 return pattern.sub(html_entity_decode_char, string)
39 def sanitize_email(email):
40 assert isinstance(email, basestring) and email
42 result = re.subn(r';|/|:', ',',
43 html_entity_decode(email or ''))[0].split(',')
45 emails = [parseaddr(email)[1]
47 for email in item.split()]
51 if validate_email(email)]
54 def is_integer_list(ids):
55 return all(isinstance(i, (int, long)) for i in ids)
58 class ResPartner(osv.Model):
59 _inherit = 'res.partner'
62 'id': fields.integer('Id', readonly=True),
63 'create_date': fields.datetime('Create Date', readonly=True),
66 class MergePartnerLine(osv.TransientModel):
67 _name = 'base.partner.merge.line'
70 'wizard_id': fields.many2one('base.partner.merge.automatic.wizard',
72 'min_id': fields.integer('MinID'),
73 'aggr_ids': fields.char('Ids', required=True),
79 class MergePartnerAutomatic(osv.TransientModel):
81 The idea behind this wizard is to create a list of potential partners to
82 merge. We use two objects, the first one is the wizard for the end-user.
83 And the second will contain the partner list to merge.
85 _name = 'base.partner.merge.automatic.wizard'
89 'group_by_email': fields.boolean('Email'),
90 'group_by_name': fields.boolean('Name'),
91 'group_by_is_company': fields.boolean('Is Company'),
92 'group_by_vat': fields.boolean('VAT'),
93 'group_by_parent_id': fields.boolean('Parent Company'),
95 'state': fields.selection([('option', 'Option'),
96 ('selection', 'Selection'),
97 ('finished', 'Finished')],
101 'number_group': fields.integer("Group of Contacts", readonly=True),
102 'current_line_id': fields.many2one('base.partner.merge.line', 'Current Line'),
103 'line_ids': fields.one2many('base.partner.merge.line', 'wizard_id', 'Lines'),
104 'partner_ids': fields.many2many('res.partner', string='Contacts'),
105 'dst_partner_id': fields.many2one('res.partner', string='Destination Contact'),
107 'exclude_contact': fields.boolean('A user associated to the contact'),
108 'exclude_journal_item': fields.boolean('Journal Items associated to the contact'),
109 'maximum_group': fields.integer("Maximum of Group of Contacts"),
112 def default_get(self, cr, uid, fields, context=None):
115 res = super(MergePartnerAutomatic, self).default_get(cr, uid, fields, context)
116 if context.get('active_model') == 'res.partner' and context.get('active_ids'):
117 partner_ids = context['active_ids']
118 res['state'] = 'selection'
119 res['partner_ids'] = partner_ids
120 res['dst_partner_id'] = self._get_ordered_partner(cr, uid, partner_ids, context=context)[-1].id
127 def get_fk_on(self, cr, table):
128 q = """ SELECT cl1.relname as table,
129 att1.attname as column
130 FROM pg_constraint as con, pg_class as cl1, pg_class as cl2,
131 pg_attribute as att1, pg_attribute as att2
132 WHERE con.conrelid = cl1.oid
133 AND con.confrelid = cl2.oid
134 AND array_lower(con.conkey, 1) = 1
135 AND con.conkey[1] = att1.attnum
136 AND att1.attrelid = cl1.oid
138 AND att2.attname = 'id'
139 AND array_lower(con.confkey, 1) = 1
140 AND con.confkey[1] = att2.attnum
141 AND att2.attrelid = cl2.oid
142 AND con.contype = 'f'
144 return cr.execute(q, (table,))
146 def _update_foreign_keys(self, cr, uid, src_partners, dst_partner, context=None):
147 _logger.debug('_update_foreign_keys for dst_partner: %s for src_partners: %r', dst_partner.id, list(map(operator.attrgetter('id'), src_partners)))
149 # find the many2one relation to a partner
150 proxy = self.pool.get('res.partner')
151 self.get_fk_on(cr, 'res_partner')
155 for table, column in cr.fetchall():
156 if 'base_partner_merge_' in table:
158 partner_ids = tuple(map(int, src_partners))
160 query = "SELECT column_name FROM information_schema.columns WHERE table_name LIKE '%s'" % (table)
161 cr.execute(query, ())
163 for data in cr.fetchall():
164 if data[0] != column:
165 columns.append(data[0])
172 if len(columns) <= 1:
175 UPDATE "%(table)s" as ___tu
181 FROM "%(table)s" as ___tw
184 ___tu.%(value)s = ___tw.%(value)s
186 for partner_id in partner_ids:
187 cr.execute(query, (dst_partner.id, partner_id, dst_partner.id))
189 cr.execute("SAVEPOINT recursive_partner_savepoint")
191 query = 'UPDATE "%(table)s" SET %(column)s = %%s WHERE %(column)s IN %%s' % query_dic
192 cr.execute(query, (dst_partner.id, partner_ids,))
194 if column == proxy._parent_name and table == 'res_partner':
196 WITH RECURSIVE cycle(id, parent_id) AS (
197 SELECT id, parent_id FROM res_partner
199 SELECT cycle.id, res_partner.parent_id
200 FROM res_partner, cycle
201 WHERE res_partner.id = cycle.parent_id AND
202 cycle.id != cycle.parent_id
204 SELECT id FROM cycle WHERE id = parent_id AND id = %s
206 cr.execute(query, (dst_partner.id,))
208 cr.execute("ROLLBACK TO SAVEPOINT recursive_partner_savepoint")
210 cr.execute("RELEASE SAVEPOINT recursive_partner_savepoint")
212 def _update_reference_fields(self, cr, uid, src_partners, dst_partner, context=None):
213 _logger.debug('_update_reference_fields for dst_partner: %s for src_partners: %r', dst_partner.id, list(map(operator.attrgetter('id'), src_partners)))
215 def update_records(model, src, field_model='model', field_id='res_id', context=None):
216 proxy = self.pool.get(model)
219 domain = [(field_model, '=', 'res.partner'), (field_id, '=', src.id)]
220 ids = proxy.search(cr, openerp.SUPERUSER_ID, domain, context=context)
221 return proxy.write(cr, openerp.SUPERUSER_ID, ids, {field_id: dst_partner.id}, context=context)
223 update_records = functools.partial(update_records, context=context)
225 for partner in src_partners:
226 update_records('calendar', src=partner, field_model='model_id.model')
227 update_records('ir.attachment', src=partner, field_model='res_model')
228 update_records('mail.followers', src=partner, field_model='res_model')
229 update_records('mail.message', src=partner)
230 update_records('marketing.campaign.workitem', src=partner, field_model='object_id.model')
231 update_records('ir.model.data', src=partner)
233 proxy = self.pool['ir.model.fields']
234 domain = [('ttype', '=', 'reference')]
235 record_ids = proxy.search(cr, openerp.SUPERUSER_ID, domain, context=context)
237 for record in proxy.browse(cr, openerp.SUPERUSER_ID, record_ids, context=context):
239 proxy_model = self.pool[record.model]
240 field_type = proxy_model._columns[record.name].__class__._type
242 # unknown model or field => skip
245 if field_type == 'function':
248 for partner in src_partners:
250 (record.name, '=', 'res.partner,%d' % partner.id)
252 model_ids = proxy_model.search(cr, openerp.SUPERUSER_ID, domain, context=context)
254 record.name: 'res.partner,%d' % dst_partner.id,
256 proxy_model.write(cr, openerp.SUPERUSER_ID, model_ids, values, context=context)
258 def _update_values(self, cr, uid, src_partners, dst_partner, context=None):
259 _logger.debug('_update_values for dst_partner: %s for src_partners: %r', dst_partner.id, list(map(operator.attrgetter('id'), src_partners)))
261 columns = dst_partner._columns
262 def write_serializer(column, item):
263 if isinstance(item, browse_record):
269 for column, field in columns.iteritems():
270 if field._type not in ('many2many', 'one2many') and not isinstance(field, fields.function):
271 for item in itertools.chain(src_partners, [dst_partner]):
273 values[column] = write_serializer(column, item[column])
275 values.pop('id', None)
276 parent_id = values.pop('parent_id', None)
277 dst_partner.write(values)
278 if parent_id and parent_id != dst_partner.id:
280 dst_partner.write({'parent_id': parent_id})
281 except (osv.except_osv, orm.except_orm):
282 _logger.info('Skip recursive partner hierarchies for parent_id %s of partner: %s', parent_id, dst_partner.id)
284 @mute_logger('openerp.osv.expression', 'openerp.osv.orm')
285 def _merge(self, cr, uid, partner_ids, dst_partner=None, context=None):
286 proxy = self.pool.get('res.partner')
288 partner_ids = proxy.exists(cr, uid, list(partner_ids), context=context)
289 if len(partner_ids) < 2:
292 if len(partner_ids) > 3:
293 raise osv.except_osv(_('Error'), _("For safety reasons, you cannot merge more than 3 contacts together. You can re-open the wizard several times if needed."))
295 if openerp.SUPERUSER_ID != uid and len(set(partner.email for partner in proxy.browse(cr, uid, partner_ids, context=context))) > 1:
296 raise osv.except_osv(_('Error'), _("All contacts must have the same email. Only the Administrator can merge contacts with different emails."))
298 if dst_partner and dst_partner.id in partner_ids:
299 src_partners = proxy.browse(cr, uid, [id for id in partner_ids if id != dst_partner.id], context=context)
301 ordered_partners = self._get_ordered_partner(cr, uid, partner_ids, context)
302 dst_partner = ordered_partners[-1]
303 src_partners = ordered_partners[:-1]
304 _logger.info("dst_partner: %s", dst_partner.id)
306 if openerp.SUPERUSER_ID != uid and self._model_is_installed(cr, uid, 'account.move.line', context=context) and \
307 self.pool.get('account.move.line').search(cr, openerp.SUPERUSER_ID, [('partner_id', 'in', [partner.id for partner in src_partners])], context=context):
308 raise osv.except_osv(_('Error'), _("Only the destination contact may be linked to existing Journal Items. Please ask the Administrator if you need to merge several contacts linked to existing Journal Items."))
310 call_it = lambda function: function(cr, uid, src_partners, dst_partner,
313 call_it(self._update_foreign_keys)
314 call_it(self._update_reference_fields)
315 call_it(self._update_values)
317 _logger.info('(uid = %s) merged the partners %r with %s', uid, list(map(operator.attrgetter('id'), src_partners)), dst_partner.id)
318 dst_partner.message_post(body='%s %s'%(_("Merged with the following partners:"), ", ".join('%s<%s>(ID %s)' % (p.name, p.email or 'n/a', p.id) for p in src_partners)))
320 for partner in src_partners:
323 def clean_emails(self, cr, uid, context=None):
325 Clean the email address of the partner, if there is an email field with
326 a mimum of two addresses, the system will create a new partner, with the
327 information of the previous one and will copy the new cleaned email into
333 proxy_model = self.pool['ir.model.fields']
334 field_ids = proxy_model.search(cr, uid, [('model', '=', 'res.partner'),
335 ('ttype', 'like', '%2many')],
337 fields = proxy_model.read(cr, uid, field_ids, context=context)
338 reset_fields = dict((field['name'], []) for field in fields)
340 proxy_partner = self.pool['res.partner']
341 context['active_test'] = False
342 ids = proxy_partner.search(cr, uid, [], context=context)
344 fields = ['name', 'var' 'partner_id' 'is_company', 'email']
345 partners = proxy_partner.read(cr, uid, ids, fields, context=context)
347 partners.sort(key=operator.itemgetter('id'))
348 partners_len = len(partners)
350 _logger.info('partner_len: %r', partners_len)
352 for idx, partner in enumerate(partners):
353 if not partner['email']:
356 percent = (idx / float(partners_len)) * 100.0
357 _logger.info('idx: %r', idx)
358 _logger.info('percent: %r', percent)
360 emails = sanitize_email(partner['email'])
361 head, tail = emails[:1], emails[1:]
362 email = head[0] if head else False
364 proxy_partner.write(cr, uid, [partner['id']],
365 {'email': email}, context=context)
368 values = dict(reset_fields, email=email)
369 proxy_partner.copy(cr, uid, partner['id'], values,
373 _logger.exception("There is a problem with this partner: %r", partner)
377 def close_cb(self, cr, uid, ids, context=None):
378 return {'type': 'ir.actions.act_window_close'}
380 def _generate_query(self, fields, maximum_group=100):
381 group_fields = ', '.join(fields)
385 if field in ['email', 'name']:
386 filters.append((field, 'IS NOT', 'NULL'))
388 criteria = ' AND '.join('%s %s %s' % (field, operator, value)
389 for field, operator, value in filters)
392 "SELECT min(id), array_agg(id)",
397 text.append('WHERE %s' % criteria)
400 "GROUP BY %s" % group_fields,
401 "HAVING COUNT(*) >= 2",
407 "LIMIT %s" % maximum_group,
410 return ' '.join(text)
412 def _compute_selected_groupby(self, this):
413 group_by_str = 'group_by_'
414 group_by_len = len(group_by_str)
418 for key in self._columns.keys()
419 if key.startswith(group_by_str)
425 if getattr(this, '%s%s' % (group_by_str, field), False)
429 raise osv.except_osv(_('Error'),
430 _("You have to specify a filter for your selection"))
434 def next_cb(self, cr, uid, ids, context=None):
436 Don't compute any thing
438 context = dict(context or {}, active_test=False)
439 this = self.browse(cr, uid, ids[0], context=context)
440 if this.current_line_id:
441 this.current_line_id.unlink()
442 return self._next_screen(cr, uid, this, context)
444 def _get_ordered_partner(self, cr, uid, partner_ids, context=None):
445 partners = self.pool.get('res.partner').browse(cr, uid, list(partner_ids), context=context)
446 ordered_partners = sorted(sorted(partners,
447 key=operator.attrgetter('create_date'), reverse=True),
448 key=operator.attrgetter('active'), reverse=True)
449 return ordered_partners
451 def _next_screen(self, cr, uid, this, context=None):
455 # in this case, we try to find the next record.
456 current_line = this.line_ids[0]
457 current_partner_ids = literal_eval(current_line.aggr_ids)
459 'current_line_id': current_line.id,
460 'partner_ids': [(6, 0, current_partner_ids)],
461 'dst_partner_id': self._get_ordered_partner(cr, uid, current_partner_ids, context)[-1].id,
462 'state': 'selection',
466 'current_line_id': False,
474 'type': 'ir.actions.act_window',
475 'res_model': this._name,
481 def _model_is_installed(self, cr, uid, model, context=None):
482 proxy = self.pool.get('ir.model')
483 domain = [('model', '=', model)]
484 return proxy.search_count(cr, uid, domain, context=context) > 0
486 def _partner_use_in(self, cr, uid, aggr_ids, models, context=None):
488 Check if there is no occurence of this group of partner in the selected
491 for model, field in models.iteritems():
492 proxy = self.pool.get(model)
493 domain = [(field, 'in', aggr_ids)]
494 if proxy.search_count(cr, uid, domain, context=context):
498 def compute_models(self, cr, uid, ids, context=None):
500 Compute the different models needed by the system if you want to exclude
503 assert is_integer_list(ids)
505 this = self.browse(cr, uid, ids[0], context=context)
508 if this.exclude_contact:
509 models['res.users'] = 'partner_id'
511 if self._model_is_installed(cr, uid, 'account.move.line', context=context) and this.exclude_journal_item:
512 models['account.move.line'] = 'partner_id'
516 def _process_query(self, cr, uid, ids, query, context=None):
518 Execute the select request and write the result in this wizard
520 proxy = self.pool.get('base.partner.merge.line')
521 this = self.browse(cr, uid, ids[0], context=context)
522 models = self.compute_models(cr, uid, ids, context=context)
526 for min_id, aggr_ids in cr.fetchall():
527 if models and self._partner_use_in(cr, uid, aggr_ids, models, context=context):
530 'wizard_id': this.id,
532 'aggr_ids': aggr_ids,
535 proxy.create(cr, uid, values, context=context)
539 'state': 'selection',
540 'number_group': counter,
545 _logger.info("counter: %s", counter)
547 def start_process_cb(self, cr, uid, ids, context=None):
550 * Compute the selected groups (with duplication)
551 * If the user has selected the 'exclude_XXX' fields, avoid the partners.
553 assert is_integer_list(ids)
555 context = dict(context or {}, active_test=False)
556 this = self.browse(cr, uid, ids[0], context=context)
557 groups = self._compute_selected_groupby(this)
558 query = self._generate_query(groups, this.maximum_group)
559 self._process_query(cr, uid, ids, query, context=context)
561 return self._next_screen(cr, uid, this, context)
563 def automatic_process_cb(self, cr, uid, ids, context=None):
564 assert is_integer_list(ids)
565 this = self.browse(cr, uid, ids[0], context=context)
566 this.start_process_cb()
569 for line in this.line_ids:
570 partner_ids = literal_eval(line.aggr_ids)
571 self._merge(cr, uid, partner_ids, context=context)
575 this.write({'state': 'finished'})
577 'type': 'ir.actions.act_window',
578 'res_model': this._name,
584 def parent_migration_process_cb(self, cr, uid, ids, context=None):
585 assert is_integer_list(ids)
587 context = dict(context or {}, active_test=False)
588 this = self.browse(cr, uid, ids[0], context=context)
593 array_agg(DISTINCT p1.id)
599 p1.email = p2.email AND
600 p1.name = p2.name AND
601 (p1.parent_id = p2.id OR p1.id = p2.parent_id)
607 CASE WHEN p1.parent_id = p2.id THEN p2.id
615 self._process_query(cr, uid, ids, query, context=context)
617 for line in this.line_ids:
618 partner_ids = literal_eval(line.aggr_ids)
619 self._merge(cr, uid, partner_ids, context=context)
623 this.write({'state': 'finished'})
636 'type': 'ir.actions.act_window',
637 'res_model': this._name,
643 def update_all_process_cb(self, cr, uid, ids, context=None):
644 assert is_integer_list(ids)
646 # WITH RECURSIVE cycle(id, parent_id) AS (
647 # SELECT id, parent_id FROM res_partner
649 # SELECT cycle.id, res_partner.parent_id
650 # FROM res_partner, cycle
651 # WHERE res_partner.id = cycle.parent_id AND
652 # cycle.id != cycle.parent_id
655 # SET parent_id = NULL
656 # WHERE id in (SELECT id FROM cycle WHERE id = parent_id);
658 this = self.browse(cr, uid, ids[0], context=context)
660 self.parent_migration_process_cb(cr, uid, ids, context=None)
663 {'group_by_vat': True, 'group_by_email': True, 'group_by_name': True},
664 # {'group_by_name': True, 'group_by_is_company': True, 'group_by_parent_id': True},
665 # {'group_by_email': True, 'group_by_is_company': True, 'group_by_parent_id': True},
666 # {'group_by_name': True, 'group_by_vat': True, 'group_by_is_company': True, 'exclude_journal_item': True},
667 # {'group_by_email': True, 'group_by_vat': True, 'group_by_is_company': True, 'exclude_journal_item': True},
668 # {'group_by_email': True, 'group_by_is_company': True, 'exclude_contact': True, 'exclude_journal_item': True},
669 # {'group_by_name': True, 'group_by_is_company': True, 'exclude_contact': True, 'exclude_journal_item': True}
672 for merge_value in list_merge:
673 id = self.create(cr, uid, merge_value, context=context)
674 self.automatic_process_cb(cr, uid, [id], context=context)
682 parent_id IS NOT NULL AND
683 is_company IS NOT NULL
693 # FROM res_partner as p2
694 # WHERE p2.email = p1.email AND
695 # p2.parent_id != p2.id
699 # p1.parent_id = p1.id
702 return self._next_screen(cr, uid, this, context)
704 def merge_cb(self, cr, uid, ids, context=None):
705 assert is_integer_list(ids)
707 context = dict(context or {}, active_test=False)
708 this = self.browse(cr, uid, ids[0], context=context)
710 partner_ids = set(map(int, this.partner_ids))
712 this.write({'state': 'finished'})
714 'type': 'ir.actions.act_window',
715 'res_model': this._name,
721 self._merge(cr, uid, partner_ids, this.dst_partner_id, context=context)
723 if this.current_line_id:
724 this.current_line_id.unlink()
726 return self._next_screen(cr, uid, this, context)
728 def auto_set_parent_id(self, cr, uid, ids, context=None):
729 assert is_integer_list(ids)
731 # select partner who have one least invoice
732 partner_treated = ['@gmail.com']
733 cr.execute(""" SELECT p.id, p.email
734 FROM res_partner as p
735 LEFT JOIN account_invoice as a
736 ON p.id = a.partner_id AND a.state in ('open','paid')
737 WHERE p.grade_id is NOT NULL
739 ORDER BY COUNT(a.id) DESC
741 re_email = re.compile(r".*@")
742 for id, email in cr.fetchall():
744 email = re_email.sub("@", email or "")
745 if not email or email in partner_treated:
747 partner_treated.append(email)
749 # don't update the partners if they are more of one who have invoice
750 cr.execute(""" SELECT *
751 FROM res_partner as p
752 WHERE p.id != %s AND p.email LIKE '%%%s' AND
753 EXISTS (SELECT * FROM account_invoice as a WHERE p.id = a.partner_id AND a.state in ('open','paid'))
756 if len(cr.fetchall()) > 1:
757 _logger.info("%s MORE OF ONE COMPANY", email)
760 # to display changed values
761 cr.execute(""" SELECT id,email
763 WHERE parent_id != %s AND id != %s AND email LIKE '%%%s'
764 """ % (id, id, email))
765 _logger.info("%r", cr.fetchall())
768 cr.execute(""" UPDATE res_partner
770 WHERE id != %s AND email LIKE '%%%s'
771 """ % (id, id, email))