X-Git-Url: http://git.inspyration.org/?a=blobdiff_plain;f=bin%2Ftools%2Fyaml_import.py;h=6e5977d8e6af2155f4804d87b854ae05f8d8b23d;hb=896c74fa23dd6fee2f8f81d89ffb8227975b0e6b;hp=cd1a1a34fd619b4f5950fe678e3a965a85dff19b;hpb=3f8b66799f6cf768320045aca6131b7d8eb6dc5c;p=odoo%2Fodoo.git diff --git a/bin/tools/yaml_import.py b/bin/tools/yaml_import.py index cd1a1a3..6e5977d 100644 --- a/bin/tools/yaml_import.py +++ b/bin/tools/yaml_import.py @@ -1,6 +1,7 @@ # -*- encoding: utf-8 -*- import types import time # used to eval time.strftime expressions +from datetime import datetime, timedelta import logging import pooler @@ -8,9 +9,13 @@ import netsvc import misc from config import config import yaml_tag - import yaml +# YAML import needs both safe and unsafe eval, but let's +# default to /safe/. +unsafe_eval = eval +from tools.safe_eval import safe_eval as eval + logger_channel = 'tests' class YamlImportException(Exception): @@ -29,7 +34,8 @@ def is_comment(node): return isinstance(node, types.StringTypes) def is_assert(node): - return _is_yaml_mapping(node, yaml_tag.Assert) + return isinstance(node, yaml_tag.Assert) \ + or _is_yaml_mapping(node, yaml_tag.Assert) def is_record(node): return _is_yaml_mapping(node, yaml_tag.Record) @@ -65,14 +71,16 @@ def is_url(node): def is_eval(node): return isinstance(node, yaml_tag.Eval) - + def is_ref(node): return isinstance(node, yaml_tag.Ref) \ or _is_yaml_mapping(node, yaml_tag.Ref) - + def is_ir_set(node): return _is_yaml_mapping(node, yaml_tag.IrSet) +def is_string(node): + return isinstance(node, basestring) class TestReport(object): def __init__(self): @@ -98,7 +106,7 @@ class TestReport(object): success += self._report[severity][True] failure += self._report[severity][False] res.append("total\t%s\t%s" % (success, failure)) - res.append("end of report (%s assertion(s) checked)" % success + failure) + res.append("end of report (%s assertion(s) checked)" % (success + failure)) return "\n".join(res) class RecordDictWrapper(dict): @@ -112,7 +120,7 @@ class RecordDictWrapper(dict): if key in self.record: return self.record[key] return dict.__getitem__(self, key) - + class YamlInterpreter(object): def __init__(self, cr, module, id_map, mode, filename, noupdate=False): self.cr = cr @@ -126,7 +134,11 @@ class YamlInterpreter(object): self.pool = pooler.get_pool(cr.dbname) self.uid = 1 self.context = {} # opererp context - self.eval_context = {'ref': self._ref(), '_ref': self._ref(), 'time': time} # added '_ref' so that record['ref'] is possible + self.eval_context = {'ref': self._ref(), + '_ref': self._ref(), # added '_ref' so that record['ref'] is possible + 'time': time, + 'datetime': datetime, + 'timedelta': timedelta} def _ref(self): return lambda xml_id: self.get_id(xml_id) @@ -135,7 +147,7 @@ class YamlInterpreter(object): model = self.pool.get(model_name) assert model, "The model %s does not exist." % (model_name,) return model - + def validate_xml_id(self, xml_id): id = xml_id if '.' in xml_id: @@ -163,9 +175,7 @@ class YamlInterpreter(object): else: module = self.module checked_xml_id = xml_id - ir_id = self.pool.get('ir.model.data')._get_id(self.cr, self.uid, module, checked_xml_id) - obj = self.pool.get('ir.model.data').read(self.cr, self.uid, ir_id, ['res_id']) - id = int(obj['res_id']) + _, id = self.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, module, checked_xml_id) self.id_map[xml_id] = id return id @@ -209,19 +219,22 @@ class YamlInterpreter(object): elif assertion.search: q = eval(assertion.search, self.eval_context) ids = self.pool.get(assertion.model).search(self.cr, self.uid, q, context=assertion.context) - if not ids: + else: raise YamlImportException('Nothing to assert: you must give either an id or a search criteria.') return ids def process_assert(self, node): - assertion, expressions = node.items()[0] + if isinstance(node, dict): + assertion, expressions = node.items()[0] + else: + assertion, expressions = node, [] if self.isnoupdate(assertion) and self.mode != 'init': self.logger.warn('This assertion was not evaluated ("%s").' % assertion.string) return model = self.get_model(assertion.model) ids = self._get_assertion_id(assertion) - if assertion.count and len(ids) != assertion.count: + if assertion.count is not None and len(ids) != assertion.count: msg = 'assertion "%s" failed!\n' \ ' Incorrect search count:\n' \ ' expected count: %d\n' \ @@ -234,12 +247,32 @@ class YamlInterpreter(object): record = model.browse(self.cr, self.uid, id, context) for test in expressions: try: - success = eval(test, self.eval_context, RecordDictWrapper(record)) + success = unsafe_eval(test, self.eval_context, RecordDictWrapper(record)) except Exception, e: + self.logger.debug('Exception during evaluation of !assert block in yaml_file %s.', self.filename, exc_info=True) raise YamlImportAbortion(e) if not success: msg = 'Assertion "%s" FAILED\ntest: %s\n' args = (assertion.string, test) + for aop in ('==', '!=', '<>', 'in', 'not in', '>=', '<=', '>', '<'): + if aop in test: + left, right = test.split(aop,1) + lmsg = '' + rmsg = '' + try: + lmsg = unsafe_eval(left, self.eval_context, RecordDictWrapper(record)) + except Exception, e: + lmsg = '' + + try: + rmsg = unsafe_eval(right, self.eval_context, RecordDictWrapper(record)) + except Exception, e: + rmsg = '' + + msg += 'values: ! %s %s %s' + args += ( lmsg, aop, rmsg ) + break + self._log_assert_failure(assertion.severity, msg, *args) return else: # all tests were successful for this assertion tag (no break) @@ -254,58 +287,89 @@ class YamlInterpreter(object): b = bool(value) else: b = default - return b - + return b + + def create_osv_memory_record(self, record, fields): + model = self.get_model(record.model) + record_dict = self._create_record(model, fields) + id_new=model.create(self.cr, self.uid, record_dict, context=self.context) + self.id_map[record.id] = int(id_new) + return record_dict + def process_record(self, node): + import osv record, fields = node.items()[0] - - self.validate_xml_id(record.id) - if self.isnoupdate(record) and self.mode != 'init': - id = self.pool.get('ir.model.data')._update_dummy(self.cr, self.uid, record.model, self.module, record.id) - # check if the resource already existed at the last update - if id: - self.id_map[record] = int(id) - return None - else: - if not self._coerce_bool(record.forcecreate): + model = self.get_model(record.model) + if isinstance(model, osv.osv.osv_memory): + record_dict=self.create_osv_memory_record(record, fields) + else: + self.validate_xml_id(record.id) + if self.isnoupdate(record) and self.mode != 'init': + id = self.pool.get('ir.model.data')._update_dummy(self.cr, 1, record.model, self.module, record.id) + # check if the resource already existed at the last update + if id: + self.id_map[record] = int(id) return None + else: + if not self._coerce_bool(record.forcecreate): + return None + + record_dict = self._create_record(model, fields) + self.logger.debug("RECORD_DICT %s" % record_dict) + #context = self.get_context(record, self.eval_context) + context = record.context #TOFIX: record.context like {'withoutemployee':True} should pass from self.eval_context. example: test_project.yml in project module + id = self.pool.get('ir.model.data')._update(self.cr, 1, record.model, \ + self.module, record_dict, record.id, noupdate=self.isnoupdate(record), mode=self.mode, context=context) + self.id_map[record.id] = int(id) + if config.get('import_partial'): + self.cr.commit() - model = self.get_model(record.model) - record_dict = self._create_record(model, fields) - self.logger.debug("RECORD_DICT %s" % record_dict) - id = self.pool.get('ir.model.data')._update(self.cr, self.uid, record.model, \ - self.module, record_dict, record.id, noupdate=self.isnoupdate(record), mode=self.mode) - self.id_map[record.id] = int(id) - if config.get('import_partial', False): - self.cr.commit() - def _create_record(self, model, fields): record_dict = {} for field_name, expression in fields.items(): field_value = self._eval_field(model, field_name, expression) record_dict[field_name] = field_value - return record_dict - + return record_dict + + def process_ref(self, node, column=None): + if node.search: + if node.model: + model_name = node.model + elif column: + model_name = column._obj + else: + raise YamlImportException('You need to give a model for the search, or a column to infer it.') + model = self.get_model(model_name) + q = eval(node.search, self.eval_context) + ids = model.search(self.cr, self.uid, q) + if node.use: + instances = model.browse(self.cr, self.uid, ids) + value = [inst[node.use] for inst in instances] + else: + value = ids + elif node.id: + value = self.get_id(node.id) + else: + value = None + return value + + def process_eval(self, node): + return eval(node.expression, self.eval_context) + def _eval_field(self, model, field_name, expression): - column = model._columns[field_name] + # TODO this should be refactored as something like model.get_field() in bin/osv + if field_name in model._columns: + column = model._columns[field_name] + elif field_name in model._inherit_fields: + column = model._inherit_fields[field_name][2] + else: + raise KeyError("Object '%s' does not contain field '%s'" % (model, field_name)) if is_ref(expression): - if expression.model: - other_model_name = expression.model - else: - other_model_name = column._obj - other_model = self.get_model(other_model_name) - if expression.search: - q = eval(expression.search, self.eval_context) - ids = other_model.search(self.cr, self.uid, q) - if expression.use: - instances = other_model.browse(self.cr, self.uid, ids) - elements = [inst[expression.use] for inst in instances] - else: - elements = ids - if column._type in ("many2many", "one2many"): - value = [(6, 0, elements)] - else: # many2one - value = self._get_first_result(elements) + elements = self.process_ref(expression, column) + if column._type in ("many2many", "one2many"): + value = [(6, 0, elements)] + else: # many2one + value = self._get_first_result(elements) elif column._type == "many2one": value = self.get_id(expression) elif column._type == "one2many": @@ -314,36 +378,46 @@ class YamlInterpreter(object): elif column._type == "many2many": ids = [self.get_id(xml_id) for xml_id in expression] value = [(6, 0, ids)] + elif column._type == "date" and is_string(expression): + # enforce ISO format for string date values, to be locale-agnostic during tests + time.strptime(expression, misc.DEFAULT_SERVER_DATE_FORMAT) + value = expression + elif column._type == "datetime" and is_string(expression): + # enforce ISO format for string datetime values, to be locale-agnostic during tests + time.strptime(expression, misc.DEFAULT_SERVER_DATETIME_FORMAT) + value = expression else: # scalar field if is_eval(expression): - value = eval(expression.expression, self.eval_context) + value = self.process_eval(expression) else: value = expression # raise YamlImportException('Unsupported column "%s" or value %s:%s' % (field_name, type(expression), expression)) return value - + def process_context(self, node): self.context = node.__dict__ if node.uid: self.uid = self.get_id(node.uid) if node.noupdate: self.noupdate = node.noupdate - + def process_python(self, node): def log(msg, *args): self.logger.log(logging.TEST, msg, *args) python, statements = node.items()[0] model = self.get_model(python.model) statements = statements.replace("\r\n", "\n") - code_context = {'self': model, 'cr': self.cr, 'uid': self.uid, 'log': log, 'context': self.context} + code_context = {'model': model, 'cr': self.cr, 'uid': self.uid, 'log': log, 'context': self.context} + code_context.update({'self': model}) # remove me when no !python block test uses 'self' anymore try: - code = compile(statements, self.filename, 'exec') - eval(code, {'ref': self.get_id}, code_context) + code_obj = compile(statements, self.filename, 'exec') + unsafe_eval(code_obj, {'ref': self.get_id}, code_context) except AssertionError, e: self._log_assert_failure(python.severity, 'AssertionError in Python code %s: %s', python.name, e) return except Exception, e: - raise YamlImportAbortion(e) + self.logger.debug('Exception during evaluation of !python block in yaml_file %s.', self.filename, exc_info=True) + raise else: self.assert_report.record(True, python.severity) @@ -370,8 +444,38 @@ class YamlInterpreter(object): uid = workflow.uid else: uid = self.uid + self.cr.execute('select distinct signal from wkf_transition') + signals=[x['signal'] for x in self.cr.dictfetchall()] + if workflow.action not in signals: + raise YamlImportException('Incorrect action %s. No such action defined' % workflow.action) wf_service = netsvc.LocalService("workflow") wf_service.trg_validate(uid, workflow.model, id, workflow.action, self.cr) + + def _eval_params(self, model, params): + args = [] + for i, param in enumerate(params): + if isinstance(param, types.ListType): + value = self._eval_params(model, param) + elif is_ref(param): + value = self.process_ref(param) + elif is_eval(param): + value = self.process_eval(param) + elif isinstance(param, types.DictionaryType): # supports XML syntax + param_model = self.get_model(param.get('model', model)) + if 'search' in param: + q = eval(param['search'], self.eval_context) + ids = param_model.search(self.cr, self.uid, q) + value = self._get_first_result(ids) + elif 'eval' in param: + local_context = {'obj': lambda x: param_model.browse(self.cr, self.uid, x, self.context)} + local_context.update(self.id_map) + value = eval(param['eval'], self.eval_context, local_context) + else: + raise YamlImportException('You must provide either a !ref or at least a "eval" or a "search" to function parameter #%d.' % i) + else: + value = param # scalar value + args.append(value) + return args def process_function(self, node): function, params = node.items()[0] @@ -379,25 +483,10 @@ class YamlInterpreter(object): return model = self.get_model(function.model) context = self.get_context(function, self.eval_context) - args = [] if function.eval: - args = eval(function.eval, self.eval_context) - for i, param in enumerate(params): - if 'model' in param: - param_model = self.get_model(param['model']) - else: - param_model = model - if 'search' in param: - q = eval(param['search'], self.eval_context) - ids = param_model.search(cr, uid, q) - value = self._get_first_result(ids) - elif 'eval' in param: - local_context = {'obj': lambda x: value_model.browse(self.cr, self.uid, x, context)} - local_context.update(self.id_map) - value = eval(param['eval'], self.eval_context, local_context) - else: - raise YamlImportException('You must provide at least a "eval" or a "search" to function parameter #%d.' % i) - args.append(value) + args = self.process_eval(function.eval) + else: + args = self._eval_params(function.model, params) method = function.name getattr(model, method)(self.cr, self.uid, *args) @@ -482,7 +571,7 @@ class YamlInterpreter(object): self._set_group_values(node, values) - pid = self.pool.get('ir.model.data')._update(self.cr, self.uid, \ + pid = self.pool.get('ir.model.data')._update(self.cr, 1, \ 'ir.ui.menu', self.module, values, node.id, mode=self.mode, \ noupdate=self.isnoupdate(node), res_id=res and res[0] or False) @@ -493,19 +582,23 @@ class YamlInterpreter(object): action_type = node.type or 'act_window' action_id = self.get_id(node.action) action = "ir.actions.%s,%d" % (action_type, action_id) - self.pool.get('ir.model.data').ir_set(self.cr, self.uid, 'action', \ + self.pool.get('ir.model.data').ir_set(self.cr, 1, 'action', \ 'tree_but_open', 'Menuitem', [('ir.ui.menu', int(parent_id))], action, True, True, xml_id=node.id) def process_act_window(self, node): + assert getattr(node, 'id'), "Attribute %s of act_window is empty !" % ('id',) + assert getattr(node, 'name'), "Attribute %s of act_window is empty !" % ('name',) + assert getattr(node, 'res_model'), "Attribute %s of act_window is empty !" % ('res_model',) self.validate_xml_id(node.id) view_id = False if node.view: view_id = self.get_id(node.view) - context = eval(node.context, self.eval_context) - + if not node.context: + node.context={} + context = eval(str(node.context), self.eval_context) values = { 'name': node.name, - 'type': type or 'ir.actions.act_window', + 'type': node.type or 'ir.actions.act_window', 'view_id': view_id, 'domain': node.domain, 'context': context, @@ -516,13 +609,14 @@ class YamlInterpreter(object): 'usage': node.usage, 'limit': node.limit, 'auto_refresh': node.auto_refresh, + 'multi': getattr(node, 'multi', False), } self._set_group_values(node, values) if node.target: values['target'] = node.target - id = self.pool.get('ir.model.data')._update(self.cr, self.uid, \ + id = self.pool.get('ir.model.data')._update(self.cr, 1, \ 'ir.actions.act_window', self.module, values, node.id, mode=self.mode) self.id_map[node.id] = int(id) @@ -530,25 +624,29 @@ class YamlInterpreter(object): keyword = 'client_action_relate' value = 'ir.actions.act_window,%s' % id replace = node.replace or True - self.pool.get('ir.model.data').ir_set(self.cr, self.uid, 'action', keyword, \ + self.pool.get('ir.model.data').ir_set(self.cr, 1, 'action', keyword, \ node.id, [node.src_model], value, replace=replace, noupdate=self.isnoupdate(node), isobject=True, xml_id=node.id) # TODO add remove ir.model.data def process_delete(self, node): - if len(node.search): - ids = self.pool.get(node.model).search(self.cr, self.uid, eval(node.search, self.eval_context)) + assert getattr(node, 'model'), "Attribute %s of delete tag is empty !" % ('model',) + if self.pool.get(node.model): + if len(node.search): + ids = self.pool.get(node.model).search(self.cr, self.uid, eval(node.search, self.eval_context)) + else: + ids = [self.get_id(node.id)] + if len(ids): + self.pool.get(node.model).unlink(self.cr, self.uid, ids) + self.pool.get('ir.model.data')._unlink(self.cr, 1, node.model, ids) else: - ids = [self.get_id(node.id)] - if len(ids): - self.pool.get(node.model).unlink(self.cr, self.uid, ids) - self.pool.get('ir.model.data')._unlink(self.cr, self.uid, node.model, ids) + self.logger.log(logging.TEST, "Record not deleted.") def process_url(self, node): self.validate_xml_id(node.id) res = {'name': node.name, 'url': node.url, 'target': node.target} - id = self.pool.get('ir.model.data')._update(self.cr, self.uid, \ + id = self.pool.get('ir.model.data')._update(self.cr, 1, \ "ir.actions.url", self.module, res, node.id, mode=self.mode) self.id_map[node.id] = int(id) # ir_set @@ -556,7 +654,7 @@ class YamlInterpreter(object): keyword = node.keyword or 'client_action_multi' value = 'ir.actions.url,%s' % id replace = node.replace or True - self.pool.get('ir.model.data').ir_set(self.cr, self.uid, 'action', \ + self.pool.get('ir.model.data').ir_set(self.cr, 1, 'action', \ keyword, node.url, ["ir.actions.url"], value, replace=replace, \ noupdate=self.isnoupdate(node), isobject=True, xml_id=node.id) @@ -566,12 +664,12 @@ class YamlInterpreter(object): _, fields = node.items()[0] res = {} for fieldname, expression in fields.items(): - if isinstance(expression, Eval): + if is_eval(expression): value = eval(expression.expression, self.eval_context) else: value = expression res[fieldname] = value - self.pool.get('ir.model.data').ir_set(self.cr, self.uid, res['key'], res['key2'], \ + self.pool.get('ir.model.data').ir_set(self.cr, 1, res['key'], res['key2'], \ res['name'], res['models'], res['value'], replace=res.get('replace',True), \ isobject=res.get('isobject', False), meta=res.get('meta',None)) @@ -580,14 +678,18 @@ class YamlInterpreter(object): for dest, f in (('name','string'), ('model','model'), ('report_name','name')): values[dest] = getattr(node, f) assert values[dest], "Attribute %s of report is empty !" % (f,) - for field,dest in (('rml','report_rml'),('xml','report_xml'),('xsl','report_xsl'),('attachment','attachment'),('attachment_use','attachment_use')): + for field,dest in (('rml','report_rml'),('file','report_rml'),('xml','report_xml'),('xsl','report_xsl'),('attachment','attachment'),('attachment_use','attachment_use')): if getattr(node, field): values[dest] = getattr(node, field) if node.auto: values['auto'] = eval(node.auto) if node.sxw: - sxw_content = misc.file_open(node.sxw).read() - values['report_sxw_content'] = sxw_content + sxw_file = misc.file_open(node.sxw) + try: + sxw_content = sxw_file.read() + values['report_sxw_content'] = sxw_content + finally: + sxw_file.close() if node.header: values['header'] = eval(node.header) values['multi'] = node.multi and eval(node.multi) @@ -596,7 +698,7 @@ class YamlInterpreter(object): self._set_group_values(node, values) - id = self.pool.get('ir.model.data')._update(self.cr, self.uid, "ir.actions.report.xml", \ + id = self.pool.get('ir.model.data')._update(self.cr, 1, "ir.actions.report.xml", \ self.module, values, xml_id, noupdate=self.isnoupdate(node), mode=self.mode) self.id_map[xml_id] = int(id) @@ -604,7 +706,7 @@ class YamlInterpreter(object): keyword = node.keyword or 'client_print_multi' value = 'ir.actions.report.xml,%s' % id replace = node.replace or True - self.pool.get('ir.model.data').ir_set(self.cr, self.uid, 'action', \ + self.pool.get('ir.model.data').ir_set(self.cr, 1, 'action', \ keyword, values['name'], [values['model']], value, replace=replace, isobject=True, xml_id=xml_id) def process_none(self): @@ -618,6 +720,8 @@ class YamlInterpreter(object): """ Processes a Yaml string. Custom tags are interpreted by 'process_' instance methods. """ + yaml_tag.add_constructors() + is_preceded_by_comment = False for node in yaml.load(yaml_string): is_preceded_by_comment = self._log(node, is_preceded_by_comment) @@ -627,7 +731,7 @@ class YamlInterpreter(object): self.logger.exception(e) except Exception, e: self.logger.exception(e) - raise e + raise def _process_node(self, node): if is_comment(node):