X-Git-Url: http://git.inspyration.org/?a=blobdiff_plain;f=openerp%2Fmodules%2Floading.py;h=5d274f6614bc47377ff498d304a6860b568d3626;hb=389257c0090a9bcd00ff22079e9fff2b32b2f59a;hp=c37016628a9874ba126fb2d9f8cc3a113d8471c2;hpb=40bce2fead943e85d6318098845e545ed8cd5427;p=odoo%2Fodoo.git diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index c370166..5d274f6 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -24,36 +24,35 @@ """ -import os, sys, imp -from os.path import join as opj +import base64 +import imp import itertools -import zipimport - -import openerp - -import openerp.osv as osv -import openerp.tools as tools -import openerp.tools.osutil as osutil -from openerp.tools.safe_eval import safe_eval as eval -import openerp.pooler as pooler -from openerp.tools.translate import _ - -import openerp.netsvc as netsvc - +import logging +import os +import re +import sys +import threading import zipfile -import openerp.release as release +import zipimport -import re -import base64 -from zipfile import PyZipFile, ZIP_DEFLATED from cStringIO import StringIO +from os.path import join as opj +from zipfile import PyZipFile, ZIP_DEFLATED -import logging +import openerp import openerp.modules.db import openerp.modules.graph import openerp.modules.migration +import openerp.netsvc as netsvc +import openerp.osv as osv +import openerp.pooler as pooler +import openerp.release as release +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +from openerp.tools.translate import _ from openerp.modules.module import \ get_modules, get_modules_with_version, \ load_information_from_description_file, \ @@ -81,6 +80,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ + logger = logging.getLogger('init.load') def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: @@ -88,29 +88,23 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= if new_query: cr.execute(new_query) - def load_init_xml(cr, m, idref, mode): - _load_data(cr, m, idref, mode, 'init_xml') - - def load_update_xml(cr, m, idref, mode): - _load_data(cr, m, idref, mode, 'update_xml') - - def load_demo_xml(cr, m, idref, mode): - _load_data(cr, m, idref, mode, 'demo_xml') - - def load_data(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'data') - - def load_demo(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'demo') + load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml') + load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml') + load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml') + load_data = lambda *args: _load_data(cr, *args, kind='data') + load_demo = lambda *args: _load_data(cr, *args, kind='demo') - def load_test(cr, module_name, idref, mode): + def load_test(module_name, idref, mode): cr.commit() if not tools.config.options['test_disable']: try: + threading.currentThread().testing = True _load_data(cr, module_name, idref, mode, 'test') except Exception, e: - logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) + logging.getLogger('init.test').exception( + 'Tests failed to execute in module %s', module_name) finally: + threading.currentThread().testing = False if tools.config.options['test_commit']: cr.commit() else: @@ -126,8 +120,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= """ for filename in package.data[kind]: - log = logging.getLogger('init') - log.info("module %s: loading %s", module_name, filename) + logger.info("module %s: loading %s", module_name, filename) _, ext = os.path.splitext(filename) pathname = os.path.join(module_name, filename) fp = tools.file_open(pathname) @@ -152,61 +145,65 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= status = {} processed_modules = [] - statusi = 0 + loaded_modules = [] pool = pooler.get_pool(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) + logger.debug('loading %d packages...', len(graph)) - # register, instanciate and initialize models for each modules - for package in graph: - if skip_modules and package.name in skip_modules: + # get db timestamp + cr.execute("select now()::timestamp") + dt_before_load = cr.fetchone()[0] + + # register, instantiate and initialize models for each modules + for index, package in enumerate(graph): + module_name = package.name + module_id = package.id + + if skip_modules and module_name in skip_modules: continue - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) + + logger.info('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') register_module_classes(package.name) - models = pool.instanciate(package.name, cr) - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + models = pool.load(cr, package) + loaded_modules.append(package.name) + if package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) - cr.commit() - # load data for each modules - modobj = pool.get('ir.module.module') - for package in graph: - status['progress'] = (float(statusi)+0.1) / len(graph) - m = package.name - mid = package.id + status['progress'] = float(index) / len(graph) - if skip_modules and m in skip_modules: - continue + # Can't put this line out of the loop: ir.module.module will be + # registered by init_module_models() above. + modobj = pool.get('ir.module.module') if perform_checks: - modobj.check(cr, 1, [mid]) + modobj.check(cr, 1, [module_id]) idref = {} - status['progress'] = (float(statusi)+0.4) / len(graph) - mode = 'update' - if hasattr(package, 'init') or package.state == 'to install': + if package.state == 'to install': mode = 'init' + else: + mode = 'update' - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + if package.state in ('to install', 'to upgrade'): if package.state=='to upgrade': # upgrading the module information - modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data)) - load_init_xml(cr, m, idref, mode) - load_update_xml(cr, m, idref, mode) - load_data(cr, m, idref, mode) - if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): - status['progress'] = (float(statusi)+0.75) / len(graph) - load_demo_xml(cr, m, idref, mode) - load_demo(cr, m, idref, mode) - cr.execute('update ir_module_module set demo=%s where id=%s', (True, mid)) + modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data)) + load_init_xml(module_name, idref, mode) + load_update_xml(module_name, idref, mode) + load_data(module_name, idref, mode) + if package.dbdemo and package.state != 'installed': + status['progress'] = (index + 0.75) / len(graph) + load_demo_xml(module_name, idref, mode) + load_demo(module_name, idref, mode) + cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) # launch tests only in demo mode, as most tests will depend # on demo data. Other tests can be added into the regular # 'data' section, but should probably not alter the data, # as there is no rollback. - load_test(cr, m, idref, mode) + load_test(module_name, idref, mode) processed_modules.append(package.name) @@ -214,22 +211,20 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= ver = release.major_version + '.' + package.data['version'] # Set new modules and dependencies - modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver}) - cr.commit() + modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver}) # Update translations for all installed languages - modobj.update_translations(cr, 1, [mid], None) - cr.commit() + modobj.update_translations(cr, 1, [module_id], None) package.state = 'installed' - for kind in ('init', 'demo', 'update'): - if hasattr(package, kind): - delattr(package, kind) - statusi += 1 + cr.commit() + + # mark new res_log records as read + cr.execute("update res_log set read=True where create_date >= %s", (dt_before_load,)) cr.commit() - return processed_modules + return loaded_modules, processed_modules def _check_module_names(cr, module_names): mod_names = set(module_names) @@ -245,97 +240,106 @@ def _check_module_names(cr, module_names): incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) +def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules): + """Loads modules marked with ``states``, adding them to ``graph`` and + ``loaded_modules`` and returns a list of installed/upgraded modules.""" + processed_modules = [] + while True: + cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),)) + module_list = [name for (name,) in cr.fetchall() if name not in graph] + new_modules_in_graph = graph.add_modules(cr, module_list, force) + logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) + loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules) + processed_modules.extend(processed) + loaded_modules.extend(loaded) + if not processed: break + return processed_modules + + def load_modules(db, force_demo=False, status=None, update_module=False): # TODO status['progress'] reporting is broken: used twice (and reset each # time to zero) in load_module_graph, not fine-grained enough. # It should be a method exposed by the pool. - initialize_sys_path() open_openerp_namespace() - cr = db.cursor() - if cr: - cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'") - if len(cr.fetchall())==0: - logger.notifyChannel("init", netsvc.LOG_INFO, "init db") - openerp.modules.db.initialize(cr) - tools.config["init"]["all"] = 1 - tools.config['update']['all'] = 1 - if not tools.config['without_demo']: - tools.config["demo"]['all'] = 1 force = [] if force_demo: force.append('demo') - # This is a brand new pool, just created in pooler.get_db_and_pool() - pool = pooler.get_pool(cr.dbname) - + cr = db.cursor() try: - processed_modules = [] + if not openerp.modules.db.is_initialized(cr): + logger.notifyChannel("init", netsvc.LOG_INFO, "init db") + openerp.modules.db.initialize(cr) + update_module = True + + # This is a brand new pool, just created in pooler.get_db_and_pool() + pool = pooler.get_pool(cr.dbname) + report = tools.assertion_report() - # NOTE: Try to also load the modules that have been marked as uninstallable previously... - STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] if 'base' in tools.config['update'] or 'all' in tools.config['update']: cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) graph = openerp.modules.graph.Graph() - graph.add_module(cr, 'base', force) + graph.add_module(cr, 'base', force_demo) if not graph: logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) - processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) + + # processed_modules: for cleanup step after install + # loaded_modules: to avoid double loading + # After load_module_graph(), 'base' has been installed or updated and its state is 'installed'. + loaded_modules, processed_modules = load_module_graph(cr, graph, status, report=report) if tools.config['load_language']: for lang in tools.config['load_language'].split(','): tools.load_language(cr, lang) # STEP 2: Mark other modules to be loaded/updated + # This is a one-shot use of tools.config[init|update] from the command line + # arguments. It is directly cleared to not interfer with later create/update + # issued via RPC. if update_module: modobj = pool.get('ir.module.module') - if ('base' in tools.config['init']) or ('base' in tools.config['update']): + if ('base' in tools.config['init']) or ('base' in tools.config['update']) \ + or ('all' in tools.config['init']) or ('all' in tools.config['update']): logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') modobj.update_list(cr, 1) - _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) + if 'all' in tools.config['init']: + ids = modobj.search(cr, 1, []) + tools.config['init'] = dict.fromkeys([m['name'] for m in modobj.read(cr, 1, ids, ['name'])], 1) - mods = [k for k in tools.config['init'] if tools.config['init'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) - if ids: - modobj.button_install(cr, 1, ids) - - mods = [k for k in tools.config['update'] if tools.config['update'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) - if ids: - modobj.button_upgrade(cr, 1, ids) + _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) - cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) + mods = [k for k in tools.config['init'] if tools.config['init'][k] and k not in ('base', 'all')] + ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) + if ids: + modobj.button_install(cr, 1, ids) # goes from 'uninstalled' to 'to install' - STATES_TO_LOAD += ['to install'] + mods = [k for k in tools.config['update'] if tools.config['update'][k] and k not in ('base', 'all')] + ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) + if ids: + modobj.button_upgrade(cr, 1, ids) # goes from 'installed' to 'to upgrade' + for kind in ('init', 'demo', 'update'): + tools.config[kind] = {} # STEP 3: Load marked modules (skipping base which was done in STEP 1) - loop_guardrail = 0 - while True: - loop_guardrail += 1 - if loop_guardrail > 100: - raise ValueError('Possible recursive module tree detected, aborting.') - cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(STATES_TO_LOAD),)) - - module_list = [name for (name,) in cr.fetchall() if name not in graph] - if not module_list: - break - - new_modules_in_graph = graph.add_modules(cr, module_list, force) - if new_modules_in_graph == 0: - # nothing to load - break - - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) - processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) + # IMPORTANT: this is done in two parts, first loading all installed or + # partially installed modules (i.e. installed/to upgrade), to + # offer a consistent system to the second part: installing + # newly selected modules. + states_to_load = ['installed', 'to upgrade'] + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed_modules.extend(processed) + if update_module: + states_to_load = ['to install'] + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed_modules.extend(processed) # load custom models cr.execute('select model from ir_model where state=%s', ('manual',)) @@ -347,16 +351,16 @@ def load_modules(db, force_demo=False, status=None, update_module=False): cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") for (model, name) in cr.fetchall(): model_obj = pool.get(model) - if model_obj and not isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name)) + if model_obj and not model_obj.is_transient(): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'Model %s (%s) has no access rules!' % (model, name)) # Temporary warning while we remove access rights on osv_memory objects, as they have # been replaced by owner-only access rights cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""") for (model, name) in cr.fetchall(): model_obj = pool.get(model) - if isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name)) + if model_obj and model_obj.is_transient(): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'The transient model %s (%s) should not have explicit access rules!' % (model, name)) cr.execute("SELECT model from ir_model") for (model,) in cr.fetchall(): @@ -364,7 +368,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False): if obj: obj._check_removed_columns(cr, log=True) else: - logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) + logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)" % model) # Cleanup orphan records pool.get('ir.model.data')._process_end(cr, 1, processed_modules) @@ -372,9 +376,6 @@ def load_modules(db, force_demo=False, status=None, update_module=False): if report.get_report(): logger.notifyChannel('init', netsvc.LOG_INFO, report) - for kind in ('init', 'demo', 'update'): - tools.config[kind] = {} - cr.commit() if update_module: # Remove records referenced from ir_model_data for modules to be