1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2013 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
34 import openerp.modules.db
35 import openerp.modules.graph
36 import openerp.modules.migration
37 import openerp.modules.registry
38 import openerp.osv as osv
39 import openerp.tools as tools
40 from openerp import SUPERUSER_ID
42 from openerp.tools.translate import _
43 from openerp.modules.module import initialize_sys_path, \
44 load_openerp_module, init_module_models, adapt_version
46 _logger = logging.getLogger(__name__)
47 _test_logger = logging.getLogger('openerp.tests')
50 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
51 """Migrates+Updates or Installs all module nodes from ``graph``
52 :param graph: graph of module nodes to load
53 :param status: status dictionary for keeping track of progress
54 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
56 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
57 :return: list of modules that were installed or updated
59 def load_test(module_name, idref, mode):
62 threading.currentThread().testing = True
63 _load_data(cr, module_name, idref, mode, 'test')
66 _test_logger.exception(
67 'module %s: an exception occurred in a test', module_name)
70 threading.currentThread().testing = False
71 if tools.config.options['test_commit']:
76 def _get_files_of_kind(kind):
78 kind = ['demo_xml', 'demo']
80 kind = ['init_xml', 'update_xml', 'data']
81 if isinstance(kind, str):
85 for f in package.data[k]:
87 if k.endswith('_xml') and not (k == 'init_xml' and not f.endswith('.xml')):
88 # init_xml, update_xml and demo_xml are deprecated except
89 # for the case of init_xml with yaml, csv and sql files as
90 # we can't specify noupdate for those file.
91 correct_key = 'demo' if k.count('demo') else 'data'
93 "module %s: key '%s' is deprecated in favor of '%s' for file '%s'.",
94 package.name, k, correct_key, f
98 def _load_data(cr, module_name, idref, mode, kind):
101 kind: data, demo, test, init_xml, update_xml, demo_xml.
103 noupdate is False, unless it is demo data or it is csv data in
107 for filename in _get_files_of_kind(kind):
108 _logger.info("module %s: loading %s", module_name, filename)
110 if kind in ('demo', 'demo_xml') or (filename.endswith('.csv') and kind in ('init', 'init_xml')):
112 tools.convert_file(cr, module_name, filename, idref, mode, noupdate, kind, report)
117 processed_modules = []
119 registry = openerp.registry(cr.dbname)
120 migrations = openerp.modules.migration.MigrationManager(cr, graph)
121 _logger.info('loading %d modules...', len(graph))
123 # Query manual fields for all models at once and save them on the registry
124 # so the initialization code for each model does not have to do it
125 # one model at a time.
126 registry.fields_by_model = {}
127 cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
128 for field in cr.dictfetchall():
129 registry.fields_by_model.setdefault(field['model'], []).append(field)
131 # register, instantiate and initialize models for each modules
132 for index, package in enumerate(graph):
133 module_name = package.name
134 module_id = package.id
136 if skip_modules and module_name in skip_modules:
139 _logger.debug('module %s: loading objects', package.name)
140 migrations.migrate_module(package, 'pre')
141 load_openerp_module(package.name)
143 models = registry.load(cr, package)
145 loaded_modules.append(package.name)
146 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
147 init_module_models(cr, package.name, models)
148 registry._init_modules.add(package.name)
149 status['progress'] = float(index) / len(graph)
151 # Can't put this line out of the loop: ir.module.module will be
152 # registered by init_module_models() above.
153 modobj = registry['ir.module.module']
156 modobj.check(cr, SUPERUSER_ID, [module_id])
161 if hasattr(package, 'init') or package.state == 'to install':
164 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
165 if package.state=='to upgrade':
166 # upgrading the module information
167 modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
168 _load_data(cr, module_name, idref, mode, kind='data')
169 has_demo = hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed')
171 status['progress'] = (index + 0.75) / len(graph)
172 _load_data(cr, module_name, idref, mode, kind='demo')
173 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
175 migrations.migrate_module(package, 'post')
178 # launch tests only in demo mode, as most tests will depend
179 # on demo data. Other tests can be added into the regular
180 # 'data' section, but should probably not alter the data,
181 # as there is no rollback.
182 if tools.config.options['test_enable']:
183 report.record_result(load_test(module_name, idref, mode))
185 # Run the `fast_suite` and `checks` tests given by the module.
186 if module_name == 'base':
187 # Also run the core tests after the database is created.
188 report.record_result(openerp.modules.module.run_unit_tests('openerp'))
189 report.record_result(openerp.modules.module.run_unit_tests(module_name))
191 processed_modules.append(package.name)
193 ver = adapt_version(package.data['version'])
194 # Set new modules and dependencies
195 modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
196 # Update translations for all installed languages
197 modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)
199 package.state = 'installed'
200 for kind in ('init', 'demo', 'update'):
201 if hasattr(package, kind):
202 delattr(package, kind)
206 # The query won't be valid for models created later (i.e. custom model
207 # created after the registry has been loaded), so empty its result.
208 registry.fields_by_model = None
212 return loaded_modules, processed_modules
214 def _check_module_names(cr, module_names):
215 mod_names = set(module_names)
216 if 'base' in mod_names:
217 # ignore dummy 'all' module
218 if 'all' in mod_names:
219 mod_names.remove('all')
221 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
222 if cr.dictfetchone()['count'] != len(mod_names):
223 # find out what module name(s) are incorrect:
224 cr.execute("SELECT name FROM ir_module_module")
225 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
226 _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
228 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
229 """Loads modules marked with ``states``, adding them to ``graph`` and
230 ``loaded_modules`` and returns a list of installed/upgraded modules."""
231 processed_modules = []
233 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
234 module_list = [name for (name,) in cr.fetchall() if name not in graph]
235 graph.add_modules(cr, module_list, force)
236 _logger.debug('Updating graph with %d more modules', len(module_list))
237 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
238 processed_modules.extend(processed)
239 loaded_modules.extend(loaded)
240 if not processed: break
241 return processed_modules
244 def load_modules(db, force_demo=False, status=None, update_module=False):
245 # TODO status['progress'] reporting is broken: used twice (and reset each
246 # time to zero) in load_module_graph, not fine-grained enough.
247 # It should be a method exposed by the registry.
248 initialize_sys_path()
256 if not openerp.modules.db.is_initialized(cr):
257 _logger.info("init db")
258 openerp.modules.db.initialize(cr)
259 tools.config["init"]["all"] = 1
260 tools.config['update']['all'] = 1
261 if not tools.config['without_demo']:
262 tools.config["demo"]['all'] = 1
264 # This is a brand new registry, just created in
265 # openerp.modules.registry.RegistryManager.new().
266 registry = openerp.registry(cr.dbname)
268 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
269 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
271 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
272 graph = openerp.modules.graph.Graph()
273 graph.add_module(cr, 'base', force)
275 _logger.critical('module base cannot be loaded! (hint: verify addons-path)')
276 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
278 # processed_modules: for cleanup step after install
279 # loaded_modules: to avoid double loading
280 report = registry._assertion_report
281 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
283 if tools.config['load_language']:
284 for lang in tools.config['load_language'].split(','):
285 tools.load_language(cr, lang)
287 # STEP 2: Mark other modules to be loaded/updated
289 modobj = registry['ir.module.module']
290 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
291 _logger.info('updating modules list')
292 modobj.update_list(cr, SUPERUSER_ID)
294 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
296 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
298 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
300 modobj.button_install(cr, SUPERUSER_ID, ids)
302 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
304 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
306 modobj.button_upgrade(cr, SUPERUSER_ID, ids)
308 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
311 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
312 # IMPORTANT: this is done in two parts, first loading all installed or
313 # partially installed modules (i.e. installed/to upgrade), to
314 # offer a consistent system to the second part: installing
315 # newly selected modules.
316 # We include the modules 'to remove' in the first step, because
317 # they are part of the "currently installed" modules. They will
318 # be dropped in STEP 6 later, before restarting the loading
320 # IMPORTANT 2: We have to loop here until all relevant modules have been
321 # processed, because in some rare cases the dependencies have
322 # changed, and modules that depend on an uninstalled module
323 # will not be processed on the first pass.
324 # It's especially useful for migrations.
325 previously_processed = -1
326 while previously_processed < len(processed_modules):
327 previously_processed = len(processed_modules)
328 processed_modules += load_marked_modules(cr, graph,
329 ['installed', 'to upgrade', 'to remove'],
330 force, status, report, loaded_modules, update_module)
332 processed_modules += load_marked_modules(cr, graph,
333 ['to install'], force, status, report,
334 loaded_modules, update_module)
337 cr.execute('select model from ir_model where state=%s', ('manual',))
338 for model in cr.dictfetchall():
339 registry['ir.model'].instanciate(cr, SUPERUSER_ID, model['model'], {})
341 # STEP 4: Finish and cleanup installations
342 if processed_modules:
343 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
344 for (model, name) in cr.fetchall():
345 if model in registry and not registry[model].is_transient() and not isinstance(registry[model], openerp.osv.orm.AbstractModel):
346 _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,1,1,1',
347 model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
349 # Temporary warning while we remove access rights on osv_memory objects, as they have
350 # been replaced by owner-only access rights
351 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
352 for (model, name) in cr.fetchall():
353 if model in registry and registry[model].is_transient():
354 _logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
356 cr.execute("SELECT model from ir_model")
357 for (model,) in cr.fetchall():
358 if model in registry:
359 registry[model]._check_removed_columns(cr, log=True)
361 _logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
363 # Cleanup orphan records
364 registry['ir.model.data']._process_end(cr, SUPERUSER_ID, processed_modules)
366 for kind in ('init', 'demo', 'update'):
367 tools.config[kind] = {}
371 # STEP 5: Cleanup menus
372 # Remove menu items that are not referenced by any of other
373 # (child) menu item, ir_values, or ir_model_data.
374 # TODO: This code could be a method of ir_ui_menu. Remove menu without actions of children
377 cr.execute('''delete from
380 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
382 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
384 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
389 _logger.info('removed %d unused menus', cr.rowcount)
391 # STEP 6: Uninstall modules to remove
393 # Remove records referenced from ir_model_data for modules to be
394 # removed (and removed the references from ir_model_data).
395 cr.execute("SELECT id FROM ir_module_module WHERE state=%s", ('to remove',))
396 mod_ids_to_remove = [x[0] for x in cr.fetchall()]
397 if mod_ids_to_remove:
398 registry['ir.module.module'].module_uninstall(cr, SUPERUSER_ID, mod_ids_to_remove)
399 # Recursive reload, should only happen once, because there should be no
400 # modules to remove next time
402 _logger.info('Reloading registry once more after uninstalling modules')
403 return openerp.modules.registry.RegistryManager.new(cr.dbname, force_demo, status, update_module)
405 # STEP 7: verify custom views on every model
407 Views = registry['ir.ui.view']
408 custom_view_test = True
409 for model in registry.models.keys():
410 if not Views._validate_custom_views(cr, SUPERUSER_ID, model):
411 custom_view_test = False
412 _logger.error('invalid custom view(s) for model %s', model)
413 report.record_result(custom_view_test)
416 _logger.error('At least one test failed when loading the modules.')
418 _logger.info('Modules loaded.')
420 # STEP 8: call _register_hook on every model
421 for model in registry.models.values():
422 model._register_hook(cr)
428 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: