1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2013 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
34 import openerp.modules.db
35 import openerp.modules.graph
36 import openerp.modules.migration
37 import openerp.modules.registry
38 import openerp.osv as osv
39 import openerp.tools as tools
40 from openerp import SUPERUSER_ID
42 from openerp.tools.translate import _
43 from openerp.modules.module import initialize_sys_path, \
44 load_openerp_module, init_module_models, adapt_version
46 _logger = logging.getLogger(__name__)
47 _test_logger = logging.getLogger('openerp.tests')
50 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
51 """Migrates+Updates or Installs all module nodes from ``graph``
52 :param graph: graph of module nodes to load
53 :param status: status dictionary for keeping track of progress
54 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
56 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
57 :return: list of modules that were installed or updated
59 def process_sql_file(cr, fp):
60 queries = fp.read().split(';')
62 new_query = ' '.join(query.split())
66 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
67 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
68 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
69 load_data = lambda *args: _load_data(cr, *args, kind='data')
70 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
72 def load_test(module_name, idref, mode):
75 threading.currentThread().testing = True
76 _load_data(cr, module_name, idref, mode, 'test')
79 _test_logger.exception(
80 'module %s: an exception occurred in a test', module_name)
83 threading.currentThread().testing = False
84 if tools.config.options['test_commit']:
89 def _load_data(cr, module_name, idref, mode, kind):
92 kind: data, demo, test, init_xml, update_xml, demo_xml.
94 noupdate is False, unless it is demo data or it is csv data in
98 for filename in package.data[kind]:
100 _test_logger.info("module %s: loading %s", module_name, filename)
102 _logger.info("module %s: loading %s", module_name, filename)
103 _, ext = os.path.splitext(filename)
104 pathname = os.path.join(module_name, filename)
105 fp = tools.file_open(pathname)
107 if kind in ('demo', 'demo_xml'):
112 if kind in ('init', 'init_xml'):
114 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
116 process_sql_file(cr, fp)
118 tools.convert_yaml_import(cr, module_name, fp, kind, idref, mode, noupdate, report)
120 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
122 pass # .js files are valid but ignored here.
124 _logger.warning("Can't load unknown file type %s.", filename)
131 processed_modules = []
133 registry = openerp.registry(cr.dbname)
134 migrations = openerp.modules.migration.MigrationManager(cr, graph)
135 _logger.info('loading %d modules...', len(graph))
137 # Query manual fields for all models at once and save them on the registry
138 # so the initialization code for each model does not have to do it
139 # one model at a time.
140 registry.fields_by_model = {}
141 cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
142 for field in cr.dictfetchall():
143 registry.fields_by_model.setdefault(field['model'], []).append(field)
145 # register, instantiate and initialize models for each modules
146 for index, package in enumerate(graph):
147 module_name = package.name
148 module_id = package.id
150 if skip_modules and module_name in skip_modules:
153 _logger.debug('module %s: loading objects', package.name)
154 migrations.migrate_module(package, 'pre')
155 load_openerp_module(package.name)
157 models = registry.load(cr, package)
159 loaded_modules.append(package.name)
160 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
161 init_module_models(cr, package.name, models)
162 registry._init_modules.add(package.name)
163 status['progress'] = float(index) / len(graph)
165 # Can't put this line out of the loop: ir.module.module will be
166 # registered by init_module_models() above.
167 modobj = registry['ir.module.module']
170 modobj.check(cr, SUPERUSER_ID, [module_id])
175 if hasattr(package, 'init') or package.state == 'to install':
178 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
179 if package.state=='to upgrade':
180 # upgrading the module information
181 modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
182 load_init_xml(module_name, idref, mode)
183 load_update_xml(module_name, idref, mode)
184 load_data(module_name, idref, mode)
185 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
186 status['progress'] = (index + 0.75) / len(graph)
187 load_demo_xml(module_name, idref, mode)
188 load_demo(module_name, idref, mode)
189 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
191 # launch tests only in demo mode, as most tests will depend
192 # on demo data. Other tests can be added into the regular
193 # 'data' section, but should probably not alter the data,
194 # as there is no rollback.
195 if tools.config.options['test_enable']:
196 report.record_result(load_test(module_name, idref, mode))
198 # Run the `fast_suite` and `checks` tests given by the module.
199 if module_name == 'base':
200 # Also run the core tests after the database is created.
201 report.record_result(openerp.modules.module.run_unit_tests('openerp'))
202 report.record_result(openerp.modules.module.run_unit_tests(module_name))
204 processed_modules.append(package.name)
206 migrations.migrate_module(package, 'post')
208 ver = adapt_version(package.data['version'])
209 # Set new modules and dependencies
210 modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
211 # Update translations for all installed languages
212 modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)
214 package.state = 'installed'
215 for kind in ('init', 'demo', 'update'):
216 if hasattr(package, kind):
217 delattr(package, kind)
221 # The query won't be valid for models created later (i.e. custom model
222 # created after the registry has been loaded), so empty its result.
223 registry.fields_by_model = None
227 return loaded_modules, processed_modules
229 def _check_module_names(cr, module_names):
230 mod_names = set(module_names)
231 if 'base' in mod_names:
232 # ignore dummy 'all' module
233 if 'all' in mod_names:
234 mod_names.remove('all')
236 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
237 if cr.dictfetchone()['count'] != len(mod_names):
238 # find out what module name(s) are incorrect:
239 cr.execute("SELECT name FROM ir_module_module")
240 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
241 _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
243 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
244 """Loads modules marked with ``states``, adding them to ``graph`` and
245 ``loaded_modules`` and returns a list of installed/upgraded modules."""
246 processed_modules = []
248 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
249 module_list = [name for (name,) in cr.fetchall() if name not in graph]
250 graph.add_modules(cr, module_list, force)
251 _logger.debug('Updating graph with %d more modules', len(module_list))
252 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
253 processed_modules.extend(processed)
254 loaded_modules.extend(loaded)
255 if not processed: break
256 return processed_modules
259 def load_modules(db, force_demo=False, status=None, update_module=False):
260 # TODO status['progress'] reporting is broken: used twice (and reset each
261 # time to zero) in load_module_graph, not fine-grained enough.
262 # It should be a method exposed by the registry.
263 initialize_sys_path()
271 if not openerp.modules.db.is_initialized(cr):
272 _logger.info("init db")
273 openerp.modules.db.initialize(cr)
274 tools.config["init"]["all"] = 1
275 tools.config['update']['all'] = 1
276 if not tools.config['without_demo']:
277 tools.config["demo"]['all'] = 1
279 # This is a brand new registry, just created in
280 # openerp.modules.registry.RegistryManager.new().
281 registry = openerp.registry(cr.dbname)
283 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
284 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
286 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
287 graph = openerp.modules.graph.Graph()
288 graph.add_module(cr, 'base', force)
290 _logger.critical('module base cannot be loaded! (hint: verify addons-path)')
291 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
293 # processed_modules: for cleanup step after install
294 # loaded_modules: to avoid double loading
295 report = registry._assertion_report
296 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
298 if tools.config['load_language']:
299 for lang in tools.config['load_language'].split(','):
300 tools.load_language(cr, lang)
302 # STEP 2: Mark other modules to be loaded/updated
304 modobj = registry['ir.module.module']
305 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
306 _logger.info('updating modules list')
307 modobj.update_list(cr, SUPERUSER_ID)
309 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
311 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
313 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
315 modobj.button_install(cr, SUPERUSER_ID, ids)
317 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
319 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
321 modobj.button_upgrade(cr, SUPERUSER_ID, ids)
323 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
326 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
327 # IMPORTANT: this is done in two parts, first loading all installed or
328 # partially installed modules (i.e. installed/to upgrade), to
329 # offer a consistent system to the second part: installing
330 # newly selected modules.
331 # We include the modules 'to remove' in the first step, because
332 # they are part of the "currently installed" modules. They will
333 # be dropped in STEP 6 later, before restarting the loading
335 states_to_load = ['installed', 'to upgrade', 'to remove']
336 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
337 processed_modules.extend(processed)
339 states_to_load = ['to install']
340 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
341 processed_modules.extend(processed)
344 cr.execute('select model from ir_model where state=%s', ('manual',))
345 for model in cr.dictfetchall():
346 registry['ir.model'].instanciate(cr, SUPERUSER_ID, model['model'], {})
348 # STEP 4: Finish and cleanup installations
349 if processed_modules:
350 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
351 for (model, name) in cr.fetchall():
352 if model in registry and not registry[model].is_transient():
353 _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,1,1,1',
354 model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
356 # Temporary warning while we remove access rights on osv_memory objects, as they have
357 # been replaced by owner-only access rights
358 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
359 for (model, name) in cr.fetchall():
360 if model in registry and registry[model].is_transient():
361 _logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
363 cr.execute("SELECT model from ir_model")
364 for (model,) in cr.fetchall():
365 if model in registry:
366 registry[model]._check_removed_columns(cr, log=True)
368 _logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
370 # Cleanup orphan records
371 registry['ir.model.data']._process_end(cr, SUPERUSER_ID, processed_modules)
373 for kind in ('init', 'demo', 'update'):
374 tools.config[kind] = {}
378 # STEP 5: Cleanup menus
379 # Remove menu items that are not referenced by any of other
380 # (child) menu item, ir_values, or ir_model_data.
381 # TODO: This code could be a method of ir_ui_menu. Remove menu without actions of children
384 cr.execute('''delete from
387 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
389 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
391 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
396 _logger.info('removed %d unused menus', cr.rowcount)
398 # STEP 6: Uninstall modules to remove
400 # Remove records referenced from ir_model_data for modules to be
401 # removed (and removed the references from ir_model_data).
402 cr.execute("SELECT id FROM ir_module_module WHERE state=%s", ('to remove',))
403 mod_ids_to_remove = [x[0] for x in cr.fetchall()]
404 if mod_ids_to_remove:
405 registry['ir.module.module'].module_uninstall(cr, SUPERUSER_ID, mod_ids_to_remove)
406 # Recursive reload, should only happen once, because there should be no
407 # modules to remove next time
409 _logger.info('Reloading registry once more after uninstalling modules')
410 return openerp.modules.registry.RegistryManager.new(cr.dbname, force_demo, status, update_module)
413 _logger.error('At least one test failed when loading the modules.')
415 _logger.info('Modules loaded.')
417 # STEP 7: call _register_hook on every model
418 for model in registry.models.values():
419 model._register_hook(cr)
425 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: