1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2012 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
34 import openerp.modules.db
35 import openerp.modules.graph
36 import openerp.modules.migration
37 import openerp.osv as osv
38 import openerp.pooler as pooler
39 import openerp.release as release
40 import openerp.tools as tools
41 from openerp import SUPERUSER_ID
43 from openerp import SUPERUSER_ID
44 from openerp.tools.translate import _
45 from openerp.modules.module import initialize_sys_path, \
46 load_openerp_module, init_module_models
48 _logger = logging.getLogger(__name__)
50 def open_openerp_namespace():
51 # See comment for open_openerp_namespace.
52 if openerp.conf.deprecation.open_openerp_namespace:
53 for k, v in list(sys.modules.items()):
54 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
55 sys.modules[k[8:]] = v
58 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
59 """Migrates+Updates or Installs all module nodes from ``graph``
60 :param graph: graph of module nodes to load
61 :param status: status dictionary for keeping track of progress
62 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
64 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
65 :return: list of modules that were installed or updated
67 def process_sql_file(cr, fp):
68 queries = fp.read().split(';')
70 new_query = ' '.join(query.split())
74 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
75 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
76 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
77 load_data = lambda *args: _load_data(cr, *args, kind='data')
78 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
80 def load_test(module_name, idref, mode):
83 threading.currentThread().testing = True
84 _load_data(cr, module_name, idref, mode, 'test')
88 'module %s: an exception occurred in a test', module_name)
91 threading.currentThread().testing = False
92 if tools.config.options['test_commit']:
97 def _load_data(cr, module_name, idref, mode, kind):
100 kind: data, demo, test, init_xml, update_xml, demo_xml.
102 noupdate is False, unless it is demo data or it is csv data in
106 for filename in package.data[kind]:
108 _logger.log(logging.TEST, "module %s: loading %s", module_name, filename)
110 _logger.info("module %s: loading %s", module_name, filename)
111 _, ext = os.path.splitext(filename)
112 pathname = os.path.join(module_name, filename)
113 fp = tools.file_open(pathname)
115 if kind in ('demo', 'demo_xml'):
120 if kind in ('init', 'init_xml'):
122 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
124 process_sql_file(cr, fp)
126 tools.convert_yaml_import(cr, module_name, fp, kind, idref, mode, noupdate, report)
128 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
130 pass # .js files are valid but ignored here.
132 _logger.warning("Can't load unknown file type %s.", filename)
139 processed_modules = []
141 pool = pooler.get_pool(cr.dbname)
142 migrations = openerp.modules.migration.MigrationManager(cr, graph)
143 _logger.debug('loading %d packages...', len(graph))
145 # Query manual fields for all models at once and save them on the registry
146 # so the initialization code for each model does not have to do it
147 # one model at a time.
148 pool.fields_by_model = {}
149 cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
150 for field in cr.dictfetchall():
151 pool.fields_by_model.setdefault(field['model'], []).append(field)
153 # register, instantiate and initialize models for each modules
154 for index, package in enumerate(graph):
155 module_name = package.name
156 module_id = package.id
158 if skip_modules and module_name in skip_modules:
161 _logger.info('module %s: loading objects', package.name)
162 migrations.migrate_module(package, 'pre')
163 load_openerp_module(package.name)
165 models = pool.load(cr, package)
167 loaded_modules.append(package.name)
168 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
169 init_module_models(cr, package.name, models)
170 pool._init_modules.add(package.name)
171 status['progress'] = float(index) / len(graph)
173 # Can't put this line out of the loop: ir.module.module will be
174 # registered by init_module_models() above.
175 modobj = pool.get('ir.module.module')
178 modobj.check(cr, SUPERUSER_ID, [module_id])
183 if hasattr(package, 'init') or package.state == 'to install':
186 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
187 if package.state=='to upgrade':
188 # upgrading the module information
189 modobj.write(cr, SUPERUSER_ID, [module_id], modobj.get_values_from_terp(package.data))
190 load_init_xml(module_name, idref, mode)
191 load_update_xml(module_name, idref, mode)
192 load_data(module_name, idref, mode)
193 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
194 status['progress'] = (index + 0.75) / len(graph)
195 load_demo_xml(module_name, idref, mode)
196 load_demo(module_name, idref, mode)
197 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
199 # launch tests only in demo mode, as most tests will depend
200 # on demo data. Other tests can be added into the regular
201 # 'data' section, but should probably not alter the data,
202 # as there is no rollback.
203 if tools.config.options['test_enable']:
204 report.record_result(load_test(module_name, idref, mode))
206 # Run the `fast_suite` and `checks` tests given by the module.
207 if module_name == 'base':
208 # Also run the core tests after the database is created.
209 report.record_result(openerp.modules.module.run_unit_tests('openerp'))
210 report.record_result(openerp.modules.module.run_unit_tests(module_name))
212 processed_modules.append(package.name)
214 migrations.migrate_module(package, 'post')
216 ver = release.major_version + '.' + package.data['version']
217 # Set new modules and dependencies
218 modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
219 # Update translations for all installed languages
220 modobj.update_translations(cr, SUPERUSER_ID, [module_id], None)
222 package.state = 'installed'
223 for kind in ('init', 'demo', 'update'):
224 if hasattr(package, kind):
225 delattr(package, kind)
229 # The query won't be valid for models created later (i.e. custom model
230 # created after the registry has been loaded), so empty its result.
231 pool.fields_by_model = None
235 return loaded_modules, processed_modules
237 def _check_module_names(cr, module_names):
238 mod_names = set(module_names)
239 if 'base' in mod_names:
240 # ignore dummy 'all' module
241 if 'all' in mod_names:
242 mod_names.remove('all')
244 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
245 if cr.dictfetchone()['count'] != len(mod_names):
246 # find out what module name(s) are incorrect:
247 cr.execute("SELECT name FROM ir_module_module")
248 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
249 _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
251 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
252 """Loads modules marked with ``states``, adding them to ``graph`` and
253 ``loaded_modules`` and returns a list of installed/upgraded modules."""
254 processed_modules = []
256 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
257 module_list = [name for (name,) in cr.fetchall() if name not in graph]
258 graph.add_modules(cr, module_list, force)
259 _logger.debug('Updating graph with %d more modules', len(module_list))
260 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
261 processed_modules.extend(processed)
262 loaded_modules.extend(loaded)
263 if not processed: break
264 return processed_modules
267 def load_modules(db, force_demo=False, status=None, update_module=False):
268 # TODO status['progress'] reporting is broken: used twice (and reset each
269 # time to zero) in load_module_graph, not fine-grained enough.
270 # It should be a method exposed by the pool.
271 initialize_sys_path()
273 open_openerp_namespace()
281 if not openerp.modules.db.is_initialized(cr):
282 _logger.info("init db")
283 openerp.modules.db.initialize(cr)
284 tools.config["init"]["all"] = 1
285 tools.config['update']['all'] = 1
286 if not tools.config['without_demo']:
287 tools.config["demo"]['all'] = 1
289 # This is a brand new pool, just created in pooler.get_db_and_pool()
290 pool = pooler.get_pool(cr.dbname)
292 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
293 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
295 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
296 graph = openerp.modules.graph.Graph()
297 graph.add_module(cr, 'base', force)
299 _logger.critical('module base cannot be loaded! (hint: verify addons-path)')
300 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
302 # processed_modules: for cleanup step after install
303 # loaded_modules: to avoid double loading
304 report = pool._assertion_report
305 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
307 if tools.config['load_language']:
308 for lang in tools.config['load_language'].split(','):
309 tools.load_language(cr, lang)
311 # STEP 2: Mark other modules to be loaded/updated
313 modobj = pool.get('ir.module.module')
314 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
315 _logger.info('updating modules list')
316 modobj.update_list(cr, SUPERUSER_ID)
318 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
320 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
322 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
324 modobj.button_install(cr, SUPERUSER_ID, ids)
326 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
328 ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
330 modobj.button_upgrade(cr, SUPERUSER_ID, ids)
332 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
335 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
336 # IMPORTANT: this is done in two parts, first loading all installed or
337 # partially installed modules (i.e. installed/to upgrade), to
338 # offer a consistent system to the second part: installing
339 # newly selected modules.
340 # We include the modules 'to remove' in the first step, because
341 # they are part of the "currently installed" modules. They will
342 # be dropped in STEP 6 later, before restarting the loading
344 states_to_load = ['installed', 'to upgrade', 'to remove']
345 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
346 processed_modules.extend(processed)
348 states_to_load = ['to install']
349 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
350 processed_modules.extend(processed)
353 cr.execute('select model from ir_model where state=%s', ('manual',))
354 for model in cr.dictfetchall():
355 pool.get('ir.model').instanciate(cr, SUPERUSER_ID, model['model'], {})
357 # STEP 4: Finish and cleanup installations
358 if processed_modules:
359 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
360 for (model, name) in cr.fetchall():
361 model_obj = pool.get(model)
362 if model_obj and not model_obj.is_transient():
363 _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,,1,1,1,1',
364 model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
366 # Temporary warning while we remove access rights on osv_memory objects, as they have
367 # been replaced by owner-only access rights
368 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
369 for (model, name) in cr.fetchall():
370 model_obj = pool.get(model)
371 if model_obj and model_obj.is_transient():
372 _logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
374 cr.execute("SELECT model from ir_model")
375 for (model,) in cr.fetchall():
376 obj = pool.get(model)
378 obj._check_removed_columns(cr, log=True)
380 _logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
382 # Cleanup orphan records
383 pool.get('ir.model.data')._process_end(cr, SUPERUSER_ID, processed_modules)
385 for kind in ('init', 'demo', 'update'):
386 tools.config[kind] = {}
390 # STEP 5: Cleanup menus
391 # Remove menu items that are not referenced by any of other
392 # (child) menu item, ir_values, or ir_model_data.
393 # TODO: This code could be a method of ir_ui_menu. Remove menu without actions of children
396 cr.execute('''delete from
399 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
401 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
403 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
408 _logger.info('removed %d unused menus', cr.rowcount)
410 # STEP 6: Uninstall modules to remove
412 # Remove records referenced from ir_model_data for modules to be
413 # removed (and removed the references from ir_model_data).
414 cr.execute("SELECT id FROM ir_module_module WHERE state=%s", ('to remove',))
415 mod_ids_to_remove = [x[0] for x in cr.fetchall()]
416 if mod_ids_to_remove:
417 pool.get('ir.module.module').module_uninstall(cr, SUPERUSER_ID, mod_ids_to_remove)
418 # Recursive reload, should only happen once, because there should be no
419 # modules to remove next time
421 _logger.info('Reloading registry once more after uninstalling modules')
422 return pooler.restart_pool(cr.dbname, force_demo, status, update_module)
425 _logger.error('At least one test failed when loading the modules.')
427 _logger.info('Modules loaded.')
432 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: