1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
38 from cStringIO import StringIO
39 from os.path import join as opj
40 from zipfile import PyZipFile, ZIP_DEFLATED
44 import openerp.modules.db
45 import openerp.modules.graph
46 import openerp.modules.migration
47 import openerp.netsvc as netsvc
48 import openerp.osv as osv
49 import openerp.pooler as pooler
50 import openerp.release as release
51 import openerp.tools as tools
52 import openerp.tools.osutil as osutil
54 from openerp.tools.safe_eval import safe_eval as eval
55 from openerp.tools.translate import _
56 from openerp.modules.module import \
57 get_modules, get_modules_with_version, \
58 load_information_from_description_file, \
59 get_module_resource, zip_directory, \
60 get_module_path, initialize_sys_path, \
61 register_module_classes, init_module_models
63 logger = netsvc.Logger()
66 def open_openerp_namespace():
67 # See comment for open_openerp_namespace.
68 if openerp.conf.deprecation.open_openerp_namespace:
69 for k, v in list(sys.modules.items()):
70 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
71 sys.modules[k[8:]] = v
74 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
75 """Migrates+Updates or Installs all module nodes from ``graph``
76 :param graph: graph of module nodes to load
77 :param status: status dictionary for keeping track of progress
78 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
79 for same cases, and even raise osv_except if certificate is invalid)
80 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
81 :return: list of modules that were installed or updated
83 logger = logging.getLogger('init.load')
84 def process_sql_file(cr, fp):
85 queries = fp.read().split(';')
87 new_query = ' '.join(query.split())
91 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
92 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
93 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
94 load_data = lambda *args: _load_data(cr, *args, kind='data')
95 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
97 def load_test(module_name, idref, mode):
99 if not tools.config.options['test_disable']:
101 threading.currentThread().testing = True
102 _load_data(cr, module_name, idref, mode, 'test')
104 logging.getLogger('init.test').exception(
105 'Tests failed to execute in module %s', module_name)
107 threading.currentThread().testing = False
108 if tools.config.options['test_commit']:
113 def _load_data(cr, module_name, idref, mode, kind):
116 kind: data, demo, test, init_xml, update_xml, demo_xml.
118 noupdate is False, unless it is demo data or it is csv data in
122 for filename in package.data[kind]:
123 logger.info("module %s: loading %s", module_name, filename)
124 _, ext = os.path.splitext(filename)
125 pathname = os.path.join(module_name, filename)
126 fp = tools.file_open(pathname)
128 if kind in ('demo', 'demo_xml'):
132 if kind in ('init', 'init_xml'):
134 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
136 process_sql_file(cr, fp)
138 tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
140 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
147 processed_modules = []
149 pool = pooler.get_pool(cr.dbname)
150 migrations = openerp.modules.migration.MigrationManager(cr, graph)
151 logger.debug('loading %d packages...', len(graph))
154 cr.execute("select now()::timestamp")
155 dt_before_load = cr.fetchone()[0]
157 # register, instantiate and initialize models for each modules
158 for index, package in enumerate(graph):
159 module_name = package.name
160 module_id = package.id
162 if skip_modules and module_name in skip_modules:
165 logger.info('module %s: loading objects', package.name)
166 migrations.migrate_module(package, 'pre')
167 register_module_classes(package.name)
168 models = pool.load(cr, package)
169 loaded_modules.append(package.name)
170 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
171 init_module_models(cr, package.name, models)
173 status['progress'] = float(index) / len(graph)
175 # Can't put this line out of the loop: ir.module.module will be
176 # registered by init_module_models() above.
177 modobj = pool.get('ir.module.module')
180 modobj.check(cr, 1, [module_id])
185 if hasattr(package, 'init') or package.state == 'to install':
188 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
189 if package.state=='to upgrade':
190 # upgrading the module information
191 modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data))
192 load_init_xml(module_name, idref, mode)
193 load_update_xml(module_name, idref, mode)
194 load_data(module_name, idref, mode)
195 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
196 status['progress'] = (index + 0.75) / len(graph)
197 load_demo_xml(module_name, idref, mode)
198 load_demo(module_name, idref, mode)
199 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
201 # launch tests only in demo mode, as most tests will depend
202 # on demo data. Other tests can be added into the regular
203 # 'data' section, but should probably not alter the data,
204 # as there is no rollback.
205 load_test(module_name, idref, mode)
207 processed_modules.append(package.name)
209 migrations.migrate_module(package, 'post')
211 ver = release.major_version + '.' + package.data['version']
212 # Set new modules and dependencies
213 modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver})
214 # Update translations for all installed languages
215 modobj.update_translations(cr, 1, [module_id], None)
217 package.state = 'installed'
218 for kind in ('init', 'demo', 'update'):
219 if hasattr(package, kind):
220 delattr(package, kind)
224 # mark new res_log records as read
225 cr.execute("update res_log set read=True where create_date >= %s", (dt_before_load,))
229 return loaded_modules, processed_modules
231 def _check_module_names(cr, module_names):
232 mod_names = set(module_names)
233 if 'base' in mod_names:
234 # ignore dummy 'all' module
235 if 'all' in mod_names:
236 mod_names.remove('all')
238 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
239 if cr.dictfetchone()['count'] != len(mod_names):
240 # find out what module name(s) are incorrect:
241 cr.execute("SELECT name FROM ir_module_module")
242 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
243 logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
245 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
246 """Loads modules marked with ``states``, adding them to ``graph`` and
247 ``loaded_modules`` and returns a list of installed/upgraded modules."""
248 processed_modules = []
250 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
251 module_list = [name for (name,) in cr.fetchall() if name not in graph]
252 new_modules_in_graph = graph.add_modules(cr, module_list, force)
253 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
254 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
255 processed_modules.extend(processed)
256 loaded_modules.extend(loaded)
257 if not processed: break
258 return processed_modules
261 def load_modules(db, force_demo=False, status=None, update_module=False):
262 # TODO status['progress'] reporting is broken: used twice (and reset each
263 # time to zero) in load_module_graph, not fine-grained enough.
264 # It should be a method exposed by the pool.
265 initialize_sys_path()
267 open_openerp_namespace()
275 if not openerp.modules.db.is_initialized(cr):
276 logger.notifyChannel("init", netsvc.LOG_INFO, "init db")
277 openerp.modules.db.initialize(cr)
278 tools.config["init"]["all"] = 1
279 tools.config['update']['all'] = 1
280 if not tools.config['without_demo']:
281 tools.config["demo"]['all'] = 1
283 # This is a brand new pool, just created in pooler.get_db_and_pool()
284 pool = pooler.get_pool(cr.dbname)
286 report = tools.assertion_report()
287 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
288 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
290 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
291 graph = openerp.modules.graph.Graph()
292 graph.add_module(cr, 'base', force)
294 logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
295 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
297 # processed_modules: for cleanup step after install
298 # loaded_modules: to avoid double loading
299 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
301 if tools.config['load_language']:
302 for lang in tools.config['load_language'].split(','):
303 tools.load_language(cr, lang)
305 # STEP 2: Mark other modules to be loaded/updated
307 modobj = pool.get('ir.module.module')
308 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
309 logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
310 modobj.update_list(cr, 1)
312 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
314 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
316 ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
318 modobj.button_install(cr, 1, ids)
320 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
322 ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
324 modobj.button_upgrade(cr, 1, ids)
326 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
329 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
330 # IMPORTANT: this is done in two parts, first loading all installed or
331 # partially installed modules (i.e. installed/to upgrade), to
332 # offer a consistent system to the second part: installing
333 # newly selected modules.
334 states_to_load = ['installed', 'to upgrade']
335 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
336 processed_modules.extend(processed)
338 states_to_load = ['to install']
339 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
340 processed_modules.extend(processed)
343 cr.execute('select model from ir_model where state=%s', ('manual',))
344 for model in cr.dictfetchall():
345 pool.get('ir.model').instanciate(cr, 1, model['model'], {})
347 # STEP 4: Finish and cleanup
348 if processed_modules:
349 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
350 for (model, name) in cr.fetchall():
351 model_obj = pool.get(model)
352 if model_obj and not model_obj.is_transient():
353 logger.notifyChannel('init', netsvc.LOG_WARNING, 'Model %s (%s) has no access rules!' % (model, name))
355 # Temporary warning while we remove access rights on osv_memory objects, as they have
356 # been replaced by owner-only access rights
357 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
358 for (model, name) in cr.fetchall():
359 model_obj = pool.get(model)
360 if model_obj and model_obj.is_transient():
361 logger.notifyChannel('init', netsvc.LOG_WARNING, 'The transient model %s (%s) should not have explicit access rules!' % (model, name))
363 cr.execute("SELECT model from ir_model")
364 for (model,) in cr.fetchall():
365 obj = pool.get(model)
367 obj._check_removed_columns(cr, log=True)
369 logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)" % model)
371 # Cleanup orphan records
372 pool.get('ir.model.data')._process_end(cr, 1, processed_modules)
374 if report.get_report():
375 logger.notifyChannel('init', netsvc.LOG_INFO, report)
377 for kind in ('init', 'demo', 'update'):
378 tools.config[kind] = {}
382 # Remove records referenced from ir_model_data for modules to be
383 # removed (and removed the references from ir_model_data).
384 cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
385 for mod_id, mod_name in cr.fetchall():
386 cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
387 for rmod, rid in cr.fetchall():
389 rmod_module= pool.get(rmod)
391 # TODO group by module so that we can delete multiple ids in a call
392 rmod_module.unlink(cr, uid, [rid])
394 logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid))
395 cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
398 # Remove menu items that are not referenced by any of other
399 # (child) menu item, ir_values, or ir_model_data.
400 # This code could be a method of ir_ui_menu.
401 # TODO: remove menu without actions of children
403 cr.execute('''delete from
406 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
408 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
410 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
415 logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,))
417 # Pretend that modules to be removed are actually uninstalled.
418 cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
424 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: