1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
37 from cStringIO import StringIO
38 from os.path import join as opj
39 from zipfile import PyZipFile, ZIP_DEFLATED
43 import openerp.modules.db
44 import openerp.modules.graph
45 import openerp.modules.migration
46 import openerp.netsvc as netsvc
47 import openerp.osv as osv
48 import openerp.pooler as pooler
49 import openerp.release as release
50 import openerp.tools as tools
51 import openerp.tools.osutil as osutil
53 from openerp.tools.safe_eval import safe_eval as eval
54 from openerp.tools.translate import _
55 from openerp.modules.module import \
56 get_modules, get_modules_with_version, \
57 load_information_from_description_file, \
58 get_module_resource, zip_directory, \
59 get_module_path, initialize_sys_path, \
60 register_module_classes, init_module_models
62 logger = netsvc.Logger()
65 def open_openerp_namespace():
66 # See comment for open_openerp_namespace.
67 if openerp.conf.deprecation.open_openerp_namespace:
68 for k, v in list(sys.modules.items()):
69 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
70 sys.modules[k[8:]] = v
73 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
74 """Migrates+Updates or Installs all module nodes from ``graph``
75 :param graph: graph of module nodes to load
76 :param status: status dictionary for keeping track of progress
77 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
78 for same cases, and even raise osv_except if certificate is invalid)
79 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
80 :return: list of modules that were installed or updated
82 logger = logging.getLogger('init.load')
83 def process_sql_file(cr, fp):
84 queries = fp.read().split(';')
86 new_query = ' '.join(query.split())
90 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
91 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
92 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
93 load_data = lambda *args: _load_data(cr, *args, kind='data')
94 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
96 def load_test(module_name, idref, mode):
98 if not tools.config.options['test_disable']:
100 _load_data(cr, module_name, idref, mode, 'test')
102 logging.getLogger('init.test').exception(
103 'Tests failed to execute in module %s', module_name)
105 if tools.config.options['test_commit']:
110 def _load_data(cr, module_name, idref, mode, kind):
113 kind: data, demo, test, init_xml, update_xml, demo_xml.
115 noupdate is False, unless it is demo data or it is csv data in
119 for filename in package.data[kind]:
120 logger.info("module %s: loading %s", module_name, filename)
121 _, ext = os.path.splitext(filename)
122 pathname = os.path.join(module_name, filename)
123 fp = tools.file_open(pathname)
125 if kind in ('demo', 'demo_xml'):
129 if kind in ('init', 'init_xml'):
131 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
133 process_sql_file(cr, fp)
135 tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
137 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
144 processed_modules = []
146 pool = pooler.get_pool(cr.dbname)
147 migrations = openerp.modules.migration.MigrationManager(cr, graph)
148 logger.debug('loading %d packages...', len(graph))
151 cr.execute("select now()::timestamp")
152 dt_before_load = cr.fetchone()[0]
154 # register, instantiate and initialize models for each modules
155 for index, package in enumerate(graph):
156 module_name = package.name
157 module_id = package.id
159 if skip_modules and module_name in skip_modules:
162 logger.info('module %s: loading objects', package.name)
163 migrations.migrate_module(package, 'pre')
164 register_module_classes(package.name)
165 models = pool.load(cr, package)
166 loaded_modules.append(package.name)
167 if package.state in ('to install', 'to upgrade'):
168 init_module_models(cr, package.name, models)
170 status['progress'] = float(index) / len(graph)
172 # Can't put this line out of the loop: ir.module.module will be
173 # registered by init_module_models() above.
174 modobj = pool.get('ir.module.module')
177 modobj.check(cr, 1, [module_id])
181 if package.state == 'to install':
186 if package.state in ('to install', 'to upgrade'):
187 if package.state=='to upgrade':
188 # upgrading the module information
189 modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data))
190 load_init_xml(module_name, idref, mode)
191 load_update_xml(module_name, idref, mode)
192 load_data(module_name, idref, mode)
193 if package.dbdemo and package.state != 'installed':
194 status['progress'] = (index + 0.75) / len(graph)
195 load_demo_xml(module_name, idref, mode)
196 load_demo(module_name, idref, mode)
197 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
199 # launch tests only in demo mode, as most tests will depend
200 # on demo data. Other tests can be added into the regular
201 # 'data' section, but should probably not alter the data,
202 # as there is no rollback.
203 load_test(module_name, idref, mode)
205 processed_modules.append(package.name)
207 migrations.migrate_module(package, 'post')
209 ver = release.major_version + '.' + package.data['version']
210 # Set new modules and dependencies
211 modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver})
212 # Update translations for all installed languages
213 modobj.update_translations(cr, 1, [module_id], None)
215 package.state = 'installed'
219 # mark new res_log records as read
220 cr.execute("update res_log set read=True where create_date >= %s", (dt_before_load,))
224 return loaded_modules, processed_modules
226 def _check_module_names(cr, module_names):
227 mod_names = set(module_names)
228 if 'base' in mod_names:
229 # ignore dummy 'all' module
230 if 'all' in mod_names:
231 mod_names.remove('all')
233 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
234 if cr.dictfetchone()['count'] != len(mod_names):
235 # find out what module name(s) are incorrect:
236 cr.execute("SELECT name FROM ir_module_module")
237 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
238 logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
240 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
241 """Loads modules marked with ``states``, adding them to ``graph`` and
242 ``loaded_modules`` and returns a list of installed/upgraded modules."""
243 processed_modules = []
245 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
246 module_list = [name for (name,) in cr.fetchall() if name not in graph]
247 new_modules_in_graph = graph.add_modules(cr, module_list, force)
248 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
249 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
250 processed_modules.extend(processed)
251 loaded_modules.extend(loaded)
252 if not processed: break
253 return processed_modules
256 def load_modules(db, force_demo=False, status=None, update_module=False):
257 # TODO status['progress'] reporting is broken: used twice (and reset each
258 # time to zero) in load_module_graph, not fine-grained enough.
259 # It should be a method exposed by the pool.
260 initialize_sys_path()
262 open_openerp_namespace()
270 if not openerp.modules.db.is_initialized(cr):
271 logger.notifyChannel("init", netsvc.LOG_INFO, "init db")
272 openerp.modules.db.initialize(cr)
275 # This is a brand new pool, just created in pooler.get_db_and_pool()
276 pool = pooler.get_pool(cr.dbname)
278 report = tools.assertion_report()
279 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
280 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
282 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
283 graph = openerp.modules.graph.Graph()
284 graph.add_module(cr, 'base', force)
286 logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
287 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
289 # processed_modules: for cleanup step after install
290 # loaded_modules: to avoid double loading
291 # After load_module_graph(), 'base' has been installed or updated and its state is 'installed'.
292 loaded_modules, processed_modules = load_module_graph(cr, graph, status, report=report)
294 if tools.config['load_language']:
295 for lang in tools.config['load_language'].split(','):
296 tools.load_language(cr, lang)
298 # STEP 2: Mark other modules to be loaded/updated
299 # This is a one-shot use of tools.config[init|update] from the command line
300 # arguments. It is directly cleared to not interfer with later create/update
303 modobj = pool.get('ir.module.module')
304 if ('base' in tools.config['init']) or ('base' in tools.config['update']) \
305 or ('all' in tools.config['init']) or ('all' in tools.config['update']):
306 logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
307 modobj.update_list(cr, 1)
309 if 'all' in tools.config['init']:
310 ids = modobj.search(cr, 1, [])
311 tools.config['init'] = dict.fromkeys([m['name'] for m in modobj.read(cr, 1, ids, ['name'])], 1)
313 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
315 mods = [k for k in tools.config['init'] if tools.config['init'][k] and k not in ('base', 'all')]
316 ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
318 modobj.button_install(cr, 1, ids) # goes from 'uninstalled' to 'to install'
320 mods = [k for k in tools.config['update'] if tools.config['update'][k] and k not in ('base', 'all')]
321 ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
323 modobj.button_upgrade(cr, 1, ids) # goes from 'installed' to 'to upgrade'
325 for kind in ('init', 'demo', 'update'):
326 tools.config[kind] = {}
328 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
329 # IMPORTANT: this is done in two parts, first loading all installed or
330 # partially installed modules (i.e. installed/to upgrade), to
331 # offer a consistent system to the second part: installing
332 # newly selected modules.
333 states_to_load = ['installed', 'to upgrade']
334 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
335 processed_modules.extend(processed)
337 states_to_load = ['to install']
338 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
339 processed_modules.extend(processed)
342 cr.execute('select model from ir_model where state=%s', ('manual',))
343 for model in cr.dictfetchall():
344 pool.get('ir.model').instanciate(cr, 1, model['model'], {})
346 # STEP 4: Finish and cleanup
347 if processed_modules:
348 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
349 for (model, name) in cr.fetchall():
350 model_obj = pool.get(model)
351 if model_obj and not model_obj.is_transient():
352 logger.notifyChannel('init', netsvc.LOG_WARNING, 'Model %s (%s) has no access rules!' % (model, name))
354 # Temporary warning while we remove access rights on osv_memory objects, as they have
355 # been replaced by owner-only access rights
356 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
357 for (model, name) in cr.fetchall():
358 model_obj = pool.get(model)
359 if model_obj and model_obj.is_transient():
360 logger.notifyChannel('init', netsvc.LOG_WARNING, 'The transient model %s (%s) should not have explicit access rules!' % (model, name))
362 cr.execute("SELECT model from ir_model")
363 for (model,) in cr.fetchall():
364 obj = pool.get(model)
366 obj._check_removed_columns(cr, log=True)
368 logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)" % model)
370 # Cleanup orphan records
371 pool.get('ir.model.data')._process_end(cr, 1, processed_modules)
373 if report.get_report():
374 logger.notifyChannel('init', netsvc.LOG_INFO, report)
378 # Remove records referenced from ir_model_data for modules to be
379 # removed (and removed the references from ir_model_data).
380 cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
381 for mod_id, mod_name in cr.fetchall():
382 cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
383 for rmod, rid in cr.fetchall():
385 rmod_module= pool.get(rmod)
387 # TODO group by module so that we can delete multiple ids in a call
388 rmod_module.unlink(cr, uid, [rid])
390 logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid))
391 cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
394 # Remove menu items that are not referenced by any of other
395 # (child) menu item, ir_values, or ir_model_data.
396 # This code could be a method of ir_ui_menu.
397 # TODO: remove menu without actions of children
399 cr.execute('''delete from
402 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
404 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
406 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
411 logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,))
413 # Pretend that modules to be removed are actually uninstalled.
414 cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
420 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: