1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
28 from os.path import join as opj
34 import openerp.osv as osv
35 import openerp.tools as tools
36 import openerp.tools.osutil as osutil
37 from openerp.tools.safe_eval import safe_eval as eval
38 import openerp.pooler as pooler
39 from openerp.tools.translate import _
41 import openerp.netsvc as netsvc
44 import openerp.release as release
48 from zipfile import PyZipFile, ZIP_DEFLATED
49 from cStringIO import StringIO
53 import openerp.modules.db
54 import openerp.modules.graph
55 import openerp.modules.migration
57 from openerp.modules.module import \
58 get_modules, get_modules_with_version, \
59 load_information_from_description_file, \
60 get_module_resource, zip_directory, \
61 get_module_path, initialize_sys_path, \
62 register_module_classes, init_module_models
64 logger = netsvc.Logger()
67 def open_openerp_namespace():
68 # See comment for open_openerp_namespace.
69 if openerp.conf.deprecation.open_openerp_namespace:
70 for k, v in list(sys.modules.items()):
71 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
72 sys.modules[k[8:]] = v
75 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
76 """Migrates+Updates or Installs all module nodes from ``graph``
77 :param graph: graph of module nodes to load
78 :param status: status dictionary for keeping track of progress
79 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
80 for same cases, and even raise osv_except if certificate is invalid)
81 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
82 :return: list of modules that were installed or updated
84 def process_sql_file(cr, fp):
85 queries = fp.read().split(';')
87 new_query = ' '.join(query.split())
91 def load_init_xml(cr, module_name, idref, mode):
92 _load_data(cr, module_name, idref, mode, 'init_xml')
94 def load_update_xml(cr, module_name, idref, mode):
95 _load_data(cr, module_name, idref, mode, 'update_xml')
97 def load_demo_xml(cr, module_name, idref, mode):
98 _load_data(cr, module_name, idref, mode, 'demo_xml')
100 def load_data(cr, module_name, idref, mode):
101 _load_data(cr, module_name, idref, mode, 'data')
103 def load_demo(cr, module_name, idref, mode):
104 _load_data(cr, module_name, idref, mode, 'demo')
106 def load_test(cr, module_name, idref, mode):
108 if not tools.config.options['test_disable']:
110 _load_data(cr, module_name, idref, mode, 'test')
112 logging.getLogger('test').exception('Tests failed to execute in module %s', module_name)
114 if tools.config.options['test_commit']:
119 def _load_data(cr, module_name, idref, mode, kind):
122 kind: data, demo, test, init_xml, update_xml, demo_xml.
124 noupdate is False, unless it is demo data or it is csv data in
128 for filename in package.data[kind]:
129 log = logging.getLogger('init')
130 log.info("module %s: loading %s", module_name, filename)
131 _, ext = os.path.splitext(filename)
132 pathname = os.path.join(module_name, filename)
133 fp = tools.file_open(pathname)
135 if kind in ('demo', 'demo_xml'):
139 if kind in ('init', 'init_xml'):
141 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
143 process_sql_file(cr, fp)
145 tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
147 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
154 processed_modules = []
157 pool = pooler.get_pool(cr.dbname)
158 migrations = openerp.modules.migration.MigrationManager(cr, graph)
159 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph))
161 # register, instantiate and initialize models for each modules
162 for package in graph:
163 module_name = package.name
164 module_id = package.id
166 if skip_modules and module_name in skip_modules:
169 logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name)
170 migrations.migrate_module(package, 'pre')
171 register_module_classes(package.name)
172 models = pool.instanciate(package.name, cr)
173 loaded_modules.append(package.name)
174 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
175 init_module_models(cr, package.name, models)
177 status['progress'] = float(statusi) / len(graph)
179 # Can't put this line out of the loop: ir.module.module will be
180 # registered by init_module_models() above.
181 modobj = pool.get('ir.module.module')
184 modobj.check(cr, 1, [module_id])
189 if hasattr(package, 'init') or package.state == 'to install':
192 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
193 if package.state=='to upgrade':
194 # upgrading the module information
195 modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data))
196 load_init_xml(cr, module_name, idref, mode)
197 load_update_xml(cr, module_name, idref, mode)
198 load_data(cr, module_name, idref, mode)
199 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
200 status['progress'] = (float(statusi)+0.75) / len(graph)
201 load_demo_xml(cr, module_name, idref, mode)
202 load_demo(cr, module_name, idref, mode)
203 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
205 # launch tests only in demo mode, as most tests will depend
206 # on demo data. Other tests can be added into the regular
207 # 'data' section, but should probably not alter the data,
208 # as there is no rollback.
209 load_test(cr, module_name, idref, mode)
211 processed_modules.append(package.name)
213 migrations.migrate_module(package, 'post')
215 ver = release.major_version + '.' + package.data['version']
216 # Set new modules and dependencies
217 modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver})
218 # Update translations for all installed languages
219 modobj.update_translations(cr, 1, [module_id], None)
221 package.state = 'installed'
222 for kind in ('init', 'demo', 'update'):
223 if hasattr(package, kind):
224 delattr(package, kind)
231 return loaded_modules, processed_modules
233 def _check_module_names(cr, module_names):
234 mod_names = set(module_names)
235 if 'base' in mod_names:
236 # ignore dummy 'all' module
237 if 'all' in mod_names:
238 mod_names.remove('all')
240 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
241 if cr.dictfetchone()['count'] != len(mod_names):
242 # find out what module name(s) are incorrect:
243 cr.execute("SELECT name FROM ir_module_module")
244 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
245 logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
247 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
248 """Loads modules marked with ``states``, adding them to ``graph`` and
249 ``loaded_modules`` and returns a list of installed/upgraded modules."""
250 processed_modules = []
252 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
253 module_list = [name for (name,) in cr.fetchall() if name not in graph]
254 new_modules_in_graph = graph.add_modules(cr, module_list, force)
255 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
256 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
257 processed_modules.extend(processed)
258 loaded_modules.extend(loaded)
259 if not processed: break
260 return processed_modules
263 def load_modules(db, force_demo=False, status=None, update_module=False):
264 # TODO status['progress'] reporting is broken: used twice (and reset each
265 # time to zero) in load_module_graph, not fine-grained enough.
266 # It should be a method exposed by the pool.
267 initialize_sys_path()
269 open_openerp_namespace()
277 if not openerp.modules.db.is_initialized(cr):
278 logger.notifyChannel("init", netsvc.LOG_INFO, "init db")
279 openerp.modules.db.initialize(cr)
280 tools.config["init"]["all"] = 1
281 tools.config['update']['all'] = 1
282 if not tools.config['without_demo']:
283 tools.config["demo"]['all'] = 1
285 # This is a brand new pool, just created in pooler.get_db_and_pool()
286 pool = pooler.get_pool(cr.dbname)
288 processed_modules = [] # for cleanup step after install
289 loaded_modules = [] # to avoid double loading
290 report = tools.assertion_report()
291 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
292 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
294 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
295 graph = openerp.modules.graph.Graph()
296 graph.add_module(cr, 'base', force)
298 logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
299 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
300 loaded, processed = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
301 processed_modules.extend(processed)
303 if tools.config['load_language']:
304 for lang in tools.config['load_language'].split(','):
305 tools.load_language(cr, lang)
307 # STEP 2: Mark other modules to be loaded/updated
309 modobj = pool.get('ir.module.module')
310 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
311 logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
312 modobj.update_list(cr, 1)
314 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
316 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
318 ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
320 modobj.button_install(cr, 1, ids)
322 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
324 ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
326 modobj.button_upgrade(cr, 1, ids)
328 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
331 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
332 # IMPORTANT: this is done in two parts, first loading all installed or
333 # partially installed modules (i.e. installed/to upgrade), to
334 # offer a consistent system to the second part: installing
335 # newly selected modules.
336 states_to_load = ['installed', 'to upgrade']
337 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
338 processed_modules.extend(processed)
340 states_to_load = ['to install']
341 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
342 processed_modules.extend(processed)
345 cr.execute('select model from ir_model where state=%s', ('manual',))
346 for model in cr.dictfetchall():
347 pool.get('ir.model').instanciate(cr, 1, model['model'], {})
349 # STEP 4: Finish and cleanup
350 if processed_modules:
351 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
352 for (model, name) in cr.fetchall():
353 model_obj = pool.get(model)
354 if model_obj and not isinstance(model_obj, osv.osv.osv_memory):
355 logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name))
357 # Temporary warning while we remove access rights on osv_memory objects, as they have
358 # been replaced by owner-only access rights
359 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
360 for (model, name) in cr.fetchall():
361 model_obj = pool.get(model)
362 if isinstance(model_obj, osv.osv.osv_memory) and not isinstance(model_obj, osv.osv.osv):
363 logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name))
365 cr.execute("SELECT model from ir_model")
366 for (model,) in cr.fetchall():
367 obj = pool.get(model)
369 obj._check_removed_columns(cr, log=True)
371 logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model)
373 # Cleanup orphan records
374 pool.get('ir.model.data')._process_end(cr, 1, processed_modules)
376 if report.get_report():
377 logger.notifyChannel('init', netsvc.LOG_INFO, report)
379 for kind in ('init', 'demo', 'update'):
380 tools.config[kind] = {}
384 # Remove records referenced from ir_model_data for modules to be
385 # removed (and removed the references from ir_model_data).
386 cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
387 for mod_id, mod_name in cr.fetchall():
388 cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
389 for rmod, rid in cr.fetchall():
391 rmod_module= pool.get(rmod)
393 # TODO group by module so that we can delete multiple ids in a call
394 rmod_module.unlink(cr, uid, [rid])
396 logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid))
397 cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
400 # Remove menu items that are not referenced by any of other
401 # (child) menu item, ir_values, or ir_model_data.
402 # This code could be a method of ir_ui_menu.
403 # TODO: remove menu without actions of children
405 cr.execute('''delete from
408 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
410 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
412 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
417 logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,))
419 # Pretend that modules to be removed are actually uninstalled.
420 cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
426 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: