1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
37 from cStringIO import StringIO
38 from os.path import join as opj
39 from zipfile import PyZipFile, ZIP_DEFLATED
43 import openerp.modules.db
44 import openerp.modules.graph
45 import openerp.modules.migration
46 import openerp.netsvc as netsvc
47 import openerp.osv as osv
48 import openerp.pooler as pooler
49 import openerp.release as release
50 import openerp.tools as tools
51 import openerp.tools.osutil as osutil
53 from openerp.tools.safe_eval import safe_eval as eval
54 from openerp.tools.translate import _
55 from openerp.modules.module import \
56 get_modules, get_modules_with_version, \
57 load_information_from_description_file, \
58 get_module_resource, zip_directory, \
59 get_module_path, initialize_sys_path, \
60 register_module_classes, init_module_models
62 logger = netsvc.Logger()
65 def open_openerp_namespace():
66 # See comment for open_openerp_namespace.
67 if openerp.conf.deprecation.open_openerp_namespace:
68 for k, v in list(sys.modules.items()):
69 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
70 sys.modules[k[8:]] = v
73 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
74 """Migrates+Updates or Installs all module nodes from ``graph``
75 :param graph: graph of module nodes to load
76 :param status: status dictionary for keeping track of progress
77 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
78 for same cases, and even raise osv_except if certificate is invalid)
79 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
80 :return: list of modules that were installed or updated
82 logger = logging.getLogger('init.load')
83 def process_sql_file(cr, fp):
84 queries = fp.read().split(';')
86 new_query = ' '.join(query.split())
90 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
91 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
92 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
93 load_data = lambda *args: _load_data(cr, *args, kind='data')
94 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
96 def load_test(module_name, idref, mode):
98 if not tools.config.options['test_disable']:
100 _load_data(cr, module_name, idref, mode, 'test')
102 logging.getLogger('init.test').exception(
103 'Tests failed to execute in module %s', module_name)
105 if tools.config.options['test_commit']:
110 def _load_data(cr, module_name, idref, mode, kind):
113 kind: data, demo, test, init_xml, update_xml, demo_xml.
115 noupdate is False, unless it is demo data or it is csv data in
119 for filename in package.data[kind]:
120 logger.info("module %s: loading %s", module_name, filename)
121 _, ext = os.path.splitext(filename)
122 pathname = os.path.join(module_name, filename)
123 fp = tools.file_open(pathname)
125 if kind in ('demo', 'demo_xml'):
129 if kind in ('init', 'init_xml'):
131 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
133 process_sql_file(cr, fp)
135 tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
137 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
144 processed_modules = []
146 pool = pooler.get_pool(cr.dbname)
147 migrations = openerp.modules.migration.MigrationManager(cr, graph)
148 logger.debug('loading %d packages...', len(graph))
150 # register, instantiate and initialize models for each modules
151 for index, package in enumerate(graph):
152 module_name = package.name
153 module_id = package.id
155 if skip_modules and module_name in skip_modules:
158 logger.info('module %s: loading objects', package.name)
159 migrations.migrate_module(package, 'pre')
160 register_module_classes(package.name)
161 models = pool.load(cr, package)
162 loaded_modules.append(package.name)
163 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
164 init_module_models(cr, package.name, models)
166 status['progress'] = float(index) / len(graph)
168 # Can't put this line out of the loop: ir.module.module will be
169 # registered by init_module_models() above.
170 modobj = pool.get('ir.module.module')
173 modobj.check(cr, 1, [module_id])
178 if hasattr(package, 'init') or package.state == 'to install':
181 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
182 if package.state=='to upgrade':
183 # upgrading the module information
184 modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data))
185 load_init_xml(module_name, idref, mode)
186 load_update_xml(module_name, idref, mode)
187 load_data(module_name, idref, mode)
188 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
189 status['progress'] = (index + 0.75) / len(graph)
190 load_demo_xml(module_name, idref, mode)
191 load_demo(module_name, idref, mode)
192 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
194 # launch tests only in demo mode, as most tests will depend
195 # on demo data. Other tests can be added into the regular
196 # 'data' section, but should probably not alter the data,
197 # as there is no rollback.
198 load_test(module_name, idref, mode)
200 processed_modules.append(package.name)
202 migrations.migrate_module(package, 'post')
204 ver = release.major_version + '.' + package.data['version']
205 # Set new modules and dependencies
206 modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver})
207 # Update translations for all installed languages
208 modobj.update_translations(cr, 1, [module_id], None)
210 package.state = 'installed'
211 for kind in ('init', 'demo', 'update'):
212 if hasattr(package, kind):
213 delattr(package, kind)
219 return loaded_modules, processed_modules
221 def _check_module_names(cr, module_names):
222 mod_names = set(module_names)
223 if 'base' in mod_names:
224 # ignore dummy 'all' module
225 if 'all' in mod_names:
226 mod_names.remove('all')
228 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
229 if cr.dictfetchone()['count'] != len(mod_names):
230 # find out what module name(s) are incorrect:
231 cr.execute("SELECT name FROM ir_module_module")
232 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
233 logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
235 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
236 """Loads modules marked with ``states``, adding them to ``graph`` and
237 ``loaded_modules`` and returns a list of installed/upgraded modules."""
238 processed_modules = []
240 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
241 module_list = [name for (name,) in cr.fetchall() if name not in graph]
242 new_modules_in_graph = graph.add_modules(cr, module_list, force)
243 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
244 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
245 processed_modules.extend(processed)
246 loaded_modules.extend(loaded)
247 if not processed: break
248 return processed_modules
251 def load_modules(db, force_demo=False, status=None, update_module=False):
252 # TODO status['progress'] reporting is broken: used twice (and reset each
253 # time to zero) in load_module_graph, not fine-grained enough.
254 # It should be a method exposed by the pool.
255 initialize_sys_path()
257 open_openerp_namespace()
265 if not openerp.modules.db.is_initialized(cr):
266 logger.notifyChannel("init", netsvc.LOG_INFO, "init db")
267 openerp.modules.db.initialize(cr)
268 tools.config["init"]["all"] = 1
269 tools.config['update']['all'] = 1
270 if not tools.config['without_demo']:
271 tools.config["demo"]['all'] = 1
273 # This is a brand new pool, just created in pooler.get_db_and_pool()
274 pool = pooler.get_pool(cr.dbname)
276 report = tools.assertion_report()
277 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
278 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
280 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
281 graph = openerp.modules.graph.Graph()
282 graph.add_module(cr, 'base', force)
284 logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
285 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
287 # processed_modules: for cleanup step after install
288 # loaded_modules: to avoid double loading
289 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
291 if tools.config['load_language']:
292 for lang in tools.config['load_language'].split(','):
293 tools.load_language(cr, lang)
295 # STEP 2: Mark other modules to be loaded/updated
297 modobj = pool.get('ir.module.module')
298 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
299 logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list')
300 modobj.update_list(cr, 1)
302 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
304 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
306 ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
308 modobj.button_install(cr, 1, ids)
310 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
312 ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
314 modobj.button_upgrade(cr, 1, ids)
316 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
319 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
320 # IMPORTANT: this is done in two parts, first loading all installed or
321 # partially installed modules (i.e. installed/to upgrade), to
322 # offer a consistent system to the second part: installing
323 # newly selected modules.
324 states_to_load = ['installed', 'to upgrade']
325 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
326 processed_modules.extend(processed)
328 states_to_load = ['to install']
329 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
330 processed_modules.extend(processed)
333 cr.execute('select model from ir_model where state=%s', ('manual',))
334 for model in cr.dictfetchall():
335 pool.get('ir.model').instanciate(cr, 1, model['model'], {})
337 # STEP 4: Finish and cleanup
338 if processed_modules:
339 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
340 for (model, name) in cr.fetchall():
341 model_obj = pool.get(model)
342 if model_obj and not model_obj.is_transient():
343 logger.notifyChannel('init', netsvc.LOG_WARNING, 'Model %s (%s) has no access rules!' % (model, name))
345 # Temporary warning while we remove access rights on osv_memory objects, as they have
346 # been replaced by owner-only access rights
347 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
348 for (model, name) in cr.fetchall():
349 model_obj = pool.get(model)
350 if model_obj and model_obj.is_transient():
351 logger.notifyChannel('init', netsvc.LOG_WARNING, 'The transient model %s (%s) should not have explicit access rules!' % (model, name))
353 cr.execute("SELECT model from ir_model")
354 for (model,) in cr.fetchall():
355 obj = pool.get(model)
357 obj._check_removed_columns(cr, log=True)
359 logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)" % model)
361 # Cleanup orphan records
362 pool.get('ir.model.data')._process_end(cr, 1, processed_modules)
364 if report.get_report():
365 logger.notifyChannel('init', netsvc.LOG_INFO, report)
367 for kind in ('init', 'demo', 'update'):
368 tools.config[kind] = {}
372 # Remove records referenced from ir_model_data for modules to be
373 # removed (and removed the references from ir_model_data).
374 cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
375 for mod_id, mod_name in cr.fetchall():
376 cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
377 for rmod, rid in cr.fetchall():
379 rmod_module= pool.get(rmod)
381 # TODO group by module so that we can delete multiple ids in a call
382 rmod_module.unlink(cr, uid, [rid])
384 logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid))
385 cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
388 # Remove menu items that are not referenced by any of other
389 # (child) menu item, ir_values, or ir_model_data.
390 # This code could be a method of ir_ui_menu.
391 # TODO: remove menu without actions of children
393 cr.execute('''delete from
396 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
398 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
400 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
405 logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,))
407 # Pretend that modules to be removed are actually uninstalled.
408 cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
414 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: