1 # -*- coding: utf-8 -*-
2 ##############################################################################
4 # OpenERP, Open Source Management Solution
5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
6 # Copyright (C) 2010-2011 OpenERP s.a. (<http://openerp.com>).
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as
10 # published by the Free Software Foundation, either version 3 of the
11 # License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 ##############################################################################
23 """ Modules (also called addons) management.
38 from cStringIO import StringIO
39 from os.path import join as opj
40 from zipfile import PyZipFile, ZIP_DEFLATED
44 import openerp.modules.db
45 import openerp.modules.graph
46 import openerp.modules.migration
47 import openerp.netsvc as netsvc
48 import openerp.osv as osv
49 import openerp.pooler as pooler
50 import openerp.release as release
51 import openerp.tools as tools
52 import openerp.tools.osutil as osutil
54 from openerp.tools.safe_eval import safe_eval as eval
55 from openerp.tools.translate import _
56 from openerp.modules.module import \
57 get_modules, get_modules_with_version, \
58 load_information_from_description_file, \
59 call_post_load_hook, \
60 get_module_resource, zip_directory, \
61 get_module_path, initialize_sys_path, \
62 register_module_classes, init_module_models
64 _logger = logging.getLogger(__name__)
66 def open_openerp_namespace():
67 # See comment for open_openerp_namespace.
68 if openerp.conf.deprecation.open_openerp_namespace:
69 for k, v in list(sys.modules.items()):
70 if k.startswith('openerp.') and sys.modules.get(k[8:]) is None:
71 sys.modules[k[8:]] = v
74 def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None):
75 """Migrates+Updates or Installs all module nodes from ``graph``
76 :param graph: graph of module nodes to load
77 :param status: status dictionary for keeping track of progress
78 :param perform_checks: whether module descriptors should be checked for validity (prints warnings
79 for same cases, and even raise osv_except if certificate is invalid)
80 :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
81 :return: list of modules that were installed or updated
83 def process_sql_file(cr, fp):
84 queries = fp.read().split(';')
86 new_query = ' '.join(query.split())
90 load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml')
91 load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml')
92 load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml')
93 load_data = lambda *args: _load_data(cr, *args, kind='data')
94 load_demo = lambda *args: _load_data(cr, *args, kind='demo')
96 def load_test(module_name, idref, mode):
98 if not tools.config.options['test_disable']:
100 threading.currentThread().testing = True
101 _load_data(cr, module_name, idref, mode, 'test')
104 'Tests failed to execute in module %s', module_name)
106 threading.currentThread().testing = False
107 if tools.config.options['test_commit']:
112 def _load_data(cr, module_name, idref, mode, kind):
115 kind: data, demo, test, init_xml, update_xml, demo_xml.
117 noupdate is False, unless it is demo data or it is csv data in
121 for filename in package.data[kind]:
122 _logger.info("module %s: loading %s", module_name, filename)
123 _, ext = os.path.splitext(filename)
124 pathname = os.path.join(module_name, filename)
125 fp = tools.file_open(pathname)
127 if kind in ('demo', 'demo_xml'):
131 if kind in ('init', 'init_xml'):
133 tools.convert_csv_import(cr, module_name, pathname, fp.read(), idref, mode, noupdate)
135 process_sql_file(cr, fp)
137 tools.convert_yaml_import(cr, module_name, fp, idref, mode, noupdate)
139 tools.convert_xml_import(cr, module_name, fp, idref, mode, noupdate, report)
146 processed_modules = []
148 pool = pooler.get_pool(cr.dbname)
149 migrations = openerp.modules.migration.MigrationManager(cr, graph)
150 _logger.debug('loading %d packages...', len(graph))
153 cr.execute("select now()::timestamp")
154 dt_before_load = cr.fetchone()[0]
156 # register, instantiate and initialize models for each modules
157 for index, package in enumerate(graph):
158 module_name = package.name
159 module_id = package.id
161 if skip_modules and module_name in skip_modules:
164 _logger.info('module %s: loading objects', package.name)
165 migrations.migrate_module(package, 'pre')
166 register_module_classes(package.name)
168 models = pool.load(cr, package)
169 loaded_modules.append(package.name)
170 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
171 init_module_models(cr, package.name, models)
173 status['progress'] = float(index) / len(graph)
175 # Can't put this line out of the loop: ir.module.module will be
176 # registered by init_module_models() above.
177 modobj = pool.get('ir.module.module')
180 modobj.check(cr, 1, [module_id])
185 if hasattr(package, 'init') or package.state == 'to install':
188 if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
189 if package.state=='to upgrade':
190 # upgrading the module information
191 modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data))
192 load_init_xml(module_name, idref, mode)
193 load_update_xml(module_name, idref, mode)
194 load_data(module_name, idref, mode)
195 if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'):
196 status['progress'] = (index + 0.75) / len(graph)
197 load_demo_xml(module_name, idref, mode)
198 load_demo(module_name, idref, mode)
199 cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id))
201 # launch tests only in demo mode, as most tests will depend
202 # on demo data. Other tests can be added into the regular
203 # 'data' section, but should probably not alter the data,
204 # as there is no rollback.
205 load_test(module_name, idref, mode)
207 processed_modules.append(package.name)
209 migrations.migrate_module(package, 'post')
211 ver = release.major_version + '.' + package.data['version']
212 # Set new modules and dependencies
213 modobj.write(cr, 1, [module_id], {'state': 'installed', 'latest_version': ver})
214 # Update translations for all installed languages
215 modobj.update_translations(cr, 1, [module_id], None)
217 package.state = 'installed'
218 for kind in ('init', 'demo', 'update'):
219 if hasattr(package, kind):
220 delattr(package, kind)
224 # mark new res_log records as read
225 cr.execute("update res_log set read=True where create_date >= %s", (dt_before_load,))
229 return loaded_modules, processed_modules
231 def _check_module_names(cr, module_names):
232 mod_names = set(module_names)
233 if 'base' in mod_names:
234 # ignore dummy 'all' module
235 if 'all' in mod_names:
236 mod_names.remove('all')
238 cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),))
239 if cr.dictfetchone()['count'] != len(mod_names):
240 # find out what module name(s) are incorrect:
241 cr.execute("SELECT name FROM ir_module_module")
242 incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
243 _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
245 def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
246 """Loads modules marked with ``states``, adding them to ``graph`` and
247 ``loaded_modules`` and returns a list of installed/upgraded modules."""
248 processed_modules = []
250 cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
251 module_list = [name for (name,) in cr.fetchall() if name not in graph]
252 new_modules_in_graph = graph.add_modules(cr, module_list, force)
253 _logger.debug('Updating graph with %d more modules', len(module_list))
254 loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
255 processed_modules.extend(processed)
256 loaded_modules.extend(loaded)
257 if not processed: break
258 return processed_modules
261 def load_modules(db, force_demo=False, status=None, update_module=False):
262 # TODO status['progress'] reporting is broken: used twice (and reset each
263 # time to zero) in load_module_graph, not fine-grained enough.
264 # It should be a method exposed by the pool.
265 initialize_sys_path()
267 open_openerp_namespace()
275 if not openerp.modules.db.is_initialized(cr):
276 _logger.info("init db")
277 openerp.modules.db.initialize(cr)
278 tools.config["init"]["all"] = 1
279 tools.config['update']['all'] = 1
280 if not tools.config['without_demo']:
281 tools.config["demo"]['all'] = 1
283 # This is a brand new pool, just created in pooler.get_db_and_pool()
284 pool = pooler.get_pool(cr.dbname)
286 report = tools.assertion_report()
287 if 'base' in tools.config['update'] or 'all' in tools.config['update']:
288 cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
290 # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
291 graph = openerp.modules.graph.Graph()
292 graph.add_module(cr, 'base', force)
294 _logger.critical('module base cannot be loaded! (hint: verify addons-path)')
295 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
297 # processed_modules: for cleanup step after install
298 # loaded_modules: to avoid double loading
299 loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
301 if tools.config['load_language']:
302 for lang in tools.config['load_language'].split(','):
303 tools.load_language(cr, lang)
305 # STEP 2: Mark other modules to be loaded/updated
307 modobj = pool.get('ir.module.module')
308 if ('base' in tools.config['init']) or ('base' in tools.config['update']):
309 _logger.info('updating modules list')
310 modobj.update_list(cr, 1)
312 _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys()))
314 mods = [k for k in tools.config['init'] if tools.config['init'][k]]
316 ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)])
318 modobj.button_install(cr, 1, ids)
320 mods = [k for k in tools.config['update'] if tools.config['update'][k]]
322 ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)])
324 modobj.button_upgrade(cr, 1, ids)
326 cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
329 # STEP 3: Load marked modules (skipping base which was done in STEP 1)
330 # IMPORTANT: this is done in two parts, first loading all installed or
331 # partially installed modules (i.e. installed/to upgrade), to
332 # offer a consistent system to the second part: installing
333 # newly selected modules.
334 states_to_load = ['installed', 'to upgrade']
335 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
336 processed_modules.extend(processed)
338 states_to_load = ['to install']
339 processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
340 processed_modules.extend(processed)
343 cr.execute('select model from ir_model where state=%s', ('manual',))
344 for model in cr.dictfetchall():
345 pool.get('ir.model').instanciate(cr, 1, model['model'], {})
347 # STEP 4: Finish and cleanup
348 if processed_modules:
349 cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
350 for (model, name) in cr.fetchall():
351 model_obj = pool.get(model)
352 if model_obj and not model_obj.is_transient():
353 _logger.warning('Model %s (%s) has no access rules!', model, name)
355 # Temporary warning while we remove access rights on osv_memory objects, as they have
356 # been replaced by owner-only access rights
357 cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""")
358 for (model, name) in cr.fetchall():
359 model_obj = pool.get(model)
360 if model_obj and model_obj.is_transient():
361 _logger.warning('The transient model %s (%s) should not have explicit access rules!', model, name)
363 cr.execute("SELECT model from ir_model")
364 for (model,) in cr.fetchall():
365 obj = pool.get(model)
367 obj._check_removed_columns(cr, log=True)
369 _logger.warning("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
371 # Cleanup orphan records
372 pool.get('ir.model.data')._process_end(cr, 1, processed_modules)
374 for kind in ('init', 'demo', 'update'):
375 tools.config[kind] = {}
379 # Remove records referenced from ir_model_data for modules to be
380 # removed (and removed the references from ir_model_data).
381 cr.execute("select id,name from ir_module_module where state=%s", ('to remove',))
382 for mod_id, mod_name in cr.fetchall():
383 cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,))
384 for rmod, rid in cr.fetchall():
386 rmod_module= pool.get(rmod)
388 # TODO group by module so that we can delete multiple ids in a call
389 rmod_module.unlink(cr, uid, [rid])
391 _logger.error('Could not locate %s to remove res=%d' % (rmod,rid))
392 cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,))
395 # Remove menu items that are not referenced by any of other
396 # (child) menu item, ir_values, or ir_model_data.
397 # This code could be a method of ir_ui_menu.
398 # TODO: remove menu without actions of children
400 cr.execute('''delete from
403 (id not IN (select parent_id from ir_ui_menu where parent_id is not null))
405 (id not IN (select res_id from ir_values where model='ir.ui.menu'))
407 (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''')
412 _logger.info('removed %d unused menus', cr.rowcount)
414 # Pretend that modules to be removed are actually uninstalled.
415 cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',))
418 _logger.info('Modules loaded.')
423 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: