openerp/service/db.py

   1 # -*- coding: utf-8 -*-
   2 from contextlib import closing
   3 from functools import wraps
   4 import logging
   5 import os
   6 import shutil
   7 import threading
   8 import traceback
   9 import tempfile
  10 import zipfile
  11
  12 import psycopg2
  13
  14 import openerp
  15 from openerp import SUPERUSER_ID
  16 import openerp.release
  17 import openerp.sql_db
  18 import openerp.tools
  19
  20 import security
  21
  22 _logger = logging.getLogger(__name__)
  23
  24 self_actions = {}
  25 self_id = 0
  26 self_id_protect = threading.Semaphore()
  27
  28 # This should be moved to openerp.modules.db, along side initialize().
  29 def _initialize_db(id, db_name, demo, lang, user_password):
  30     try:
  31         self_actions[id]['progress'] = 0
  32         db = openerp.sql_db.db_connect(db_name)
  33         with closing(db.cursor()) as cr:
  34             # TODO this should be removed as it is done by RegistryManager.new().
  35             openerp.modules.db.initialize(cr)
  36             openerp.tools.config['lang'] = lang
  37             cr.commit()
  38
  39         registry = openerp.modules.registry.RegistryManager.new(
  40             db_name, demo, self_actions[id], update_module=True)
  41
  42         with closing(db.cursor()) as cr:
  43             if lang:
  44                 modobj = registry['ir.module.module']
  45                 mids = modobj.search(cr, SUPERUSER_ID, [('state', '=', 'installed')])
  46                 modobj.update_translations(cr, SUPERUSER_ID, mids, lang)
  47
  48             # update admin's password and lang
  49             values = {'password': user_password, 'lang': lang}
  50             registry['res.users'].write(cr, SUPERUSER_ID, [SUPERUSER_ID], values)
  51
  52             cr.execute('SELECT login, password FROM res_users ORDER BY login')
  53             self_actions[id].update(users=cr.dictfetchall(), clean=True)
  54             cr.commit()
  55
  56     except Exception, e:
  57         self_actions[id].update(clean=False, exception=e)
  58         _logger.exception('CREATE DATABASE failed:')
  59         self_actions[id]['traceback'] = traceback.format_exc()
  60
  61 def dispatch(method, params):
  62     if method in ['create', 'get_progress', 'drop', 'dump', 'restore', 'rename',
  63                   'change_admin_password', 'migrate_databases',
  64                   'create_database', 'duplicate_database']:
  65         passwd = params[0]
  66         params = params[1:]
  67         security.check_super(passwd)
  68     elif method in ['db_exist', 'list', 'list_lang', 'server_version']:
  69         # params = params
  70         # No security check for these methods
  71         pass
  72     else:
  73         raise KeyError("Method not found: %s" % method)
  74     fn = globals()['exp_' + method]
  75     return fn(*params)
  76
  77 def _create_empty_database(name):
  78     db = openerp.sql_db.db_connect('postgres')
  79     with closing(db.cursor()) as cr:
  80         chosen_template = openerp.tools.config['db_template']
  81         cr.execute("SELECT datname FROM pg_database WHERE datname = %s",
  82                    (name,))
  83         if cr.fetchall():
  84             raise openerp.exceptions.Warning("database %r already exists!" % (name,))
  85         else:
  86             cr.autocommit(True)     # avoid transaction block
  87             cr.execute("""CREATE DATABASE "%s" ENCODING 'unicode' TEMPLATE "%s" """ % (name, chosen_template))
  88
  89 def exp_create(db_name, demo, lang, user_password='admin'):
  90     self_id_protect.acquire()
  91     global self_id
  92     self_id += 1
  93     id = self_id
  94     self_id_protect.release()
  95
  96     self_actions[id] = {'clean': False}
  97
  98     _create_empty_database(db_name)
  99
 100     _logger.info('CREATE DATABASE %s', db_name.lower())
 101     create_thread = threading.Thread(target=_initialize_db,
 102                                      args=(id, db_name, demo, lang, user_password))
 103     create_thread.start()
 104     self_actions[id]['thread'] = create_thread
 105     return id
 106
 107 def exp_create_database(db_name, demo, lang, user_password='admin'):
 108     """ Similar to exp_create but blocking."""
 109     self_id_protect.acquire()
 110     global self_id
 111     self_id += 1
 112     id = self_id
 113     self_id_protect.release()
 114
 115     self_actions[id] = {'clean': False}
 116
 117     _logger.info('Create database `%s`.', db_name)
 118     _create_empty_database(db_name)
 119     _initialize_db(id, db_name, demo, lang, user_password)
 120     return True
 121
 122 def exp_duplicate_database(db_original_name, db_name):
 123     _logger.info('Duplicate database `%s` to `%s`.', db_original_name, db_name)
 124     openerp.sql_db.close_db(db_original_name)
 125     db = openerp.sql_db.db_connect('postgres')
 126     with closing(db.cursor()) as cr:
 127         cr.autocommit(True)     # avoid transaction block
 128         cr.execute("""CREATE DATABASE "%s" ENCODING 'unicode' TEMPLATE "%s" """ % (db_name, db_original_name))
 129
 130     from_fs = openerp.tools.config.filestore(db_original_name)
 131     to_fs = openerp.tools.config.filestore(db_name)
 132     if os.path.exists(from_fs) and not os.path.exists(to_fs):
 133         shutil.copy(from_fs, to_fs)
 134     return True
 135
 136 def exp_get_progress(id):
 137     if self_actions[id]['thread'].isAlive():
 138 #       return openerp.modules.init_progress[db_name]
 139         return min(self_actions[id].get('progress', 0), 0.95), []
 140     else:
 141         clean = self_actions[id]['clean']
 142         if clean:
 143             users = self_actions[id]['users']
 144             for user in users:
 145                 # Remove the None passwords as they can't be marshalled by XML-RPC.
 146                 if user['password'] is None:
 147                     user['password'] = ''
 148             self_actions.pop(id)
 149             return 1.0, users
 150         else:
 151             a = self_actions.pop(id)
 152             exc, tb = a['exception'], a['traceback']
 153             raise Exception, exc, tb
 154
 155 def exp_drop(db_name):
 156     if db_name not in exp_list(True):
 157         return False
 158     openerp.modules.registry.RegistryManager.delete(db_name)
 159     openerp.sql_db.close_db(db_name)
 160
 161     db = openerp.sql_db.db_connect('postgres')
 162     with closing(db.cursor()) as cr:
 163         cr.autocommit(True)     # avoid transaction block
 164         # Try to terminate all other connections that might prevent
 165         # dropping the database
 166         try:
 167             # PostgreSQL 9.2 renamed pg_stat_activity.procpid to pid:
 168             # http://www.postgresql.org/docs/9.2/static/release-9-2.html#AEN110389
 169             pid_col = 'pid' if cr._cnx.server_version >= 90200 else 'procpid'
 170
 171             cr.execute("""SELECT pg_terminate_backend(%(pid_col)s)
 172                           FROM pg_stat_activity
 173                           WHERE datname = %%s AND
 174                                 %(pid_col)s != pg_backend_pid()""" % {'pid_col': pid_col},
 175                        (db_name,))
 176         except Exception:
 177             pass
 178
 179         try:
 180             cr.execute('DROP DATABASE "%s"' % db_name)
 181         except Exception, e:
 182             _logger.error('DROP DB: %s failed:\n%s', db_name, e)
 183             raise Exception("Couldn't drop database %s: %s" % (db_name, e))
 184         else:
 185             _logger.info('DROP DB: %s', db_name)
 186
 187     fs = openerp.tools.config.filestore(db_name)
 188     if os.path.exists(fs):
 189         shutil.rmtree(fs)
 190     return True
 191
 192 def _set_pg_password_in_environment(func):
 193     """ On systems where pg_restore/pg_dump require an explicit
 194     password (i.e. when not connecting via unix sockets, and most
 195     importantly on Windows), it is necessary to pass the PG user
 196     password in the environment or in a special .pgpass file.
 197
 198     This decorator handles setting
 199     :envvar:`PGPASSWORD` if it is not already
 200     set, and removing it afterwards.
 201
 202     See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
 203
 204     .. note:: This is not thread-safe, and should never be enabled for
 205          SaaS (giving SaaS users the super-admin password is not a good idea
 206          anyway)
 207     """
 208     @wraps(func)
 209     def wrapper(*args, **kwargs):
 210         if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
 211             return func(*args, **kwargs)
 212         else:
 213             os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
 214             try:
 215                 return func(*args, **kwargs)
 216             finally:
 217                 del os.environ['PGPASSWORD']
 218     return wrapper
 219
 220 def exp_dump(db_name):
 221     with tempfile.TemporaryFile() as t:
 222         dump_db(db_name, t)
 223         t.seek(0)
 224         return t.read().encode('base64')
 225
 226 @_set_pg_password_in_environment
 227 def dump_db(db, stream):
 228     """Dump database `db` into file-like object `stream`"""
 229     with openerp.tools.osutil.tempdir() as dump_dir:
 230         registry = openerp.modules.registry.RegistryManager.get(db)
 231         with registry.cursor() as cr:
 232             filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
 233             if os.path.exists(filestore):
 234                 shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
 235
 236         dump_file = os.path.join(dump_dir, 'dump.sql')
 237         cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
 238         if openerp.tools.config['db_user']:
 239             cmd.append('--username=' + openerp.tools.config['db_user'])
 240         if openerp.tools.config['db_host']:
 241             cmd.append('--host=' + openerp.tools.config['db_host'])
 242         if openerp.tools.config['db_port']:
 243             cmd.append('--port=' + str(openerp.tools.config['db_port']))
 244         cmd.append(db)
 245
 246         if openerp.tools.exec_pg_command(*cmd):
 247             _logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
 248                           'password on the server. You may need to create a .pgpass file for '
 249                           'authentication, or specify `db_password` in the server configuration '
 250                           'file.', db)
 251             raise Exception("Couldn't dump database")
 252
 253         openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
 254
 255     _logger.info('DUMP DB successful: %s', db)
 256
 257 def exp_restore(db_name, data, copy=False):
 258     data_file = tempfile.NamedTemporaryFile(delete=False)
 259     try:
 260         data_file.write(data.decode('base64'))
 261         data_file.close()
 262         restore_db(db_name, data_file.name, copy=copy)
 263     finally:
 264         os.unlink(data_file.name)
 265     return True
 266
 267 @_set_pg_password_in_environment
 268 def restore_db(db, dump_file, copy=False):
 269     assert isinstance(db, basestring)
 270     if exp_db_exist(db):
 271         _logger.warning('RESTORE DB: %s already exists', db)
 272         raise Exception("Database already exists")
 273
 274     _create_empty_database(db)
 275
 276     filestore_path = None
 277     with openerp.tools.osutil.tempdir() as dump_dir:
 278         if zipfile.is_zipfile(dump_file):
 279             # v8 format
 280             with zipfile.ZipFile(dump_file, 'r') as z:
 281                 # only extract known members!
 282                 filestore = [m for m in z.namelist() if m.startswith('filestore/')]
 283                 z.extractall(dump_dir, ['dump.sql'] + filestore)
 284
 285                 if filestore:
 286                     filestore_path = os.path.join(dump_dir, 'filestore')
 287
 288             pg_cmd = 'psql'
 289             pg_args = ['-q', '-f', os.path.join(dump_dir, 'dump.sql')]
 290
 291         else:
 292             # <= 7.0 format (raw pg_dump output)
 293             pg_cmd = 'pg_restore'
 294             pg_args = ['--no-owner', dump_file]
 295
 296         args = []
 297         if openerp.tools.config['db_user']:
 298             args.append('--username=' + openerp.tools.config['db_user'])
 299         if openerp.tools.config['db_host']:
 300             args.append('--host=' + openerp.tools.config['db_host'])
 301         if openerp.tools.config['db_port']:
 302             args.append('--port=' + str(openerp.tools.config['db_port']))
 303         args.append('--dbname=' + db)
 304         pg_args = args + pg_args
 305
 306         if openerp.tools.exec_pg_command(pg_cmd, *pg_args):
 307             raise Exception("Couldn't restore database")
 308
 309         registry = openerp.modules.registry.RegistryManager.new(db)
 310         with registry.cursor() as cr:
 311             if copy:
 312                 # if it's a copy of a database, force generation of a new dbuuid
 313                 registry['ir.config_parameter'].init(cr, force=True)
 314             if filestore_path:
 315                 filestore_dest = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
 316                 shutil.move(filestore_path, filestore_dest)
 317
 318             if openerp.tools.config['unaccent']:
 319                 try:
 320                     with cr.savepoint():
 321                         cr.execute("CREATE EXTENSION unaccent")
 322                 except psycopg2.Error:
 323                     pass
 324
 325     _logger.info('RESTORE DB: %s', db)
 326
 327 def exp_rename(old_name, new_name):
 328     openerp.modules.registry.RegistryManager.delete(old_name)
 329     openerp.sql_db.close_db(old_name)
 330
 331     db = openerp.sql_db.db_connect('postgres')
 332     with closing(db.cursor()) as cr:
 333         cr.autocommit(True)     # avoid transaction block
 334         try:
 335             cr.execute('ALTER DATABASE "%s" RENAME TO "%s"' % (old_name, new_name))
 336             _logger.info('RENAME DB: %s -> %s', old_name, new_name)
 337         except Exception, e:
 338             _logger.error('RENAME DB: %s -> %s failed:\n%s', old_name, new_name, e)
 339             raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
 340
 341     old_fs = openerp.tools.config.filestore(old_name)
 342     new_fs = openerp.tools.config.filestore(new_name)
 343     if os.path.exists(old_fs) and not os.path.exists(new_fs):
 344         shutil.move(old_fs, new_fs)
 345     return True
 346
 347 @openerp.tools.mute_logger('openerp.sql_db')
 348 def exp_db_exist(db_name):
 349     ## Not True: in fact, check if connection to database is possible. The database may exists
 350     return bool(openerp.sql_db.db_connect(db_name))
 351
 352 def exp_list(document=False):
 353     if not openerp.tools.config['list_db'] and not document:
 354         raise openerp.exceptions.AccessDenied()
 355     chosen_template = openerp.tools.config['db_template']
 356     templates_list = tuple(set(['template0', 'template1', 'postgres', chosen_template]))
 357     db = openerp.sql_db.db_connect('postgres')
 358     with closing(db.cursor()) as cr:
 359         try:
 360             db_user = openerp.tools.config["db_user"]
 361             if not db_user and os.name == 'posix':
 362                 import pwd
 363                 db_user = pwd.getpwuid(os.getuid())[0]
 364             if not db_user:
 365                 cr.execute("select usename from pg_user where usesysid=(select datdba from pg_database where datname=%s)", (openerp.tools.config["db_name"],))
 366                 res = cr.fetchone()
 367                 db_user = res and str(res[0])
 368             if db_user:
 369                 cr.execute("select datname from pg_database where datdba=(select usesysid from pg_user where usename=%s) and datname not in %s order by datname", (db_user, templates_list))
 370             else:
 371                 cr.execute("select datname from pg_database where datname not in %s order by datname", (templates_list,))
 372             res = [openerp.tools.ustr(name) for (name,) in cr.fetchall()]
 373         except Exception:
 374             res = []
 375     res.sort()
 376     return res
 377
 378 def exp_change_admin_password(new_password):
 379     openerp.tools.config['admin_passwd'] = new_password
 380     openerp.tools.config.save()
 381     return True
 382
 383 def exp_list_lang():
 384     return openerp.tools.scan_languages()
 385
 386 def exp_server_version():
 387     """ Return the version of the server
 388         Used by the client to verify the compatibility with its own version
 389     """
 390     return openerp.release.version
 391
 392 def exp_migrate_databases(databases):
 393     for db in databases:
 394         _logger.info('migrate database %s', db)
 395         openerp.tools.config['update']['base'] = True
 396         openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True)
 397     return True
 398
 399 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: