import base64
import errno
import logging
+import shutil
from StringIO import StringIO
import psycopg2
import random
import string
import pooler
-import netsvc
import nodes
from content_index import cntIndex
We have to consider 3 cases of data /retrieval/:
Given (context,path) we need to access the file (aka. node).
given (directory, context), we need one of its children (for listings, views)
- given (ir.attachment, context), we needs its data and metadata (node).
+ given (ir.attachment, context), we need its data and metadata (node).
For data /storage/ we have the cases:
Have (ir.attachment, context), we modify the file (save, update, rename etc).
if mode.endswith('b'):
mode = mode[:-1]
self.mode = mode
+ self._size = os.stat(path).st_size
- for attr in ('closed', 'read', 'write', 'seek', 'tell'):
+ for attr in ('closed', 'read', 'write', 'seek', 'tell', 'next'):
setattr(self,attr, getattr(self.__file, attr))
+ def size(self):
+ return self._size
+
+ def __iter__(self):
+ return self
+
def close(self):
# TODO: locking in init, close()
fname = self.__file.name
" WHERE id = %s",
(fsize, par.file_id))
par.content_length = fsize
- par.content_type = mime
cr.commit()
cr.close()
except Exception:
"""
def __init__(self, parent, ira_browse, mode):
nodes.node_descriptor.__init__(self, parent)
+ self._size = 0L
if mode.endswith('b'):
mode = mode[:-1]
cr = ira_browse._cr # reuse the cursor of the browse object, just now
cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,))
data = cr.fetchone()[0]
+ if data:
+ self._size = len(data)
StringIO.__init__(self, data)
elif mode in ('w', 'w+'):
StringIO.__init__(self, None)
raise IOError(errno.EINVAL, "Invalid file mode")
self.mode = mode
+ def size(self):
+ return self._size
+
def close(self):
# we now open a *separate* cursor, to update the data.
# FIXME: this may be improved, for concurrency handling
par = self._get_parent()
- uid = par.context.uid
+ # uid = par.context.uid
cr = pooler.get_db(par.context.dbname).cursor()
try:
if self.mode in ('w', 'w+', 'r+'):
" WHERE id = %s",
(out, len(data), par.file_id))
cr.commit()
- except Exception, e:
+ except Exception:
logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
raise
finally:
"""
def __init__(self, parent, ira_browse, mode):
nodes.node_descriptor.__init__(self, parent)
+ self._size = 0L
if mode.endswith('b'):
mode = mode[:-1]
if mode in ('r', 'r+'):
- StringIO.__init__(self, base64.decodestring(ira_browse.db_datas))
+ data = base64.decodestring(ira_browse.db_datas)
+ if data:
+ self._size = len(data)
+ StringIO.__init__(self, data)
elif mode in ('w', 'w+'):
StringIO.__init__(self, None)
# at write, we start at 0 (= overwrite), but have the original
raise IOError(errno.EINVAL, "Invalid file mode")
self.mode = mode
+ def size(self):
+ return self._size
+
def close(self):
# we now open a *separate* cursor, to update the data.
# FIXME: this may be improved, for concurrency handling
par = self._get_parent()
- uid = par.context.uid
+ # uid = par.context.uid
cr = pooler.get_db(par.context.dbname).cursor()
try:
if self.mode in ('w', 'w+', 'r+'):
cr.execute('UPDATE ir_attachment SET db_datas = %s::bytea, file_size=%s, ' \
'index_content = %s, file_type = %s ' \
'WHERE id = %s',
- (base64.encodestring(out), len(out), icont_u, mime, par.file_id))
+ (base64.encodestring(data), len(data), icont_u, mime, par.file_id))
elif self.mode == 'a':
- out = self.getvalue()
+ data = self.getvalue()
# Yes, we're obviously using the wrong representation for storing our
# data as base64-in-bytea
cr.execute("UPDATE ir_attachment " \
"SET db_datas = encode( (COALESCE(decode(encode(db_datas,'escape'),'base64'),'') || decode(%s, 'base64')),'base64')::bytea , " \
" file_size = COALESCE(file_size, 0) + %s " \
" WHERE id = %s",
- (base64.encodestring(out), len(out), par.file_id))
+ (base64.encodestring(data), len(data), par.file_id))
cr.commit()
- except Exception, e:
+ except Exception:
logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
raise
finally:
filename = random_name()
return os.path.join(flag, filename)
+ def __prepare_realpath(self, cr, file_node, ira, store_path, do_create=True):
+ """ Cleanup path for realstore, create dirs if needed
+
+ @param file_node the node
+ @param ira ir.attachment browse of the file_node
+ @param store_path the path of the parent storage object, list
+ @param do_create create the directories, if needed
+
+ @return tuple(path "/var/filestore/real/dir/", npath ['dir','fname.ext'] )
+ """
+ file_node.fix_ppath(cr, ira)
+ npath = file_node.full_path() or []
+ # npath may contain empty elements, for root directory etc.
+ npath = filter(lambda x: x is not None, npath)
+
+ # if self._debug:
+ # self._doclog.debug('Npath: %s', npath)
+ for n in npath:
+ if n == '..':
+ raise ValueError("Invalid '..' element in path")
+ for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?',):
+ if ch in n:
+ raise ValueError("Invalid char %s in path %s" %(ch, n))
+ dpath = [store_path,]
+ dpath += npath[:-1]
+ path = os.path.join(*dpath)
+ if not os.path.isdir(path):
+ self._doclog.debug("Create dirs: %s", path)
+ os.makedirs(path)
+ return path, npath
+
def get_data(self, cr, uid, id, file_node, context=None, fil_obj=None):
""" retrieve the contents of some file_node having storage_id = id
optionally, fil_obj could point to the browse object of the file
(ir.attachment)
"""
- if not context:
- context = {}
- boo = self.browse(cr, uid, id, context)
+ boo = self.browse(cr, uid, id, context=context)
+ if not boo.online:
+ raise IOError(errno.EREMOTE, 'medium offline')
+
if fil_obj:
ira = fil_obj
else:
"""
if context is None:
context = {}
- boo = self.browse(cr, uid, id, context)
+ boo = self.browse(cr, uid, id, context=context)
if not boo.online:
- raise RuntimeError('media offline')
+ raise IOError(errno.EREMOTE, 'medium offline')
+
+ if boo.readonly and mode not in ('r', 'rb'):
+ raise IOError(errno.EPERM, "Readonly medium")
ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
if boo.type == 'filestore':
return nodefd_db64(file_node, ira_browse=ira, mode=mode)
elif boo.type == 'realstore':
- if not ira.store_fname:
- # On a migrated db, some files may have the wrong storage type
- # try to fix their directory.
- if ira.file_size:
- self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id)
- sfname = ira.name
- fpath = os.path.join(boo.path,ira.store_fname or ira.name)
- if (not os.path.exists(fpath)) and mode in ('r','r+'):
+ path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path,
+ do_create = (mode[0] in ('w','a')) )
+ fpath = os.path.join(path, npath[-1])
+ if (not os.path.exists(fpath)) and mode[0] == 'r':
raise IOError("File not found: %s" % fpath)
+ elif mode[0] in ('w', 'a') and not ira.store_fname:
+ store_fname = os.path.join(*npath)
+ cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s',
+ (store_fname, ira.id))
return nodefd_file(file_node, path=fpath, mode=mode)
elif boo.type == 'virtual':
raise TypeError("No %s storage" % boo.type)
def __get_data_3(self, cr, uid, boo, ira, context):
- if not boo.online:
- raise RuntimeError('media offline')
if boo.type == 'filestore':
if not ira.store_fname:
# On a migrated db, some files may have the wrong storage type
# try to fix their directory.
if ira.file_size:
self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id)
- sfname = ira.name
+ # sfname = ira.name
fpath = os.path.join(boo.path,ira.store_fname or ira.name)
if os.path.exists(fpath):
return file(fpath,'rb').read()
elif not ira.store_fname:
return None
else:
- raise IOError("File not found: %s" % fpath)
+ raise IOError(errno.ENOENT, "File not found: %s" % fpath)
elif boo.type == 'virtual':
raise ValueError('Virtual storage does not support static files')
This function MUST be used from an ir.attachment. It wouldn't make sense
to store things persistently for other types (dynamic).
"""
- if not context:
- context = {}
- boo = self.browse(cr, uid, id, context)
+ boo = self.browse(cr, uid, id, context=context)
if fil_obj:
ira = fil_obj
else:
ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
if not boo.online:
- raise RuntimeError('media offline')
+ raise IOError(errno.EREMOTE, 'medium offline')
+
+ if boo.readonly:
+ raise IOError(errno.EPERM, "Readonly medium")
+
self._doclog.debug( "Store data for ir.attachment #%d" % ira.id)
store_fname = None
fname = None
try:
store_fname = self.__get_random_fname(path)
fname = os.path.join(path, store_fname)
- fp = file(fname, 'wb')
- fp.write(data)
- fp.close()
+ fp = open(fname, 'wb')
+ try:
+ fp.write(data)
+ finally:
+ fp.close()
self._doclog.debug( "Saved data to %s" % fname)
filesize = len(data) # os.stat(fname).st_size
(out, file_node.file_id))
elif boo.type == 'realstore':
try:
- file_node.fix_ppath(cr, ira)
- npath = file_node.full_path() or []
- # npath may contain empty elements, for root directory etc.
- for i, n in enumerate(npath):
- if n == None:
- del npath[i]
- for n in npath:
- for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
- if ch in n:
- raise ValueError("Invalid char %s in path %s" %(ch, n))
- dpath = [boo.path,]
- dpath += npath[:-1]
- path = os.path.join(*dpath)
- if not os.path.isdir(path):
- os.makedirs(path)
+ path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path, do_create=True)
fname = os.path.join(path, npath[-1])
- fp = file(fname,'wb')
- fp.write(data)
- fp.close()
+ fp = open(fname,'wb')
+ try:
+ fp.write(data)
+ finally:
+ fp.close()
self._doclog.debug("Saved data to %s", fname)
filesize = len(data) # os.stat(fname).st_size
store_fname = os.path.join(*npath)
files that have to be removed, too. """
if not storage_bo.online:
- raise RuntimeError('media offline')
+ raise IOError(errno.EREMOTE, 'medium offline')
+
+ if storage_bo.readonly:
+ raise IOError(errno.EPERM, "Readonly medium")
if storage_bo.type == 'filestore':
fname = fil_bo.store_fname
if ktype == 'file':
try:
os.unlink(fname)
- except Exception, e:
+ except Exception:
self._doclog.warning("Could not remove file %s, please remove manually.", fname, exc_info=True)
else:
self._doclog.warning("Unknown unlink key %s" % ktype)
"""
sbro = self.browse(cr, uid, file_node.storage_id, context=context)
assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+
+ if not sbro.online:
+ raise IOError(errno.EREMOTE, 'medium offline')
+ if sbro.readonly:
+ raise IOError(errno.EPERM, "Readonly medium")
+
if sbro.type in ('filestore', 'db', 'db64'):
# nothing to do for a rename, allow to change the db field
return { 'name': new_name, 'datas_fname': new_name }
elif sbro.type == 'realstore':
- fname = fil_bo.store_fname
+ ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
+
+ path, npath = self.__prepare_realpath(cr, file_node, ira, sbro.path, do_create=False)
+ fname = ira.store_fname
+
if not fname:
- return ValueError("Tried to rename a non-stored file")
- path = storage_bo.path
- oldpath = os.path.join(path, fname)
-
- for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
- if ch in new_name:
- raise ValueError("Invalid char %s in name %s" %(ch, new_name))
-
- file_node.fix_ppath(cr, ira)
- npath = file_node.full_path() or []
- dpath = [path,]
- dpath.extend(npath[:-1])
- dpath.append(new_name)
- newpath = os.path.join(*dpath)
- # print "old, new paths:", oldpath, newpath
+ self._doclog.warning("Trying to rename a non-stored file")
+ if fname != os.path.join(*npath):
+ self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(npath))
+
+ oldpath = os.path.join(path, npath[-1])
+ newpath = os.path.join(path, new_name)
os.rename(oldpath, newpath)
- return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name }
+ store_path = npath[:-1]
+ store_path.append(new_name)
+ store_fname = os.path.join(*store_path)
+ return { 'name': new_name, 'datas_fname': new_name, 'store_fname': store_fname }
else:
- raise TypeError("No %s storage" % boo.type)
+ raise TypeError("No %s storage" % sbro.type)
def simple_move(self, cr, uid, file_node, ndir_bro, context=None):
""" A preparation for a file move.
sbro = self.browse(cr, uid, file_node.storage_id, context=context)
assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+ if not sbro.online:
+ raise IOError(errno.EREMOTE, 'medium offline')
+
+ if sbro.readonly:
+ raise IOError(errno.EPERM, "Readonly medium")
+
par = ndir_bro
psto = None
while par:
# nothing to do for a rename, allow to change the db field
return { 'parent_id': ndir_bro.id }
elif sbro.type == 'realstore':
- raise NotImplementedError("Cannot move in realstore, yet") # TODO
- fname = fil_bo.store_fname
+ ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
+
+ path, opath = self.__prepare_realpath(cr, file_node, ira, sbro.path, do_create=False)
+ fname = ira.store_fname
+
if not fname:
- return ValueError("Tried to rename a non-stored file")
- path = storage_bo.path
- oldpath = os.path.join(path, fname)
+ self._doclog.warning("Trying to rename a non-stored file")
+ if fname != os.path.join(*opath):
+ self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(opath))
+
+ oldpath = os.path.join(path, opath[-1])
- for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
- if ch in new_name:
- raise ValueError("Invalid char %s in name %s" %(ch, new_name))
-
- file_node.fix_ppath(cr, ira)
- npath = file_node.full_path() or []
- dpath = [path,]
- dpath.extend(npath[:-1])
- dpath.append(new_name)
- newpath = os.path.join(*dpath)
- # print "old, new paths:", oldpath, newpath
- os.rename(oldpath, newpath)
- return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name }
+ npath = [sbro.path,] + (ndir_bro.get_full_path() or [])
+ npath = filter(lambda x: x is not None, npath)
+ newdir = os.path.join(*npath)
+ if not os.path.isdir(newdir):
+ self._doclog.debug("Must create dir %s", newdir)
+ os.makedirs(newdir)
+ npath.append(opath[-1])
+ newpath = os.path.join(*npath)
+
+ self._doclog.debug("Going to move %s from %s to %s", opath[-1], oldpath, newpath)
+ shutil.move(oldpath, newpath)
+
+ store_path = npath[1:] + [opath[-1],]
+ store_fname = os.path.join(*store_path)
+
+ return { 'store_fname': store_fname }
else:
- raise TypeError("No %s storage" % boo.type)
+ raise TypeError("No %s storage" % sbro.type)
document_storage()
projects / odoo / odoo.git / blobdiff