[ARVADOS] updated: 5549904bbd5dec9bafe60e36d4ea1abe6b791f19
git at public.curoverse.com
git at public.curoverse.com
Thu Apr 16 16:26:44 EDT 2015
Summary of changes:
sdk/python/arvados/arvfile.py | 48 +++++--
sdk/python/arvados/collection.py | 22 ++--
services/fuse/arvados_fuse/__init__.py | 184 +++++++++++++++++++++-----
services/fuse/arvados_fuse/fusedir.py | 7 +-
services/fuse/arvados_fuse/fusefile.py | 14 ++
services/fuse/tests/test_mount.py | 231 ++++++++++++++++++++++++++++++++-
6 files changed, 446 insertions(+), 60 deletions(-)
via 5549904bbd5dec9bafe60e36d4ea1abe6b791f19 (commit)
from 23a940aaf47411cd07615b3301d24d3868a11bce (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5549904bbd5dec9bafe60e36d4ea1abe6b791f19
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Apr 16 16:26:41 2015 -0400
3198: Support for mkdir, rmdir, unlink, rename. Improve exception catching.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 13bb507..b4102f4 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -310,7 +310,7 @@ class _BufferBlock(object):
self.buffer_view = None
self.buffer_block = None
else:
- raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
+ raise AssertionError("Invalid state change from %s to %s" % (self.state, nextstate))
@synchronized
def state(self):
@@ -354,7 +354,7 @@ def must_be_writable(orig_func):
@functools.wraps(orig_func)
def must_be_writable_wrapper(self, *args, **kwargs):
if not self.writable():
- raise IOError((errno.EROFS, "Collection must be writable."))
+ raise IOError(errno.EROFS, "Collection must be writable.")
return orig_func(self, *args, **kwargs)
return must_be_writable_wrapper
@@ -719,7 +719,7 @@ class ArvadosFile(object):
self._segments = new_segs
self._modified = True
elif size > self.size():
- raise IOError("truncate() does not support extending the file size")
+ raise IOError(errno.EINVAL, "truncate() does not support extending the file size")
def readfrom(self, offset, size, num_retries):
"""Read upto `size` bytes from the file starting at `offset`."""
@@ -800,12 +800,17 @@ class ArvadosFile(object):
replace_range(self._segments, offset, len(data), self._current_bblock.blockid, self._current_bblock.write_pointer - len(data))
+ self.parent.notify(MOD, self.parent, self.name, (self, self))
+
+ return len(data)
+
@synchronized
def flush(self):
- if self._current_bblock:
- self._repack_writes()
- self.parent._my_block_manager().commit_bufferblock(self._current_bblock)
- self.parent.notify(MOD, self.parent, self.name, (self, self))
+ if self.modified():
+ if self._current_bblock and self._current_bblock.state() == _BufferBlock.WRITABLE:
+ self._repack_writes()
+ self.parent._my_block_manager().commit_bufferblock(self._current_bblock)
+ self.parent.notify(MOD, self.parent, self.name, (self, self))
@must_be_writable
@synchronized
@@ -872,16 +877,32 @@ class ArvadosFileReader(ArvadosFileReaderBase):
@_FileLikeObjectBase._before_close
@retry_method
- def read(self, size, num_retries=None):
- """Read up to `size` bytes from the stream, starting at the current file position."""
- data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
- self._filepos += len(data)
- return data
+ def read(self, size=None, num_retries=None):
+ """Read up to `size` bytes from the file and return the result.
+
+ Starts at the current file position. If `size` is None, read the
+ entire remainder of the file.
+ """
+ if size is None:
+ data = []
+ rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+ while rd:
+ data.append(rd)
+ self._filepos += len(rd)
+ rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+ return ''.join(data)
+ else:
+ data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
+ self._filepos += len(data)
+ return data
@_FileLikeObjectBase._before_close
@retry_method
def readfrom(self, offset, size, num_retries=None):
- """Read up to `size` bytes from the stream, starting at the current file position."""
+ """Read up to `size` bytes from the stream, starting at the specified file offset.
+
+ This method does not change the file position.
+ """
return self.arvadosfile.readfrom(offset, size, num_retries)
def flush(self):
@@ -907,6 +928,7 @@ class ArvadosFileWriter(ArvadosFileReader):
else:
self.arvadosfile.writeto(self._filepos, data, num_retries)
self._filepos += len(data)
+ return len(data)
@_FileLikeObjectBase._before_close
@retry_method
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 8091e8b..d610c35 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -555,7 +555,7 @@ class RichCollectionBase(CollectionBase):
if isinstance(item, RichCollectionBase):
return item.find_or_create(pathcomponents[1], create_type)
else:
- raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
+ raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
else:
return self
@@ -581,9 +581,9 @@ class RichCollectionBase(CollectionBase):
else:
return item
else:
- raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
+ raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
- def mkdirs(path):
+ def mkdirs(self, path):
"""Recursive subcollection create.
Like `os.mkdirs()`. Will create intermediate subcollections needed to
@@ -616,7 +616,7 @@ class RichCollectionBase(CollectionBase):
create = (mode != "r")
if create and not self.writable():
- raise IOError((errno.EROFS, "Collection is read only"))
+ raise IOError(errno.EROFS, "Collection is read only")
if create:
arvfile = self.find_or_create(path, FILE)
@@ -624,9 +624,9 @@ class RichCollectionBase(CollectionBase):
arvfile = self.find(path)
if arvfile is None:
- raise IOError((errno.ENOENT, "File not found"))
+ raise IOError(errno.ENOENT, "File not found")
if not isinstance(arvfile, ArvadosFile):
- raise IOError((errno.EISDIR, "Path must refer to a file."))
+ raise IOError(errno.EISDIR, "Path must refer to a file.")
if mode[0] == "w":
arvfile.truncate(0)
@@ -721,10 +721,10 @@ class RichCollectionBase(CollectionBase):
pathcomponents = path.split("/", 1)
item = self._items.get(pathcomponents[0])
if item is None:
- raise IOError((errno.ENOENT, "File not found"))
+ raise IOError(errno.ENOENT, "File not found")
if len(pathcomponents) == 1:
if isinstance(self._items[pathcomponents[0]], RichCollectionBase) and len(self._items[pathcomponents[0]]) > 0 and not recursive:
- raise IOError((errno.ENOTEMPTY, "Subcollection not empty"))
+ raise IOError(errno.ENOTEMPTY, "Subcollection not empty")
deleteditem = self._items[pathcomponents[0]]
del self._items[pathcomponents[0]]
self._modified = True
@@ -757,7 +757,7 @@ class RichCollectionBase(CollectionBase):
"""
if target_name in self and not overwrite:
- raise IOError((errno.EEXIST, "File already exists"))
+ raise IOError(errno.EEXIST, "File already exists")
modified_from = None
if target_name in self:
@@ -800,7 +800,7 @@ class RichCollectionBase(CollectionBase):
if isinstance(source, basestring):
source_obj = source_collection.find(source)
if source_obj is None:
- raise IOError((errno.ENOENT, "File not found"))
+ raise IOError(errno.ENOENT, "File not found")
sourcecomponents = source.split("/")
else:
source_obj = source
@@ -1380,6 +1380,7 @@ class Collection(RichCollectionBase):
segments = []
streamoffset = 0L
state = BLOCKS
+ self.mkdirs(stream_name)
continue
if state == BLOCKS:
@@ -1432,6 +1433,7 @@ class Subcollection(RichCollectionBase):
self.lock = self.root_collection().lock
self._manifest_text = None
self.name = name
+ self.num_retries = parent.num_retries
def root_collection(self):
return self.parent.root_collection()
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b376d64..67878d7 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -22,42 +22,48 @@ import threading
import itertools
import ciso8601
import collections
+import functools
from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
from fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
-# log_handler = logging.StreamHandler()
-# llogger = logging.getLogger('llfuse')
-# llogger.addHandler(log_handler)
-# llogger.setLevel(logging.DEBUG)
+log_handler = logging.StreamHandler()
+llogger = logging.getLogger('llfuse')
+llogger.addHandler(log_handler)
+llogger.setLevel(logging.DEBUG)
-class FileHandle(object):
- """Connects a numeric file handle to a File object that has
+class Handle(object):
+ """Connects a numeric file handle to a File or Directory object that has
been opened by the client."""
- def __init__(self, fh, fileobj):
+ def __init__(self, fh, obj):
self.fh = fh
- self.fileobj = fileobj
- self.fileobj.inc_use()
+ self.obj = obj
+ self.obj.inc_use()
def release(self):
- self.fileobj.dec_use()
+ self.obj.dec_use()
+ def flush(self):
+ with llfuse.lock_released:
+ return self.obj.flush()
-class DirectoryHandle(object):
+
+class FileHandle(Handle):
+ """Connects a numeric file handle to a File object that has
+ been opened by the client."""
+ pass
+
+
+class DirectoryHandle(Handle):
"""Connects a numeric file handle to a Directory object that has
been opened by the client."""
def __init__(self, fh, dirobj, entries):
- self.fh = fh
+ super(DirectoryHandle, self).__init__(fh, dirobj)
self.entries = entries
- self.dirobj = dirobj
- self.dirobj.inc_use()
-
- def release(self):
- self.dirobj.dec_use()
class InodeCache(object):
@@ -106,6 +112,7 @@ class InodeCache(object):
if obj.persisted() and obj._cache_priority in self._entries:
self._remove(obj, True)
+
class Inodes(object):
"""Manage the set of inodes. This is the mapping from a numeric id
to a concrete File or Directory object"""
@@ -148,6 +155,21 @@ class Inodes(object):
llfuse.invalidate_inode(entry.inode)
del self._entries[entry.inode]
+def catch_exceptions(orig_func):
+ @functools.wraps(orig_func)
+ def catch_exceptions_wrapper(self, *args, **kwargs):
+ try:
+ return orig_func(self, *args, **kwargs)
+ except llfuse.FUSEError:
+ raise
+ except EnvironmentError as e:
+ raise llfuse.FUSEError(e.errno)
+ except Exception:
+ _logger.exception("")
+ raise llfuse.FUSEError(errno.EIO)
+
+ return catch_exceptions_wrapper
+
class Operations(llfuse.Operations):
"""This is the main interface with llfuse.
@@ -187,6 +209,7 @@ class Operations(llfuse.Operations):
def access(self, inode, mode, ctx):
return True
+ @catch_exceptions
def getattr(self, inode):
if inode not in self.inodes:
raise llfuse.FUSEError(errno.ENOENT)
@@ -225,6 +248,7 @@ class Operations(llfuse.Operations):
return entry
+ @catch_exceptions
def lookup(self, parent_inode, name):
name = unicode(name, self.encoding)
_logger.debug("arv-mount lookup: parent_inode %i name %s",
@@ -246,24 +270,26 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.ENOENT)
+ @catch_exceptions
def open(self, inode, flags):
if inode in self.inodes:
p = self.inodes[inode]
else:
raise llfuse.FUSEError(errno.ENOENT)
- if (flags & os.O_WRONLY) or (flags & os.O_RDWR):
- raise llfuse.FUSEError(errno.EROFS)
-
if isinstance(p, Directory):
raise llfuse.FUSEError(errno.EISDIR)
+ if ((flags & os.O_WRONLY) or (flags & os.O_RDWR)) and not p.writable():
+ raise llfuse.FUSEError(errno.EPERM)
+
fh = self._filehandles_counter
self._filehandles_counter += 1
self._filehandles[fh] = FileHandle(fh, p)
self.inodes.touch(p)
return fh
+ @catch_exceptions
def read(self, fh, off, size):
_logger.debug("arv-mount read %i %i %i", fh, off, size)
if fh in self._filehandles:
@@ -271,20 +297,40 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.EBADF)
- self.inodes.touch(handle.fileobj)
+ self.inodes.touch(handle.obj)
try:
with llfuse.lock_released:
- return handle.fileobj.readfrom(off, size, self.num_retries)
+ return handle.obj.readfrom(off, size, self.num_retries)
except arvados.errors.NotFoundError as e:
_logger.warning("Block not found: " + str(e))
raise llfuse.FUSEError(errno.EIO)
- except Exception:
- _logger.exception("Read error")
- raise llfuse.FUSEError(errno.EIO)
+ @catch_exceptions
+ def write(self, fh, off, buf):
+ _logger.debug("arv-mount write %i %i %i", fh, off, len(buf))
+ if fh in self._filehandles:
+ handle = self._filehandles[fh]
+ else:
+ raise llfuse.FUSEError(errno.EBADF)
+
+ if not handle.obj.writable():
+ raise llfuse.FUSEError(errno.EPERM)
+
+ self.inodes.touch(handle.obj)
+
+ with llfuse.lock_released:
+ return handle.obj.writeto(off, buf, self.num_retries)
+
+ @catch_exceptions
def release(self, fh):
if fh in self._filehandles:
+ try:
+ self._filehandles[fh].flush()
+ except EnvironmentError as e:
+ raise llfuse.FUSEError(e.errno)
+ except Exception:
+ _logger.exception("Flush error")
self._filehandles[fh].release()
del self._filehandles[fh]
self.inodes.cap_cache()
@@ -292,6 +338,7 @@ class Operations(llfuse.Operations):
def releasedir(self, fh):
self.release(fh)
+ @catch_exceptions
def opendir(self, inode):
_logger.debug("arv-mount opendir: inode %i", inode)
@@ -316,7 +363,7 @@ class Operations(llfuse.Operations):
self._filehandles[fh] = DirectoryHandle(fh, p, [('.', p), ('..', parent)] + list(p.items()))
return fh
-
+ @catch_exceptions
def readdir(self, fh, off):
_logger.debug("arv-mount readdir: fh %i off %i", fh, off)
@@ -325,7 +372,7 @@ class Operations(llfuse.Operations):
else:
raise llfuse.FUSEError(errno.EBADF)
- _logger.debug("arv-mount handle.dirobj %s", handle.dirobj)
+ _logger.debug("arv-mount handle.dirobj %s", handle.obj)
e = off
while e < len(handle.entries):
@@ -336,6 +383,7 @@ class Operations(llfuse.Operations):
pass
e += 1
+ @catch_exceptions
def statfs(self):
st = llfuse.StatvfsData()
st.f_bsize = 64 * 1024
@@ -351,12 +399,78 @@ class Operations(llfuse.Operations):
st.f_frsize = 0
return st
- # The llfuse documentation recommends only overloading functions that
- # are actually implemented, as the default implementation will raise ENOSYS.
- # However, there is a bug in the llfuse default implementation of create()
- # "create() takes exactly 5 positional arguments (6 given)" which will crash
- # arv-mount.
- # The workaround is to implement it with the proper number of parameters,
- # and then everything works out.
+ def _check_writable(self, inode_parent):
+ if inode_parent in self.inodes:
+ p = self.inodes[inode_parent]
+ else:
+ raise llfuse.FUSEError(errno.ENOENT)
+
+ if not isinstance(p, Directory):
+ raise llfuse.FUSEError(errno.ENOTDIR)
+
+ if not p.writable():
+ raise llfuse.FUSEError(errno.EPERM)
+
+ if not isinstance(p, CollectionDirectoryBase):
+ raise llfuse.FUSEError(errno.EPERM)
+
+ return p
+
+ @catch_exceptions
def create(self, inode_parent, name, mode, flags, ctx):
- raise llfuse.FUSEError(errno.EROFS)
+ p = self._check_writable(inode_parent)
+
+ with llfuse.lock_released:
+ p.collection.open(name, "w")
+
+ # The file entry should have been implicitly created by callback.
+ f = p[name]
+
+ fh = self._filehandles_counter
+ self._filehandles_counter += 1
+ self._filehandles[fh] = FileHandle(fh, f)
+ self.inodes.touch(p)
+
+ return (fh, self.getattr(f.inode))
+
+ @catch_exceptions
+ def mkdir(self, inode_parent, name, mode, ctx):
+ p = self._check_writable(inode_parent)
+
+ with llfuse.lock_released:
+ p.collection.mkdirs(name)
+
+ # The dir entry should have been implicitly created by callback.
+ d = p[name]
+
+ return self.getattr(d.inode)
+
+ @catch_exceptions
+ def unlink(self, inode_parent, name):
+ p = self._check_writable(inode_parent)
+
+ with llfuse.lock_released:
+ p.collection.remove(name)
+
+ def rmdir(self, inode_parent, name):
+ self.unlink(inode_parent, name)
+
+ @catch_exceptions
+ def rename(self, inode_parent_old, name_old, inode_parent_new, name_new):
+ src = self._check_writable(inode_parent_old)
+ dest = self._check_writable(inode_parent_new)
+
+ with llfuse.lock_released:
+ dest.collection.copy(name_old, name_new, source_collection=src.collection, overwrite=True)
+ src.collection.remove(name_old)
+
+ @catch_exceptions
+ def flush(self, fh):
+ if fh in self._filehandles:
+ self._filehandles[fh].flush()
+
+ def fsync(self, fh, datasync):
+ self.flush(fh)
+
+ def fsyncdir(self, fh, datasync):
+ self.flush(fh)
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 53f53c7..51c652e 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -167,6 +167,8 @@ class Directory(FreshBase):
def writable(self):
return False
+ def flush(self):
+ pass
class CollectionDirectoryBase(Directory):
def __init__(self, parent_inode, inodes, collection):
@@ -195,6 +197,7 @@ class CollectionDirectoryBase(Directory):
ent = self._entries[name]
llfuse.invalidate_entry(self.inode, name)
llfuse.invalidate_inode(ent.inode)
+ _logger.warn("Finished handling event")
def populate(self, mtime):
self._mtime = mtime
@@ -205,6 +208,8 @@ class CollectionDirectoryBase(Directory):
def writable(self):
return self.collection.writable()
+ def flush(self):
+ self.collection.root_collection().save()
class CollectionDirectory(CollectionDirectoryBase):
"""Represents the root of a directory tree holding a collection."""
@@ -229,7 +234,7 @@ class CollectionDirectory(CollectionDirectoryBase):
return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
def writable(self):
- return self.collection.writable() if self.collection else self._writable
+ return self.collection.writable() if self.collection is not None else self._writable
# Used by arv-web.py to switch the contents of the CollectionDirectory
def change_collection(self, new_locator):
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index fcb7711..07de680 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -21,6 +21,9 @@ class File(FreshBase):
def readfrom(self, off, size, num_retries=0):
return ''
+ def writeto(self, off, size, num_retries=0):
+ raise Exception("Not writable")
+
def mtime(self):
return self._mtime
@@ -30,6 +33,9 @@ class File(FreshBase):
def writable(self):
return False
+ def flush(self):
+ pass
+
class FuseArvadosFile(File):
"""Wraps a ArvadosFile."""
@@ -43,12 +49,20 @@ class FuseArvadosFile(File):
def readfrom(self, off, size, num_retries=0):
return self.arvfile.readfrom(off, size, num_retries)
+ def writeto(self, off, buf, num_retries=0):
+ return self.arvfile.writeto(off, buf, num_retries)
+
def stale(self):
return False
def writable(self):
return self.arvfile.writable()
+ def flush(self):
+ if self.writable():
+ self.arvfile.flush()
+ return self.arvfile.parent.root_collection().save()
+
class StringFile(File):
"""Wrap a simple string as a file"""
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 9a0674b..bb7d871 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -16,6 +16,8 @@ import logging
import run_test_server
+logger = logging.getLogger('arvados.arv-mount')
+
class MountTestBase(unittest.TestCase):
def setUp(self):
self.keeptmp = tempfile.mkdtemp()
@@ -29,7 +31,7 @@ class MountTestBase(unittest.TestCase):
operations = fuse.Operations(os.getuid(), os.getgid(), inode_cache=2)
operations.inodes.add_entry(root_class(
llfuse.ROOT_INODE, operations.inodes, self.api, 0, **root_kwargs))
- llfuse.init(operations, self.mounttmp, [])
+ llfuse.init(operations, self.mounttmp, ['debug'])
threading.Thread(None, llfuse.main).start()
# wait until the driver is finished initializing
operations.initlock.wait()
@@ -319,6 +321,8 @@ class FuseUpdateFileTest(MountTestBase):
with collection.open("file1.txt", "w") as f:
f.write("blub")
+ collection.save_new()
+
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(None, collection)
@@ -342,6 +346,8 @@ class FuseAddFileToCollectionTest(MountTestBase):
with collection.open("file1.txt", "w") as f:
f.write("blub")
+ collection.save_new()
+
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(None, collection)
@@ -364,6 +370,8 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
with collection.open("file2.txt", "w") as f:
f.write("plnp")
+ collection.save_new()
+
m = self.make_mount(fuse.CollectionDirectory)
with llfuse.lock:
m.new_collection(None, collection)
@@ -376,6 +384,227 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
d1 = os.listdir(self.mounttmp)
self.assertEqual(["file1.txt"], d1)
+class FuseCreateFileTest(MountTestBase):
+ def runTest(self):
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertEqual(collection2["manifest_text"], "")
+
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ self.assertNotIn("file1.txt", collection)
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+ pass
+
+ self.assertIn("file1.txt", collection)
+
+ d1 = os.listdir(self.mounttmp)
+ self.assertEqual(["file1.txt"], d1)
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
+
+
+class FuseWriteFileTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ self.assertNotIn("file1.txt", collection)
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ with collection.open("file1.txt") as f:
+ self.assertEqual(f.read(), "Hello world!")
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "r") as f:
+ self.assertEqual(f.read(), "Hello world!")
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseUpdateFileTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "r+") as f:
+ self.assertEqual(f.read(), "Hello world!")
+ f.seek(0)
+ f.write("Hola mundo!")
+ f.seek(0)
+ self.assertEqual(f.read(), "Hola mundo!!")
+
+ with open(os.path.join(self.mounttmp, "file1.txt"), "r") as f:
+ self.assertEqual(f.read(), "Hola mundo!")
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseMkdirTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ with self.assertRaises(IOError):
+ with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+ with self.assertRaises(OSError):
+ os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+ d1 = os.listdir(self.mounttmp)
+ self.assertEqual(["testdir"], d1)
+
+ with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual(["file1.txt"], d1)
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseRmTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+ with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ # Starting manifest
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+ # Can't delete because it's not empty
+ with self.assertRaises(OSError):
+ os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+ d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual(["file1.txt"], d1)
+
+ # Delete file
+ os.remove(os.path.join(self.mounttmp, "testdir", "file1.txt"))
+
+ # Make sure it's empty
+ d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual([], d1)
+
+ # Try to delete it again
+ with self.assertRaises(OSError):
+ os.remove(os.path.join(self.mounttmp, "testdir", "file1.txt"))
+
+ # Can't have empty directories :-( so manifest will be empty.
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertEqual(collection2["manifest_text"], "")
+
+ # Should be able to delete now that it is empty
+ os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+ # Make sure it's empty
+ d1 = os.listdir(os.path.join(self.mounttmp))
+ self.assertEqual([], d1)
+
+ # Try to delete it again
+ with self.assertRaises(OSError):
+ os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+ # manifest should be empty now.
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertEqual(collection2["manifest_text"], "")
+
+
+class FuseMvTest(MountTestBase):
+ def runTest(self):
+ arvados.logger.setLevel(logging.DEBUG)
+
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(None, collection)
+ self.assertTrue(m.writable())
+
+ os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+ with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+ f.write("Hello world!")
+
+ # Starting manifest
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+ d1 = os.listdir(os.path.join(self.mounttmp))
+ self.assertEqual(["testdir"], d1)
+ d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual(["file1.txt"], d1)
+
+ os.rename(os.path.join(self.mounttmp, "testdir", "file1.txt"), os.path.join(self.mounttmp, "file1.txt"))
+
+ d1 = os.listdir(os.path.join(self.mounttmp))
+ self.assertEqual(["file1.txt", "testdir"], sorted(d1))
+ d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+ self.assertEqual([], d1)
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+ self.assertRegexpMatches(collection2["manifest_text"],
+ r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
class FuseUnitTest(unittest.TestCase):
def test_sanitize_filename(self):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list