[ARVADOS] updated: 5549904bbd5dec9bafe60e36d4ea1abe6b791f19

git at public.curoverse.com git at public.curoverse.com
Thu Apr 16 16:26:44 EDT 2015


Summary of changes:
 sdk/python/arvados/arvfile.py          |  48 +++++--
 sdk/python/arvados/collection.py       |  22 ++--
 services/fuse/arvados_fuse/__init__.py | 184 +++++++++++++++++++++-----
 services/fuse/arvados_fuse/fusedir.py  |   7 +-
 services/fuse/arvados_fuse/fusefile.py |  14 ++
 services/fuse/tests/test_mount.py      | 231 ++++++++++++++++++++++++++++++++-
 6 files changed, 446 insertions(+), 60 deletions(-)

       via  5549904bbd5dec9bafe60e36d4ea1abe6b791f19 (commit)
      from  23a940aaf47411cd07615b3301d24d3868a11bce (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 5549904bbd5dec9bafe60e36d4ea1abe6b791f19
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Apr 16 16:26:41 2015 -0400

    3198: Support for mkdir, rmdir, unlink, rename.  Improve exception catching.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 13bb507..b4102f4 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -310,7 +310,7 @@ class _BufferBlock(object):
                 self.buffer_view = None
                 self.buffer_block = None
         else:
-            raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
+            raise AssertionError("Invalid state change from %s to %s" % (self.state, nextstate))
 
     @synchronized
     def state(self):
@@ -354,7 +354,7 @@ def must_be_writable(orig_func):
     @functools.wraps(orig_func)
     def must_be_writable_wrapper(self, *args, **kwargs):
         if not self.writable():
-            raise IOError((errno.EROFS, "Collection must be writable."))
+            raise IOError(errno.EROFS, "Collection must be writable.")
         return orig_func(self, *args, **kwargs)
     return must_be_writable_wrapper
 
@@ -719,7 +719,7 @@ class ArvadosFile(object):
             self._segments = new_segs
             self._modified = True
         elif size > self.size():
-            raise IOError("truncate() does not support extending the file size")
+            raise IOError(errno.EINVAL, "truncate() does not support extending the file size")
 
     def readfrom(self, offset, size, num_retries):
         """Read upto `size` bytes from the file starting at `offset`."""
@@ -800,12 +800,17 @@ class ArvadosFile(object):
 
         replace_range(self._segments, offset, len(data), self._current_bblock.blockid, self._current_bblock.write_pointer - len(data))
 
+        self.parent.notify(MOD, self.parent, self.name, (self, self))
+
+        return len(data)
+
     @synchronized
     def flush(self):
-        if self._current_bblock:
-            self._repack_writes()
-            self.parent._my_block_manager().commit_bufferblock(self._current_bblock)
-        self.parent.notify(MOD, self.parent, self.name, (self, self))
+        if self.modified():
+            if self._current_bblock and self._current_bblock.state() == _BufferBlock.WRITABLE:
+                self._repack_writes()
+                self.parent._my_block_manager().commit_bufferblock(self._current_bblock)
+            self.parent.notify(MOD, self.parent, self.name, (self, self))
 
     @must_be_writable
     @synchronized
@@ -872,16 +877,32 @@ class ArvadosFileReader(ArvadosFileReaderBase):
 
     @_FileLikeObjectBase._before_close
     @retry_method
-    def read(self, size, num_retries=None):
-        """Read up to `size` bytes from the stream, starting at the current file position."""
-        data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
-        self._filepos += len(data)
-        return data
+    def read(self, size=None, num_retries=None):
+        """Read up to `size` bytes from the file and return the result.
+
+        Starts at the current file position.  If `size` is None, read the
+        entire remainder of the file.
+        """
+        if size is None:
+            data = []
+            rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+            while rd:
+                data.append(rd)
+                self._filepos += len(rd)
+                rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
+            return ''.join(data)
+        else:
+            data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
+            self._filepos += len(data)
+            return data
 
     @_FileLikeObjectBase._before_close
     @retry_method
     def readfrom(self, offset, size, num_retries=None):
-        """Read up to `size` bytes from the stream, starting at the current file position."""
+        """Read up to `size` bytes from the stream, starting at the specified file offset.
+
+        This method does not change the file position.
+        """
         return self.arvadosfile.readfrom(offset, size, num_retries)
 
     def flush(self):
@@ -907,6 +928,7 @@ class ArvadosFileWriter(ArvadosFileReader):
         else:
             self.arvadosfile.writeto(self._filepos, data, num_retries)
             self._filepos += len(data)
+        return len(data)
 
     @_FileLikeObjectBase._before_close
     @retry_method
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 8091e8b..d610c35 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -555,7 +555,7 @@ class RichCollectionBase(CollectionBase):
                 if isinstance(item, RichCollectionBase):
                     return item.find_or_create(pathcomponents[1], create_type)
                 else:
-                    raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
+                    raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
         else:
             return self
 
@@ -581,9 +581,9 @@ class RichCollectionBase(CollectionBase):
                 else:
                     return item
             else:
-                raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
+                raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
 
-    def mkdirs(path):
+    def mkdirs(self, path):
         """Recursive subcollection create.
 
         Like `os.mkdirs()`.  Will create intermediate subcollections needed to
@@ -616,7 +616,7 @@ class RichCollectionBase(CollectionBase):
         create = (mode != "r")
 
         if create and not self.writable():
-            raise IOError((errno.EROFS, "Collection is read only"))
+            raise IOError(errno.EROFS, "Collection is read only")
 
         if create:
             arvfile = self.find_or_create(path, FILE)
@@ -624,9 +624,9 @@ class RichCollectionBase(CollectionBase):
             arvfile = self.find(path)
 
         if arvfile is None:
-            raise IOError((errno.ENOENT, "File not found"))
+            raise IOError(errno.ENOENT, "File not found")
         if not isinstance(arvfile, ArvadosFile):
-            raise IOError((errno.EISDIR, "Path must refer to a file."))
+            raise IOError(errno.EISDIR, "Path must refer to a file.")
 
         if mode[0] == "w":
             arvfile.truncate(0)
@@ -721,10 +721,10 @@ class RichCollectionBase(CollectionBase):
         pathcomponents = path.split("/", 1)
         item = self._items.get(pathcomponents[0])
         if item is None:
-            raise IOError((errno.ENOENT, "File not found"))
+            raise IOError(errno.ENOENT, "File not found")
         if len(pathcomponents) == 1:
             if isinstance(self._items[pathcomponents[0]], RichCollectionBase) and len(self._items[pathcomponents[0]]) > 0 and not recursive:
-                raise IOError((errno.ENOTEMPTY, "Subcollection not empty"))
+                raise IOError(errno.ENOTEMPTY, "Subcollection not empty")
             deleteditem = self._items[pathcomponents[0]]
             del self._items[pathcomponents[0]]
             self._modified = True
@@ -757,7 +757,7 @@ class RichCollectionBase(CollectionBase):
         """
 
         if target_name in self and not overwrite:
-            raise IOError((errno.EEXIST, "File already exists"))
+            raise IOError(errno.EEXIST, "File already exists")
 
         modified_from = None
         if target_name in self:
@@ -800,7 +800,7 @@ class RichCollectionBase(CollectionBase):
         if isinstance(source, basestring):
             source_obj = source_collection.find(source)
             if source_obj is None:
-                raise IOError((errno.ENOENT, "File not found"))
+                raise IOError(errno.ENOENT, "File not found")
             sourcecomponents = source.split("/")
         else:
             source_obj = source
@@ -1380,6 +1380,7 @@ class Collection(RichCollectionBase):
                 segments = []
                 streamoffset = 0L
                 state = BLOCKS
+                self.mkdirs(stream_name)
                 continue
 
             if state == BLOCKS:
@@ -1432,6 +1433,7 @@ class Subcollection(RichCollectionBase):
         self.lock = self.root_collection().lock
         self._manifest_text = None
         self.name = name
+        self.num_retries = parent.num_retries
 
     def root_collection(self):
         return self.parent.root_collection()
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b376d64..67878d7 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -22,42 +22,48 @@ import threading
 import itertools
 import ciso8601
 import collections
+import functools
 
 from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
 from fusefile import StringFile, FuseArvadosFile
 
 _logger = logging.getLogger('arvados.arvados_fuse')
 
-# log_handler = logging.StreamHandler()
-# llogger = logging.getLogger('llfuse')
-# llogger.addHandler(log_handler)
-# llogger.setLevel(logging.DEBUG)
+log_handler = logging.StreamHandler()
+llogger = logging.getLogger('llfuse')
+llogger.addHandler(log_handler)
+llogger.setLevel(logging.DEBUG)
 
-class FileHandle(object):
-    """Connects a numeric file handle to a File object that has
+class Handle(object):
+    """Connects a numeric file handle to a File or Directory object that has
     been opened by the client."""
 
-    def __init__(self, fh, fileobj):
+    def __init__(self, fh, obj):
         self.fh = fh
-        self.fileobj = fileobj
-        self.fileobj.inc_use()
+        self.obj = obj
+        self.obj.inc_use()
 
     def release(self):
-        self.fileobj.dec_use()
+        self.obj.dec_use()
 
+    def flush(self):
+        with llfuse.lock_released:
+            return self.obj.flush()
 
-class DirectoryHandle(object):
+
+class FileHandle(Handle):
+    """Connects a numeric file handle to a File  object that has
+    been opened by the client."""
+    pass
+
+
+class DirectoryHandle(Handle):
     """Connects a numeric file handle to a Directory object that has
     been opened by the client."""
 
     def __init__(self, fh, dirobj, entries):
-        self.fh = fh
+        super(DirectoryHandle, self).__init__(fh, dirobj)
         self.entries = entries
-        self.dirobj = dirobj
-        self.dirobj.inc_use()
-
-    def release(self):
-        self.dirobj.dec_use()
 
 
 class InodeCache(object):
@@ -106,6 +112,7 @@ class InodeCache(object):
         if obj.persisted() and obj._cache_priority in self._entries:
             self._remove(obj, True)
 
+
 class Inodes(object):
     """Manage the set of inodes.  This is the mapping from a numeric id
     to a concrete File or Directory object"""
@@ -148,6 +155,21 @@ class Inodes(object):
         llfuse.invalidate_inode(entry.inode)
         del self._entries[entry.inode]
 
+def catch_exceptions(orig_func):
+    @functools.wraps(orig_func)
+    def catch_exceptions_wrapper(self, *args, **kwargs):
+        try:
+            return orig_func(self, *args, **kwargs)
+        except llfuse.FUSEError:
+            raise
+        except EnvironmentError as e:
+            raise llfuse.FUSEError(e.errno)
+        except Exception:
+            _logger.exception("")
+            raise llfuse.FUSEError(errno.EIO)
+
+    return catch_exceptions_wrapper
+
 
 class Operations(llfuse.Operations):
     """This is the main interface with llfuse.
@@ -187,6 +209,7 @@ class Operations(llfuse.Operations):
     def access(self, inode, mode, ctx):
         return True
 
+    @catch_exceptions
     def getattr(self, inode):
         if inode not in self.inodes:
             raise llfuse.FUSEError(errno.ENOENT)
@@ -225,6 +248,7 @@ class Operations(llfuse.Operations):
 
         return entry
 
+    @catch_exceptions
     def lookup(self, parent_inode, name):
         name = unicode(name, self.encoding)
         _logger.debug("arv-mount lookup: parent_inode %i name %s",
@@ -246,24 +270,26 @@ class Operations(llfuse.Operations):
         else:
             raise llfuse.FUSEError(errno.ENOENT)
 
+    @catch_exceptions
     def open(self, inode, flags):
         if inode in self.inodes:
             p = self.inodes[inode]
         else:
             raise llfuse.FUSEError(errno.ENOENT)
 
-        if (flags & os.O_WRONLY) or (flags & os.O_RDWR):
-            raise llfuse.FUSEError(errno.EROFS)
-
         if isinstance(p, Directory):
             raise llfuse.FUSEError(errno.EISDIR)
 
+        if ((flags & os.O_WRONLY) or (flags & os.O_RDWR)) and not p.writable():
+            raise llfuse.FUSEError(errno.EPERM)
+
         fh = self._filehandles_counter
         self._filehandles_counter += 1
         self._filehandles[fh] = FileHandle(fh, p)
         self.inodes.touch(p)
         return fh
 
+    @catch_exceptions
     def read(self, fh, off, size):
         _logger.debug("arv-mount read %i %i %i", fh, off, size)
         if fh in self._filehandles:
@@ -271,20 +297,40 @@ class Operations(llfuse.Operations):
         else:
             raise llfuse.FUSEError(errno.EBADF)
 
-        self.inodes.touch(handle.fileobj)
+        self.inodes.touch(handle.obj)
 
         try:
             with llfuse.lock_released:
-                return handle.fileobj.readfrom(off, size, self.num_retries)
+                return handle.obj.readfrom(off, size, self.num_retries)
         except arvados.errors.NotFoundError as e:
             _logger.warning("Block not found: " + str(e))
             raise llfuse.FUSEError(errno.EIO)
-        except Exception:
-            _logger.exception("Read error")
-            raise llfuse.FUSEError(errno.EIO)
 
+    @catch_exceptions
+    def write(self, fh, off, buf):
+        _logger.debug("arv-mount write %i %i %i", fh, off, len(buf))
+        if fh in self._filehandles:
+            handle = self._filehandles[fh]
+        else:
+            raise llfuse.FUSEError(errno.EBADF)
+
+        if not handle.obj.writable():
+            raise llfuse.FUSEError(errno.EPERM)
+
+        self.inodes.touch(handle.obj)
+
+        with llfuse.lock_released:
+            return handle.obj.writeto(off, buf, self.num_retries)
+
+    @catch_exceptions
     def release(self, fh):
         if fh in self._filehandles:
+            try:
+                self._filehandles[fh].flush()
+            except EnvironmentError as e:
+                raise llfuse.FUSEError(e.errno)
+            except Exception:
+                _logger.exception("Flush error")
             self._filehandles[fh].release()
             del self._filehandles[fh]
         self.inodes.cap_cache()
@@ -292,6 +338,7 @@ class Operations(llfuse.Operations):
     def releasedir(self, fh):
         self.release(fh)
 
+    @catch_exceptions
     def opendir(self, inode):
         _logger.debug("arv-mount opendir: inode %i", inode)
 
@@ -316,7 +363,7 @@ class Operations(llfuse.Operations):
         self._filehandles[fh] = DirectoryHandle(fh, p, [('.', p), ('..', parent)] + list(p.items()))
         return fh
 
-
+    @catch_exceptions
     def readdir(self, fh, off):
         _logger.debug("arv-mount readdir: fh %i off %i", fh, off)
 
@@ -325,7 +372,7 @@ class Operations(llfuse.Operations):
         else:
             raise llfuse.FUSEError(errno.EBADF)
 
-        _logger.debug("arv-mount handle.dirobj %s", handle.dirobj)
+        _logger.debug("arv-mount handle.dirobj %s", handle.obj)
 
         e = off
         while e < len(handle.entries):
@@ -336,6 +383,7 @@ class Operations(llfuse.Operations):
                     pass
             e += 1
 
+    @catch_exceptions
     def statfs(self):
         st = llfuse.StatvfsData()
         st.f_bsize = 64 * 1024
@@ -351,12 +399,78 @@ class Operations(llfuse.Operations):
         st.f_frsize = 0
         return st
 
-    # The llfuse documentation recommends only overloading functions that
-    # are actually implemented, as the default implementation will raise ENOSYS.
-    # However, there is a bug in the llfuse default implementation of create()
-    # "create() takes exactly 5 positional arguments (6 given)" which will crash
-    # arv-mount.
-    # The workaround is to implement it with the proper number of parameters,
-    # and then everything works out.
+    def _check_writable(self, inode_parent):
+        if inode_parent in self.inodes:
+            p = self.inodes[inode_parent]
+        else:
+            raise llfuse.FUSEError(errno.ENOENT)
+
+        if not isinstance(p, Directory):
+            raise llfuse.FUSEError(errno.ENOTDIR)
+
+        if not p.writable():
+            raise llfuse.FUSEError(errno.EPERM)
+
+        if not isinstance(p, CollectionDirectoryBase):
+            raise llfuse.FUSEError(errno.EPERM)
+
+        return p
+
+    @catch_exceptions
     def create(self, inode_parent, name, mode, flags, ctx):
-        raise llfuse.FUSEError(errno.EROFS)
+        p = self._check_writable(inode_parent)
+
+        with llfuse.lock_released:
+            p.collection.open(name, "w")
+
+        # The file entry should have been implicitly created by callback.
+        f = p[name]
+
+        fh = self._filehandles_counter
+        self._filehandles_counter += 1
+        self._filehandles[fh] = FileHandle(fh, f)
+        self.inodes.touch(p)
+
+        return (fh, self.getattr(f.inode))
+
+    @catch_exceptions
+    def mkdir(self, inode_parent, name, mode, ctx):
+        p = self._check_writable(inode_parent)
+
+        with llfuse.lock_released:
+            p.collection.mkdirs(name)
+
+        # The dir entry should have been implicitly created by callback.
+        d = p[name]
+
+        return self.getattr(d.inode)
+
+    @catch_exceptions
+    def unlink(self, inode_parent, name):
+        p = self._check_writable(inode_parent)
+
+        with llfuse.lock_released:
+            p.collection.remove(name)
+
+    def rmdir(self, inode_parent, name):
+        self.unlink(inode_parent, name)
+
+    @catch_exceptions
+    def rename(self, inode_parent_old, name_old, inode_parent_new, name_new):
+        src = self._check_writable(inode_parent_old)
+        dest = self._check_writable(inode_parent_new)
+
+        with llfuse.lock_released:
+            dest.collection.copy(name_old, name_new, source_collection=src.collection, overwrite=True)
+            src.collection.remove(name_old)
+
+    @catch_exceptions
+    def flush(self, fh):
+        if fh in self._filehandles:
+            self._filehandles[fh].flush()
+
+    def fsync(self, fh, datasync):
+        self.flush(fh)
+
+    def fsyncdir(self, fh, datasync):
+        self.flush(fh)
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 53f53c7..51c652e 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -167,6 +167,8 @@ class Directory(FreshBase):
     def writable(self):
         return False
 
+    def flush(self):
+        pass
 
 class CollectionDirectoryBase(Directory):
     def __init__(self, parent_inode, inodes, collection):
@@ -195,6 +197,7 @@ class CollectionDirectoryBase(Directory):
                     ent = self._entries[name]
                     llfuse.invalidate_entry(self.inode, name)
                     llfuse.invalidate_inode(ent.inode)
+        _logger.warn("Finished handling event")
 
     def populate(self, mtime):
         self._mtime = mtime
@@ -205,6 +208,8 @@ class CollectionDirectoryBase(Directory):
     def writable(self):
         return self.collection.writable()
 
+    def flush(self):
+        self.collection.root_collection().save()
 
 class CollectionDirectory(CollectionDirectoryBase):
     """Represents the root of a directory tree holding a collection."""
@@ -229,7 +234,7 @@ class CollectionDirectory(CollectionDirectoryBase):
         return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
 
     def writable(self):
-        return self.collection.writable() if self.collection else self._writable
+        return self.collection.writable() if self.collection is not None else self._writable
 
     # Used by arv-web.py to switch the contents of the CollectionDirectory
     def change_collection(self, new_locator):
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index fcb7711..07de680 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -21,6 +21,9 @@ class File(FreshBase):
     def readfrom(self, off, size, num_retries=0):
         return ''
 
+    def writeto(self, off, size, num_retries=0):
+        raise Exception("Not writable")
+
     def mtime(self):
         return self._mtime
 
@@ -30,6 +33,9 @@ class File(FreshBase):
     def writable(self):
         return False
 
+    def flush(self):
+        pass
+
 class FuseArvadosFile(File):
     """Wraps a ArvadosFile."""
 
@@ -43,12 +49,20 @@ class FuseArvadosFile(File):
     def readfrom(self, off, size, num_retries=0):
         return self.arvfile.readfrom(off, size, num_retries)
 
+    def writeto(self, off, buf, num_retries=0):
+        return self.arvfile.writeto(off, buf, num_retries)
+
     def stale(self):
         return False
 
     def writable(self):
         return self.arvfile.writable()
 
+    def flush(self):
+        if self.writable():
+            self.arvfile.flush()
+            return self.arvfile.parent.root_collection().save()
+
 
 class StringFile(File):
     """Wrap a simple string as a file"""
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 9a0674b..bb7d871 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -16,6 +16,8 @@ import logging
 
 import run_test_server
 
+logger = logging.getLogger('arvados.arv-mount')
+
 class MountTestBase(unittest.TestCase):
     def setUp(self):
         self.keeptmp = tempfile.mkdtemp()
@@ -29,7 +31,7 @@ class MountTestBase(unittest.TestCase):
         operations = fuse.Operations(os.getuid(), os.getgid(), inode_cache=2)
         operations.inodes.add_entry(root_class(
             llfuse.ROOT_INODE, operations.inodes, self.api, 0, **root_kwargs))
-        llfuse.init(operations, self.mounttmp, [])
+        llfuse.init(operations, self.mounttmp, ['debug'])
         threading.Thread(None, llfuse.main).start()
         # wait until the driver is finished initializing
         operations.initlock.wait()
@@ -319,6 +321,8 @@ class FuseUpdateFileTest(MountTestBase):
         with collection.open("file1.txt", "w") as f:
             f.write("blub")
 
+        collection.save_new()
+
         m = self.make_mount(fuse.CollectionDirectory)
         with llfuse.lock:
             m.new_collection(None, collection)
@@ -342,6 +346,8 @@ class FuseAddFileToCollectionTest(MountTestBase):
         with collection.open("file1.txt", "w") as f:
             f.write("blub")
 
+        collection.save_new()
+
         m = self.make_mount(fuse.CollectionDirectory)
         with llfuse.lock:
             m.new_collection(None, collection)
@@ -364,6 +370,8 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
         with collection.open("file2.txt", "w") as f:
             f.write("plnp")
 
+        collection.save_new()
+
         m = self.make_mount(fuse.CollectionDirectory)
         with llfuse.lock:
             m.new_collection(None, collection)
@@ -376,6 +384,227 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
         d1 = os.listdir(self.mounttmp)
         self.assertEqual(["file1.txt"], d1)
 
+class FuseCreateFileTest(MountTestBase):
+    def runTest(self):
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertEqual(collection2["manifest_text"], "")
+
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        self.assertNotIn("file1.txt", collection)
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+            pass
+
+        self.assertIn("file1.txt", collection)
+
+        d1 = os.listdir(self.mounttmp)
+        self.assertEqual(["file1.txt"], d1)
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
+
+
+class FuseWriteFileTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        self.assertNotIn("file1.txt", collection)
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        with collection.open("file1.txt") as f:
+            self.assertEqual(f.read(), "Hello world!")
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "r") as f:
+            self.assertEqual(f.read(), "Hello world!")
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseUpdateFileTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "r+") as f:
+            self.assertEqual(f.read(), "Hello world!")
+            f.seek(0)
+            f.write("Hola mundo!")
+            f.seek(0)
+            self.assertEqual(f.read(), "Hola mundo!!")
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "r") as f:
+            self.assertEqual(f.read(), "Hola mundo!")
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseMkdirTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        with self.assertRaises(IOError):
+            with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+                f.write("Hello world!")
+
+        os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+        with self.assertRaises(OSError):
+            os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+        d1 = os.listdir(self.mounttmp)
+        self.assertEqual(["testdir"], d1)
+
+        with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual(["file1.txt"], d1)
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
+class FuseRmTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+        with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        # Starting manifest
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+        # Can't delete because it's not empty
+        with self.assertRaises(OSError):
+            os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+        d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual(["file1.txt"], d1)
+
+        # Delete file
+        os.remove(os.path.join(self.mounttmp, "testdir", "file1.txt"))
+
+        # Make sure it's empty
+        d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual([], d1)
+
+        # Try to delete it again
+        with self.assertRaises(OSError):
+            os.remove(os.path.join(self.mounttmp, "testdir", "file1.txt"))
+
+        # Can't have empty directories :-( so manifest will be empty.
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertEqual(collection2["manifest_text"], "")
+
+        # Should be able to delete now that it is empty
+        os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+        # Make sure it's empty
+        d1 = os.listdir(os.path.join(self.mounttmp))
+        self.assertEqual([], d1)
+
+        # Try to delete it again
+        with self.assertRaises(OSError):
+            os.rmdir(os.path.join(self.mounttmp, "testdir"))
+
+        # manifest should be empty now.
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertEqual(collection2["manifest_text"], "")
+
+
+class FuseMvTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(None, collection)
+        self.assertTrue(m.writable())
+
+        os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+        with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        # Starting manifest
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+        d1 = os.listdir(os.path.join(self.mounttmp))
+        self.assertEqual(["testdir"], d1)
+        d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual(["file1.txt"], d1)
+
+        os.rename(os.path.join(self.mounttmp, "testdir", "file1.txt"), os.path.join(self.mounttmp, "file1.txt"))
+
+        d1 = os.listdir(os.path.join(self.mounttmp))
+        self.assertEqual(["file1.txt", "testdir"], sorted(d1))
+        d1 = os.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual([], d1)
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
 
 class FuseUnitTest(unittest.TestCase):
     def test_sanitize_filename(self):

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list