[ARVADOS] updated: 499d871d59a5c05cf1034580e7c10ba37cefef4a
git at public.curoverse.com
git at public.curoverse.com
Tue Apr 14 16:56:16 EDT 2015
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 22 ++++++++++++----------
services/fuse/arvados_fuse/fusedir.py | 25 ++++++++++++++-----------
services/fuse/arvados_fuse/fusefile.py | 20 ++++++++++----------
3 files changed, 36 insertions(+), 31 deletions(-)
via 499d871d59a5c05cf1034580e7c10ba37cefef4a (commit)
via 4ac1cbf8618fc51f1413597c2fb406c1bead3376 (commit)
via 2139ee84699c7c535b75c5a21ff220d772c62504 (commit)
from 416a7c3a1b96abf7982362682048481f2afda0c9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 499d871d59a5c05cf1034580e7c10ba37cefef4a
Merge: 4ac1cbf 2139ee8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Apr 14 16:42:03 2015 -0400
Merge branch '3198-inode-cache' into 3198-writable-fuse
commit 4ac1cbf8618fc51f1413597c2fb406c1bead3376
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Apr 14 16:41:40 2015 -0400
3198: Use new Collection/ArvadosFile API.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 83b4710..f1bf9cc 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -24,7 +24,7 @@ import ciso8601
import collections
from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory
-from fusefile import StreamReaderFile, StringFile
+from fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -157,7 +157,7 @@ class Operations(llfuse.Operations):
"""
- def __init__(self, uid, gid, encoding="utf-8", inode_cache=1000):
+ def __init__(self, uid, gid, encoding="utf-8", inode_cache=1000, num_retries=7):
super(Operations, self).__init__()
self.inodes = Inodes(inode_cache)
@@ -173,6 +173,8 @@ class Operations(llfuse.Operations):
# is fully initialized should wait() on this event object.
self.initlock = threading.Event()
+ self.num_retries = num_retries
+
def init(self):
# Allow threads that are waiting for the driver to be finished
# initializing to continue
@@ -196,7 +198,7 @@ class Operations(llfuse.Operations):
entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
if isinstance(e, Directory):
entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFDIR
- elif isinstance(e, StreamReaderFile):
+ elif isinstance(e, FuseArvadosFile):
entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFREG
else:
entry.st_mode |= stat.S_IFREG
@@ -266,12 +268,12 @@ class Operations(llfuse.Operations):
try:
with llfuse.lock_released:
- return handle.fileobj.readfrom(off, size)
+ return handle.fileobj.readfrom(off, size, self.num_retries)
except arvados.errors.NotFoundError as e:
_logger.warning("Block not found: " + str(e))
raise llfuse.FUSEError(errno.EIO)
except Exception:
- _logger.exception()
+ _logger.exception("Read error")
raise llfuse.FUSEError(errno.EIO)
def release(self, fh):
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 08a5168..c1cfaa4 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -6,9 +6,10 @@ import arvados
import apiclient
import functools
-from fusefile import StringFile, StreamReaderFile, ObjectFile
+from fusefile import StringFile, ObjectFile, FuseArvadosFile
from fresh import FreshBase, convertTime, use_counter
+import arvados.collection
from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -184,6 +185,17 @@ class CollectionDirectory(Directory):
def same(self, i):
return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
+ @staticmethod
+ def populate(inodes, cwd, collection, mtime):
+ for entry, item in collection.items():
+ entry = sanitize_filename(entry)
+ if isinstance(item, arvados.collection.RichCollectionBase):
+ cwd._entries[entry] = inodes.add_entry(Directory(cwd.inode, inodes))
+ cwd._mtime = mtime
+ CollectionDirectory.populate(inodes, cwd._entries[entry], item, mtime)
+ else:
+ cwd._entries[entry] = inodes.add_entry(FuseArvadosFile(cwd.inode, item, mtime))
+
# Used by arv-web.py to switch the contents of the CollectionDirectory
def change_collection(self, new_locator):
"""Switch the contents of the CollectionDirectory.
@@ -205,16 +217,7 @@ class CollectionDirectory(Directory):
if self.collection_object_file is not None:
self.collection_object_file.update(self.collection_object)
- for s in coll_reader.all_streams():
- cwd = self
- for part in s.name().split('/'):
- if part != '' and part != '.':
- partname = sanitize_filename(part)
- if partname not in cwd._entries:
- cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode, self.inodes))
- cwd = cwd._entries[partname]
- for k, v in s.files().items():
- cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.mtime()))
+ CollectionDirectory.populate(self.inodes, self, coll_reader, self.mtime())
def update(self):
try:
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index efe31c3..9af341f 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -18,7 +18,7 @@ class File(FreshBase):
def size(self):
return 0
- def readfrom(self, off, size):
+ def readfrom(self, off, size, num_retries=0):
return ''
def mtime(self):
@@ -28,18 +28,18 @@ class File(FreshBase):
return True
-class StreamReaderFile(File):
- """Wraps a StreamFileReader as a file."""
+class FuseArvadosFile(File):
+ """Wraps a ArvadosFile."""
- def __init__(self, parent_inode, reader, _mtime):
- super(StreamReaderFile, self).__init__(parent_inode, _mtime)
- self.reader = reader
+ def __init__(self, parent_inode, arvfile, _mtime):
+ super(FuseArvadosFile, self).__init__(parent_inode, _mtime)
+ self.arvfile = arvfile
def size(self):
- return self.reader.size()
+ return self.arvfile.size()
- def readfrom(self, off, size):
- return self.reader.readfrom(off, size)
+ def readfrom(self, off, size, num_retries=0):
+ return self.arvfile.readfrom(off, size, num_retries)
def stale(self):
return False
@@ -54,7 +54,7 @@ class StringFile(File):
def size(self):
return len(self.contents)
- def readfrom(self, off, size):
+ def readfrom(self, off, size, num_retries=0):
return self.contents[off:(off+size)]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list