[ARVADOS] updated: 20ade56019456b41c98021c2ed5a848bd8d018bb
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 16 16:21:02 EDT 2015
Summary of changes:
.../tutorial-keep-mount.html.textile.liquid | 21 +++++++++++-
sdk/python/arvados/_ranges.py | 4 +--
sdk/python/arvados/arvfile.py | 40 ++++++++++++----------
services/fuse/arvados_fuse/__init__.py | 8 +++--
services/fuse/bin/arv-mount | 10 ++++--
services/fuse/tests/test_mount.py | 15 ++++----
6 files changed, 64 insertions(+), 34 deletions(-)
via 20ade56019456b41c98021c2ed5a848bd8d018bb (commit)
from a830b5b560251c3143a7b1fd60db3f50a7021b34 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 20ade56019456b41c98021c2ed5a848bd8d018bb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 16 15:35:31 2015 -0400
3198: Add enable_write flag to FUSE and --enable-write and --read-only to
arv-mount. Update documentation. Code cleanup in locators_and_ranges() and
set_state().
diff --git a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
index a41fede..c219544 100644
--- a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
@@ -14,7 +14,7 @@ h2. Arv-mount
* You can browse, open and read Keep entries as if they are regular files.
* It is easy for existing tools to access files in Keep.
-* Data is downloaded on demand. It is not necessary to download an entire file or collection to start processing.
+* Data is streamed on demand. It is not necessary to download an entire file or collection to start processing.
The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by their Keep locator. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
@@ -34,3 +34,22 @@ var-GS000016015-ASM.tsv.bz2
The last line unmounts Keep. Subdirectories will no longer be accessible.
Within each directory on Keep, there is a @.arvados#collection@ file that does not show up with @ls at . Its contents include, for instance, the @portable_data_hash@, which is the same as the Keep locator.
+
+h3. Modifying files and directories in Keep
+
+By default, all files in the Keep mount are read only. However, @arv-mount --enable-write@ enables you to perform the following operations using normal Unix command line tools (@touch@, @mv@, @rm@, @mkdir@, @rmdir@) and your own programs using standard POSIX file system APIs:
+
+* Create, update, rename and delete individual files within collections
+* Create and delete subdirectories inside collections
+* Move files and directories within and between collections
+* Create and delete collections within a project (using @mkdir@ and @rmdir@ in a project directory)
+
+Not supported:
+
+* Symlinks, hard links
+* Changing permissions
+* Extended attributes
+
+If multiple clients try to modify the same file in the same collection, this result in a conflict. In this case, the most recent file wins, and the "loser" will be renamed to a conflict file in the form @name~YYYYMMDD-HHMMSS~conflict~@.
+
+Please note this feature is in beta testing. In particular, the conflict mechanism is itself currently subject to race condiditions with potential for data loss when a collection is being modified simultaneously by multiple clients. This issue will be resolved in future development.
diff --git a/sdk/python/arvados/_ranges.py b/sdk/python/arvados/_ranges.py
index 371d003..83437b2 100644
--- a/sdk/python/arvados/_ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -99,9 +99,7 @@ def locators_and_ranges(data_locators, range_start, range_size, limit=None):
# We should always start at the first segment due to the binary
# search.
- while i < len(data_locators):
- if limit and len(resp) > limit:
- break
+ while i < len(data_locators) and len(resp) != limit:
dl = data_locators[i]
block_start = dl.range_start
block_size = dl.range_size
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 7d6d676..7cd64aa 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -311,28 +311,30 @@ class _BufferBlock(object):
else:
raise AssertionError("Buffer block is not writable")
+ STATE_TRANSITIONS = frozenset([
+ (WRITABLE, PENDING),
+ (PENDING, COMMITTED),
+ (PENDING, ERROR),
+ (ERROR, PENDING)])
+
@synchronized
def set_state(self, nextstate, val=None):
- if ((self._state == _BufferBlock.WRITABLE and nextstate == _BufferBlock.PENDING) or
- (self._state == _BufferBlock.PENDING and nextstate == _BufferBlock.COMMITTED) or
- (self._state == _BufferBlock.PENDING and nextstate == _BufferBlock.ERROR) or
- (self._state == _BufferBlock.ERROR and nextstate == _BufferBlock.PENDING)):
- self._state = nextstate
-
- if self._state == _BufferBlock.PENDING:
- self.wait_for_commit.clear()
-
- if self._state == _BufferBlock.COMMITTED:
- self._locator = val
- self.buffer_view = None
- self.buffer_block = None
- self.wait_for_commit.set()
-
- if self._state == _BufferBlock.ERROR:
- self.error = val
- self.wait_for_commit.set()
- else:
+ if (self._state, nextstate) not in self.STATE_TRANSITIONS:
raise StateChangeError("Invalid state change from %s to %s" % (self.state, nextstate), self.state, nextstate)
+ self._state = nextstate
+
+ if self._state == _BufferBlock.PENDING:
+ self.wait_for_commit.clear()
+
+ if self._state == _BufferBlock.COMMITTED:
+ self._locator = val
+ self.buffer_view = None
+ self.buffer_block = None
+ self.wait_for_commit.set()
+
+ if self._state == _BufferBlock.ERROR:
+ self.error = val
+ self.wait_for_commit.set()
@synchronized
def state(self):
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 91e1907..a2b91a6 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -259,7 +259,7 @@ class Operations(llfuse.Operations):
"""
- def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4):
+ def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4, enable_write=False):
super(Operations, self).__init__()
if not inode_cache:
@@ -267,6 +267,7 @@ class Operations(llfuse.Operations):
self.inodes = Inodes(inode_cache, encoding=encoding)
self.uid = uid
self.gid = gid
+ self.enable_write = enable_write
# dict of inode to filehandle
self._filehandles = {}
@@ -349,7 +350,7 @@ class Operations(llfuse.Operations):
if isinstance(e, FuseArvadosFile):
entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
- if e.writable():
+ if self.enable_write and e.writable():
entry.st_mode |= stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
entry.st_nlink = 1
@@ -536,6 +537,9 @@ class Operations(llfuse.Operations):
return st
def _check_writable(self, inode_parent):
+ if not self.enable_write:
+ raise llfuse.FUSEError(errno.EROFS)
+
if inode_parent in self.inodes:
p = self.inodes[inode_parent]
else:
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 6e38728..3498500 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -50,6 +50,9 @@ with "--".
parser.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
parser.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
+ parser.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
+ parser.add_argument('--enable-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
+
parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
dest="exec_args", metavar=('command', 'args', '...', '--'),
help="""Mount, run a command, then unmount and exit""")
@@ -84,14 +87,17 @@ with "--".
arvados.logger.setLevel(logging.DEBUG)
logger.debug("arv-mount debugging enabled")
+ logger.warn("enable write is %s", args.enable_write)
+
try:
# Create the request handler
operations = Operations(os.getuid(),
os.getgid(),
encoding=args.encoding,
- inode_cache=InodeCache(cap=args.directory_cache))
+ inode_cache=InodeCache(cap=args.directory_cache),
+ enable_write=args.enable_write)
api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
- keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
+ keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
usr = api.users().current().execute(num_retries=args.retries)
now = time.time()
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 31d4e7a..ac5af5b 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -37,7 +37,7 @@ class MountTestBase(unittest.TestCase):
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def make_mount(self, root_class, **root_kwargs):
- self.operations = fuse.Operations(os.getuid(), os.getgid())
+ self.operations = fuse.Operations(os.getuid(), os.getgid(), enable_write=True)
self.operations.inodes.add_entry(root_class(
llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
llfuse.init(self.operations, self.mounttmp, [])
@@ -47,7 +47,8 @@ class MountTestBase(unittest.TestCase):
return self.operations.inodes[llfuse.ROOT_INODE]
def tearDown(self):
- self.pool.close()
+ self.pool.terminate()
+ self.pool.join()
del self.pool
# llfuse.close is buggy, so use fusermount instead.
@@ -519,7 +520,7 @@ class FuseUpdateFileTest(MountTestBase):
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
- # See note in FuseWriteFileTest
+ # See note in MountTestBase.setUp
self.pool.apply(fuseUpdateFileTestHelper, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -802,7 +803,7 @@ class FuseFileConflictTest(MountTestBase):
with collection2.open("file1.txt", "w") as f:
f.write("foo")
- # See comment in FuseWriteFileTest
+ # See note in MountTestBase.setUp
self.pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
@@ -838,7 +839,7 @@ class FuseUnlinkOpenFileTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- # See comment in FuseWriteFileTest
+ # See note in MountTestBase.setUp
self.pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
self.assertEqual(collection.manifest_text(), "")
@@ -879,7 +880,7 @@ class FuseMvFileBetweenCollectionsTest(MountTestBase):
m = self.make_mount(fuse.MagicDirectory)
- # See comment in FuseWriteFileTest
+ # See note in MountTestBase.setUp
self.pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
collection1.manifest_locator(),
collection2.manifest_locator()))
@@ -950,7 +951,7 @@ class FuseMvDirBetweenCollectionsTest(MountTestBase):
m = self.make_mount(fuse.MagicDirectory)
- # See comment in FuseWriteFileTest
+ # See note in MountTestBase.setUp
self.pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
collection1.manifest_locator(),
collection2.manifest_locator()))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list