[ARVADOS] updated: 615100bbbff1cd2d97f25bdd948cf9c82b3332af
git at public.curoverse.com
git at public.curoverse.com
Wed Nov 25 10:54:32 EST 2015
Summary of changes:
.../tutorial-keep-mount.html.textile.liquid | 4 +-
sdk/python/tests/run_test_server.py | 16 +-
services/arv-web/arv-web.py | 2 +-
services/fuse/arvados_fuse/__init__.py | 95 +++---
services/fuse/arvados_fuse/command.py | 338 +++++++++++++++++++++
services/fuse/arvados_fuse/crunchstat.py | 63 ++++
services/fuse/arvados_fuse/fresh.py | 2 +
services/fuse/arvados_fuse/fusedir.py | 75 ++++-
services/fuse/arvados_fuse/fusefile.py | 39 ++-
services/fuse/bin/arv-mount | 292 +-----------------
services/fuse/tests/integration_test.py | 68 +++++
services/fuse/tests/mount_test_base.py | 5 +-
services/fuse/tests/test_command_args.py | 189 ++++++++++++
services/fuse/tests/test_mount.py | 2 +-
services/fuse/tests/test_tmp_collection.py | 124 ++++++++
15 files changed, 971 insertions(+), 343 deletions(-)
create mode 100644 services/fuse/arvados_fuse/command.py
create mode 100644 services/fuse/arvados_fuse/crunchstat.py
create mode 100644 services/fuse/tests/integration_test.py
create mode 100644 services/fuse/tests/test_command_args.py
create mode 100644 services/fuse/tests/test_tmp_collection.py
via 615100bbbff1cd2d97f25bdd948cf9c82b3332af (commit)
via 346a55897f4e4a2a26a1a84665f860c000b3da7d (commit)
via 6d86e355c251a68dd870e22ea1cdd43eea10c390 (commit)
via 6264e41963708efbc19d94a0a4f53e0d53dcb7ad (commit)
via 9e5589c31eb118a957f49e8c7a2613c4673395c9 (commit)
via 7cb15d038a126d4d16e6bfea670161baed0a1d7d (commit)
via 1cea943fbc95efdfa1ff9faa8f7c28882560b848 (commit)
via 2a5d2064c5399f209bcf1bf1a47f2807ff5e16f1 (commit)
via 98802c14566d90722ffee033dfe3301354d983ef (commit)
via 2b31c00d8bd9e12127eefaa91a51e3ceee5ca76b (commit)
via a11c0e77a0d114d50a997db93174465c9aa85f5a (commit)
via 47c7337cc9c33a19208946deceecc5ce266ae741 (commit)
via 7134fe6c4ae22c56b45232840918be9e05f9445a (commit)
via fcdaf1cedc195d3835a96d89471837ae321c5063 (commit)
from 0c8cf79d48283ecbe376ab958ad2ac90bbb34e59 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 615100bbbff1cd2d97f25bdd948cf9c82b3332af
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 25 10:49:57 2015 -0500
5824: Move run_test_server diag messages from stdout to stderr.
refs #5824
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index adb8652..d6febdd 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -98,7 +98,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
# Pidfile exists, but we can't parse it. Perhaps the
# server has created the file but hasn't written its PID
# yet?
- print("Parse error reading pidfile {}: {}".format(pidfile, error))
+ print("Parse error reading pidfile {}: {}".format(pidfile, error),
+ file=sys.stderr)
time.sleep(0.1)
now = time.time()
@@ -113,7 +114,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
try:
if now >= startTERM:
os.kill(server_pid, signal.SIGTERM)
- print("Sent SIGTERM to {} ({})".format(server_pid, pidfile))
+ print("Sent SIGTERM to {} ({})".format(server_pid, pidfile),
+ file=sys.stderr)
except OSError as error:
if error.errno == errno.ESRCH:
# Thrown by os.getpgid() or os.kill() if the process
@@ -124,7 +126,8 @@ def kill_server_pid(pidfile, wait=10, passenger_root=False):
now = time.time()
print("Server PID {} ({}) did not exit, giving up after {}s".
- format(server_pid, pidfile, wait))
+ format(server_pid, pidfile, wait),
+ file=sys.stderr)
def find_available_port():
"""Return an IPv4 port number that is not in use right now.
@@ -155,7 +158,8 @@ def _wait_until_port_listens(port, timeout=10):
subprocess.check_output(['which', 'lsof'])
except subprocess.CalledProcessError:
print("WARNING: No `lsof` -- cannot wait for port to listen. "+
- "Sleeping 0.5 and hoping for the best.")
+ "Sleeping 0.5 and hoping for the best.",
+ file=sys.stderr)
time.sleep(0.5)
return
deadline = time.time() + timeout
@@ -695,7 +699,9 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.action not in actions:
- print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
+ print("Unrecognized action '{}'. Actions are: {}.".
+ format(args.action, actions),
+ file=sys.stderr)
sys.exit(1)
if args.action == 'start':
stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
commit 346a55897f4e4a2a26a1a84665f860c000b3da7d
Merge: 0c8cf79 6d86e35
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 25 10:45:24 2015 -0500
Merge branch '7751-mount-tmp' refs #7751
commit 6d86e355c251a68dd870e22ea1cdd43eea10c390
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 24 13:45:02 2015 -0500
7751: Fix race by telling fuse not to cache the .arvados#collection dirent.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b7c0091..65f117b 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -407,8 +407,8 @@ class Operations(llfuse.Operations):
entry = llfuse.EntryAttributes()
entry.st_ino = inode
entry.generation = 0
- entry.entry_timeout = 60
- entry.attr_timeout = 60
+ entry.entry_timeout = 60 if e.allow_dirent_cache else 0
+ entry.attr_timeout = 60 if e.allow_attr_cache else 0
entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
if isinstance(e, Directory):
diff --git a/services/fuse/arvados_fuse/fresh.py b/services/fuse/arvados_fuse/fresh.py
index ec2d47a..2075741 100644
--- a/services/fuse/arvados_fuse/fresh.py
+++ b/services/fuse/arvados_fuse/fresh.py
@@ -67,6 +67,8 @@ class FreshBase(object):
self.cache_priority = None
self.cache_size = 0
self.cache_uuid = None
+ self.allow_attr_cache = True
+ self.allow_dirent_cache = True
# Mark the value as stale
def invalidate(self):
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index d4c075a..e731327 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -112,6 +112,13 @@ class FuncToJSONFile(StringFile):
super(FuncToJSONFile, self).__init__(parent_inode, "", 0)
self.func = func
+ # invalidate_inode() and invalidate_entry() are asynchronous
+ # with no callback to wait for. In order to guarantee
+ # userspace programs don't get stale data that was generated
+ # before the last invalidate(), we must disallow dirent
+ # caching entirely.
+ self.allow_dirent_cache = False
+
def size(self):
self._update()
return super(FuncToJSONFile, self).size()
diff --git a/services/fuse/tests/test_tmp_collection.py b/services/fuse/tests/test_tmp_collection.py
index e403a2c..60eba1b 100644
--- a/services/fuse/tests/test_tmp_collection.py
+++ b/services/fuse/tests/test_tmp_collection.py
@@ -110,12 +110,15 @@ class TmpCollectionTest(IntegrationTest):
r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:bar 3:3:foo\n$'),
('foo', None,
r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:bar\n$'),
+ ('bar', None,
+ r'^$'),
]
- for fn, content, expect in ops:
- path = os.path.join(tmpdir, fn)
- if content is None:
- os.unlink(path)
- else:
- with open(path, 'w') as f:
- f.write(content)
- self.assertRegexpMatches(current_manifest(tmpdir), expect)
+ for _ in range(10):
+ for fn, content, expect in ops:
+ path = os.path.join(tmpdir, fn)
+ if content is None:
+ os.unlink(path)
+ else:
+ with open(path, 'w') as f:
+ f.write(content)
+ self.assertRegexpMatches(current_manifest(tmpdir), expect)
commit 6264e41963708efbc19d94a0a4f53e0d53dcb7ad
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 24 13:43:42 2015 -0500
7751: Refactor TmpCollectionDirectory: generate .arvados#collection less often.
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 04c2d50..a2135b3 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -8,8 +8,9 @@ import functools
import threading
from apiclient import errors as apiclient_errors
import errno
+import time
-from fusefile import StringFile, ObjectFile, FuseArvadosFile
+from fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
from fresh import FreshBase, convertTime, use_counter, check_update
import arvados.collection
@@ -483,43 +484,36 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
job output.
"""
+ class UnsaveableCollection(arvados.collection.Collection):
+ def save(self):
+ pass
+ def save_new(self):
+ pass
+
def __init__(self, parent_inode, inodes, api_client, num_retries):
- collection = arvados.collection.Collection(
+ collection = self.UnsaveableCollection(
api_client=api_client,
keep_client=api_client.keep)
- collection.save = self._commit_collection
- collection.save_new = self._commit_collection
super(TmpCollectionDirectory, self).__init__(
parent_inode, inodes, collection)
self.collection_record_file = None
- self._subscribed = False
- self._update_collection_record()
-
- def update(self, *args, **kwargs):
- if not self._subscribed:
- with llfuse.lock_released:
- self.populate(self.mtime())
- self._subscribed = True
-
- @use_counter
- def _commit_collection(self):
- """Commit the data blocks, but don't save the collection to API.
+ self.populate(self.mtime())
- Update the content of the special .arvados#collection file, if
- it has been instantiated.
- """
- self.collection.flush()
- self._update_collection_record()
- if self.collection_record_file is not None:
- self.collection_record_file.update(self.collection_record)
+ def on_event(self, *args, **kwargs):
+ super(TmpCollectionDirectory, self).on_event(*args, **kwargs)
+ if self.collection_record_file:
+ with llfuse.lock:
+ self.collection_record_file.invalidate()
self.inodes.invalidate_inode(self.collection_record_file.inode)
+ _logger.debug("%s invalidated collection record", self)
- def _update_collection_record(self):
- self.collection_record = {
- "uuid": None,
- "manifest_text": self.collection.manifest_text(),
- "portable_data_hash": self.collection.portable_data_hash(),
- }
+ def collection_record(self):
+ with llfuse.lock_released:
+ return {
+ "uuid": None,
+ "manifest_text": self.collection.manifest_text(),
+ "portable_data_hash": self.collection.portable_data_hash(),
+ }
def __contains__(self, k):
return (k == '.arvados#collection' or
@@ -529,18 +523,26 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
def __getitem__(self, item):
if item == '.arvados#collection':
if self.collection_record_file is None:
- self.collection_record_file = ObjectFile(
+ self.collection_record_file = FuncToJSONFile(
self.inode, self.collection_record)
self.inodes.add_entry(self.collection_record_file)
return self.collection_record_file
return super(TmpCollectionDirectory, self).__getitem__(item)
+ def persisted(self):
+ return False
+
def writable(self):
return True
def finalize(self):
self.collection.stop_threads()
+ def invalidate(self):
+ if self.collection_record_file:
+ self.collection_record_file.invalidate()
+ super(TmpCollectionDirectory, self).invalidate()
+
class MagicDirectory(Directory):
"""A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index 4d472cf..d4c075a 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -1,7 +1,8 @@
-import logging
-import re
import json
import llfuse
+import logging
+import re
+import time
from fresh import FreshBase, convertTime
@@ -99,3 +100,30 @@ class ObjectFile(StringFile):
def persisted(self):
return True
+
+
+class FuncToJSONFile(StringFile):
+ """File content is the return value of a given function, encoded as JSON.
+
+ The function is called at the time the file is read. The result is
+ cached until invalidate() is called.
+ """
+ def __init__(self, parent_inode, func):
+ super(FuncToJSONFile, self).__init__(parent_inode, "", 0)
+ self.func = func
+
+ def size(self):
+ self._update()
+ return super(FuncToJSONFile, self).size()
+
+ def readfrom(self, *args, **kwargs):
+ self._update()
+ return super(FuncToJSONFile, self).readfrom(*args, **kwargs)
+
+ def _update(self):
+ if not self.stale():
+ return
+ self._mtime = time.time()
+ obj = self.func()
+ self.contents = json.dumps(obj, indent=4, sort_keys=True) + "\n"
+ self.fresh()
commit 9e5589c31eb118a957f49e8c7a2613c4673395c9
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 24 10:41:15 2015 -0500
7751: Fix stack trace on shutdown caused by race between destroy() and llfuse unmount.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index a540ebd..b7c0091 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -324,6 +324,12 @@ class Operations(llfuse.Operations):
# is fully initialized should wait() on this event object.
self.initlock = threading.Event()
+ # If we get overlapping shutdown events (e.g., fusermount -u
+ # -z and operations.destroy()) llfuse calls forget() on inodes
+ # that have already been deleted. To avoid this, we make
+ # forget() a no-op if called after destroy().
+ self._shutdown_started = threading.Event()
+
self.num_retries = num_retries
self.read_counter = arvados.keep.Counter()
@@ -340,11 +346,13 @@ class Operations(llfuse.Operations):
@catch_exceptions
def destroy(self):
- if self.events:
- self.events.close()
- self.events = None
+ with llfuse.lock:
+ self._shutdown_started.set()
+ if self.events:
+ self.events.close()
+ self.events = None
- self.inodes.clear()
+ self.inodes.clear()
def access(self, inode, mode, ctx):
return True
@@ -468,6 +476,8 @@ class Operations(llfuse.Operations):
@catch_exceptions
def forget(self, inodes):
+ if self._shutdown_started.is_set():
+ return
for inode, nlookup in inodes:
ent = self.inodes[inode]
_logger.debug("arv-mount forget: inode %i nlookup %i ref_count %i", inode, nlookup, ent.ref_count)
commit 7cb15d038a126d4d16e6bfea670161baed0a1d7d
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 23 17:42:40 2015 -0500
7751: Fix shell wrapper code path.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index e12d62a..93bcd20 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -125,9 +125,9 @@ class Mount(object):
def run(self):
if self.args.exec_args:
- self._run_exec(self.args)
+ self._run_exec()
else:
- self._run_standalone(self.args)
+ self._run_standalone()
def _fuse_options(self):
"""FUSE mount options; see mount.fuse(8)"""
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index f8fc96f..671d6a9 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-import arvados_fuse
+import arvados_fuse.command
if __name__ == '__main__':
args = arvados_fuse.command.ArgumentParser().parse_args()
commit 1cea943fbc95efdfa1ff9faa8f7c28882560b848
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 23 16:31:37 2015 -0500
7751: Lowercase method name for PEP-0008.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index ba74111..e12d62a 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -123,7 +123,7 @@ class Mount(object):
subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
self.operations.destroy()
- def Run(self):
+ def run(self):
if self.args.exec_args:
self._run_exec(self.args)
else:
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 05b4c50..f8fc96f 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -4,4 +4,4 @@ import arvados_fuse
if __name__ == '__main__':
args = arvados_fuse.command.ArgumentParser().parse_args()
- arvados_fuse.command.Mount(args).Run()
+ arvados_fuse.command.Mount(args).run()
commit 2a5d2064c5399f209bcf1bf1a47f2807ff5e16f1
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 22 05:26:13 2015 -0500
7751: Suppress help messages when testing usage errors.
diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py
index 62987cc..19d56a9 100644
--- a/services/fuse/tests/test_command_args.py
+++ b/services/fuse/tests/test_command_args.py
@@ -1,12 +1,14 @@
import arvados
import arvados_fuse
import arvados_fuse.command
+import contextlib
import functools
import json
import llfuse
import logging
import os
import run_test_server
+import sys
import tempfile
import unittest
@@ -22,6 +24,15 @@ def noexit(func):
raise SystemExitCaught
return wrapper
+ at contextlib.contextmanager
+def nostderr():
+ orig, sys.stderr = sys.stderr, open(os.devnull, 'w')
+ try:
+ yield
+ finally:
+ sys.stderr = orig
+
+
class MountArgsTest(unittest.TestCase):
def setUp(self):
self.mntdir = tempfile.mkdtemp()
@@ -126,10 +137,11 @@ class MountArgsTest(unittest.TestCase):
['--by-id', '--project', gid],
['--mount-tmp', 'foo', '--by-id'],
]:
- with self.assertRaises(SystemExit):
- args = arvados_fuse.command.ArgumentParser().parse_args(
- badargs + ['--foreground', self.mntdir])
- arvados_fuse.command.Mount(args)
+ with nostderr():
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args(
+ badargs + ['--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
@noexit
def test_project(self):
uuid = run_test_server.fixture('groups')['aproject']['uuid']
@@ -169,8 +181,9 @@ class MountArgsTest(unittest.TestCase):
def test_custom_unsupported_layouts(self):
for name in ['.', '..', '', 'foo/bar', '/foo']:
- with self.assertRaises(SystemExit):
- args = arvados_fuse.command.ArgumentParser().parse_args([
- '--mount-tmp', name,
- '--foreground', self.mntdir])
- arvados_fuse.command.Mount(args)
+ with nostderr():
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', name,
+ '--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
commit 98802c14566d90722ffee033dfe3301354d983ef
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 18:37:32 2015 -0500
7751: Update explanation of special .arvados#collection file. Fix wayward use of "Keep locator".
diff --git a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
index 73c02ef..32cc3ca 100644
--- a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Arv-mount
* It is easy for existing tools to access files in Keep.
* Data is streamed on demand. It is not necessary to download an entire file or collection to start processing.
-The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by their Keep locator. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
+The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by UUID or portable data hash. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
<notextile>
<pre><code>~$ <span class="userinput">mkdir -p keep</span>
@@ -33,7 +33,7 @@ var-GS000016015-ASM.tsv.bz2
The last line unmounts Keep. Subdirectories will no longer be accessible.
-Within each directory on Keep, there is a @.arvados#collection@ file that does not show up with @ls at . Its contents include, for instance, the @portable_data_hash@, which is the same as the Keep locator.
+In the top level directory of each collection, arv-mount provides a special file called @.arvados#collection@ that contains a JSON-formatted API record for the collection. This can be used to determine the collection's @portable_data_hash@, @uuid@, etc. This file does not show up in @ls@ or @ls -a at .
h3. Modifying files and directories in Keep
commit 2b31c00d8bd9e12127eefaa91a51e3ceee5ca76b
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 18:09:55 2015 -0500
7751: Return an empty array from InodeCache.find() instead of None.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 55f1ad7..a540ebd 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -192,8 +192,8 @@ class InodeCache(object):
if obj.persisted() and obj.cache_priority in self._entries:
self._remove(obj, True)
- def find(self, uuid):
- return self._by_uuid.get(uuid)
+ def find_by_uuid(self, uuid):
+ return self._by_uuid.get(uuid, [])
def clear(self):
self._entries.clear()
@@ -356,34 +356,37 @@ class Operations(llfuse.Operations):
@catch_exceptions
def on_event(self, ev):
- if 'event_type' in ev:
- with llfuse.lock:
- items = self.inodes.inode_cache.find(ev["object_uuid"])
- if items is not None:
- for item in items:
- item.invalidate()
- if ev["object_kind"] == "arvados#collection":
- new_attr = ev.get("properties") and ev["properties"].get("new_attributes") and ev["properties"]["new_attributes"]
-
- # new_attributes.modified_at currently lacks subsecond precision (see #6347) so use event_at which
- # should always be the same.
- #record_version = (new_attr["modified_at"], new_attr["portable_data_hash"]) if new_attr else None
- record_version = (ev["event_at"], new_attr["portable_data_hash"]) if new_attr else None
-
- item.update(to_record_version=record_version)
- else:
- item.update()
-
- oldowner = ev.get("properties") and ev["properties"].get("old_attributes") and ev["properties"]["old_attributes"].get("owner_uuid")
- olditemparent = self.inodes.inode_cache.find(oldowner)
- if olditemparent is not None:
- olditemparent.invalidate()
- olditemparent.update()
-
- itemparent = self.inodes.inode_cache.find(ev["object_owner_uuid"])
- if itemparent is not None:
- itemparent.invalidate()
- itemparent.update()
+ if 'event_type' not in ev:
+ return
+ with llfuse.lock:
+ for item in self.inodes.inode_cache.find_by_uuid(ev["object_uuid"]):
+ item.invalidate()
+ if ev["object_kind"] == "arvados#collection":
+ new_attr = (ev.get("properties") and
+ ev["properties"].get("new_attributes") and
+ ev["properties"]["new_attributes"])
+
+ # new_attributes.modified_at currently lacks
+ # subsecond precision (see #6347) so use event_at
+ # which should always be the same.
+ record_version = (
+ (ev["event_at"], new_attr["portable_data_hash"])
+ if new_attr else None)
+
+ item.update(to_record_version=record_version)
+ else:
+ item.update()
+
+ oldowner = (
+ ev.get("properties") and
+ ev["properties"].get("old_attributes") and
+ ev["properties"]["old_attributes"].get("owner_uuid"))
+ newowner = ev["object_owner_uuid"]
+ for parent in (
+ self.inodes.inode_cache.find_by_uuid(oldowner) +
+ self.inodes.inode_cache.find_by_uuid(newowner)):
+ parent.invalidate()
+ parent.update()
@catch_exceptions
commit a11c0e77a0d114d50a997db93174465c9aa85f5a
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 03:38:10 2015 -0500
7751: Test mount arguments.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index e9d8bb5..ba74111 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -187,16 +187,31 @@ class Mount(object):
dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
mount_readme = False
- if self.args.mode is not None and (
- self.args.mount_by_id or
- self.args.mount_by_pdh or
- self.args.mount_by_tag or
- self.args.mount_home or
- self.args.mount_shared or
- self.args.mount_tmp or
- self.args.mount_collection):
- sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
- format(self.args.mode))
+ if self.args.collection is not None:
+ # Set up the request handler with the collection at the root
+ self.args.mode = 'collection'
+ dir_class = CollectionDirectory
+ dir_args.append(self.args.collection)
+ elif self.args.project is not None:
+ self.args.mode = 'project'
+ dir_class = ProjectDirectory
+ dir_args.append(
+ self.api.groups().get(uuid=self.args.project).execute(
+ num_retries=self.args.retries))
+
+ if (self.args.mount_by_id or
+ self.args.mount_by_pdh or
+ self.args.mount_by_tag or
+ self.args.mount_home or
+ self.args.mount_shared or
+ self.args.mount_tmp):
+ if self.args.mode is not None:
+ sys.exit(
+ "Cannot combine '{}' mode with custom --mount-* options.".
+ format(self.args.mode))
+ elif self.args.mode is None:
+ # If no --mount-custom or custom mount args, --all is the default
+ self.args.mode = 'all'
if self.args.mode in ['by_id', 'by_pdh']:
# Set up the request handler with the 'magic directory' at the root
@@ -217,15 +232,6 @@ class Mount(object):
self.args.mount_home = ['home']
self.args.mount_shared = ['shared']
mount_readme = True
- elif self.args.collection is not None:
- # Set up the request handler with the collection at the root
- dir_class = CollectionDirectory
- dir_args.append(self.args.collection)
- elif self.args.project is not None:
- dir_class = ProjectDirectory
- dir_args.append(
- self.api.groups().get(uuid=self.args.project).execute(
- num_retries=self.args.retries))
if dir_class is not None:
self.operations.inodes.add_entry(dir_class(*dir_args))
@@ -252,7 +258,7 @@ class Mount(object):
text = self._readme_text(
arvados.config.get('ARVADOS_API_HOST'),
usr['email'])
- self._add_mount(e, StringFile(e.inode, text, now))
+ self._add_mount(e, 'README', StringFile(e.inode, text, now))
def _add_mount(self, tld, name, ent):
if name in ['', '.', '..'] or '/' in name:
@@ -271,6 +277,7 @@ From here, the following directories are available:
by_tag/ Access to Keep collections organized by tag.
home/ The contents of your home project.
shared/ Projects shared with you.
+
'''.format(api_host, user_email)
def _run_exec(self):
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 21961a5..04c2d50 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -556,12 +556,13 @@ class MagicDirectory(Directory):
README_TEXT = """
This directory provides access to Arvados collections as subdirectories listed
by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
-the form '1234567890abcdefghijklmnopqrstuv+123').
+the form '1234567890abcdef0123456789abcdef+123').
Note that this directory will appear empty until you attempt to access a
specific collection subdirectory (such as trying to 'cd' into it), at which
point the collection will actually be looked up on the server and the directory
will appear if it exists.
+
""".lstrip()
def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False):
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 9652709..faa4a55 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -47,7 +47,7 @@ class IntegrationTest(unittest.TestCase):
def setUp(self):
self.mnt = tempfile.mkdtemp()
- run_test_server.authorize_with("admin")
+ run_test_server.authorize_with('active')
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def tearDown(self):
diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py
new file mode 100644
index 0000000..62987cc
--- /dev/null
+++ b/services/fuse/tests/test_command_args.py
@@ -0,0 +1,176 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import functools
+import json
+import llfuse
+import logging
+import os
+import run_test_server
+import tempfile
+import unittest
+
+def noexit(func):
+ """If argparse or arvados_fuse tries to exit, fail the test instead"""
+ class SystemExitCaught(StandardError):
+ pass
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ return func(*args, **kwargs)
+ except SystemExit:
+ raise SystemExitCaught
+ return wrapper
+
+class MountArgsTest(unittest.TestCase):
+ def setUp(self):
+ self.mntdir = tempfile.mkdtemp()
+ run_test_server.authorize_with('active')
+
+ def tearDown(self):
+ os.rmdir(self.mntdir)
+
+ def lookup(self, mnt, *path):
+ ent = mnt.operations.inodes[llfuse.ROOT_INODE]
+ for p in path:
+ ent = ent[p]
+ return ent
+
+ def check_ent_type(self, cls, *path):
+ ent = self.lookup(self.mnt, *path)
+ self.assertEqual(ent.__class__, cls)
+ return ent
+
+ @noexit
+ def test_default_all(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+ e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
+
+ e = self.check_ent_type(arvados_fuse.StringFile, 'README')
+ readme = e.readfrom(0, -1)
+ self.assertRegexpMatches(readme, r'active-user at arvados\.local')
+ self.assertRegexpMatches(readme, r'\n$')
+
+ e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
+ txt = e.readfrom(0, -1)
+ self.assertRegexpMatches(txt, r'portable data hash')
+ self.assertRegexpMatches(txt, r'\n$')
+
+ @noexit
+ def test_by_id(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-id',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_id')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.MagicDirectory)
+ self.assertEqual(e.pdh_only, False)
+
+ @noexit
+ def test_by_pdh(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-pdh',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_pdh')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.MagicDirectory)
+ self.assertEqual(e.pdh_only, True)
+
+ @noexit
+ def test_by_tag(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-tag',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_tag')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.TagsDirectory)
+
+ @noexit
+ def test_collection(self, id_type='uuid'):
+ c = run_test_server.fixture('collections')['public_text_file']
+ cid = c[id_type]
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--collection', cid,
+ '--foreground', self.mntdir])
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.CollectionDirectory)
+ self.assertEqual(e.collection_locator, cid)
+
+ def test_collection_pdh(self):
+ self.test_collection('portable_data_hash')
+
+ @noexit
+ def test_home(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--home',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'home')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory)
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ def test_mutually_exclusive_args(self):
+ cid = run_test_server.fixture('collections')['public_text_file']['uuid']
+ gid = run_test_server.fixture('groups')['aproject']['uuid']
+ for badargs in [
+ ['--mount-tmp', 'foo', '--collection', cid],
+ ['--mount-tmp', 'foo', '--project', gid],
+ ['--collection', cid, '--project', gid],
+ ['--by-id', '--project', gid],
+ ['--mount-tmp', 'foo', '--by-id'],
+ ]:
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args(
+ badargs + ['--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
+ @noexit
+ def test_project(self):
+ uuid = run_test_server.fixture('groups')['aproject']['uuid']
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--project', uuid,
+ '--foreground', self.mntdir])
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory)
+ self.assertEqual(e.project_object['uuid'], uuid)
+
+ @noexit
+ def test_shared(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--shared',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'shared')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.SharedDirectory)
+ self.assertEqual(e.current_user['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ @noexit
+ def test_custom(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'foo',
+ '--mount-tmp', 'bar',
+ '--mount-home', 'my_home',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ self.mnt = arvados_fuse.command.Mount(args)
+ self.check_ent_type(arvados_fuse.Directory)
+ self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
+ self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ def test_custom_unsupported_layouts(self):
+ for name in ['.', '..', '', 'foo/bar', '/foo']:
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', name,
+ '--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
commit 47c7337cc9c33a19208946deceecc5ce266ae741
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 19 20:38:34 2015 -0500
7751: Add tests for --mount-tmp option.
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
new file mode 100644
index 0000000..9652709
--- /dev/null
+++ b/services/fuse/tests/integration_test.py
@@ -0,0 +1,68 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import functools
+import inspect
+import multiprocessing
+import os
+import sys
+import tempfile
+import unittest
+import run_test_server
+
+def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
+ class Test(unittest.TestCase):
+ def runTest(self, *args, **kwargs):
+ getattr(getattr(sys.modules[modName], clsName), funcName)(self, *args, **kwargs)
+ Test().runTest(*args, **kwargs)
+
+
+class IntegrationTest(unittest.TestCase):
+ def pool_test(self, *args, **kwargs):
+ """Run a static method as a unit test, in a different process.
+
+ If called by method 'foobar', the static method '_foobar' of
+ the same class will be called in the other process.
+ """
+ modName = inspect.getmodule(self).__name__
+ clsName = self.__class__.__name__
+ funcName = inspect.currentframe().f_back.f_code.co_name
+ pool = multiprocessing.Pool(1)
+ try:
+ pool.apply(
+ wrap_static_test_method,
+ (modName, clsName, '_'+funcName, args, kwargs))
+ finally:
+ pool.terminate()
+ pool.join()
+
+ @classmethod
+ def setUpClass(cls):
+ run_test_server.run()
+ run_test_server.run_keep(enforce_permissions=True, num_servers=2)
+
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop_keep(num_servers=2)
+
+ def setUp(self):
+ self.mnt = tempfile.mkdtemp()
+ run_test_server.authorize_with("admin")
+ self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
+
+ def tearDown(self):
+ os.rmdir(self.mnt)
+ run_test_server.reset()
+
+ @staticmethod
+ def mount(argv):
+ """Decorator. Sets up a FUSE mount at self.mnt with the given args."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(self, *args, **kwargs):
+ with arvados_fuse.command.Mount(
+ arvados_fuse.command.ArgumentParser().parse_args(
+ argv + ['--foreground', self.mnt])):
+ return func(self, *args, **kwargs)
+ return wrapper
+ return decorator
diff --git a/services/fuse/tests/test_tmp_collection.py b/services/fuse/tests/test_tmp_collection.py
new file mode 100644
index 0000000..e403a2c
--- /dev/null
+++ b/services/fuse/tests/test_tmp_collection.py
@@ -0,0 +1,121 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import json
+import logging
+import os
+import tempfile
+import unittest
+
+from .integration_test import IntegrationTest
+from .mount_test_base import MountTestBase
+
+logger = logging.getLogger('arvados.arv-mount')
+
+
+class TmpCollectionArgsTest(unittest.TestCase):
+ def setUp(self):
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ os.rmdir(self.tmpdir)
+
+ def test_tmp_only(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'tmp1',
+ '--mount-tmp', 'tmp2',
+ self.tmpdir,
+ ])
+ self.assertIn(args.mode, [None, 'custom'])
+ self.assertEqual(['tmp1', 'tmp2'], args.mount_tmp)
+ for mtype in ['home', 'shared', 'by_id', 'by_pdh', 'by_tag']:
+ self.assertEqual([], getattr(args, 'mount_'+mtype))
+
+ def test_tmp_and_home(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'test_tmp',
+ '--mount-home', 'test_home',
+ self.tmpdir,
+ ])
+ self.assertIn(args.mode, [None, 'custom'])
+ self.assertEqual(['test_tmp'], args.mount_tmp)
+ self.assertEqual(['test_home'], args.mount_home)
+
+ def test_no_tmp(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ self.tmpdir,
+ ])
+ self.assertEqual([], args.mount_tmp)
+
+
+def current_manifest(tmpdir):
+ return json.load(open(
+ os.path.join(tmpdir, '.arvados#collection')
+ ))['manifest_text']
+
+
+class TmpCollectionTest(IntegrationTest):
+ mnt_args = [
+ '--read-write',
+ '--mount-tmp', 'zzz',
+ ]
+
+ @IntegrationTest.mount(argv=mnt_args+['--mount-tmp', 'yyy'])
+ def test_two_tmp(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'),
+ os.path.join(self.mnt, 'yyy'))
+ @staticmethod
+ def _test_two_tmp(self, zzz, yyy):
+ self.assertEqual(current_manifest(zzz), "")
+ self.assertEqual(current_manifest(yyy), "")
+ with open(os.path.join(zzz, 'foo'), 'w') as f:
+ f.write('foo')
+ self.assertNotEqual(current_manifest(zzz), "")
+ self.assertEqual(current_manifest(yyy), "")
+ os.unlink(os.path.join(zzz, 'foo'))
+ with open(os.path.join(yyy, 'bar'), 'w') as f:
+ f.write('bar')
+ self.assertEqual(current_manifest(zzz), "")
+ self.assertNotEqual(current_manifest(yyy), "")
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_empty(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_empty(self, tmpdir):
+ self.assertEqual(current_manifest(tmpdir), "")
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_onefile(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_onefile(self, tmpdir):
+ with open(os.path.join(tmpdir, 'foo'), 'w') as f:
+ f.write('foo')
+ self.assertRegexpMatches(
+ current_manifest(tmpdir),
+ r'^\. acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:foo\n$')
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_snapshots(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_snapshots(self, tmpdir):
+ ops = [
+ ('foo', 'bar',
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:foo\n$'),
+ ('foo', 'foo',
+ r'^\. acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:foo\n$'),
+ ('bar', 'bar',
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:bar 3:3:foo\n$'),
+ ('foo', None,
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:bar\n$'),
+ ]
+ for fn, content, expect in ops:
+ path = os.path.join(tmpdir, fn)
+ if content is None:
+ os.unlink(path)
+ else:
+ with open(path, 'w') as f:
+ f.write(content)
+ self.assertRegexpMatches(current_manifest(tmpdir), expect)
commit 7134fe6c4ae22c56b45232840918be9e05f9445a
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 19 17:37:25 2015 -0500
7751: Move code from arv-mount executable to module.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
new file mode 100644
index 0000000..e9d8bb5
--- /dev/null
+++ b/services/fuse/arvados_fuse/command.py
@@ -0,0 +1,331 @@
+import argparse
+import arvados
+import daemon
+import llfuse
+import logging
+import os
+import signal
+import subprocess
+import sys
+import time
+
+import arvados.commands._util as arv_cmd
+from arvados_fuse import crunchstat
+from arvados_fuse import *
+
+class ArgumentParser(argparse.ArgumentParser):
+ def __init__(self):
+ super(ArgumentParser, self).__init__(
+ parents=[arv_cmd.retry_opt],
+ description='''Mount Keep data under the local filesystem. Default mode is --home''',
+ epilog="""
+ Note: When using the --exec feature, you must either specify the
+ mountpoint before --exec, or mark the end of your --exec arguments
+ with "--".
+ """)
+ self.add_argument('mountpoint', type=str, help="""Mount point.""")
+ self.add_argument('--allow-other', action='store_true',
+ help="""Let other users read the mount""")
+
+ mode = self.add_mutually_exclusive_group()
+
+ mode.add_argument('--all', action='store_const', const='all', dest='mode',
+ help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
+ mode.add_argument('--custom', action='store_const', const=None, dest='mode',
+ help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
+ mode.add_argument('--home', action='store_const', const='home', dest='mode',
+ help="""Mount only the user's home project.""")
+ mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
+ help="""Mount only list of projects shared with the user.""")
+ mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
+ help="""Mount subdirectories listed by tag.""")
+ mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
+ help="""Mount subdirectories listed by portable data hash or uuid.""")
+ mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
+ help="""Mount subdirectories listed by portable data hash.""")
+ mode.add_argument('--project', type=str, metavar='UUID',
+ help="""Mount the specified project.""")
+ mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
+ help="""Mount only the specified collection.""")
+
+ mounts = self.add_argument_group('Custom mount options')
+ mounts.add_argument('--mount-by-pdh',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
+ mounts.add_argument('--mount-by-id',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
+ mounts.add_argument('--mount-by-tag',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
+ mounts.add_argument('--mount-home',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount the current user's home project at mountpoint/PATH.")
+ mounts.add_argument('--mount-shared',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount projects shared with the current user at mountpoint/PATH.")
+ mounts.add_argument('--mount-tmp',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
+
+ self.add_argument('--debug', action='store_true', help="""Debug mode""")
+ self.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
+ self.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
+ self.add_argument('--encoding', type=str, help="Character encoding to use for filesystem, default is utf-8 (see Python codec registry for list of available encodings)", default="utf-8")
+
+ self.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
+ self.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
+
+ self.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
+ self.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
+
+ self.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
+
+ self.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
+ dest="exec_args", metavar=('command', 'args', '...', '--'),
+ help="""Mount, run a command, then unmount and exit""")
+
+
+class Mount(object):
+ def __init__(self, args, logger=logging.getLogger('arvados.arv-mount')):
+ self.logger = logger
+ self.args = args
+
+ self.args.mountpoint = os.path.realpath(self.args.mountpoint)
+ if self.args.logfile:
+ self.args.logfile = os.path.realpath(self.args.logfile)
+
+ # Daemonize as early as possible, so we don't accidentally close
+ # file descriptors we're using.
+ self.daemon_ctx = None
+ if not (self.args.exec_args or self.args.foreground):
+ os.chdir(self.args.mountpoint)
+ self.daemon_ctx = daemon.DaemonContext(working_directory='.')
+ self.daemon_ctx.open()
+
+ try:
+ self._setup_logging()
+ self._setup_api()
+ self._setup_mount()
+ except Exception as e:
+ self.logger.exception("arv-mount: exception during setup: %s", e)
+ exit(1)
+
+ def __enter__(self):
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+ if self.args.mode != 'by_pdh':
+ self.operations.listen_for_events()
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+ self.operations.initlock.wait()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
+ self.operations.destroy()
+
+ def Run(self):
+ if self.args.exec_args:
+ self._run_exec(self.args)
+ else:
+ self._run_standalone(self.args)
+
+ def _fuse_options(self):
+ """FUSE mount options; see mount.fuse(8)"""
+ opts = [optname for optname in ['allow_other', 'debug']
+ if getattr(self.args, optname)]
+ # Increase default read/write size from 4KiB to 128KiB
+ opts += ["big_writes", "max_read=131072"]
+ return opts
+
+ def _setup_logging(self):
+ # Configure a log handler based on command-line switches.
+ if self.args.logfile:
+ log_handler = logging.FileHandler(self.args.logfile)
+ elif self.daemon_ctx:
+ log_handler = logging.NullHandler()
+ else:
+ log_handler = None
+
+ if log_handler is not None:
+ arvados.logger.removeHandler(arvados.log_handler)
+ arvados.logger.addHandler(log_handler)
+
+ if self.args.debug:
+ arvados.logger.setLevel(logging.DEBUG)
+ self.logger.debug("arv-mount debugging enabled")
+
+ self.logger.info("enable write is %s", self.args.enable_write)
+
+ def _setup_api(self):
+ self.api = arvados.safeapi.ThreadSafeApiCache(
+ apiconfig=arvados.config.settings(),
+ keep_params={
+ "block_cache": arvados.keep.KeepBlockCache(self.args.file_cache)
+ })
+
+ def _setup_mount(self):
+ self.operations = Operations(
+ os.getuid(),
+ os.getgid(),
+ api_client=self.api,
+ encoding=self.args.encoding,
+ inode_cache=InodeCache(cap=self.args.directory_cache),
+ enable_write=self.args.enable_write)
+
+ if self.args.crunchstat_interval:
+ statsthread = threading.Thread(
+ target=crunchstat.statlogger,
+ args=(self.args.crunchstat_interval,
+ self.api.keep,
+ self.operations))
+ statsthread.daemon = True
+ statsthread.start()
+
+ usr = self.api.users().current().execute(num_retries=self.args.retries)
+ now = time.time()
+ dir_class = None
+ dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
+ mount_readme = False
+
+ if self.args.mode is not None and (
+ self.args.mount_by_id or
+ self.args.mount_by_pdh or
+ self.args.mount_by_tag or
+ self.args.mount_home or
+ self.args.mount_shared or
+ self.args.mount_tmp or
+ self.args.mount_collection):
+ sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
+ format(self.args.mode))
+
+ if self.args.mode in ['by_id', 'by_pdh']:
+ # Set up the request handler with the 'magic directory' at the root
+ dir_class = MagicDirectory
+ dir_args.append(self.args.mode == 'by_pdh')
+ elif self.args.mode == 'by_tag':
+ dir_class = TagsDirectory
+ elif self.args.mode == 'shared':
+ dir_class = SharedDirectory
+ dir_args.append(usr)
+ elif self.args.mode == 'home':
+ dir_class = ProjectDirectory
+ dir_args.append(usr)
+ dir_args.append(True)
+ elif self.args.mode == 'all':
+ self.args.mount_by_id = ['by_id']
+ self.args.mount_by_tag = ['by_tag']
+ self.args.mount_home = ['home']
+ self.args.mount_shared = ['shared']
+ mount_readme = True
+ elif self.args.collection is not None:
+ # Set up the request handler with the collection at the root
+ dir_class = CollectionDirectory
+ dir_args.append(self.args.collection)
+ elif self.args.project is not None:
+ dir_class = ProjectDirectory
+ dir_args.append(
+ self.api.groups().get(uuid=self.args.project).execute(
+ num_retries=self.args.retries))
+
+ if dir_class is not None:
+ self.operations.inodes.add_entry(dir_class(*dir_args))
+ return
+
+ e = self.operations.inodes.add_entry(Directory(
+ llfuse.ROOT_INODE, self.operations.inodes))
+ dir_args[0] = e.inode
+
+ for name in self.args.mount_by_id:
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ for name in self.args.mount_by_pdh:
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=True))
+ for name in self.args.mount_by_tag:
+ self._add_mount(e, name, TagsDirectory(*dir_args))
+ for name in self.args.mount_home:
+ self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ for name in self.args.mount_shared:
+ self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ for name in self.args.mount_tmp:
+ self._add_mount(e, name, TmpCollectionDirectory(*dir_args))
+
+ if mount_readme:
+ text = self._readme_text(
+ arvados.config.get('ARVADOS_API_HOST'),
+ usr['email'])
+ self._add_mount(e, StringFile(e.inode, text, now))
+
+ def _add_mount(self, tld, name, ent):
+ if name in ['', '.', '..'] or '/' in name:
+ sys.exit("Mount point '{}' is not supported.".format(name))
+ tld._entries[name] = self.operations.inodes.add_entry(ent)
+
+ def _readme_text(self, api_host, user_email):
+ return '''
+Welcome to Arvados! This directory provides file system access to
+files and objects available on the Arvados installation located at
+'{}' using credentials for user '{}'.
+
+From here, the following directories are available:
+
+ by_id/ Access to Keep collections by uuid or portable data hash (see by_id/README for details).
+ by_tag/ Access to Keep collections organized by tag.
+ home/ The contents of your home project.
+ shared/ Projects shared with you.
+'''.format(api_host, user_email)
+
+ def _run_exec(self):
+ # Initialize the fuse connection
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+
+ # Subscribe to change events from API server
+ if self.args.mode != 'by_pdh':
+ self.operations.listen_for_events()
+
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ self.operations.initlock.wait()
+
+ rc = 255
+ try:
+ sp = subprocess.Popen(self.args.exec_args, shell=False)
+
+ # forward signals to the process.
+ signal.signal(signal.SIGINT, lambda signum, frame: sp.send_signal(signum))
+ signal.signal(signal.SIGTERM, lambda signum, frame: sp.send_signal(signum))
+ signal.signal(signal.SIGQUIT, lambda signum, frame: sp.send_signal(signum))
+
+ # wait for process to complete.
+ rc = sp.wait()
+
+ # restore default signal handlers.
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
+ signal.signal(signal.SIGQUIT, signal.SIG_DFL)
+ except Exception as e:
+ self.logger.exception(
+ 'arv-mount: exception during exec %s', self.args.exec_args)
+ try:
+ rc = e.errno
+ except AttributeError:
+ pass
+ finally:
+ subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
+ self.operations.destroy()
+ exit(rc)
+
+ def _run_standalone(self):
+ try:
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+
+ # Subscribe to change events from API server
+ self.operations.listen_for_events()
+
+ llfuse.main()
+ except Exception as e:
+ self.logger.exception('arv-mount: exception during mount: %s', e)
+ exit(getattr(e, 'errno', 1))
+ finally:
+ self.operations.destroy()
+ exit(0)
diff --git a/services/fuse/arvados_fuse/crunchstat.py b/services/fuse/arvados_fuse/crunchstat.py
new file mode 100644
index 0000000..67d2ccc
--- /dev/null
+++ b/services/fuse/arvados_fuse/crunchstat.py
@@ -0,0 +1,63 @@
+import sys
+import time
+
+class Stat(object):
+ def __init__(self, prefix, interval,
+ egr_name, ing_name,
+ egr_func, ing_func):
+ self.prefix = prefix
+ self.interval = interval
+ self.egr_name = egr_name
+ self.ing_name = ing_name
+ self.egress = egr_func
+ self.ingress = ing_func
+ self.egr_prev = self.egress()
+ self.ing_prev = self.ingress()
+
+ def update(self):
+ egr = self.egress()
+ ing = self.ingress()
+
+ delta = " -- interval %.4f seconds %d %s %d %s" % (self.interval,
+ egr - self.egr_prev,
+ self.egr_name,
+ ing - self.ing_prev,
+ self.ing_name)
+
+ sys.stderr.write("crunchstat: %s %d %s %d %s%s\n" % (self.prefix,
+ egr,
+ self.egr_name,
+ ing,
+ self.ing_name,
+ delta))
+
+ self.egr_prev = egr
+ self.ing_prev = ing
+
+
+def statlogger(interval, keep, ops):
+ calls = Stat("keepcalls", interval, "put", "get",
+ keep.put_counter.get,
+ keep.get_counter.get)
+ net = Stat("net:keep0", interval, "tx", "rx",
+ keep.upload_counter.get,
+ keep.download_counter.get)
+ cache = Stat("keepcache", interval, "hit", "miss",
+ keep.hits_counter.get,
+ keep.misses_counter.get)
+ fuseops = Stat("fuseops", interval,"write", "read",
+ ops.write_ops_counter.get,
+ ops.read_ops_counter.get)
+ blk = Stat("blkio:0:0", interval, "write", "read",
+ ops.write_counter.get,
+ ops.read_counter.get)
+
+ while True:
+ time.sleep(interval)
+ calls.update()
+ net.update()
+ cache.update()
+ fuseops.update()
+ blk.update()
+
+
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index a679905..05b4c50 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -1,350 +1,7 @@
#!/usr/bin/env python
-import argparse
-import arvados
-import daemon
-import logging
-import os
-import signal
-import subprocess
-import sys
-import time
-
-import arvados.commands._util as arv_cmd
-from arvados_fuse import *
-from arvados.safeapi import ThreadSafeApiCache
-import arvados.keep
-
-logger = logging.getLogger('arvados.arv-mount')
-
-class Stat(object):
- def __init__(self, prefix, interval,
- egr_name, ing_name,
- egr_func, ing_func):
- self.prefix = prefix
- self.interval = interval
- self.egr_name = egr_name
- self.ing_name = ing_name
- self.egress = egr_func
- self.ingress = ing_func
- self.egr_prev = self.egress()
- self.ing_prev = self.ingress()
-
- def update(self):
- egr = self.egress()
- ing = self.ingress()
-
- delta = " -- interval %.4f seconds %d %s %d %s" % (self.interval,
- egr - self.egr_prev,
- self.egr_name,
- ing - self.ing_prev,
- self.ing_name)
-
- sys.stderr.write("crunchstat: %s %d %s %d %s%s\n" % (self.prefix,
- egr,
- self.egr_name,
- ing,
- self.ing_name,
- delta))
-
- self.egr_prev = egr
- self.ing_prev = ing
-
-
-def statlogger(interval, keep, ops):
- calls = Stat("keepcalls", interval, "put", "get",
- keep.put_counter.get,
- keep.get_counter.get)
- net = Stat("net:keep0", interval, "tx", "rx",
- keep.upload_counter.get,
- keep.download_counter.get)
- cache = Stat("keepcache", interval, "hit", "miss",
- keep.hits_counter.get,
- keep.misses_counter.get)
- fuseops = Stat("fuseops", interval,"write", "read",
- ops.write_ops_counter.get,
- ops.read_ops_counter.get)
- blk = Stat("blkio:0:0", interval, "write", "read",
- ops.write_counter.get,
- ops.read_counter.get)
-
- while True:
- time.sleep(interval)
- calls.update()
- net.update()
- cache.update()
- fuseops.update()
- blk.update()
-
+import arvados_fuse
if __name__ == '__main__':
- # Handle command line parameters
- parser = argparse.ArgumentParser(
- parents=[arv_cmd.retry_opt],
- description='''Mount Keep data under the local filesystem. Default mode is --home''',
- epilog="""
-Note: When using the --exec feature, you must either specify the
-mountpoint before --exec, or mark the end of your --exec arguments
-with "--".
-""")
- parser.add_argument('mountpoint', type=str, help="""Mount point.""")
- parser.add_argument('--allow-other', action='store_true',
- help="""Let other users read the mount""")
-
- mount_mode = parser.add_mutually_exclusive_group()
-
- mount_mode.add_argument('--all', action='store_const', const='all', dest='mode',
- help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
- mount_mode.add_argument('--custom', action='store_const', const=None, dest='mode',
- help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
- mount_mode.add_argument('--home', action='store_const', const='home', dest='mode',
- help="""Mount only the user's home project.""")
- mount_mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
- help="""Mount only list of projects shared with the user.""")
- mount_mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
- help="""Mount subdirectories listed by tag.""")
- mount_mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
- help="""Mount subdirectories listed by portable data hash or uuid.""")
- mount_mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
- help="""Mount subdirectories listed by portable data hash.""")
- mount_mode.add_argument('--project', type=str, metavar='UUID',
- help="""Mount the specified project.""")
- mount_mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
- help="""Mount only the specified collection.""")
-
- mounts = parser.add_argument_group('Custom mount options')
- mounts.add_argument('--mount-by-pdh',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
- mounts.add_argument('--mount-by-id',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
- mounts.add_argument('--mount-by-tag',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
- mounts.add_argument('--mount-home',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount the current user's home project at mountpoint/PATH.")
- mounts.add_argument('--mount-shared',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount projects shared with the current user at mountpoint/PATH.")
- mounts.add_argument('--mount-tmp',
- type=str, metavar='PATH', action='append', default=[],
- help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
-
- parser.add_argument('--debug', action='store_true', help="""Debug mode""")
- parser.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
- parser.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
- parser.add_argument('--encoding', type=str, help="Character encoding to use for filesystem, default is utf-8 (see Python codec registry for list of available encodings)", default="utf-8")
-
- parser.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
- parser.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
-
- parser.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
- parser.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
-
- parser.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
-
- parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
- dest="exec_args", metavar=('command', 'args', '...', '--'),
- help="""Mount, run a command, then unmount and exit""")
-
- args = parser.parse_args()
- args.mountpoint = os.path.realpath(args.mountpoint)
- if args.logfile:
- args.logfile = os.path.realpath(args.logfile)
-
- # Daemonize as early as possible, so we don't accidentally close
- # file descriptors we're using.
- if not (args.exec_args or args.foreground):
- os.chdir(args.mountpoint)
- daemon_ctx = daemon.DaemonContext(working_directory='.')
- daemon_ctx.open()
- else:
- daemon_ctx = None
-
- # Configure a log handler based on command-line switches.
- if args.logfile:
- log_handler = logging.FileHandler(args.logfile)
- elif daemon_ctx:
- log_handler = logging.NullHandler()
- else:
- log_handler = None
-
- if log_handler is not None:
- arvados.logger.removeHandler(arvados.log_handler)
- arvados.logger.addHandler(log_handler)
-
- if args.debug:
- arvados.logger.setLevel(logging.DEBUG)
- logger.debug("arv-mount debugging enabled")
-
- logger.info("enable write is %s", args.enable_write)
-
- try:
- api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
- keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
-
- # Create the request handler
- operations = Operations(os.getuid(),
- os.getgid(),
- api_client=api,
- encoding=args.encoding,
- inode_cache=InodeCache(cap=args.directory_cache),
- enable_write=args.enable_write)
-
- if args.crunchstat_interval:
- statsthread = threading.Thread(target=statlogger, args=(args.crunchstat_interval, api.keep, operations))
- statsthread.daemon = True
- statsthread.start()
-
- usr = api.users().current().execute(num_retries=args.retries)
- now = time.time()
- dir_class = None
- dir_args = [llfuse.ROOT_INODE, operations.inodes, api, args.retries]
- mount_readme = False
-
- if args.mode is not None and (
- args.mount_by_id or
- args.mount_by_pdh or
- args.mount_by_tag or
- args.mount_home or
- args.mount_shared or
- args.mount_tmp or
- args.mount_collection):
- sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
- format(args.mode))
-
- if args.mode in ['by_id', 'by_pdh']:
- # Set up the request handler with the 'magic directory' at the root
- dir_class = MagicDirectory
- dir_args.append(args.mode == 'by_pdh')
- elif args.mode == 'by_tag':
- dir_class = TagsDirectory
- elif args.mode == 'shared':
- dir_class = SharedDirectory
- dir_args.append(usr)
- elif args.mode == 'home':
- dir_class = ProjectDirectory
- dir_args.append(usr)
- dir_args.append(True)
- elif args.mode == 'all':
- args.mount_by_id = ['by_id']
- args.mount_by_tag = ['by_tag']
- args.mount_home = ['home']
- args.mount_shared = ['shared']
- mount_readme = True
- elif args.collection is not None:
- # Set up the request handler with the collection at the root
- dir_class = CollectionDirectory
- dir_args.append(args.collection)
- elif args.project is not None:
- dir_class = ProjectDirectory
- dir_args.append(api.groups().get(uuid=args.project).execute(
- num_retries=args.retries))
-
- if dir_class is not None:
- operations.inodes.add_entry(dir_class(*dir_args))
- else:
- e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE, operations.inodes))
- dir_args[0] = e.inode
-
- def addMount(tld, name, ent):
- if name in ['', '.', '..'] or '/' in name:
- sys.exit("Mount point '{}' is not supported.".format(name))
- tld._entries[name] = operations.inodes.add_entry(ent)
-
- for name in args.mount_by_id:
- addMount(e, name, MagicDirectory(*dir_args, pdh_only=False))
- for name in args.mount_by_pdh:
- addMount(e, name, MagicDirectory(*dir_args, pdh_only=True))
- for name in args.mount_by_tag:
- addMount(e, name, TagsDirectory(*dir_args))
- for name in args.mount_home:
- addMount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
- for name in args.mount_shared:
- addMount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
- for name in args.mount_tmp:
- addMount(e, name, TmpCollectionDirectory(*dir_args))
-
- if mount_readme:
- text = '''
-Welcome to Arvados! This directory provides file system access to
-files and objects available on the Arvados installation located at
-'{}' using credentials for user '{}'.
-
-From here, the following directories are available:
-
- by_id/ Access to Keep collections by uuid or portable data hash (see by_id/README for details).
- by_tag/ Access to Keep collections organized by tag.
- home/ The contents of your home project.
- shared/ Projects shared with you.
-'''.format(arvados.config.get('ARVADOS_API_HOST'), usr['email'])
- addMount(e, StringFile(e.inode, text, now))
-
- except Exception:
- logger.exception("arv-mount: exception during API setup")
- exit(1)
-
- # FUSE options, see mount.fuse(8)
- opts = [optname for optname in ['allow_other', 'debug']
- if getattr(args, optname)]
-
- # Increase default read/write size from 4KiB to 128KiB
- opts += ["big_writes", "max_read=131072"]
-
- if args.exec_args:
- # Initialize the fuse connection
- llfuse.init(operations, args.mountpoint, opts)
-
- # Subscribe to change events from API server
- if args.mode != 'by_pdh':
- operations.listen_for_events()
-
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- rc = 255
- try:
- sp = subprocess.Popen(args.exec_args, shell=False)
-
- # forward signals to the process.
- signal.signal(signal.SIGINT, lambda signum, frame: sp.send_signal(signum))
- signal.signal(signal.SIGTERM, lambda signum, frame: sp.send_signal(signum))
- signal.signal(signal.SIGQUIT, lambda signum, frame: sp.send_signal(signum))
-
- # wait for process to complete.
- rc = sp.wait()
-
- # restore default signal handlers.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
- signal.signal(signal.SIGQUIT, signal.SIG_DFL)
- except Exception as e:
- logger.exception('arv-mount: exception during exec %s',
- args.exec_args)
- try:
- rc = e.errno
- except AttributeError:
- pass
- finally:
- subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
- operations.destroy()
-
- exit(rc)
- else:
- try:
- llfuse.init(operations, args.mountpoint, opts)
-
- # Subscribe to change events from API server
- operations.listen_for_events()
-
- llfuse.main()
- except Exception as e:
- logger.exception('arv-mount: exception during mount')
- exit(getattr(e, 'errno', 1))
- finally:
- operations.destroy()
+ args = arvados_fuse.command.ArgumentParser().parse_args()
+ arvados_fuse.command.Mount(args).Run()
commit fcdaf1cedc195d3835a96d89471837ae321c5063
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 18 14:32:18 2015 -0500
7751: Add --mount-tmp option.
diff --git a/services/arv-web/arv-web.py b/services/arv-web/arv-web.py
index 482a577..5a95e27 100755
--- a/services/arv-web/arv-web.py
+++ b/services/arv-web/arv-web.py
@@ -83,7 +83,7 @@ class ArvWeb(object):
def run_fuse_mount(self):
self.mountdir = tempfile.mkdtemp()
- self.operations = Operations(os.getuid(), os.getgid(), "utf-8")
+ self.operations = Operations(os.getuid(), os.getgid(), self.api, "utf-8")
self.cdir = CollectionDirectory(llfuse.ROOT_INODE, self.operations.inodes, self.api, 2, self.collection)
self.operations.inodes.add_entry(self.cdir)
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index fd25aa9..55f1ad7 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -76,7 +76,7 @@ import Queue
llfuse.capi._notify_queue = Queue.Queue()
-from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
+from fusedir import sanitize_filename, Directory, CollectionDirectory, TmpCollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
from fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -304,9 +304,11 @@ class Operations(llfuse.Operations):
"""
- def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4, enable_write=False):
+ def __init__(self, uid, gid, api_client, encoding="utf-8", inode_cache=None, num_retries=4, enable_write=False):
super(Operations, self).__init__()
+ self._api_client = api_client
+
if not inode_cache:
inode_cache = InodeCache(cap=256*1024*1024)
self.inodes = Inodes(inode_cache, encoding=encoding)
@@ -347,8 +349,8 @@ class Operations(llfuse.Operations):
def access(self, inode, mode, ctx):
return True
- def listen_for_events(self, api_client):
- self.events = arvados.events.subscribe(api_client,
+ def listen_for_events(self):
+ self.events = arvados.events.subscribe(self._api_client,
[["event_type", "in", ["create", "update", "delete"]]],
self.on_event)
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index fdc93fb..21961a5 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -474,6 +474,74 @@ class CollectionDirectory(CollectionDirectoryBase):
self.collection.stop_threads()
+class TmpCollectionDirectory(CollectionDirectoryBase):
+ """A directory backed by an Arvados collection that never gets saved.
+
+ This supports using Keep as scratch space. A userspace program can
+ read the .arvados#collection file to get a current manifest in
+ order to save a snapshot of the scratch data or use it as a crunch
+ job output.
+ """
+
+ def __init__(self, parent_inode, inodes, api_client, num_retries):
+ collection = arvados.collection.Collection(
+ api_client=api_client,
+ keep_client=api_client.keep)
+ collection.save = self._commit_collection
+ collection.save_new = self._commit_collection
+ super(TmpCollectionDirectory, self).__init__(
+ parent_inode, inodes, collection)
+ self.collection_record_file = None
+ self._subscribed = False
+ self._update_collection_record()
+
+ def update(self, *args, **kwargs):
+ if not self._subscribed:
+ with llfuse.lock_released:
+ self.populate(self.mtime())
+ self._subscribed = True
+
+ @use_counter
+ def _commit_collection(self):
+ """Commit the data blocks, but don't save the collection to API.
+
+ Update the content of the special .arvados#collection file, if
+ it has been instantiated.
+ """
+ self.collection.flush()
+ self._update_collection_record()
+ if self.collection_record_file is not None:
+ self.collection_record_file.update(self.collection_record)
+ self.inodes.invalidate_inode(self.collection_record_file.inode)
+
+ def _update_collection_record(self):
+ self.collection_record = {
+ "uuid": None,
+ "manifest_text": self.collection.manifest_text(),
+ "portable_data_hash": self.collection.portable_data_hash(),
+ }
+
+ def __contains__(self, k):
+ return (k == '.arvados#collection' or
+ super(TmpCollectionDirectory, self).__contains__(k))
+
+ @use_counter
+ def __getitem__(self, item):
+ if item == '.arvados#collection':
+ if self.collection_record_file is None:
+ self.collection_record_file = ObjectFile(
+ self.inode, self.collection_record)
+ self.inodes.add_entry(self.collection_record_file)
+ return self.collection_record_file
+ return super(TmpCollectionDirectory, self).__getitem__(item)
+
+ def writable(self):
+ return True
+
+ def finalize(self):
+ self.collection.stop_threads()
+
+
class MagicDirectory(Directory):
"""A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 44bc698..a679905 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -7,6 +7,7 @@ import logging
import os
import signal
import subprocess
+import sys
import time
import arvados.commands._util as arv_cmd
@@ -92,17 +93,44 @@ with "--".
mount_mode = parser.add_mutually_exclusive_group()
- mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default).""")
- mount_mode.add_argument('--home', action='store_true', help="""Mount only the user's home project.""")
- mount_mode.add_argument('--shared', action='store_true', help="""Mount only list of projects shared with the user.""")
- mount_mode.add_argument('--by-tag', action='store_true',
+ mount_mode.add_argument('--all', action='store_const', const='all', dest='mode',
+ help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
+ mount_mode.add_argument('--custom', action='store_const', const=None, dest='mode',
+ help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
+ mount_mode.add_argument('--home', action='store_const', const='home', dest='mode',
+ help="""Mount only the user's home project.""")
+ mount_mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
+ help="""Mount only list of projects shared with the user.""")
+ mount_mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
help="""Mount subdirectories listed by tag.""")
- mount_mode.add_argument('--by-id', action='store_true',
+ mount_mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
help="""Mount subdirectories listed by portable data hash or uuid.""")
- mount_mode.add_argument('--by-pdh', action='store_true',
+ mount_mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
help="""Mount subdirectories listed by portable data hash.""")
- mount_mode.add_argument('--project', type=str, help="""Mount a specific project.""")
- mount_mode.add_argument('--collection', type=str, help="""Mount only the specified collection.""")
+ mount_mode.add_argument('--project', type=str, metavar='UUID',
+ help="""Mount the specified project.""")
+ mount_mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
+ help="""Mount only the specified collection.""")
+
+ mounts = parser.add_argument_group('Custom mount options')
+ mounts.add_argument('--mount-by-pdh',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
+ mounts.add_argument('--mount-by-id',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
+ mounts.add_argument('--mount-by-tag',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
+ mounts.add_argument('--mount-home',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount the current user's home project at mountpoint/PATH.")
+ mounts.add_argument('--mount-shared',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount projects shared with the current user at mountpoint/PATH.")
+ mounts.add_argument('--mount-tmp',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
parser.add_argument('--debug', action='store_true', help="""Debug mode""")
parser.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
@@ -154,14 +182,16 @@ with "--".
logger.info("enable write is %s", args.enable_write)
try:
+ api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
+ keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
+
# Create the request handler
operations = Operations(os.getuid(),
os.getgid(),
+ api_client=api,
encoding=args.encoding,
inode_cache=InodeCache(cap=args.directory_cache),
enable_write=args.enable_write)
- api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
- keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
if args.crunchstat_interval:
statsthread = threading.Thread(target=statlogger, args=(args.crunchstat_interval, api.keep, operations))
@@ -172,19 +202,38 @@ with "--".
now = time.time()
dir_class = None
dir_args = [llfuse.ROOT_INODE, operations.inodes, api, args.retries]
- if args.by_id or args.by_pdh:
+ mount_readme = False
+
+ if args.mode is not None and (
+ args.mount_by_id or
+ args.mount_by_pdh or
+ args.mount_by_tag or
+ args.mount_home or
+ args.mount_shared or
+ args.mount_tmp or
+ args.mount_collection):
+ sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
+ format(args.mode))
+
+ if args.mode in ['by_id', 'by_pdh']:
# Set up the request handler with the 'magic directory' at the root
dir_class = MagicDirectory
- dir_args.append(args.by_pdh)
- elif args.by_tag:
+ dir_args.append(args.mode == 'by_pdh')
+ elif args.mode == 'by_tag':
dir_class = TagsDirectory
- elif args.shared:
+ elif args.mode == 'shared':
dir_class = SharedDirectory
dir_args.append(usr)
- elif args.home:
+ elif args.mode == 'home':
dir_class = ProjectDirectory
dir_args.append(usr)
dir_args.append(True)
+ elif args.mode == 'all':
+ args.mount_by_id = ['by_id']
+ args.mount_by_tag = ['by_tag']
+ args.mount_home = ['home']
+ args.mount_shared = ['shared']
+ mount_readme = True
elif args.collection is not None:
# Set up the request handler with the collection at the root
dir_class = CollectionDirectory
@@ -200,19 +249,29 @@ with "--".
e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE, operations.inodes))
dir_args[0] = e.inode
- e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(*dir_args))
-
- e._entries['by_tag'] = operations.inodes.add_entry(TagsDirectory(*dir_args))
-
- dir_args.append(usr)
- dir_args.append(True)
- e._entries['home'] = operations.inodes.add_entry(ProjectDirectory(*dir_args))
- e._entries['shared'] = operations.inodes.add_entry(SharedDirectory(*dir_args))
-
- text = '''
-Welcome to Arvados! This directory provides file system access to files and objects
-available on the Arvados installation located at '{}'
-using credentials for user '{}'.
+ def addMount(tld, name, ent):
+ if name in ['', '.', '..'] or '/' in name:
+ sys.exit("Mount point '{}' is not supported.".format(name))
+ tld._entries[name] = operations.inodes.add_entry(ent)
+
+ for name in args.mount_by_id:
+ addMount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ for name in args.mount_by_pdh:
+ addMount(e, name, MagicDirectory(*dir_args, pdh_only=True))
+ for name in args.mount_by_tag:
+ addMount(e, name, TagsDirectory(*dir_args))
+ for name in args.mount_home:
+ addMount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ for name in args.mount_shared:
+ addMount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ for name in args.mount_tmp:
+ addMount(e, name, TmpCollectionDirectory(*dir_args))
+
+ if mount_readme:
+ text = '''
+Welcome to Arvados! This directory provides file system access to
+files and objects available on the Arvados installation located at
+'{}' using credentials for user '{}'.
From here, the following directories are available:
@@ -221,9 +280,7 @@ From here, the following directories are available:
home/ The contents of your home project.
shared/ Projects shared with you.
'''.format(arvados.config.get('ARVADOS_API_HOST'), usr['email'])
-
- e._entries["README"] = operations.inodes.add_entry(StringFile(e.inode, text, now))
-
+ addMount(e, StringFile(e.inode, text, now))
except Exception:
logger.exception("arv-mount: exception during API setup")
@@ -241,8 +298,8 @@ From here, the following directories are available:
llfuse.init(operations, args.mountpoint, opts)
# Subscribe to change events from API server
- if not args.by_pdh:
- operations.listen_for_events(api)
+ if args.mode != 'by_pdh':
+ operations.listen_for_events()
t = threading.Thread(None, lambda: llfuse.main())
t.start()
@@ -283,7 +340,7 @@ From here, the following directories are available:
llfuse.init(operations, args.mountpoint, opts)
# Subscribe to change events from API server
- operations.listen_for_events(api)
+ operations.listen_for_events()
llfuse.main()
except Exception as e:
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 3b7cbaa..9fb24db 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -35,7 +35,10 @@ class MountTestBase(unittest.TestCase):
self.api = api if api else arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def make_mount(self, root_class, **root_kwargs):
- self.operations = fuse.Operations(os.getuid(), os.getgid(), enable_write=True)
+ self.operations = fuse.Operations(
+ os.getuid(), os.getgid(),
+ api_client=self.api,
+ enable_write=True)
self.operations.inodes.add_entry(root_class(
llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
llfuse.init(self.operations, self.mounttmp, [])
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 1d7b908..05c8685 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -703,7 +703,7 @@ class FuseUpdateFromEventTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- self.operations.listen_for_events(self.api)
+ self.operations.listen_for_events()
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list