[ARVADOS] created: af17ab93eac126058eac64ef69cd7ed5980ca9f0
git at public.curoverse.com
git at public.curoverse.com
Fri Nov 20 18:27:35 EST 2015
at af17ab93eac126058eac64ef69cd7ed5980ca9f0 (commit)
commit af17ab93eac126058eac64ef69cd7ed5980ca9f0
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 18:37:32 2015 -0500
7751: Update explanation of special .arvados#collection file. Fix wayward use of "Keep locator".
diff --git a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
index 73c02ef..32cc3ca 100644
--- a/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Arv-mount
* It is easy for existing tools to access files in Keep.
* Data is streamed on demand. It is not necessary to download an entire file or collection to start processing.
-The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by their Keep locator. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
+The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory. To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by UUID or portable data hash. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
<notextile>
<pre><code>~$ <span class="userinput">mkdir -p keep</span>
@@ -33,7 +33,7 @@ var-GS000016015-ASM.tsv.bz2
The last line unmounts Keep. Subdirectories will no longer be accessible.
-Within each directory on Keep, there is a @.arvados#collection@ file that does not show up with @ls at . Its contents include, for instance, the @portable_data_hash@, which is the same as the Keep locator.
+In the top level directory of each collection, arv-mount provides a special file called @.arvados#collection@ that contains a JSON-formatted API record for the collection. This can be used to determine the collection's @portable_data_hash@, @uuid@, etc. This file does not show up in @ls@ or @ls -a at .
h3. Modifying files and directories in Keep
commit 8670ed65540036975de9a28bfbfaf0b4df9c19d8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 18:09:55 2015 -0500
7751: Return an empty array from InodeCache.find() instead of None.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 55f1ad7..a540ebd 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -192,8 +192,8 @@ class InodeCache(object):
if obj.persisted() and obj.cache_priority in self._entries:
self._remove(obj, True)
- def find(self, uuid):
- return self._by_uuid.get(uuid)
+ def find_by_uuid(self, uuid):
+ return self._by_uuid.get(uuid, [])
def clear(self):
self._entries.clear()
@@ -356,34 +356,37 @@ class Operations(llfuse.Operations):
@catch_exceptions
def on_event(self, ev):
- if 'event_type' in ev:
- with llfuse.lock:
- items = self.inodes.inode_cache.find(ev["object_uuid"])
- if items is not None:
- for item in items:
- item.invalidate()
- if ev["object_kind"] == "arvados#collection":
- new_attr = ev.get("properties") and ev["properties"].get("new_attributes") and ev["properties"]["new_attributes"]
-
- # new_attributes.modified_at currently lacks subsecond precision (see #6347) so use event_at which
- # should always be the same.
- #record_version = (new_attr["modified_at"], new_attr["portable_data_hash"]) if new_attr else None
- record_version = (ev["event_at"], new_attr["portable_data_hash"]) if new_attr else None
-
- item.update(to_record_version=record_version)
- else:
- item.update()
-
- oldowner = ev.get("properties") and ev["properties"].get("old_attributes") and ev["properties"]["old_attributes"].get("owner_uuid")
- olditemparent = self.inodes.inode_cache.find(oldowner)
- if olditemparent is not None:
- olditemparent.invalidate()
- olditemparent.update()
-
- itemparent = self.inodes.inode_cache.find(ev["object_owner_uuid"])
- if itemparent is not None:
- itemparent.invalidate()
- itemparent.update()
+ if 'event_type' not in ev:
+ return
+ with llfuse.lock:
+ for item in self.inodes.inode_cache.find_by_uuid(ev["object_uuid"]):
+ item.invalidate()
+ if ev["object_kind"] == "arvados#collection":
+ new_attr = (ev.get("properties") and
+ ev["properties"].get("new_attributes") and
+ ev["properties"]["new_attributes"])
+
+ # new_attributes.modified_at currently lacks
+ # subsecond precision (see #6347) so use event_at
+ # which should always be the same.
+ record_version = (
+ (ev["event_at"], new_attr["portable_data_hash"])
+ if new_attr else None)
+
+ item.update(to_record_version=record_version)
+ else:
+ item.update()
+
+ oldowner = (
+ ev.get("properties") and
+ ev["properties"].get("old_attributes") and
+ ev["properties"]["old_attributes"].get("owner_uuid"))
+ newowner = ev["object_owner_uuid"]
+ for parent in (
+ self.inodes.inode_cache.find_by_uuid(oldowner) +
+ self.inodes.inode_cache.find_by_uuid(newowner)):
+ parent.invalidate()
+ parent.update()
@catch_exceptions
commit dc5f441e4b0b3832809ead096c1e30b5e6c74f22
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 03:38:10 2015 -0500
7751: Test mount arguments.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index e9d8bb5..ba74111 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -187,16 +187,31 @@ class Mount(object):
dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
mount_readme = False
- if self.args.mode is not None and (
- self.args.mount_by_id or
- self.args.mount_by_pdh or
- self.args.mount_by_tag or
- self.args.mount_home or
- self.args.mount_shared or
- self.args.mount_tmp or
- self.args.mount_collection):
- sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
- format(self.args.mode))
+ if self.args.collection is not None:
+ # Set up the request handler with the collection at the root
+ self.args.mode = 'collection'
+ dir_class = CollectionDirectory
+ dir_args.append(self.args.collection)
+ elif self.args.project is not None:
+ self.args.mode = 'project'
+ dir_class = ProjectDirectory
+ dir_args.append(
+ self.api.groups().get(uuid=self.args.project).execute(
+ num_retries=self.args.retries))
+
+ if (self.args.mount_by_id or
+ self.args.mount_by_pdh or
+ self.args.mount_by_tag or
+ self.args.mount_home or
+ self.args.mount_shared or
+ self.args.mount_tmp):
+ if self.args.mode is not None:
+ sys.exit(
+ "Cannot combine '{}' mode with custom --mount-* options.".
+ format(self.args.mode))
+ elif self.args.mode is None:
+ # If no --mount-custom or custom mount args, --all is the default
+ self.args.mode = 'all'
if self.args.mode in ['by_id', 'by_pdh']:
# Set up the request handler with the 'magic directory' at the root
@@ -217,15 +232,6 @@ class Mount(object):
self.args.mount_home = ['home']
self.args.mount_shared = ['shared']
mount_readme = True
- elif self.args.collection is not None:
- # Set up the request handler with the collection at the root
- dir_class = CollectionDirectory
- dir_args.append(self.args.collection)
- elif self.args.project is not None:
- dir_class = ProjectDirectory
- dir_args.append(
- self.api.groups().get(uuid=self.args.project).execute(
- num_retries=self.args.retries))
if dir_class is not None:
self.operations.inodes.add_entry(dir_class(*dir_args))
@@ -252,7 +258,7 @@ class Mount(object):
text = self._readme_text(
arvados.config.get('ARVADOS_API_HOST'),
usr['email'])
- self._add_mount(e, StringFile(e.inode, text, now))
+ self._add_mount(e, 'README', StringFile(e.inode, text, now))
def _add_mount(self, tld, name, ent):
if name in ['', '.', '..'] or '/' in name:
@@ -271,6 +277,7 @@ From here, the following directories are available:
by_tag/ Access to Keep collections organized by tag.
home/ The contents of your home project.
shared/ Projects shared with you.
+
'''.format(api_host, user_email)
def _run_exec(self):
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 21961a5..04c2d50 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -556,12 +556,13 @@ class MagicDirectory(Directory):
README_TEXT = """
This directory provides access to Arvados collections as subdirectories listed
by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
-the form '1234567890abcdefghijklmnopqrstuv+123').
+the form '1234567890abcdef0123456789abcdef+123').
Note that this directory will appear empty until you attempt to access a
specific collection subdirectory (such as trying to 'cd' into it), at which
point the collection will actually be looked up on the server and the directory
will appear if it exists.
+
""".lstrip()
def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False):
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 9652709..faa4a55 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -47,7 +47,7 @@ class IntegrationTest(unittest.TestCase):
def setUp(self):
self.mnt = tempfile.mkdtemp()
- run_test_server.authorize_with("admin")
+ run_test_server.authorize_with('active')
self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def tearDown(self):
diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py
new file mode 100644
index 0000000..62987cc
--- /dev/null
+++ b/services/fuse/tests/test_command_args.py
@@ -0,0 +1,176 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import functools
+import json
+import llfuse
+import logging
+import os
+import run_test_server
+import tempfile
+import unittest
+
+def noexit(func):
+ """If argparse or arvados_fuse tries to exit, fail the test instead"""
+ class SystemExitCaught(StandardError):
+ pass
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ return func(*args, **kwargs)
+ except SystemExit:
+ raise SystemExitCaught
+ return wrapper
+
+class MountArgsTest(unittest.TestCase):
+ def setUp(self):
+ self.mntdir = tempfile.mkdtemp()
+ run_test_server.authorize_with('active')
+
+ def tearDown(self):
+ os.rmdir(self.mntdir)
+
+ def lookup(self, mnt, *path):
+ ent = mnt.operations.inodes[llfuse.ROOT_INODE]
+ for p in path:
+ ent = ent[p]
+ return ent
+
+ def check_ent_type(self, cls, *path):
+ ent = self.lookup(self.mnt, *path)
+ self.assertEqual(ent.__class__, cls)
+ return ent
+
+ @noexit
+ def test_default_all(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+ e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
+
+ e = self.check_ent_type(arvados_fuse.StringFile, 'README')
+ readme = e.readfrom(0, -1)
+ self.assertRegexpMatches(readme, r'active-user at arvados\.local')
+ self.assertRegexpMatches(readme, r'\n$')
+
+ e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
+ txt = e.readfrom(0, -1)
+ self.assertRegexpMatches(txt, r'portable data hash')
+ self.assertRegexpMatches(txt, r'\n$')
+
+ @noexit
+ def test_by_id(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-id',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_id')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.MagicDirectory)
+ self.assertEqual(e.pdh_only, False)
+
+ @noexit
+ def test_by_pdh(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-pdh',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_pdh')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.MagicDirectory)
+ self.assertEqual(e.pdh_only, True)
+
+ @noexit
+ def test_by_tag(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--by-tag',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'by_tag')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.TagsDirectory)
+
+ @noexit
+ def test_collection(self, id_type='uuid'):
+ c = run_test_server.fixture('collections')['public_text_file']
+ cid = c[id_type]
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--collection', cid,
+ '--foreground', self.mntdir])
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.CollectionDirectory)
+ self.assertEqual(e.collection_locator, cid)
+
+ def test_collection_pdh(self):
+ self.test_collection('portable_data_hash')
+
+ @noexit
+ def test_home(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--home',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'home')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory)
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ def test_mutually_exclusive_args(self):
+ cid = run_test_server.fixture('collections')['public_text_file']['uuid']
+ gid = run_test_server.fixture('groups')['aproject']['uuid']
+ for badargs in [
+ ['--mount-tmp', 'foo', '--collection', cid],
+ ['--mount-tmp', 'foo', '--project', gid],
+ ['--collection', cid, '--project', gid],
+ ['--by-id', '--project', gid],
+ ['--mount-tmp', 'foo', '--by-id'],
+ ]:
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args(
+ badargs + ['--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
+ @noexit
+ def test_project(self):
+ uuid = run_test_server.fixture('groups')['aproject']['uuid']
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--project', uuid,
+ '--foreground', self.mntdir])
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory)
+ self.assertEqual(e.project_object['uuid'], uuid)
+
+ @noexit
+ def test_shared(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--shared',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, 'shared')
+ self.mnt = arvados_fuse.command.Mount(args)
+ e = self.check_ent_type(arvados_fuse.SharedDirectory)
+ self.assertEqual(e.current_user['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ @noexit
+ def test_custom(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'foo',
+ '--mount-tmp', 'bar',
+ '--mount-home', 'my_home',
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ self.mnt = arvados_fuse.command.Mount(args)
+ self.check_ent_type(arvados_fuse.Directory)
+ self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
+ self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
+ e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
+ self.assertEqual(e.project_object['uuid'],
+ run_test_server.fixture('users')['active']['uuid'])
+
+ def test_custom_unsupported_layouts(self):
+ for name in ['.', '..', '', 'foo/bar', '/foo']:
+ with self.assertRaises(SystemExit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', name,
+ '--foreground', self.mntdir])
+ arvados_fuse.command.Mount(args)
commit 857b016d1407e8a3df47ceec469b046808e99571
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 19 20:38:34 2015 -0500
7751: Add tests for --mount-tmp option.
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
new file mode 100644
index 0000000..9652709
--- /dev/null
+++ b/services/fuse/tests/integration_test.py
@@ -0,0 +1,68 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import functools
+import inspect
+import multiprocessing
+import os
+import sys
+import tempfile
+import unittest
+import run_test_server
+
+def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
+ class Test(unittest.TestCase):
+ def runTest(self, *args, **kwargs):
+ getattr(getattr(sys.modules[modName], clsName), funcName)(self, *args, **kwargs)
+ Test().runTest(*args, **kwargs)
+
+
+class IntegrationTest(unittest.TestCase):
+ def pool_test(self, *args, **kwargs):
+ """Run a static method as a unit test, in a different process.
+
+ If called by method 'foobar', the static method '_foobar' of
+ the same class will be called in the other process.
+ """
+ modName = inspect.getmodule(self).__name__
+ clsName = self.__class__.__name__
+ funcName = inspect.currentframe().f_back.f_code.co_name
+ pool = multiprocessing.Pool(1)
+ try:
+ pool.apply(
+ wrap_static_test_method,
+ (modName, clsName, '_'+funcName, args, kwargs))
+ finally:
+ pool.terminate()
+ pool.join()
+
+ @classmethod
+ def setUpClass(cls):
+ run_test_server.run()
+ run_test_server.run_keep(enforce_permissions=True, num_servers=2)
+
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop_keep(num_servers=2)
+
+ def setUp(self):
+ self.mnt = tempfile.mkdtemp()
+ run_test_server.authorize_with("admin")
+ self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
+
+ def tearDown(self):
+ os.rmdir(self.mnt)
+ run_test_server.reset()
+
+ @staticmethod
+ def mount(argv):
+ """Decorator. Sets up a FUSE mount at self.mnt with the given args."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(self, *args, **kwargs):
+ with arvados_fuse.command.Mount(
+ arvados_fuse.command.ArgumentParser().parse_args(
+ argv + ['--foreground', self.mnt])):
+ return func(self, *args, **kwargs)
+ return wrapper
+ return decorator
diff --git a/services/fuse/tests/test_tmp_collection.py b/services/fuse/tests/test_tmp_collection.py
new file mode 100644
index 0000000..e403a2c
--- /dev/null
+++ b/services/fuse/tests/test_tmp_collection.py
@@ -0,0 +1,121 @@
+import arvados
+import arvados_fuse
+import arvados_fuse.command
+import json
+import logging
+import os
+import tempfile
+import unittest
+
+from .integration_test import IntegrationTest
+from .mount_test_base import MountTestBase
+
+logger = logging.getLogger('arvados.arv-mount')
+
+
+class TmpCollectionArgsTest(unittest.TestCase):
+ def setUp(self):
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ os.rmdir(self.tmpdir)
+
+ def test_tmp_only(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'tmp1',
+ '--mount-tmp', 'tmp2',
+ self.tmpdir,
+ ])
+ self.assertIn(args.mode, [None, 'custom'])
+ self.assertEqual(['tmp1', 'tmp2'], args.mount_tmp)
+ for mtype in ['home', 'shared', 'by_id', 'by_pdh', 'by_tag']:
+ self.assertEqual([], getattr(args, 'mount_'+mtype))
+
+ def test_tmp_and_home(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--mount-tmp', 'test_tmp',
+ '--mount-home', 'test_home',
+ self.tmpdir,
+ ])
+ self.assertIn(args.mode, [None, 'custom'])
+ self.assertEqual(['test_tmp'], args.mount_tmp)
+ self.assertEqual(['test_home'], args.mount_home)
+
+ def test_no_tmp(self):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ self.tmpdir,
+ ])
+ self.assertEqual([], args.mount_tmp)
+
+
+def current_manifest(tmpdir):
+ return json.load(open(
+ os.path.join(tmpdir, '.arvados#collection')
+ ))['manifest_text']
+
+
+class TmpCollectionTest(IntegrationTest):
+ mnt_args = [
+ '--read-write',
+ '--mount-tmp', 'zzz',
+ ]
+
+ @IntegrationTest.mount(argv=mnt_args+['--mount-tmp', 'yyy'])
+ def test_two_tmp(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'),
+ os.path.join(self.mnt, 'yyy'))
+ @staticmethod
+ def _test_two_tmp(self, zzz, yyy):
+ self.assertEqual(current_manifest(zzz), "")
+ self.assertEqual(current_manifest(yyy), "")
+ with open(os.path.join(zzz, 'foo'), 'w') as f:
+ f.write('foo')
+ self.assertNotEqual(current_manifest(zzz), "")
+ self.assertEqual(current_manifest(yyy), "")
+ os.unlink(os.path.join(zzz, 'foo'))
+ with open(os.path.join(yyy, 'bar'), 'w') as f:
+ f.write('bar')
+ self.assertEqual(current_manifest(zzz), "")
+ self.assertNotEqual(current_manifest(yyy), "")
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_empty(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_empty(self, tmpdir):
+ self.assertEqual(current_manifest(tmpdir), "")
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_onefile(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_onefile(self, tmpdir):
+ with open(os.path.join(tmpdir, 'foo'), 'w') as f:
+ f.write('foo')
+ self.assertRegexpMatches(
+ current_manifest(tmpdir),
+ r'^\. acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:foo\n$')
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_snapshots(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_snapshots(self, tmpdir):
+ ops = [
+ ('foo', 'bar',
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:foo\n$'),
+ ('foo', 'foo',
+ r'^\. acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:foo\n$'),
+ ('bar', 'bar',
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? acbd18db4cc2f85cedef654fccc4a4d8\+3(\+\S+)? 0:3:bar 3:3:foo\n$'),
+ ('foo', None,
+ r'^\. 37b51d194a7513e45b56f6524f2d51f2\+3(\+\S+)? 0:3:bar\n$'),
+ ]
+ for fn, content, expect in ops:
+ path = os.path.join(tmpdir, fn)
+ if content is None:
+ os.unlink(path)
+ else:
+ with open(path, 'w') as f:
+ f.write(content)
+ self.assertRegexpMatches(current_manifest(tmpdir), expect)
commit 64ea7e9876848463e876a85347212d9390954c11
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 19 17:37:25 2015 -0500
7751: Move code from arv-mount executable to module.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
new file mode 100644
index 0000000..e9d8bb5
--- /dev/null
+++ b/services/fuse/arvados_fuse/command.py
@@ -0,0 +1,331 @@
+import argparse
+import arvados
+import daemon
+import llfuse
+import logging
+import os
+import signal
+import subprocess
+import sys
+import time
+
+import arvados.commands._util as arv_cmd
+from arvados_fuse import crunchstat
+from arvados_fuse import *
+
+class ArgumentParser(argparse.ArgumentParser):
+ def __init__(self):
+ super(ArgumentParser, self).__init__(
+ parents=[arv_cmd.retry_opt],
+ description='''Mount Keep data under the local filesystem. Default mode is --home''',
+ epilog="""
+ Note: When using the --exec feature, you must either specify the
+ mountpoint before --exec, or mark the end of your --exec arguments
+ with "--".
+ """)
+ self.add_argument('mountpoint', type=str, help="""Mount point.""")
+ self.add_argument('--allow-other', action='store_true',
+ help="""Let other users read the mount""")
+
+ mode = self.add_mutually_exclusive_group()
+
+ mode.add_argument('--all', action='store_const', const='all', dest='mode',
+ help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
+ mode.add_argument('--custom', action='store_const', const=None, dest='mode',
+ help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
+ mode.add_argument('--home', action='store_const', const='home', dest='mode',
+ help="""Mount only the user's home project.""")
+ mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
+ help="""Mount only list of projects shared with the user.""")
+ mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
+ help="""Mount subdirectories listed by tag.""")
+ mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
+ help="""Mount subdirectories listed by portable data hash or uuid.""")
+ mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
+ help="""Mount subdirectories listed by portable data hash.""")
+ mode.add_argument('--project', type=str, metavar='UUID',
+ help="""Mount the specified project.""")
+ mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
+ help="""Mount only the specified collection.""")
+
+ mounts = self.add_argument_group('Custom mount options')
+ mounts.add_argument('--mount-by-pdh',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
+ mounts.add_argument('--mount-by-id',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
+ mounts.add_argument('--mount-by-tag',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
+ mounts.add_argument('--mount-home',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount the current user's home project at mountpoint/PATH.")
+ mounts.add_argument('--mount-shared',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount projects shared with the current user at mountpoint/PATH.")
+ mounts.add_argument('--mount-tmp',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
+
+ self.add_argument('--debug', action='store_true', help="""Debug mode""")
+ self.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
+ self.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
+ self.add_argument('--encoding', type=str, help="Character encoding to use for filesystem, default is utf-8 (see Python codec registry for list of available encodings)", default="utf-8")
+
+ self.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
+ self.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
+
+ self.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
+ self.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
+
+ self.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
+
+ self.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
+ dest="exec_args", metavar=('command', 'args', '...', '--'),
+ help="""Mount, run a command, then unmount and exit""")
+
+
+class Mount(object):
+ def __init__(self, args, logger=logging.getLogger('arvados.arv-mount')):
+ self.logger = logger
+ self.args = args
+
+ self.args.mountpoint = os.path.realpath(self.args.mountpoint)
+ if self.args.logfile:
+ self.args.logfile = os.path.realpath(self.args.logfile)
+
+ # Daemonize as early as possible, so we don't accidentally close
+ # file descriptors we're using.
+ self.daemon_ctx = None
+ if not (self.args.exec_args or self.args.foreground):
+ os.chdir(self.args.mountpoint)
+ self.daemon_ctx = daemon.DaemonContext(working_directory='.')
+ self.daemon_ctx.open()
+
+ try:
+ self._setup_logging()
+ self._setup_api()
+ self._setup_mount()
+ except Exception as e:
+ self.logger.exception("arv-mount: exception during setup: %s", e)
+ exit(1)
+
+ def __enter__(self):
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+ if self.args.mode != 'by_pdh':
+ self.operations.listen_for_events()
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+ self.operations.initlock.wait()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
+ self.operations.destroy()
+
+ def Run(self):
+ if self.args.exec_args:
+ self._run_exec(self.args)
+ else:
+ self._run_standalone(self.args)
+
+ def _fuse_options(self):
+ """FUSE mount options; see mount.fuse(8)"""
+ opts = [optname for optname in ['allow_other', 'debug']
+ if getattr(self.args, optname)]
+ # Increase default read/write size from 4KiB to 128KiB
+ opts += ["big_writes", "max_read=131072"]
+ return opts
+
+ def _setup_logging(self):
+ # Configure a log handler based on command-line switches.
+ if self.args.logfile:
+ log_handler = logging.FileHandler(self.args.logfile)
+ elif self.daemon_ctx:
+ log_handler = logging.NullHandler()
+ else:
+ log_handler = None
+
+ if log_handler is not None:
+ arvados.logger.removeHandler(arvados.log_handler)
+ arvados.logger.addHandler(log_handler)
+
+ if self.args.debug:
+ arvados.logger.setLevel(logging.DEBUG)
+ self.logger.debug("arv-mount debugging enabled")
+
+ self.logger.info("enable write is %s", self.args.enable_write)
+
+ def _setup_api(self):
+ self.api = arvados.safeapi.ThreadSafeApiCache(
+ apiconfig=arvados.config.settings(),
+ keep_params={
+ "block_cache": arvados.keep.KeepBlockCache(self.args.file_cache)
+ })
+
+ def _setup_mount(self):
+ self.operations = Operations(
+ os.getuid(),
+ os.getgid(),
+ api_client=self.api,
+ encoding=self.args.encoding,
+ inode_cache=InodeCache(cap=self.args.directory_cache),
+ enable_write=self.args.enable_write)
+
+ if self.args.crunchstat_interval:
+ statsthread = threading.Thread(
+ target=crunchstat.statlogger,
+ args=(self.args.crunchstat_interval,
+ self.api.keep,
+ self.operations))
+ statsthread.daemon = True
+ statsthread.start()
+
+ usr = self.api.users().current().execute(num_retries=self.args.retries)
+ now = time.time()
+ dir_class = None
+ dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
+ mount_readme = False
+
+ if self.args.mode is not None and (
+ self.args.mount_by_id or
+ self.args.mount_by_pdh or
+ self.args.mount_by_tag or
+ self.args.mount_home or
+ self.args.mount_shared or
+ self.args.mount_tmp or
+ self.args.mount_collection):
+ sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
+ format(self.args.mode))
+
+ if self.args.mode in ['by_id', 'by_pdh']:
+ # Set up the request handler with the 'magic directory' at the root
+ dir_class = MagicDirectory
+ dir_args.append(self.args.mode == 'by_pdh')
+ elif self.args.mode == 'by_tag':
+ dir_class = TagsDirectory
+ elif self.args.mode == 'shared':
+ dir_class = SharedDirectory
+ dir_args.append(usr)
+ elif self.args.mode == 'home':
+ dir_class = ProjectDirectory
+ dir_args.append(usr)
+ dir_args.append(True)
+ elif self.args.mode == 'all':
+ self.args.mount_by_id = ['by_id']
+ self.args.mount_by_tag = ['by_tag']
+ self.args.mount_home = ['home']
+ self.args.mount_shared = ['shared']
+ mount_readme = True
+ elif self.args.collection is not None:
+ # Set up the request handler with the collection at the root
+ dir_class = CollectionDirectory
+ dir_args.append(self.args.collection)
+ elif self.args.project is not None:
+ dir_class = ProjectDirectory
+ dir_args.append(
+ self.api.groups().get(uuid=self.args.project).execute(
+ num_retries=self.args.retries))
+
+ if dir_class is not None:
+ self.operations.inodes.add_entry(dir_class(*dir_args))
+ return
+
+ e = self.operations.inodes.add_entry(Directory(
+ llfuse.ROOT_INODE, self.operations.inodes))
+ dir_args[0] = e.inode
+
+ for name in self.args.mount_by_id:
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ for name in self.args.mount_by_pdh:
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=True))
+ for name in self.args.mount_by_tag:
+ self._add_mount(e, name, TagsDirectory(*dir_args))
+ for name in self.args.mount_home:
+ self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ for name in self.args.mount_shared:
+ self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ for name in self.args.mount_tmp:
+ self._add_mount(e, name, TmpCollectionDirectory(*dir_args))
+
+ if mount_readme:
+ text = self._readme_text(
+ arvados.config.get('ARVADOS_API_HOST'),
+ usr['email'])
+ self._add_mount(e, StringFile(e.inode, text, now))
+
+ def _add_mount(self, tld, name, ent):
+ if name in ['', '.', '..'] or '/' in name:
+ sys.exit("Mount point '{}' is not supported.".format(name))
+ tld._entries[name] = self.operations.inodes.add_entry(ent)
+
+ def _readme_text(self, api_host, user_email):
+ return '''
+Welcome to Arvados! This directory provides file system access to
+files and objects available on the Arvados installation located at
+'{}' using credentials for user '{}'.
+
+From here, the following directories are available:
+
+ by_id/ Access to Keep collections by uuid or portable data hash (see by_id/README for details).
+ by_tag/ Access to Keep collections organized by tag.
+ home/ The contents of your home project.
+ shared/ Projects shared with you.
+'''.format(api_host, user_email)
+
+ def _run_exec(self):
+ # Initialize the fuse connection
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+
+ # Subscribe to change events from API server
+ if self.args.mode != 'by_pdh':
+ self.operations.listen_for_events()
+
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ self.operations.initlock.wait()
+
+ rc = 255
+ try:
+ sp = subprocess.Popen(self.args.exec_args, shell=False)
+
+ # forward signals to the process.
+ signal.signal(signal.SIGINT, lambda signum, frame: sp.send_signal(signum))
+ signal.signal(signal.SIGTERM, lambda signum, frame: sp.send_signal(signum))
+ signal.signal(signal.SIGQUIT, lambda signum, frame: sp.send_signal(signum))
+
+ # wait for process to complete.
+ rc = sp.wait()
+
+ # restore default signal handlers.
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
+ signal.signal(signal.SIGQUIT, signal.SIG_DFL)
+ except Exception as e:
+ self.logger.exception(
+ 'arv-mount: exception during exec %s', self.args.exec_args)
+ try:
+ rc = e.errno
+ except AttributeError:
+ pass
+ finally:
+ subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
+ self.operations.destroy()
+ exit(rc)
+
+ def _run_standalone(self):
+ try:
+ llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+
+ # Subscribe to change events from API server
+ self.operations.listen_for_events()
+
+ llfuse.main()
+ except Exception as e:
+ self.logger.exception('arv-mount: exception during mount: %s', e)
+ exit(getattr(e, 'errno', 1))
+ finally:
+ self.operations.destroy()
+ exit(0)
diff --git a/services/fuse/arvados_fuse/crunchstat.py b/services/fuse/arvados_fuse/crunchstat.py
new file mode 100644
index 0000000..67d2ccc
--- /dev/null
+++ b/services/fuse/arvados_fuse/crunchstat.py
@@ -0,0 +1,63 @@
+import sys
+import time
+
+class Stat(object):
+ def __init__(self, prefix, interval,
+ egr_name, ing_name,
+ egr_func, ing_func):
+ self.prefix = prefix
+ self.interval = interval
+ self.egr_name = egr_name
+ self.ing_name = ing_name
+ self.egress = egr_func
+ self.ingress = ing_func
+ self.egr_prev = self.egress()
+ self.ing_prev = self.ingress()
+
+ def update(self):
+ egr = self.egress()
+ ing = self.ingress()
+
+ delta = " -- interval %.4f seconds %d %s %d %s" % (self.interval,
+ egr - self.egr_prev,
+ self.egr_name,
+ ing - self.ing_prev,
+ self.ing_name)
+
+ sys.stderr.write("crunchstat: %s %d %s %d %s%s\n" % (self.prefix,
+ egr,
+ self.egr_name,
+ ing,
+ self.ing_name,
+ delta))
+
+ self.egr_prev = egr
+ self.ing_prev = ing
+
+
+def statlogger(interval, keep, ops):
+ calls = Stat("keepcalls", interval, "put", "get",
+ keep.put_counter.get,
+ keep.get_counter.get)
+ net = Stat("net:keep0", interval, "tx", "rx",
+ keep.upload_counter.get,
+ keep.download_counter.get)
+ cache = Stat("keepcache", interval, "hit", "miss",
+ keep.hits_counter.get,
+ keep.misses_counter.get)
+ fuseops = Stat("fuseops", interval,"write", "read",
+ ops.write_ops_counter.get,
+ ops.read_ops_counter.get)
+ blk = Stat("blkio:0:0", interval, "write", "read",
+ ops.write_counter.get,
+ ops.read_counter.get)
+
+ while True:
+ time.sleep(interval)
+ calls.update()
+ net.update()
+ cache.update()
+ fuseops.update()
+ blk.update()
+
+
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index a679905..05b4c50 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -1,350 +1,7 @@
#!/usr/bin/env python
-import argparse
-import arvados
-import daemon
-import logging
-import os
-import signal
-import subprocess
-import sys
-import time
-
-import arvados.commands._util as arv_cmd
-from arvados_fuse import *
-from arvados.safeapi import ThreadSafeApiCache
-import arvados.keep
-
-logger = logging.getLogger('arvados.arv-mount')
-
-class Stat(object):
- def __init__(self, prefix, interval,
- egr_name, ing_name,
- egr_func, ing_func):
- self.prefix = prefix
- self.interval = interval
- self.egr_name = egr_name
- self.ing_name = ing_name
- self.egress = egr_func
- self.ingress = ing_func
- self.egr_prev = self.egress()
- self.ing_prev = self.ingress()
-
- def update(self):
- egr = self.egress()
- ing = self.ingress()
-
- delta = " -- interval %.4f seconds %d %s %d %s" % (self.interval,
- egr - self.egr_prev,
- self.egr_name,
- ing - self.ing_prev,
- self.ing_name)
-
- sys.stderr.write("crunchstat: %s %d %s %d %s%s\n" % (self.prefix,
- egr,
- self.egr_name,
- ing,
- self.ing_name,
- delta))
-
- self.egr_prev = egr
- self.ing_prev = ing
-
-
-def statlogger(interval, keep, ops):
- calls = Stat("keepcalls", interval, "put", "get",
- keep.put_counter.get,
- keep.get_counter.get)
- net = Stat("net:keep0", interval, "tx", "rx",
- keep.upload_counter.get,
- keep.download_counter.get)
- cache = Stat("keepcache", interval, "hit", "miss",
- keep.hits_counter.get,
- keep.misses_counter.get)
- fuseops = Stat("fuseops", interval,"write", "read",
- ops.write_ops_counter.get,
- ops.read_ops_counter.get)
- blk = Stat("blkio:0:0", interval, "write", "read",
- ops.write_counter.get,
- ops.read_counter.get)
-
- while True:
- time.sleep(interval)
- calls.update()
- net.update()
- cache.update()
- fuseops.update()
- blk.update()
-
+import arvados_fuse
if __name__ == '__main__':
- # Handle command line parameters
- parser = argparse.ArgumentParser(
- parents=[arv_cmd.retry_opt],
- description='''Mount Keep data under the local filesystem. Default mode is --home''',
- epilog="""
-Note: When using the --exec feature, you must either specify the
-mountpoint before --exec, or mark the end of your --exec arguments
-with "--".
-""")
- parser.add_argument('mountpoint', type=str, help="""Mount point.""")
- parser.add_argument('--allow-other', action='store_true',
- help="""Let other users read the mount""")
-
- mount_mode = parser.add_mutually_exclusive_group()
-
- mount_mode.add_argument('--all', action='store_const', const='all', dest='mode',
- help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
- mount_mode.add_argument('--custom', action='store_const', const=None, dest='mode',
- help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
- mount_mode.add_argument('--home', action='store_const', const='home', dest='mode',
- help="""Mount only the user's home project.""")
- mount_mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
- help="""Mount only list of projects shared with the user.""")
- mount_mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
- help="""Mount subdirectories listed by tag.""")
- mount_mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
- help="""Mount subdirectories listed by portable data hash or uuid.""")
- mount_mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
- help="""Mount subdirectories listed by portable data hash.""")
- mount_mode.add_argument('--project', type=str, metavar='UUID',
- help="""Mount the specified project.""")
- mount_mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
- help="""Mount only the specified collection.""")
-
- mounts = parser.add_argument_group('Custom mount options')
- mounts.add_argument('--mount-by-pdh',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
- mounts.add_argument('--mount-by-id',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
- mounts.add_argument('--mount-by-tag',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
- mounts.add_argument('--mount-home',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount the current user's home project at mountpoint/PATH.")
- mounts.add_argument('--mount-shared',
- type=str, metavar='PATH', action='append', default=[],
- help="Mount projects shared with the current user at mountpoint/PATH.")
- mounts.add_argument('--mount-tmp',
- type=str, metavar='PATH', action='append', default=[],
- help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
-
- parser.add_argument('--debug', action='store_true', help="""Debug mode""")
- parser.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
- parser.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
- parser.add_argument('--encoding', type=str, help="Character encoding to use for filesystem, default is utf-8 (see Python codec registry for list of available encodings)", default="utf-8")
-
- parser.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
- parser.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
-
- parser.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
- parser.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
-
- parser.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
-
- parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
- dest="exec_args", metavar=('command', 'args', '...', '--'),
- help="""Mount, run a command, then unmount and exit""")
-
- args = parser.parse_args()
- args.mountpoint = os.path.realpath(args.mountpoint)
- if args.logfile:
- args.logfile = os.path.realpath(args.logfile)
-
- # Daemonize as early as possible, so we don't accidentally close
- # file descriptors we're using.
- if not (args.exec_args or args.foreground):
- os.chdir(args.mountpoint)
- daemon_ctx = daemon.DaemonContext(working_directory='.')
- daemon_ctx.open()
- else:
- daemon_ctx = None
-
- # Configure a log handler based on command-line switches.
- if args.logfile:
- log_handler = logging.FileHandler(args.logfile)
- elif daemon_ctx:
- log_handler = logging.NullHandler()
- else:
- log_handler = None
-
- if log_handler is not None:
- arvados.logger.removeHandler(arvados.log_handler)
- arvados.logger.addHandler(log_handler)
-
- if args.debug:
- arvados.logger.setLevel(logging.DEBUG)
- logger.debug("arv-mount debugging enabled")
-
- logger.info("enable write is %s", args.enable_write)
-
- try:
- api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
- keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
-
- # Create the request handler
- operations = Operations(os.getuid(),
- os.getgid(),
- api_client=api,
- encoding=args.encoding,
- inode_cache=InodeCache(cap=args.directory_cache),
- enable_write=args.enable_write)
-
- if args.crunchstat_interval:
- statsthread = threading.Thread(target=statlogger, args=(args.crunchstat_interval, api.keep, operations))
- statsthread.daemon = True
- statsthread.start()
-
- usr = api.users().current().execute(num_retries=args.retries)
- now = time.time()
- dir_class = None
- dir_args = [llfuse.ROOT_INODE, operations.inodes, api, args.retries]
- mount_readme = False
-
- if args.mode is not None and (
- args.mount_by_id or
- args.mount_by_pdh or
- args.mount_by_tag or
- args.mount_home or
- args.mount_shared or
- args.mount_tmp or
- args.mount_collection):
- sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
- format(args.mode))
-
- if args.mode in ['by_id', 'by_pdh']:
- # Set up the request handler with the 'magic directory' at the root
- dir_class = MagicDirectory
- dir_args.append(args.mode == 'by_pdh')
- elif args.mode == 'by_tag':
- dir_class = TagsDirectory
- elif args.mode == 'shared':
- dir_class = SharedDirectory
- dir_args.append(usr)
- elif args.mode == 'home':
- dir_class = ProjectDirectory
- dir_args.append(usr)
- dir_args.append(True)
- elif args.mode == 'all':
- args.mount_by_id = ['by_id']
- args.mount_by_tag = ['by_tag']
- args.mount_home = ['home']
- args.mount_shared = ['shared']
- mount_readme = True
- elif args.collection is not None:
- # Set up the request handler with the collection at the root
- dir_class = CollectionDirectory
- dir_args.append(args.collection)
- elif args.project is not None:
- dir_class = ProjectDirectory
- dir_args.append(api.groups().get(uuid=args.project).execute(
- num_retries=args.retries))
-
- if dir_class is not None:
- operations.inodes.add_entry(dir_class(*dir_args))
- else:
- e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE, operations.inodes))
- dir_args[0] = e.inode
-
- def addMount(tld, name, ent):
- if name in ['', '.', '..'] or '/' in name:
- sys.exit("Mount point '{}' is not supported.".format(name))
- tld._entries[name] = operations.inodes.add_entry(ent)
-
- for name in args.mount_by_id:
- addMount(e, name, MagicDirectory(*dir_args, pdh_only=False))
- for name in args.mount_by_pdh:
- addMount(e, name, MagicDirectory(*dir_args, pdh_only=True))
- for name in args.mount_by_tag:
- addMount(e, name, TagsDirectory(*dir_args))
- for name in args.mount_home:
- addMount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
- for name in args.mount_shared:
- addMount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
- for name in args.mount_tmp:
- addMount(e, name, TmpCollectionDirectory(*dir_args))
-
- if mount_readme:
- text = '''
-Welcome to Arvados! This directory provides file system access to
-files and objects available on the Arvados installation located at
-'{}' using credentials for user '{}'.
-
-From here, the following directories are available:
-
- by_id/ Access to Keep collections by uuid or portable data hash (see by_id/README for details).
- by_tag/ Access to Keep collections organized by tag.
- home/ The contents of your home project.
- shared/ Projects shared with you.
-'''.format(arvados.config.get('ARVADOS_API_HOST'), usr['email'])
- addMount(e, StringFile(e.inode, text, now))
-
- except Exception:
- logger.exception("arv-mount: exception during API setup")
- exit(1)
-
- # FUSE options, see mount.fuse(8)
- opts = [optname for optname in ['allow_other', 'debug']
- if getattr(args, optname)]
-
- # Increase default read/write size from 4KiB to 128KiB
- opts += ["big_writes", "max_read=131072"]
-
- if args.exec_args:
- # Initialize the fuse connection
- llfuse.init(operations, args.mountpoint, opts)
-
- # Subscribe to change events from API server
- if args.mode != 'by_pdh':
- operations.listen_for_events()
-
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- rc = 255
- try:
- sp = subprocess.Popen(args.exec_args, shell=False)
-
- # forward signals to the process.
- signal.signal(signal.SIGINT, lambda signum, frame: sp.send_signal(signum))
- signal.signal(signal.SIGTERM, lambda signum, frame: sp.send_signal(signum))
- signal.signal(signal.SIGQUIT, lambda signum, frame: sp.send_signal(signum))
-
- # wait for process to complete.
- rc = sp.wait()
-
- # restore default signal handlers.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
- signal.signal(signal.SIGQUIT, signal.SIG_DFL)
- except Exception as e:
- logger.exception('arv-mount: exception during exec %s',
- args.exec_args)
- try:
- rc = e.errno
- except AttributeError:
- pass
- finally:
- subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
- operations.destroy()
-
- exit(rc)
- else:
- try:
- llfuse.init(operations, args.mountpoint, opts)
-
- # Subscribe to change events from API server
- operations.listen_for_events()
-
- llfuse.main()
- except Exception as e:
- logger.exception('arv-mount: exception during mount')
- exit(getattr(e, 'errno', 1))
- finally:
- operations.destroy()
+ args = arvados_fuse.command.ArgumentParser().parse_args()
+ arvados_fuse.command.Mount(args).Run()
commit 2b2b4c47b856fb494583c1e593b039d817d49806
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 18 14:32:18 2015 -0500
7751: Add --mount-tmp option.
diff --git a/services/arv-web/arv-web.py b/services/arv-web/arv-web.py
index 482a577..5a95e27 100755
--- a/services/arv-web/arv-web.py
+++ b/services/arv-web/arv-web.py
@@ -83,7 +83,7 @@ class ArvWeb(object):
def run_fuse_mount(self):
self.mountdir = tempfile.mkdtemp()
- self.operations = Operations(os.getuid(), os.getgid(), "utf-8")
+ self.operations = Operations(os.getuid(), os.getgid(), self.api, "utf-8")
self.cdir = CollectionDirectory(llfuse.ROOT_INODE, self.operations.inodes, self.api, 2, self.collection)
self.operations.inodes.add_entry(self.cdir)
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index fd25aa9..55f1ad7 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -76,7 +76,7 @@ import Queue
llfuse.capi._notify_queue = Queue.Queue()
-from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
+from fusedir import sanitize_filename, Directory, CollectionDirectory, TmpCollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
from fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -304,9 +304,11 @@ class Operations(llfuse.Operations):
"""
- def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4, enable_write=False):
+ def __init__(self, uid, gid, api_client, encoding="utf-8", inode_cache=None, num_retries=4, enable_write=False):
super(Operations, self).__init__()
+ self._api_client = api_client
+
if not inode_cache:
inode_cache = InodeCache(cap=256*1024*1024)
self.inodes = Inodes(inode_cache, encoding=encoding)
@@ -347,8 +349,8 @@ class Operations(llfuse.Operations):
def access(self, inode, mode, ctx):
return True
- def listen_for_events(self, api_client):
- self.events = arvados.events.subscribe(api_client,
+ def listen_for_events(self):
+ self.events = arvados.events.subscribe(self._api_client,
[["event_type", "in", ["create", "update", "delete"]]],
self.on_event)
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index fdc93fb..21961a5 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -474,6 +474,74 @@ class CollectionDirectory(CollectionDirectoryBase):
self.collection.stop_threads()
+class TmpCollectionDirectory(CollectionDirectoryBase):
+ """A directory backed by an Arvados collection that never gets saved.
+
+ This supports using Keep as scratch space. A userspace program can
+ read the .arvados#collection file to get a current manifest in
+ order to save a snapshot of the scratch data or use it as a crunch
+ job output.
+ """
+
+ def __init__(self, parent_inode, inodes, api_client, num_retries):
+ collection = arvados.collection.Collection(
+ api_client=api_client,
+ keep_client=api_client.keep)
+ collection.save = self._commit_collection
+ collection.save_new = self._commit_collection
+ super(TmpCollectionDirectory, self).__init__(
+ parent_inode, inodes, collection)
+ self.collection_record_file = None
+ self._subscribed = False
+ self._update_collection_record()
+
+ def update(self, *args, **kwargs):
+ if not self._subscribed:
+ with llfuse.lock_released:
+ self.populate(self.mtime())
+ self._subscribed = True
+
+ @use_counter
+ def _commit_collection(self):
+ """Commit the data blocks, but don't save the collection to API.
+
+ Update the content of the special .arvados#collection file, if
+ it has been instantiated.
+ """
+ self.collection.flush()
+ self._update_collection_record()
+ if self.collection_record_file is not None:
+ self.collection_record_file.update(self.collection_record)
+ self.inodes.invalidate_inode(self.collection_record_file.inode)
+
+ def _update_collection_record(self):
+ self.collection_record = {
+ "uuid": None,
+ "manifest_text": self.collection.manifest_text(),
+ "portable_data_hash": self.collection.portable_data_hash(),
+ }
+
+ def __contains__(self, k):
+ return (k == '.arvados#collection' or
+ super(TmpCollectionDirectory, self).__contains__(k))
+
+ @use_counter
+ def __getitem__(self, item):
+ if item == '.arvados#collection':
+ if self.collection_record_file is None:
+ self.collection_record_file = ObjectFile(
+ self.inode, self.collection_record)
+ self.inodes.add_entry(self.collection_record_file)
+ return self.collection_record_file
+ return super(TmpCollectionDirectory, self).__getitem__(item)
+
+ def writable(self):
+ return True
+
+ def finalize(self):
+ self.collection.stop_threads()
+
+
class MagicDirectory(Directory):
"""A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 44bc698..a679905 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -7,6 +7,7 @@ import logging
import os
import signal
import subprocess
+import sys
import time
import arvados.commands._util as arv_cmd
@@ -92,17 +93,44 @@ with "--".
mount_mode = parser.add_mutually_exclusive_group()
- mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default).""")
- mount_mode.add_argument('--home', action='store_true', help="""Mount only the user's home project.""")
- mount_mode.add_argument('--shared', action='store_true', help="""Mount only list of projects shared with the user.""")
- mount_mode.add_argument('--by-tag', action='store_true',
+ mount_mode.add_argument('--all', action='store_const', const='all', dest='mode',
+ help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default if no --mount-* arguments are given).""")
+ mount_mode.add_argument('--custom', action='store_const', const=None, dest='mode',
+ help="""Mount a top level meta-directory with subdirectories as specified by additional --mount-* arguments (default if any --mount-* arguments are given).""")
+ mount_mode.add_argument('--home', action='store_const', const='home', dest='mode',
+ help="""Mount only the user's home project.""")
+ mount_mode.add_argument('--shared', action='store_const', const='shared', dest='mode',
+ help="""Mount only list of projects shared with the user.""")
+ mount_mode.add_argument('--by-tag', action='store_const', const='by_tag', dest='mode',
help="""Mount subdirectories listed by tag.""")
- mount_mode.add_argument('--by-id', action='store_true',
+ mount_mode.add_argument('--by-id', action='store_const', const='by_id', dest='mode',
help="""Mount subdirectories listed by portable data hash or uuid.""")
- mount_mode.add_argument('--by-pdh', action='store_true',
+ mount_mode.add_argument('--by-pdh', action='store_const', const='by_pdh', dest='mode',
help="""Mount subdirectories listed by portable data hash.""")
- mount_mode.add_argument('--project', type=str, help="""Mount a specific project.""")
- mount_mode.add_argument('--collection', type=str, help="""Mount only the specified collection.""")
+ mount_mode.add_argument('--project', type=str, metavar='UUID',
+ help="""Mount the specified project.""")
+ mount_mode.add_argument('--collection', type=str, metavar='UUID_or_PDH',
+ help="""Mount only the specified collection.""")
+
+ mounts = parser.add_argument_group('Custom mount options')
+ mounts.add_argument('--mount-by-pdh',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/P where P is the collection's portable data hash.")
+ mounts.add_argument('--mount-by-id',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount each readable collection at mountpoint/PATH/UUID and mountpoint/PATH/PDH where PDH is the collection's portable data hash and UUID is its UUID.")
+ mounts.add_argument('--mount-by-tag',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount all collections with tag TAG at mountpoint/PATH/TAG/UUID.")
+ mounts.add_argument('--mount-home',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount the current user's home project at mountpoint/PATH.")
+ mounts.add_argument('--mount-shared',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Mount projects shared with the current user at mountpoint/PATH.")
+ mounts.add_argument('--mount-tmp',
+ type=str, metavar='PATH', action='append', default=[],
+ help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
parser.add_argument('--debug', action='store_true', help="""Debug mode""")
parser.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
@@ -154,14 +182,16 @@ with "--".
logger.info("enable write is %s", args.enable_write)
try:
+ api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
+ keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
+
# Create the request handler
operations = Operations(os.getuid(),
os.getgid(),
+ api_client=api,
encoding=args.encoding,
inode_cache=InodeCache(cap=args.directory_cache),
enable_write=args.enable_write)
- api = ThreadSafeApiCache(apiconfig=arvados.config.settings(),
- keep_params={"block_cache": arvados.keep.KeepBlockCache(args.file_cache)})
if args.crunchstat_interval:
statsthread = threading.Thread(target=statlogger, args=(args.crunchstat_interval, api.keep, operations))
@@ -172,19 +202,38 @@ with "--".
now = time.time()
dir_class = None
dir_args = [llfuse.ROOT_INODE, operations.inodes, api, args.retries]
- if args.by_id or args.by_pdh:
+ mount_readme = False
+
+ if args.mode is not None and (
+ args.mount_by_id or
+ args.mount_by_pdh or
+ args.mount_by_tag or
+ args.mount_home or
+ args.mount_shared or
+ args.mount_tmp or
+ args.mount_collection):
+ sys.exit("Cannot combine '{}' mode with custom --mount-* options.".
+ format(args.mode))
+
+ if args.mode in ['by_id', 'by_pdh']:
# Set up the request handler with the 'magic directory' at the root
dir_class = MagicDirectory
- dir_args.append(args.by_pdh)
- elif args.by_tag:
+ dir_args.append(args.mode == 'by_pdh')
+ elif args.mode == 'by_tag':
dir_class = TagsDirectory
- elif args.shared:
+ elif args.mode == 'shared':
dir_class = SharedDirectory
dir_args.append(usr)
- elif args.home:
+ elif args.mode == 'home':
dir_class = ProjectDirectory
dir_args.append(usr)
dir_args.append(True)
+ elif args.mode == 'all':
+ args.mount_by_id = ['by_id']
+ args.mount_by_tag = ['by_tag']
+ args.mount_home = ['home']
+ args.mount_shared = ['shared']
+ mount_readme = True
elif args.collection is not None:
# Set up the request handler with the collection at the root
dir_class = CollectionDirectory
@@ -200,19 +249,29 @@ with "--".
e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE, operations.inodes))
dir_args[0] = e.inode
- e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(*dir_args))
-
- e._entries['by_tag'] = operations.inodes.add_entry(TagsDirectory(*dir_args))
-
- dir_args.append(usr)
- dir_args.append(True)
- e._entries['home'] = operations.inodes.add_entry(ProjectDirectory(*dir_args))
- e._entries['shared'] = operations.inodes.add_entry(SharedDirectory(*dir_args))
-
- text = '''
-Welcome to Arvados! This directory provides file system access to files and objects
-available on the Arvados installation located at '{}'
-using credentials for user '{}'.
+ def addMount(tld, name, ent):
+ if name in ['', '.', '..'] or '/' in name:
+ sys.exit("Mount point '{}' is not supported.".format(name))
+ tld._entries[name] = operations.inodes.add_entry(ent)
+
+ for name in args.mount_by_id:
+ addMount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ for name in args.mount_by_pdh:
+ addMount(e, name, MagicDirectory(*dir_args, pdh_only=True))
+ for name in args.mount_by_tag:
+ addMount(e, name, TagsDirectory(*dir_args))
+ for name in args.mount_home:
+ addMount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ for name in args.mount_shared:
+ addMount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ for name in args.mount_tmp:
+ addMount(e, name, TmpCollectionDirectory(*dir_args))
+
+ if mount_readme:
+ text = '''
+Welcome to Arvados! This directory provides file system access to
+files and objects available on the Arvados installation located at
+'{}' using credentials for user '{}'.
From here, the following directories are available:
@@ -221,9 +280,7 @@ From here, the following directories are available:
home/ The contents of your home project.
shared/ Projects shared with you.
'''.format(arvados.config.get('ARVADOS_API_HOST'), usr['email'])
-
- e._entries["README"] = operations.inodes.add_entry(StringFile(e.inode, text, now))
-
+ addMount(e, StringFile(e.inode, text, now))
except Exception:
logger.exception("arv-mount: exception during API setup")
@@ -241,8 +298,8 @@ From here, the following directories are available:
llfuse.init(operations, args.mountpoint, opts)
# Subscribe to change events from API server
- if not args.by_pdh:
- operations.listen_for_events(api)
+ if args.mode != 'by_pdh':
+ operations.listen_for_events()
t = threading.Thread(None, lambda: llfuse.main())
t.start()
@@ -283,7 +340,7 @@ From here, the following directories are available:
llfuse.init(operations, args.mountpoint, opts)
# Subscribe to change events from API server
- operations.listen_for_events(api)
+ operations.listen_for_events()
llfuse.main()
except Exception as e:
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 3b7cbaa..9fb24db 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -35,7 +35,10 @@ class MountTestBase(unittest.TestCase):
self.api = api if api else arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def make_mount(self, root_class, **root_kwargs):
- self.operations = fuse.Operations(os.getuid(), os.getgid(), enable_write=True)
+ self.operations = fuse.Operations(
+ os.getuid(), os.getgid(),
+ api_client=self.api,
+ enable_write=True)
self.operations.inodes.add_entry(root_class(
llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
llfuse.init(self.operations, self.mounttmp, [])
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 1d7b908..05c8685 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -703,7 +703,7 @@ class FuseUpdateFromEventTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- self.operations.listen_for_events(self.api)
+ self.operations.listen_for_events()
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual([], sorted(d1))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list