[ARVADOS] updated: 4b0934a2bb95509468fc4b6132177d47cc7988a8
git at public.curoverse.com
git at public.curoverse.com
Mon Dec 1 11:56:00 EST 2014
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 50 +++++++++----------
services/fuse/tests/test_mount.py | 90 ++++++++++++++--------------------
2 files changed, 62 insertions(+), 78 deletions(-)
via 4b0934a2bb95509468fc4b6132177d47cc7988a8 (commit)
via 98fd0639a6703084ad9877d9713b8fa4a8dfb03d (commit)
via f20ae9938c84f48954246a732be7c2f02ff848af (commit)
from 2bca6945ae26ff552979838ea4db379e6ab3dca6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4b0934a2bb95509468fc4b6132177d47cc7988a8
Merge: 2bca694 98fd063
Author: Brett Smith <brett at curoverse.com>
Date: Mon Dec 1 11:55:31 2014 -0500
Merge branch '4501-fuse-by-id-subdirectory-wip'
Closes #4501, #4675.
commit 98fd0639a6703084ad9877d9713b8fa4a8dfb03d
Author: Brett Smith <brett at curoverse.com>
Date: Tue Nov 25 15:24:08 2014 -0500
4501: FUSE exposes by_id subdirectory when run with --by-id.
This makes it possible for people to develop code against their own
normal Keep mount, and then have it work as expected in Crunch (which
uses --by-id for backward compatibility).
I considered adding a second inode for by_id that pointed to the same
underlying MagicDirectory instance. Unfortunately, the current
implementation would make it difficult to avoid exposing infinitely
recursing by_id subdirectories, and that seemed like a bigger problem
than the relatively small overhead of having two MagicDirectory
instances. (The Keep client will use comparatively more RAM, and it
will use the same block cache for both directories, which should keep
it relatively under control.)
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 9154c82..b2a3b2e 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -5,7 +5,6 @@
import os
import sys
import llfuse
-from llfuse import FUSEError
import errno
import stat
import threading
@@ -20,6 +19,8 @@ import time
import _strptime
import calendar
import threading
+import itertools
+
from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -391,18 +392,8 @@ class MagicDirectory(Directory):
to readdir().
'''
- def __init__(self, parent_inode, inodes, api, num_retries):
- super(MagicDirectory, self).__init__(parent_inode)
- self.inodes = inodes
- self.api = api
- self.num_retries = num_retries
- # Have to defer creating readme_file because at this point we don't
- # yet have an inode assigned.
- self.readme_file = None
-
- def create_readme(self):
- if self.readme_file is None:
- text = '''This directory provides access to Arvados collections as subdirectories listed
+ README_TEXT = '''
+This directory provides access to Arvados collections as subdirectories listed
by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
the form '1234567890abcdefghijklmnopqrstuv+123').
@@ -410,13 +401,27 @@ Note that this directory will appear empty until you attempt to access a
specific collection subdirectory (such as trying to 'cd' into it), at which
point the collection will actually be looked up on the server and the directory
will appear if it exists.
-'''
- self.readme_file = self.inodes.add_entry(StringFile(self.inode, text, time.time()))
- self._entries["README"] = self.readme_file
+'''.lstrip()
- def __contains__(self, k):
- self.create_readme()
+ def __init__(self, parent_inode, inodes, api, num_retries):
+ super(MagicDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ self.num_retries = num_retries
+
+ def __setattr__(self, name, value):
+ super(MagicDirectory, self).__setattr__(name, value)
+ # When we're assigned an inode, add a README.
+ if ((name == 'inode') and (self.inode is not None) and
+ (not self._entries)):
+ self._entries['README'] = self.inodes.add_entry(
+ StringFile(self.inode, self.README_TEXT, time.time()))
+ # If we're the root directory, add an identical by_id subdirectory.
+ if self.inode == llfuse.ROOT_INODE:
+ self._entries['by_id'] = self.inodes.add_entry(MagicDirectory(
+ self.inode, self.inodes, self.api, self.num_retries))
+ def __contains__(self, k):
if k in self._entries:
return True
@@ -435,10 +440,6 @@ will appear if it exists.
_logger.debug('arv-mount exception keep %s', e)
return False
- def items(self):
- self.create_readme()
- return self._entries.items()
-
def __getitem__(self, item):
if item in self:
return self._entries[item]
@@ -692,7 +693,7 @@ class Inodes(object):
def __init__(self):
self._entries = {}
- self._counter = llfuse.ROOT_INODE
+ self._counter = itertools.count(llfuse.ROOT_INODE)
def __getitem__(self, item):
return self._entries[item]
@@ -710,9 +711,8 @@ class Inodes(object):
return k in self._entries
def add_entry(self, entry):
- entry.inode = self._counter
+ entry.inode = next(self._counter)
self._entries[entry.inode] = entry
- self._counter += 1
return entry
def del_entry(self, entry):
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index b4eb27f..f0f375c 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -140,8 +140,6 @@ class FuseMagicTest(MountTestBase):
operations = fuse.Operations(os.getuid(), os.getgid())
e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
- self.mounttmp = tempfile.mkdtemp()
-
llfuse.init(operations, self.mounttmp, [])
t = threading.Thread(None, lambda: llfuse.main())
t.start()
@@ -150,17 +148,20 @@ class FuseMagicTest(MountTestBase):
operations.initlock.wait()
# now check some stuff
- d1 = os.listdir(self.mounttmp)
- d1.sort()
- self.assertEqual(['README'], d1)
-
- d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
- d2.sort()
- self.assertEqual(['thing1.txt'], d2)
-
- d3 = os.listdir(self.mounttmp)
- d3.sort()
- self.assertEqual([self.testcollection, 'README'], d3)
+ mount_ls = os.listdir(self.mounttmp)
+ self.assertIn('README', mount_ls)
+ self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
+ arvados.util.uuid_pattern.match(fn)
+ for fn in mount_ls),
+ "new FUSE MagicDirectory lists Collection")
+ self.assertDirContents(self.testcollection, ['thing1.txt'])
+ self.assertDirContents(os.path.join('by_id', self.testcollection),
+ ['thing1.txt'])
+ mount_ls = os.listdir(self.mounttmp)
+ self.assertIn('README', mount_ls)
+ self.assertIn(self.testcollection, mount_ls)
+ self.assertIn(self.testcollection,
+ os.listdir(os.path.join(self.mounttmp, 'by_id')))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
commit f20ae9938c84f48954246a732be7c2f02ff848af
Author: Brett Smith <brett at curoverse.com>
Date: Tue Nov 25 12:03:32 2014 -0500
4501: Restore FUSE FuseTagsUpdateTest.
This test hasn't been running in a while because it defined
runRealTest rather than runTest. The old code predates Collection
UUIDs, so it had to be updated to deal with those.
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 092fed5..b4eb27f 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -36,6 +36,12 @@ class MountTestBase(unittest.TestCase):
os.rmdir(self.mounttmp)
shutil.rmtree(self.keeptmp)
+ def assertDirContents(self, subdir, expect_content):
+ path = self.mounttmp
+ if subdir:
+ path = os.path.join(path, subdir)
+ self.assertEqual(sorted(expect_content), sorted(os.listdir(path)))
+
class FuseMountTest(MountTestBase):
def setUp(self):
@@ -80,12 +86,6 @@ class FuseMountTest(MountTestBase):
self.testcollection = cw.finish()
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
- def assertDirContents(self, subdir, expect_content):
- path = self.mounttmp
- if subdir:
- path = os.path.join(path, subdir)
- self.assertEqual(sorted(expect_content), sorted(os.listdir(path)))
-
def runTest(self):
# Create the request handler
operations = fuse.Operations(os.getuid(), os.getgid())
@@ -196,7 +196,14 @@ class FuseTagsTest(MountTestBase):
class FuseTagsUpdateTest(MountTestBase):
- def runRealTest(self):
+ def tag_collection(self, coll_uuid, tag_name):
+ return self.api.links().create(
+ body={'link': {'head_uuid': coll_uuid,
+ 'link_class': 'tag',
+ 'name': tag_name,
+ }}).execute()
+
+ def runTest(self):
operations = fuse.Operations(os.getuid(), os.getgid())
e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, poll_time=1))
@@ -206,46 +213,22 @@ class FuseTagsUpdateTest(MountTestBase):
# wait until the driver is finished initializing
operations.initlock.wait()
+ self.assertIn('foo_tag', os.listdir(self.mounttmp))
- d1 = os.listdir(self.mounttmp)
- d1.sort()
- self.assertEqual(['foo_tag'], d1)
-
- self.api.links().create(body={'link': {
- 'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
- 'link_class': 'tag',
- 'name': 'bar_tag'
- }}).execute()
-
+ bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
+ self.tag_collection(bar_uuid, 'fuse_test_tag')
time.sleep(1)
+ self.assertIn('fuse_test_tag', os.listdir(self.mounttmp))
+ self.assertDirContents('fuse_test_tag', [bar_uuid])
- d2 = os.listdir(self.mounttmp)
- d2.sort()
- self.assertEqual(['bar_tag', 'foo_tag'], d2)
-
- d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
- d3.sort()
- self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
-
- l = self.api.links().create(body={'link': {
- 'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
- 'link_class': 'tag',
- 'name': 'bar_tag'
- }}).execute()
-
+ baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
+ l = self.tag_collection(baz_uuid, 'fuse_test_tag')
time.sleep(1)
-
- d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
- d4.sort()
- self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
+ self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
self.api.links().delete(uuid=l['uuid']).execute()
-
time.sleep(1)
-
- d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
- d5.sort()
- self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
+ self.assertDirContents('fuse_test_tag', [bar_uuid])
class FuseSharedTest(MountTestBase):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list