[ARVADOS] created: c70fc67b1c98dc044abf55bf7dc1d10a486e8775

git at public.curoverse.com git at public.curoverse.com
Tue Nov 25 15:24:42 EST 2014


        at  c70fc67b1c98dc044abf55bf7dc1d10a486e8775 (commit)


commit c70fc67b1c98dc044abf55bf7dc1d10a486e8775
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Nov 25 15:24:08 2014 -0500

    4501: FUSE exposes by_id subdirectory when run with --by-id.
    
    This makes it possible for people to develop code against their own
    normal Keep mount, and then have it work as expected in Crunch (which
    uses --by-id for backward compatibility).
    
    I considered adding a second inode for by_id that pointed to the same
    underlying MagicDirectory instance.  Unfortunately, the current
    implementation would make it difficult to avoid exposing infinitely
    recursing by_id subdirectories, and that seemed like a bigger problem
    than the relatively small overhead of having two MagicDirectory
    instances.  (The Keep client will use comparatively more RAM, and it
    will use the same block cache for both directories, which should keep
    it relatively under control.)

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 9154c82..b2a3b2e 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -5,7 +5,6 @@
 import os
 import sys
 import llfuse
-from llfuse import FUSEError
 import errno
 import stat
 import threading
@@ -20,6 +19,8 @@ import time
 import _strptime
 import calendar
 import threading
+import itertools
+
 from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
 
 _logger = logging.getLogger('arvados.arvados_fuse')
@@ -391,18 +392,8 @@ class MagicDirectory(Directory):
     to readdir().
     '''
 
-    def __init__(self, parent_inode, inodes, api, num_retries):
-        super(MagicDirectory, self).__init__(parent_inode)
-        self.inodes = inodes
-        self.api = api
-        self.num_retries = num_retries
-        # Have to defer creating readme_file because at this point we don't
-        # yet have an inode assigned.
-        self.readme_file = None
-
-    def create_readme(self):
-        if self.readme_file is None:
-            text = '''This directory provides access to Arvados collections as subdirectories listed
+    README_TEXT = '''
+This directory provides access to Arvados collections as subdirectories listed
 by uuid (in the form 'zzzzz-4zz18-1234567890abcde') or portable data hash (in
 the form '1234567890abcdefghijklmnopqrstuv+123').
 
@@ -410,13 +401,27 @@ Note that this directory will appear empty until you attempt to access a
 specific collection subdirectory (such as trying to 'cd' into it), at which
 point the collection will actually be looked up on the server and the directory
 will appear if it exists.
-'''
-            self.readme_file = self.inodes.add_entry(StringFile(self.inode, text, time.time()))
-            self._entries["README"] = self.readme_file
+'''.lstrip()
 
-    def __contains__(self, k):
-        self.create_readme()
+    def __init__(self, parent_inode, inodes, api, num_retries):
+        super(MagicDirectory, self).__init__(parent_inode)
+        self.inodes = inodes
+        self.api = api
+        self.num_retries = num_retries
+
+    def __setattr__(self, name, value):
+        super(MagicDirectory, self).__setattr__(name, value)
+        # When we're assigned an inode, add a README.
+        if ((name == 'inode') and (self.inode is not None) and
+              (not self._entries)):
+            self._entries['README'] = self.inodes.add_entry(
+                StringFile(self.inode, self.README_TEXT, time.time()))
+            # If we're the root directory, add an identical by_id subdirectory.
+            if self.inode == llfuse.ROOT_INODE:
+                self._entries['by_id'] = self.inodes.add_entry(MagicDirectory(
+                        self.inode, self.inodes, self.api, self.num_retries))
 
+    def __contains__(self, k):
         if k in self._entries:
             return True
 
@@ -435,10 +440,6 @@ will appear if it exists.
             _logger.debug('arv-mount exception keep %s', e)
             return False
 
-    def items(self):
-        self.create_readme()
-        return self._entries.items()
-
     def __getitem__(self, item):
         if item in self:
             return self._entries[item]
@@ -692,7 +693,7 @@ class Inodes(object):
 
     def __init__(self):
         self._entries = {}
-        self._counter = llfuse.ROOT_INODE
+        self._counter = itertools.count(llfuse.ROOT_INODE)
 
     def __getitem__(self, item):
         return self._entries[item]
@@ -710,9 +711,8 @@ class Inodes(object):
         return k in self._entries
 
     def add_entry(self, entry):
-        entry.inode = self._counter
+        entry.inode = next(self._counter)
         self._entries[entry.inode] = entry
-        self._counter += 1
         return entry
 
     def del_entry(self, entry):
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index b4eb27f..b36ed0b 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -140,8 +140,6 @@ class FuseMagicTest(MountTestBase):
         operations = fuse.Operations(os.getuid(), os.getgid())
         e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0))
 
-        self.mounttmp = tempfile.mkdtemp()
-
         llfuse.init(operations, self.mounttmp, [])
         t = threading.Thread(None, lambda: llfuse.main())
         t.start()
@@ -150,17 +148,15 @@ class FuseMagicTest(MountTestBase):
         operations.initlock.wait()
 
         # now check some stuff
-        d1 = os.listdir(self.mounttmp)
-        d1.sort()
-        self.assertEqual(['README'], d1)
-
-        d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
-        d2.sort()
-        self.assertEqual(['thing1.txt'], d2)
-
-        d3 = os.listdir(self.mounttmp)
-        d3.sort()
-        self.assertEqual([self.testcollection, 'README'], d3)
+        self.assertIn('README', os.listdir(self.mounttmp))
+        self.assertDirContents(self.testcollection, ['thing1.txt'])
+        self.assertDirContents(os.path.join('by_id', self.testcollection),
+                               ['thing1.txt'])
+        mount_ls = os.listdir(self.mounttmp)
+        self.assertIn('README', mount_ls)
+        self.assertIn(self.testcollection, mount_ls)
+        self.assertIn(self.testcollection,
+                      os.listdir(os.path.join(self.mounttmp, 'by_id')))
 
         files = {}
         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'

commit 2e90667a8dcfad24f071c109097ba11840f5763d
Author: Brett Smith <brett at curoverse.com>
Date:   Tue Nov 25 12:03:32 2014 -0500

    4501: Restore FUSE FuseTagsUpdateTest.
    
    This test hasn't been running in a while because it defined
    runRealTest rather than runTest.  The old code predates Collection
    UUIDs, so it had to be updated to deal with those.

diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 092fed5..b4eb27f 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -36,6 +36,12 @@ class MountTestBase(unittest.TestCase):
         os.rmdir(self.mounttmp)
         shutil.rmtree(self.keeptmp)
 
+    def assertDirContents(self, subdir, expect_content):
+        path = self.mounttmp
+        if subdir:
+            path = os.path.join(path, subdir)
+        self.assertEqual(sorted(expect_content), sorted(os.listdir(path)))
+
 
 class FuseMountTest(MountTestBase):
     def setUp(self):
@@ -80,12 +86,6 @@ class FuseMountTest(MountTestBase):
         self.testcollection = cw.finish()
         self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
 
-    def assertDirContents(self, subdir, expect_content):
-        path = self.mounttmp
-        if subdir:
-            path = os.path.join(path, subdir)
-        self.assertEqual(sorted(expect_content), sorted(os.listdir(path)))
-
     def runTest(self):
         # Create the request handler
         operations = fuse.Operations(os.getuid(), os.getgid())
@@ -196,7 +196,14 @@ class FuseTagsTest(MountTestBase):
 
 
 class FuseTagsUpdateTest(MountTestBase):
-    def runRealTest(self):
+    def tag_collection(self, coll_uuid, tag_name):
+        return self.api.links().create(
+            body={'link': {'head_uuid': coll_uuid,
+                           'link_class': 'tag',
+                           'name': tag_name,
+        }}).execute()
+
+    def runTest(self):
         operations = fuse.Operations(os.getuid(), os.getgid())
         e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, poll_time=1))
 
@@ -206,46 +213,22 @@ class FuseTagsUpdateTest(MountTestBase):
 
         # wait until the driver is finished initializing
         operations.initlock.wait()
+        self.assertIn('foo_tag', os.listdir(self.mounttmp))
 
-        d1 = os.listdir(self.mounttmp)
-        d1.sort()
-        self.assertEqual(['foo_tag'], d1)
-
-        self.api.links().create(body={'link': {
-            'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
-            'link_class': 'tag',
-            'name': 'bar_tag'
-        }}).execute()
-
+        bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
+        self.tag_collection(bar_uuid, 'fuse_test_tag')
         time.sleep(1)
+        self.assertIn('fuse_test_tag', os.listdir(self.mounttmp))
+        self.assertDirContents('fuse_test_tag', [bar_uuid])
 
-        d2 = os.listdir(self.mounttmp)
-        d2.sort()
-        self.assertEqual(['bar_tag', 'foo_tag'], d2)
-
-        d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-        d3.sort()
-        self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
-
-        l = self.api.links().create(body={'link': {
-            'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
-            'link_class': 'tag',
-            'name': 'bar_tag'
-        }}).execute()
-
+        baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
+        l = self.tag_collection(baz_uuid, 'fuse_test_tag')
         time.sleep(1)
-
-        d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-        d4.sort()
-        self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
+        self.assertDirContents('fuse_test_tag', [bar_uuid, baz_uuid])
 
         self.api.links().delete(uuid=l['uuid']).execute()
-
         time.sleep(1)
-
-        d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
-        d5.sort()
-        self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
+        self.assertDirContents('fuse_test_tag', [bar_uuid])
 
 
 class FuseSharedTest(MountTestBase):

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list