[ARVADOS] updated: 349e1ee218d7e888c6c1bcb07f6537f0bdc85012

git at public.curoverse.com git at public.curoverse.com
Thu Jun 11 15:22:43 EDT 2015


Summary of changes:
 sdk/python/arvados/arvfile.py          |  2 +-
 sdk/python/arvados/collection.py       |  5 ++-
 services/fuse/arvados_fuse/__init__.py | 19 ++++++---
 services/fuse/arvados_fuse/fresh.py    |  3 ++
 services/fuse/arvados_fuse/fusedir.py  |  6 +++
 services/fuse/bin/arv-mount            |  3 ++
 services/fuse/tests/test_mount.py      | 76 ++++++++++++++++++----------------
 7 files changed, 71 insertions(+), 43 deletions(-)

       via  349e1ee218d7e888c6c1bcb07f6537f0bdc85012 (commit)
      from  aff4a730ad890564ee05c2395c4ebb49458e3cdc (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 349e1ee218d7e888c6c1bcb07f6537f0bdc85012
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 11 15:22:32 2015 -0400

    3198: Additional fixes based on review feedback.
    
    * Adjust how ~conflict~ files are renamed
    * Add finalize() on fuse inodes() to terminate put and get threads.
    * Improve naming of some test case helpers
    * Relax regex match on authorization token

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 5befa19..ffe3d35 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -553,7 +553,7 @@ class _BlockManager(object):
         :sync:
           If `sync` is True, upload the block synchronously.
           If `sync` is False, upload the block asynchronously.  This will
-          return immediately unless if the upload queue is at capacity, in
+          return immediately unless the upload queue is at capacity, in
           which case it will wait on an upload queue slot.
 
         """
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 6677ca6..5dde8f7 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1007,7 +1007,7 @@ class RichCollectionBase(CollectionBase):
             path = change[1]
             initial = change[2]
             local = self.find(path)
-            conflictpath = "%s~conflict-%s~" % (path, time.strftime("%Y-%m-%d-%H:%M:%S",
+            conflictpath = "%s~%s~conflict~" % (path, time.strftime("%Y%m%d-%H%M%S",
                                                                     time.gmtime()))
             if event_type == ADD:
                 if local is None:
@@ -1307,6 +1307,9 @@ class Collection(RichCollectionBase):
         if exc_type is None:
             if self.writable() and self._has_collection_uuid():
                 self.save()
+        self.stop_threads()
+
+    def stop_threads(self):
         if self._block_manager is not None:
             self._block_manager.stop_threads()
 
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 46c5a1b..abe9821 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -72,10 +72,11 @@ from fusefile import StringFile, FuseArvadosFile
 
 _logger = logging.getLogger('arvados.arvados_fuse')
 
-log_handler = logging.StreamHandler()
-llogger = logging.getLogger('llfuse')
-llogger.addHandler(log_handler)
-llogger.setLevel(logging.DEBUG)
+# Uncomment this to enable llfuse debug logging.
+# log_handler = logging.StreamHandler()
+# llogger = logging.getLogger('llfuse')
+# llogger.addHandler(log_handler)
+# llogger.setLevel(logging.DEBUG)
 
 class Handle(object):
     """Connects a numeric file handle to a File or Directory object that has
@@ -145,7 +146,6 @@ class InodeCache(object):
         return True
 
     def cap_cache(self):
-        #_logger.debug("InodeCache total is %i cap is %i", self._total, self.cap)
         if self._total > self.cap:
             for key in list(self._entries.keys()):
                 if self._total < self.cap or len(self._entries) < self.min_entries:
@@ -220,6 +220,7 @@ class Inodes(object):
             _logger.debug("Deleting inode %i", entry.inode)
             self.inode_cache.unmanage(entry)
             llfuse.invalidate_inode(entry.inode)
+            entry.finalize()
             del self._entries[entry.inode]
             entry.inode = None
         else:
@@ -284,15 +285,21 @@ class Operations(llfuse.Operations):
         # initializing to continue
         self.initlock.set()
 
+    @catch_exceptions
     def destroy(self):
         if self.events:
             self.events.close()
+            self.events = None
+
+        for k,v in self.inodes.items():
+            v.finalize()
+        self.inodes = None
 
     def access(self, inode, mode, ctx):
         return True
 
     def listen_for_events(self, api_client):
-        self.event = arvados.events.subscribe(api_client,
+        self.events = arvados.events.subscribe(api_client,
                                  [["event_type", "in", ["create", "update", "delete"]]],
                                  self.on_event)
 
diff --git a/services/fuse/arvados_fuse/fresh.py b/services/fuse/arvados_fuse/fresh.py
index 6ecf35c..ec2d47a 100644
--- a/services/fuse/arvados_fuse/fresh.py
+++ b/services/fuse/arvados_fuse/fresh.py
@@ -115,3 +115,6 @@ class FreshBase(object):
 
     def uuid(self):
         return None
+
+    def finalize(self):
+        pass
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 85f4bca..26e7fc1 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -465,6 +465,12 @@ class CollectionDirectory(CollectionDirectoryBase):
         # footprint directly would be more accurate, but also more complicated.
         return self._manifest_size * 128
 
+    def finalize(self):
+        if self.collection is not None:
+            if self.writable():
+                self.collection.save()
+            self.collection.stop_threads()
+
 
 class MagicDirectory(Directory):
     """A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index c3f4ab0..6e38728 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -197,6 +197,7 @@ From here, the following directories are available:
                 pass
         finally:
             subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
+            operations.destroy()
 
         exit(rc)
     else:
@@ -210,3 +211,5 @@ From here, the following directories are available:
         except Exception as e:
             logger.exception('arv-mount: exception during mount')
             exit(getattr(e, 'errno', 1))
+        finally:
+            operations.destroy()
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 82d8ec7..b4f3599 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -52,13 +52,16 @@ class MountTestBase(unittest.TestCase):
 
         # llfuse.close is buggy, so use fusermount instead.
         #llfuse.close(unmount=True)
+
         count = 0
         success = 1
         while (count < 9 and success != 0):
           success = subprocess.call(["fusermount", "-u", self.mounttmp])
-          time.sleep(0.5)
+          time.sleep(0.1)
           count += 1
 
+        self.operations.destroy()
+
         os.rmdir(self.mounttmp)
         shutil.rmtree(self.keeptmp)
         run_test_server.reset()
@@ -328,7 +331,7 @@ class FuseHomeTest(MountTestBase):
         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 
 
-def fuseModifyFileTestHelper1(mounttmp):
+def fuseModifyFileTestHelperReadStartContents(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             d1 = llfuse.listdir(mounttmp)
@@ -337,7 +340,7 @@ def fuseModifyFileTestHelper1(mounttmp):
                 self.assertEqual("blub", f.read())
     Test().runTest()
 
-def fuseModifyFileTestHelper2(mounttmp):
+def fuseModifyFileTestHelperReadEndContents(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             d1 = llfuse.listdir(mounttmp)
@@ -358,12 +361,12 @@ class FuseModifyFileTest(MountTestBase):
         with llfuse.lock:
             m.new_collection(collection.api_response(), collection)
 
-        self.pool.apply(fuseModifyFileTestHelper1, (self.mounttmp,))
+        self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
 
         with collection.open("file1.txt", "w") as f:
             f.write("plnp")
 
-        self.pool.apply(fuseModifyFileTestHelper2, (self.mounttmp,))
+        self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
 
 
 class FuseAddFileToCollectionTest(MountTestBase):
@@ -445,17 +448,17 @@ class FuseCreateFileTest(MountTestBase):
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
+            r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
 
 
-def fuseWriteFileTestHelper1(mounttmp):
+def fuseWriteFileTestHelperWriteFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
                 f.write("Hello world!")
     Test().runTest()
 
-def fuseWriteFileTestHelper2(mounttmp):
+def fuseWriteFileTestHelperReadFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
@@ -474,16 +477,16 @@ class FuseWriteFileTest(MountTestBase):
 
         self.assertNotIn("file1.txt", collection)
 
-        self.pool.apply(fuseWriteFileTestHelper1, (self.mounttmp,))
+        self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
 
         with collection.open("file1.txt") as f:
             self.assertEqual(f.read(), "Hello world!")
 
-        self.pool.apply(fuseWriteFileTestHelper2, (self.mounttmp,))
+        self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
 
 def fuseUpdateFileTestHelper(mounttmp):
@@ -521,7 +524,7 @@ class FuseUpdateFileTest(MountTestBase):
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A[a-f0-9]{40}@[a-f0-9]{8} 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:11:file1\.txt 22:1:file1\.txt$')
+            r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
 
 
 def fuseMkdirTestHelper(mounttmp):
@@ -561,10 +564,10 @@ class FuseMkdirTest(MountTestBase):
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
 
-def fuseRmTestHelper1(mounttmp):
+def fuseRmTestHelperWriteFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             os.mkdir(os.path.join(mounttmp, "testdir"))
@@ -574,7 +577,7 @@ def fuseRmTestHelper1(mounttmp):
 
     Test().runTest()
 
-def fuseRmTestHelper2(mounttmp):
+def fuseRmTestHelperDeleteFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             # Can't delete because it's not empty
@@ -597,7 +600,7 @@ def fuseRmTestHelper2(mounttmp):
 
     Test().runTest()
 
-def fuseRmTestHelper3(mounttmp):
+def fuseRmTestHelperRmdir(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             # Should be able to delete now that it is empty
@@ -623,26 +626,26 @@ class FuseRmTest(MountTestBase):
             m.new_collection(collection.api_response(), collection)
         self.assertTrue(m.writable())
 
-        self.pool.apply(fuseRmTestHelper1, (self.mounttmp,))
+        self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
 
         # Starting manifest
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
-        self.pool.apply(fuseRmTestHelper2, (self.mounttmp,))
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
+        self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
 
         # Can't have empty directories :-( so manifest will be empty.
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertEqual(collection2["manifest_text"], "")
 
-        self.pool.apply(fuseRmTestHelper3, (self.mounttmp,))
+        self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
 
         # manifest should be empty now.
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertEqual(collection2["manifest_text"], "")
 
 
-def fuseMvFileTestHelper1(mounttmp):
+def fuseMvFileTestHelperWriteFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             os.mkdir(os.path.join(mounttmp, "testdir"))
@@ -652,7 +655,7 @@ def fuseMvFileTestHelper1(mounttmp):
 
     Test().runTest()
 
-def fuseMvFileTestHelper2(mounttmp):
+def fuseMvFileTestHelperMoveFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
             d1 = llfuse.listdir(os.path.join(mounttmp))
@@ -679,18 +682,18 @@ class FuseMvFileTest(MountTestBase):
             m.new_collection(collection.api_response(), collection)
         self.assertTrue(m.writable())
 
-        self.pool.apply(fuseMvFileTestHelper1, (self.mounttmp,))
+        self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
 
         # Starting manifest
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
-        self.pool.apply(fuseMvFileTestHelper2, (self.mounttmp,))
+        self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
 
 def fuseRenameTestHelper(mounttmp):
@@ -718,7 +721,7 @@ class FuseRenameTest(MountTestBase):
         # Starting manifest
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
         d1 = llfuse.listdir(os.path.join(self.mounttmp))
         self.assertEqual(["testdir"], d1)
@@ -734,13 +737,11 @@ class FuseRenameTest(MountTestBase):
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
         self.assertRegexpMatches(collection2["manifest_text"],
-            r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+            r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
 
 class FuseUpdateFromEventTest(MountTestBase):
     def runTest(self):
-        arvados.logger.setLevel(logging.DEBUG)
-
         collection = arvados.collection.Collection(api_client=self.api)
         collection.save_new()
 
@@ -778,7 +779,7 @@ def fuseFileConflictTestHelper(mounttmp):
                 self.assertEqual(f.read(), "bar")
 
             self.assertRegexpMatches(d1[1],
-                r'file1\.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~')
+                r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
 
             with open(os.path.join(mounttmp, d1[1]), "r") as f:
                 self.assertEqual(f.read(), "foo")
@@ -886,7 +887,7 @@ class FuseMvFileBetweenCollectionsTest(MountTestBase):
         collection1.update()
         collection2.update()
 
-        self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+        self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
         self.assertEqual(collection2.manifest_text(), "")
 
         self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
@@ -897,7 +898,10 @@ class FuseMvFileBetweenCollectionsTest(MountTestBase):
         collection2.update()
 
         self.assertEqual(collection1.manifest_text(), "")
-        self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file2\.txt$")
+        self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
+
+        collection1.stop_threads()
+        collection2.stop_threads()
 
 
 def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
@@ -954,7 +958,7 @@ class FuseMvDirBetweenCollectionsTest(MountTestBase):
         collection1.update()
         collection2.update()
 
-        self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+        self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
         self.assertEqual(collection2.manifest_text(), "")
 
         self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
@@ -965,8 +969,10 @@ class FuseMvDirBetweenCollectionsTest(MountTestBase):
         collection2.update()
 
         self.assertEqual(collection1.manifest_text(), "")
-        self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+        self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
 
+        collection1.stop_threads()
+        collection2.stop_threads()
 
 def fuseProjectMkdirTestHelper1(mounttmp):
     class Test(unittest.TestCase):

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list