[ARVADOS] updated: 349e1ee218d7e888c6c1bcb07f6537f0bdc85012
git at public.curoverse.com
git at public.curoverse.com
Thu Jun 11 15:22:43 EDT 2015
Summary of changes:
sdk/python/arvados/arvfile.py | 2 +-
sdk/python/arvados/collection.py | 5 ++-
services/fuse/arvados_fuse/__init__.py | 19 ++++++---
services/fuse/arvados_fuse/fresh.py | 3 ++
services/fuse/arvados_fuse/fusedir.py | 6 +++
services/fuse/bin/arv-mount | 3 ++
services/fuse/tests/test_mount.py | 76 ++++++++++++++++++----------------
7 files changed, 71 insertions(+), 43 deletions(-)
via 349e1ee218d7e888c6c1bcb07f6537f0bdc85012 (commit)
from aff4a730ad890564ee05c2395c4ebb49458e3cdc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 349e1ee218d7e888c6c1bcb07f6537f0bdc85012
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 11 15:22:32 2015 -0400
3198: Additional fixes based on review feedback.
* Adjust how ~conflict~ files are renamed
* Add finalize() on fuse inodes() to terminate put and get threads.
* Improve naming of some test case helpers
* Relax regex match on authorization token
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 5befa19..ffe3d35 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -553,7 +553,7 @@ class _BlockManager(object):
:sync:
If `sync` is True, upload the block synchronously.
If `sync` is False, upload the block asynchronously. This will
- return immediately unless if the upload queue is at capacity, in
+ return immediately unless the upload queue is at capacity, in
which case it will wait on an upload queue slot.
"""
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 6677ca6..5dde8f7 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1007,7 +1007,7 @@ class RichCollectionBase(CollectionBase):
path = change[1]
initial = change[2]
local = self.find(path)
- conflictpath = "%s~conflict-%s~" % (path, time.strftime("%Y-%m-%d-%H:%M:%S",
+ conflictpath = "%s~%s~conflict~" % (path, time.strftime("%Y%m%d-%H%M%S",
time.gmtime()))
if event_type == ADD:
if local is None:
@@ -1307,6 +1307,9 @@ class Collection(RichCollectionBase):
if exc_type is None:
if self.writable() and self._has_collection_uuid():
self.save()
+ self.stop_threads()
+
+ def stop_threads(self):
if self._block_manager is not None:
self._block_manager.stop_threads()
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 46c5a1b..abe9821 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -72,10 +72,11 @@ from fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
-log_handler = logging.StreamHandler()
-llogger = logging.getLogger('llfuse')
-llogger.addHandler(log_handler)
-llogger.setLevel(logging.DEBUG)
+# Uncomment this to enable llfuse debug logging.
+# log_handler = logging.StreamHandler()
+# llogger = logging.getLogger('llfuse')
+# llogger.addHandler(log_handler)
+# llogger.setLevel(logging.DEBUG)
class Handle(object):
"""Connects a numeric file handle to a File or Directory object that has
@@ -145,7 +146,6 @@ class InodeCache(object):
return True
def cap_cache(self):
- #_logger.debug("InodeCache total is %i cap is %i", self._total, self.cap)
if self._total > self.cap:
for key in list(self._entries.keys()):
if self._total < self.cap or len(self._entries) < self.min_entries:
@@ -220,6 +220,7 @@ class Inodes(object):
_logger.debug("Deleting inode %i", entry.inode)
self.inode_cache.unmanage(entry)
llfuse.invalidate_inode(entry.inode)
+ entry.finalize()
del self._entries[entry.inode]
entry.inode = None
else:
@@ -284,15 +285,21 @@ class Operations(llfuse.Operations):
# initializing to continue
self.initlock.set()
+ @catch_exceptions
def destroy(self):
if self.events:
self.events.close()
+ self.events = None
+
+ for k,v in self.inodes.items():
+ v.finalize()
+ self.inodes = None
def access(self, inode, mode, ctx):
return True
def listen_for_events(self, api_client):
- self.event = arvados.events.subscribe(api_client,
+ self.events = arvados.events.subscribe(api_client,
[["event_type", "in", ["create", "update", "delete"]]],
self.on_event)
diff --git a/services/fuse/arvados_fuse/fresh.py b/services/fuse/arvados_fuse/fresh.py
index 6ecf35c..ec2d47a 100644
--- a/services/fuse/arvados_fuse/fresh.py
+++ b/services/fuse/arvados_fuse/fresh.py
@@ -115,3 +115,6 @@ class FreshBase(object):
def uuid(self):
return None
+
+ def finalize(self):
+ pass
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 85f4bca..26e7fc1 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -465,6 +465,12 @@ class CollectionDirectory(CollectionDirectoryBase):
# footprint directly would be more accurate, but also more complicated.
return self._manifest_size * 128
+ def finalize(self):
+ if self.collection is not None:
+ if self.writable():
+ self.collection.save()
+ self.collection.stop_threads()
+
class MagicDirectory(Directory):
"""A special directory that logically contains the set of all extant keep locators.
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index c3f4ab0..6e38728 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -197,6 +197,7 @@ From here, the following directories are available:
pass
finally:
subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
+ operations.destroy()
exit(rc)
else:
@@ -210,3 +211,5 @@ From here, the following directories are available:
except Exception as e:
logger.exception('arv-mount: exception during mount')
exit(getattr(e, 'errno', 1))
+ finally:
+ operations.destroy()
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 82d8ec7..b4f3599 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -52,13 +52,16 @@ class MountTestBase(unittest.TestCase):
# llfuse.close is buggy, so use fusermount instead.
#llfuse.close(unmount=True)
+
count = 0
success = 1
while (count < 9 and success != 0):
success = subprocess.call(["fusermount", "-u", self.mounttmp])
- time.sleep(0.5)
+ time.sleep(0.1)
count += 1
+ self.operations.destroy()
+
os.rmdir(self.mounttmp)
shutil.rmtree(self.keeptmp)
run_test_server.reset()
@@ -328,7 +331,7 @@ class FuseHomeTest(MountTestBase):
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
-def fuseModifyFileTestHelper1(mounttmp):
+def fuseModifyFileTestHelperReadStartContents(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
d1 = llfuse.listdir(mounttmp)
@@ -337,7 +340,7 @@ def fuseModifyFileTestHelper1(mounttmp):
self.assertEqual("blub", f.read())
Test().runTest()
-def fuseModifyFileTestHelper2(mounttmp):
+def fuseModifyFileTestHelperReadEndContents(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
d1 = llfuse.listdir(mounttmp)
@@ -358,12 +361,12 @@ class FuseModifyFileTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- self.pool.apply(fuseModifyFileTestHelper1, (self.mounttmp,))
+ self.pool.apply(fuseModifyFileTestHelperReadStartContents, (self.mounttmp,))
with collection.open("file1.txt", "w") as f:
f.write("plnp")
- self.pool.apply(fuseModifyFileTestHelper2, (self.mounttmp,))
+ self.pool.apply(fuseModifyFileTestHelperReadEndContents, (self.mounttmp,))
class FuseAddFileToCollectionTest(MountTestBase):
@@ -445,17 +448,17 @@ class FuseCreateFileTest(MountTestBase):
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
+ r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:file1\.txt$')
-def fuseWriteFileTestHelper1(mounttmp):
+def fuseWriteFileTestHelperWriteFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
f.write("Hello world!")
Test().runTest()
-def fuseWriteFileTestHelper2(mounttmp):
+def fuseWriteFileTestHelperReadFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
@@ -474,16 +477,16 @@ class FuseWriteFileTest(MountTestBase):
self.assertNotIn("file1.txt", collection)
- self.pool.apply(fuseWriteFileTestHelper1, (self.mounttmp,))
+ self.pool.apply(fuseWriteFileTestHelperWriteFile, (self.mounttmp,))
with collection.open("file1.txt") as f:
self.assertEqual(f.read(), "Hello world!")
- self.pool.apply(fuseWriteFileTestHelper2, (self.mounttmp,))
+ self.pool.apply(fuseWriteFileTestHelperReadFile, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
def fuseUpdateFileTestHelper(mounttmp):
@@ -521,7 +524,7 @@ class FuseUpdateFileTest(MountTestBase):
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A[a-f0-9]{40}@[a-f0-9]{8} 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:11:file1\.txt 22:1:file1\.txt$')
+ r'\. daaef200ebb921e011e3ae922dd3266b\+11\+A\S+ 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:11:file1\.txt 22:1:file1\.txt$')
def fuseMkdirTestHelper(mounttmp):
@@ -561,10 +564,10 @@ class FuseMkdirTest(MountTestBase):
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
-def fuseRmTestHelper1(mounttmp):
+def fuseRmTestHelperWriteFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
os.mkdir(os.path.join(mounttmp, "testdir"))
@@ -574,7 +577,7 @@ def fuseRmTestHelper1(mounttmp):
Test().runTest()
-def fuseRmTestHelper2(mounttmp):
+def fuseRmTestHelperDeleteFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
# Can't delete because it's not empty
@@ -597,7 +600,7 @@ def fuseRmTestHelper2(mounttmp):
Test().runTest()
-def fuseRmTestHelper3(mounttmp):
+def fuseRmTestHelperRmdir(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
# Should be able to delete now that it is empty
@@ -623,26 +626,26 @@ class FuseRmTest(MountTestBase):
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
- self.pool.apply(fuseRmTestHelper1, (self.mounttmp,))
+ self.pool.apply(fuseRmTestHelperWriteFile, (self.mounttmp,))
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
- self.pool.apply(fuseRmTestHelper2, (self.mounttmp,))
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
+ self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
# Can't have empty directories :-( so manifest will be empty.
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertEqual(collection2["manifest_text"], "")
- self.pool.apply(fuseRmTestHelper3, (self.mounttmp,))
+ self.pool.apply(fuseRmTestHelperRmdir, (self.mounttmp,))
# manifest should be empty now.
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertEqual(collection2["manifest_text"], "")
-def fuseMvFileTestHelper1(mounttmp):
+def fuseMvFileTestHelperWriteFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
os.mkdir(os.path.join(mounttmp, "testdir"))
@@ -652,7 +655,7 @@ def fuseMvFileTestHelper1(mounttmp):
Test().runTest()
-def fuseMvFileTestHelper2(mounttmp):
+def fuseMvFileTestHelperMoveFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
d1 = llfuse.listdir(os.path.join(mounttmp))
@@ -679,18 +682,18 @@ class FuseMvFileTest(MountTestBase):
m.new_collection(collection.api_response(), collection)
self.assertTrue(m.writable())
- self.pool.apply(fuseMvFileTestHelper1, (self.mounttmp,))
+ self.pool.apply(fuseMvFileTestHelperWriteFile, (self.mounttmp,))
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
- self.pool.apply(fuseMvFileTestHelper2, (self.mounttmp,))
+ self.pool.apply(fuseMvFileTestHelperMoveFile, (self.mounttmp,))
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
def fuseRenameTestHelper(mounttmp):
@@ -718,7 +721,7 @@ class FuseRenameTest(MountTestBase):
# Starting manifest
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
d1 = llfuse.listdir(os.path.join(self.mounttmp))
self.assertEqual(["testdir"], d1)
@@ -734,13 +737,11 @@ class FuseRenameTest(MountTestBase):
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
- r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+ r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
class FuseUpdateFromEventTest(MountTestBase):
def runTest(self):
- arvados.logger.setLevel(logging.DEBUG)
-
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
@@ -778,7 +779,7 @@ def fuseFileConflictTestHelper(mounttmp):
self.assertEqual(f.read(), "bar")
self.assertRegexpMatches(d1[1],
- r'file1\.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~')
+ r'file1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~')
with open(os.path.join(mounttmp, d1[1]), "r") as f:
self.assertEqual(f.read(), "foo")
@@ -886,7 +887,7 @@ class FuseMvFileBetweenCollectionsTest(MountTestBase):
collection1.update()
collection2.update()
- self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+ self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
self.assertEqual(collection2.manifest_text(), "")
self.pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
@@ -897,7 +898,10 @@ class FuseMvFileBetweenCollectionsTest(MountTestBase):
collection2.update()
self.assertEqual(collection1.manifest_text(), "")
- self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file2\.txt$")
+ self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file2\.txt$")
+
+ collection1.stop_threads()
+ collection2.stop_threads()
def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
@@ -954,7 +958,7 @@ class FuseMvDirBetweenCollectionsTest(MountTestBase):
collection1.update()
collection2.update()
- self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+ self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
self.assertEqual(collection2.manifest_text(), "")
self.pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
@@ -965,8 +969,10 @@ class FuseMvDirBetweenCollectionsTest(MountTestBase):
collection2.update()
self.assertEqual(collection1.manifest_text(), "")
- self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+ self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$")
+ collection1.stop_threads()
+ collection2.stop_threads()
def fuseProjectMkdirTestHelper1(mounttmp):
class Test(unittest.TestCase):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list