[ARVADOS] created: 5ac2b460fdfc000398415dfa50a5ac3bfedce0d7
Git user
git at public.curoverse.com
Wed Apr 20 13:53:46 EDT 2016
at 5ac2b460fdfc000398415dfa50a5ac3bfedce0d7 (commit)
commit 5ac2b460fdfc000398415dfa50a5ac3bfedce0d7
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 18 09:27:01 2016 -0400
8345: Use multiprocessing pool to get dir listings.
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 55d3092..e573ddf 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -47,6 +47,10 @@ class MountTestBase(unittest.TestCase):
raise
llfuse.close()
+ def listmountdir(self, *subdirs):
+ return self.pool.apply(os.listdir,
+ (os.path.join(self.mounttmp, *subdirs), ))
+
def make_mount(self, root_class, **root_kwargs):
self.operations = fuse.Operations(
os.getuid(), os.getgid(),
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index e534e32..73da16d 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -127,7 +127,7 @@ class FuseMagicTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.MagicDirectory)
- mount_ls = llfuse.listdir(self.mounttmp)
+ mount_ls = self.listmountdir()
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
arvados.util.uuid_pattern.match(fn)
@@ -136,11 +136,10 @@ class FuseMagicTest(MountTestBase):
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
- mount_ls = llfuse.listdir(self.mounttmp)
+ mount_ls = self.listmountdir()
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
- self.assertIn(self.testcollection,
- llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
+ self.assertIn(self.testcollection, self.listmountdir('by_id'))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
@@ -154,15 +153,15 @@ class FuseTagsTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.TagsDirectory)
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
d1.sort()
self.assertEqual(['foo_tag'], d1)
- d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
+ d2 = self.listmountdir('foo_tag')
d2.sort()
self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
- d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
+ d3 = self.listmountdir('foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7')
d3.sort()
self.assertEqual(['foo'], d3)
@@ -178,12 +177,12 @@ class FuseTagsUpdateTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.TagsDirectory, poll_time=1)
- self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
+ self.assertIn('foo_tag', self.listmountdir())
bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
self.tag_collection(bar_uuid, 'fuse_test_tag')
time.sleep(1)
- self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
+ self.assertIn('fuse_test_tag', self.listmountdir())
self.assertDirContents('fuse_test_tag', [bar_uuid])
baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
@@ -204,14 +203,14 @@ class FuseSharedTest(MountTestBase):
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
# to the current user)
- shared_dirs = llfuse.listdir(self.mounttmp)
+ shared_dirs = self.listmountdir()
shared_dirs.sort()
self.assertIn('FUSE User', shared_dirs)
# fuse_user_objs is a list of the objects owned by the FUSE
# test user (which present as files in the 'FUSE User'
# directory)
- fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
+ fuse_user_objs = self.listmountdir('FUSE User')
fuse_user_objs.sort()
self.assertEqual(['FUSE Test Project', # project owned by user
'collection #1 owned by FUSE', # collection owned by user
@@ -220,7 +219,7 @@ class FuseSharedTest(MountTestBase):
], fuse_user_objs)
# test_proj_files is a list of the files in the FUSE Test Project.
- test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
+ test_proj_files = self.listmountdir('FUSE User', 'FUSE Test Project')
test_proj_files.sort()
self.assertEqual(['collection in FUSE project',
'pipeline instance in FUSE project.pipelineInstance',
@@ -255,10 +254,10 @@ class FuseHomeTest(MountTestBase):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertIn('Unrestricted public data', d1)
- d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
+ d2 = self.listmountdir('Unrestricted public data')
public_project = run_test_server.fixture('groups')[
'anonymously_accessible_project']
found_in = 0
@@ -278,14 +277,14 @@ class FuseHomeTest(MountTestBase):
self.assertNotEqual(0, found_in)
self.assertNotEqual(0, found_not_in)
- d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
+ d3 = self.listmountdir('Unrestricted public data', 'GNU General Public License, version 3')
self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
def fuseModifyFileTestHelperReadStartContents(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertEqual(["file1.txt"], d1)
with open(os.path.join(mounttmp, "file1.txt")) as f:
self.assertEqual("blub", f.read())
@@ -294,7 +293,7 @@ def fuseModifyFileTestHelperReadStartContents(mounttmp):
def fuseModifyFileTestHelperReadEndContents(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertEqual(["file1.txt"], d1)
with open(os.path.join(mounttmp, "file1.txt")) as f:
self.assertEqual("plnp", f.read())
@@ -332,13 +331,13 @@ class FuseAddFileToCollectionTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt"], d1)
with collection.open("file2.txt", "w") as f:
f.write("plnp")
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
@@ -357,12 +356,12 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
collection.remove("file2.txt")
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt"], d1)
@@ -394,7 +393,7 @@ class FuseCreateFileTest(MountTestBase):
self.assertIn("file1.txt", collection)
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -494,13 +493,13 @@ def fuseMkdirTestHelper(mounttmp):
with self.assertRaises(OSError):
os.mkdir(os.path.join(mounttmp, "testdir"))
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertEqual(["testdir"], d1)
with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
f.write("Hello world!")
- d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+ d1 = os.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
Test().runTest()
@@ -539,14 +538,14 @@ def fuseRmTestHelperDeleteFile(mounttmp):
with self.assertRaises(OSError):
os.rmdir(os.path.join(mounttmp, "testdir"))
- d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+ d1 = os.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
# Delete file
os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
# Make sure it's empty
- d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+ d1 = os.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual([], d1)
# Try to delete it again
@@ -562,7 +561,7 @@ def fuseRmTestHelperRmdir(mounttmp):
os.rmdir(os.path.join(mounttmp, "testdir"))
# Make sure it's empty
- d1 = llfuse.listdir(os.path.join(mounttmp))
+ d1 = os.listdir(os.path.join(mounttmp))
self.assertEqual([], d1)
# Try to delete it again
@@ -613,16 +612,16 @@ def fuseMvFileTestHelperWriteFile(mounttmp):
def fuseMvFileTestHelperMoveFile(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
- d1 = llfuse.listdir(os.path.join(mounttmp))
+ d1 = os.listdir(os.path.join(mounttmp))
self.assertEqual(["testdir"], d1)
- d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+ d1 = os.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual(["file1.txt"], d1)
os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
- d1 = llfuse.listdir(os.path.join(mounttmp))
+ d1 = os.listdir(os.path.join(mounttmp))
self.assertEqual(["file1.txt", "testdir"], sorted(d1))
- d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+ d1 = os.listdir(os.path.join(mounttmp, "testdir"))
self.assertEqual([], d1)
Test().runTest()
@@ -678,16 +677,16 @@ class FuseRenameTest(MountTestBase):
self.assertRegexpMatches(collection2["manifest_text"],
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
- d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ d1 = self.listmountdir()
self.assertEqual(["testdir"], d1)
- d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
+ d1 = self.listmountdir("testdir")
self.assertEqual(["file1.txt"], d1)
os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
- d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ d1 = self.listmountdir()
self.assertEqual(["testdir2"], sorted(d1))
- d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
+ d1 = self.listmountdir("testdir2")
self.assertEqual(["file1.txt"], d1)
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -706,7 +705,7 @@ class FuseUpdateFromEventTest(MountTestBase):
self.operations.listen_for_events()
- d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ d1 = self.listmountdir()
self.assertEqual([], sorted(d1))
with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
@@ -717,7 +716,7 @@ class FuseUpdateFromEventTest(MountTestBase):
# should show up via event bus notify
- d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ d1 = self.listmountdir()
self.assertEqual(["file1.txt"], sorted(d1))
@@ -727,7 +726,7 @@ def fuseFileConflictTestHelper(mounttmp):
with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
f.write("bar")
- d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
+ d1 = sorted(os.listdir(os.path.join(mounttmp)))
self.assertEqual(len(d1), 2)
with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
@@ -750,7 +749,7 @@ class FuseFileConflictTest(MountTestBase):
with llfuse.lock:
m.new_collection(collection.api_response(), collection)
- d1 = llfuse.listdir(os.path.join(self.mounttmp))
+ d1 = self.listmountdir()
self.assertEqual([], sorted(d1))
with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
@@ -767,12 +766,12 @@ def fuseUnlinkOpenFileTest(mounttmp):
with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
f.write("foo")
- d1 = llfuse.listdir(os.path.join(mounttmp))
+ d1 = os.listdir(os.path.join(mounttmp))
self.assertEqual(["file1.txt"], sorted(d1))
os.remove(os.path.join(mounttmp, "file1.txt"))
- d1 = llfuse.listdir(os.path.join(mounttmp))
+ d1 = os.listdir(os.path.join(mounttmp))
self.assertEqual([], sorted(d1))
f.seek(0)
@@ -955,29 +954,29 @@ class FuseProjectMkdirRmdirTest(MountTestBase):
self.make_mount(fuse.ProjectDirectory,
project_object=self.api.users().current().execute())
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertNotIn('testcollection', d1)
self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertIn('testcollection', d1)
self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
- d1 = llfuse.listdir(self.mounttmp)
+ d1 = self.listmountdir()
self.assertNotIn('testcollection', d1)
def fuseProjectMvTestHelper1(mounttmp):
class Test(unittest.TestCase):
def runTest(self):
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
os.mkdir(os.path.join(mounttmp, "testcollection"))
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertIn('testcollection', d1)
with self.assertRaises(OSError):
@@ -985,10 +984,10 @@ def fuseProjectMvTestHelper1(mounttmp):
os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
- d1 = llfuse.listdir(mounttmp)
+ d1 = os.listdir(mounttmp)
self.assertNotIn('testcollection', d1)
- d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
+ d1 = os.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
self.assertIn('testcollection', d1)
Test().runTest()
@@ -1030,9 +1029,9 @@ class MagicDirApiError(FuseMagicTest):
self.operations.inodes.inode_cache.min_entries = 2
with self.assertRaises(OSError):
- llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
+ self.listmountdir(self.testcollection)
- llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
+ self.listmountdir(self.testcollection)
class FuseUnitTest(unittest.TestCase):
@@ -1087,7 +1086,7 @@ class FuseMagicTestPDHOnly(MountTestBase):
else:
self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
- mount_ls = llfuse.listdir(self.mounttmp)
+ mount_ls = self.listmountdir()
self.assertIn('README', mount_ls)
self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
arvados.util.uuid_pattern.match(fn)
@@ -1098,11 +1097,10 @@ class FuseMagicTestPDHOnly(MountTestBase):
self.assertDirContents(self.testcollection, ['thing1.txt'])
self.assertDirContents(os.path.join('by_id', self.testcollection),
['thing1.txt'])
- mount_ls = llfuse.listdir(self.mounttmp)
+ mount_ls = self.listmountdir()
self.assertIn('README', mount_ls)
self.assertIn(self.testcollection, mount_ls)
- self.assertIn(self.testcollection,
- llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
+ self.assertIn(self.testcollection, self.listmountdir('by_id'))
files = {}
files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list