[ARVADOS] created: a1819526b85ce37c7d3ae421c2f5329c1c245c7f
git at public.curoverse.com
git at public.curoverse.com
Fri Jun 26 00:26:53 EDT 2015
at a1819526b85ce37c7d3ae421c2f5329c1c245c7f (commit)
commit a1819526b85ce37c7d3ae421c2f5329c1c245c7f
Author: radhika <radhika at curoverse.com>
Date: Fri Jun 26 00:25:12 2015 -0400
6219: first draft at fuse performance testing.
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
new file mode 100644
index 0000000..37e0d51
--- /dev/null
+++ b/services/fuse/tests/mount_test_base.py
@@ -0,0 +1,74 @@
+import arvados
+import arvados.safeapi
+import arvados_fuse as fuse
+import glob
+import json
+import llfuse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+import logging
+import multiprocessing
+import run_test_server
+
+logger = logging.getLogger('arvados.arv-mount')
+
+class MountTestBase(unittest.TestCase):
+ def setUp(self):
+ # The underlying C implementation of open() makes a fstat() syscall
+ # with the GIL still held. When the GETATTR message comes back to
+ # llfuse (which in these tests is in the same interpreter process) it
+ # can't acquire the GIL, so it can't service the fstat() call, so it
+ # deadlocks. The workaround is to run some of our test code in a
+ # separate process. Forturnately the multiprocessing module makes this
+ # relatively easy.
+ self.pool = multiprocessing.Pool(1)
+
+ self.keeptmp = tempfile.mkdtemp()
+ os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ self.mounttmp = tempfile.mkdtemp()
+ run_test_server.run()
+ run_test_server.authorize_with("admin")
+ self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
+
+ def make_mount(self, root_class, **root_kwargs):
+ self.operations = fuse.Operations(os.getuid(), os.getgid(), enable_write=True)
+ self.operations.inodes.add_entry(root_class(
+ llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
+ llfuse.init(self.operations, self.mounttmp, [])
+ threading.Thread(None, llfuse.main).start()
+ # wait until the driver is finished initializing
+ self.operations.initlock.wait()
+ return self.operations.inodes[llfuse.ROOT_INODE]
+
+ def tearDown(self):
+ self.pool.terminate()
+ self.pool.join()
+ del self.pool
+
+ # llfuse.close is buggy, so use fusermount instead.
+ #llfuse.close(unmount=True)
+
+ count = 0
+ success = 1
+ while (count < 9 and success != 0):
+ success = subprocess.call(["fusermount", "-u", self.mounttmp])
+ time.sleep(0.1)
+ count += 1
+
+ self.operations.destroy()
+
+ os.rmdir(self.mounttmp)
+ shutil.rmtree(self.keeptmp)
+ run_test_server.reset()
+
+ def assertDirContents(self, subdir, expect_content):
+ path = self.mounttmp
+ if subdir:
+ path = os.path.join(path, subdir)
+ self.assertEqual(sorted(expect_content), sorted(llfuse.listdir(path)))
diff --git a/services/fuse/tests/performance/test_collection_performance.py b/services/fuse/tests/performance/test_collection_performance.py
new file mode 100644
index 0000000..165b5da
--- /dev/null
+++ b/services/fuse/tests/performance/test_collection_performance.py
@@ -0,0 +1,147 @@
+import arvados
+import arvados.safeapi
+import arvados_fuse as fuse
+import glob
+import json
+import llfuse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+import logging
+import multiprocessing
+from .. import run_test_server
+from ..mount_test_base import MountTestBase
+
+logger = logging.getLogger('arvados.arv-mount')
+
+from performance_profiler import profiled
+
+ at profiled
+def fuseCreateCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, blocks_per_file=1, bytes_per_block=1):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ data = 'x' * blocks_per_file * bytes_per_block
+ names = 'file0.txt'
+ for i in range(1, files_per_stream):
+ names += ',file' + str(i) + '.txt'
+ file_names = names.split(',')
+
+ for i in range(0, streams):
+ with self.assertRaises(IOError):
+ with open(os.path.join(mounttmp, "./stream", "file0.txt"), "w") as f:
+ f.write(data)
+
+ os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
+
+ with self.assertRaises(OSError):
+ os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
+
+ # Create files
+ for j in range(0, files_per_stream):
+ with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
+ f.write(data)
+
+ d1 = llfuse.listdir(os.path.join(mounttmp, "./stream" + str(i)))
+ self.assertEqual(sorted(file_names), sorted(d1))
+
+ Test().runTest()
+
+ at profiled
+def fuseMoveFileFromCollectionWithManyFiles(mounttmp, stream, filename):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ d1 = llfuse.listdir(os.path.join(mounttmp, stream))
+ self.assertIn(filename, d1)
+
+ os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved-from-'+stream+'-'+filename))
+
+ d1 = llfuse.listdir(os.path.join(mounttmp))
+ self.assertIn('moved-from-'+stream+'-'+filename, d1)
+
+ d1 = llfuse.listdir(os.path.join(mounttmp, stream))
+ self.assertNotIn(filename, d1)
+
+ Test().runTest()
+
+ at profiled
+def fuseDeleteFileFromCollectionWithManyFiles(mounttmp, stream, filename):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ d1 = llfuse.listdir(os.path.join(mounttmp, stream))
+
+ # Delete file
+ os.remove(os.path.join(mounttmp, stream, filename))
+
+ # Try to delete it again
+ with self.assertRaises(OSError):
+ os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
+
+ Test().runTest()
+
+# Create a collection with two streams, each with 200 files
+class CreateCollectionWithManyFilesAndRenameMoveAndDeleteFile(MountTestBase):
+ def runTest(self):
+ collection = arvados.collection.Collection(api_client=self.api)
+ collection.save_new()
+
+ m = self.make_mount(fuse.CollectionDirectory)
+ with llfuse.lock:
+ m.new_collection(collection.api_response(), collection)
+ self.assertTrue(m.writable())
+
+ streams = 2
+ files_per_stream = 200
+
+ self.pool.apply(fuseCreateCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, 1, 1,))
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+
+ for i in range(0, streams):
+ self.assertIn('./stream' + str(i), collection2["manifest_text"])
+
+ for i in range(0, files_per_stream):
+ self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
+
+ # Move file0.txt out of the streams into .
+ self.pool.apply(fuseMoveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream0', 'file0.txt',))
+ self.pool.apply(fuseMoveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream1', 'file0.txt',))
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+
+ manifest_streams = collection2['manifest_text'].split('\n')
+ self.assertEqual(4, len(manifest_streams))
+
+ self.assertIn('moved-from-stream0-file0.txt', manifest_streams[0])
+ self.assertIn('moved-from-stream1-file0.txt', manifest_streams[0])
+
+ self.assertNotIn('file0.txt', manifest_streams[1])
+ self.assertNotIn('file0.txt', manifest_streams[2])
+
+ for i in range(1, files_per_stream):
+ self.assertIn('file' + str(i) + '.txt', manifest_streams[1])
+ self.assertIn('file' + str(i) + '.txt', manifest_streams[2])
+
+ # Delete 'file1.txt' from both the streams
+ self.pool.apply(fuseDeleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream0', 'file1.txt'))
+ self.pool.apply(fuseDeleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream1', 'file1.txt'))
+
+ collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+
+ manifest_streams = collection2['manifest_text'].split('\n')
+ self.assertEqual(4, len(manifest_streams))
+
+ self.assertIn('moved-from-stream0-file0.txt', manifest_streams[0])
+ self.assertIn('moved-from-stream1-file0.txt', manifest_streams[0])
+
+ self.assertNotIn('file1.txt', manifest_streams[1])
+ self.assertNotIn('file1.txt', manifest_streams[2])
+
+ for i in range(2, files_per_stream):
+ self.assertIn('file' + str(i) + '.txt', manifest_streams[1])
+ self.assertIn('file' + str(i) + '.txt', manifest_streams[2])
+
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index ac5af5b..48aefbf 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -16,62 +16,9 @@ import logging
import multiprocessing
import run_test_server
-logger = logging.getLogger('arvados.arv-mount')
+from mount_test_base import MountTestBase
-class MountTestBase(unittest.TestCase):
- def setUp(self):
- # The underlying C implementation of open() makes a fstat() syscall
- # with the GIL still held. When the GETATTR message comes back to
- # llfuse (which in these tests is in the same interpreter process) it
- # can't acquire the GIL, so it can't service the fstat() call, so it
- # deadlocks. The workaround is to run some of our test code in a
- # separate process. Forturnately the multiprocessing module makes this
- # relatively easy.
- self.pool = multiprocessing.Pool(1)
-
- self.keeptmp = tempfile.mkdtemp()
- os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
- self.mounttmp = tempfile.mkdtemp()
- run_test_server.run()
- run_test_server.authorize_with("admin")
- self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
-
- def make_mount(self, root_class, **root_kwargs):
- self.operations = fuse.Operations(os.getuid(), os.getgid(), enable_write=True)
- self.operations.inodes.add_entry(root_class(
- llfuse.ROOT_INODE, self.operations.inodes, self.api, 0, **root_kwargs))
- llfuse.init(self.operations, self.mounttmp, [])
- threading.Thread(None, llfuse.main).start()
- # wait until the driver is finished initializing
- self.operations.initlock.wait()
- return self.operations.inodes[llfuse.ROOT_INODE]
-
- def tearDown(self):
- self.pool.terminate()
- self.pool.join()
- del self.pool
-
- # llfuse.close is buggy, so use fusermount instead.
- #llfuse.close(unmount=True)
-
- count = 0
- success = 1
- while (count < 9 and success != 0):
- success = subprocess.call(["fusermount", "-u", self.mounttmp])
- time.sleep(0.1)
- count += 1
-
- self.operations.destroy()
-
- os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
- run_test_server.reset()
-
- def assertDirContents(self, subdir, expect_content):
- path = self.mounttmp
- if subdir:
- path = os.path.join(path, subdir)
- self.assertEqual(sorted(expect_content), sorted(llfuse.listdir(path)))
+logger = logging.getLogger('arvados.arv-mount')
class FuseMountTest(MountTestBase):
commit f5cb4310db63948ea63329415ade2ed8eef529e2
Merge: 46a6199 1ec9e37
Author: radhika <radhika at curoverse.com>
Date: Thu Jun 25 22:19:39 2015 -0400
Merge branch 'master' into 6219-fuse-performance-testing
commit 46a6199f3a40a24ee145adc390500190b17a6395
Merge: 20ade56 69107d2
Author: radhika <radhika at curoverse.com>
Date: Wed Jun 24 15:36:11 2015 -0400
Merge branch 'master' into 3198-writable-fuse
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list