[ARVADOS] updated: 3b06308d1704a14c10f4821494085dffa8fe6ea6
git at public.curoverse.com
git at public.curoverse.com
Mon Jun 29 16:31:51 EDT 2015
Summary of changes:
.../performance/test_collection_performance.py | 115 ++++++++++-----------
1 file changed, 53 insertions(+), 62 deletions(-)
via 3b06308d1704a14c10f4821494085dffa8fe6ea6 (commit)
from 79be7ea20c46c6d005de5e0a24bf7b47ccfd43f6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3b06308d1704a14c10f4821494085dffa8fe6ea6
Author: radhika <radhika at curoverse.com>
Date: Mon Jun 29 16:19:08 2015 -0400
6219: magic dir test in progress
diff --git a/services/fuse/tests/performance/test_collection_performance.py b/services/fuse/tests/performance/test_collection_performance.py
index fe92084..445d823 100644
--- a/services/fuse/tests/performance/test_collection_performance.py
+++ b/services/fuse/tests/performance/test_collection_performance.py
@@ -21,34 +21,23 @@ logger = logging.getLogger('arvados.arv-mount')
from performance_profiler import profiled
+ at profiled
def fuseCreateCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, blocks_per_file=1, bytes_per_block=1, data='x'):
class Test(unittest.TestCase):
def runTest(self):
- names = 'file0.txt'
- for i in range(1, files_per_stream):
- names += ',file' + str(i) + '.txt'
- file_names = names.split(',')
+ file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
for i in range(0, streams):
- with self.assertRaises(IOError):
- with open(os.path.join(mounttmp, "./stream", "file0.txt"), "w") as f:
- f.write(data)
-
os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
- with self.assertRaises(OSError):
- os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
-
# Create files
for j in range(0, files_per_stream):
with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
f.write(data)
- d1 = llfuse.listdir(os.path.join(mounttmp, "./stream" + str(i)))
- self.assertEqual(sorted(file_names), sorted(d1))
-
Test().runTest()
+ at profiled
def fuseReadContentsFromCollectionWithManyFiles(mounttmp, streams, files_per_stream, content):
class Test(unittest.TestCase):
def runTest(self):
@@ -60,39 +49,36 @@ def fuseReadContentsFromCollectionWithManyFiles(mounttmp, streams, files_per_str
Test().runTest()
+ at profiled
def fuseMoveFileFromCollectionWithManyFiles(mounttmp, stream, filename):
class Test(unittest.TestCase):
def runTest(self):
d1 = llfuse.listdir(os.path.join(mounttmp, stream))
self.assertIn(filename, d1)
- os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved-from-'+stream+'-'+filename))
+ os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, filename))
d1 = llfuse.listdir(os.path.join(mounttmp))
- self.assertIn('moved-from-'+stream+'-'+filename, d1)
+ self.assertIn(filename, d1)
d1 = llfuse.listdir(os.path.join(mounttmp, stream))
self.assertNotIn(filename, d1)
Test().runTest()
+ at profiled
def fuseDeleteFileFromCollectionWithManyFiles(mounttmp, stream, filename):
class Test(unittest.TestCase):
def runTest(self):
- d1 = llfuse.listdir(os.path.join(mounttmp, stream))
-
- # Delete file
os.remove(os.path.join(mounttmp, stream, filename))
- # Try to delete it again
- with self.assertRaises(OSError):
- os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
-
Test().runTest()
# Create a collection with two streams, each with 200 files
class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
- @profiled
+ def setUp(self):
+ super(CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
+
def test_CreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
collection = arvados.collection.Collection(api_client=self.api)
collection.save_new()
@@ -132,7 +118,7 @@ class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
self.assertEqual(4, len(manifest_streams))
for i in range(0, streams):
- self.assertIn('moved-from-stream'+str(i)+'-file0.txt', manifest_streams[0])
+ self.assertIn('file0.txt', manifest_streams[0])
for i in range(0, streams):
self.assertNotIn('file0.txt', manifest_streams[i+1])
@@ -151,7 +137,7 @@ class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
self.assertEqual(4, len(manifest_streams))
for i in range(0, streams):
- self.assertIn('moved-from-stream'+str(i)+'-file0.txt', manifest_streams[0])
+ self.assertIn('file0.txt', manifest_streams[0])
self.assertNotIn('file1.txt', collection2['manifest_text'])
@@ -160,21 +146,22 @@ class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
-class UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
+ at profiled
+def magicDirTest_MoveFileFromCollectionWithManyFiles(mounttmp, collection1, collection2, stream, filename):
+ class Test(unittest.TestCase):
+ def runTest(self):
+ #os.rename(os.path.join(mounttmp, collection1, stream, filename), os.path.join(mounttmp, collection2, stream, filename))
+ print('TBD')
+
+ Test().runTest()
+
+class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
def setUp(self):
- super(UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
+ super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
- @profiled
- def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
+ def magicDirTest_createCollectionWithManyFiles(self, streams=1, files_per_stream=1, blocks_per_file=1, bytes_per_block=1, data='x'):
# Create collection
cw = arvados.CollectionWriter()
-
- streams = 2
- files_per_stream = 200
- blocks_per_file = 1
- bytes_per_block = 1
-
- data = 'x' * blocks_per_file * bytes_per_block
for i in range(0, streams):
cw.start_new_stream('./stream' + str(i))
for j in range(0, files_per_stream):
@@ -183,42 +170,46 @@ class UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBa
self.testcollection = cw.finish()
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
+ return self.testcollection
- # Mount FuseMagicDir
- self.make_mount(fuse.MagicDirectory)
-
- mount_ls = llfuse.listdir(self.mounttmp)
- self.assertIn('README', mount_ls)
-
- self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
- arvados.util.uuid_pattern.match(fn)
- for fn in mount_ls),
- "new FUSE MagicDirectory lists Collection")
-
- names = 'stream0'
- for i in range(1, streams):
- names += ',stream' + str(i)
- stream_names = names.split(',')
-
- names = 'file0.txt'
- for i in range(1, files_per_stream):
- names += ',file' + str(i) + '.txt'
- file_names = names.split(',')
+ @profiled
+ def magicDirTest_readCollectionContents(self, collection, streams=1, files_per_stream=1, blocks_per_file=1, bytes_per_block=1, data='x'):
+ stream_names = ["stream%i" % i for i in range(0, streams)]
+ file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
- self.assertDirContents(self.testcollection, stream_names)
- self.assertDirContents(os.path.join('by_id', self.testcollection), stream_names)
+ self.assertDirContents(collection, stream_names)
+ self.assertDirContents(os.path.join('by_id', collection), stream_names)
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
- self.assertIn(self.testcollection, mount_ls)
- self.assertIn(self.testcollection,
+ self.assertIn(collection, mount_ls)
+ self.assertIn(collection,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
files = {}
for i in range(0, streams):
for j in range(0, files_per_stream):
- files[os.path.join(self.mounttmp, self.testcollection, 'stream'+str(i)+'/file'+str(j)+'.txt')] = data
+ files[os.path.join(self.mounttmp, collection, 'stream'+str(i)+'/file'+str(j)+'.txt')] = data
for k, v in files.items():
with open(os.path.join(self.mounttmp, k)) as f:
self.assertEqual(v, f.read())
+
+ def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
+ streams = 2
+ files_per_stream = 200
+ blocks_per_file = 1
+ bytes_per_block = 1
+
+ data = 'x' * blocks_per_file * bytes_per_block
+
+ collection1 = self.magicDirTest_createCollectionWithManyFiles(streams, files_per_stream, blocks_per_file, bytes_per_block, data)
+ collection2 = self.magicDirTest_createCollectionWithManyFiles()
+
+ # Mount FuseMagicDir
+ self.make_mount(fuse.MagicDirectory)
+
+ self.magicDirTest_readCollectionContents(collection1, streams, files_per_stream, blocks_per_file, bytes_per_block, data)
+
+ # Move file0.txt out of the streams into .
+ self.pool.apply(magicDirTest_MoveFileFromCollectionWithManyFiles, (self.mounttmp, collection1, collection2, 'stream0', 'file1.txt',))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list