[ARVADOS] updated: 06f92553e100b58d2d72a3190fb8e57abac820bb
git at public.curoverse.com
git at public.curoverse.com
Fri Feb 21 11:01:24 EST 2014
Summary of changes:
sdk/python/arvados/collection.py | 25 +++++++++++--------------
sdk/python/arvados/stream.py | 16 +++++++++++++++-
sdk/python/test_collections.py | 28 ++++++++++++++++++++++++----
3 files changed, 50 insertions(+), 19 deletions(-)
via 06f92553e100b58d2d72a3190fb8e57abac820bb (commit)
from b6888eaa68820d4107a73fb1fea136c00866f915 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 06f92553e100b58d2d72a3190fb8e57abac820bb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 21 11:02:34 2014 -0500
Added StreamFileReader.as_manifest() back and added test for it.
Refactored manifest text generation a bit.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index d790580..b48b6df 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -57,8 +57,7 @@ def normalize_stream(s, stream):
if len(stream[f]) == 0:
stream_tokens.append("0:0:{0}".format(fout))
- return stream_tokens
-
+ return stream_tokens
def normalize(collection):
streams = {}
@@ -121,13 +120,9 @@ class CollectionReader(object):
# now regenerate the manifest text based on the normalized stream
#print "normalizing", self._manifest_text
- self._manifest_text = ''
- for stream in self._streams:
- self._manifest_text += stream[0].replace(' ', '\\040')
- for t in stream[1:]:
- self._manifest_text += (" " + t.replace(' ', '\\040'))
- self._manifest_text += "\n"
- #print "result ", self._manifest_text
+ self._manifest_text = ''.join([StreamReader(stream).manifest_text() for stream in self._streams])
+ #print "result", self._manifest_text
+
def all_streams(self):
self._populate()
@@ -275,19 +270,21 @@ class CollectionWriter(object):
def finish(self):
return Keep.put(self.manifest_text())
-
def manifest_text(self):
self.finish_current_stream()
manifest = ''
+
for stream in self._finished_streams:
if not re.search(r'^\.(/.*)?$', stream[0]):
manifest += './'
manifest += stream[0].replace(' ', '\\040')
- for locator in stream[1]:
- manifest += " %s" % locator
- for sfile in stream[2]:
- manifest += " %d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040'))
+ manifest += ' ' + ' '.join(stream[1])
+ manifest += ' ' + ' '.join("%d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040')) for sfile in stream[2])
manifest += "\n"
+
+ #print 'writer',manifest
+ #print 'after reader',CollectionReader(manifest).manifest_text()
+
return CollectionReader(manifest).manifest_text()
def data_locators(self):
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 6ee12e1..dc90d8e 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -193,8 +193,14 @@ class StreamFileReader(object):
if data != '':
yield data
+ def as_manifest(self):
+ manifest_text = ['.']
+ manifest_text.extend([d[LOCATOR] for d in self._stream._data_locators])
+ manifest_text.extend(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], self.name().replace(' ', '\\040')) for seg in self.segments])
+ return arvados.CollectionReader(' '.join(manifest_text) + '\n').manifest_text()
+
class StreamReader(object):
- def __init__(self, tokens, keep=None, debug=False):
+ def __init__(self, tokens, keep=None, debug=False, _empty=False):
self._stream_name = None
self._data_locators = []
self._files = collections.OrderedDict()
@@ -258,3 +264,11 @@ class StreamReader(object):
for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size):
data += self._keep.get(locator)[segmentoffset:segmentoffset+segmentsize]
return data
+
+ def manifest_text(self):
+ manifest_text = [self.name().replace(' ', '\\040')]
+ manifest_text.extend([d[LOCATOR] for d in self._data_locators])
+ manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
+ for seg in f.segments])
+ for f in self._files.values()])
+ return ' '.join(manifest_text) + '\n'
diff --git a/sdk/python/test_collections.py b/sdk/python/test_collections.py
index 50687b0..3dfc72f 100644
--- a/sdk/python/test_collections.py
+++ b/sdk/python/test_collections.py
@@ -42,7 +42,7 @@ class LocalCollectionReaderTest(unittest.TestCase):
os.environ['KEEP_LOCAL_STORE'] = '/tmp'
LocalCollectionWriterTest().runTest()
def runTest(self):
- cr = arvados.CollectionReader('23ca013983d6239e98931cc779e68426+114')
+ cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114')
got = []
for s in cr.all_streams():
for f in s.all_files():
@@ -68,7 +68,7 @@ class LocalCollectionManifestSubsetTest(unittest.TestCase):
os.environ['KEEP_LOCAL_STORE'] = '/tmp'
LocalCollectionWriterTest().runTest()
def runTest(self):
- self._runTest('23ca013983d6239e98931cc779e68426+114',
+ self._runTest('d6c3b8e571f1b81ebb150a45ed06c884+114',
[[3, '.', 'bar.txt', 'bar'],
[3, '.', 'foo.txt', 'foo'],
[3, './baz', 'baz.txt', 'baz']])
@@ -130,8 +130,6 @@ class LocalCollectionEmptyFileTest(unittest.TestCase):
cw.start_new_file('zero.txt')
cw.write('')
- print 'stuff'
-
self.assertEqual(cw.manifest_text(), ". 0:0:zero.txt\n")
self.check_manifest_file_sizes(cw.manifest_text(), [0])
cw = arvados.CollectionWriter()
@@ -166,6 +164,7 @@ class LocalCollectionBZ2DecompressionTest(unittest.TestCase):
bz2_manifest = cw.manifest_text()
cr = arvados.CollectionReader(bz2_manifest)
+
got = 0
for x in list(cr.all_files())[0].readlines():
self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
@@ -451,3 +450,24 @@ class StreamReaderTest(unittest.TestCase):
self.assertEqual(sr.readfrom(25, 5), content[25:30])
self.assertEqual(sr.readfrom(30, 5), '')
+class ExtractFileTest(unittest.TestCase):
+ def runTest(self):
+ m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
+. 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt
+. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt
+. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
+. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt"""
+
+ m2 = arvados.CollectionReader(m1)
+
+ self.assertEqual(m2.manifest_text(),
+ ". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
+
+ self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md5sum.txt'].as_manifest(),
+ ". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
+ self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md6sum.txt'].as_manifest(),
+ ". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
+ self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md7sum.txt'].as_manifest(),
+ ". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
+ self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md9sum.txt'].as_manifest(),
+ ". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list