[ARVADOS] updated: fa739e32888d42bb99cef4781dc0c3f1c6190ac7
git at public.curoverse.com
git at public.curoverse.com
Sun Feb 16 22:13:39 EST 2014
Summary of changes:
sdk/python/arvados/stream.py | 45 +++++++++++++++++++---
sdk/python/test_collections.py | 81 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 117 insertions(+), 9 deletions(-)
via fa739e32888d42bb99cef4781dc0c3f1c6190ac7 (commit)
from a9c427bdcdb215119bb28d2b53b0f8b462048e72 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fa739e32888d42bb99cef4781dc0c3f1c6190ac7
Author: Peter Amstutz <tetron at cosmos>
Date: Sun Feb 16 22:14:04 2014 -0500
Unit tests for arvados.locators_and_ranges
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 3843411..7dd5736 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -27,25 +27,57 @@ BLOCKSIZE = 1
OFFSET = 2
SEGMENTSIZE = 3
-def locators_and_ranges(data_locators, range_start, range_size):
- '''returns list of [block locator, blocksize, segment offset, segment size] that satisfies the range'''
+def locators_and_ranges(data_locators, range_start, range_size, debug=False):
+ '''
+ Get blocks that are covered by the range
+ data_locators: list of [locator, block_size, block_start], assumes that blocks are in order and contigous
+ range_start: start of range
+ range_size: size of range
+ returns list of [block locator, blocksize, segment offset, segment size] that satisfies the range
+ '''
+ if range_size == 0:
+ return []
resp = []
range_start = long(range_start)
range_size = long(range_size)
range_end = range_start + range_size
block_start = 0L
- for locator, block_size, block_start in data_locators:
+
+ hi = len(data_locators)
+ lo = 0
+ i = int((hi + lo) / 2)
+ block_size = data_locators[i][BLOCKSIZE]
+ block_start = data_locators[i][OFFSET]
+ block_end = block_start + block_size
+ if debug: print '---'
+ while not (range_start >= block_start and range_start <= block_end):
+ if lo == i:
+ break
+ if range_start > block_start:
+ lo = i
+ else:
+ hi = i
+ i = int((hi + lo) / 2)
+ if debug: print lo, i, hi
+ block_size = data_locators[i][BLOCKSIZE]
+ block_start = data_locators[i][OFFSET]
+ block_end = block_start + block_size
+
+ while i < len(data_locators):
+ locator, block_size, block_start = data_locators[i]
block_end = block_start + block_size
- if range_end < block_start:
+ if debug:
+ print locator, "range_start", range_start, "block_start", block_start, "range_end", range_end, "block_end", block_end
+ if range_end <= block_start:
# range ends before this block starts, so don't look at any more locators
break
- if range_start > block_end:
+ if range_start >= block_end:
# range starts after this block ends, so go to next block
next
elif range_start >= block_start and range_end <= block_end:
# range starts and ends in this block
resp.append([locator, block_size, range_start - block_start, range_size])
- elif range_start >= block_start:
+ elif range_start >= block_start and range_end > block_end:
# range starts in this block
resp.append([locator, block_size, range_start - block_start, block_end - range_start])
elif range_start < block_start and range_end > block_end:
@@ -55,6 +87,7 @@ def locators_and_ranges(data_locators, range_start, range_size):
# range starts in a previous block and ends in this block
resp.append([locator, block_size, 0L, range_end - block_start])
block_start = block_end
+ i += 1
return resp
diff --git a/sdk/python/test_collections.py b/sdk/python/test_collections.py
index a352527..08589dd 100644
--- a/sdk/python/test_collections.py
+++ b/sdk/python/test_collections.py
@@ -208,9 +208,6 @@ class LocalCollectionGzipDecompressionTest(unittest.TestCase):
"decompression returned %d lines instead of %d" % (got, n_lines_in))
class NormalizedCollectionTest(unittest.TestCase):
- def setUp(self):
- pass
-
def runTest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
@@ -257,3 +254,81 @@ class NormalizedCollectionTest(unittest.TestCase):
m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
"""
self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8)
+
+class LocatorsAndRangesTest(unittest.TestCase):
+ def runTest(self):
+ blocks2 = [['a', 10, 0],
+ ['b', 10, 10],
+ ['c', 10, 20],
+ ['d', 10, 30],
+ ['e', 10, 40],
+ ['f', 10, 50]]
+
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [['a', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [['b', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [['c', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [['d', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [['e', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [['f', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
+
+ blocks3 = [['a', 10, 0],
+ ['b', 10, 10],
+ ['c', 10, 20],
+ ['d', 10, 30],
+ ['e', 10, 40],
+ ['f', 10, 50],
+ ['g', 10, 60]]
+
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [['a', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [['b', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [['c', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [['d', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [['e', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [['f', 10, 2, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [['g', 10, 2, 2]])
+
+
+ blocks = [['a', 10, 0],
+ ['b', 15, 10],
+ ['c', 5, 25]]
+ self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10],
+ ['b', 15, 0, 1]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9],
+ ['b', 15, 0, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10],
+ ['b', 15, 0, 15]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10],
+ ['b', 15, 0, 15],
+ ['c', 5, 0, 5]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [['a', 10, 1, 9],
+ ['b', 15, 0, 15],
+ ['c', 5, 0, 5]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10],
+ ['b', 15, 0, 15],
+ ['c', 5, 0, 5]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2],
+ ['b', 15, 0, 15]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2],
+ ['b', 15, 0, 15],
+ ['c', 5, 0, 3]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1],
+ ['b', 15, 0, 14]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14],
+ ['c', 5, 0, 1]])
+
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list