[ARVADOS] updated: ad638d5e0e28ab260df272a33bdeaa2b56a96054
git at public.curoverse.com
git at public.curoverse.com
Thu Dec 18 22:41:58 EST 2014
Summary of changes:
sdk/python/arvados/arvfile.py | 26 +++---
sdk/python/arvados/collection.py | 21 +++--
sdk/python/arvados/ranges.py | 60 +++++++++----
sdk/python/arvados/stream.py | 71 ++++++++-------
sdk/python/tests/test_collections.py | 169 ++++++++++++++++++-----------------
5 files changed, 193 insertions(+), 154 deletions(-)
via ad638d5e0e28ab260df272a33bdeaa2b56a96054 (commit)
from 7abed45cc9b1c5e816584b9f267f6c448384ffed (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ad638d5e0e28ab260df272a33bdeaa2b56a96054
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Dec 18 22:43:23 2014 -0500
3198: Refactoring. Added Range object instead of fiddling with arrays, should
make the code easier to comprehend.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 593d4fb..f544eaa 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -169,7 +169,7 @@ class StreamFileReader(ArvadosFileReaderBase):
def _size(self):
n = self.segments[-1]
- return n[OFFSET] + n[BLOCKSIZE]
+ return n.range_start + n.range_size
@ArvadosFileBase._before_close
@retry_method
@@ -181,9 +181,10 @@ class StreamFileReader(ArvadosFileReaderBase):
data = ''
available_chunks = locators_and_ranges(self.segments, self._filepos, size)
if available_chunks:
- locator, blocksize, segmentoffset, segmentsize = available_chunks[0]
- data = self._stream._readfrom(locator+segmentoffset, segmentsize,
- num_retries=num_retries)
+ lr = available_chunks[0]
+ data = self._stream._readfrom(lr.locator+lr.segment_offset,
+ lr.segment_size,
+ num_retries=num_retries)
self._filepos += len(data)
return data
@@ -203,9 +204,9 @@ class StreamFileReader(ArvadosFileReaderBase):
def as_manifest(self):
manifest_text = ['.']
- manifest_text.extend([d[LOCATOR] for d in self._stream._data_locators])
- manifest_text.extend(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], self.name().replace(' ', '\\040')) for seg in self.segments])
- return CollectionReader(' '.join(manifest_text) + '\n').manifest_text(normalize=True)
+ manifest_text.extend([d.locator for d in self._stream._data_locators])
+ manifest_text.extend(["{}:{}:{}".format(seg.locator, seg.range_size, self.name().replace(' ', '\\040')) for seg in self.segments])
+ return manifest_text #arvados.CollectionReader(' '.join(manifest_text) + '\n').manifest_text(normalize=True)
class ArvadosFile(ArvadosFileReaderBase):
@@ -225,7 +226,7 @@ class ArvadosFile(ArvadosFileReaderBase):
fileoffset = 0L
for seg in segs:
- for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._stream._data_locators, seg[LOCATOR]+seg[OFFSET], seg[SEGMENTSIZE]):
+ for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._stream._data_locators, seg.locator+seg.range_start, seg[SEGMENTSIZE]):
newstream.append([locator, blocksize, streamoffset])
self.segments.append([streamoffset+segmentoffset, segmentsize, fileoffset])
streamoffset += blocksize
@@ -259,6 +260,9 @@ class ArvadosFile(ArvadosFileReaderBase):
pass
def add_segment(self, blocks, pos, size):
- for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(blocks, pos, size):
- last = self.segments[-1] if self.segments else [0, 0, 0]
- self.segments.append([locator, segmentsize, last[OFFSET]+last[BLOCKSIZE], segmentoffset])
+ for lr in locators_and_ranges(blocks, pos, size):
+ last = self.segments[-1] if self.segments else Range(0, 0, 0)
+ r = Range(lr.locator, last.range_start+last.range_size, lr.segment_size)
+ r.block_size = lr.block_size
+ r.segment_offset = lr.segment_offset
+ self.segments.append(r)
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 077115e..b5a9ef9 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -9,6 +9,7 @@ from stat import *
from .arvfile import ArvadosFileBase, split, ArvadosFile
from keep import *
from .stream import StreamReader, normalize_stream
+from .ranges import Range
import config
import errors
import util
@@ -181,7 +182,7 @@ class CollectionReader(CollectionBase):
if filename not in streams[streamname]:
streams[streamname][filename] = []
for r in f.segments:
- streams[streamname][filename].extend(s.locators_and_ranges(r[0], r[1]))
+ streams[streamname][filename].extend(s.locators_and_ranges(r.locator, r.range_size))
self._streams = [normalize_stream(s, streams[s])
for s in sorted(streams)]
@@ -690,10 +691,10 @@ def import_manifest(manifest_text):
continue
if state == BLOCKS:
- s = re.match(r'[0-9a-f]{32}\+(\d)+(\+\S+)*', tok)
+ s = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
if s:
blocksize = long(s.group(1))
- blocks.append([tok, blocksize, streamoffset])
+ blocks.append(Range(tok, streamoffset, blocksize))
streamoffset += blocksize
else:
state = SEGMENTS
@@ -718,12 +719,18 @@ def import_manifest(manifest_text):
def export_manifest(item, stream_name="."):
buf = ""
- print item
if isinstance(item, Collection):
- for i, j in item.items.values():
- buf += export_manifest(j, stream_name)
+ stream = {}
+ for k,v in item.items.items():
+ if isinstance(item, Collection):
+ buf += export_manifest(v, stream_name)
+ else:
+ if isinstance(item, ArvadosFile):
+ buf += str(item.segments)
+ #stream[k] = [[s.locator, s[4], s[], s[]] for s in item.segments]
else:
buf += stream_name
buf += " "
- buf += item.segments
+ buf += str(item.segments)
+ buf += "\n"
return buf
diff --git a/sdk/python/arvados/ranges.py b/sdk/python/arvados/ranges.py
index dbd5e3d..acc1c50 100644
--- a/sdk/python/arvados/ranges.py
+++ b/sdk/python/arvados/ranges.py
@@ -1,7 +1,11 @@
-LOCATOR = 0
-BLOCKSIZE = 1
-OFFSET = 2
-SEGMENTSIZE = 3
+class Range(object):
+ def __init__(self, locator, range_start, range_size):
+ self.locator = locator
+ self.range_start = range_start
+ self.range_size = range_size
+
+ def __repr__(self):
+ return "[\"%s\", %i, %i]" % (self.locator, self.range_size, self.range_start)
def first_block(data_locators, range_start, range_size, debug=False):
block_start = 0L
@@ -12,8 +16,8 @@ def first_block(data_locators, range_start, range_size, debug=False):
hi = len(data_locators)
lo = 0
i = int((hi + lo) / 2)
- block_size = data_locators[i][BLOCKSIZE]
- block_start = data_locators[i][OFFSET]
+ block_size = data_locators[i].range_size
+ block_start = data_locators[i].range_start
block_end = block_start + block_size
if debug: print '---'
@@ -30,19 +34,35 @@ def first_block(data_locators, range_start, range_size, debug=False):
hi = i
i = int((hi + lo) / 2)
if debug: print lo, i, hi
- block_size = data_locators[i][BLOCKSIZE]
- block_start = data_locators[i][OFFSET]
+ block_size = data_locators[i].range_size
+ block_start = data_locators[i].range_start
block_end = block_start + block_size
return i
+class LocatorAndRange(object):
+ def __init__(self, locator, block_size, segment_offset, segment_size):
+ self.locator = locator
+ self.block_size = block_size
+ self.segment_offset = segment_offset
+ self.segment_size = segment_size
+
+ def __eq__(self, other):
+ return (self.locator == other.locator and
+ self.block_size == other.block_size and
+ self.segment_offset == other.segment_offset and
+ self.segment_size == other.segment_size)
+
+ def __repr__(self):
+ return "[\"%s\", %i, %i, %i]" % (self.locator, self.block_size, self.segment_offset, self.segment_size)
+
def locators_and_ranges(data_locators, range_start, range_size, debug=False):
'''
Get blocks that are covered by the range
- data_locators: list of [locator, block_size, block_start], assumes that blocks are in order and contigous
+ data_locators: list of Range objects, assumes that blocks are in order and contigous
range_start: start of range
range_size: size of range
- returns list of [block locator, blocksize, segment offset, segment size] that satisfies the range
+ returns list of LocatorAndRange objects
'''
if range_size == 0:
return []
@@ -56,10 +76,12 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False):
return []
while i < len(data_locators):
- locator, block_size, block_start = data_locators[i]
+ dl = data_locators[i]
+ block_start = dl.range_start
+ block_size = dl.range_size
block_end = block_start + block_size
if debug:
- print locator, "range_start", range_start, "block_start", block_start, "range_end", range_end, "block_end", block_end
+ print dl.locator, "range_start", range_start, "block_start", block_start, "range_end", range_end, "block_end", block_end
if range_end <= block_start:
# range ends before this block starts, so don't look at any more locators
break
@@ -71,16 +93,16 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False):
if range_start >= block_start and range_end <= block_end:
# range starts and ends in this block
- resp.append([locator, block_size, range_start - block_start, range_size])
+ resp.append(LocatorAndRange(dl.locator, block_size, range_start - block_start, range_size))
elif range_start >= block_start and range_end > block_end:
# range starts in this block
- resp.append([locator, block_size, range_start - block_start, block_end - range_start])
+ resp.append(LocatorAndRange(dl.locator, block_size, range_start - block_start, block_end - range_start))
elif range_start < block_start and range_end > block_end:
# range starts in a previous block and extends to further blocks
- resp.append([locator, block_size, 0L, block_size])
+ resp.append(LocatorAndRange(dl.locator, block_size, 0L, block_size))
elif range_start < block_start and range_end <= block_end:
# range starts in a previous block and ends in this block
- resp.append([locator, block_size, 0L, range_end - block_start])
+ resp.append(LocatorAndRange(dl.locator, block_size, 0L, range_end - block_start))
block_start = block_end
i += 1
return resp
@@ -88,7 +110,7 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False):
def replace_range(data_locators, range_start, range_size, new_locator, debug=False):
'''
Replace a file segment range with a new segment.
- data_locators: list of [locator, segment_size, segment_start], assumes that segments are in order and contigous
+ data_locators: list of Range objects, assumes that segments are in order and contigous
range_start: start of range
range_size: size of range
new_locator: locator for new segment to be inserted
@@ -102,9 +124,9 @@ def replace_range(data_locators, range_start, range_size, new_locator, debug=Fal
range_end = range_start + range_size
last = data_locators[-1]
- if (last[OFFSET]+last[BLOCKSIZE]) == range_start:
+ if (last.range_start+last.range_size) == range_start:
# extend last segment
- last[BLOCKSIZE] += range_size
+ last.range_size += range_size
return
i = first_block(data_locators, range_start, range_size, debug)
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index ecccefa..d28efcf 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -16,7 +16,8 @@ import errors
def normalize_stream(s, stream):
'''
s is the stream name
- stream is a StreamReader object
+ stream is a dict mapping each filename to a list in the form [block locator, block size, segment offset (from beginning of block), segment size]
+ returns the stream as a list of tokens
'''
stream_tokens = [s]
sortedfiles = list(stream.keys())
@@ -24,29 +25,33 @@ def normalize_stream(s, stream):
blocks = {}
streamoffset = 0L
+ # Go through each file and add each referenced block exactly once.
for f in sortedfiles:
for b in stream[f]:
- if b[arvados.LOCATOR] not in blocks:
- stream_tokens.append(b[arvados.LOCATOR])
- blocks[b[arvados.LOCATOR]] = streamoffset
- streamoffset += b[arvados.BLOCKSIZE]
+ if b.locator not in blocks:
+ stream_tokens.append(b.locator)
+ blocks[b.locator] = streamoffset
+ streamoffset += b.block_size
+ # Add the empty block if the stream is otherwise empty.
if len(stream_tokens) == 1:
stream_tokens.append(config.EMPTY_BLOCK_LOCATOR)
for f in sortedfiles:
+ # Add in file segments
current_span = None
fout = f.replace(' ', '\\040')
for segment in stream[f]:
- segmentoffset = blocks[segment[arvados.LOCATOR]] + segment[arvados.OFFSET]
+ # Collapse adjacent segments
+ streamoffset = blocks[segment.locator] + segment.segment_offset
if current_span is None:
- current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]]
+ current_span = [streamoffset, streamoffset + segment.segment_size]
else:
- if segmentoffset == current_span[1]:
- current_span[1] += segment[arvados.SEGMENTSIZE]
+ if streamoffset == current_span[1]:
+ current_span[1] += segment.segment_size
else:
stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
- current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]]
+ current_span = [streamoffset, streamoffset + segment.segment_size]
if current_span is not None:
stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
@@ -78,7 +83,7 @@ class StreamReader(object):
s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
if s:
blocksize = long(s.group(1))
- self._data_locators.append([tok, blocksize, streamoffset])
+ self._data_locators.append(Range(tok, streamoffset, blocksize))
streamoffset += blocksize
continue
@@ -88,10 +93,10 @@ class StreamReader(object):
size = long(s.group(2))
name = s.group(3).replace('\\040', ' ')
if name not in self._files:
- self._files[name] = StreamFileReader(self, [[pos, size, 0]], name)
+ self._files[name] = StreamFileReader(self, [Range(pos, 0, size)], name)
else:
filereader = self._files[name]
- filereader.segments.append([pos, size, filereader.size()])
+ filereader.segments.append(Range(pos, filereader.size(), size))
continue
raise errors.SyntaxError("Invalid manifest format")
@@ -107,7 +112,7 @@ class StreamReader(object):
def _size(self):
n = self._data_locators[-1]
- return n[OFFSET] + n[BLOCKSIZE]
+ return n.range_start + n.range_size
def size(self):
return self._size()
@@ -131,19 +136,19 @@ class StreamReader(object):
if self._keep is None:
self._keep = KeepClient(num_retries=self.num_retries)
data = []
- for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size):
- data.append(self._keepget(locator, num_retries=num_retries)[segmentoffset:segmentoffset+segmentsize])
+ for lr in locators_and_ranges(self._data_locators, start, size):
+ data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size])
return ''.join(data)
def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
if strip:
for d in self._data_locators:
- m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR])
+ m = re.match(r'^[0-9a-f]{32}\+\d+', d.locator)
manifest_text.append(m.group(0))
else:
- manifest_text.extend([d[LOCATOR] for d in self._data_locators])
- manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name.replace(' ', '\\040'))
+ manifest_text.extend([d.locator for d in self._data_locators])
+ manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
for seg in f.segments])
for f in self._files.values()])
return ' '.join(manifest_text) + '\n'
@@ -204,8 +209,8 @@ class BufferBlock(object):
# def _init_bufferblock(self):
# last = self._data_locators[-1]
-# streamoffset = last[OFFSET] + last[BLOCKSIZE]
-# if last[BLOCKSIZE] == 0:
+# streamoffset = last.range_start + last.range_size
+# if last.range_size == 0:
# del self._data_locators[-1]
# self.current_bblock = BufferBlock("bufferblock%i" % len(self.bufferblocks), streamoffset)
# self.bufferblocks[self.current_bblock.locator] = self.current_bblock
@@ -225,34 +230,34 @@ class BufferBlock(object):
# while i < len(tmp_segs):
# # Go through each segment and identify segments that include the buffer block
# s = tmp_segs[i]
-# if s[LOCATOR] < self.current_bblock.locator_list_entry[OFFSET] and (s[LOCATOR] + s[BLOCKSIZE]) > self.current_bblock.locator_list_entry[OFFSET]:
+# if s[LOCATOR] < self.current_bblock.locator_list_entry.range_start and (s[LOCATOR] + s.range_size) > self.current_bblock.locator_list_entry.range_start:
# # The segment straddles the previous block and the current buffer block. Split the segment.
-# b1 = self.current_bblock.locator_list_entry[OFFSET] - s[LOCATOR]
-# b2 = (s[LOCATOR] + s[BLOCKSIZE]) - self.current_bblock.locator_list_entry[OFFSET]
-# bb_seg = [self.current_bblock.locator_list_entry[OFFSET], b2, s[OFFSET]+b1]
-# tmp_segs[i] = [s[LOCATOR], b1, s[OFFSET]]
+# b1 = self.current_bblock.locator_list_entry.range_start - s[LOCATOR]
+# b2 = (s[LOCATOR] + s.range_size) - self.current_bblock.locator_list_entry.range_start
+# bb_seg = [self.current_bblock.locator_list_entry.range_start, b2, s.range_start+b1]
+# tmp_segs[i] = [s[LOCATOR], b1, s.range_start]
# tmp_segs.insert(i+1, bb_seg)
# bufferblock_segs.append(bb_seg)
# i += 1
-# elif s[LOCATOR] >= self.current_bblock.locator_list_entry[OFFSET]:
+# elif s[LOCATOR] >= self.current_bblock.locator_list_entry.range_start:
# # The segment's data is in the buffer block.
# bufferblock_segs.append(s)
# i += 1
# # Now sum up the segments to get the total bytes
# # of the file referencing into the buffer block.
-# write_total = sum([s[BLOCKSIZE] for s in bufferblock_segs])
+# write_total = sum([s.range_size for s in bufferblock_segs])
-# if write_total < self.current_bblock.locator_list_entry[BLOCKSIZE]:
+# if write_total < self.current_bblock.locator_list_entry.range_size:
# # There is more data in the buffer block than is actually accounted for by segments, so
# # re-pack into a new buffer by copying over to a new buffer block.
# new_bb = BufferBlock(self.current_bblock.locator,
-# self.current_bblock.locator_list_entry[OFFSET],
+# self.current_bblock.locator_list_entry.range_start,
# starting_size=write_total)
# for t in bufferblock_segs:
-# t_start = t[LOCATOR] - self.current_bblock.locator_list_entry[OFFSET]
-# t_end = t_start + t[BLOCKSIZE]
-# t[0] = self.current_bblock.locator_list_entry[OFFSET] + new_bb.write_pointer
+# t_start = t[LOCATOR] - self.current_bblock.locator_list_entry.range_start
+# t_end = t_start + t.range_size
+# t[0] = self.current_bblock.locator_list_entry.range_start + new_bb.write_pointer
# new_bb.append(self.current_bblock.buffer_block[t_start:t_end])
# self.current_bblock = new_bb
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 9882db9..5ed8716 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -14,6 +14,7 @@ import unittest
import run_test_server
import arvados_testutil as tutil
+from arvados.ranges import Range, LocatorAndRange
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
@@ -210,97 +211,97 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
self.assertEqual(arvados.CollectionReader(m8, self.api_client).manifest_text(normalize=True), m8)
def test_locators_and_ranges(self):
- blocks2 = [['a', 10, 0],
- ['b', 10, 10],
- ['c', 10, 20],
- ['d', 10, 30],
- ['e', 10, 40],
- ['f', 10, 50]]
-
- self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [['a', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [['b', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [['c', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [['d', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [['e', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [['f', 10, 2, 2]])
+ blocks2 = [Range('a', 0, 10),
+ Range('b', 10, 10),
+ Range('c', 20, 10),
+ Range('d', 30, 10),
+ Range('e', 40, 10),
+ Range('f', 50, 10)]
+
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [['a', 10, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [['b', 10, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [['c', 10, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [['d', 10, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [['e', 10, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [['f', 10, 0, 2]])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [LocatorAndRange('a', 10, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [LocatorAndRange('b', 10, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [LocatorAndRange('c', 10, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [LocatorAndRange('d', 10, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [LocatorAndRange('e', 10, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [LocatorAndRange('f', 10, 0, 2)])
self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), [])
self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [['a', 10, 9, 1], ['b', 10, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [['b', 10, 9, 1], ['c', 10, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [['c', 10, 9, 1], ['d', 10, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [['d', 10, 9, 1], ['e', 10, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [['e', 10, 9, 1], ['f', 10, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [['f', 10, 9, 1]])
-
-
- blocks3 = [['a', 10, 0],
- ['b', 10, 10],
- ['c', 10, 20],
- ['d', 10, 30],
- ['e', 10, 40],
- ['f', 10, 50],
- ['g', 10, 60]]
-
- self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [['a', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [['b', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [['c', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [['d', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [['e', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [['f', 10, 2, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [['g', 10, 2, 2]])
-
-
- blocks = [['a', 10, 0],
- ['b', 15, 10],
- ['c', 5, 25]]
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [LocatorAndRange('a', 10, 9, 1), LocatorAndRange('b', 10, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [LocatorAndRange('b', 10, 9, 1), LocatorAndRange('c', 10, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [LocatorAndRange('c', 10, 9, 1), LocatorAndRange('d', 10, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [LocatorAndRange('d', 10, 9, 1), LocatorAndRange('e', 10, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [LocatorAndRange('e', 10, 9, 1), LocatorAndRange('f', 10, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [LocatorAndRange('f', 10, 9, 1)])
+
+
+ blocks3 = [Range('a', 0, 10),
+ Range('b', 10, 10),
+ Range('c', 20, 10),
+ Range('d', 30, 10),
+ Range('e', 40, 10),
+ Range('f', 50, 10),
+ Range('g', 60, 10)]
+
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [LocatorAndRange('g', 10, 2, 2)])
+
+
+ blocks = [Range('a', 0, 10),
+ Range('b', 10, 15),
+ Range('c', 25, 5)]
self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10],
- ['b', 15, 0, 1]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9],
- ['b', 15, 0, 2]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10],
- ['b', 15, 0, 15]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10],
- ['b', 15, 0, 15],
- ['c', 5, 0, 5]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [['a', 10, 1, 9],
- ['b', 15, 0, 15],
- ['c', 5, 0, 5]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10],
- ['b', 15, 0, 15],
- ['c', 5, 0, 5]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2],
- ['b', 15, 0, 15]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2],
- ['b', 15, 0, 15],
- ['c', 5, 0, 3]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1],
- ['b', 15, 0, 14]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]])
- self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14],
- ['c', 5, 0, 1]])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [LocatorAndRange('a', 10, 0, 5)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [LocatorAndRange('a', 10, 3, 5)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [LocatorAndRange('a', 10, 0, 10)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 1)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [LocatorAndRange('a', 10, 1, 9),
+ LocatorAndRange('b', 15, 0, 2)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [LocatorAndRange('a', 10, 1, 9),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [LocatorAndRange('b', 15, 5, 5)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [LocatorAndRange('a', 10, 8, 2),
+ LocatorAndRange('b', 15, 0, 15)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [LocatorAndRange('a', 10, 8, 2),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 3)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [LocatorAndRange('c', 5, 1, 2)])
+
+ self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [LocatorAndRange('a', 10, 9, 1),
+ LocatorAndRange('b', 15, 0, 14)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [LocatorAndRange('b', 15, 0, 15)])
+ self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [LocatorAndRange('b', 15, 1, 14),
+ LocatorAndRange('c', 5, 0, 1)])
class MockKeep(object):
def __init__(self, content, num_retries=0):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list