[ARVADOS] updated: d251327b574e981cc02ff04c80d609df5c9ec30d
git at public.curoverse.com
git at public.curoverse.com
Thu Jan 1 22:59:59 EST 2015
Summary of changes:
sdk/python/arvados/arvfile.py | 25 +++++++-------
sdk/python/arvados/collection.py | 44 +++++++++++++++++------
sdk/python/arvados/keep.py | 6 ++--
sdk/python/tests/arvados_testutil.py | 32 +++++++++++++----
sdk/python/tests/test_keep_client.py | 67 +++++++++++++++++++-----------------
5 files changed, 109 insertions(+), 65 deletions(-)
via d251327b574e981cc02ff04c80d609df5c9ec30d (commit)
from 20f65ff9a7c6c6f73b152d122b70fb44ea5a21a4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d251327b574e981cc02ff04c80d609df5c9ec30d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 1 23:01:27 2015 -0500
3198: Added Collection.rename (needs test). Fixing tests broken on account of
using a requests session object so mocking requests.get and requests.put
doesn't work any more.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 5c6916e..c46019a 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -357,18 +357,17 @@ class BlockManager(object):
self._prefetch_queue.put(locator)
class ArvadosFile(object):
- def __init__(self, parent, stream=[], segments=[], keep=None):
+ def __init__(self, parent, stream=[], segments=[]):
'''
stream: a list of Range objects representing a block stream
segments: a list of Range objects representing segments
'''
self.parent = parent
self._modified = True
- self._segments = []
+ self.segments = []
for s in segments:
self.add_segment(stream, s.range_start, s.range_size)
self._current_bblock = None
- self._keep = keep
def set_unmodified(self):
self._modified = False
@@ -378,7 +377,7 @@ class ArvadosFile(object):
def truncate(self, size):
new_segs = []
- for r in self._segments:
+ for r in self.segments:
range_end = r.range_start+r.range_size
if r.range_start >= size:
# segment is past the trucate size, all done
@@ -391,7 +390,7 @@ class ArvadosFile(object):
else:
new_segs.append(r)
- self._segments = new_segs
+ self.segments = new_segs
self._modified = True
def readfrom(self, offset, size, num_retries):
@@ -399,10 +398,10 @@ class ArvadosFile(object):
return ''
data = []
- for lr in locators_and_ranges(self._segments, offset, size + config.KEEP_BLOCK_SIZE):
+ for lr in locators_and_ranges(self.segments, offset, size + config.KEEP_BLOCK_SIZE):
self.parent._my_block_manager().block_prefetch(lr.locator)
- for lr in locators_and_ranges(self._segments, offset, size):
+ for lr in locators_and_ranges(self.segments, offset, size):
# TODO: if data is empty, wait on block get, otherwise only
# get more data if the block is already in the cache.
data.append(self.parent._my_block_manager().get_block(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size])
@@ -414,7 +413,7 @@ class ArvadosFile(object):
a previous buffered write). Re-pack the buffer block for efficiency
and to avoid leaking information.
'''
- segs = self._segments
+ segs = self.segments
# Sum up the segments to get the total bytes of the file referencing
# into the buffer block.
@@ -453,18 +452,18 @@ class ArvadosFile(object):
self._current_bblock = self.parent._my_block_manager().alloc_bufferblock()
self._current_bblock.append(data)
- replace_range(self._segments, offset, len(data), self._current_bblock.blockid, self._current_bblock.write_pointer - len(data))
+ replace_range(self.segments, offset, len(data), self._current_bblock.blockid, self._current_bblock.write_pointer - len(data))
def add_segment(self, blocks, pos, size):
self._modified = True
for lr in locators_and_ranges(blocks, pos, size):
- last = self._segments[-1] if self._segments else Range(0, 0, 0)
+ last = self.segments[-1] if self.segments else Range(0, 0, 0)
r = Range(lr.locator, last.range_start+last.range_size, lr.segment_size, lr.segment_offset)
- self._segments.append(r)
+ self.segments.append(r)
def size(self):
- if self._segments:
- n = self._segments[-1]
+ if self.segments:
+ n = self.segments[-1]
return n.range_start + n.range_size
else:
return 0
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index b07a643..ea18123 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -644,7 +644,7 @@ class Collection(CollectionBase):
def __init__(self, manifest_locator_or_text=None, parent=None, api_client=None,
keep_client=None, num_retries=0, block_manager=None):
- self._parent = parent
+ self.parent = parent
self._items = None
self._api_client = api_client
self._keep_client = keep_client
@@ -668,24 +668,24 @@ class Collection(CollectionBase):
def _my_api(self):
if self._api_client is None:
- if self._parent is not None:
- return self._parent._my_api()
+ if self.parent is not None:
+ return self.parent._my_api()
self._api_client = arvados.api('v1')
self._keep_client = None # Make a new one with the new api.
return self._api_client
def _my_keep(self):
if self._keep_client is None:
- if self._parent is not None:
- return self._parent._my_keep()
+ if self.parent is not None:
+ return self.parent._my_keep()
self._keep_client = KeepClient(api_client=self._my_api(),
num_retries=self.num_retries)
return self._keep_client
def _my_block_manager(self):
if self._block_manager is None:
- if self._parent is not None:
- return self._parent._my_block_manager()
+ if self.parent is not None:
+ return self.parent._my_block_manager()
self._block_manager = BlockManager(self._my_keep())
return self._block_manager
@@ -768,7 +768,7 @@ class Collection(CollectionBase):
self._block_manager.stop_threads()
@_populate_first
- def find(self, path, create=False):
+ def find(self, path, create=False, create_collection=False):
p = path.split("/")
if p[0] == '.':
del p[0]
@@ -779,7 +779,10 @@ class Collection(CollectionBase):
# item must be a file
if item is None and create:
# create new file
- item = ArvadosFile(self, keep=self._keep_client)
+ if create_collection:
+ item = Collection(parent=self, num_retries=self.num_retries)
+ else:
+ item = ArvadosFile(self)
self._items[p[0]] = item
return item
else:
@@ -933,6 +936,25 @@ class Collection(CollectionBase):
self._manifest_locator = self._api_response["uuid"]
self.set_unmodified()
+ @_populate_first
+ def rename(self, old, new):
+ old_path, old_fn = os.path.split(old)
+ old_col = self.find(path)
+ if old_col is None:
+ raise IOError((errno.ENOENT, "File not found"))
+ if not isinstance(old_p, Collection):
+ raise IOError((errno.ENOTDIR, "Parent in path is a file, not a directory"))
+ if old_fn in old_col:
+ new_path, new_fn = os.path.split(new)
+ new_col = self.find(new_path, create=True, create_collection=True)
+ if not isinstance(new_col, Collection):
+ raise IOError((errno.ENOTDIR, "Destination is a file, not a directory"))
+ ent = old_col[old_fn]
+ del old_col[old_fn]
+ ent.parent = new_col
+ new_col[new_fn] = ent
+ else:
+ raise IOError((errno.ENOENT, "File not found"))
def import_manifest(manifest_text, into_collection=None, api_client=None, keep=None, num_retries=None):
if into_collection is not None:
@@ -998,7 +1020,7 @@ def export_manifest(item, stream_name=".", portable_locators=False):
for k in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
v = item[k]
st = []
- for s in v._segments:
+ for s in v.segments:
loc = s.locator
if loc.startswith("bufferblock"):
loc = v.parent._my_block_manager()._bufferblocks[loc].locator()
@@ -1014,7 +1036,7 @@ def export_manifest(item, stream_name=".", portable_locators=False):
buf += export_manifest(item[k], stream_name=os.path.join(stream_name, k), portable_locators=portable_locators)
elif isinstance(item, ArvadosFile):
st = []
- for s in item._segments:
+ for s in item.segments:
loc = s.locator
if loc.startswith("bufferblock"):
loc = item._bufferblocks[loc].calculate_locator()
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 7080465..36ec56c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -404,7 +404,7 @@ class KeepClient(object):
def __init__(self, api_client=None, proxy=None,
timeout=DEFAULT_TIMEOUT, proxy_timeout=DEFAULT_PROXY_TIMEOUT,
api_token=None, local_store=None, block_cache=None,
- num_retries=0):
+ num_retries=0, session=None):
"""Initialize a new KeepClient.
Arguments:
@@ -462,7 +462,7 @@ class KeepClient(object):
self.put = self.local_store_put
else:
self.num_retries = num_retries
- self.session = requests.Session()
+ self.session = session if session is not None else requests.Session()
if proxy:
if not proxy.endswith('/'):
proxy += '/'
@@ -614,7 +614,7 @@ class KeepClient(object):
hint_roots = ['http://keep.{}.arvadosapi.com/'.format(hint[2:])
for hint in locator.hints if hint.startswith('K@')]
# Map root URLs their KeepService objects.
- roots_map = {root: self.KeepService(root) for root in hint_roots}
+ roots_map = {root: self.KeepService(root, self.session) for root in hint_roots}
blob = None
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index aa7e632..5d58ea8 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -39,13 +39,31 @@ def fake_requests_response(code, body, **headers):
r.raw = io.BytesIO(body)
return r
-def mock_get_responses(body, *codes, **headers):
- return mock.patch('requests.get', side_effect=(
- fake_requests_response(code, body, **headers) for code in codes))
-
-def mock_put_responses(body, *codes, **headers):
- return mock.patch('requests.put', side_effect=(
- fake_requests_response(code, body, **headers) for code in codes))
+class MockSession(object):
+ def __init__(self, body, codes, headers):
+ if isinstance(body, list):
+ self.body = body
+ else:
+ self.body = [body for c in codes]
+ self.codes = codes
+ self.headers = headers
+ self.n = -1
+
+ def get(self, url, **kwargs):
+ self.n += 1
+ return fake_requests_response(self.codes[self.n], self.body[self.n], **self.headers)
+
+ def put(self, url, **kwargs):
+ self.n += 1
+ return fake_requests_response(self.codes[self.n], self.body[self.n], **self.headers)
+
+# def mock_get_responses(body, *codes, **headers):
+# return mock.patch('requests.get', side_effect=(
+# fake_requests_response(code, body, **headers) for code in codes))
+
+# def mock_put_responses(body, *codes, **headers):
+# return mock.patch('requests.put', side_effect=(
+# fake_requests_response(code, body, **headers) for code in codes))
def mock_requestslib_responses(method, body, *codes, **headers):
return mock.patch(method, side_effect=(
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 982e4b4..108b74c 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -416,11 +416,18 @@ class KeepClientRetryTestMixin(object):
def setUp(self):
self.client_kwargs = {'proxy': self.PROXY_ADDR, 'local_store': ''}
+ self.mock_session = None
+
+ def mock_get_responses(self, body, *codes, **headers):
+ self.mock_session = tutil.MockSession(body, codes, headers)
+
+ def mock_put_responses(self, body, *codes, **headers):
+ self.mock_session = tutil.MockSession(body, codes, headers)
def new_client(self, **caller_kwargs):
kwargs = self.client_kwargs.copy()
kwargs.update(caller_kwargs)
- return arvados.KeepClient(**kwargs)
+ return arvados.KeepClient(session=self.mock_session, **kwargs)
def run_method(self, *args, **kwargs):
raise NotImplementedError("test subclasses must define run_method")
@@ -436,29 +443,29 @@ class KeepClientRetryTestMixin(object):
self.assertRaises(error_class, self.run_method, *args, **kwargs)
def test_immediate_success(self):
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 200):
- self.check_success()
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 200)
+ self.check_success()
def test_retry_then_success(self):
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200):
- self.check_success(num_retries=3)
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200)
+ self.check_success(num_retries=3)
def test_no_default_retry(self):
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200):
- self.check_exception()
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200)
+ self.check_exception()
def test_no_retry_after_permanent_error(self):
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 403, 200):
- self.check_exception(num_retries=3)
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 403, 200)
+ self.check_exception(num_retries=3)
def test_error_after_retries_exhausted(self):
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 500, 200):
- self.check_exception(num_retries=1)
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 500, 200)
+ self.check_exception(num_retries=1)
def test_num_retries_instance_fallback(self):
self.client_kwargs['num_retries'] = 3
- with self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200):
- self.check_success()
+ self.TEST_PATCHER(self.DEFAULT_EXPECT, 500, 200)
+ self.check_success()
@tutil.skip_sleep
@@ -466,15 +473,15 @@ class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_DATA
DEFAULT_EXCEPTION = arvados.errors.KeepReadError
HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K at xyzzy'
- TEST_PATCHER = staticmethod(tutil.mock_get_responses)
+ TEST_PATCHER = KeepClientRetryTestMixin.mock_get_responses
def run_method(self, locator=KeepClientRetryTestMixin.TEST_LOCATOR,
*args, **kwargs):
return self.new_client().get(locator, *args, **kwargs)
def test_specific_exception_when_not_found(self):
- with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 200):
- self.check_exception(arvados.errors.NotFoundError, num_retries=3)
+ self.mock_get_responses(self.DEFAULT_EXPECT, 404, 200)
+ self.check_exception(arvados.errors.NotFoundError, num_retries=3)
def test_general_exception_with_mixed_errors(self):
# get should raise a NotFoundError if no server returns the block,
@@ -482,16 +489,16 @@ class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
# This test rigs up 50/50 disagreement between two servers, and
# checks that it does not become a NotFoundError.
client = self.new_client()
- with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 500):
- with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
- client.get(self.HINTED_LOCATOR)
- self.assertNotIsInstance(
- exc_check.exception, arvados.errors.NotFoundError,
- "mixed errors raised NotFoundError")
+ self.mock_get_responses(self.DEFAULT_EXPECT, 404, 500)
+ with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
+ client.get(self.HINTED_LOCATOR)
+ self.assertNotIsInstance(
+ exc_check.exception, arvados.errors.NotFoundError,
+ "mixed errors raised NotFoundError")
def test_hint_server_can_succeed_without_retries(self):
- with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 200, 500):
- self.check_success(locator=self.HINTED_LOCATOR)
+ self.mock_get_responses(self.DEFAULT_EXPECT, 404, 200, 500)
+ self.check_success(locator=self.HINTED_LOCATOR)
def test_try_next_server_after_timeout(self):
side_effects = [
@@ -502,22 +509,20 @@ class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
self.check_success(locator=self.HINTED_LOCATOR)
def test_retry_data_with_wrong_checksum(self):
- side_effects = (tutil.fake_requests_response(200, s)
- for s in ['baddata', self.TEST_DATA])
- with mock.patch('requests.get', side_effect=side_effects):
- self.check_success(locator=self.HINTED_LOCATOR)
+ self.mock_get_responses(['baddata', self.TEST_DATA], 200, 200)
+ self.check_success(locator=self.HINTED_LOCATOR)
@tutil.skip_sleep
class KeepClientRetryPutTestCase(KeepClientRetryTestMixin, unittest.TestCase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_LOCATOR
DEFAULT_EXCEPTION = arvados.errors.KeepWriteError
- TEST_PATCHER = staticmethod(tutil.mock_put_responses)
+ TEST_PATCHER = KeepClientRetryTestMixin.mock_put_responses
def run_method(self, data=KeepClientRetryTestMixin.TEST_DATA,
copies=1, *args, **kwargs):
return self.new_client().put(data, copies, *args, **kwargs)
def test_do_not_send_multiple_copies_to_same_server(self):
- with tutil.mock_put_responses(self.DEFAULT_EXPECT, 200):
- self.check_exception(copies=2, num_retries=3)
+ self.mock_put_responses(self.DEFAULT_EXPECT, 200)
+ self.check_exception(copies=2, num_retries=3)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list