[ARVADOS] updated: 8eb65b21a7b7eea1f987d7fec9cb8ac65de2aeb2
git at public.curoverse.com
git at public.curoverse.com
Thu May 14 22:56:08 EDT 2015
Summary of changes:
sdk/python/arvados/keep.py | 24 ++++++++++++++++++------
sdk/python/tests/arvados_testutil.py | 4 +++-
sdk/python/tests/test_keep_client.py | 10 ++++++++++
3 files changed, 31 insertions(+), 7 deletions(-)
via 8eb65b21a7b7eea1f987d7fec9cb8ac65de2aeb2 (commit)
from 46b2b9208c6f20c54938ca2312fa56a7f3147472 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8eb65b21a7b7eea1f987d7fec9cb8ac65de2aeb2
Author: Radhika Chippada <radhika at curoverse.com>
Date: Thu May 14 22:55:25 2015 -0400
4717: use keep_services -> read_only flag in python sdk.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 48d115c..2846630 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -648,6 +648,10 @@ class KeepClient(object):
'uuid': 'proxy',
'_service_root': proxy,
}]
+ self._writable_services = [{
+ 'uuid': 'proxy',
+ '_service_root': proxy,
+ }]
self.using_proxy = True
self._static_services_list = True
else:
@@ -659,6 +663,7 @@ class KeepClient(object):
self.api_token = api_client.api_token
self._gateway_services = {}
self._keep_services = None
+ self._writable_services = None
self.using_proxy = None
self._static_services_list = False
@@ -711,6 +716,9 @@ class KeepClient(object):
self._keep_services = [
ks for ks in accessible
if ks.get('service_type') in ['disk', 'proxy']]
+ self._writable_services = [
+ ks for ks in accessible
+ if (ks.get('service_type') in ['disk', 'proxy']) and (True != ks.get('read_only'))]
_logger.debug(str(self._keep_services))
self.using_proxy = any(ks.get('service_type') == 'proxy'
@@ -725,7 +733,7 @@ class KeepClient(object):
"""
return hashlib.md5(data_hash + service_uuid[-15:]).hexdigest()
- def weighted_service_roots(self, locator, force_rebuild=False):
+ def weighted_service_roots(self, locator, force_rebuild=False, need_writable=False):
"""Return an array of Keep service endpoints, in the order in
which they should be probed when reading or writing data with
the given hash+hints.
@@ -750,21 +758,24 @@ class KeepClient(object):
# Sort the available local services by weight (heaviest first)
# for this locator, and return their service_roots (base URIs)
# in that order.
+ use_services = self._keep_services
+ if (need_writable == True):
+ use_services = self._writable_services
sorted_roots.extend([
svc['_service_root'] for svc in sorted(
- self._keep_services,
+ use_services,
reverse=True,
key=lambda svc: self._service_weight(locator.md5sum, svc['uuid']))])
_logger.debug("{}: {}".format(locator, sorted_roots))
return sorted_roots
- def map_new_services(self, roots_map, locator, force_rebuild, **headers):
+ def map_new_services(self, roots_map, locator, force_rebuild, need_writable, **headers):
# roots_map is a dictionary, mapping Keep service root strings
# to KeepService objects. Poll for Keep services, and add any
# new ones to roots_map. Return the current list of local
# root strings.
headers.setdefault('Authorization', "OAuth2 %s" % (self.api_token,))
- local_roots = self.weighted_service_roots(locator, force_rebuild)
+ local_roots = self.weighted_service_roots(locator, force_rebuild, need_writable)
for root in local_roots:
if root not in roots_map:
roots_map[root] = self.KeepService(
@@ -858,7 +869,8 @@ class KeepClient(object):
try:
sorted_roots = self.map_new_services(
roots_map, locator,
- force_rebuild=(tries_left < num_retries))
+ force_rebuild=(tries_left < num_retries),
+ need_writable=False)
except Exception as error:
loop.save_result(error)
continue
@@ -939,7 +951,7 @@ class KeepClient(object):
try:
local_roots = self.map_new_services(
roots_map, locator,
- force_rebuild=(tries_left < num_retries), **headers)
+ force_rebuild=(tries_left < num_retries), need_writable=True, **headers)
except Exception as error:
loop.save_result(error)
continue
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index d138e59..6e2a078 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -141,7 +141,8 @@ class ApiClientMock(object):
service_host=None,
service_port=None,
service_ssl_flag=False,
- additional_services=[]):
+ additional_services=[],
+ read_only=False):
if api_mock is None:
api_mock = self.api_client_mock()
body = {
@@ -153,6 +154,7 @@ class ApiClientMock(object):
'service_port': service_port or 65535-i,
'service_ssl_flag': service_ssl_flag,
'service_type': service_type,
+ 'read_only': read_only,
} for i in range(0, count)] + additional_services
}
self._mock_api_call(api_mock.keep_services().accessible, status, body)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 419f1ce..c44379b 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -463,6 +463,16 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
+ def test_proxy_put_with_no_writable_services(self):
+ data = 'test with no writable services'
+ data_loc = '{}+{}'.format(hashlib.md5(data).hexdigest(), len(data))
+ api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1)
+ with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
+ self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
+ keep_client = arvados.KeepClient(api_client=api_client)
+ keep_client.put(data)
+ self.assertEqual(True, ("no Keep services available" in str(exc_check.exception)))
+ self.assertEqual(0, len(exc_check.exception.request_errors()))
class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
DATA = 'x' * 2**10
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list