[ARVADOS] created: 17df259dd045c498d0e6db6a784654008be1378d
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 10 17:33:44 EST 2015
at 17df259dd045c498d0e6db6a784654008be1378d (commit)
commit 17df259dd045c498d0e6db6a784654008be1378d
Merge: 7354fa4 9aef039
Author: sguthrie <sallyeguthrie at gmail.com>
Date: Tue Nov 10 17:33:25 2015 -0500
Merge branch 'master' of git.curoverse.com:arvados into 7235-python-keep-client-timeout
commit 7354fa4e0e56bec396388054cd1e150a2935c0df
Author: sguthrie <sallyeguthrie at gmail.com>
Date: Tue Nov 10 15:23:18 2015 -0500
First edits to tests. Cannot get test suite to behave or be timely
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 8ed86fd..78f8cf2 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -225,11 +225,13 @@ class KeepBlockCache(object):
class KeepClient(object):
# Default Keep server connection timeout: 2 seconds
- # Default Keep server read timeout: 300 seconds
+ # Default Keep server read timeout: 64 seconds
+ # Default Keep server bandwidth minimum: 32 KiB per second
# Default Keep proxy connection timeout: 20 seconds
- # Default Keep proxy read timeout: 300 seconds
- DEFAULT_TIMEOUT = (2, 300)
- DEFAULT_PROXY_TIMEOUT = (20, 300)
+ # Default Keep proxy read timeout: 64 seconds
+ # Default Keep proxy bandwidth minimum: 32 KiB per second
+ DEFAULT_TIMEOUT = (2, 64, 32)
+ DEFAULT_PROXY_TIMEOUT = (20, 64, 32)
class ThreadLimiter(object):
"""
@@ -490,11 +492,13 @@ class KeepClient(object):
if not timeouts:
return
elif isinstance(timeouts, tuple):
- conn_t, xfer_t = timeouts
+ conn_t, xfer_t, bandwidth_kib_per_s = timeouts
else:
conn_t, xfer_t = (timeouts, timeouts)
+ bandwidth_kib_per_s = self.DEFAULT_TIMEOUT[2]
curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(conn_t*1000))
- curl.setopt(pycurl.TIMEOUT_MS, int(xfer_t*1000))
+ curl.setopt(pycurl.LOW_SPEED_TIME, int(xfer_t))
+ curl.setopt(pycurl.LOW_SPEED_LIMIT, int(bandwidth_kib_per_s*1024))
def _headerfunction(self, header_line):
header_line = header_line.decode('iso-8859-1')
@@ -598,20 +602,20 @@ class KeepClient(object):
:timeout:
The initial timeout (in seconds) for HTTP requests to Keep
- non-proxy servers. A tuple of two floats is interpreted as
- (connection_timeout, read_timeout): see
+ non-proxy servers. A tuple of three floats is interpreted as
+ (connection_timeout, read_timeout, minimum_bandwidth): see
http://docs.python-requests.org/en/latest/user/advanced/#timeouts.
Because timeouts are often a result of transient server load, the
actual connection timeout will be increased by a factor of two on
each retry.
- Default: (2, 300).
+ Default: (2, 64, 32).
:proxy_timeout:
The initial timeout (in seconds) for HTTP requests to
- Keep proxies. A tuple of two floats is interpreted as
- (connection_timeout, read_timeout). The behavior described
- above for adjusting connection timeouts on retry also applies.
- Default: (20, 300).
+ Keep proxies. A tuple of three floats is interpreted as
+ (connection_timeout, read_timeout, minimum_bandwidth). The behavior
+ described above for adjusting connection timeouts on retry also applies.
+ Default: (20, 64, 32).
:api_token:
If you're not using an API client, but only talking
@@ -698,7 +702,7 @@ class KeepClient(object):
# TODO(twp): the timeout should be a property of a
# KeepService, not a KeepClient. See #4488.
t = self.proxy_timeout if self.using_proxy else self.timeout
- return (t[0] * (1 << attempt_number), t[1])
+ return (t[0] * (1 << attempt_number), t[1], t[2])
def build_services_list(self, force_rebuild=False):
if (self._static_services_list or
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index ef724ed..a19eca9 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -22,7 +22,13 @@ class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
'response_body': 0,
# before returning from handler (thus setting response EOF)
'response_close': 0,
+ 'mid_write': 0,
+ 'mid_read': 0,
}
+ #If self.bandwidth = None, function at maximum bandwidth
+ #Otherwise, self.bandwidth is the maximum number of bytes per second to
+ # operate at.
+ self.bandwidth = None
super(Server, self).__init__(*args, **kwargs)
def setdelays(self, **kwargs):
@@ -31,6 +37,10 @@ class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
self.delays.get(k) # NameError if unknown key
self.delays[k] = v
+ def setbandwidth(self, bandwidth):
+ """For future requests, impose bandwidth limits."""
+ self.bandwidth = bandwidth*1024.0
+
def _sleep_at_least(self, seconds):
"""Sleep for given time, even if signals are received."""
wake = time.time() + seconds
@@ -40,10 +50,61 @@ class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
todo = wake - time.time()
def _do_delay(self, k):
+ #if self.delays[k] > 0:
+ # print "Delaying %f seconds for %s delay" % (self.delays[k], k)
self._sleep_at_least(self.delays[k])
class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+ def wfile_bandwidth_write(self, data_to_write):
+ BYTES_PER_WRITE = 10
+ if self.server.bandwidth == None and self.server.delays['mid_write'] == 0:
+ self.wfile.write(data_to_write)
+ else:
+ outage_happened = False
+ num_bytes = len(data_to_write)
+ num_sent_bytes = 0
+ while num_sent_bytes < num_bytes:
+ if num_sent_bytes > 1000 and not outage_happened:
+ self.server._do_delay('mid_write')
+ outage_happened = True
+ num_write_bytes = min(BYTES_PER_WRITE, num_bytes - num_sent_bytes)
+ if self.server.bandwidth == None:
+ wait = 0
+ else:
+ wait = num_write_bytes / self.server.bandwidth
+ #print "Bandwidth: %f KiB. Want to write total of %f bytes. Writing %f bytes now. Waiting %f seconds" % (self.server.bandwidth/1024.0, num_bytes, num_write_bytes, wait)
+ self.server._sleep_at_least(wait)
+ self.wfile.write(data_to_write[num_sent_bytes:num_sent_bytes+num_write_bytes])
+ num_sent_bytes += num_write_bytes
+ return None
+
+ def rfile_bandwidth_read(self, bytes_to_read):
+ BYTES_PER_READ = 10
+ if self.server.bandwidth == None and self.server.delays['mid_read'] == 0:
+ return self.rfile.read(bytes_to_read)
+ else:
+ data = ''
+ outage_happened = False
+ bytes_read = 0
+ while bytes_to_read > bytes_read:
+ if bytes_read > 1000 and not outage_happened:
+ self.server._do_delay('mid_read')
+ outage_happened = True
+ next_bytes_to_read = min(BYTES_PER_READ, bytes_to_read - bytes_read)
+ t0 = time.time()
+ data += self.rfile.read(next_bytes_to_read)
+ time_spent_getting_data = time.time() - t0
+ if self.server.bandwidth == None:
+ wait = 0
+ else:
+ wait = next_bytes_to_read / self.server.bandwidth - time_spent_getting_data
+ #print "Bandwidth: %f KiB. Wanted total of %f bytes. Now reading %f bytes. Have read %f bytes. Waiting %f seconds" % (self.server.bandwidth/1024.0, bytes_to_read, next_bytes_to_read, len(data), wait)
+ if wait > 0:
+ self.server._sleep_at_least(wait)
+ bytes_read += next_bytes_to_read
+ return data
+
def handle(self, *args, **kwargs):
self.server._do_delay('request')
return super(Handler, self).handle(*args, **kwargs)
@@ -60,7 +121,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
self.send_header('Content-type', 'application/octet-stream')
self.end_headers()
self.server._do_delay('response_body')
- self.wfile.write(self.server.store[datahash])
+ #print "Writing to client"
+ self.wfile_bandwidth_write(self.server.store[datahash])
self.server._do_delay('response_close')
def do_PUT(self):
@@ -70,11 +132,11 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
# 2.7 BaseHTTPRequestHandler was patched to support 100 Continue, but
# reading the actual code that ships in Debian it clearly is not, so we
# need to send the response on the socket directly.
-
- self.wfile.write("%s %d %s\r\n\r\n" %
+ #print "Writing continue"
+ self.wfile_bandwidth_write("%s %d %s\r\n\r\n" %
(self.protocol_version, 100, "Continue"))
-
- data = self.rfile.read(int(self.headers.getheader('content-length')))
+ #print "Reading input from client"
+ data = self.rfile_bandwidth_read(int(self.headers.getheader('content-length')))
datahash = hashlib.md5(data).hexdigest()
self.server.store[datahash] = data
self.server._do_delay('response')
@@ -82,7 +144,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
self.send_header('Content-type', 'text/plain')
self.end_headers()
self.server._do_delay('response_body')
- self.wfile.write(datahash + '+' + str(len(data)))
+ #print "Write output hash"
+ self.wfile_bandwidth_write(datahash + '+' + str(len(data)))
self.server._do_delay('response_close')
def log_request(self, *args, **kwargs):
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 9046892..a08398a 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -287,8 +287,11 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000))
self.assertEqual(
- mock.responses[0].getopt(pycurl.TIMEOUT_MS),
- int(arvados.KeepClient.DEFAULT_TIMEOUT[1]*1000))
+ mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
+ int(arvados.KeepClient.DEFAULT_TIMEOUT[1]))
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
+ int(arvados.KeepClient.DEFAULT_TIMEOUT[2]*1024))
def test_put_timeout(self):
api_client = self.mock_keep_services(count=1)
@@ -301,8 +304,11 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000))
self.assertEqual(
- mock.responses[0].getopt(pycurl.TIMEOUT_MS),
- int(arvados.KeepClient.DEFAULT_TIMEOUT[1]*1000))
+ mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
+ int(arvados.KeepClient.DEFAULT_TIMEOUT[1]))
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
+ int(arvados.KeepClient.DEFAULT_TIMEOUT[2]*1024))
def test_proxy_get_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
@@ -315,8 +321,11 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000))
self.assertEqual(
- mock.responses[0].getopt(pycurl.TIMEOUT_MS),
- int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]*1000))
+ mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
+ int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]))
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
+ int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2]*1024))
def test_proxy_put_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
@@ -329,8 +338,11 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000))
self.assertEqual(
- mock.responses[0].getopt(pycurl.TIMEOUT_MS),
- int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]*1000))
+ mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
+ int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]))
+ self.assertEqual(
+ mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
+ int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2]*1024))
def check_no_services_error(self, verb, exc_class):
api_client = mock.MagicMock(name='api_client')
@@ -540,8 +552,15 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
- DATA = 'x' * 2**10
-
+ DATA = 'x' * 2**12
+ #Needs to be above 11 to hit timeout time on lowest bandwidth
+ BANDWIDTH_LOW_LIM = 32
+ BANDWIDTH_DELTA = 1
+ TIMEOUT_TIME = 64
+ #Needs to be low to trigger bandwidth errors before we run out of data
+ TIMEOUT_DELTA = 1
+ #This is a variable determined by trial-and-error which probably depends on
+ # the implementation of keepstub.py
class assertTakesBetween(unittest.TestCase):
def __init__(self, tmin, tmax):
self.tmin = tmin
@@ -554,6 +573,16 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
self.assertGreater(time.time() - self.t0, self.tmin)
self.assertLess(time.time() - self.t0, self.tmax)
+ class assertTakesGreater(unittest.TestCase):
+ def __init__(self, tmin):
+ self.tmin = tmin
+
+ def __enter__(self):
+ self.t0 = time.time()
+
+ def __exit__(self, *args, **kwargs):
+ self.assertGreater(time.time() - self.t0, self.tmin)
+
def setUp(self):
sock = socket.socket()
sock.bind(('0.0.0.0', 0))
@@ -572,7 +601,7 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
def tearDown(self):
self.server.shutdown()
- def keepClient(self, timeouts=(0.1, 1.0)):
+ def keepClient(self, timeouts=(0.1, TIMEOUT_TIME, BANDWIDTH_LOW_LIM)):
return arvados.KeepClient(
api_client=self.api_client,
timeout=timeouts)
@@ -587,42 +616,131 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
)
with self.assertTakesBetween(0.1, 0.5):
with self.assertRaises(arvados.errors.KeepWriteError):
- self.keepClient((0.1, 1)).put(self.DATA, copies=1, num_retries=0)
-
- def test_timeout_slow_request(self):
- self.server.setdelays(request=0.2)
- self._test_200ms()
-
- def test_timeout_slow_response(self):
- self.server.setdelays(response=0.2)
- self._test_200ms()
-
- def test_timeout_slow_response_body(self):
- self.server.setdelays(response_body=0.2)
- self._test_200ms()
-
- def _test_200ms(self):
- """Connect should be t<100ms, request should be 200ms <= t < 300ms"""
-
- # Allow 100ms to connect, then 1s for response. Everything
- # should work, and everything should take at least 200ms to
- # return.
- kc = self.keepClient((.1, 1))
- with self.assertTakesBetween(.2, .3):
- loc = kc.put(self.DATA, copies=1, num_retries=0)
- with self.assertTakesBetween(.2, .3):
- self.assertEqual(self.DATA, kc.get(loc, num_retries=0))
-
- # Allow 1s to connect, then 100ms for response. Nothing should
- # work, and everything should take at least 100ms to return.
- kc = self.keepClient((1, .1))
- with self.assertTakesBetween(.1, .2):
- with self.assertRaises(arvados.errors.KeepReadError):
+ self.keepClient().put(self.DATA, copies=1, num_retries=0)
+
+ def test_low_bandwidth_no_delays_success(self):
+ self.server.setbandwidth(self.BANDWIDTH_LOW_LIM+self.BANDWIDTH_DELTA)
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.assertEqual(self.DATA, kc.get(loc, num_retries=0))
+
+ def test_too_low_bandwidth_no_delays_failure(self):
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setbandwidth(self.BANDWIDTH_LOW_LIM-self.BANDWIDTH_DELTA)
+ # Check that lessening bandwidth corresponds to failing
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
kc.get(loc, num_retries=0)
- with self.assertTakesBetween(.1, .2):
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepWriteError):
kc.put(self.DATA, copies=1, num_retries=0)
+ def test_low_bandwidth_with_server_response_delay_failure(self):
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
+ self.server.setdelays(response=self.TIMEOUT_DELTA)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
+
+ def test_low_bandwidth_with_server_mid_delay_failure(self):
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
+ self.server.setdelays(mid_write=self.TIMEOUT_DELTA, mid_read=self.TIMEOUT_DELTA)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
+
+ def test_med_bandwidth_with_distributed_delays_success(self):
+ safe_delay_time = self.TIMEOUT_TIME/2.0-self.TIMEOUT_DELTA
+ self.server.setbandwidth(2*self.BANDWIDTH_LOW_LIM)
+ self.server.setdelays(response=safe_delay_time,
+ mid_write=safe_delay_time, mid_read=safe_delay_time)
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.assertEqual(self.DATA, kc.get(loc, num_retries=0))
+
+ def test_med_bandwidth_with_delays_failure(self):
+ safe_delay_time = self.TIMEOUT_TIME/2.0 - self.TIMEOUT_DELTA
+ not_safe_delay_time = self.TIMEOUT_TIME/2.0 + self.TIMEOUT_DELTA
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setbandwidth(2*self.BANDWIDTH_LOW_LIM)
+ self.server.setdelays(response=safe_delay_time, mid_write=not_safe_delay_time, mid_read=not_safe_delay_time)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
+
+ def test_high_bandwidth_large_server_delay_failure(self):
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setbandwidth(62*self.BANDWIDTH_LOW_LIM)
+ self.server.setdelays(response=self.TIMEOUT_TIME-self.TIMEOUT_DELTA)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
+
+ def test_high_bandwidth_large_server_delay_success(self):
+ self.server.setbandwidth(63*self.BANDWIDTH_LOW_LIM)
+ kc = self.keepClient()
+ self.server.setdelays(response=self.TIMEOUT_TIME-self.TIMEOUT_DELTA)
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.assertEqual(self.DATA, kc.get(loc, num_retries=0))
+
+ def test_high_bandwidth_server_delay_success(self):
+ self.server.setbandwidth(63*self.BANDWIDTH_LOW_LIM)
+ kc = self.keepClient()
+ self.server.setdelays(response=self.TIMEOUT_TIME-self.TIMEOUT_DELTA*2,
+ mid_read=self.TIMEOUT_TIME-self.TIMEOUT_DELTA*2,
+ mid_write=self.TIMEOUT_TIME-self.TIMEOUT_DELTA*2)
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.assertEqual(self.DATA, kc.get(loc, num_retries=0))
+
+ def test_high_bandwidth_server_start_delay_failure(self):
+ #Check that if the response takes a little longer than expected,
+ # get and put fail
+ self.server.setbandwidth(63*self.BANDWIDTH_LOW_LIM)
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setdelays(response=self.TIMEOUT_TIME+self.TIMEOUT_DELTA,
+ mid_read=self.TIMEOUT_TIME-self.TIMEOUT_DELTA,
+ mid_write=self.TIMEOUT_TIME-self.TIMEOUT_DELTA)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
+
+ def test_high_bandwidth_server_mid_delay_failure(self):
+ #Check that if the server hiccups for too long, get and put fail
+ self.server.setbandwidth(63*self.BANDWIDTH_LOW_LIM)
+ kc = self.keepClient()
+ loc = kc.put(self.DATA, copies=1, num_retries=0)
+ self.server.setdelays(response=self.TIMEOUT_TIME-self.TIMEOUT_DELTA,
+ mid_read=self.TIMEOUT_TIME+self.TIMEOUT_DELTA,
+ mid_write=self.TIMEOUT_TIME+self.TIMEOUT_DELTA)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepReadError) as e:
+ kc.get(loc, num_retries=0)
+ with self.assertTakesGreater(self.TIMEOUT_TIME):
+ with self.assertRaises(arvados.errors.KeepWriteError):
+ kc.put(self.DATA, copies=1, num_retries=0)
class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
def mock_disks_and_gateways(self, disks=3, gateways=1):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list