[ARVADOS] updated: 1.2.0-190-gb8addea7d
Git user
git at public.curoverse.com
Wed Oct 31 15:20:19 EDT 2018
Summary of changes:
sdk/python/arvados/collection.py | 25 ++++++++---------
sdk/python/arvados/keep.py | 14 +++++-----
sdk/python/tests/test_keep_client.py | 52 ++++++++++++++++++++++++++++++------
3 files changed, 63 insertions(+), 28 deletions(-)
via b8addea7d6c8c464a2743191561470332eeaba13 (commit)
via 335d735bae14dedddeab261bbbb06f03380b52e5 (commit)
via 195f50e456b9bdc5293573bb4224d18a5bd0df7d (commit)
from 4c669f3d78a990863c5c053f98f790e027da5a96 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b8addea7d6c8c464a2743191561470332eeaba13
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Oct 31 15:43:46 2018 -0300
14259: Iterate just once when scanning for remote blocks.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 55797bdfe..13301b297 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1037,18 +1037,19 @@ class RichCollectionBase(CollectionBase):
different subdirectories.
"""
- for filename in [f for f in self.keys() if isinstance(self[f], ArvadosFile)]:
- for s in self[filename].segments():
- if '+R' in s.locator:
- try:
- loc = remote_blocks[s.locator]
- except KeyError:
- loc = self._my_keep().refresh_signature(s.locator)
- remote_blocks[s.locator] = loc
- s.locator = loc
- self.set_committed(False)
- for dirname in [d for d in self.keys() if isinstance(self[d], RichCollectionBase)]:
- remote_blocks = self[dirname]._copy_remote_blocks(remote_blocks)
+ for item in self:
+ if isinstance(self[item], ArvadosFile):
+ for s in self[item].segments():
+ if '+R' in s.locator:
+ try:
+ loc = remote_blocks[s.locator]
+ except KeyError:
+ loc = self._my_keep().refresh_signature(s.locator)
+ remote_blocks[s.locator] = loc
+ s.locator = loc
+ self.set_committed(False)
+ elif isinstance(self[item], RichCollectionBase):
+ remote_blocks = self[item]._copy_remote_blocks(remote_blocks)
return remote_blocks
@synchronized
commit 335d735bae14dedddeab261bbbb06f03380b52e5
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Oct 31 15:16:22 2018 -0300
14259: Don't enforce bandwidth timeouts on keep client HEAD requests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 9618ee5ce..1b6376e9b 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -377,7 +377,7 @@ class KeepClient(object):
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
if method == "HEAD":
curl.setopt(pycurl.NOBODY, True)
- self._setcurltimeouts(curl, timeout)
+ self._setcurltimeouts(curl, timeout, method=="HEAD")
try:
curl.perform()
@@ -516,7 +516,7 @@ class KeepClient(object):
self.upload_counter.add(len(body))
return True
- def _setcurltimeouts(self, curl, timeouts):
+ def _setcurltimeouts(self, curl, timeouts, ignore_bandwidth=False):
if not timeouts:
return
elif isinstance(timeouts, tuple):
@@ -529,8 +529,9 @@ class KeepClient(object):
conn_t, xfer_t = (timeouts, timeouts)
bandwidth_bps = KeepClient.DEFAULT_TIMEOUT[2]
curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(conn_t*1000))
- curl.setopt(pycurl.LOW_SPEED_TIME, int(math.ceil(xfer_t)))
- curl.setopt(pycurl.LOW_SPEED_LIMIT, int(math.ceil(bandwidth_bps)))
+ if not ignore_bandwidth:
+ curl.setopt(pycurl.LOW_SPEED_TIME, int(math.ceil(xfer_t)))
+ curl.setopt(pycurl.LOW_SPEED_LIMIT, int(math.ceil(bandwidth_bps)))
def _headerfunction(self, header_line):
if isinstance(header_line, bytes):
@@ -1094,10 +1095,7 @@ class KeepClient(object):
# Always cache the result, then return it if we succeeded.
if loop.success():
- if method == "HEAD":
- return blob or True
- else:
- return blob
+ return blob
finally:
if slot is not None:
slot.set(blob)
commit 195f50e456b9bdc5293573bb4224d18a5bd0df7d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Oct 31 15:12:14 2018 -0300
14259: Test updates & additions related to keep client HEAD requests
* Test updates to confirm that when doing HEAD requests, no bandwidth timeout
are enforced.
* Test additions to prove that HEAD request responses don't get cached.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 192a21cd6..d6b3a2a12 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -412,10 +412,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000))
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
- int(arvados.KeepClient.DEFAULT_TIMEOUT[1]))
+ None)
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
- int(arvados.KeepClient.DEFAULT_TIMEOUT[2]))
+ None)
def test_proxy_get_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
@@ -446,10 +446,10 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000))
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
- int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]))
+ None)
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
- int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2]))
+ None)
def test_proxy_put_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
@@ -561,6 +561,43 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
@tutil.skip_sleep
+class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def setUp(self):
+ self.api_client = self.mock_keep_services(count=2)
+ self.keep_client = arvados.KeepClient(api_client=self.api_client)
+ self.data = b'xyzzy'
+ self.locator = '1271ed5ef305aadabc605b1609e24c52'
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_get_request_cache(self, get_mock):
+ with tutil.mock_keep_responses(self.data, 200, 200):
+ self.keep_client.get(self.locator)
+ self.keep_client.get(self.locator)
+ # Request already cached, don't require more than one request
+ get_mock.assert_called_once()
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_head_request_cache(self, get_mock):
+ with tutil.mock_keep_responses(self.data, 200, 200):
+ self.keep_client.head(self.locator)
+ self.keep_client.head(self.locator)
+ # Don't cache HEAD requests so that they're not confused with GET reqs
+ self.assertEqual(2, get_mock.call_count)
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_head_and_then_get_return_different_responses(self, get_mock):
+ head_resp = None
+ get_resp = None
+ get_mock.side_effect = ['first response', 'second response']
+ with tutil.mock_keep_responses(self.data, 200, 200):
+ head_resp = self.keep_client.head(self.locator)
+ get_resp = self.keep_client.get(self.locator)
+ self.assertEqual('first response', head_resp)
+ # First reponse was not cached because it was from a HEAD request.
+ self.assertNotEqual(head_resp, get_resp)
+
+
+ at tutil.skip_sleep
class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
def setUp(self):
self.api_client = self.mock_keep_services(count=2)
@@ -849,7 +886,7 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
loc = kc.put(self.DATA, copies=1, num_retries=0)
self.server.setbandwidth(0.5*self.BANDWIDTH_LOW_LIM)
with self.assertTakesGreater(self.TIMEOUT_TIME):
- with self.assertRaises(arvados.errors.KeepReadError) as e:
+ with self.assertRaises(arvados.errors.KeepReadError):
kc.get(loc, num_retries=0)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepWriteError):
@@ -861,14 +898,13 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
self.server.setdelays(response=self.TIMEOUT_TIME)
with self.assertTakesGreater(self.TIMEOUT_TIME):
- with self.assertRaises(arvados.errors.KeepReadError) as e:
+ with self.assertRaises(arvados.errors.KeepReadError):
kc.get(loc, num_retries=0)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepWriteError):
kc.put(self.DATA, copies=1, num_retries=0)
with self.assertTakesGreater(self.TIMEOUT_TIME):
- with self.assertRaises(arvados.errors.KeepReadError) as e:
- kc.head(loc, num_retries=0)
+ kc.head(loc, num_retries=0)
def test_low_bandwidth_with_server_mid_delay_failure(self):
kc = self.keepClient()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list