[ARVADOS] updated: 6f6590fc776e42b3da8e09fbcda0b47c3c991227
Git user
git at public.curoverse.com
Tue Jul 26 10:05:43 EDT 2016
Summary of changes:
sdk/python/arvados/keep.py | 26 ++++++++++++++++----------
sdk/python/tests/run_test_server.py | 4 ++--
sdk/python/tests/test_keep_client.py | 16 ++++++++++++----
3 files changed, 30 insertions(+), 16 deletions(-)
via 6f6590fc776e42b3da8e09fbcda0b47c3c991227 (commit)
via 7fe7645d6ee146dd5e9b09daf44f1b63533c3f0d (commit)
via 47b03669f3bccebe70cbd6fd1d9d39013f80b60d (commit)
from 2aefffe61af4bb93659ca1cd9a9a027858658bec (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6f6590fc776e42b3da8e09fbcda0b47c3c991227
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Jul 26 11:03:57 2016 -0300
9551: Replaced ARVADOS_KEEP_PROXY with ARVADOS_KEEP_SERVICES new var on tests. Added proxy URI validation test.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 723d09d..db7835b 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -720,7 +720,7 @@ class KeepClient(object):
if not proxy_uris[i].endswith('/'):
proxy_uris[i] += '/'
# URL validation
- url = urlparse(proxy_uris[i])
+ url = urlparse.urlparse(proxy_uris[i])
if not (url.scheme and url.netloc):
raise arvados.errors.ArgumentError("Invalid proxy URI: {}".format(proxy_uris[i]))
self.api_token = api_token
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 45cc3ca..e72f67d 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -476,7 +476,7 @@ def run_keep_proxy():
'service_type': 'proxy',
'service_ssl_flag': False,
}}).execute()
- os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
+ os.environ["ARVADOS_KEEP_SERVICES"] = "http://localhost:{}".format(port)
_setport('keepproxy', port)
_wait_until_port_listens(port)
@@ -664,7 +664,7 @@ class TestCaseWithServers(unittest.TestCase):
cls._orig_environ = os.environ.copy()
cls._orig_config = arvados.config.settings().copy()
cls._cleanup_funcs = []
- os.environ.pop('ARVADOS_KEEP_PROXY', None)
+ os.environ.pop('ARVADOS_KEEP_SERVICES', None)
os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
for server_kwargs, start_func, stop_func in (
(cls.MAIN_SERVER, run, reset),
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 9258aa5..908539b 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -240,8 +240,8 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
super(KeepProxyTestCase, self).tearDown()
def test_KeepProxyTest1(self):
- # Will use ARVADOS_KEEP_PROXY environment variable that is set by
- # setUpClass().
+ # Will use ARVADOS_KEEP_SERVICES environment variable that
+ # is set by setUpClass().
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='')
baz_locator = keep_client.put('baz')
@@ -273,11 +273,19 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
def test_KeepProxyTestMultipleURIs(self):
# Test using ARVADOS_KEEP_SERVICES env var overriding any
# existing proxy setting and setting multiple proxies
- arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'foo bar baz'
+ arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'http://10.0.0.1 https://foo.example.org:1234/'
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='')
uris = [x['_service_root'] for x in keep_client._keep_services]
- self.assertEqual(uris, ['foo/', 'bar/', 'baz/'])
+ self.assertEqual(uris, ['http://10.0.0.1/',
+ 'https://foo.example.org:1234/'])
+
+ def test_KeepProxyTestInvalidURI(self):
+ arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'bad.uri.org'
+ with self.assertRaises(arvados.errors.ArgumentError):
+ keep_client = arvados.KeepClient(api_client=self.api_client,
+ local_store='')
+
class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
def get_service_roots(self, api_client):
commit 7fe7645d6ee146dd5e9b09daf44f1b63533c3f0d
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Jul 26 10:34:12 2016 -0300
9551: Proxy URLs validation. Iteration optimization builind _keep_services list
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 0edbc1c..723d09d 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -12,6 +12,7 @@ import ssl
import sys
import threading
import timer
+import urlparse
import arvados
import arvados.config as config
@@ -714,17 +715,21 @@ class KeepClient(object):
self.num_retries = num_retries
self.max_replicas_per_service = None
if proxy:
- proxy_uris = proxy.split(' ')
+ proxy_uris = proxy.split()
for i in range(len(proxy_uris)):
if not proxy_uris[i].endswith('/'):
proxy_uris[i] += '/'
+ # URL validation
+ url = urlparse(proxy_uris[i])
+ if not (url.scheme and url.netloc):
+ raise arvados.errors.ArgumentError("Invalid proxy URI: {}".format(proxy_uris[i]))
self.api_token = api_token
self._gateway_services = {}
self._keep_services = [{
- 'uuid': "00000-bi6l4-%015d" % proxy_uris.index(uri),
+ 'uuid': "00000-bi6l4-%015d" % idx,
'service_type': 'proxy',
'_service_root': uri,
- } for uri in proxy_uris]
+ } for idx, uri in enumerate(proxy_uris)]
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
commit 47b03669f3bccebe70cbd6fd1d9d39013f80b60d
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Jul 26 09:39:17 2016 -0300
9551: Change proxy parameter priority over environment variables. Updated docstring to mention ARVADOS_KEEP_SERVICES.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 5416379..0edbc1c 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -635,8 +635,9 @@ class KeepClient(object):
:proxy:
If specified, this KeepClient will send requests to this Keep
proxy. Otherwise, KeepClient will fall back to the setting of the
- ARVADOS_KEEP_PROXY configuration setting. If you want to ensure
- KeepClient does not use a proxy, pass in an empty string.
+ ARVADOS_KEEP_SERVICES or ARVADOS_KEEP_PROXY configuration settings.
+ If you want to KeepClient does not use a proxy, pass in an empty
+ string.
:timeout:
The initial timeout (in seconds) for HTTP requests to Keep
@@ -678,11 +679,11 @@ class KeepClient(object):
put() are called. Default 0.
"""
self.lock = threading.Lock()
- if config.get('ARVADOS_KEEP_SERVICES'):
- # ARVADOS_KEEP_SERVICES overrides proxy settings
- proxy = config.get('ARVADOS_KEEP_SERVICES')
- elif proxy is None:
- proxy = config.get('ARVADOS_KEEP_PROXY')
+ if proxy is None:
+ if config.get('ARVADOS_KEEP_SERVICES'):
+ proxy = config.get('ARVADOS_KEEP_SERVICES')
+ else:
+ proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
if api_client is None:
api_token = config.get('ARVADOS_API_TOKEN')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list