[ARVADOS] updated: c0c0d769cd812e87efa392649380dc5ba8a25cd4
git at public.curoverse.com
git at public.curoverse.com
Fri May 23 16:47:52 EDT 2014
Summary of changes:
apps/workbench/app/assets/javascripts/editable.js | 4 ++
.../app/assets/javascripts/pipeline_instances.js | 79 +++++++++++-----------
apps/workbench/app/assets/javascripts/selection.js | 2 +-
.../controllers/pipeline_instances_controller.rb | 6 +-
apps/workbench/app/helpers/application_helper.rb | 4 +-
.../pipeline_instances/_show_components.html.erb | 8 ++-
.../_show_components_editable.html.erb | 2 +-
.../views/pipeline_instances/_show_inputs.html.erb | 50 ++++++++++++++
.../views/pipeline_templates/_show_recent.html.erb | 39 +++++------
.../test/integration/pipeline_instances_test.rb | 56 +++++++++++++++
apps/workbench/test/integration/smoke_test.rb | 4 ++
apps/workbench/test/integration_helper.rb | 16 +++++
sdk/python/arvados/api.py | 5 +-
sdk/python/arvados/keep.py | 3 +-
sdk/python/test_keep_client.py | 72 ++++++++++++++++----
services/api/test/fixtures/collections.yml | 8 +--
services/api/test/fixtures/pipeline_templates.yml | 4 +-
.../keep/src/arvados.org/keepproxy/keepproxy.go | 10 +++
18 files changed, 283 insertions(+), 89 deletions(-)
create mode 100644 apps/workbench/app/views/pipeline_instances/_show_inputs.html.erb
create mode 100644 apps/workbench/test/integration/pipeline_instances_test.rb
via c0c0d769cd812e87efa392649380dc5ba8a25cd4 (commit)
via 293b52241445edf9eb86e0a0f9b63fccc3aa7fbd (commit)
via b06affa05ffbb121512ae179feb06c0bf90541bd (commit)
via 8570c81e79a9c00a314e2cf28a008b9aeae65037 (commit)
via 8815915612d1d8fef2001083bec2a24b3ed2eb3f (commit)
via 7331dca3c9f4a75e291b9b975f7cd570ad2b3f96 (commit)
via b668cb88f24085f92858b268120650c28d3a79af (commit)
via ec4e4a338f45ce39b4081f67c2b991c15fdf0fcd (commit)
via 14eddaec927be590682e226b9998c3e616c0b72e (commit)
via 216733543e5348ef8ef25ee5edbe6ed2dd35f5d3 (commit)
via 34ab7b208c7eb14ccae1e31831286acf9ea29487 (commit)
via 45d037864f655373dacd681d753ea3270f4ba997 (commit)
via 744ca00ccd1e7fda9d4210b18c1f4a734f3b2261 (commit)
via b28565c8aa08cbf70762fa69e49c5067fcb57e96 (commit)
via 140944b37fac0f631e8366222b2cbc5d5373ba8a (commit)
via e35cb0f48bbead47ad0c628c280bfc2cc32035ac (commit)
via d61f27791aa739dfc93c2f953236fffd4f0fcf6c (commit)
via 89096c06922b406f7157082de410e40960f5c73e (commit)
via 9d6ce80869e187a7c5a574ea5a5272bb89dd81ce (commit)
via e2dc7f5b9068e23822391bd3cf987b6f14e83ec0 (commit)
via 1797ee0dea063ff59361740ee512c84d086dacaf (commit)
from ed101805b04c70bcbf1b070dceba2aee72170377 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c0c0d769cd812e87efa392649380dc5ba8a25cd4
Merge: ed10180 293b522
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri May 23 16:47:45 2014 -0400
Merge branch 'master' into 2751-python-sdk-keep-proxy-support refs #2751
diff --cc sdk/python/arvados/api.py
index 30acdc4,30acdc4..699c319
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@@ -19,6 -19,6 +19,10 @@@ class CredentialsFromEnv(object)
from httplib import BadStatusLine
if 'headers' not in kwargs:
kwargs['headers'] = {}
++
++ if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
++ kwargs['headers']['X-External-Client'] = '1'
++
kwargs['headers']['Authorization'] = 'OAuth2 %s' % config.get('ARVADOS_API_TOKEN', 'ARVADOS_API_TOKEN_not_set')
try:
return self.orig_http_request(uri, **kwargs)
@@@ -89,4 -89,4 +93,3 @@@ def api(version=None, cache=True)
'arvados', apiVersion, http=http, discoveryServiceUrl=url)
http.cache = None
return services[version]
--
diff --cc sdk/python/arvados/keep.py
index 7618cff,e414d26..6561c70
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@@ -111,13 -111,7 +111,14 @@@ class KeepClient(object)
url = self.args['service_root'] + self.args['data_hash']
api_token = config.get('ARVADOS_API_TOKEN')
headers = {'Authorization': "OAuth2 %s" % api_token}
+
+ if self.args['using_proxy']:
+ # We're using a proxy, so tell the proxy how many copies we
+ # want it to store
+ headers['X-Keep-Desired-Replication'] = str(self.args['want_copies'])
+
try:
++ logging.debug("Uploading to {}".format(url))
resp, content = h.request(url.encode('utf-8'), 'PUT',
headers=headers,
body=self.args['data'])
@@@ -166,37 -147,17 +167,37 @@@
def shuffled_service_roots(self, hash):
if self.service_roots == None:
self.lock.acquire()
- try:
- keep_disks = arvados.api().keep_disks().list().execute()['items']
- roots = (("http%s://%s:%d/" %
- ('s' if f['service_ssl_flag'] else '',
- f['service_host'],
- f['service_port']))
- for f in keep_disks)
- self.service_roots = sorted(set(roots))
- logging.debug(str(self.service_roots))
- finally:
- self.lock.release()
+
+ # Override normal keep disk lookup with an explict proxy
+ # configuration.
+ keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
- if keep_proxy_env != None:
++ if keep_proxy_env != None and len(keep_proxy_env) > 0:
+ if keep_proxy_env[-1:] != '/':
+ keep_proxy_env += "/"
+ self.service_roots = [keep_proxy_env]
+ self.using_proxy = True
+ else:
+ try:
+ try:
+ keep_services = arvados.api().keep_services().accessible().execute()['items']
+ except:
+ keep_services = arvados.api().keep_disks().list().execute()['items']
+
+ if len(keep_services) == 0:
+ raise arvados.errors.NoKeepServersError()
+
+ if 'service_type' in keep_services[0] and keep_services[0]['service_type'] == 'proxy':
+ self.using_proxy = True
+
+ roots = (("http%s://%s:%d/" %
+ ('s' if f['service_ssl_flag'] else '',
+ f['service_host'],
+ f['service_port']))
+ for f in keep_services)
+ self.service_roots = sorted(set(roots))
+ logging.debug(str(self.service_roots))
+ finally:
+ self.lock.release()
# Build an ordering with which to query the Keep servers based on the
# contents of the hash.
diff --cc sdk/python/test_keep_client.py
index bed1d4e,aa79b0d..c1afa7a
--- a/sdk/python/test_keep_client.py
+++ b/sdk/python/test_keep_client.py
@@@ -15,8 -14,8 +15,12 @@@ class KeepTestCase(unittest.TestCase)
del os.environ['KEEP_LOCAL_STORE']
except KeyError:
pass
++
run_test_server.run()
run_test_server.run_keep()
++ arvados.keep.global_client_object = None
++ arvados.config._settings = None
++ run_test_server.authorize_with("admin")
@classmethod
def tearDownClass(cls):
@@@ -67,19 -65,3 +71,59 @@@
self.assertEqual(arvados.Keep.get(blob_locator),
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
+
- def test_KeepProxyTest(self):
++
++class KeepProxyTestCase(unittest.TestCase):
++ @classmethod
++ def setUpClass(cls):
++ super(KeepProxyTestCase, cls).setUpClass()
+ try:
- run_test_server.run_keep_proxy("admin")
-
- baz_locator = arvados.Keep.put('baz')
- self.assertEqual(baz_locator,
- '73feffa4b7f6bb68e44cf984c85f6e88+3',
- 'wrong md5 hash from Keep.put("foo"): ' + baz_locator)
- self.assertEqual(arvados.Keep.get(baz_locator),
- 'baz',
- 'wrong content from Keep.get(md5("baz"))')
-
- self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
- finally:
- run_test_server.stop_keep_proxy()
++ del os.environ['KEEP_LOCAL_STORE']
++ except KeyError:
++ pass
++ run_test_server.run()
++ run_test_server.run_keep()
++ arvados.keep.global_client_object = None
++ arvados.config._settings = None
++ run_test_server.run_keep_proxy("admin")
++
++ @classmethod
++ def tearDownClass(cls):
++ super(KeepProxyTestCase, cls).tearDownClass()
++ run_test_server.stop()
++ run_test_server.stop_keep()
++ run_test_server.stop_keep_proxy()
++ os.environ["ARVADOS_KEEP_PROXY"] = ""
++ os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
++
++ def test_KeepProxyTest1(self):
++ # Will use ARVADOS_KEEP_PROXY environment variable that is set by
++ # run_keep_proxy()
++
++ baz_locator = arvados.Keep.put('baz')
++ self.assertEqual(baz_locator,
++ '73feffa4b7f6bb68e44cf984c85f6e88+3',
++ 'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
++ self.assertEqual(arvados.Keep.get(baz_locator),
++ 'baz',
++ 'wrong content from Keep.get(md5("baz"))')
++
++ self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
++
++ def test_KeepProxyTest2(self):
++ os.environ["ARVADOS_KEEP_PROXY"] = ""
++ os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
++ arvados.config._settings = None
++
++ # Will send X-External-Client to server and get back the proxy from
++ # keep_services/accessible
++
++ baz_locator = arvados.Keep.put('baz2')
++ self.assertEqual(baz_locator,
++ '91f372a266fe2bf2823cb8ec7fda31ce+4',
++ 'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
++ self.assertEqual(arvados.Keep.get(baz_locator),
++ 'baz2',
++ 'wrong content from Keep.get(md5("baz2"))')
++
++ self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list