[ARVADOS] updated: ce2bf31ba0831d9704f2edf82297f701b3b994df
git at public.curoverse.com
git at public.curoverse.com
Fri Oct 30 15:45:18 EDT 2015
Summary of changes:
README | 4 +-
apps/workbench/app/assets/javascripts/add_group.js | 44 ++
apps/workbench/app/models/group.rb | 4 +
apps/workbench/app/models/keep_disk.rb | 2 +-
apps/workbench/app/models/keep_service.rb | 2 +-
apps/workbench/app/models/link.rb | 4 +
apps/workbench/app/models/node.rb | 2 +-
apps/workbench/app/models/user.rb | 4 +
apps/workbench/app/models/virtual_machine.rb | 7 +-
.../_getting_started_popup.html.erb | 2 +-
.../app/views/users/_add_group_modal.html.erb | 27 ++
.../workbench/app/views/users/_show_admin.html.erb | 9 +-
.../test/integration/application_layout_test.rb | 16 +-
apps/workbench/test/integration/errors_test.rb | 12 -
apps/workbench/test/integration/users_test.rb | 22 +
.../test/integration/virtual_machines_test.rb | 14 -
doc/_config.yml | 3 +-
doc/_includes/_run_command_simple_example.liquid | 15 +-
doc/_includes/_ssh_addkey.liquid | 2 +-
.../_tutorial_bwa_sortsam_pipeline.liquid | 2 +-
doc/api/schema/Job.html.textile.liquid | 2 +-
doc/index.html.liquid | 2 +-
...onfigure-azure-blob-storage.html.textile.liquid | 62 +++
doc/install/install-keep-web.html.textile.liquid | 25 +-
doc/install/install-keepproxy.html.textile.liquid | 5 +-
doc/install/install-keepstore.html.textile.liquid | 63 ++-
doc/install/install-sso.html.textile.liquid | 3 +-
.../ssh-access-unix.html.textile.liquid | 2 +-
.../ssh-access-windows.html.textile.liquid | 6 +-
.../vm-login-with-webshell.html.textile.liquid | 2 +-
doc/user/index.html.textile.liquid | 4 +-
doc/user/topics/arv-docker.html.textile.liquid | 2 +-
...tutorial-pipeline-workbench.html.textile.liquid | 6 +-
sdk/cli/bin/crunch-job | 59 ++-
sdk/go/arvadosclient/arvadosclient_test.go | 2 +-
sdk/go/arvadostest/fixtures.go | 19 +-
sdk/go/arvadostest/run_servers.go | 22 +-
sdk/go/auth/auth.go | 12 +-
sdk/go/crunchrunner/crunchrunner.go | 356 +++++++++++++++++
sdk/go/crunchrunner/crunchrunner_test.go | 445 +++++++++++++++++++++
sdk/go/crunchrunner/upload.go | 217 ++++++++++
sdk/go/crunchrunner/upload_test.go | 140 +++++++
sdk/go/keepclient/collectionreader.go | 131 +++---
sdk/go/keepclient/collectionreader_test.go | 16 +-
sdk/go/keepclient/keepclient.go | 48 ++-
sdk/go/keepclient/keepclient_test.go | 68 +++-
sdk/go/keepclient/support.go | 73 ++--
sdk/python/arvados/__init__.py | 65 ++-
sdk/python/arvados/api.py | 11 +
sdk/python/arvados/commands/run.py | 4 +-
sdk/python/arvados/keep.py | 33 +-
sdk/python/tests/arvados_testutil.py | 4 +
sdk/python/tests/nginx.conf | 2 +-
sdk/python/tests/run_test_server.py | 43 +-
sdk/python/tests/test_api.py | 86 ++--
sdk/python/tests/test_keep_client.py | 201 ++++++----
sdk/python/tests/test_retry_job_helpers.py | 105 +++++
services/api/Gemfile | 2 +-
services/api/Gemfile.lock | 7 +-
services/api/config/application.yml.example | 4 +-
services/api/test/fixtures/collections.yml | 8 +-
services/arv-git-httpd/git_handler_test.go | 6 +-
services/crunchstat/crunchstat_test.go | 2 +-
services/datamanager/datamanager_test.go | 4 +-
services/datamanager/keep/keep.go | 2 +-
services/keep-web/.gitignore | 2 +-
services/keep-web/anonymous.go | 24 ++
services/keep-web/doc.go | 100 +++--
services/keep-web/handler.go | 46 ++-
services/keep-web/handler_test.go | 102 ++++-
services/keep-web/server_test.go | 57 +--
services/keepproxy/keepproxy.go | 22 +-
services/keepproxy/keepproxy_test.go | 129 +++++-
services/keepstore/pull_worker_integration_test.go | 2 +-
.../arvnodeman/computenode/dispatch/slurm.py | 14 +-
.../tests/test_computenode_dispatch_slurm.py | 9 +
tools/keep-rsync/keep-rsync.go | 58 +--
tools/keep-rsync/keep-rsync_test.go | 99 ++++-
78 files changed, 2698 insertions(+), 544 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/add_group.js
create mode 100644 apps/workbench/app/views/users/_add_group_modal.html.erb
create mode 100644 doc/install/configure-azure-blob-storage.html.textile.liquid
create mode 100644 sdk/go/crunchrunner/crunchrunner.go
create mode 100644 sdk/go/crunchrunner/crunchrunner_test.go
create mode 100644 sdk/go/crunchrunner/upload.go
create mode 100644 sdk/go/crunchrunner/upload_test.go
create mode 100644 sdk/python/tests/test_retry_job_helpers.py
create mode 100644 services/keep-web/anonymous.go
via ce2bf31ba0831d9704f2edf82297f701b3b994df (commit)
via ecc0d50421458b1ccd175b4e1e9fc539a98088ed (commit)
via 6fbf2a60cf4a41156cf97c98e36ff54ec76f8401 (commit)
via a4acb3ae95b2fc7f4b5f1e174c910a54cc6681da (commit)
via 87640a81e725d0246837994acbb3a696d14401c6 (commit)
via 0477296dadd43b7fff9d8cf282bfa183fca3e138 (commit)
via 4103db7eade3ce5e06889adf12264fa03980d9d7 (commit)
via 8454069c654cdf3fb26d129cd27a4fbfbe8dfa3c (commit)
via 6e76e3322d66f609dabcd34c98cba34bd739e089 (commit)
via e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98 (commit)
via f0fdf6e7bdfd252b22c6347c292ddd511bfa31eb (commit)
via a58956e93618463e01733acaa9e8d6ce0afbc5ab (commit)
via d70d9b256c5268f6aac6ef1952ac7d5381ee6e46 (commit)
via 6cef86f2d748be96695293b48afac61e2ae640df (commit)
via 585bd329a7aea0cad03b99431e14fc6ca919a76f (commit)
via 77460b2190e84df4178c25f014bbf136d559922e (commit)
via b90d3ea4986a1af59395f223e0320f07cc91c272 (commit)
via 0975ea0b42e84ff40f36461f93b90b26abebf04b (commit)
via 89c2982376a7275e873d203c949d13b40fbd3bc4 (commit)
via 3445ed729dbccd21878373e01477be79e74a8453 (commit)
via 0f668000c8cacec50804cb0019dbe7d7dc1d2b36 (commit)
via a8caa6e1b359893e99226a8b21f356eee6edd529 (commit)
via d954b40925d8315ede49e9a05c05c0ba24a74ba9 (commit)
via a62773375ef74bbdd6a77f5dfa2023e122e717db (commit)
via 405c8505ccf36f7a1cfec68f305fae2edc439fa0 (commit)
via c33fa89b10dad03c8dce3459358ff7fe4825aa99 (commit)
via f76d7ea1946e2ed67c8d64e13f49c885c5add83b (commit)
via 2667ae2edcb79651eb55d85e21914ddf6638bb09 (commit)
via a5d766f5b4596d9b7b4996d5ccce43f46d61799a (commit)
via 0bb6625febf6838867e105e3e7f66a485b639c76 (commit)
via c5db21f4d720e931ed7bff0c5da53caf0cafd2f4 (commit)
via 61c91fe02574ad751c2c957f6ca0d2e1e21105f9 (commit)
via 38f28f90703cce64f124e26d8f61e10073505b2d (commit)
via a88b81005608f38bb269693d961f29bad5a9eb35 (commit)
via 2c149e2e0a4d00831f111aedaaf47320100c3a00 (commit)
via f4bc86376bd9f0f4cd10aad9296763a089cecd2a (commit)
via 9290961beb18476f71aa0ae12478fe50493e8ad9 (commit)
via bea3b865873ab4769d9786137037680a8f280560 (commit)
via 49645de7263ac010f25af3d79e177c4962e59578 (commit)
via 50484a71693dd3458b8d2323a8c4e118f37121a9 (commit)
via 2b43b263f8cd5cbd931d884bfa961a7dba7a4b9a (commit)
via ef11ca4b86beb111acb6a99d4eadbc7b0a0d27ef (commit)
via 9379a6ccce1d9787ced772e82732f1d5de8bc756 (commit)
via d0f91b34eedec9af266ec4877a3005dd627ad38e (commit)
via 5b743caf9a5913c62ec1f525f4e95d1c31f76233 (commit)
via 3e4f591181a38d739d94cf9321f874c19937d199 (commit)
via 6e0145efb9929b5a854fb3461161460b0ac75fba (commit)
via 5c478bbd77d326e06a44729792b433764fb39b02 (commit)
via a93b0bd7195895129411789f0db00a62d8c7336f (commit)
via 916172f535ee28f608433d536a9ef7eb2d70df39 (commit)
via 77b2a1d82d032f7e29f9c477d32127a4435ec13e (commit)
via cf36ef35f234d1b35bb4212345f7cfc4d6e47578 (commit)
via 0fa7cd5bfdab924f316046f923aec282b699d8a1 (commit)
via dbf51c6ab20fab93c3f71910c13de9d4a0501bad (commit)
via 352b2fa89337c6381aee45d173f95e5c7c0653e0 (commit)
via 4e98b0adbda5d6525825e72f67d9542c47147d0a (commit)
via 7d887106d3eabb9844c4a687403a18581167a823 (commit)
via f5ab6191a96ce31defe0269b9bc739c6492c1a90 (commit)
via 92768ce858673678aa7924f83ad41e2a9f8dd678 (commit)
via 31eb1bdc31e1d030844a6fdc7f4ba4286ec79d4f (commit)
via a81ba64fcf67efcdb1323402612ca3fe5abf7b92 (commit)
via 34941aef29a05cb6b3653e1d53a3c80591fabb7b (commit)
via 8cbdec0e12dbdde43405a61e3259c16840233212 (commit)
via 7011a3022e1378fcd3897b42f841726459d2fa13 (commit)
via 38d8d205385181811235502dbad7c6e27fcea2a7 (commit)
via 5d3de212473f9fbf0e797c1fc03ec1dbf8f532ef (commit)
via 53b1b5bbe6d6ae007e8ce546ba2539e0c061e25a (commit)
via 036656d9d65ee62771d0b47b5b3407aece833c8b (commit)
via 1dd5b73a66b107d16ae4ccc0e97e5b4a18caef17 (commit)
via dc3915c4eafa0e7c69278de5df659b2b4338f2d2 (commit)
via 0d428c09204f37f9c276c31e54bdb7acc5c80c02 (commit)
via 5fb633ee021c613d99280e8958a6598602041011 (commit)
via 30bfda443a98efc1a717f35258e3c3ffd7369d7c (commit)
via 7600537f3f34ee88a76688dbb0e1d73723905fa7 (commit)
via 4df91a01ad793744d7370b1ad5837543ed76c242 (commit)
via 5a4cb9d3957905a02716761edb1be662edad0312 (commit)
via f15a9a1ad40f589d3c40f856a95e1d5556ee7ca1 (commit)
via 98d7f56eb8b518444c732b53f18b0390a40dc224 (commit)
via f0ea4324260fb4dc6df693d9548285bb64b3b69f (commit)
via 689f463bbd231fba8e32b6d46a963d0dacf0e509 (commit)
via 78068d0f47e28b8caae3a16fb5a5a5c4037a287b (commit)
via 4c2855c80c11b60be9e5c6af384a0a90fb66ae16 (commit)
via 5130d0237e8a0be9d0593f222181317e3c9ec57a (commit)
via b23dbeaa9a3900e6a5fe6ed2df53d191317f31ec (commit)
via 38625a4ac1665055951265c1373a0e38a36a125d (commit)
via 22c45aed63a8c497508b132ca62dac599e3af06f (commit)
via 5da8705c0633e8a79b00ceab5904eb4c632963d3 (commit)
via 5f93585b16f2b5ac9c2e86f8dea876b310cbcf57 (commit)
via da453701654115387e6b11189c6a830f24abf715 (commit)
via 9a2bed0f3878e1332f800987292f9ec4316bd9a7 (commit)
via 3d4717f6f48957d58d46fa0ecb4e65e3f49b1858 (commit)
via 3da10f0c0f5a0c0c91d49436a5995c890b03d228 (commit)
via 6b1b4d80445f0e03f89c46a167bebefe7bcf97c0 (commit)
via 329da35b297dc38f3f32198ed1c7e09bbace7c0e (commit)
via 66dd4459251971acceae41622e2e1221892a8872 (commit)
via b534fd13b3446067ddf6f8bd3c5b19c3463bc655 (commit)
via d379c467be58c66b2f1e7acafc97634b269a1542 (commit)
via 5f822d9281e5afea05b1c87e59740687c0ee6692 (commit)
via 6de9152319167bb32602501e02e18a89e333336d (commit)
via 17568511436d6721de01870fead8995796243ee8 (commit)
via 680d70e6df8e6bf30309b908eaacdf6476d37dc2 (commit)
via 6761445a08454ab023dd0c9603b36d61577ea9d5 (commit)
via 831b63c1b17650fb28112360c5e123d91c7d546b (commit)
via 08a15a1e6b8e6f2e44f18328c1f6dd25343cffc2 (commit)
via 9385f4e8b60658460d47a1051ef0e8b3be342cba (commit)
via f2691496d53aa6367d4b777b67881b05b50ed85f (commit)
via 25af384efacda9f3c552eebfee42e14b785e9a16 (commit)
via c31abb2aa1a1e4493ee1f3ed2d63163cf7b4cafb (commit)
via 0f74b22fb1b64f983debb78d7946eb26728fcc52 (commit)
via eb220f290840b6b28e1979972a3a672dbf17b117 (commit)
via e0f940ec72a9ed000185196f8d01419302b3cb59 (commit)
via a9e08c498ac8126026d1226f34913f87c8e590a0 (commit)
via 716d6859bda672865c1266818bbc9814cfa9e64e (commit)
via c9018ad3b7f67f2f15a6a6671bf271858926586e (commit)
via d26a444780cafcc06d20d42cef114f608e0e114d (commit)
via 7cc39786e4951c102a9d10052096db8b953d9e66 (commit)
via 43e2110902218494d1f0d69936bdb98456557248 (commit)
via 88be56a262ab073e6aa96bdf6f80464731562a33 (commit)
via 1e74db260a84317d58969b9b530d0d87a325da9c (commit)
via 89b0d40ef05354a2b298a96ec3fc7a879f7f5328 (commit)
via 1c4ba204361de15430236b84b10c9ac5c68665ae (commit)
from 0f535af4fbcb7a9f6ca1e25675b31cc9656d88f3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ce2bf31ba0831d9704f2edf82297f701b3b994df
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 30 15:44:43 2015 -0400
5824: Give an anonymous token to keepproxy, not an admin token.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 4514442..cdfd93a 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -405,10 +405,9 @@ def run_keep_proxy():
return
stop_keep_proxy()
- admin_token = auth_token('admin')
port = find_available_port()
env = os.environ.copy()
- env['ARVADOS_API_TOKEN'] = admin_token
+ env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
logf = open(_fifo2stderr('keepproxy'), 'w')
kp = subprocess.Popen(
['keepproxy',
@@ -419,7 +418,7 @@ def run_keep_proxy():
api = arvados.api(
version='v1',
host=os.environ['ARVADOS_API_HOST'],
- token=admin_token,
+ token=auth_token('admin'),
insecure=True)
for d in api.keep_services().list(
filters=[['service_type','=','proxy']]).execute()['items']:
commit ecc0d50421458b1ccd175b4e1e9fc539a98088ed
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 30 15:38:49 2015 -0400
5824: Send keepproxy and keepstore logs to stderr via fifo, just like nginx_access_log.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index cb824e8..4514442 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -139,6 +139,26 @@ def _wait_until_port_listens(port, timeout=10):
format(port, timeout),
file=sys.stderr)
+def _fifo2stderr(label):
+ """Create a fifo, and copy it to stderr, prepending label to each line.
+
+ Return value is the path to the new FIFO.
+
+ +label+ should contain only alphanumerics: it is also used as part
+ of the FIFO filename.
+ """
+ fifo = os.path.join(TEST_TMPDIR, label+'.fifo')
+ try:
+ os.remove(fifo)
+ except OSError as error:
+ if error.errno != errno.ENOENT:
+ raise
+ os.mkfifo(fifo, 0700)
+ subprocess.Popen(
+ ['sed', '-e', 's/^/['+label+'] /', fifo],
+ stdout=sys.stderr)
+ return fifo
+
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
@@ -307,9 +327,10 @@ def _start_keep(n, keep_args):
for arg, val in keep_args.iteritems():
keep_cmd.append("{}={}".format(arg, val))
- logf = open(os.path.join(TEST_TMPDIR, 'keep{}.log'.format(n)), 'a+')
+ logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
kp0 = subprocess.Popen(
keep_cmd, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+
with open(_pidfile('keep{}'.format(n)), 'w') as f:
f.write(str(kp0.pid))
@@ -388,11 +409,12 @@ def run_keep_proxy():
port = find_available_port()
env = os.environ.copy()
env['ARVADOS_API_TOKEN'] = admin_token
+ logf = open(_fifo2stderr('keepproxy'), 'w')
kp = subprocess.Popen(
['keepproxy',
'-pid='+_pidfile('keepproxy'),
'-listen=:{}'.format(port)],
- env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
api = arvados.api(
version='v1',
@@ -477,7 +499,7 @@ def run_nginx():
nginxconf['GITSSLPORT'] = find_available_port()
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
- nginxconf['ACCESSLOG'] = os.path.join(TEST_TMPDIR, 'nginx_access_log.fifo')
+ nginxconf['ACCESSLOG'] = _fifo2stderr('nginx_access_log')
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
@@ -490,22 +512,12 @@ def run_nginx():
env = os.environ.copy()
env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
- try:
- os.remove(nginxconf['ACCESSLOG'])
- except OSError as error:
- if error.errno != errno.ENOENT:
- raise
-
- os.mkfifo(nginxconf['ACCESSLOG'], 0700)
nginx = subprocess.Popen(
['nginx',
'-g', 'error_log stderr info;',
'-g', 'pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
- cat_access = subprocess.Popen(
- ['cat', nginxconf['ACCESSLOG']],
- stdout=sys.stderr)
_setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
_setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
_setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
commit 6fbf2a60cf4a41156cf97c98e36ff54ec76f8401
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 30 14:33:44 2015 -0400
5824: Enable anonymous use of keep-web in integration tests.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index f8f8b18..cb824e8 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -451,6 +451,7 @@ def run_keep_web():
env.pop('ARVADOS_API_TOKEN', None)
keepweb = subprocess.Popen(
['keep-web',
+ '-anonymous-token='+fixture('api_client_authorizations')['anonymous']['api_token'],
'-attachment-only-host=localhost:'+str(keepwebport),
'-address=:'+str(keepwebport)],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
commit a4acb3ae95b2fc7f4b5f1e174c910a54cc6681da
Merge: 0f535af 87640a8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Oct 30 14:30:13 2015 -0400
Merge branch '5824-keep-web' into 5824-keep-web-workbench
Conflicts:
sdk/python/tests/run_test_server.py
services/keepproxy/keepproxy_test.go
diff --cc sdk/python/tests/run_test_server.py
index 591b500,d325b4e..f8f8b18
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@@ -496,7 -471,9 +502,10 @@@ def run_nginx()
'-g', 'pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ cat_access = subprocess.Popen(
+ ['cat', nginxconf['ACCESSLOG']],
+ stdout=sys.stderr)
+ _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
_setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
_setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
diff --cc services/keepproxy/keepproxy_test.go
index f350e0b,2c75ec1..e4f09b4
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@@ -28,8 -30,12 +28,14 @@@ var _ = Suite(&ServerRequiredSuite{}
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
+ // Gocheck boilerplate
+ var _ = Suite(&NoKeepServerSuite{})
+
+ // Test with no keepserver to simulate errors
+ type NoKeepServerSuite struct{}
+
+var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
+
// Wait (up to 1 second) for keepproxy to listen on a port. This
// avoids a race condition where we hit a "connection refused" error
// because we start testing the proxy too soon.
@@@ -65,27 -83,80 +83,27 @@@ func (s *NoKeepServerSuite) TearDownSui
arvadostest.StopAPI()
}
-func setupProxyService() {
-
- client := &http.Client{Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
-
- var req *http.Request
- var err error
- if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
- panic(err.Error())
- }
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
-
- reader, writer := io.Pipe()
-
- req.Body = reader
-
- go func() {
- data := url.Values{}
- data.Set("keep_service", `{
- "service_host": "localhost",
- "service_port": 29950,
- "service_ssl_flag": false,
- "service_type": "proxy"
-}`)
-
- writer.Write([]byte(data.Encode()))
- writer.Close()
- }()
-
- var resp *http.Response
- if resp, err = client.Do(req); err != nil {
- panic(err.Error())
- }
- if resp.StatusCode != 200 {
- panic(resp.Status)
- }
-}
+func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
+ args = append([]string{"keepproxy"}, args...)
+ os.Args = append(args, "-listen=:0")
+ listener = nil
+ go main()
+ waitForListener()
-func runProxy(c *C, args []string, port int, bogusClientToken bool) *keepclient.KeepClient {
- if bogusClientToken {
- os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
- }
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc := keepclient.KeepClient{
- Arvados: &arv,
- Want_replicas: 2,
- Using_proxy: true,
- Client: &http.Client{},
- }
- locals := map[string]string{
- "proxy": fmt.Sprintf("http://localhost:%v", port),
- }
- writableLocals := map[string]string{
- "proxy": fmt.Sprintf("http://localhost:%v", port),
- }
- kc.SetServiceRoots(locals, writableLocals, nil)
- c.Check(kc.Using_proxy, Equals, true)
- c.Check(len(kc.LocalRoots()), Equals, 1)
- for _, root := range kc.LocalRoots() {
- c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
- }
- log.Print("keepclient created")
if bogusClientToken {
- arvadostest.ResetEnv()
+ arv.ApiToken = "bogus-token"
}
-
- {
- os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
- listener = nil
- go main()
+ kc := keepclient.New(&arv)
+ sr := map[string]string{
+ TestProxyUUID: "http://" + listener.Addr().String(),
}
+ kc.SetServiceRoots(sr, sr, sr)
+ kc.Arvados.External = true
+ kc.Using_proxy = true
- return kc
+ return &kc
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
@@@ -370,6 -500,87 +400,87 @@@ func (s *ServerRequiredSuite) TestGetIn
}
// GetIndex with invalid prefix
- _, err = kc.GetIndex("proxy", "xyz")
+ _, err = kc.GetIndex(TestProxyUUID, "xyz")
c.Assert((err != nil), Equals, true)
}
+
+ func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
+ kc := runProxy(c, []string{"keepproxy"}, 28852, false)
+ waitForListener()
+ defer closeListener()
+
+ // Put a test block
+ hash, rep, err := kc.PutB([]byte("foo"))
+ c.Check(err, Equals, nil)
+ c.Check(rep, Equals, 2)
+
+ for _, token := range []string{
+ "nosuchtoken",
+ "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
+ } {
+ // Change token to given bad token
+ kc.Arvados.ApiToken = token
+
+ // Ask should result in error
+ _, _, err = kc.Ask(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ := err.(keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, false)
+ c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
+
+ // Get should result in error
+ _, _, _, err = kc.Get(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ = err.(keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, false)
+ c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
+ }
+ }
+
+ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
+
+ // keepclient with no such keep server
+ kc := keepclient.New(&arv)
+ locals := map[string]string{
+ "proxy": "http://localhost:12345",
+ }
+ kc.SetServiceRoots(locals, nil, nil)
+
+ // Ask should result in temporary connection refused error
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+ _, _, err = kc.Ask(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ := err.(*keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, true)
+ c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
+
+ // Get should result in temporary connection refused error
+ _, _, _, err = kc.Get(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ = err.(*keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, true)
+ c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
+ }
+
+ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
+ kc := runProxy(c, []string{"keepproxy"}, 29999, false)
+ waitForListener()
+ defer closeListener()
+
+ // Ask should result in temporary connection refused error
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+ _, _, err := kc.Ask(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ := err.(*keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, true)
+ c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
+
+ // Get should result in temporary connection refused error
+ _, _, _, err = kc.Get(hash)
+ c.Check(err, NotNil)
+ errNotFound, _ = err.(*keepclient.ErrNotFound)
+ c.Check(errNotFound.Temporary(), Equals, true)
+ c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
+ }
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list