[ARVADOS] updated: 82780d422b0f8a4ee4e4df52673cc88e7bb936a5
git at public.curoverse.com
git at public.curoverse.com
Mon Jul 27 10:51:17 EDT 2015
Summary of changes:
.gitignore | 2 +
apps/workbench/Gemfile | 2 +-
apps/workbench/Gemfile.lock | 78 +-
.../app/assets/javascripts/select_modal.js | 5 +-
apps/workbench/app/assets/javascripts/users.js | 2 +-
.../app/controllers/actions_controller.rb | 3 +-
.../app/controllers/application_controller.rb | 1 +
.../app/controllers/projects_controller.rb | 20 +-
apps/workbench/app/controllers/users_controller.rb | 9 +-
.../app/controllers/virtual_machines_controller.rb | 10 +
.../app/views/application/_show_recent.html.erb | 16 +-
apps/workbench/app/views/layouts/body.html.erb | 4 +-
.../pipeline_instances/_running_component.html.erb | 3 +-
.../_show_components_running.html.erb | 12 +-
.../app/views/users/_manage_repositories.html.erb | 1 +
.../views/users/_manage_virtual_machines.html.erb | 21 +-
.../app/views/users/_setup_popup.html.erb | 7 +-
.../workbench/app/views/users/_show_admin.html.erb | 8 +-
.../app/views/virtual_machines/webshell.html.erb | 49 +
apps/workbench/config/application.default.yml | 13 +
apps/workbench/config/load_config.rb | 12 +-
apps/workbench/config/routes.rb | 1 +
apps/workbench/public/webshell/README | 3 +
apps/workbench/public/webshell/enabled.gif | Bin 0 -> 847 bytes
apps/workbench/public/webshell/keyboard.html | 62 +
apps/workbench/public/webshell/keyboard.png | Bin 0 -> 808 bytes
apps/workbench/public/webshell/shell_in_a_box.js | 4835 ++++++++++++++++++++
apps/workbench/public/webshell/styles.css | 272 ++
.../controllers/application_controller_test.rb | 16 +
.../controllers/collections_controller_test.rb | 83 +-
.../test/controllers/jobs_controller_test.rb | 4 +
.../test/controllers/projects_controller_test.rb | 117 +
.../test/controllers/users_controller_test.rb | 45 +
.../test/helpers/collections_helper_test.rb | 1 +
.../test/integration/anonymous_access_test.rb | 10 -
.../test/integration/application_layout_test.rb | 39 +
.../test/integration/collection_upload_test.rb | 7 -
.../workbench/test/integration/collections_test.rb | 45 -
apps/workbench/test/integration/projects_test.rb | 141 -
apps/workbench/test/integration/search_box_test.rb | 13 +-
.../test/integration/user_manage_account_test.rb | 2 +
apps/workbench/test/integration/users_test.rb | 56 +-
crunch_scripts/crunchutil/vwd.py | 6 +
crunch_scripts/run-command | 9 +-
doc/_config.yml | 4 +
doc/_includes/_arv_copy_expectations.liquid | 3 +
doc/_includes/_install_debian_key.liquid | 4 +
doc/_includes/_install_redhat_key.liquid | 6 +
doc/_includes/_note_python27_sc.liquid | 5 +
doc/_includes/_tutorial_expectations.liquid | 2 +-
doc/images/add-new-repository.png | Bin 0 -> 9575 bytes
doc/images/added-new-repository.png | Bin 0 -> 14210 bytes
doc/images/api-token-host.png | Bin 0 -> 25207 bytes
doc/images/repositories-panel.png | Bin 0 -> 32812 bytes
doc/images/vm-access-with-webshell.png | Bin 0 -> 61121 bytes
doc/install/install-api-server.html.textile.liquid | 261 +-
.../install-arv-git-httpd.html.textile.liquid | 38 +-
.../install-compute-node.html.textile.liquid | 44 +-
.../install-crunch-dispatch.html.textile.liquid | 34 +-
doc/install/install-keepdl.html.textile.liquid | 64 +
doc/install/install-keepproxy.html.textile.liquid | 18 +-
doc/install/install-keepstore.html.textile.liquid | 19 +-
...l-manual-prerequisites-ruby.html.textile.liquid | 51 +-
...nstall-manual-prerequisites.html.textile.liquid | 52 +-
.../install-shell-server.html.textile.liquid | 51 +-
doc/install/install-sso.html.textile.liquid | 51 +-
.../install-workbench-app.html.textile.liquid | 165 +-
doc/sdk/perl/index.html.textile.liquid | 40 +-
doc/sdk/python/sdk-python.html.textile.liquid | 61 +-
.../check-environment.html.textile.liquid | 2 +-
.../ssh-access-unix.html.textile.liquid | 2 +-
.../ssh-access-windows.html.textile.liquid | 2 +-
.../vm-login-with-webshell.html.textile.liquid | 19 +
doc/user/reference/api-tokens.html.textile.liquid | 4 +-
doc/user/topics/arv-copy.html.textile.liquid | 80 +
doc/user/topics/run-command.html.textile.liquid | 13 +
.../add-new-repository.html.textile.liquid | 42 +
.../tutorial-firstscript.html.textile.liquid | 5 +-
.../tutorial-keep-mount.html.textile.liquid | 22 +-
.../tutorial-submit-job.html.textile.liquid | 35 +-
docker/api/Dockerfile | 4 +-
docker/api/update-gitolite.rb | 11 +-
docker/base/Dockerfile | 14 +-
docker/build_tools/Makefile | 14 +-
docker/compute/Dockerfile | 6 +-
docker/doc/Dockerfile | 4 +-
docker/java-bwa-samtools/Dockerfile | 4 +-
docker/jobs/Dockerfile | 29 +-
docker/{base => jobs}/apt.arvados.org.list | 0
docker/keepproxy/Dockerfile | 4 +-
docker/mkimage-debootstrap.sh | 11 +-
docker/passenger/Dockerfile | 4 +-
docker/postgresql/Dockerfile | 2 +-
docker/shell/Dockerfile | 4 +-
docker/slurm/Dockerfile | 4 +-
docker/workbench/Dockerfile | 4 +-
sdk/cli/bin/crunch-job | 154 +-
sdk/cwl/.gitignore | 1 +
sdk/cwl/README.rst | 1 +
sdk/cwl/arvados_cwl/__init__.py | 295 ++
sdk/{python/bin/arv-ls => cwl/bin/cwl-runner} | 2 +-
sdk/cwl/gittaggers.py | 1 +
{services/fuse => sdk/cwl}/setup.py | 26 +-
sdk/go/arvadosclient/arvadosclient.go | 22 +-
sdk/go/arvadosclient/collectionreader.go | 24 -
sdk/go/arvadostest/fixtures.go | 19 +
sdk/go/auth/auth.go | 28 +-
sdk/go/blockdigest/blockdigest.go | 64 +-
sdk/go/blockdigest/blockdigest_test.go | 93 +-
sdk/go/blockdigest/testing.go | 16 +
sdk/go/httpserver/log.go | 11 +-
sdk/go/keepclient/collectionreader.go | 186 +
sdk/go/keepclient/collectionreader_test.go | 117 +
sdk/go/keepclient/hashcheck.go | 10 +-
sdk/go/keepclient/support.go | 1 -
sdk/go/logger/logger.go | 7 +-
sdk/go/logger/main/testlogger.go | 29 -
sdk/go/logger/util.go | 20 +
sdk/go/manifest/manifest.go | 149 +-
sdk/go/manifest/manifest_test.go | 131 +-
sdk/pam/arvados_pam.py | 100 +
sdk/pam/debian/arvados_pam | 10 +
sdk/pam/debian/shellinabox | 136 +
sdk/python/arvados/_ranges.py | 22 +-
sdk/python/arvados/api.py | 6 +-
sdk/python/arvados/arvfile.py | 399 +-
sdk/python/arvados/collection.py | 327 +-
sdk/python/arvados/commands/put.py | 5 +-
sdk/python/arvados/commands/run.py | 123 +-
sdk/python/arvados/commands/ws.py | 9 +-
sdk/python/arvados/events.py | 41 +-
sdk/python/arvados/keep.py | 4 +-
sdk/python/arvados/util.py | 36 +-
sdk/python/setup.py | 2 -
sdk/python/tests/manifest_examples.py | 21 +
.../python/tests/performance}/__init__.py | 0
.../tests/performance/performance_profiler.py | 49 +
sdk/python/tests/performance/test_a_sample.py | 15 +
sdk/python/tests/run_test_server.py | 44 +-
sdk/python/tests/test_arv_put.py | 16 +-
sdk/python/tests/test_arv_ws.py | 13 +
sdk/python/tests/test_arvfile.py | 178 +-
sdk/python/tests/test_benchmark_collections.py | 97 +
sdk/python/tests/test_collections.py | 38 +-
sdk/python/tests/test_util.py | 18 +-
sdk/python/tests/test_websockets.py | 98 +-
services/api/Gemfile | 2 +-
services/api/Gemfile.lock | 6 +-
services/api/Rakefile | 31 +
.../arvados/v1/repositories_controller.rb | 19 +-
.../arvados/v1/virtual_machines_controller.rb | 44 +-
.../app/controllers/user_sessions_controller.rb | 4 +-
services/api/app/mailers/user_notifier.rb | 2 +-
services/api/app/models/collection.rb | 10 +
services/api/app/models/node.rb | 14 +-
.../api/app/views/static/login_failure.html.erb | 2 +-
.../views/user_notifier/account_is_setup.text.erb | 7 +-
services/api/config/application.default.yml | 25 +
services/api/config/application.yml.example | 3 +
services/api/config/database.yml.sample | 3 +
services/api/config/initializers/load_config.rb | 16 +
.../api/config/initializers/omniauth.rb.example | 13 -
services/api/config/initializers/omniauth_init.rb | 19 +
services/api/db/structure.sql | 3 +-
services/api/script/crunch-dispatch.rb | 69 +-
services/api/test/fixtures/collections.yml | 2 +-
services/api/test/fixtures/links.yml | 43 +
services/api/test/fixtures/nodes.yml | 24 +
.../functional/arvados/v1/links_controller_test.rb | 2 +-
.../functional/arvados/v1/nodes_controller_test.rb | 9 +
.../arvados/v1/repositories_controller_test.rb | 11 +
.../functional/arvados/v1/users_controller_test.rb | 10 +-
.../arvados/v1/virtual_machines_controller_test.rb | 64 +
services/api/test/helpers/manifest_examples.rb | 2 +-
services/api/test/unit/collection_test.rb | 18 +-
services/api/test/unit/node_test.rb | 50 +
services/api/test/unit/user_notifier_test.rb | 8 +-
services/datamanager/collection/collection.go | 109 +-
services/datamanager/collection/collection_test.go | 123 +
services/datamanager/collection/testing.go | 60 +
services/datamanager/datamanager.go | 95 +-
services/datamanager/keep/keep.go | 90 +-
services/datamanager/loggerutil/loggerutil.go | 13 +-
services/datamanager/summary/canonical_string.go | 27 +
services/datamanager/summary/file.go | 120 +
services/datamanager/summary/pull_list.go | 194 +
services/datamanager/summary/pull_list_test.go | 279 ++
services/datamanager/summary/summary.go | 267 ++
services/datamanager/summary/summary_test.go | 220 +
services/fuse/README.rst | 4 +
services/fuse/arvados_fuse/__init__.py | 433 +-
services/fuse/arvados_fuse/fresh.py | 52 +-
services/fuse/arvados_fuse/fusedir.py | 457 +-
services/fuse/arvados_fuse/fusefile.py | 54 +-
services/fuse/bin/arv-mount | 23 +-
services/fuse/setup.py | 4 +-
services/fuse/tests/fstest.py | 133 +
services/fuse/tests/mount_test_base.py | 72 +
services/fuse/tests/{ => performance}/__init__.py | 0
.../fuse/tests/performance/performance_profiler.py | 1 +
.../performance/test_collection_performance.py | 477 ++
services/fuse/tests/prof.py | 17 +
services/fuse/tests/test_inodes.py | 5 +-
services/fuse/tests/test_mount.py | 803 +++-
services/keepdl/doc.go | 157 +
services/keepdl/handler.go | 125 +-
services/keepdl/handler_test.go | 209 +
services/keepdl/server_test.go | 55 +-
services/keepstore/bufferpool.go | 19 +
services/keepstore/handler_test.go | 3 +
services/keepstore/handlers.go | 75 +-
services/keepstore/keepstore.go | 12 +-
services/keepstore/keepstore_test.go | 37 -
services/keepstore/trash_worker.go | 13 +-
services/keepstore/trash_worker_test.go | 29 +
services/keepstore/volume_unix.go | 2 +-
services/keepstore/volume_unix_test.go | 20 +
217 files changed, 14043 insertions(+), 1957 deletions(-)
create mode 100644 apps/workbench/app/views/virtual_machines/webshell.html.erb
create mode 100644 apps/workbench/public/webshell/README
create mode 100644 apps/workbench/public/webshell/enabled.gif
create mode 100644 apps/workbench/public/webshell/keyboard.html
create mode 100644 apps/workbench/public/webshell/keyboard.png
create mode 100644 apps/workbench/public/webshell/shell_in_a_box.js
create mode 100644 apps/workbench/public/webshell/styles.css
create mode 100644 doc/_includes/_arv_copy_expectations.liquid
create mode 100644 doc/_includes/_install_debian_key.liquid
create mode 100644 doc/_includes/_install_redhat_key.liquid
create mode 100644 doc/_includes/_note_python27_sc.liquid
create mode 100644 doc/images/add-new-repository.png
create mode 100644 doc/images/added-new-repository.png
create mode 100644 doc/images/api-token-host.png
create mode 100644 doc/images/repositories-panel.png
create mode 100644 doc/images/vm-access-with-webshell.png
create mode 100644 doc/install/install-keepdl.html.textile.liquid
create mode 100644 doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
create mode 100644 doc/user/topics/arv-copy.html.textile.liquid
create mode 100644 doc/user/tutorials/add-new-repository.html.textile.liquid
copy docker/{base => jobs}/apt.arvados.org.list (100%)
create mode 120000 sdk/cwl/.gitignore
create mode 100644 sdk/cwl/README.rst
create mode 100644 sdk/cwl/arvados_cwl/__init__.py
copy sdk/{python/bin/arv-ls => cwl/bin/cwl-runner} (70%)
create mode 120000 sdk/cwl/gittaggers.py
copy {services/fuse => sdk/cwl}/setup.py (59%)
delete mode 100644 sdk/go/arvadosclient/collectionreader.go
create mode 100644 sdk/go/arvadostest/fixtures.go
create mode 100644 sdk/go/blockdigest/testing.go
create mode 100644 sdk/go/keepclient/collectionreader.go
create mode 100644 sdk/go/keepclient/collectionreader_test.go
delete mode 100644 sdk/go/logger/main/testlogger.go
create mode 100644 sdk/go/logger/util.go
create mode 100644 sdk/pam/arvados_pam.py
create mode 100644 sdk/pam/debian/arvados_pam
create mode 100644 sdk/pam/debian/shellinabox
create mode 100644 sdk/python/tests/manifest_examples.py
copy {services/fuse/tests => sdk/python/tests/performance}/__init__.py (100%)
create mode 100644 sdk/python/tests/performance/performance_profiler.py
create mode 100644 sdk/python/tests/performance/test_a_sample.py
create mode 100644 sdk/python/tests/test_arv_ws.py
create mode 100644 sdk/python/tests/test_benchmark_collections.py
delete mode 100644 services/api/config/initializers/omniauth.rb.example
create mode 100644 services/api/config/initializers/omniauth_init.rb
create mode 100644 services/datamanager/collection/collection_test.go
create mode 100644 services/datamanager/collection/testing.go
create mode 100644 services/datamanager/summary/canonical_string.go
create mode 100644 services/datamanager/summary/file.go
create mode 100644 services/datamanager/summary/pull_list.go
create mode 100644 services/datamanager/summary/pull_list_test.go
create mode 100644 services/datamanager/summary/summary.go
create mode 100644 services/datamanager/summary/summary_test.go
create mode 100644 services/fuse/tests/fstest.py
create mode 100644 services/fuse/tests/mount_test_base.py
copy services/fuse/tests/{ => performance}/__init__.py (100%)
create mode 120000 services/fuse/tests/performance/performance_profiler.py
create mode 100644 services/fuse/tests/performance/test_collection_performance.py
create mode 100644 services/fuse/tests/prof.py
create mode 100644 services/keepdl/doc.go
create mode 100644 services/keepdl/handler_test.go
discards e3d48bb942c3ccea6f9c7a2ab0c5db5233e7b4a7 (commit)
discards 40ec18b8fc06ee9c37e88be8c55e9c46af546efd (commit)
discards e48e9554fb8b90ef03c037ea6f0135160258997d (commit)
discards 1ce1b0a4e586bb8d665f63c05a11de1332ac2ba3 (commit)
discards e7edfc19e29bb61aa077b76b6857653e2eaaf5d3 (commit)
discards 7b8223b2b7ec8ba611bfc5250d3183cf297a0ebe (commit)
discards 65d773097bc4b67c1a3eb0f4b17f0b1659a4b1c7 (commit)
discards b11f4cdc04da273482b68b1e6e44156a0bb05771 (commit)
discards 489911981539dcbde1e6b37264cc2a952316f114 (commit)
via 82780d422b0f8a4ee4e4df52673cc88e7bb936a5 (commit)
via 9a02abe887e3c09115a000f9f63666db9fe96172 (commit)
via 18277495a1593a0a9a7d67b39f5fb47c62979f2c (commit)
via cddb2c4abda1607ff36f248ddf9a291191114598 (commit)
via 6c011d450d6e320892989cc37605f7cf67dc3034 (commit)
via 1f9ebffd675488ef594a3ebc81f4c6eb63da7887 (commit)
via 5269624aeb5cd29dcb8b488bf8a297fdb6c12e0e (commit)
via f3ae4368efc15797d1d9c6179ca4ba460bdd5da0 (commit)
via d40945c358950d9e43e4ad0aa1ec9fd02353090d (commit)
via bed6a3d54844b6ba41f5f7803f3b289783e02e5f (commit)
via 2534cab13807ee2614400365b1cb6a4649c6678e (commit)
via 12c47d0c0dc38a9e1d1e5a0e953a226a1a0557c6 (commit)
via 847293a0e90fed989b9dea9a99f00126415530b3 (commit)
via ad05e4948fab822910fbf57f60b739f100a5cdb1 (commit)
via d9434ed5ae6f129227a74cc85dc15fa6bdf199ac (commit)
via 8b8eb200a3e8d52f1fb98142771412447bb2911e (commit)
via 99146504cd618769de59176ee7458c8973877242 (commit)
via ca8e6a099f43d0b227fd1983dabbdaa6cbfb7246 (commit)
via e024bcf310e61819a75e1ef3e45cf99b6457cfb0 (commit)
via cb185aea60905a237d5a0729d1240ce4c961d7f4 (commit)
via b92203411f6f6adaef1c2af62495830f13f4fa14 (commit)
via dd18605e7fe7c537dc84a3e3174d18ec0320c95a (commit)
via 71a3acee61652424e7d556a3bdf22f76ad0a2f5d (commit)
via 3bfb7144d19cc75bb99e93bc3681cd2b9e4733fe (commit)
via 0964a1c6e5da3b1dbf83c361e767f6db36cc0c7c (commit)
via 4b8ecdc4d88c9dccbbcf07b5037ba4dcf4ea2d20 (commit)
via fe0f4c45a3e3bf2c28f389dce064aafd92a30b0b (commit)
via fe0e18020d6e817893417220df2afe3683393591 (commit)
via f46a1e9489373ce076c71f7f52609a4e4a9c050a (commit)
via 11990115642928c703077e7089a4415b36a363b9 (commit)
via 12e199bee6aab07b730bf0264f91394154986072 (commit)
via 35157cf6e2d6d423eff031e0d952b5b45bf07383 (commit)
via 9844eb7f071daea026b617ceb2eac79bb595e9e8 (commit)
via 6995d15780c0e46c02acc23481de7bed906d152f (commit)
via 151064a2cbd152bb81ab1f8fdf102b36bc13ca2a (commit)
via 9b2bf1dddd21c991623e4b2b8b412e1f3fb05d34 (commit)
via 68b018141677e7147fe2b52a88561dc7d57e6d79 (commit)
via 111c659adefb766d3773e48ae839515f8007f67c (commit)
via 6d1fb016135e6126c1d5c17ca02f210ba98dfc13 (commit)
via b97c921a3bc9ce6186a80dd4157c6132321e6374 (commit)
via 91a77b7ce2abca7d4297b2dfa5eca54fc78a1425 (commit)
via 1794be475e5bb94d8dd6764329731bacc2a67f2b (commit)
via 8c1a3773b9ae07e8f37e3567a9e7c225e47d2ea5 (commit)
via 9e4957c362c398c4276833075f54b19fa4050041 (commit)
via 8c8b6b67c412633c762af1769c71af56c2310f5c (commit)
via 0bb9d575d4c9609693d71fa47f8958af10f40d2b (commit)
via e6e3409439ef6d72a701e409b1c079900e006913 (commit)
via 847f47a1a86d40dc6ae5d13f62039e55d1afa36d (commit)
via d591acbac423d324cf00e5930851fff6957a19d9 (commit)
via 2b41829bf0a889558c320121710ef3fd2e90ef7e (commit)
via 623be7f83c1a88dd2eeae5ee6028daac47ad0e1c (commit)
via 106df13f4470b75e6fb4114c94b32b6fc68736f4 (commit)
via 7879727a44d761e633f7282fad6073549495be8a (commit)
via 17e8c59deca04137f66ece7837f96599ff82f4de (commit)
via ddb83a220cd6f3b62171c3c374c58e553d82a5d6 (commit)
via c4c8e9b0350aedf985dd99168ce6d3b5afb60acb (commit)
via 6a2f46b152dd122df40012a487b4f4c2f5e37197 (commit)
via cd25f265349e86fab2fbe81959747ce112986806 (commit)
via 2ad8f9ccb5d02e76437e48f2193fbacc4a9ca2ad (commit)
via 6e5e1c725a10ea791f00d073003175ac3c29a6c7 (commit)
via b23101649f5320c7f4f5c68f3d7745373e316249 (commit)
via b08e97355a492d9b4710721b5105e981e49e8238 (commit)
via 61d432f332dd9314f8c4cd57c2393e80e8ecef59 (commit)
via a1ed06fc35101b054efa62a757c92c3c4d14ff06 (commit)
via ee29db4af2c800924fa06b7bbc1f3058ba3cfb94 (commit)
via 70e832c1c5976614c4da5c2987ecba7217c3e83a (commit)
via e0ab53eaacc78f74e3f446f2cd5f58b8add29898 (commit)
via f7c152d3ca9a753812bd7729ceb6cb8a23b12fce (commit)
via 0b19163c8633defab5988f8b775833d3358ebaaf (commit)
via 4a35a164fcddb66e04df2afe4686783fd3b35510 (commit)
via 37e6d6e690446496a43ea7d3da48cb145c2e9628 (commit)
via 7253b0d4cdf9612a4c0de0ca849979cec5e8d382 (commit)
via edb109c244d25fb9a963a2146953710fd4d77099 (commit)
via caa9e89d531f03838e64d36d050be9d96e7c6c96 (commit)
via 1d5e009a31e0b462b7bfcf4c0e5dc36db423c90f (commit)
via 78e3248417510aecbac9e66a22d48e32a80181bf (commit)
via 61187968f0e9268430b967ce37a79d9384ec7e3c (commit)
via 77aeb6fdedab3d2aac25120e5a99317155c9f26e (commit)
via 94afae23dd927b2d9d6c3472f90bc00f0310bbf5 (commit)
via a1e6ebc7e1afc04e8a1de0c0bb8e304eda6ecf2c (commit)
via 84698df0ff6dcd834c080840f5d7e95a1ce086a9 (commit)
via e621ee479da9664e912a6e74f24342877533872c (commit)
via 1c6515a4445e9fd79961c9c8caf21528f16d9399 (commit)
via 706f34d335cecb9f321dda8abaf8718fa951e908 (commit)
via 9ac57b0bc6cb5d90da57c943df489401c63b7a7f (commit)
via 210883bcb2428ad0262a4b064c56612de0a22ef6 (commit)
via 4999699d7614f214eb8718b85c5c30e9fb382c23 (commit)
via acacdd86095218aa2bb4cc4cd564d9d7f135da3c (commit)
via 85b29e4db84f53e390c7964f43caf91d3a764510 (commit)
via 702ce6c3da9a1b09a2ed546fd3da775d21bd703b (commit)
via fb9730d1da1eab233e4e7ea01c1015cd70ba6cf7 (commit)
via db2584826ed9aadb162b8bd6d25e164565bbab8b (commit)
via 6ba447b83ef96f0f52db5eebd04fd22e5a0e1c74 (commit)
via 12ca84522ca29c237c477ab1974299d04715b09d (commit)
via 1856a3a1e9c95b4db4742ab53f737e91dbf46cff (commit)
via 25e646a708d1d91aebcf8db80b8ae1fafa044034 (commit)
via 041af47977925c319ad3b6a809089eb64ffdd738 (commit)
via 22f68d307a3229fbf842fba0f61ea0ae0330832b (commit)
via a45c162ac02bb261fc65d3d59b446f0610c3ab8f (commit)
via ca8e8224203ba5be3a9a190862e7c8ac2ac94e7e (commit)
via 7748c0b5eb966b8d05eb2156b4445f552ed3839d (commit)
via 7365909fd21f8016e7b676cfca2a1ad28781e690 (commit)
via 8e1477dea24ed50b09c055092314fb6522c5a114 (commit)
via df070cb903ec46ce51eb610d44530369a824b12b (commit)
via c790738ae71771a5574b166b3e93a1bca9b89bf6 (commit)
via ed8897b942f147afccf8eeac1025861ffe2f1690 (commit)
via 97374cec874aaaaeb92eeb962bf580bdba199be9 (commit)
via 318e3d183c3800863731a20a10f1b8bf9cc82280 (commit)
via 14dbdda455cb0e49b8848575337c5e7806747ee7 (commit)
via c3a7bb61e982ff5b0747204b79c4ca759c19b537 (commit)
via 46da2daa12366c10d7e175de8c46d964a2e06aac (commit)
via 0d9da683cb9572f6b5ba3f65376066938e701fb4 (commit)
via 66c19e11db2626bd82eb755ea6552ce5caec69af (commit)
via 1d82713672c3f6304b8f5d7d014ee39fa15bc579 (commit)
via 2c3c9f64426e825295aeb1f4265d67429ee14cf6 (commit)
via e59b42354a10079ecd579a1dbe53c39a20d05313 (commit)
via beab0d6bf936becea2a92c6778c2008d451db0fc (commit)
via 0628f0c1d83fc71d4b0913f4f3fb90e4ad1632f2 (commit)
via 421c879e077ce8f644553ba3a1481cb55529ee33 (commit)
via 39c75ea686e2326508fd8e3d0be31cdde7906597 (commit)
via 8b8673b66d593742deb718ab5933fa2c2a1d8672 (commit)
via 8f589475096eb42dd1eccfbfbfc1fd5bc8f4e8ba (commit)
via 7adf48e3633942e40b3943db8a7a31ec23d12a5b (commit)
via d5dbaeb59cb702793b926a63cabbeaa37f96dcfd (commit)
via 66380d0e89c00559123ceda8e74e3b1487f4a95a (commit)
via 603992242f91426818fd56317c6adf4521f9500c (commit)
via 9053b511d514aa3e902259d3070ba439bfae6613 (commit)
via 5e27876fa4d3faf3b973282bfb4f152c02345bdc (commit)
via f78434fcb802949eaae131adf625950ad9981ede (commit)
via 0e57453d2b637a3d105d4e3d67031f3915f9d302 (commit)
via 5d17dfc6124a19ff7c9ebce607699d3e3f415bad (commit)
via bb8a8e8ab7dc201dccf8f2f3cf243e63ba8d14cd (commit)
via 4a5c16b70d01e063ae8fb82ee576a542b3a2376f (commit)
via 9d52ea83290cab293229815286579ab6a1584f9e (commit)
via 1d8068392d44ae35969362fb77d85e6270ff9c27 (commit)
via 9007e6691362f389e2fc282a63233562bffb8b05 (commit)
via 726cc580bb8901fb97021421cc71e3a55b37aeac (commit)
via ad2375606991dedb5cd5a574641b9cc57245539d (commit)
via ee42981c6282567f787e33523a1bcf805d7d178a (commit)
via 7e5b8ba9e260669bc6fd85c201c4f771bceaa1b0 (commit)
via 5106490f8cb4d3e6aaa8da2ae46283c1ef64a027 (commit)
via 7bd12f51c864ec78fc93f0fa95acf796a4999afc (commit)
via 6196d3280c04dabf2f347a55ed5d034e6bf5aa39 (commit)
via becec5efdb1a5e031f20d30393dcccf87232118d (commit)
via 0581bd67385a585beb2d6ee5392ce7cf9d526873 (commit)
via e3ab22d08738c5aedb3e021e47959c1548f62ead (commit)
via 5b311db10a1f745d2b7018f4fd1ce462550f1bc6 (commit)
via f42c7e3d3344104206ca0b8669e2b07a6b30388e (commit)
via a00eab83a1a2636f5c18be8109f73bf050d1ec88 (commit)
via 340a9ffa3a84a8dc4d7b4413136c4f1719eb5591 (commit)
via 20524cb833e736d4baed4dcadca6fcc6d8bcc7a8 (commit)
via 5ee1f6c05fe8b491afead3697a05401511a4d4b4 (commit)
via 5fa837af6e73abbcf70e66c6de785e3a24259328 (commit)
via 6e3e3c5c11a673f2347876368993da9a3715d8f3 (commit)
via c47db46c7f2aac3ad1b099d8716c44dbb9f8bbf6 (commit)
via abe3b885cf4b123ad6e36f6d4d9ae6695e0fa32e (commit)
via 101003562d2b5af0fc90a115fbbad98898de3d50 (commit)
via a4926dfa88f36c529096f30b223a434479cd0eb8 (commit)
via 6929750b2f8f1f0b01b40f0ac516ec5f9e431c4d (commit)
via 264391e43f3fa3a73c1c274dd9970963e68b8ace (commit)
via 959ab858881801060b292a71b6e694c4801ef2eb (commit)
via ad7c6e10909fd4ac10466d62a7b3e5a77804a233 (commit)
via 5b349796c7ddf23188c92dbe98e4ce75a2ac6ee6 (commit)
via c830768910678af42a92d8d0b63ac96d5636361c (commit)
via 97f5239f053b1691d7f2cb56230386921f8ea4d4 (commit)
via 1088f78459a6ac24b91673625ef72976dfb99fd3 (commit)
via 64736a9fa518af448b98e6277185acc269bd5ade (commit)
via dee0d839af1a7cd1eca4b31f7f2371ff3b0803ed (commit)
via 2414412bb3360e02a72e5459711cf077a5e50380 (commit)
via eb4fb5479e86795128e71ebd1bb478b50ff6d7c9 (commit)
via f19af5689832a7b28a60f433cea850ca06841230 (commit)
via 1db6a725fdadd1c92d27a459b2ba2820e5722e97 (commit)
via c5e726ce0161166fcdac2eeda5e9939926152b7a (commit)
via 1d73050943da34f9983c7b71eb9cd76a1f0aed5e (commit)
via 8d0355a42caf66f40fe3007a43dc2c8b88712083 (commit)
via 4e416c9872af5237494958d1bacd33eb17821732 (commit)
via 4f77c7788c6fc3a6cc9cd90ff231d837fdec7cc4 (commit)
via e76418b037477b700037652bec9dcba98839e14b (commit)
via 99c4b804af44311d95e0b0ab72521471a1166347 (commit)
via 484f12ee2767535a87272c3899967c29b1e13651 (commit)
via 3cf223a0d4374e6fc743439826f49adaf29f21cb (commit)
via 19891e472215cf9e976158beb56ba22b8d581d7b (commit)
via 3d3f25043f3f270a675391237a2a2a73495e1e37 (commit)
via 45c2b55b04626ab62d725f280893e270502e6011 (commit)
via a5c2ae6bf4687becf0cae82b45362e092c11c8da (commit)
via c416e64c8f6a3a1e66daaf8ed2da0b9251df30cb (commit)
via c01e6aeda1af59597bfddf046628bb4802f2d671 (commit)
via de98daba898ad2dfa58c9e810d98dadf4d208b95 (commit)
via fc2dc01f5e98b3c1d663e78f882eed20962de9d7 (commit)
via 5a9684ac02fb535d1ad5cc80a6390d758415d0d5 (commit)
via 76e03aef476139403bab1df2ed038761159ceff1 (commit)
via 2ecf7efbab5ceb082739dcffd98c18bb4b14447e (commit)
via 9fbb2bbd4e2a0f1d15e1db3f3d606cdedae825a7 (commit)
via e56ea5c6c9244d2baaef9a24efc5eaad5bdf290c (commit)
via 71ba12ca158c39c17187f15c26279ec00d461dc5 (commit)
via 85de04ad9268e45bad459a606bfabfca4f6fad8c (commit)
via 158b18f036fc5f3f5848390d7ef53c6493aeb5a9 (commit)
via acfbb64831047dcae895e57ceff3d0a834c91bf2 (commit)
via 260e4dc4c2686fbcf3a6e3979c817d5a2a765c67 (commit)
via 8375bf07eb52a68c8881164b7f9f89d4a454e3b3 (commit)
via 5608a875c36101c791e35c474abf5e3900aad071 (commit)
via e06ea339d3d0c0f6fad81128b3cab34cdd4bd36f (commit)
via 734a66939892064fe9c663fd746cb7371c7d84e4 (commit)
via 2359eabb2e84dd9aa5109332c37b0f50aee896bc (commit)
via 3eedcb355e3dfdc4b8006083cfbcf2610b9d895d (commit)
via 25d4718345b22916d1b865c164437934a2a6cddd (commit)
via 9df4975aa288c0847d69c25474fe4711ba5b91f3 (commit)
via da80f4a198e734313a7991466244083dabd64b00 (commit)
via c428f31ab63f8414973848455ab7c44ed4a1d51b (commit)
via cfcfa1c48d95a07bb961baeef8d7658ddecce41e (commit)
via fdb63be9fde8ea69e78e68f77bb0ab00a79a8d6f (commit)
via c0082c57cb7c5e67115e7b03c8c85f74b5b29d0d (commit)
via cdfd69a42adf6ed017a443773eccfd8f021fea32 (commit)
via 770927dbfa1fb16b3075d1581dc66f4e7a623631 (commit)
via fa92bc75955f1a81652241addf3c6b24c594b55e (commit)
via 80a9fa664c8370436a0190f483410e5af05e26c3 (commit)
via f721928e89bb06f6df21432da521c9163ff722e3 (commit)
via c74c5d27e5c79b475180e87552bafcfefb5aa9f7 (commit)
via 62f27bcb9975216848c975c20979b0837ffdb4b1 (commit)
via 57922375aa60a80e5af5c1e5baa0205df9dfdfb1 (commit)
via 50abee31d6dad724c0d4b8a452eeb330364cde85 (commit)
via 67b77eb3b201786e632f0e6c6df9a9c5eb9fd402 (commit)
via 09fcf10ff841e5032145936385b406412674a368 (commit)
via b67167e763608af2909ce4e4e25c03d0e7db8b84 (commit)
via 160aa31a20a754b165f0184d712fba8b65519125 (commit)
via 0db099d96a60ffafdaf0cdf14e92a4dc579e723e (commit)
via cf8786f78aa3d8af225f1eaf2bd7c0a17cfa33fc (commit)
via 87e9c0e26ff55582a993a8e11902e8657647f59a (commit)
via 6e7c96b2bdf0498028bf6ee3902289e81ec1f2d7 (commit)
via 76e74a7feb2142ff48a1189957626cc8f6deb360 (commit)
via 0ebc3631838c34c0307fe73beb8e8037b0110bcf (commit)
via 8048e6daaeb0628b243855b606093f95dded9d29 (commit)
via 15ccbbab5a621a1cbfeaeb7c65ee35f72da3efb2 (commit)
via 53755b6cb7a0ba9a73a7f1a54c219140e82863d8 (commit)
via 508d3b100a3faa9711e42a32ddeaade5da2470da (commit)
via e4895820d9302048cc41e4b119450bbb8c01f70f (commit)
via 25e66d0972d323e6bd865f9705cb3f075b727290 (commit)
via 34ec4990bb0deaa6c7bef5b9793da3c6836b79e2 (commit)
via 59e45dbc9ef35513c9a32785158274b86d4e003d (commit)
via d0a2405dd75f02ee40714d112e8ba162ac01ad40 (commit)
via fdb655af9d3ace45edc08357b3328a1f8231e449 (commit)
via 5f18d6be31b253030d884e1e3dad1cf255dd5bab (commit)
via 0effb9301167df51d071e52562b3acfe7513498b (commit)
via 56753cd6748386be0cd298267486a687f22f067d (commit)
via deff6c2601a3872a141f2aa7ebdf81e0427c94cb (commit)
via 094c9628eed32ebd425382398126d337f83f3bcb (commit)
via fc69579b197cb963209111620f0a908c7811e1db (commit)
via 88393d2fb7ed29877b4a1bd2a899ffc05d7dfe9a (commit)
via 9d209cb34089febeaadeab572a1b4c8d9d485741 (commit)
via 3b06308d1704a14c10f4821494085dffa8fe6ea6 (commit)
via c5e5db25e95289376a0df35695ab2b5f48131b1f (commit)
via 8f10d54cf1d1b03d566366bed3a04ceaa315dd5b (commit)
via 26b4ea9922d7d9f6c71595960df1c3d0b6c37fc6 (commit)
via 18c93ee7a21912e9759b127a3ec73bb81c15f23d (commit)
via b17c7d97723d99909868d3435b449ab4e95f1708 (commit)
via 554300e76561021bd26f9143cdece13dd80341cd (commit)
via 79be7ea20c46c6d005de5e0a24bf7b47ccfd43f6 (commit)
via 416a99c2039e879cefc67dd0764b8544ef6c2d53 (commit)
via 9b910084faf3db6fa2071af604620e7d45d12a6c (commit)
via 665b0fbe5f57866f9d0183a08e713fe07e8db8de (commit)
via 2d4198a095e193102daa2710c6b2baba7be7c9ce (commit)
via 3465b6a80446e631479bb72d62739833506dc0de (commit)
via a1819526b85ce37c7d3ae421c2f5329c1c245c7f (commit)
via f5cb4310db63948ea63329415ade2ed8eef529e2 (commit)
via 1ec9e3707d464e96e0a1c44d3d074842b3b5051b (commit)
via 5b3187552676947ee74e4b652e7a04d3d9b9a3a4 (commit)
via 41538c32327e28e00206c3dbb2317e92a8731958 (commit)
via 46a6199f3a40a24ee145adc390500190b17a6395 (commit)
via 69107d28a38609607112a3355de61e0f61ed4f51 (commit)
via 3104ab72159ede44043fcd71dc0094c97f9c4251 (commit)
via a8a79114bdb1b02d5c071a1efce2796f633976ed (commit)
via d7e0ab968120be2807c8cc4adf41c85e887e0b05 (commit)
via 628654aa8c43b00472867975031923747d163aef (commit)
via eac37febf548d1d103661ded6e1a0e21e64ba7cd (commit)
via ad7679cfe57733940f8461097ee01bfd97997ce6 (commit)
via 1f40455a7bd764c517c7f1ddb8b4b41b4a2f7ee8 (commit)
via b269c28f1d54e8609f36c8aeb77a2b6025172066 (commit)
via 24b4d1ad90558332cd5251b265a54c21ffdbfd36 (commit)
via ae63accf8df1fdc458603fdf0c259c4bf0f25231 (commit)
via 372aaff2b572ce772fafc506e9c57d465eb823f6 (commit)
via 83d820df9f30c281b4babbc8b05fdea4e2b81d01 (commit)
via d1dbdfd430be8a1bdd21fc1f02f8fe5e2d989092 (commit)
via 08284382b53f621c09c4ffc87d82fa0261a69d32 (commit)
via 5386f6657234f3c24a4783cf63ab85016eda85b8 (commit)
via 657235e9e2b90c837efe809fc014fe6fe0cb9b23 (commit)
via daed47277f97a1e972904b1fdcd16f8ce38a4e6a (commit)
via 970bcc6f77a7b1ff14a7ec124e3004d89b1f173a (commit)
via 2a5192a38e2d5cde2f7b974c0f6ec5a615d6e139 (commit)
via 3d078f8d6387e64eb6f0b844d2fe784a0be45230 (commit)
via bb42f7d7d8320474c3e05e9e2185e7923feb1606 (commit)
via d6c2c9f1446c35a23f7dc1f73de398c4484e0cbe (commit)
via 9020972ed0ea1395a9978363a34ddc8dfa77ee48 (commit)
via ad1f68c37d46e56916e18996e14405ed9dab7d35 (commit)
via e81225f0a847dfd14d97acbab775a9b3e0e6cb9b (commit)
via 82a3d53af36a8374a4a1f28a19c69feabe8ca217 (commit)
via ac7939b70866c8ed8bcdd9f0854c6c845534ba78 (commit)
via 9d9f6578ada5768205150757f241c66009347884 (commit)
via 88ce56692e75cebdedc983fcb00aa48ea60aba8b (commit)
via e8db6685b64c6853eb3c5d3ee27ba58168c3c8e8 (commit)
via 20ade56019456b41c98021c2ed5a848bd8d018bb (commit)
via 0dd85fcd31ef5f251dcf143fef4118d6ea56f700 (commit)
via 61c3f86eb779ab8e723e43354eddafe219bc27d9 (commit)
via 451a6d3590ed7c049ad6ce24f8f6c01685d7d3d3 (commit)
via 88038b52936824be2bd2127c03d54e647cd1ff7e (commit)
via a830b5b560251c3143a7b1fd60db3f50a7021b34 (commit)
via 05ff1098f0e9eda5d642a1249f8b3a236656320c (commit)
via 27dc00515f48ed69b4d5e26ff64805b8cda4ccd3 (commit)
via 349e1ee218d7e888c6c1bcb07f6537f0bdc85012 (commit)
via aff4a730ad890564ee05c2395c4ebb49458e3cdc (commit)
via f9b617b7c8245d1e0eedaafc181501a6ac344657 (commit)
via 900b548097c68649ae2874ded5849f1d8164384c (commit)
via 3812c3e89d4048a91db28b4b9276f0c45dba7201 (commit)
via f7cf309e7d7003b5bf9407af81e9a2b1374cde8e (commit)
via 6a59d473c1574eec4db1f83b5d1a963b4f976e5a (commit)
via 982f7db5b67194ddb3b3dd1fae594784f58b35b8 (commit)
via c63ff55687f32dfdff01b9827b411b3757d48ee7 (commit)
via 47d1bdc0960af5bfc8f2793c352f60483539c389 (commit)
via d7635fbe751b2d00dd722a038723577f344406e1 (commit)
via 5824ee2e5198dd46a7813fe2adbd380a114f9ac4 (commit)
via 999c05af58ef1bb6f2b6804301f6ef8d02544a2b (commit)
via b2329e2ce27a24a2d964743a87091413b0e5858e (commit)
via 6eb3d1fb8fe71623fa63da46c250184cf2e4fbb8 (commit)
via b9bf39b18d1e161b6b971270d15c1024ece86243 (commit)
via 5a662d84f00c0c2693c18d333bab9d0fdda7e28e (commit)
via 28e6503a62fbbff41224843c8c39f4052242c8cd (commit)
via 19c43cebdfa1b701b031577f62c2a7b060dc8b0e (commit)
via faefe8fc192886b35be484d5a6bb9f5a6a87b53f (commit)
via 27616fe74103c079a84ac34b2adb83f1952c5772 (commit)
via fea4d2ba4ab741daff3fd17d910b72539a50a447 (commit)
via 2403cd9fd5a508252fe4570d6eed9c9c4efa366a (commit)
via 3809452aef74876da9d2644fe6c824a22527d6ac (commit)
via 3d1d497b23a53370417385eb75a5ba7207364ff7 (commit)
via a04932c206425f4eb2f46e36f0acf4b7b194b865 (commit)
via 32f8850d66e388fe0c086f8e1e4c74658c34fad0 (commit)
via ae85b0b33bcae6e36dcfa66d2fca9c70583c54b6 (commit)
via a592559241a69087a35361177d9aa81c8a2c3e79 (commit)
via 2fc2ce33d7b857d42be0cd94354e90933b84ff1b (commit)
via aed4d13a2fd3a97679c512eab2b4a9e5e24df112 (commit)
via 8a0e311e4f8c402c551d61f6290e5bdca149a619 (commit)
via 06d87aa4fe72ad86c94593e4909be08bad6acb35 (commit)
via 682dd5b6cc23a455766a7651e3e841257660b31c (commit)
via d2f68bd1e108c3f2dda2322c427050d019b17e04 (commit)
via ef6773029872db261f1b3bf3fb2ca86f47dfdcfe (commit)
via 296e1b20ecf2c283508712332f163c06464b3b5d (commit)
via f69d2824c997c53caa11d30ba816768bad52e12b (commit)
via 4077a9af0985d3c85f2f2de2bb7a0f6be581e71e (commit)
via 32131dfa999fe658e5e61f465a5badf71271e2d2 (commit)
via 53f785c298338645b6880f22f26b0c36a7cfab4d (commit)
via 5549904bbd5dec9bafe60e36d4ea1abe6b791f19 (commit)
via 23a940aaf47411cd07615b3301d24d3868a11bce (commit)
via f91b1eaef727a11769edb8d02410b43386652b0d (commit)
via 499d871d59a5c05cf1034580e7c10ba37cefef4a (commit)
via 4ac1cbf8618fc51f1413597c2fb406c1bead3376 (commit)
via 0191604164c265a7b972a5802680477f20451d9d (commit)
via 8e1a832244c109351a20a7181da0f65c73e63987 (commit)
via 290ddb6fb0776106cbd68a68f5753452437f357f (commit)
via b1233abce0628266a1805e52d9f9fc651c2a5c59 (commit)
via f09da35ed284ebc1ed1e941f3e3cb63b06a35d51 (commit)
via a56406d730f2a07dd442b9e99ef9dab7b7d81895 (commit)
via a5a4f79e91aa8bba1794394646808f6d4c444661 (commit)
via d3f9fad0cc83a9af47589894998a781db9c60989 (commit)
via 6b5b6890158830b26161b3879a0d1eeaa122659f (commit)
via 248d2ebb4ab7ea3d9060838bcfbfe3b9330da5ee (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (e3d48bb942c3ccea6f9c7a2ab0c5db5233e7b4a7)
\
N -- N -- N (82780d422b0f8a4ee4e4df52673cc88e7bb936a5)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 82780d422b0f8a4ee4e4df52673cc88e7bb936a5
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 27 10:50:49 2015 -0400
5824: fixup keepdl docs.
diff --git a/services/keepdl/doc.go b/services/keepdl/doc.go
index 65c7f19..62743db 100644
--- a/services/keepdl/doc.go
+++ b/services/keepdl/doc.go
@@ -1,28 +1,128 @@
// Keepdl provides read-only HTTP access to files stored in Keep. It
// serves public data to anonymous and unauthenticated clients, and
-// accepts authentication via Arvados tokens. It can be installed
-// anywhere with access to Keep services, typically behind a web proxy
-// that provides SSL support.
+// serves private data to clients that supply Arvados API tokens. It
+// can be installed anywhere with access to Keep services, typically
+// behind a web proxy that supports TLS.
//
-// Given that this amounts to a web hosting service for arbitrary
-// content, it is vital to ensure that at least one of the following is
-// true:
+// Starting the server
//
-// Usage
-//
-// Listening:
+// Serve HTTP requests at port 1234 on all interfaces:
//
// keepdl -address=:1234
//
-// Start an HTTP server on port 1234.
+// Serve HTTP requests at port 1234 on the interface with IP address 1.2.3.4:
//
// keepdl -address=1.2.3.4:1234
//
-// Start an HTTP server on port 1234, on the interface with IP address 1.2.3.4.
+// Proxy configuration
//
// Keepdl does not support SSL natively. Typically, it is installed
// behind a proxy like nginx.
//
+// Here is an example nginx configuration.
+//
+// http {
+// upstream keepdl {
+// server localhost:1234;
+// }
+// server {
+// listen *:443 ssl;
+// server_name dl.example.com *.dl.example.com ~.*--dl.example.com;
+// ssl_certificate /root/wildcard.example.com.crt;
+// ssl_certificate_key /root/wildcard.example.com.key;
+// location / {
+// proxy_pass http://keepdl;
+// proxy_set_header Host $host;
+// proxy_set_header X-Forwarded-For $remote_addr;
+// }
+// }
+// }
+//
+// It is not necessary to run keepdl on the same host as the nginx
+// proxy. However, TLS is not used between nginx and keepdl, so
+// intervening networks must be secured by other means.
+//
+// Download URLs
+//
+// The following URL patterns are supported for public collections
+// (i.e., collections which can be served by keepdl without making use
+// of any credentials supplied by the client).
+//
+// http://dl.example.com/c=uuid_or_pdh/path/file.txt
+// http://dl.example.com/c=uuid_or_pdh/path/t=TOKEN/file.txt
+//
+// The following URL patterns are supported for all collections:
+//
+// http://uuid_or_pdh--dl.example.com/path/file.txt
+// http://uuid_or_pdh--dl.example.com/t=TOKEN/path/file.txt
+//
+// In the latter set, the string "--" can be replaced with "." with
+// identical results, provided the upstream proxy is configured to
+// present clients with a suitable certificate, and forward requests
+// to keepdl, for the relevant server names:
+//
+// http://uuid_or_pdh--dl.example.com/path/file.txt
+// http://uuid_or_pdh.dl.example.com/path/file.txt
+//
+// The first form is intended for cases where the cost or effort of
+// deploying a wildcard TLS certificate is greater than zero.
+//
+// In all of the above forms, the "dl.example.com" part can be
+// anything at all.
+//
+// In all of the above forms, the "uuid_or_pdh" part can be either a
+// collection UUID or a portable data hash with the "+" character
+// replaced by "-".
+//
+// Authorization mechanisms
+//
+// A token can be provided in an Authorization header:
+//
+// Authorization: OAuth2 o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+//
+// A base64-encoded token can be provided in a cookie named "api_token":
+//
+// Cookie: api_token=bzA3ajRweDdSbEpLNEN1TVlwN0MwTERUNEN6UjFKMXFCRTVBdm83ZUNjVWpPVGlreEs=
+//
+// A token can be provided in an URL-encoded query string:
+//
+// GET /foo.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+//
+// A suitably encoded token can be provided in a POST body if the
+// request has a content type of application/x-www-form-urlencoded or
+// multipart/form-data:
+//
+// POST /foo.txt
+// Content-Type: application/x-www-form-urlencoded
+// [...]
+// api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+//
+// Compatibility
+//
+// Client-provided authorization tokens are ignored if the client does
+// not provide a Host header.
+//
+// In order to use the query string or a POST form authorization
+// mechanisms, the client must follow 303 redirects; the client must
+// accept cookies with a 303 response and send those cookies when
+// performing the redirect; and either the client or an intervening
+// proxy must resolve a relative URL ("//host/path") if given in a
+// response Location header.
+//
+// Intranet mode
+//
+// Normally, Keepdl accepts requests for multiple collections using
+// the same host name, provided the client's credentials are not being
+// used. This provides insufficient XSS protection in an installation
+// where the "anonymously accessible" data is not truly public, but
+// merely protected by network topology.
+//
+// In such cases -- for example, a site which is not reachable from
+// the internet, where some data is world-readable from Arvados's
+// perspective but is intended to be available only to users within
+// the local network -- the upstream proxy should configured to return
+// 401 for all paths beginning with "/c=".
+//
package main
// TODO(TC): Implement
@@ -31,7 +131,7 @@ package main
//
// Normally, Keepdl is installed using a wildcard DNS entry and a
// wildcard HTTPS certificate, serving data from collection X at
-// ``https://X.dl.example.com/path/file.ext''.
+// ``https://X--dl.example.com/path/file.ext''.
//
// It will also serve publicly accessible data at
// ``https://dl.example.com/collections/X/path/file.txt'', but it does not
diff --git a/services/keepdl/handler_test.go b/services/keepdl/handler_test.go
index 06d6da8..2479597 100644
--- a/services/keepdl/handler_test.go
+++ b/services/keepdl/handler_test.go
@@ -85,6 +85,8 @@ func authzViaPOST(r *http.Request, tok string) {
url.Values{"api_token": {tok}}.Encode()))
}
+// Try some combinations of {url, token} using the given authorization
+// mechanism, and verify the result is correct.
func doVhostRequests(c *check.C, authz authorizer) {
hostPath := arvadostest.FooCollection + ".example.com/foo"
for _, tok := range []string{
@@ -110,6 +112,11 @@ func doVhostRequests(c *check.C, authz authorizer) {
} else {
c.Check(code >= 400, check.Equals, true)
c.Check(code < 500, check.Equals, true)
+ if tok == "" {
+ c.Check(code, check.Equals, http.StatusUnauthorized)
+ } else {
+ c.Check(code, check.Equals, http.StatusNotFound)
+ }
c.Check(body, check.Equals, "")
}
}
commit 9a02abe887e3c09115a000f9f63666db9fe96172
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 27 10:50:24 2015 -0400
5824: fixup sdk
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 3c7888a..34210fe 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -2,6 +2,7 @@ package auth
import (
"encoding/base64"
+ "log"
"net/http"
"net/url"
"strings"
@@ -77,6 +78,7 @@ func (a *Credentials) loadTokenFromCookie(r *http.Request) {
if err != nil || len(cookie.Value) == 0 {
return
}
+ log.Printf("%+v", r.Header)
token, err := DecodeTokenCookie(cookie.Value)
if err != nil {
return
commit 18277495a1593a0a9a7d67b39f5fb47c62979f2c
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 27 07:22:00 2015 -0400
5824: more keepdl vhost.
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
index ce7ac9a..03b3e26 100644
--- a/services/keepdl/handler.go
+++ b/services/keepdl/handler.go
@@ -2,6 +2,7 @@ package main
import (
"fmt"
+ "html"
"io"
"mime"
"net/http"
@@ -29,32 +30,47 @@ func init() {
// return s if s is a UUID or a PDH, otherwise ""
func parseCollectionIdFromDNSName(s string) string {
- if !arvadosclient.UUIDMatcher.MatchString(s) && !arvadosclient.PDHMatcher.MatchString(s) {
+ // Strip domain.
+ if i := strings.IndexRune(s, '.'); i >= 0 {
+ s = s[:i]
+ }
+ // Names like {uuid}--dl.example.com serve the same purpose as
+ // {uuid}.dl.example.com but can reduce cost/effort of using
+ // [additional] wildcard certificates.
+ if i := strings.Index(s, "--"); i >= 0 {
+ s = s[:i]
+ }
+ if !arvadosclient.UUIDMatch(s) && !arvadosclient.PDHMatch(s) {
return ""
}
return s
}
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
- var statusCode int
+ var statusCode = 0
var statusText string
w := httpserver.WrapResponseWriter(wOrig)
defer func() {
- if statusCode > 0 {
- if w.WroteStatus() == 0 {
- w.WriteHeader(statusCode)
- } else {
- httpserver.Log(r.RemoteAddr, "WARNING",
- fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
- }
+ if statusCode == 0 {
+ statusCode = w.WroteStatus()
+ } else if w.WroteStatus() == 0 {
+ w.WriteHeader(statusCode)
+ } else if w.WroteStatus() != statusCode {
+ httpserver.Log(r.RemoteAddr, "WARNING",
+ fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
}
if statusText == "" {
statusText = http.StatusText(statusCode)
}
- httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.String())
+ httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
}()
+ if r.Method != "GET" && r.Method != "POST" {
+ statusCode, statusText = http.StatusMethodNotAllowed, r.Method
+ return
+ }
+
arv := clientPool.Get()
if arv == nil {
statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
@@ -70,43 +86,56 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var reqTokens []string
var pathToken bool
- if targetId = parseCollectionIdFromDNSName(strings.Split(r.Host, ".")[0]); targetId != "" {
+ if targetId = parseCollectionIdFromDNSName(r.Host); targetId != "" {
// "http://{id}.domain.example.com/{path}" form
if t := r.FormValue("api_token"); t != "" {
- // ...with explicit token in query string,
- // ?api_token={token}. We must encrypt the
+ // ...with explicit token in query string or
+ // form in POST body. We must encrypt the
// token such that it can only be used for
- // this collection; put it in a cookie; and
- // redirect to the same URL without the query
- // param. If we don't encrypt it, the served
- // content could have JavaScript code that
- // reads the user's real token and uses it for
- // something other than reading other files in
- // the same collection. If we put it anywhere
- // in the Location bar, it looks like it can
- // be bookmarked or shared.
+ // this collection; put it in an HttpOnly
+ // cookie; and redirect to the same URL with
+ // the query param redacted, and method =
+ // GET.
+ //
+ // The HttpOnly flag is necessary to prevent
+ // JavaScript code (included in, or loaded by,
+ // a page in the collection being served) from
+ // employing the user's token beyond reading
+ // other files in the same domain, i.e., same
+ // the collection.
+ //
+ // The 303 redirect is necessary in the case
+ // of a GET request to avoid exposing the
+ // token in the Location bar, and in the case
+ // of a POST request to avoid raising warnings
+ // when the user refreshes the resulting page.
http.SetCookie(w, &http.Cookie{
Name: "api_token",
- Value: t,
+ Value: auth.EncodeTokenCookie([]byte(t)),
Path: "/",
- Domain: r.Host,
Expires: time.Now().AddDate(10,0,0),
})
redir := (&url.URL{Host: r.Host, Path: r.URL.Path}).String()
+
w.Header().Add("Location", redir)
- statusCode, statusText = http.StatusFound, ""
+ statusCode, statusText = http.StatusSeeOther, redir
+ w.WriteHeader(statusCode)
+ io.WriteString(w, `<A href="`)
+ io.WriteString(w, html.EscapeString(redir))
+ io.WriteString(w, `">Continue</A>`)
return
} else if strings.HasPrefix(pathParts[0], "t=") {
// ...with explicit token in path,
// "{...}.com/t={token}/{path}". This form
// must only be used to pass scoped tokens
// that give permission for a single
- // collection (see above note about
- // redirects).
+ // collection. See FormValue case above.
tokens = []string{pathParts[0][2:]}
targetPath = pathParts[1:]
+ pathToken = true
} else {
- // ...without explicit token
+ // ...with cookie, Authorization header, or
+ // no token at all
reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
tokens = append(reqTokens, anonymousTokens...)
targetPath = pathParts
@@ -136,7 +165,6 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
found := false
for _, arv.ApiToken = range tokens {
err := arv.Get("collections", targetId, nil, &collection)
- httpserver.Log(err)
if err == nil {
// Success
found = true
@@ -181,8 +209,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// someone trying (anonymously) to download public
// data that has been deleted. Allow a referrer to
// provide this context somehow?
- statusCode = http.StatusUnauthorized
w.Header().Add("WWW-Authenticate", "Basic realm=\"dl\"")
+ statusCode = http.StatusUnauthorized
return
}
@@ -212,6 +240,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
+ w.WriteHeader(http.StatusOK)
_, err = io.Copy(w, rdr)
if err != nil {
statusCode, statusText = http.StatusBadGateway, err.Error()
diff --git a/services/keepdl/handler_test.go b/services/keepdl/handler_test.go
new file mode 100644
index 0000000..06d6da8
--- /dev/null
+++ b/services/keepdl/handler_test.go
@@ -0,0 +1,202 @@
+package main
+
+import (
+ "html"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "regexp"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&UnitSuite{})
+
+type UnitSuite struct {}
+
+func mustParseURL(s string) *url.URL {
+ r, err := url.Parse(s)
+ if err != nil {
+ panic("parse URL: " + s)
+ }
+ return r
+}
+
+func (s *IntegrationSuite) TestVhost404(c *check.C) {
+ for _, testURL := range []string{
+ arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
+ arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
+ } {
+ resp := httptest.NewRecorder()
+ req := &http.Request{
+ Method: "GET",
+ URL: mustParseURL(testURL),
+ }
+ (&handler{}).ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Body.String(), check.Equals, "")
+ }
+}
+
+type authorizer func(*http.Request, string)
+
+func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
+ doVhostRequests(c, authzViaAuthzHeader)
+}
+func authzViaAuthzHeader(r *http.Request, tok string) {
+ r.Header.Add("Authorization", "OAuth2 " + tok)
+}
+
+func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
+ doVhostRequests(c, authzViaCookieValue)
+}
+func authzViaCookieValue(r *http.Request, tok string) {
+ r.AddCookie(&http.Cookie{
+ Name: "api_token",
+ Value: auth.EncodeTokenCookie([]byte(tok)),
+ })
+}
+
+func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
+ doVhostRequests(c, authzViaPath)
+}
+func authzViaPath(r *http.Request, tok string) {
+ r.URL.Path = "/t=" + tok + r.URL.Path
+}
+
+func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
+ doVhostRequests(c, authzViaQueryString)
+}
+func authzViaQueryString(r *http.Request, tok string) {
+ r.URL.RawQuery = "api_token=" + tok
+}
+
+func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
+ doVhostRequests(c, authzViaPOST)
+}
+func authzViaPOST(r *http.Request, tok string) {
+ r.Method = "POST"
+ r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+ r.Body = ioutil.NopCloser(strings.NewReader(
+ url.Values{"api_token": {tok}}.Encode()))
+}
+
+func doVhostRequests(c *check.C, authz authorizer) {
+ hostPath := arvadostest.FooCollection + ".example.com/foo"
+ for _, tok := range []string{
+ arvadostest.ActiveToken,
+ arvadostest.ActiveToken[:15],
+ arvadostest.SpectatorToken,
+ "bogus",
+ "",
+ } {
+ u := mustParseURL("http://" + hostPath)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ Header: http.Header{},
+ }
+ authz(req, tok)
+ resp := doReq(req)
+ code, body := resp.Code, resp.Body.String()
+ if tok == arvadostest.ActiveToken {
+ c.Check(code, check.Equals, http.StatusOK)
+ c.Check(body, check.Equals, "foo")
+ } else {
+ c.Check(code >= 400, check.Equals, true)
+ c.Check(code < 500, check.Equals, true)
+ c.Check(body, check.Equals, "")
+ }
+ }
+}
+
+func doReq(req *http.Request) *httptest.ResponseRecorder {
+ resp := httptest.NewRecorder()
+ (&handler{}).ServeHTTP(resp, req)
+ if resp.Code != http.StatusSeeOther {
+ return resp
+ }
+ cookies := (&http.Response{Header: resp.Header()}).Cookies()
+ u, _ := req.URL.Parse(resp.Header().Get("Location"))
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ Header: http.Header{},
+ }
+ for _, c := range cookies {
+ req.AddCookie(c)
+ }
+ return doReq(req)
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ arvadostest.FooCollection + ".example.com/foo",
+ "?api_token=" + arvadostest.ActiveToken,
+ "text/plain",
+ "",
+ http.StatusOK,
+ )
+}
+
+func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "POST",
+ arvadostest.FooCollection + ".example.com/foo",
+ "",
+ "application/x-www-form-urlencoded",
+ url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
+ http.StatusOK,
+ )
+}
+
+func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "POST",
+ arvadostest.FooCollection + ".example.com/foo",
+ "",
+ "application/x-www-form-urlencoded",
+ url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
+ http.StatusNotFound,
+ )
+}
+
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, body string, expectStatus int) {
+ u, _ := url.Parse(`http://` + hostPath + queryString)
+ req := &http.Request{
+ Method: method,
+ Host: u.Host,
+ URL: u,
+ Header: http.Header{"Content-Type": {contentType}},
+ Body: ioutil.NopCloser(strings.NewReader(body)),
+ }
+
+ resp := httptest.NewRecorder()
+ (&handler{}).ServeHTTP(resp, req)
+ c.Assert(resp.Code, check.Equals, http.StatusSeeOther)
+ c.Check(resp.Body.String(), check.Matches, `.*href="//` + regexp.QuoteMeta(html.EscapeString(hostPath)) + `".*`)
+ cookies := (&http.Response{Header: resp.Header()}).Cookies()
+
+ u, _ = u.Parse(resp.Header().Get("Location"))
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ Header: http.Header{},
+ }
+ for _, c := range cookies {
+ req.AddCookie(c)
+ }
+
+ resp = httptest.NewRecorder()
+ (&handler{}).ServeHTTP(resp, req)
+ c.Check(resp.Header().Get("Location"), check.Equals, "")
+ c.Check(resp.Code, check.Equals, expectStatus)
+ if expectStatus == http.StatusOK {
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ }
+}
commit cddb2c4abda1607ff36f248ddf9a291191114598
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 27 07:21:28 2015 -0400
5824: SDK fixup
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index dfc1655..9bdf54c 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -16,10 +16,13 @@ import (
"strings"
)
+type StringMatcher func(string) bool
+
+var UUIDMatch StringMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
+var PDHMatch StringMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
-var UUIDMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`)
-var PDHMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`)
// Indicates an error that was returned by the API server.
type APIServerError struct {
@@ -136,12 +139,11 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a
parameters = make(Dict)
}
- parameters["format"] = "json"
-
vals := make(url.Values)
for k, v := range parameters {
- m, err := json.Marshal(v)
- if err == nil {
+ if s, ok := v.(string); ok {
+ vals.Set(k, s)
+ } else if m, err := json.Marshal(v); err == nil {
vals.Set(k, string(m))
}
}
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 10a3121..3c7888a 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -1,17 +1,13 @@
package auth
import (
- "crypto"
- "crypto/rsa"
- "crypto/sha1"
- "log"
+ "encoding/base64"
"net/http"
"net/url"
"strings"
)
type Credentials struct {
- DecryptKey crypto.PrivateKey
Tokens []string
}
@@ -25,6 +21,15 @@ func NewCredentialsFromHTTPRequest(r *http.Request) *Credentials {
return c
}
+// EncodeTokenCookie accepts a token and returns a byte slice suitable
+// for use as a cookie value, such that it will be decoded correctly
+// by LoadTokensFromHTTPRequest.
+var EncodeTokenCookie func([]byte) string = base64.URLEncoding.EncodeToString
+
+// DecodeTokenCookie accepts a cookie value and returns the encoded
+// token.
+var DecodeTokenCookie func(string) ([]byte, error) = base64.URLEncoding.DecodeString
+
// LoadTokensFromHttpRequest loads all tokens it can find in the
// headers and query string of an http query.
func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
@@ -56,9 +61,7 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
a.Tokens = append(a.Tokens, val...)
}
- if a.DecryptKey != nil {
- a.loadEncryptedCookieToken(r)
- }
+ a.loadTokenFromCookie(r)
// TODO: Load token from Rails session cookie (if Rails site
// secret is known)
@@ -69,21 +72,14 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// the request body. This has to be requested explicitly by the
// application.
-const label = []byte{}
-
-func (a *Credentials) loadEncryptedCookieToken(r *http.Request) error {
+func (a *Credentials) loadTokenFromCookie(r *http.Request) {
cookie, err := r.Cookie("api_token")
if err != nil || len(cookie.Value) == 0 {
- return err
- }
- log.Printf("Cookie: %+v", cookie) // XXX
- encToken, err := base64.URLEncoding.DecodeString(cookie.Value)
- if err != nil {
- return err
+ return
}
- token, err := rsa.DecryptOAEP(sha1.New(), rand.Reader, a.PrivateKey, encToken, label)
+ token, err := DecodeTokenCookie(cookie.Value)
if err != nil {
- return err
+ return
}
a.Tokens = append(a.Tokens, string(token))
}
diff --git a/sdk/go/httpserver/log.go b/sdk/go/httpserver/log.go
index 7bee887..cdfc595 100644
--- a/sdk/go/httpserver/log.go
+++ b/sdk/go/httpserver/log.go
@@ -1,20 +1,21 @@
package httpserver
import (
+ "fmt"
"log"
- "strings"
)
-var escaper = strings.NewReplacer("\"", "\\\"", "\\", "\\\\", "\n", "\\n")
-
// Log calls log.Println but first transforms strings so they are
// safer to write in logs (e.g., 'foo"bar' becomes
-// '"foo\"bar"'). Non-string args are left alone.
+// '"foo\"bar"'). Arguments that aren't strings and don't have a
+// (String() string) method are left alone.
func Log(args ...interface{}) {
newargs := make([]interface{}, len(args))
for i, arg := range args {
if s, ok := arg.(string); ok {
- newargs[i] = "\"" + escaper.Replace(s) + "\""
+ newargs[i] = fmt.Sprintf("%+q", s)
+ } else if s, ok := arg.(fmt.Stringer); ok {
+ newargs[i] = fmt.Sprintf("%+q", s.String())
} else {
newargs[i] = arg
}
commit 6c011d450d6e320892989cc37605f7cf67dc3034
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 23 11:58:28 2015 -0400
5824: WIP on vhost.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index c262be1..dfc1655 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -16,9 +16,10 @@ import (
"strings"
)
-// Errors
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
+var UUIDMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`)
+var PDHMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`)
// Indicates an error that was returned by the API server.
type APIServerError struct {
@@ -296,14 +297,9 @@ func (this ArvadosClient) List(resource string, parameters Dict, output interfac
return this.Call("GET", resource, "", "", parameters, output)
}
-// API Discovery
-//
-// parameter - name of parameter to be discovered
-// return
-// value - value of the discovered parameter
-// err - error accessing the resource, or nil if no error
-var API_DISCOVERY_RESOURCE string = "discovery/v1/apis/arvados/v1/rest"
+const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest"
+// Discovery returns the value of the given parameter in the discovery document.
func (this *ArvadosClient) Discovery(parameter string) (value interface{}, err error) {
if len(this.DiscoveryDoc) == 0 {
this.DiscoveryDoc = make(Dict)
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 87b28f8..3040e0a 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -7,6 +7,8 @@ const (
FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
NonexistentCollection = "zzzzz-4zz18-totallynotexist"
HelloWorldCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+ FooPdh = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
+ HelloWorldPdh = "55713e6a34081eb03609e7ad5fcad129+62"
PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 73feffa4b7f6bb68e44cf984c85f6e88+3+Z+K at xyzzy acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero at 0 0:1:f 1:0:zero at 1 1:4:ooba 4:0:zero at 4 5:1:r 5:4:rbaz 9:0:zero at 9\n" +
"./overlapReverse acbd18db4cc2f85cedef654fccc4a4d8+3 acbd18db4cc2f85cedef654fccc4a4d8+3 5:1:o 4:2:oo 2:4:ofoo\n" +
"./segmented acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:1:frob 5:1:frob 1:1:frob 1:2:oof 0:1:oof 5:0:frob 3:1:frob\n" +
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index 4a719e9..10a3121 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -1,13 +1,18 @@
package auth
import (
+ "crypto"
+ "crypto/rsa"
+ "crypto/sha1"
+ "log"
"net/http"
"net/url"
"strings"
)
type Credentials struct {
- Tokens []string
+ DecryptKey crypto.PrivateKey
+ Tokens []string
}
func NewCredentials() *Credentials {
@@ -51,6 +56,10 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
a.Tokens = append(a.Tokens, val...)
}
+ if a.DecryptKey != nil {
+ a.loadEncryptedCookieToken(r)
+ }
+
// TODO: Load token from Rails session cookie (if Rails site
// secret is known)
}
@@ -59,3 +68,22 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// LoadTokensFromHttpRequest() that [or how] we should read and parse
// the request body. This has to be requested explicitly by the
// application.
+
+const label = []byte{}
+
+func (a *Credentials) loadEncryptedCookieToken(r *http.Request) error {
+ cookie, err := r.Cookie("api_token")
+ if err != nil || len(cookie.Value) == 0 {
+ return err
+ }
+ log.Printf("Cookie: %+v", cookie) // XXX
+ encToken, err := base64.URLEncoding.DecodeString(cookie.Value)
+ if err != nil {
+ return err
+ }
+ token, err := rsa.DecryptOAEP(sha1.New(), rand.Reader, a.PrivateKey, encToken, label)
+ if err != nil {
+ return err
+ }
+ a.Tokens = append(a.Tokens, string(token))
+}
diff --git a/sdk/go/keepclient/hashcheck.go b/sdk/go/keepclient/hashcheck.go
index 1f696d9..8f4b9c3 100644
--- a/sdk/go/keepclient/hashcheck.go
+++ b/sdk/go/keepclient/hashcheck.go
@@ -1,8 +1,3 @@
-// Lightweight implementation of io.ReadCloser that checks the contents read
-// from the underlying io.Reader a against checksum hash. To avoid reading the
-// entire contents into a buffer up front, the hash is updated with each read,
-// and the actual checksum is not checked until the underlying reader returns
-// EOF.
package keepclient
import (
@@ -14,6 +9,11 @@ import (
var BadChecksum = errors.New("Reader failed checksum")
+// Lightweight implementation of io.ReadCloser that checks the contents read
+// from the underlying io.Reader a against checksum hash. To avoid reading the
+// entire contents into a buffer up front, the hash is updated with each read,
+// and the actual checksum is not checked until the underlying reader returns
+// EOF.
type HashCheckingReader struct {
// The underlying data source
io.Reader
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 7b96341..b467d06 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -1,4 +1,3 @@
-/* Internal methods to support keepclient.go */
package keepclient
import (
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index b6dff50..f104d9a 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -9,6 +9,8 @@ import (
"fmt"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"log"
+ "regexp"
+ "strconv"
"strings"
)
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
index 04af920..ce7ac9a 100644
--- a/services/keepdl/handler.go
+++ b/services/keepdl/handler.go
@@ -5,8 +5,10 @@ import (
"io"
"mime"
"net/http"
+ "net/url"
"os"
"strings"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
@@ -25,6 +27,14 @@ func init() {
anonymousTokens = []string{}
}
+// return s if s is a UUID or a PDH, otherwise ""
+func parseCollectionIdFromDNSName(s string) string {
+ if !arvadosclient.UUIDMatcher.MatchString(s) && !arvadosclient.PDHMatcher.MatchString(s) {
+ return ""
+ }
+ return s
+}
+
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var statusCode int
var statusText string
@@ -42,7 +52,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if statusText == "" {
statusText = http.StatusText(statusCode)
}
- httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.URL.Path)
+ httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.String())
}()
arv := clientPool.Get()
@@ -54,17 +64,57 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path[1:], "/")
- if len(pathParts) < 3 || pathParts[0] != "collections" || pathParts[1] == "" || pathParts[2] == "" {
- statusCode = http.StatusNotFound
- return
- }
-
var targetId string
var targetPath []string
var tokens []string
var reqTokens []string
var pathToken bool
- if len(pathParts) >= 5 && pathParts[1] == "download" {
+
+ if targetId = parseCollectionIdFromDNSName(strings.Split(r.Host, ".")[0]); targetId != "" {
+ // "http://{id}.domain.example.com/{path}" form
+ if t := r.FormValue("api_token"); t != "" {
+ // ...with explicit token in query string,
+ // ?api_token={token}. We must encrypt the
+ // token such that it can only be used for
+ // this collection; put it in a cookie; and
+ // redirect to the same URL without the query
+ // param. If we don't encrypt it, the served
+ // content could have JavaScript code that
+ // reads the user's real token and uses it for
+ // something other than reading other files in
+ // the same collection. If we put it anywhere
+ // in the Location bar, it looks like it can
+ // be bookmarked or shared.
+ http.SetCookie(w, &http.Cookie{
+ Name: "api_token",
+ Value: t,
+ Path: "/",
+ Domain: r.Host,
+ Expires: time.Now().AddDate(10,0,0),
+ })
+ redir := (&url.URL{Host: r.Host, Path: r.URL.Path}).String()
+ w.Header().Add("Location", redir)
+ statusCode, statusText = http.StatusFound, ""
+ return
+ } else if strings.HasPrefix(pathParts[0], "t=") {
+ // ...with explicit token in path,
+ // "{...}.com/t={token}/{path}". This form
+ // must only be used to pass scoped tokens
+ // that give permission for a single
+ // collection (see above note about
+ // redirects).
+ tokens = []string{pathParts[0][2:]}
+ targetPath = pathParts[1:]
+ } else {
+ // ...without explicit token
+ reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
+ tokens = append(reqTokens, anonymousTokens...)
+ targetPath = pathParts
+ }
+ } else if len(pathParts) < 3 || pathParts[0] != "collections" || pathParts[1] == "" || pathParts[2] == "" {
+ statusCode = http.StatusNotFound
+ return
+ } else if len(pathParts) >= 5 && pathParts[1] == "download" {
// "/collections/download/{id}/{token}/path..." form:
// Don't use our configured anonymous tokens,
// Authorization headers, etc. Just use the token in
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
index fa2674a..5864315 100644
--- a/services/keepdl/server_test.go
+++ b/services/keepdl/server_test.go
@@ -3,9 +3,6 @@ package main
import (
"crypto/md5"
"fmt"
- "net/http"
- "net/http/httptest"
- "net/url"
"os/exec"
"strings"
"testing"
@@ -17,21 +14,12 @@ import (
)
var _ = check.Suite(&IntegrationSuite{})
-var _ = check.Suite(&UnitSuite{})
// IntegrationSuite tests need an API server and a keepdl server
type IntegrationSuite struct {
testServer *server
}
-func mustParseURL(s string) url.URL {
- r, err := url.Parse(s)
- if err != nil {
- panic("parse URL: " + s)
- }
- return r
-}
-
func (s *IntegrationSuite) TestNoToken(c *check.C) {
for _, token := range []string{
"",
@@ -53,21 +41,6 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
}
}
-func (s *UnitSuite) TestVhost404(c *check.C) {
- for _, testURL := range []string{
- bogusCollection + ".example.com/theperthcountyconspiracy",
- bogusCollection + ".example.com/theperthcountyconspiracy?t=" + spectatorToken,
- } {
- resp := httptest.NewRecorder()
- req := http.Request{
- URL: mustParseURL(testURL),
- }
- handler{}.ServeHTTP(resp, req)
- c.Check(resp.Body.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
- }
-}
-
// TODO: Move most cases to functional tests -- at least use Go's own
// http client instead of forking curl. Just leave enough of an
// integration test to assure that the documented way of invoking curl
commit 1f9ebffd675488ef594a3ebc81f4c6eb63da7887
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 24 23:33:08 2015 -0400
5824: add (*KeepClient)CollectionFileReader()
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
new file mode 100644
index 0000000..87b28f8
--- /dev/null
+++ b/sdk/go/arvadostest/fixtures.go
@@ -0,0 +1,17 @@
+package arvadostest
+
+const (
+ SpectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
+ ActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ AnonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
+ FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ NonexistentCollection = "zzzzz-4zz18-totallynotexist"
+ HelloWorldCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+ PathologicalManifest = ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 73feffa4b7f6bb68e44cf984c85f6e88+3+Z+K at xyzzy acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero at 0 0:1:f 1:0:zero at 1 1:4:ooba 4:0:zero at 4 5:1:r 5:4:rbaz 9:0:zero at 9\n" +
+ "./overlapReverse acbd18db4cc2f85cedef654fccc4a4d8+3 acbd18db4cc2f85cedef654fccc4a4d8+3 5:1:o 4:2:oo 2:4:ofoo\n" +
+ "./segmented acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:1:frob 5:1:frob 1:1:frob 1:2:oof 0:1:oof 5:0:frob 3:1:frob\n" +
+ `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:baz` + "\n" +
+ `./foo\040b\141r acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:b\141z\040w\141z` + "\n" +
+ "./foo acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:zero 0:3:foo\n" +
+ ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo/zero 0:3:foo/foo\n"
+)
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
new file mode 100644
index 0000000..150d1de
--- /dev/null
+++ b/sdk/go/keepclient/collectionreader.go
@@ -0,0 +1,186 @@
+package keepclient
+
+import (
+ "container/list"
+ "errors"
+ "io"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/manifest"
+)
+
+var (
+ ErrNoManifest = errors.New("Collection has no manifest")
+ ErrNotImplemented = errors.New("Not implemented")
+)
+
+// CollectionFileReader returns an io.Reader that reads file content
+// from a collection. The filename must be given relative to the root
+// of the collection, without a leading "./".
+func (kc *KeepClient) CollectionFileReader(collection map[string]interface{}, filename string) (io.ReadCloser, error) {
+ mText, ok := collection["manifest_text"].(string)
+ if !ok {
+ return nil, ErrNoManifest
+ }
+ m := manifest.Manifest{Text: mText}
+ rdrChan := make(chan *cfReader)
+ go func() {
+ var r *cfReader
+ for seg := range m.FileSegmentIterByName(filename) {
+ if r == nil {
+ // We've just discovered that the
+ // requested filename does appear in
+ // the manifest, so we can return a
+ // real reader (not nil) from
+ // CollectionFileReader().
+ r = newCFReader(kc)
+ rdrChan <- r
+ }
+ r.Todo <- seg
+ }
+ if r == nil {
+ // File not found
+ rdrChan <- nil
+ return
+ }
+ close(r.Todo)
+ }()
+ // Before returning a reader, wait until we know whether the
+ // file exists here:
+ r := <-rdrChan
+ if r == nil {
+ return nil, os.ErrNotExist
+ }
+ return r, nil
+}
+
+type cfReader struct {
+ // CollectionFileReader's manifest-reading worker sends
+ // FileSegments to this channel, then closes it:
+ Todo chan *manifest.FileSegment
+ keepClient *KeepClient
+ todoIsDone bool // true when totalSize is final and
+ // nothing more will be added to toGet
+ toGet *list.List
+ bufChan chan []byte
+ buf []byte
+ wakeupGetter *sync.Cond
+ totalSize uint64
+ err error // first error encountered
+}
+
+func (r *cfReader) Read(outbuf []byte) (n int, err error) {
+ if r.err != nil {
+ return 0, r.err
+ }
+ for r.buf == nil || len(r.buf) == 0 {
+ var ok bool
+ r.buf, ok = <-r.bufChan
+ if r.err != nil {
+ return 0, r.err
+ } else if !ok {
+ return 0, io.EOF
+ }
+ }
+ if len(r.buf) > len(outbuf) {
+ n = len(outbuf)
+ } else {
+ n = len(r.buf)
+ }
+ copy(outbuf[:n], r.buf[:n])
+ r.buf = r.buf[n:]
+ return
+}
+
+func (r *cfReader) Close() error {
+ r.Len() // Wait for toGet to fill up
+ r.wakeupGetter.L.Lock()
+ r.toGet.Init()
+ r.wakeupGetter.L.Unlock()
+ r.wakeupGetter.Broadcast()
+ r.buf = nil
+ for _ = range r.bufChan {
+ }
+ return r.err
+}
+
+func (r *cfReader) Len() uint64 {
+ // Wait (if necessary) for countTodo() to finish
+ r.wakeupGetter.L.Lock()
+ for !r.todoIsDone {
+ r.wakeupGetter.Wait()
+ }
+ r.wakeupGetter.L.Unlock()
+ return r.totalSize
+}
+
+// countTodo receives FileSegments from Todo and pushes them onto
+// toGet, updating totalSize along the way.
+func (r *cfReader) countTodo() {
+ for fs := range r.Todo {
+ r.wakeupGetter.L.Lock()
+ r.totalSize += uint64(fs.Len)
+ r.toGet.PushBack(fs)
+ r.wakeupGetter.Broadcast()
+ r.wakeupGetter.L.Unlock()
+ }
+ r.todoIsDone = true
+ r.wakeupGetter.Broadcast()
+}
+
+// Pop a *FileSegment from the front of the toGet list (first waiting,
+// if necessary, for one to appear). If the toGet list is empty and no
+// more segments will appear, return nil.
+func (r *cfReader) getNextFileSegment() *manifest.FileSegment {
+ r.wakeupGetter.L.Lock()
+ defer r.wakeupGetter.L.Unlock()
+ for {
+ element := r.toGet.Front()
+ switch {
+ case element != nil:
+ r.toGet.Remove(element)
+ return element.Value.(*manifest.FileSegment)
+ case r.todoIsDone:
+ return nil
+ default:
+ r.wakeupGetter.Wait()
+ }
+ }
+}
+
+func (r *cfReader) readBlocks() {
+ defer close(r.bufChan)
+ for fs := r.getNextFileSegment(); fs != nil; fs = r.getNextFileSegment() {
+ rdr, _, _, err := r.keepClient.Get(fs.Locator)
+ if err != nil {
+ r.err = err
+ return
+ }
+ var buf = make([]byte, fs.Offset+fs.Len)
+ _, err = io.ReadFull(rdr, buf)
+ if err != nil {
+ r.err = err
+ return
+ }
+ for bOff, bLen := fs.Offset, 1<<20; bOff <= fs.Offset+fs.Len && bLen > 0; bOff += bLen {
+ if bOff+bLen > fs.Offset+fs.Len {
+ bLen = fs.Offset + fs.Len - bOff
+ }
+ r.bufChan <- buf[bOff : bOff+bLen]
+ bOff += bLen
+ }
+ }
+}
+
+func newCFReader(kc *KeepClient) (r *cfReader) {
+ r = new(cfReader)
+ r.keepClient = kc
+ r.toGet = list.New()
+ r.Todo = make(chan *manifest.FileSegment)
+ r.bufChan = make(chan []byte)
+ r.wakeupGetter = sync.NewCond(&sync.Mutex{})
+ go r.countTodo()
+ go r.readBlocks()
+ return
+}
diff --git a/sdk/go/keepclient/collectionreader_test.go b/sdk/go/keepclient/collectionreader_test.go
new file mode 100644
index 0000000..4c9bb05
--- /dev/null
+++ b/sdk/go/keepclient/collectionreader_test.go
@@ -0,0 +1,117 @@
+package keepclient
+
+import (
+ "crypto/md5"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+// IntegrationSuite tests need an API server
+type IntegrationSuite struct{}
+
+type SuccessHandler struct {
+ disk map[string][]byte
+}
+
+func (h SuccessHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ switch req.Method {
+ case "PUT":
+ buf, err := ioutil.ReadAll(req.Body)
+ if err != nil {
+ resp.WriteHeader(500)
+ return
+ }
+ pdh := fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf))
+ h.disk[pdh] = buf
+ resp.Write([]byte(pdh))
+ case "GET":
+ pdh := req.URL.Path[1:]
+ if buf, ok := h.disk[pdh]; !ok {
+ resp.WriteHeader(404)
+ } else {
+ resp.Write(buf)
+ }
+ default:
+ resp.WriteHeader(406)
+ }
+}
+
+type rdrTest struct {
+ mt string // manifest text
+ f string // filename
+ want interface{} // error or string to expect
+}
+
+func (s *ServerRequiredSuite) TestCollectionReaderContent(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveToken
+
+ kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, check.IsNil)
+
+ {
+ localRoots := make(map[string]string)
+ h := SuccessHandler{disk: make(map[string][]byte)}
+ for i, k := range RunSomeFakeKeepServers(h, 4) {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ }
+ kc.SetServiceRoots(localRoots, localRoots, nil)
+ kc.PutB([]byte("foo"))
+ kc.PutB([]byte("bar"))
+ kc.PutB([]byte("Hello world\n"))
+ kc.PutB([]byte(""))
+ }
+
+ mt := arvadostest.PathologicalManifest
+
+ for _, testCase := range []rdrTest{
+ {mt: mt, f: "zzzz", want: os.ErrNotExist},
+ {mt: mt, f: "frob", want: os.ErrNotExist},
+ {mt: mt, f: "/segmented/frob", want: os.ErrNotExist},
+ {mt: mt, f: "./segmented/frob", want: os.ErrNotExist},
+ {mt: mt, f: "/f", want: os.ErrNotExist},
+ {mt: mt, f: "./f", want: os.ErrNotExist},
+ {mt: mt, f: "foo bar//baz", want: os.ErrNotExist},
+ {mt: mt, f: "foo/zero", want: ""},
+ {mt: mt, f: "zero at 0", want: ""},
+ {mt: mt, f: "zero at 1", want: ""},
+ {mt: mt, f: "zero at 4", want: ""},
+ {mt: mt, f: "zero at 9", want: ""},
+ {mt: mt, f: "f", want: "f"},
+ {mt: mt, f: "ooba", want: "ooba"},
+ {mt: mt, f: "overlapReverse/o", want: "o"},
+ {mt: mt, f: "overlapReverse/oo", want: "oo"},
+ {mt: mt, f: "overlapReverse/ofoo", want: "ofoo"},
+ {mt: mt, f: "foo bar/baz", want: "foo"},
+ {mt: mt, f: "segmented/frob", want: "frob"},
+ {mt: mt, f: "segmented/oof", want: "oof"},
+ } {
+ rdr, err := kc.CollectionFileReader(map[string]interface{}{"manifest_text": testCase.mt}, testCase.f)
+ switch want := testCase.want.(type) {
+ case error:
+ c.Check(rdr, check.IsNil)
+ c.Check(err, check.Equals, want)
+ case string:
+ buf := make([]byte, len(want))
+ n, err := io.ReadFull(rdr, buf)
+ c.Check(err, check.IsNil)
+ for i := 0; i < 4; i++ {
+ c.Check(string(buf), check.Equals, want)
+ n, err = rdr.Read(buf)
+ c.Check(n, check.Equals, 0)
+ c.Check(err, check.Equals, io.EOF)
+ }
+ c.Check(rdr.Close(), check.Equals, nil)
+ }
+ }
+}
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 4e816cd..b6dff50 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -5,25 +5,183 @@
package manifest
import (
+ "errors"
+ "fmt"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"log"
"strings"
)
+var ErrInvalidToken = errors.New("Invalid token")
+
+var LocatorPattern = regexp.MustCompile(
+ "^[0-9a-fA-F]{32}\\+[0-9]+(\\+[A-Z][A-Za-z0-9 at _-]+)*$")
+
type Manifest struct {
Text string
}
+type BlockLocator struct {
+ Digest blockdigest.BlockDigest
+ Size int
+ Hints []string
+}
+
+type DataSegment struct {
+ BlockLocator
+ Locator string
+ StreamOffset uint64
+}
+
+// FileSegment is a portion of a file that is contained within a
+// single block.
+type FileSegment struct {
+ Locator string
+ // Offset (within this block) of this data segment
+ Offset int
+ Len int
+}
+
// Represents a single line from a manifest.
type ManifestStream struct {
StreamName string
Blocks []string
- Files []string
+ FileTokens []string
+}
+
+var escapeSeq = regexp.MustCompile(`\\([0-9]{3}|\\)`)
+
+func unescapeSeq(seq string) string {
+ if seq == `\\` {
+ return `\`
+ }
+ i, err := strconv.ParseUint(seq[1:], 8, 8)
+ if err != nil {
+ // Invalid escape sequence: can't unescape.
+ return seq
+ }
+ return string([]byte{byte(i)})
+}
+
+func UnescapeName(s string) string {
+ return escapeSeq.ReplaceAllStringFunc(s, unescapeSeq)
+}
+
+func ParseBlockLocator(s string) (b BlockLocator, err error) {
+ if !LocatorPattern.MatchString(s) {
+ err = fmt.Errorf("String \"%s\" does not match BlockLocator pattern "+
+ "\"%s\".",
+ s,
+ LocatorPattern.String())
+ } else {
+ tokens := strings.Split(s, "+")
+ var blockSize int64
+ var blockDigest blockdigest.BlockDigest
+ // We expect both of the following to succeed since LocatorPattern
+ // restricts the strings appropriately.
+ blockDigest, err = blockdigest.FromString(tokens[0])
+ if err != nil {
+ return
+ }
+ blockSize, err = strconv.ParseInt(tokens[1], 10, 0)
+ if err != nil {
+ return
+ }
+ b.Digest = blockDigest
+ b.Size = int(blockSize)
+ b.Hints = tokens[2:]
+ }
+ return
+}
+
+func parseFileToken(tok string) (segPos, segLen uint64, name string, err error) {
+ parts := strings.SplitN(tok, ":", 3)
+ if len(parts) != 3 {
+ err = ErrInvalidToken
+ return
+ }
+ segPos, err = strconv.ParseUint(parts[0], 10, 64)
+ if err != nil {
+ return
+ }
+ segLen, err = strconv.ParseUint(parts[1], 10, 64)
+ if err != nil {
+ return
+ }
+ name = UnescapeName(parts[2])
+ return
+}
+
+func (s *ManifestStream) FileSegmentIterByName(filepath string) <-chan *FileSegment {
+ ch := make(chan *FileSegment)
+ go func() {
+ s.sendFileSegmentIterByName(filepath, ch)
+ close(ch)
+ }()
+ return ch
+}
+
+func (s *ManifestStream) sendFileSegmentIterByName(filepath string, ch chan<- *FileSegment) {
+ blockLens := make([]int, 0, len(s.Blocks))
+ // This is what streamName+"/"+fileName will look like:
+ target := "./" + filepath
+ for _, fTok := range s.FileTokens {
+ wantPos, wantLen, name, err := parseFileToken(fTok)
+ if err != nil {
+ // Skip (!) invalid file tokens.
+ continue
+ }
+ if s.StreamName+"/"+name != target {
+ continue
+ }
+ if wantLen == 0 {
+ ch <- &FileSegment{Locator: "d41d8cd98f00b204e9800998ecf8427e+0", Offset: 0, Len: 0}
+ continue
+ }
+ // Linear search for blocks containing data for this
+ // file
+ var blockPos uint64 = 0 // position of block in stream
+ for i, loc := range s.Blocks {
+ if blockPos >= wantPos+wantLen {
+ break
+ }
+ if len(blockLens) <= i {
+ blockLens = blockLens[:i+1]
+ b, err := ParseBlockLocator(loc)
+ if err != nil {
+ // Unparseable locator -> unusable
+ // stream.
+ ch <- nil
+ return
+ }
+ blockLens[i] = b.Size
+ }
+ blockLen := uint64(blockLens[i])
+ if blockPos+blockLen <= wantPos {
+ blockPos += blockLen
+ continue
+ }
+ fseg := FileSegment{
+ Locator: loc,
+ Offset: 0,
+ Len: blockLens[i],
+ }
+ if blockPos < wantPos {
+ fseg.Offset = int(wantPos - blockPos)
+ fseg.Len -= fseg.Offset
+ }
+ if blockPos+blockLen > wantPos+wantLen {
+ fseg.Len = int(wantPos+wantLen-blockPos) - fseg.Offset
+ }
+ ch <- &fseg
+ blockPos += blockLen
+ }
+ }
}
func parseManifestStream(s string) (m ManifestStream) {
tokens := strings.Split(s, " ")
- m.StreamName = tokens[0]
+ m.StreamName = UnescapeName(tokens[0])
tokens = tokens[1:]
var i int
for i = range tokens {
@@ -32,7 +190,7 @@ func parseManifestStream(s string) (m ManifestStream) {
}
}
m.Blocks = tokens[:i]
- m.Files = tokens[i:]
+ m.FileTokens = tokens[i:]
return
}
@@ -58,6 +216,20 @@ func (m *Manifest) StreamIter() <-chan ManifestStream {
return ch
}
+func (m *Manifest) FileSegmentIterByName(filepath string) <-chan *FileSegment {
+ ch := make(chan *FileSegment)
+ go func() {
+ for stream := range m.StreamIter() {
+ if !strings.HasPrefix("./"+filepath, stream.StreamName+"/") {
+ continue
+ }
+ stream.sendFileSegmentIterByName(filepath, ch)
+ }
+ close(ch)
+ }()
+ return ch
+}
+
// Blocks may appear mulitple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 8cfe3d9..364648d 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -1,10 +1,13 @@
package manifest
import (
- "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"io/ioutil"
+ "reflect"
"runtime"
"testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
)
func getStackTrace() string {
@@ -60,7 +63,7 @@ func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
expectEqual(t, actual.StreamName, expected.StreamName)
expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
- expectStringSlicesEqual(t, actual.Files, expected.Files)
+ expectStringSlicesEqual(t, actual.FileTokens, expected.FileTokens)
}
func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected blockdigest.BlockLocator) {
@@ -72,8 +75,19 @@ func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected
func TestParseManifestStreamSimple(t *testing.T) {
m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
expectManifestStream(t, m, ManifestStream{StreamName: ".",
- Blocks: []string{"365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
- Files: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
+ Blocks: []string{"365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
+ FileTokens: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
+}
+
+func TestParseBlockLocatorSimple(t *testing.T) {
+ b, err := ParseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K at qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
+ if err != nil {
+ t.Fatalf("Unexpected error parsing block locator: %v", err)
+ }
+ expectBlockLocator(t, b, BlockLocator{Digest: blockdigest.AssertFromString("365f83f5f808896ec834c8b595288735"),
+ Size: 2310,
+ Hints: []string{"K at qr1hi",
+ "Af0c9a66381f3b028677411926f0be1c6282fe67c at 542b5ddf"}})
}
func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
@@ -88,8 +102,8 @@ func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
expectManifestStream(t,
firstStream,
ManifestStream{StreamName: ".",
- Blocks: []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475 at 5441920c"},
- Files: []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
+ Blocks: []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475 at 5441920c"},
+ FileTokens: []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
received, ok := <-streamIter
if ok {
@@ -126,3 +140,58 @@ func TestBlockIterLongManifest(t *testing.T) {
Size: 31367794,
Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db at 5441920c"}})
}
+
+func TestUnescape(t *testing.T) {
+ for _, testCase := range [][]string{
+ {`\040`, ` `},
+ {`\009`, `\009`},
+ {`\\\040\\`, `\ \`},
+ {`\\040\`, `\040\`},
+ } {
+ in := testCase[0]
+ expect := testCase[1]
+ got := UnescapeName(in)
+ if expect != got {
+ t.Errorf("For '%s' got '%s' instead of '%s'", in, got, expect)
+ }
+ }
+}
+
+type fsegtest struct {
+ mt string // manifest text
+ f string // filename
+ want []FileSegment // segments should be received on channel
+}
+
+func TestFileSegmentIterByName(t *testing.T) {
+ mt := arvadostest.PathologicalManifest
+ for _, testCase := range []fsegtest{
+ {mt: mt, f: "zzzz", want: nil},
+ // This case is too sensitive: it would be acceptable
+ // (even preferable) to return only one empty segment.
+ {mt: mt, f: "foo/zero", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
+ {mt: mt, f: "zero at 0", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
+ {mt: mt, f: "zero at 1", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
+ {mt: mt, f: "zero at 4", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
+ {mt: mt, f: "zero at 9", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
+ {mt: mt, f: "f", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
+ {mt: mt, f: "ooba", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 2}}},
+ {mt: mt, f: "overlapReverse/o", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}}},
+ {mt: mt, f: "overlapReverse/oo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}}},
+ {mt: mt, f: "overlapReverse/ofoo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
+ {mt: mt, f: "foo bar/baz", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
+ // This case is too sensitive: it would be better to
+ // omit the empty segment.
+ {mt: mt, f: "segmented/frob", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}, {"37b51d194a7513e45b56f6524f2d51f2+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 1}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 1}}},
+ {mt: mt, f: "segmented/oof", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
+ } {
+ m := Manifest{Text: testCase.mt}
+ var got []FileSegment
+ for fs := range m.FileSegmentIterByName(testCase.f) {
+ got = append(got, *fs)
+ }
+ if !reflect.DeepEqual(got, testCase.want) {
+ t.Errorf("For %#v:\n got %#v\n want %#v", testCase.f, got, testCase.want)
+ }
+ }
+}
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
index 48e3640..04af920 100644
--- a/services/keepdl/handler.go
+++ b/services/keepdl/handler.go
@@ -11,6 +11,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
var clientPool = arvadosclient.MakeClientPool()
@@ -136,17 +137,20 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
filename := strings.Join(targetPath, "/")
- rdr, err := arvadosclient.CollectionFileReader(collection, filename)
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ return
+ }
+ rdr, err := kc.CollectionFileReader(collection, filename)
if os.IsNotExist(err) {
statusCode = http.StatusNotFound
return
- } else if err == arvadosclient.ErrNotImplemented {
- statusCode = http.StatusNotImplemented
- return
} else if err != nil {
statusCode, statusText = http.StatusBadGateway, err.Error()
return
}
+ defer rdr.Close()
// One or both of these can be -1 if not found:
basenamePos := strings.LastIndex(filename, "/")
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
index 1b2bdf3..fa2674a 100644
--- a/services/keepdl/server_test.go
+++ b/services/keepdl/server_test.go
@@ -19,17 +19,7 @@ import (
var _ = check.Suite(&IntegrationSuite{})
var _ = check.Suite(&UnitSuite{})
-const (
- spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
- activeToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
- anonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
- fooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
- bogusCollection = "zzzzz-4zz18-totallynotexist"
- hwCollection = "zzzzz-4zz18-4en62shvi99lxd4"
- bogusPdh = "01234567890123456789012345678901+1234"
-)
-
-// IntegrationSuite tests need an API server and an arv-git-httpd server
+// IntegrationSuite tests need an API server and a keepdl server
type IntegrationSuite struct {
testServer *server
}
@@ -47,12 +37,12 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
"",
"bogustoken",
} {
- hdr, body := s.runCurl(c, token, "/collections/"+fooCollection+"/foo")
+ hdr, body := s.runCurl(c, token, "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
c.Check(body, check.Equals, "")
if token != "" {
- hdr, body = s.runCurl(c, token, "/collections/download/"+fooCollection+"/"+token+"/foo")
+ hdr, body = s.runCurl(c, token, "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
}
@@ -90,46 +80,46 @@ func (s *IntegrationSuite) Test404(c *check.C) {
"/download",
"/collections",
"/collections/",
- "/collections/" + fooCollection,
- "/collections/" + fooCollection + "/",
+ "/collections/" + arvadostest.FooCollection,
+ "/collections/" + arvadostest.FooCollection + "/",
// Non-existent file in collection
- "/collections/" + fooCollection + "/theperthcountyconspiracy",
- "/collections/download/" + fooCollection + "/" + activeToken + "/theperthcountyconspiracy",
+ "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
+ "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
// Non-existent collection
- "/collections/" + bogusCollection,
- "/collections/" + bogusCollection + "/",
- "/collections/" + bogusCollection + "/theperthcountyconspiracy",
- "/collections/download/" + bogusCollection + "/" + activeToken + "/theperthcountyconspiracy",
+ "/collections/" + arvadostest.NonexistentCollection,
+ "/collections/" + arvadostest.NonexistentCollection + "/",
+ "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
+ "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
} {
- hdr, body := s.runCurl(c, activeToken, uri)
+ hdr, body := s.runCurl(c, arvadostest.ActiveToken, uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
c.Check(body, check.Equals, "")
}
}
func (s *IntegrationSuite) Test200(c *check.C) {
- anonymousTokens = []string{anonymousToken}
+ anonymousTokens = []string{arvadostest.AnonymousToken}
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
- arv.ApiToken = activeToken
+ arv.ApiToken = arvadostest.ActiveToken
kc, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, check.Equals, nil)
kc.PutB([]byte("Hello world\n"))
kc.PutB([]byte("foo"))
for _, spec := range [][]string{
// My collection
- {activeToken, "/collections/" + fooCollection + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
- {"", "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
- {"tokensobogus", "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
- {activeToken, "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
- {anonymousToken, "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {arvadostest.ActiveToken, "/collections/" + arvadostest.FooCollection + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"", "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"tokensobogus", "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {arvadostest.ActiveToken, "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {arvadostest.AnonymousToken, "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
// Anonymously accessible user agreement. These should
// start working when CollectionFileReader provides
// real data instead of fake/stub data.
- {"", "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
- {activeToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
- {spectatorToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
- {spectatorToken, "/collections/download/"+hwCollection+"/"+spectatorToken+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {"", "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {arvadostest.ActiveToken, "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {arvadostest.SpectatorToken, "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {arvadostest.SpectatorToken, "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
} {
hdr, body := s.runCurl(c, spec[0], spec[1])
if strings.HasPrefix(hdr, "HTTP/1.1 501 Not Implemented\r\n") && body == "" {
commit 5269624aeb5cd29dcb8b488bf8a297fdb6c12e0e
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 23 01:20:28 2015 -0400
5824: Start unit tests for vhost URLs.
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
index 66c6812..1b2bdf3 100644
--- a/services/keepdl/server_test.go
+++ b/services/keepdl/server_test.go
@@ -3,6 +3,9 @@ package main
import (
"crypto/md5"
"fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
"os/exec"
"strings"
"testing"
@@ -14,6 +17,7 @@ import (
)
var _ = check.Suite(&IntegrationSuite{})
+var _ = check.Suite(&UnitSuite{})
const (
spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
@@ -22,6 +26,7 @@ const (
fooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
bogusCollection = "zzzzz-4zz18-totallynotexist"
hwCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+ bogusPdh = "01234567890123456789012345678901+1234"
)
// IntegrationSuite tests need an API server and an arv-git-httpd server
@@ -29,6 +34,14 @@ type IntegrationSuite struct {
testServer *server
}
+func mustParseURL(s string) url.URL {
+ r, err := url.Parse(s)
+ if err != nil {
+ panic("parse URL: " + s)
+ }
+ return r
+}
+
func (s *IntegrationSuite) TestNoToken(c *check.C) {
for _, token := range []string{
"",
@@ -50,6 +63,21 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
}
}
+func (s *UnitSuite) TestVhost404(c *check.C) {
+ for _, testURL := range []string{
+ bogusCollection + ".example.com/theperthcountyconspiracy",
+ bogusCollection + ".example.com/theperthcountyconspiracy?t=" + spectatorToken,
+ } {
+ resp := httptest.NewRecorder()
+ req := http.Request{
+ URL: mustParseURL(testURL),
+ }
+ handler{}.ServeHTTP(resp, req)
+ c.Check(resp.Body.Code, check.Equals, http.StatusNotFound)
+ c.Check(resp.Body.String(), check.Equals, "")
+ }
+}
+
// TODO: Move most cases to functional tests -- at least use Go's own
// http client instead of forking curl. Just leave enough of an
// integration test to assure that the documented way of invoking curl
commit f3ae4368efc15797d1d9c6179ca4ba460bdd5da0
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 23 01:09:02 2015 -0400
5824: Add doc.go
diff --git a/services/keepdl/doc.go b/services/keepdl/doc.go
new file mode 100644
index 0000000..65c7f19
--- /dev/null
+++ b/services/keepdl/doc.go
@@ -0,0 +1,57 @@
+// Keepdl provides read-only HTTP access to files stored in Keep. It
+// serves public data to anonymous and unauthenticated clients, and
+// accepts authentication via Arvados tokens. It can be installed
+// anywhere with access to Keep services, typically behind a web proxy
+// that provides SSL support.
+//
+// Given that this amounts to a web hosting service for arbitrary
+// content, it is vital to ensure that at least one of the following is
+// true:
+//
+// Usage
+//
+// Listening:
+//
+// keepdl -address=:1234
+//
+// Start an HTTP server on port 1234.
+//
+// keepdl -address=1.2.3.4:1234
+//
+// Start an HTTP server on port 1234, on the interface with IP address 1.2.3.4.
+//
+// Keepdl does not support SSL natively. Typically, it is installed
+// behind a proxy like nginx.
+//
+package main
+
+// TODO(TC): Implement
+//
+// Trusted content
+//
+// Normally, Keepdl is installed using a wildcard DNS entry and a
+// wildcard HTTPS certificate, serving data from collection X at
+// ``https://X.dl.example.com/path/file.ext''.
+//
+// It will also serve publicly accessible data at
+// ``https://dl.example.com/collections/X/path/file.txt'', but it does not
+// accept any kind of credentials at paths like these.
+//
+// In "trust all content" mode, Keepdl will accept credentials (API
+// tokens) and serve any collection X at
+// "https://dl.example.com/collections/X/path/file.ext". This is
+// UNSAFE except in the special case where everyone who is able write
+// ANY data to Keep, and every JavaScript and HTML file written to
+// Keep, is also trusted to read ALL of the data in Keep.
+//
+// In such cases you can enable trust-all-content mode.
+//
+// keepdl -trust-all-content [...]
+//
+// In the general case, this should not be enabled: A web page stored
+// in collection X can execute JavaScript code that uses the current
+// viewer's credentials to download additional data -- data which is
+// accessible to the current viewer, but not to the author of
+// collection X -- from the same origin (``https://dl.example.com/'')
+// and upload it to some other site chosen by the author of collection
+// X.
commit d40945c358950d9e43e4ad0aa1ec9fd02353090d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jun 23 19:12:58 2015 -0400
5824: Add install doc
diff --git a/doc/install/install-keepdl.html.textile.liquid b/doc/install/install-keepdl.html.textile.liquid
new file mode 100644
index 0000000..6730dff
--- /dev/null
+++ b/doc/install/install-keepdl.html.textile.liquid
@@ -0,0 +1,64 @@
+---
+layout: default
+navsection: installguide
+title: Install download server
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+The keepdl server provides read-only HTTP access to files stored in Keep. It serves public data to anonymous and unauthenticated clients, and accepts authentication via Arvados tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides SSL support.
+
+By convention, we use the following hostname for the download service:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|dl. at uuid_prefix@.your.domain|
+</div>
+
+This hostname should resolve from anywhere on the internet.
+
+h2. Install keepdl
+
+First add the Arvados apt repository, and then install the keepdl package.
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" | sudo tee /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">sudo /usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
+~$ <span class="userinput">sudo /usr/bin/apt-get update</span>
+~$ <span class="userinput">sudo /usr/bin/apt-get install keepdl</span>
+</code></pre>
+</notextile>
+
+Verify that @keepdl@ is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keepdl -h</span>
+Usage of keepdl:
+ -address="0.0.0.0:80": Address to listen on, "host:port".
+</code></pre>
+</notextile>
+
+We recommend running @arv-git-httpd@ under "runit":https://packages.debian.org/search?keywords=runit or something similar.
+
+Your @run@ script should look something like this:
+
+<notextile>
+<pre><code>export ARVADOS_API_HOST=<span class="userinput">uuid_prefix</span>.your.domain
+exec sudo -u nobody keepdl -address=:9002 2>&1
+</code></pre>
+</notextile>
+
+h3. Set up a reverse proxy with SSL support
+
+The keepdl service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
+
+This is best achieved by putting a reverse proxy with SSL support in front of keepdl, running on port 443 and passing requests to keepdl on port 9002 (or whatever port you chose in your run script).
+
+h3. Tell the API server about the keepdl service
+
+In your API server's config/application.yml file, add the following entry:
+
+<notextile>
+<pre><code>keepdl: dl.<span class="userinput">uuid_prefix</span>.your.domain
+</code></pre>
+</notextile>
commit bed6a3d54844b6ba41f5f7803f3b289783e02e5f
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 17 02:47:49 2015 -0400
5824: Assign MIME type by file extension. closes #6327
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
index bbcd53c..48e3640 100644
--- a/services/keepdl/handler.go
+++ b/services/keepdl/handler.go
@@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
+ "mime"
"net/http"
"os"
"strings"
@@ -146,6 +147,17 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
statusCode, statusText = http.StatusBadGateway, err.Error()
return
}
+
+ // One or both of these can be -1 if not found:
+ basenamePos := strings.LastIndex(filename, "/")
+ extPos := strings.LastIndex(filename, ".")
+ if extPos > basenamePos {
+ // Now extPos is safely >= 0.
+ if t := mime.TypeByExtension(filename[extPos:]); t != "" {
+ w.Header().Set("Content-Type", t)
+ }
+ }
+
_, err = io.Copy(w, rdr)
if err != nil {
statusCode, statusText = http.StatusBadGateway, err.Error()
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
index 1c36f98..66c6812 100644
--- a/services/keepdl/server_test.go
+++ b/services/keepdl/server_test.go
@@ -109,6 +109,12 @@ func (s *IntegrationSuite) Test200(c *check.C) {
continue
}
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ if strings.HasSuffix(spec[1], ".txt") {
+ c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
+ // TODO: Check some types that aren't
+ // automatically detected by Go's http server
+ // by sniffing the content.
+ }
c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec[2])
}
}
commit 2534cab13807ee2614400365b1cb6a4649c6678e
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 23 00:02:11 2015 -0400
5824: Add keepdl.
diff --git a/services/keepdl/.gitignore b/services/keepdl/.gitignore
new file mode 100644
index 0000000..173e306
--- /dev/null
+++ b/services/keepdl/.gitignore
@@ -0,0 +1 @@
+keepdl
diff --git a/services/keepdl/handler.go b/services/keepdl/handler.go
new file mode 100644
index 0000000..bbcd53c
--- /dev/null
+++ b/services/keepdl/handler.go
@@ -0,0 +1,153 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/auth"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var clientPool = arvadosclient.MakeClientPool()
+
+var anonymousTokens []string
+
+type handler struct{}
+
+func init() {
+ // TODO(TC): Get anonymousTokens from flags
+ anonymousTokens = []string{}
+}
+
+func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
+ var statusCode int
+ var statusText string
+
+ w := httpserver.WrapResponseWriter(wOrig)
+ defer func() {
+ if statusCode > 0 {
+ if w.WroteStatus() == 0 {
+ w.WriteHeader(statusCode)
+ } else {
+ httpserver.Log(r.RemoteAddr, "WARNING",
+ fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
+ }
+ }
+ if statusText == "" {
+ statusText = http.StatusText(statusCode)
+ }
+ httpserver.Log(r.RemoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.URL.Path)
+ }()
+
+ arv := clientPool.Get()
+ if arv == nil {
+ statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
+ return
+ }
+ defer clientPool.Put(arv)
+
+ pathParts := strings.Split(r.URL.Path[1:], "/")
+
+ if len(pathParts) < 3 || pathParts[0] != "collections" || pathParts[1] == "" || pathParts[2] == "" {
+ statusCode = http.StatusNotFound
+ return
+ }
+
+ var targetId string
+ var targetPath []string
+ var tokens []string
+ var reqTokens []string
+ var pathToken bool
+ if len(pathParts) >= 5 && pathParts[1] == "download" {
+ // "/collections/download/{id}/{token}/path..." form:
+ // Don't use our configured anonymous tokens,
+ // Authorization headers, etc. Just use the token in
+ // the path.
+ targetId = pathParts[2]
+ tokens = []string{pathParts[3]}
+ targetPath = pathParts[4:]
+ pathToken = true
+ } else {
+ // "/collections/{id}/path..." form
+ targetId = pathParts[1]
+ reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
+ tokens = append(reqTokens, anonymousTokens...)
+ targetPath = pathParts[2:]
+ }
+
+ tokenResult := make(map[string]int)
+ collection := make(map[string]interface{})
+ found := false
+ for _, arv.ApiToken = range tokens {
+ err := arv.Get("collections", targetId, nil, &collection)
+ httpserver.Log(err)
+ if err == nil {
+ // Success
+ found = true
+ break
+ }
+ if srvErr, ok := err.(arvadosclient.APIServerError); ok {
+ switch srvErr.HttpStatusCode {
+ case 404, 401:
+ // Token broken or insufficient to
+ // retrieve collection
+ tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
+ continue
+ }
+ }
+ // Something more serious is wrong
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ return
+ }
+ if !found {
+ if pathToken {
+ // The URL is a "secret sharing link", but it
+ // didn't work out. Asking the client for
+ // additional credentials would just be
+ // confusing.
+ statusCode = http.StatusNotFound
+ return
+ }
+ for _, t := range reqTokens {
+ if tokenResult[t] == 404 {
+ // The client provided valid token(s), but the
+ // collection was not found.
+ statusCode = http.StatusNotFound
+ return
+ }
+ }
+ // The client's token was invalid (e.g., expired), or
+ // the client didn't even provide one. Propagate the
+ // 401 to encourage the client to use a [different]
+ // token.
+ //
+ // TODO(TC): This response would be confusing to
+ // someone trying (anonymously) to download public
+ // data that has been deleted. Allow a referrer to
+ // provide this context somehow?
+ statusCode = http.StatusUnauthorized
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"dl\"")
+ return
+ }
+
+ filename := strings.Join(targetPath, "/")
+ rdr, err := arvadosclient.CollectionFileReader(collection, filename)
+ if os.IsNotExist(err) {
+ statusCode = http.StatusNotFound
+ return
+ } else if err == arvadosclient.ErrNotImplemented {
+ statusCode = http.StatusNotImplemented
+ return
+ } else if err != nil {
+ statusCode, statusText = http.StatusBadGateway, err.Error()
+ return
+ }
+ _, err = io.Copy(w, rdr)
+ if err != nil {
+ statusCode, statusText = http.StatusBadGateway, err.Error()
+ }
+}
diff --git a/services/keepdl/main.go b/services/keepdl/main.go
new file mode 100644
index 0000000..d780cc3
--- /dev/null
+++ b/services/keepdl/main.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+ "flag"
+ "log"
+ "os"
+)
+
+func init() {
+ // MakeArvadosClient returns an error if this env var isn't
+ // available as a default token (even if we explicitly set a
+ // different token before doing anything with the client). We
+ // set this dummy value during init so it doesn't clobber the
+ // one used by "run test servers".
+ os.Setenv("ARVADOS_API_TOKEN", "xxx")
+}
+
+func main() {
+ flag.Parse()
+ srv := &server{}
+ if err := srv.Start(); err != nil {
+ log.Fatal(err)
+ }
+ log.Println("Listening at", srv.Addr)
+ if err := srv.Wait(); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/services/keepdl/server.go b/services/keepdl/server.go
new file mode 100644
index 0000000..44da00f
--- /dev/null
+++ b/services/keepdl/server.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+ "flag"
+ "net/http"
+
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+)
+
+var address string
+
+func init() {
+ flag.StringVar(&address, "address", "0.0.0.0:80",
+ "Address to listen on, \"host:port\".")
+}
+
+type server struct {
+ httpserver.Server
+}
+
+func (srv *server) Start() error {
+ mux := http.NewServeMux()
+ mux.Handle("/", &handler{})
+ srv.Handler = mux
+ srv.Addr = address
+ return srv.Server.Start()
+}
diff --git a/services/keepdl/server_test.go b/services/keepdl/server_test.go
new file mode 100644
index 0000000..1c36f98
--- /dev/null
+++ b/services/keepdl/server_test.go
@@ -0,0 +1,170 @@
+package main
+
+import (
+ "crypto/md5"
+ "fmt"
+ "os/exec"
+ "strings"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+const (
+ spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
+ activeToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ anonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
+ fooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ bogusCollection = "zzzzz-4zz18-totallynotexist"
+ hwCollection = "zzzzz-4zz18-4en62shvi99lxd4"
+)
+
+// IntegrationSuite tests need an API server and an arv-git-httpd server
+type IntegrationSuite struct {
+ testServer *server
+}
+
+func (s *IntegrationSuite) TestNoToken(c *check.C) {
+ for _, token := range []string{
+ "",
+ "bogustoken",
+ } {
+ hdr, body := s.runCurl(c, token, "/collections/"+fooCollection+"/foo")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
+ c.Check(body, check.Equals, "")
+
+ if token != "" {
+ hdr, body = s.runCurl(c, token, "/collections/download/"+fooCollection+"/"+token+"/foo")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+ c.Check(body, check.Equals, "")
+ }
+
+ hdr, body = s.runCurl(c, token, "/bad-route")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
+ c.Check(body, check.Equals, "")
+ }
+}
+
+// TODO: Move most cases to functional tests -- at least use Go's own
+// http client instead of forking curl. Just leave enough of an
+// integration test to assure that the documented way of invoking curl
+// really works against the server.
+func (s *IntegrationSuite) Test404(c *check.C) {
+ for _, uri := range []string{
+ // Routing errors
+ "/",
+ "/foo",
+ "/download",
+ "/collections",
+ "/collections/",
+ "/collections/" + fooCollection,
+ "/collections/" + fooCollection + "/",
+ // Non-existent file in collection
+ "/collections/" + fooCollection + "/theperthcountyconspiracy",
+ "/collections/download/" + fooCollection + "/" + activeToken + "/theperthcountyconspiracy",
+ // Non-existent collection
+ "/collections/" + bogusCollection,
+ "/collections/" + bogusCollection + "/",
+ "/collections/" + bogusCollection + "/theperthcountyconspiracy",
+ "/collections/download/" + bogusCollection + "/" + activeToken + "/theperthcountyconspiracy",
+ } {
+ hdr, body := s.runCurl(c, activeToken, uri)
+ c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
+ c.Check(body, check.Equals, "")
+ }
+}
+
+func (s *IntegrationSuite) Test200(c *check.C) {
+ anonymousTokens = []string{anonymousToken}
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = activeToken
+ kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, check.Equals, nil)
+ kc.PutB([]byte("Hello world\n"))
+ kc.PutB([]byte("foo"))
+ for _, spec := range [][]string{
+ // My collection
+ {activeToken, "/collections/" + fooCollection + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"", "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {"tokensobogus", "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {activeToken, "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ {anonymousToken, "/collections/download/" + fooCollection + "/" + activeToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"},
+ // Anonymously accessible user agreement. These should
+ // start working when CollectionFileReader provides
+ // real data instead of fake/stub data.
+ {"", "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {activeToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {spectatorToken, "/collections/"+hwCollection+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ {spectatorToken, "/collections/download/"+hwCollection+"/"+spectatorToken+"/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"},
+ } {
+ hdr, body := s.runCurl(c, spec[0], spec[1])
+ if strings.HasPrefix(hdr, "HTTP/1.1 501 Not Implemented\r\n") && body == "" {
+ c.Log("Not implemented!")
+ continue
+ }
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec[2])
+ }
+}
+
+// Return header block and body.
+func (s *IntegrationSuite) runCurl(c *check.C, token, uri string, args ...string) (hdr, body string) {
+ curlArgs := []string{"--silent", "--show-error", "--include"}
+ if token != "" {
+ curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ }
+ curlArgs = append(curlArgs, args...)
+ curlArgs = append(curlArgs, "http://"+s.testServer.Addr+uri)
+ c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
+ output, err := exec.Command("curl", curlArgs...).CombinedOutput()
+ // Without "-f", curl exits 0 as long as it gets a valid HTTP
+ // response from the server, even if the response status
+ // indicates that the request failed. In our test suite, we
+ // always expect a valid HTTP response, and we parse the
+ // headers ourselves. If curl exits non-zero, our testing
+ // environment is broken.
+ c.Assert(err, check.Equals, nil)
+ hdrsAndBody := strings.SplitN(string(output), "\r\n\r\n", 2)
+ c.Assert(len(hdrsAndBody), check.Equals, 2)
+ hdr = hdrsAndBody[0]
+ body = hdrsAndBody[1]
+ return
+}
+
+func (s *IntegrationSuite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+}
+
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
+}
+
+func (s *IntegrationSuite) SetUpTest(c *check.C) {
+ arvadostest.ResetEnv()
+ s.testServer = &server{}
+ var err error
+ address = "127.0.0.1:0"
+ err = s.testServer.Start()
+ c.Assert(err, check.Equals, nil)
+}
+
+func (s *IntegrationSuite) TearDownTest(c *check.C) {
+ var err error
+ if s.testServer != nil {
+ err = s.testServer.Close()
+ }
+ c.Check(err, check.Equals, nil)
+}
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list