[ARVADOS] updated: cddcee0f27a7f27d4a7c1bc58a6d1b384b83ec53
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 15 17:01:02 EDT 2015
Summary of changes:
.../test/integration/collection_upload_test.rb | 28 +++++++++++++++-------
sdk/go/keepclient/keepclient.go | 2 +-
2 files changed, 20 insertions(+), 10 deletions(-)
discards 0a38db3319886a477c7c204396df2ffa1cfd61d2 (commit)
discards 448da21288cc106e9095279925dbf691668e3ce0 (commit)
discards b12996a9d9ad7bf527b40639b078b3cea6640a9d (commit)
discards 600f427244a0242d26e9fe027916e4146034dbf4 (commit)
discards 44a382de4e7b8986c17e5baeae94f8e521d923b6 (commit)
discards 1f04571a18a218c0968c92609a481913ec134171 (commit)
discards 9fa083d19ca4c88c7e2802852b6a89da0575c73e (commit)
discards 35d3f6566426986cc6da4a5ec4c4844cbbd17ce2 (commit)
via cddcee0f27a7f27d4a7c1bc58a6d1b384b83ec53 (commit)
via 34a97b7577fec489eb9e866ccc9c2f5df709b52e (commit)
via 0381cd5f601ac996e47a0e9a6887926b3222f85f (commit)
via acd0ee16f6ebf6341dc1df5ba37c9896f720dcb8 (commit)
via 670fa72d01d23fc2d10c7ad61dab961a49e1772d (commit)
via e6616df2a2d933fd5fd5ab63124d6ad254c0ef56 (commit)
via f1a1faf919edce78261d0ac252758d1bf01d69e3 (commit)
via 8ad914c4d6976d4d862300514d65d555126481ca (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (0a38db3319886a477c7c204396df2ffa1cfd61d2)
\
N -- N -- N (cddcee0f27a7f27d4a7c1bc58a6d1b384b83ec53)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit cddcee0f27a7f27d4a7c1bc58a6d1b384b83ec53
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Oct 14 04:07:37 2015 -0400
5824: Update bundle
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index 20b8d61..8b2118c 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -74,7 +74,7 @@ GEM
rack (>= 1.0.0)
rack-test (>= 0.5.4)
xpath (~> 2.0)
- childprocess (0.5.5)
+ childprocess (0.5.6)
ffi (~> 1.0, >= 1.0.11)
cliver (0.3.2)
coffee-rails (4.1.0)
@@ -98,7 +98,7 @@ GEM
fast_stack (0.1.0)
rake
rake-compiler
- ffi (1.9.6)
+ ffi (1.9.10)
flamegraph (0.1.0)
fast_stack
google-api-client (0.6.4)
@@ -139,7 +139,7 @@ GEM
metaclass (~> 0.0.1)
morrisjs-rails (0.5.1)
railties (> 3.1, < 5)
- multi_json (1.11.1)
+ multi_json (1.11.2)
multipart-post (1.2.0)
net-scp (1.2.1)
net-ssh (>= 2.6.5)
@@ -192,7 +192,7 @@ GEM
ref (1.0.5)
ruby-debug-passenger (0.2.0)
ruby-prof (0.15.2)
- rubyzip (1.1.6)
+ rubyzip (1.1.7)
rvm-capistrano (1.5.5)
capistrano (~> 2.15.4)
sass (3.4.9)
@@ -202,7 +202,7 @@ GEM
sprockets (>= 2.8, < 4.0)
sprockets-rails (>= 2.0, < 4.0)
tilt (~> 1.1)
- selenium-webdriver (2.44.0)
+ selenium-webdriver (2.48.1)
childprocess (~> 0.5)
multi_json (~> 1.0)
rubyzip (~> 1.0)
@@ -239,7 +239,7 @@ GEM
execjs (>= 0.3.0)
json (>= 1.8.0)
uuidtools (2.1.5)
- websocket (1.2.1)
+ websocket (1.2.2)
websocket-driver (0.5.1)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.1)
@@ -294,3 +294,6 @@ DEPENDENCIES
therubyracer
uglifier (>= 1.0.3)
wiselinks
+
+BUNDLED WITH
+ 1.10.6
commit 34a97b7577fec489eb9e866ccc9c2f5df709b52e
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Oct 13 10:52:06 2015 -0400
5824: Use keep-web in Workbench integration tests
diff --git a/apps/workbench/test/helpers/download_helper.rb b/apps/workbench/test/helpers/download_helper.rb
new file mode 100644
index 0000000..21fb4cd
--- /dev/null
+++ b/apps/workbench/test/helpers/download_helper.rb
@@ -0,0 +1,21 @@
+module DownloadHelper
+ module_function
+
+ def path
+ Rails.root.join 'tmp', 'downloads'
+ end
+
+ def clear
+ FileUtils.rm_f path
+ begin
+ Dir.mkdir path
+ rescue Errno::EEXIST
+ end
+ end
+
+ def done
+ Dir[path.join '*'].reject do |f|
+ /\.part$/ =~ f
+ end
+ end
+end
diff --git a/apps/workbench/test/integration/collection_upload_test.rb b/apps/workbench/test/integration/collection_upload_test.rb
index 62efee4..5e407ce 100644
--- a/apps/workbench/test/integration/collection_upload_test.rb
+++ b/apps/workbench/test/integration/collection_upload_test.rb
@@ -7,9 +7,19 @@ class CollectionUploadTest < ActionDispatch::IntegrationTest
io.write content
end
end
+ # Database reset doesn't restore KeepServices; we have to
+ # save/restore manually.
+ use_token :admin do
+ @keep_services = KeepService.all.to_a
+ end
end
teardown do
+ use_token :admin do
+ @keep_services.each do |ks|
+ KeepService.find(ks.uuid).update_attributes(ks.attributes)
+ end
+ end
testfiles.each do |filename, _|
File.unlink(testfile_path filename)
end
@@ -64,10 +74,9 @@ class CollectionUploadTest < ActionDispatch::IntegrationTest
test "Report mixed-content error" do
skip 'Test suite does not use TLS'
need_selenium "to make file uploads work"
- begin
- use_token :admin
- proxy = KeepService.find(api_fixture('keep_services')['proxy']['uuid'])
- proxy.update_attributes service_ssl_flag: false
+ use_token :admin do
+ KeepService.where(service_type: 'proxy').first.
+ update_attributes(service_ssl_flag: false)
end
visit page_with_token 'active', sandbox_path
find('.nav-tabs a', text: 'Upload').click
@@ -82,11 +91,12 @@ class CollectionUploadTest < ActionDispatch::IntegrationTest
test "Report network error" do
need_selenium "to make file uploads work"
- begin
- use_token :admin
- proxy = KeepService.find(api_fixture('keep_services')['proxy']['uuid'])
- # Even if you somehow do port>2^16, surely nx.example.net won't respond
- proxy.update_attributes service_host: 'nx.example.net', service_port: 99999
+ use_token :admin do
+ # Even if you somehow do port>2^16, surely nx.example.net won't
+ # respond
+ KeepService.where(service_type: 'proxy').first.
+ update_attributes(service_host: 'nx.example.net',
+ service_port: 99999)
end
visit page_with_token 'active', sandbox_path
find('.nav-tabs a', text: 'Upload').click
diff --git a/apps/workbench/test/integration/download_test.rb b/apps/workbench/test/integration/download_test.rb
new file mode 100644
index 0000000..3f8eaf2
--- /dev/null
+++ b/apps/workbench/test/integration/download_test.rb
@@ -0,0 +1,38 @@
+require 'integration_helper'
+require 'helpers/download_helper'
+
+class DownloadTest < ActionDispatch::IntegrationTest
+ setup do
+ portfile = File.expand_path '../../../../../tmp/keep-web-ssl.port', __FILE__
+ @kwport = File.read portfile
+ Rails.configuration.keep_web_url = "https://localhost:#{@kwport}/c=%{uuid_or_pdh}"
+ CollectionsController.any_instance.expects(:file_enumerator).never
+
+ # Make sure Capybara can download files.
+ need_selenium 'for downloading', :selenium_with_download
+ DownloadHelper.clear
+
+ # Keep data isn't populated by fixtures, so we have to write any
+ # data we expect to read.
+ unless /^acbd/ =~ `echo -n foo | arv-put --no-progress --raw -` && $?.success?
+ raise $?.to_s
+ end
+ end
+
+ test "download from keep-web with a reader token" do
+ uuid = api_fixture('collections')['foo_file']['uuid']
+ token = api_fixture('api_client_authorizations')['active_all_collections']['api_token']
+ visit "/collections/download/#{uuid}/#{token}/"
+ within "#collection_files" do
+ click_link "foo"
+ end
+ data = nil
+ tries = 0
+ while tries < 20
+ sleep 0.1
+ tries += 1
+ data = File.read(DownloadHelper.path.join 'foo') rescue nil
+ end
+ assert_equal 'foo', data
+ end
+end
diff --git a/apps/workbench/test/integration_helper.rb b/apps/workbench/test/integration_helper.rb
index 39fdf4b..5750a1b 100644
--- a/apps/workbench/test/integration_helper.rb
+++ b/apps/workbench/test/integration_helper.rb
@@ -19,6 +19,17 @@ Capybara.register_driver :poltergeist_without_file_api do |app|
Capybara::Poltergeist::Driver.new app, POLTERGEIST_OPTS.merge(extensions: [js])
end
+Capybara.register_driver :selenium_with_download do |app|
+ profile = Selenium::WebDriver::Firefox::Profile.new
+ profile['browser.download.dir'] = DownloadHelper.path.to_s
+ profile['browser.download.downloadDir'] = DownloadHelper.path.to_s
+ profile['browser.download.defaultFolder'] = DownloadHelper.path.to_s
+ profile['browser.download.folderList'] = 2 # "save to user-defined location"
+ profile['browser.download.manager.showWhenStarting'] = false
+ profile['browser.helperApps.alwaysAsk.force'] = false
+ Capybara::Selenium::Driver.new app, profile: profile
+end
+
module WaitForAjax
Capybara.default_wait_time = 5
def wait_for_ajax
@@ -73,8 +84,8 @@ module HeadlessHelper
end
end
- def need_selenium reason=nil
- Capybara.current_driver = :selenium
+ def need_selenium reason=nil, driver=:selenium
+ Capybara.current_driver = driver
unless ENV['ARVADOS_TEST_HEADFUL'] or @headless
@headless = HeadlessSingleton.get
@headless.start
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index 89d15c6..41592af 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -176,7 +176,10 @@ class ApiServerForTests
# though it doesn't need to start up a new server).
env_script = check_output %w(python ./run_test_server.py start --auth admin)
check_output %w(python ./run_test_server.py start_arv-git-httpd)
+ check_output %w(python ./run_test_server.py start_keep-web)
check_output %w(python ./run_test_server.py start_nginx)
+ # This one isn't a no-op, even under run-tests.sh.
+ check_output %w(python ./run_test_server.py start_keep)
end
test_env = {}
env_script.each_line do |line|
@@ -192,9 +195,11 @@ class ApiServerForTests
def stop_test_server
Dir.chdir PYTHON_TESTS_DIR do
+ check_output %w(python ./run_test_server.py stop_keep)
# These are no-ops if we're running within run-tests.sh
check_output %w(python ./run_test_server.py stop_nginx)
check_output %w(python ./run_test_server.py stop_arv-git-httpd)
+ check_output %w(python ./run_test_server.py stop_keep-web)
check_output %w(python ./run_test_server.py stop)
end
@@server_is_running = false
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 6196605..885f84e 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -28,4 +28,18 @@ http {
proxy_pass http://keepproxy;
}
}
+ upstream keep-web {
+ server localhost:{{KEEPWEBPORT}};
+ }
+ server {
+ listen *:{{KEEPWEBSSLPORT}} ssl default_server;
+ server_name ~^(?<request_host>.*)$;
+ ssl_certificate {{SSLCERT}};
+ ssl_certificate_key {{SSLKEY}};
+ location / {
+ proxy_pass http://keep-web;
+ proxy_set_header Host $request_host:{{KEEPWEBPORT}};
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+ }
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 5d0c42a..b8de60f 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -437,10 +437,35 @@ def stop_arv_git_httpd():
return
kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
+def run_keep_web():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ stop_keep_web()
+
+ keepwebport = find_available_port()
+ env = os.environ.copy()
+ env.pop('ARVADOS_API_TOKEN', None)
+ keepweb = subprocess.Popen(
+ ['keep-web',
+ '-attachment-only-host=localhost:'+str(keepwebport),
+ '-address=:'+str(keepwebport)],
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ with open(_pidfile('keep-web'), 'w') as f:
+ f.write(str(keepweb.pid))
+ _setport('keep-web', keepwebport)
+ _wait_until_port_listens(keepwebport)
+
+def stop_keep_web():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('keep-web'), wait=0)
+
def run_nginx():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
nginxconf = {}
+ nginxconf['KEEPWEBPORT'] = _getport('keep-web')
+ nginxconf['KEEPWEBSSLPORT'] = find_available_port()
nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
nginxconf['GITPORT'] = _getport('arv-git-httpd')
@@ -464,6 +489,7 @@ def run_nginx():
'-g', 'pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT'])
_setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
_setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
@@ -563,7 +589,8 @@ class TestCaseWithServers(unittest.TestCase):
for server_kwargs, start_func, stop_func in (
(cls.MAIN_SERVER, run, reset),
(cls.KEEP_SERVER, run_keep, stop_keep),
- (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
+ (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy),
+ (cls.KEEP_WEB_SERVER, run_keep_web, stop_keep_web)):
if server_kwargs is not None:
start_func(**server_kwargs)
cls._cleanup_funcs.append(stop_func)
@@ -589,6 +616,7 @@ if __name__ == "__main__":
'start', 'stop',
'start_keep', 'stop_keep',
'start_keep_proxy', 'stop_keep_proxy',
+ 'start_keep-web', 'stop_keep-web',
'start_arv-git-httpd', 'stop_arv-git-httpd',
'start_nginx', 'stop_nginx',
]
@@ -625,6 +653,10 @@ if __name__ == "__main__":
run_arv_git_httpd()
elif args.action == 'stop_arv-git-httpd':
stop_arv_git_httpd()
+ elif args.action == 'start_keep-web':
+ run_keep_web()
+ elif args.action == 'stop_keep-web':
+ stop_keep_web()
elif args.action == 'start_nginx':
run_nginx()
elif args.action == 'stop_nginx':
diff --git a/services/api/app/controllers/database_controller.rb b/services/api/app/controllers/database_controller.rb
index 64818da..21c8e47 100644
--- a/services/api/app/controllers/database_controller.rb
+++ b/services/api/app/controllers/database_controller.rb
@@ -29,6 +29,10 @@ class DatabaseController < ApplicationController
fixturesets = Dir.glob(Rails.root.join('test', 'fixtures', '*.yml')).
collect { |yml| yml.match(/([^\/]*)\.yml$/)[1] }
+ # Don't reset keep_services: clients need to discover our
+ # integration-testing keepstores, not test fixtures.
+ fixturesets -= %w[keep_services]
+
table_names = '"' + ActiveRecord::Base.connection.tables.join('","') + '"'
attempts_left = 20
diff --git a/services/api/test/fixtures/api_client_authorizations.yml b/services/api/test/fixtures/api_client_authorizations.yml
index 9199d17..ecb9adb 100644
--- a/services/api/test/fixtures/api_client_authorizations.yml
+++ b/services/api/test/fixtures/api_client_authorizations.yml
@@ -87,7 +87,7 @@ active_all_collections:
user: active
api_token: activecollectionsabcdefghijklmnopqrstuvwxyz1234567
expires_at: 2038-01-01 00:00:00
- scopes: ["GET /arvados/v1/collections/", "GET /arvados/v1/keep_disks"]
+ scopes: ["GET /arvados/v1/collections/", "GET /arvados/v1/keep_services", "GET /arvados/v1/keep_services/"]
active_userlist:
api_client: untrusted
commit 0381cd5f601ac996e47a0e9a6887926b3222f85f
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 12 19:15:06 2015 -0400
5824: Add option to redirect Workbench downloads to a keep-web service
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index e01151c..38b58a1 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -1,4 +1,6 @@
require "arvados/keep"
+require "uri"
+require "cgi"
class CollectionsController < ApplicationController
include ActionController::Live
@@ -130,11 +132,27 @@ class CollectionsController < ApplicationController
usable_token = find_usable_token(tokens) do
coll = Collection.find(params[:uuid])
end
+ if usable_token.nil?
+ # Response already rendered.
+ return
+ end
+
+ if Rails.configuration.keep_web_url
+ opts = {}
+ if usable_token == params[:reader_token]
+ opts[:path_token] = usable_token
+ elsif usable_token == Rails.configuration.anonymous_user_token
+ # Don't pass a token at all
+ else
+ # We pass the current user's real token only if it's necessary
+ # to read the collection.
+ opts[:query_token] = usable_token
+ end
+ return redirect_to keep_web_url(params[:uuid], params[:file], opts)
+ end
file_name = params[:file].andand.sub(/^(\.\/|\/|)/, './')
- if usable_token.nil?
- return # Response already rendered.
- elsif file_name.nil? or not coll.manifest.has_file?(file_name)
+ if file_name.nil? or not coll.manifest.has_file?(file_name)
return render_not_found
end
@@ -305,6 +323,21 @@ class CollectionsController < ApplicationController
return nil
end
+ def keep_web_url(uuid_or_pdh, file, opts)
+ fmt = {uuid_or_pdh: uuid_or_pdh.sub('+', '-')}
+ uri = URI.parse(Rails.configuration.keep_web_url % fmt)
+ uri.path += '/' unless uri.path.end_with? '/'
+ if opts[:path_token]
+ uri.path += 't=' + opts[:path_token] + '/'
+ end
+ uri.path += '_/'
+ uri.path += CGI::escape(file)
+ if opts[:query_token]
+ uri.query = 'api_token=' + CGI::escape(opts[:query_token])
+ end
+ uri.to_s
+ end
+
# Note: several controller and integration tests rely on stubbing
# file_enumerator to return fake file content.
def file_enumerator opts
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 00959bb..5504fd2 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -225,3 +225,11 @@ common:
# E.g., using a name-based proxy server to forward connections to shell hosts:
# https://%{hostname}.webshell.uuid_prefix.arvadosapi.com/
shell_in_a_box_url: false
+
+ # Format of download/preview links. If false, use Workbench's
+ # download facility.
+ #
+ # Examples:
+ # keep_web_url: https://%{uuid_or_pdh}.dl.zzzzz.your.domain
+ # keep_web_url: https://%{uuid_or_pdh}--dl.zzzzz.your.domain
+ keep_web_url: false
diff --git a/apps/workbench/test/controllers/collections_controller_test.rb b/apps/workbench/test/controllers/collections_controller_test.rb
index 13644e0..b4e7dd3 100644
--- a/apps/workbench/test/controllers/collections_controller_test.rb
+++ b/apps/workbench/test/controllers/collections_controller_test.rb
@@ -514,4 +514,55 @@ class CollectionsControllerTest < ActionController::TestCase
get :show, {id: api_fixture('collections')['user_agreement']['uuid']}, session_for(:active)
assert_not_includes @response.body, '<a href="#Upload"'
end
+
+ def setup_for_keep_web cfg='https://%{uuid_or_pdh}.dl.zzzzz.example'
+ Rails.configuration.keep_web_url = cfg
+ @controller.expects(:file_enumerator).never
+ end
+
+ %w(uuid portable_data_hash).each do |id_type|
+ test "Redirect to keep_web_url via #{id_type}" do
+ setup_for_keep_web
+ tok = api_fixture('api_client_authorizations')['active']['api_token']
+ id = api_fixture('collections')['w_a_z_file'][id_type]
+ get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
+ assert_response :redirect
+ assert_equal "https://#{id.sub '+', '-'}.dl.zzzzz.example/_/w+a+z?api_token=#{tok}", @response.redirect_url
+ end
+
+ test "Redirect to keep_web_url via #{id_type} with reader token" do
+ setup_for_keep_web
+ tok = api_fixture('api_client_authorizations')['active']['api_token']
+ id = api_fixture('collections')['w_a_z_file'][id_type]
+ get :show_file, {uuid: id, file: "w a z", reader_token: tok}, session_for(:expired)
+ assert_response :redirect
+ assert_equal "https://#{id.sub '+', '-'}.dl.zzzzz.example/t=#{tok}/_/w+a+z", @response.redirect_url
+ end
+
+ test "Redirect to keep_web_url via #{id_type} with no token" do
+ setup_for_keep_web
+ Rails.configuration.anonymous_user_token =
+ api_fixture('api_client_authorizations')['anonymous']['api_token']
+ id = api_fixture('collections')['public_text_file'][id_type]
+ get :show_file, {uuid: id, file: "Hello World.txt"}
+ assert_response :redirect
+ assert_equal "https://#{id.sub '+', '-'}.dl.zzzzz.example/_/Hello+World.txt", @response.redirect_url
+ end
+
+ test "Redirect to keep_web_url via #{id_type} using -attachment-only-host mode" do
+ setup_for_keep_web 'https://dl.zzzzz.example/c=%{uuid_or_pdh}'
+ tok = api_fixture('api_client_authorizations')['active']['api_token']
+ id = api_fixture('collections')['w_a_z_file'][id_type]
+ get :show_file, {uuid: id, file: "w a z"}, session_for(:active)
+ assert_response :redirect
+ assert_equal "https://dl.zzzzz.example/c=#{id.sub '+', '-'}/_/w+a+z?api_token=#{tok}", @response.redirect_url
+ end
+ end
+
+ test "No redirect to keep_web_url if collection not found" do
+ setup_for_keep_web
+ id = api_fixture('collections')['w_a_z_file']['uuid']
+ get :show_file, {uuid: id, file: "w a z"}, session_for(:spectator)
+ assert_response 404
+ end
end
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 8ae9490..cc47ebe 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -94,8 +94,8 @@
// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/foo
// http://zzzzz-4zz18-znfnqtbbv4spc3w.dl.example.com/_/foo
// http://zzzzz-4zz18-znfnqtbbv4spc3w--dl.example.com/_/foo
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3+45--foo.example.com/foo
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3+45--.invalid/foo
+// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo
+// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo
//
// An additional form is supported specifically to make it more
// convenient to maintain support for existing Workbench download
commit acd0ee16f6ebf6341dc1df5ba37c9896f720dcb8
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 1 22:16:51 2015 -0400
5824: gofmt
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index b39a941..9751cd1 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -21,9 +21,9 @@ import (
type handler struct{}
var (
- clientPool = arvadosclient.MakeClientPool()
- trustAllContent = false
- anonymousTokens []string
+ clientPool = arvadosclient.MakeClientPool()
+ trustAllContent = false
+ anonymousTokens []string
attachmentOnlyHost = ""
)
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index a64aeb5..9b5ab2a 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -16,7 +16,7 @@ import (
var _ = check.Suite(&UnitSuite{})
-type UnitSuite struct {}
+type UnitSuite struct{}
func mustParseURL(s string) *url.URL {
r, err := url.Parse(s)
@@ -34,7 +34,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
resp := httptest.NewRecorder()
req := &http.Request{
Method: "GET",
- URL: mustParseURL(testURL),
+ URL: mustParseURL(testURL),
}
(&handler{}).ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
@@ -52,7 +52,7 @@ func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
doVhostRequests(c, authzViaAuthzHeader)
}
func authzViaAuthzHeader(r *http.Request, tok string) int {
- r.Header.Add("Authorization", "OAuth2 " + tok)
+ r.Header.Add("Authorization", "OAuth2 "+tok)
return http.StatusUnauthorized
}
@@ -61,7 +61,7 @@ func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
}
func authzViaCookieValue(r *http.Request, tok string) int {
r.AddCookie(&http.Cookie{
- Name: "api_token",
+ Name: "api_token",
Value: auth.EncodeTokenCookie([]byte(tok)),
})
return http.StatusUnauthorized
@@ -120,8 +120,8 @@ func doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string)
u := mustParseURL("http://" + hostPath)
req := &http.Request{
Method: "GET",
- Host: u.Host,
- URL: u,
+ Host: u.Host,
+ URL: u,
Header: http.Header{},
}
failCode := authz(req, tok)
@@ -157,8 +157,8 @@ func doReq(req *http.Request) *httptest.ResponseRecorder {
u, _ := req.URL.Parse(resp.Header().Get("Location"))
req = &http.Request{
Method: "GET",
- Host: u.Host,
- URL: u,
+ Host: u.Host,
+ URL: u,
Header: http.Header{},
}
for _, c := range cookies {
@@ -169,8 +169,8 @@ func doReq(req *http.Request) *httptest.ResponseRecorder {
func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
- arvadostest.FooCollection + ".example.com/foo",
- "?api_token=" + arvadostest.ActiveToken,
+ arvadostest.FooCollection+".example.com/foo",
+ "?api_token="+arvadostest.ActiveToken,
"text/plain",
"",
http.StatusOK,
@@ -179,8 +179,8 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com/c=" + arvadostest.FooCollection + "/foo",
- "?api_token=" + arvadostest.ActiveToken,
+ "example.com/c="+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
"text/plain",
"",
http.StatusBadRequest,
@@ -193,8 +193,8 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
}(trustAllContent)
trustAllContent = true
s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com/c=" + arvadostest.FooCollection + "/foo",
- "?api_token=" + arvadostest.ActiveToken,
+ "example.com/c="+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
"text/plain",
"",
http.StatusOK,
@@ -208,16 +208,16 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
attachmentOnlyHost = "example.com:1234"
s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com/c=" + arvadostest.FooCollection + "/foo",
- "?api_token=" + arvadostest.ActiveToken,
+ "example.com/c="+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
"text/plain",
"",
http.StatusBadRequest,
)
resp := s.testVhostRedirectTokenToCookie(c, "GET",
- "example.com:1234/c=" + arvadostest.FooCollection + "/foo",
- "?api_token=" + arvadostest.ActiveToken,
+ "example.com:1234/c="+arvadostest.FooCollection+"/foo",
+ "?api_token="+arvadostest.ActiveToken,
"text/plain",
"",
http.StatusOK,
@@ -227,7 +227,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
- arvadostest.FooCollection + ".example.com/foo",
+ arvadostest.FooCollection+".example.com/foo",
"",
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
@@ -237,7 +237,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
- arvadostest.FooCollection + ".example.com/foo",
+ arvadostest.FooCollection+".example.com/foo",
"",
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
@@ -249,10 +249,10 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
u, _ := url.Parse(`http://` + hostPath + queryString)
req := &http.Request{
Method: method,
- Host: u.Host,
- URL: u,
+ Host: u.Host,
+ URL: u,
Header: http.Header{"Content-Type": {contentType}},
- Body: ioutil.NopCloser(strings.NewReader(body)),
+ Body: ioutil.NopCloser(strings.NewReader(body)),
}
resp := httptest.NewRecorder()
@@ -261,14 +261,14 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
c.Assert(resp.Code, check.Equals, expectStatus)
return resp
}
- c.Check(resp.Body.String(), check.Matches, `.*href="//` + regexp.QuoteMeta(html.EscapeString(hostPath)) + `".*`)
+ c.Check(resp.Body.String(), check.Matches, `.*href="//`+regexp.QuoteMeta(html.EscapeString(hostPath))+`".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
u, _ = u.Parse(resp.Header().Get("Location"))
req = &http.Request{
Method: "GET",
- Host: u.Host,
- URL: u,
+ Host: u.Host,
+ URL: u,
Header: http.Header{},
}
for _, c := range cookies {
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index fdbb50e..740d243 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -105,14 +105,14 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
err = arv.Create("collections",
map[string]interface{}{
"collection": map[string]interface{}{
- "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
+ "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
"manifest_text": mtext,
},
}, &coll)
c.Assert(err, check.Equals, nil)
uuid := coll["uuid"].(string)
- hdr, body, size := s.runCurl(c, arv.ApiToken, uuid + ".dl.example.com", "/testdata.bin")
+ hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".dl.example.com", "/testdata.bin")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
@@ -139,82 +139,82 @@ func (s *IntegrationSuite) Test200(c *check.C) {
for _, spec := range []curlCase{
// My collection
{
- auth: arvadostest.ActiveToken,
- host: arvadostest.FooCollection + "--dl.example.com",
- path: "/foo",
+ auth: arvadostest.ActiveToken,
+ host: arvadostest.FooCollection + "--dl.example.com",
+ path: "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".dl.example.com",
- path: "/t=" + arvadostest.ActiveToken + "/foo",
+ host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".dl.example.com",
+ path: "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
+ path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
+ path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: "tokensobogus",
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: "tokensobogus",
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: arvadostest.ActiveToken,
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: arvadostest.ActiveToken,
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: arvadostest.AnonymousToken,
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: arvadostest.AnonymousToken,
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
// Anonymously accessible user agreement
{
- path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
- path: "/Hello%20world.txt",
+ host: arvadostest.HelloWorldCollection + ".dl.example.com",
+ path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
- path: "/_/Hello%20world.txt",
+ host: arvadostest.HelloWorldCollection + ".dl.example.com",
+ path: "/_/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.ActiveToken,
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ auth: arvadostest.ActiveToken,
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- host: arvadostest.HelloWorldCollection + "--dl.example.com",
- path: "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ host: arvadostest.HelloWorldCollection + "--dl.example.com",
+ path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
} {
@@ -238,7 +238,7 @@ func (s *IntegrationSuite) Test200(c *check.C) {
func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
- curlArgs = append(curlArgs, "--resolve", host + ":" + testPort + ":" + testHost)
+ curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
if token != "" {
curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
}
commit 670fa72d01d23fc2d10c7ad61dab961a49e1772d
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Sep 7 03:43:59 2015 -0400
5824: Add -attachment-only-host feature.
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 993b9db..8ae9490 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -173,6 +173,19 @@
// (``https://dl.example.com/'') and upload it to some other site
// chosen by the author of collection X.
//
+// Attachment-Only host
+//
+// It is possible to serve untrusted content and accept user
+// credentials at the same origin as long as the content is only
+// downloaded, never executed by browsers. A single origin (hostname
+// and port) can be designated as an "attachment-only" origin: cookies
+// will be accepted and all responses will have a
+// "Content-Disposition: attachment" header. This behavior is invoked
+// only when the designated origin matches exactly the Host header
+// provided by the client or upstream proxy.
+//
+// keep-web -attachment-only-host domain.example:9999
+//
// Trust All Content mode
//
// In "trust all content" mode, Keep-web will accept credentials (API
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index c5d439a..b39a941 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -24,11 +24,14 @@ var (
clientPool = arvadosclient.MakeClientPool()
trustAllContent = false
anonymousTokens []string
+ attachmentOnlyHost = ""
)
func init() {
flag.BoolVar(&trustAllContent, "trust-all-content", false,
"Serve non-public content from a single origin. Dangerous: read docs before using!")
+ flag.StringVar(&attachmentOnlyHost, "attachment-only-host", "",
+ "Accept credentials, and add \"Content-Disposition: attachment\" response headers, for requests at this hostname:port. Prohibiting inline display makes it possible to serve untrusted and non-public content from a single origin, i.e., without wildcard DNS or SSL.")
}
// return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
@@ -111,8 +114,16 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var tokens []string
var reqTokens []string
var pathToken bool
+ var attachment bool
credentialsOK := trustAllContent
+ if r.Host != "" && r.Host == attachmentOnlyHost {
+ credentialsOK = true
+ attachment = true
+ } else if r.FormValue("disposition") == "attachment" {
+ attachment = true
+ }
+
if targetId = parseCollectionIdFromDNSName(r.Host); targetId != "" {
// http://ID.dl.example/PATH...
credentialsOK = true
@@ -293,6 +304,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", rdr.Len()))
+ if attachment {
+ w.Header().Set("Content-Disposition", "attachment")
+ }
w.WriteHeader(http.StatusOK)
_, err = io.Copy(w, rdr)
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index e2f8edd..a64aeb5 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -201,6 +201,30 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
)
}
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
+ defer func(orig string) {
+ attachmentOnlyHost = orig
+ }(attachmentOnlyHost)
+ attachmentOnlyHost = "example.com:1234"
+
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c=" + arvadostest.FooCollection + "/foo",
+ "?api_token=" + arvadostest.ActiveToken,
+ "text/plain",
+ "",
+ http.StatusBadRequest,
+ )
+
+ resp := s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com:1234/c=" + arvadostest.FooCollection + "/foo",
+ "?api_token=" + arvadostest.ActiveToken,
+ "text/plain",
+ "",
+ http.StatusOK,
+ )
+ c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
+}
+
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection + ".example.com/foo",
@@ -221,7 +245,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
)
}
-func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, body string, expectStatus int) {
+func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, body string, expectStatus int) *httptest.ResponseRecorder {
u, _ := url.Parse(`http://` + hostPath + queryString)
req := &http.Request{
Method: method,
@@ -235,7 +259,7 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
(&handler{}).ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
c.Assert(resp.Code, check.Equals, expectStatus)
- return
+ return resp
}
c.Check(resp.Body.String(), check.Matches, `.*href="//` + regexp.QuoteMeta(html.EscapeString(hostPath)) + `".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
@@ -258,4 +282,5 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
if expectStatus == http.StatusOK {
c.Check(resp.Body.String(), check.Equals, "foo")
}
+ return resp
}
commit e6616df2a2d933fd5fd5ab63124d6ad254c0ef56
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Sep 7 02:39:10 2015 -0400
5824: Implement "trust all content" mode.
diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go
index 2f45781..993b9db 100644
--- a/services/keep-web/doc.go
+++ b/services/keep-web/doc.go
@@ -49,7 +49,7 @@
// The following "same origin" URL patterns are supported for public
// collections (i.e., collections which can be served by keep-web
// without making use of any credentials supplied by the client). See
-// "Same-origin mode" below.
+// "Same-origin URLs" below.
//
// http://dl.example.com/c=uuid_or_pdh/path/file.txt
// http://dl.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
@@ -163,7 +163,7 @@
// the local network -- the upstream proxy should configured to return
// 401 for all paths beginning with "/c=".
//
-// Same-origin mode
+// Same-origin URLs
//
// Without the same-origin protection outlined above, a web page
// stored in collection X could execute JavaScript code that uses the
@@ -173,19 +173,7 @@
// (``https://dl.example.com/'') and upload it to some other site
// chosen by the author of collection X.
//
-package main
-
-// TODO(TC): Implement?
-//
-// Trusted content
-//
-// Normally, Keep-web is installed using a wildcard DNS entry and a
-// wildcard HTTPS certificate, serving data from collection X at
-// ``https://X--dl.example.com/path/file.ext''.
-//
-// It will also serve publicly accessible data at
-// ``https://dl.example.com/collections/X/path/file.txt'', but it does not
-// accept any kind of credentials at paths like these.
+// Trust All Content mode
//
// In "trust all content" mode, Keep-web will accept credentials (API
// tokens) and serve any collection X at
@@ -198,4 +186,4 @@ package main
//
// keep-web -trust-all-content [...]
//
-// In the general case, this should not be enabled:
+package main
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 600e685..c5d439a 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,6 +1,7 @@
package main
import (
+ "flag"
"fmt"
"html"
"io"
@@ -19,8 +20,16 @@ import (
type handler struct{}
-var clientPool = arvadosclient.MakeClientPool()
-var anonymousTokens []string
+var (
+ clientPool = arvadosclient.MakeClientPool()
+ trustAllContent = false
+ anonymousTokens []string
+)
+
+func init() {
+ flag.BoolVar(&trustAllContent, "trust-all-content", false,
+ "Serve non-public content from a single origin. Dangerous: read docs before using!")
+}
// return a UUID or PDH if s begins with a UUID or URL-encoded PDH;
// otherwise return "".
@@ -102,7 +111,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var tokens []string
var reqTokens []string
var pathToken bool
- var credentialsOK bool
+ credentialsOK := trustAllContent
if targetId = parseCollectionIdFromDNSName(r.Host); targetId != "" {
// http://ID.dl.example/PATH...
@@ -139,7 +148,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
if !credentialsOK {
// It is not safe to copy the provided token
// into a cookie unless the current vhost
- // (origin) serves only a single collection.
+ // (origin) serves only a single collection or
+ // we are in trustAllContent mode.
statusCode = http.StatusBadRequest
return
}
@@ -160,7 +170,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
Name: "api_token",
Value: auth.EncodeTokenCookie([]byte(t)),
Path: "/",
- Expires: time.Now().AddDate(10,0,0),
+ Expires: time.Now().AddDate(10, 0, 0),
HttpOnly: true,
})
redir := (&url.URL{Host: r.Host, Path: r.URL.Path}).String()
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 50fd717..e2f8edd 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -177,6 +177,30 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
)
}
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c=" + arvadostest.FooCollection + "/foo",
+ "?api_token=" + arvadostest.ActiveToken,
+ "text/plain",
+ "",
+ http.StatusBadRequest,
+ )
+}
+
+func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
+ defer func(orig bool) {
+ trustAllContent = orig
+ }(trustAllContent)
+ trustAllContent = true
+ s.testVhostRedirectTokenToCookie(c, "GET",
+ "example.com/c=" + arvadostest.FooCollection + "/foo",
+ "?api_token=" + arvadostest.ActiveToken,
+ "text/plain",
+ "",
+ http.StatusOK,
+ )
+}
+
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection + ".example.com/foo",
@@ -209,7 +233,10 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
resp := httptest.NewRecorder()
(&handler{}).ServeHTTP(resp, req)
- c.Assert(resp.Code, check.Equals, http.StatusSeeOther)
+ if resp.Code != http.StatusSeeOther {
+ c.Assert(resp.Code, check.Equals, expectStatus)
+ return
+ }
c.Check(resp.Body.String(), check.Matches, `.*href="//` + regexp.QuoteMeta(html.EscapeString(hostPath)) + `".*`)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
commit f1a1faf919edce78261d0ac252758d1bf01d69e3
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Aug 30 02:45:58 2015 -0400
5824: Add read-error and lots-of-blocks tests.
diff --git a/sdk/go/keepclient/collectionreader_test.go b/sdk/go/keepclient/collectionreader_test.go
index 51710b7..94e41e2 100644
--- a/sdk/go/keepclient/collectionreader_test.go
+++ b/sdk/go/keepclient/collectionreader_test.go
@@ -2,21 +2,48 @@ package keepclient
import (
"crypto/md5"
+ "crypto/rand"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
+ "strconv"
+ "strings"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
-var _ = check.Suite(&IntegrationSuite{})
+var _ = check.Suite(&CollectionReaderUnit{})
-// IntegrationSuite tests need an API server
-type IntegrationSuite struct{}
+type CollectionReaderUnit struct {
+ arv arvadosclient.ArvadosClient
+ kc *KeepClient
+ handler SuccessHandler
+}
+
+func (s *CollectionReaderUnit) SetUpTest(c *check.C) {
+ var err error
+ s.arv, err = arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ s.arv.ApiToken = arvadostest.ActiveToken
+
+ s.kc, err = MakeKeepClient(&s.arv)
+ c.Assert(err, check.IsNil)
+
+ s.handler = SuccessHandler{
+ disk: make(map[string][]byte),
+ lock: make(chan struct{}, 1),
+ ops: new(int),
+ }
+ localRoots := make(map[string]string)
+ for i, k := range RunSomeFakeKeepServers(s.handler, 4) {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ }
+ s.kc.SetServiceRoots(localRoots, localRoots, nil)
+}
type SuccessHandler struct {
disk map[string][]byte
@@ -64,33 +91,11 @@ type rdrTest struct {
want interface{} // error or string to expect
}
-func StubWithFakeServers(kc *KeepClient, h http.Handler) {
- localRoots := make(map[string]string)
- for i, k := range RunSomeFakeKeepServers(h, 4) {
- localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
- }
- kc.SetServiceRoots(localRoots, localRoots, nil)
-}
-
-func (s *ServerRequiredSuite) TestCollectionReaderContent(c *check.C) {
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.IsNil)
- arv.ApiToken = arvadostest.ActiveToken
-
- kc, err := MakeKeepClient(&arv)
- c.Assert(err, check.IsNil)
-
- {
- h := SuccessHandler{
- disk: make(map[string][]byte),
- lock: make(chan struct{}, 1),
- }
- StubWithFakeServers(kc, h)
- kc.PutB([]byte("foo"))
- kc.PutB([]byte("bar"))
- kc.PutB([]byte("Hello world\n"))
- kc.PutB([]byte(""))
- }
+func (s *CollectionReaderUnit) TestCollectionReaderContent(c *check.C) {
+ s.kc.PutB([]byte("foo"))
+ s.kc.PutB([]byte("bar"))
+ s.kc.PutB([]byte("Hello world\n"))
+ s.kc.PutB([]byte(""))
mt := arvadostest.PathologicalManifest
@@ -116,7 +121,7 @@ func (s *ServerRequiredSuite) TestCollectionReaderContent(c *check.C) {
{mt: mt, f: "segmented/frob", want: "frob"},
{mt: mt, f: "segmented/oof", want: "oof"},
} {
- rdr, err := kc.CollectionFileReader(map[string]interface{}{"manifest_text": testCase.mt}, testCase.f)
+ rdr, err := s.kc.CollectionFileReader(map[string]interface{}{"manifest_text": testCase.mt}, testCase.f)
switch want := testCase.want.(type) {
case error:
c.Check(rdr, check.IsNil)
@@ -136,21 +141,34 @@ func (s *ServerRequiredSuite) TestCollectionReaderContent(c *check.C) {
}
}
-func (s *ServerRequiredSuite) TestCollectionReaderCloseEarly(c *check.C) {
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.IsNil)
- arv.ApiToken = arvadostest.ActiveToken
-
- kc, err := MakeKeepClient(&arv)
- c.Assert(err, check.IsNil)
-
- h := SuccessHandler{
- disk: make(map[string][]byte),
- lock: make(chan struct{}, 1),
- ops: new(int),
+func (s *CollectionReaderUnit) TestCollectionReaderManyBlocks(c *check.C) {
+ h := md5.New()
+ buf := make([]byte, 4096)
+ locs := make([]string, len(buf))
+ filesize := 0
+ for i := 0; i < len(locs); i++ {
+ _, err := io.ReadFull(rand.Reader, buf[:i])
+ c.Assert(err, check.IsNil)
+ h.Write(buf[:i])
+ locs[i], _, err = s.kc.PutB(buf[:i])
+ c.Assert(err, check.IsNil)
+ filesize += i
}
- StubWithFakeServers(kc, h)
- kc.PutB([]byte("foo"))
+ manifest := "./random " + strings.Join(locs, " ") + " 0:" + strconv.Itoa(filesize) + ":bytes.bin\n"
+ dataMD5 := h.Sum(nil)
+
+ checkMD5 := md5.New()
+ rdr, err := s.kc.CollectionFileReader(map[string]interface{}{"manifest_text": manifest}, "random/bytes.bin")
+ c.Check(err, check.IsNil)
+ _, err = io.Copy(checkMD5, rdr)
+ c.Check(err, check.IsNil)
+ _, err = rdr.Read(make([]byte, 1))
+ c.Check(err, check.Equals, io.EOF)
+ c.Check(checkMD5.Sum(nil), check.DeepEquals, dataMD5)
+}
+
+func (s *CollectionReaderUnit) TestCollectionReaderCloseEarly(c *check.C) {
+ s.kc.PutB([]byte("foo"))
mt := ". "
for i := 0; i < 1000; i++ {
@@ -161,23 +179,45 @@ func (s *ServerRequiredSuite) TestCollectionReaderCloseEarly(c *check.C) {
// Grab the stub server's lock, ensuring our cfReader doesn't
// get anything back from its first call to kc.Get() before we
// have a chance to call Close().
- h.lock <- struct{}{}
- opsBeforeRead := *h.ops
+ s.handler.lock <- struct{}{}
+ opsBeforeRead := *s.handler.ops
- rdr, err := kc.CollectionFileReader(map[string]interface{}{"manifest_text": mt}, "foo1000.txt")
+ rdr, err := s.kc.CollectionFileReader(map[string]interface{}{"manifest_text": mt}, "foo1000.txt")
c.Assert(err, check.IsNil)
+
+ firstReadDone := make(chan struct{})
+ go func() {
+ rdr.Read(make([]byte, 6))
+ firstReadDone <- struct{}{}
+ }()
err = rdr.Close()
c.Assert(err, check.IsNil)
c.Assert(rdr.Error(), check.IsNil)
// Release the stub server's lock. The first GET operation will proceed.
- <-h.lock
+ <-s.handler.lock
+
+ // Make sure our first read operation consumes the data
+ // received from the first GET.
+ <-firstReadDone
// doGet() should close toRead before sending any more bufs to it.
- if what, ok := <-rdr.toRead; ok {
- c.Errorf("Got %+v, expected toRead to be closed", what)
+ if what, ok := <-rdr.toRead; ok {
+ c.Errorf("Got %q, expected toRead to be closed", string(what))
}
// Stub should have handled exactly one GET request.
- c.Assert(*h.ops, check.Equals, opsBeforeRead+1)
+ c.Assert(*s.handler.ops, check.Equals, opsBeforeRead+1)
+}
+
+func (s *CollectionReaderUnit) TestCollectionReaderDataError(c *check.C) {
+ manifest := ". ffffffffffffffffffffffffffffffff+1 0:1:notfound.txt\n"
+ buf := make([]byte, 1)
+ rdr, err := s.kc.CollectionFileReader(map[string]interface{}{"manifest_text": manifest}, "notfound.txt")
+ c.Check(err, check.IsNil)
+ for i := 0; i < 2; i++ {
+ _, err = io.ReadFull(rdr, buf)
+ c.Check(err, check.Not(check.IsNil))
+ c.Check(err, check.Not(check.Equals), io.EOF)
+ }
}
commit 8ad914c4d6976d4d862300514d65d555126481ca
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Aug 30 02:44:18 2015 -0400
5824: Turn off debug printfs unless enabled by calling program.
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 8b7cf41..78090c8 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -11,7 +11,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"net/http"
"os"
"regexp"
@@ -161,7 +160,7 @@ func (kc *KeepClient) Get(locator string) (io.ReadCloser, int64, string, error)
Check: locator[0:32],
}, resp.ContentLength, url, nil
}
- log.Printf("DEBUG: GET %s failed: %v", locator, errs)
+ DebugPrintf("DEBUG: GET %s failed: %v", locator, errs)
return nil, 0, "", BlockNotFound
}
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index ee60d28..462fea7 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -126,10 +126,8 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
-
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), []byte{0})
writer.Write([]byte("foo"))
writer.Close()
@@ -153,15 +151,14 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
tr := streamer.AsyncStreamFromReader(512, reader)
defer tr.Close()
br1 := tr.MakeStreamReader()
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
+ go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, []byte{0})
writer.Write([]byte("foo"))
writer.Close()
@@ -193,10 +190,9 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
hash := "acbd18db4cc2f85cedef654fccc4a4d8"
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, []byte{0})
writer.Write([]byte("foo"))
writer.Close()
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 51e3e08..ab7e3c4 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -2,18 +2,23 @@ package keepclient
import (
"crypto/md5"
+ "crypto/rand"
"errors"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"net"
"net/http"
"strings"
"time"
)
+// Function used to emit debug messages. The easiest way to enable
+// keepclient debug messages in your application is to assign
+// log.Printf to DebugPrintf.
+var DebugPrintf = func(string, ...interface{}) {}
+
type keepService struct {
Uuid string `json:"uuid"`
Hostname string `json:"service_host"`
@@ -150,13 +155,13 @@ type uploadStatus struct {
}
func (this KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
- upload_status chan<- uploadStatus, expectedLength int64, requestId string) {
+ upload_status chan<- uploadStatus, expectedLength int64, requestId []byte) {
var req *http.Request
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
- log.Printf("[%v] Error creating request PUT %v error: %v", requestId, url, err.Error())
+ DebugPrintf("[%x] Error creating request PUT %v error: %v", requestId, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
body.Close()
return
@@ -181,7 +186,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
- log.Printf("[%v] Upload failed %v error: %v", requestId, url, err.Error())
+ DebugPrintf("[%x] Upload failed %v error: %v", requestId, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
@@ -197,13 +202,13 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read
respbody, err2 := ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
- log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, err2.Error(), response)
+ DebugPrintf("[%x] Upload %v error: %v response: %v", requestId, url, err2.Error(), response)
upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
- log.Printf("[%v] Upload %v success", requestId, url)
+ DebugPrintf("[%x] Upload %v success", requestId, url)
upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
- log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, resp.StatusCode, response)
+ DebugPrintf("[%x] Upload %v error: %v response: %v", requestId, url, resp.StatusCode, response)
upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
@@ -213,9 +218,10 @@ func (this KeepClient) putReplicas(
tr *streamer.AsyncStream,
expectedLength int64) (locator string, replicas int, err error) {
- // Take the hash of locator and timestamp in order to identify this
- // specific transaction in log statements.
- requestId := fmt.Sprintf("%x", md5.Sum([]byte(locator+time.Now().String())))[0:8]
+ // Generate an arbitrary ID to identify this specific
+ // transaction in debug logs.
+ requestId := make([]byte, 4)
+ io.ReadFull(rand.Reader, requestId)
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
@@ -253,7 +259,7 @@ func (this KeepClient) putReplicas(
for active*replicasPerThread < remaining_replicas {
// Start some upload requests
if next_server < len(sv) {
- log.Printf("[%v] Begin upload %s to %s", requestId, hash, sv[next_server])
+ DebugPrintf("[%x] Begin upload %s to %s", requestId, hash, sv[next_server])
go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestId)
next_server += 1
active += 1
@@ -265,8 +271,7 @@ func (this KeepClient) putReplicas(
}
}
}
- log.Printf("[%v] Replicas remaining to write: %v active uploads: %v",
- requestId, remaining_replicas, active)
+ DebugPrintf("[%x] Replicas remaining to write: %v active uploads: %v", requestId, remaining_replicas, active)
// Now wait for something to happen.
status := <-upload_status
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 8e734f7..c8f3795 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -84,6 +84,7 @@ func main() {
log.Fatalf("Error setting up arvados client %s", err.Error())
}
+ keepclient.DebugPrintf = log.Printf
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
log.Fatalf("Error setting up keep client %s", err.Error())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list