[ARVADOS] updated: a64718801b50797c9af35da3e78255463f4dae1d
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 2 15:11:00 EDT 2015
Summary of changes:
.../app/controllers/projects_controller.rb | 11 +-
apps/workbench/app/models/arvados_base.rb | 6 +-
apps/workbench/app/views/layouts/body.html.erb | 6 +
apps/workbench/app/views/projects/public.html.erb | 29 +++
apps/workbench/config/application.default.yml | 2 +-
apps/workbench/config/routes.rb | 2 +
.../test/controllers/projects_controller_test.rb | 32 +++
.../test/integration/anonymous_access_test.rb | 5 +
.../test/integration/application_layout_test.rb | 2 +
.../collection_unit_test.rb | 7 +-
.../collections_controller_test.rb | 4 +-
.../collections_perf_test.rb | 16 +-
doc/_config.yml | 3 +-
.../_tutorial_bwa_sortsam_pipeline.liquid | 15 +-
doc/index.html.liquid | 2 +-
.../install-compute-node.html.textile.liquid | 103 +++++++++
.../install-crunch-dispatch.html.textile.liquid | 70 +++++-
...l-manual-prerequisites-ruby.html.textile.liquid | 3 +-
.../getting_started/workbench.html.textile.liquid | 2 +-
doc/user/index.html.textile.liquid | 2 +-
doc/user/reference/api-tokens.html.textile.liquid | 2 +-
...nning-pipeline-command-line.html.textile.liquid | 6 +-
doc/user/topics/tutorial-job1.html.textile.liquid | 8 +-
.../running-external-program.html.textile.liquid | 17 +-
.../tutorial-firstscript.html.textile.liquid | 2 +-
.../tutorial-submit-job.html.textile.liquid | 6 +-
docker/api/Dockerfile | 4 +-
docker/api/apache2_vhost.in | 2 -
docker/api/application.yml.in | 11 +-
docker/api/arvados-clients.yml.in | 2 +-
.../{keep_server_1.json => keep_server_0.json.in} | 3 +-
.../{keep_server_0.json => keep_server_1.json.in} | 2 +-
docker/api/omniauth.rb.in | 2 +-
docker/api/setup-gitolite.sh.in | 4 +-
docker/arvdock | 58 +++--
docker/build_tools/Makefile | 11 +-
docker/build_tools/build.rb | 52 ++---
docker/config.yml.example | 12 +-
docker/doc/Dockerfile | 5 +-
docker/doc/{apache2_vhost => apache2_vhost.in} | 5 +-
docker/sso/apache2_vhost.in | 2 -
docker/workbench/Dockerfile | 4 +-
docker/workbench/apache2_vhost.in | 13 +-
docker/workbench/application.yml.in | 6 +-
sdk/cli/bin/crunch-job | 237 +++++++++++++++------
sdk/python/arvados/api.py | 21 ++
sdk/python/arvados/commands/arv_copy.py | 5 +-
sdk/python/tests/test_api.py | 18 ++
services/api/app/models/arvados_model.rb | 21 +-
services/api/app/models/node.rb | 86 +++++---
services/api/config/application.default.yml | 25 ++-
...50526180251_leading_space_on_full_text_index.rb | 41 ++++
services/api/db/structure.sql | 18 +-
.../api/test/unit/collection_performance_test.rb | 4 +-
services/api/test/unit/node_test.rb | 49 +++++
55 files changed, 830 insertions(+), 256 deletions(-)
create mode 100644 apps/workbench/app/views/projects/public.html.erb
create mode 100644 doc/install/install-compute-node.html.textile.liquid
rename docker/api/{keep_server_1.json => keep_server_0.json.in} (59%)
rename docker/api/{keep_server_0.json => keep_server_1.json.in} (59%)
rename docker/doc/{apache2_vhost => apache2_vhost.in} (62%)
create mode 100644 services/api/db/migrate/20150526180251_leading_space_on_full_text_index.rb
discards dab62e9006a117b29314f1c2db9aa10b665cc4d7 (commit)
discards ccfa3ebf6aa31f59cf6289669e542174c77a2e2a (commit)
discards cc988c81ad543eb4e8ad88416e4c2fcb937abd11 (commit)
discards 0ded94e10f668c961c257199d7b6d08af234b75a (commit)
via a64718801b50797c9af35da3e78255463f4dae1d (commit)
via 9b04f22c681af43af7bd1da9aa61ba30cb212f45 (commit)
via 00d96755d50310f05ca7cf263bdb879a01ef5074 (commit)
via 249f231c21384d6fb3ebe391591a93098b839f58 (commit)
via bd76ea1ee2b8ef0cd7a29ba2a02d2e77ce239e99 (commit)
via 469356d1a60754381e33736ac4f80e1a1e593a7f (commit)
via 121625abcf70672531b35dc4092a4597d8eca4be (commit)
via 76e42f169c6e278c1d8cefe9fb7c03cc70892bac (commit)
via c6fe632f972e4610ff7f35f83a5d7dcd2d6e7ecb (commit)
via 178d3f36265e0e9e9cc0bb6ac8c7c47a9c701687 (commit)
via 9413eb733015601af699f2027d9a7a5bad3f3dea (commit)
via b25916b93d720f80b5bdf16f018d83c13c6c3001 (commit)
via 1ec1d552c77e18e2912e400ae395ca00f4e51c3c (commit)
via 1e0d770315a7b01c316b6a4c314bff58856bfe02 (commit)
via 5e0dd2c6b8f080c81ff6077e629f5ec9f377802f (commit)
via 7a53d874994a5a9af273cee1329d9635b7e03edb (commit)
via 1b5c30eb957594e00a09df745df7630f661e3807 (commit)
via 32038d56fbfe5b635b1c53f247aea9abcca2285c (commit)
via c51a3888a01543d0835119574960a02fd7d35994 (commit)
via 3101fea587bc08e0f4f00a583d1d9e2c953417c1 (commit)
via 6fed84983e6e973eefff66a126f4bb7811c44d29 (commit)
via e73222a8d0c18b159ae3d8b53b54474650bdda16 (commit)
via d9214af111bb44deed1246ec97624b6bd8d970c5 (commit)
via 59d68bda41fcc83c47795a19dbe1d0c180952a08 (commit)
via 15009c109bee16bc09c9bbc11f7df0a677a0cf23 (commit)
via a2eb98fa68672d6966233d7864c328feb8df3939 (commit)
via b3e2782f72e0ca381f3f8c508227ebd7a2ef0c92 (commit)
via 9b67268ff45d2a552d21fa39f5086180f537ab4a (commit)
via d3fcdc7afcaf32886d2b492e136f1790e8e93ce4 (commit)
via 872cc3d79d99f5fc4b26b970c3d269ca363ea27c (commit)
via 79564b0ac7d03327cc351bbd6df544ab1f776380 (commit)
via 0f125dd51c7dc13047672dee5362866f31885e0a (commit)
via 1a0ba80d44b4be962f898ee1458faea9d4a59137 (commit)
via 71f1ccba7367ca9d000e53c0b86c0448600350f4 (commit)
via 80d57cdcc9fcae93da2b507942815d9de6dd3351 (commit)
via fbcc5ab5e7a5ad65b81e3965f93b1679d1b1e06f (commit)
via b4c2a96ed66c98cb5a07bd06138f755ae5675fb7 (commit)
via a7d558d6c2db242a6555c7f397cb4d91618aa13c (commit)
via 3bf22ae161deadd56d20c4165d2ec569de2dcdef (commit)
via 8b658d96a7bb087406fee97ee4ba9feb830abfd4 (commit)
via 5d26f7e8bf84d5ed055dc1e88996a25c4db80e85 (commit)
via ae3bb0033a24a38489c49ffc26e5a5e8fd93c160 (commit)
via 9b44bba316583b68f326920d378fbbc77b4567a2 (commit)
via 59129909453452693bd6e7be144cb9417479e98f (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (dab62e9006a117b29314f1c2db9aa10b665cc4d7)
\
N -- N -- N (a64718801b50797c9af35da3e78255463f4dae1d)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a64718801b50797c9af35da3e78255463f4dae1d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jun 2 14:40:26 2015 -0400
6087: Reset changed-attrs list after saving. Fix only-send-changed-attrs logic.
diff --git a/apps/workbench/app/models/arvados_base.rb b/apps/workbench/app/models/arvados_base.rb
index 2fca11e..68f0ebd 100644
--- a/apps/workbench/app/models/arvados_base.rb
+++ b/apps/workbench/app/models/arvados_base.rb
@@ -171,8 +171,8 @@ class ArvadosBase < ActiveRecord::Base
def save
obdata = {}
self.class.columns.each do |col|
- unless self.send(col.name.to_sym).nil? and !self.changed.include?(col.name)
- obdata[col.name.to_sym] = self.send(col.name.to_sym)
+ if new_record? or changed.include? col.name
+ obdata[col.name.to_sym] = self.send(col.name.to_sym)
end
end
obdata.delete :id
@@ -199,6 +199,7 @@ class ArvadosBase < ActiveRecord::Base
end
@new_record = false
+ changes_applied
self
end
@@ -281,6 +282,7 @@ class ArvadosBase < ActiveRecord::Base
end
@all_links = nil
@new_record = false
+ changes_applied
self
end
commit 9b04f22c681af43af7bd1da9aa61ba30cb212f45
Author: Tom Clegg <tom at curoverse.com>
Date: Mon May 25 13:25:22 2015 -0400
6087: Add big-manifest tests, with some finer-grained performance numbers on stderr.
diff --git a/apps/workbench/app/models/arvados_api_client.rb b/apps/workbench/app/models/arvados_api_client.rb
index 17c5ec6..ca09aa7 100644
--- a/apps/workbench/app/models/arvados_api_client.rb
+++ b/apps/workbench/app/models/arvados_api_client.rb
@@ -134,7 +134,7 @@ class ArvadosApiClient
header = {"Accept" => "application/json"}
- profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]} #{query[:filters]} #{query[:order]}" }
+ profile_checkpoint { "Prepare request #{query["_method"] or "POST"} #{url} #{query[:uuid]} #{query.inspect[0,256]}" }
msg = @client_mtx.synchronize do
begin
@api_client.post(url, query, header: header)
@@ -143,6 +143,12 @@ class ArvadosApiClient
end
end
profile_checkpoint 'API transaction'
+ if @@profiling_enabled
+ if msg.headers['X-Runtime']
+ Rails.logger.info "API server: #{msg.headers['X-Runtime']} runtime reported"
+ end
+ Rails.logger.info "Content-Encoding #{msg.headers['Content-Encoding'].inspect}, Content-Length #{msg.headers['Content-Length'].inspect}, actual content size #{msg.content.size}"
+ end
begin
resp = Oj.load(msg.content, :symbol_keys => true)
diff --git a/apps/workbench/test/helpers/manifest_examples.rb b/apps/workbench/test/helpers/manifest_examples.rb
new file mode 120000
index 0000000..cb908ef
--- /dev/null
+++ b/apps/workbench/test/helpers/manifest_examples.rb
@@ -0,0 +1 @@
+../../../../services/api/test/helpers/manifest_examples.rb
\ No newline at end of file
diff --git a/apps/workbench/test/helpers/time_block.rb b/apps/workbench/test/helpers/time_block.rb
new file mode 120000
index 0000000..afb43e7
--- /dev/null
+++ b/apps/workbench/test/helpers/time_block.rb
@@ -0,0 +1 @@
+../../../../services/api/test/helpers/time_block.rb
\ No newline at end of file
diff --git a/apps/workbench/test/integration_performance/collection_unit_test.rb b/apps/workbench/test/integration_performance/collection_unit_test.rb
new file mode 100644
index 0000000..6cf14b5
--- /dev/null
+++ b/apps/workbench/test/integration_performance/collection_unit_test.rb
@@ -0,0 +1,71 @@
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class Blob
+end
+
+class BigCollectionTest < ActiveSupport::TestCase
+ include ManifestExamples
+
+ setup do
+ Blob.stubs(:sign_locator).returns 'd41d8cd98f00b204e9800998ecf8427e+0'
+ end
+
+ teardown do
+ Thread.current[:arvados_api_client] = nil
+ end
+
+ # You can try with compress=false here too, but at last check it
+ # didn't make a significant difference.
+ [true].each do |compress|
+ test "crud cycle for collection with big manifest (compress=#{compress})" do
+ Rails.configuration.api_response_compression = compress
+ Thread.current[:arvados_api_client] = nil
+ crudtest
+ end
+ end
+
+ def crudtest
+ use_token :active
+ bigmanifest = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 0)
+ end
+ c = time_block "new (manifest size = #{bigmanifest.length>>20}MiB)" do
+ Collection.new manifest_text: bigmanifest
+ end
+ time_block 'create' do
+ c.save!
+ end
+ time_block 'read' do
+ Collection.find c.uuid
+ end
+ time_block 'read(cached)' do
+ Collection.find c.uuid
+ end
+ time_block 'list' do
+ list = Collection.select(['uuid', 'manifest_text']).filter [['uuid','=',c.uuid]]
+ assert_equal 1, list.count
+ assert_equal c.uuid, list.first.uuid
+ assert_not_nil list.first.manifest_text
+ end
+ time_block 'update(name-only)' do
+ manifest_text_length = c.manifest_text.length
+ c.update_attributes name: 'renamed during test case'
+ assert_equal c.manifest_text.length, manifest_text_length
+ end
+ time_block 'update' do
+ c.manifest_text += ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:empty.txt\n"
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ time_block 'read(404)' do
+ assert_empty Collection.filter([['uuid','=',c.uuid]])
+ end
+ end
+end
diff --git a/apps/workbench/test/integration_performance/collections_controller_test.rb b/apps/workbench/test/integration_performance/collections_controller_test.rb
new file mode 100644
index 0000000..190d5e6
--- /dev/null
+++ b/apps/workbench/test/integration_performance/collections_controller_test.rb
@@ -0,0 +1,71 @@
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class Blob
+end
+
+class BigCollectionsControllerTest < ActionController::TestCase
+ include ManifestExamples
+
+ setup do
+ Blob.stubs(:sign_locator).returns 'd41d8cd98f00b204e9800998ecf8427e+0'
+ end
+
+ test "combine two big and two small collections" do
+ @controller = ActionsController.new
+ bigmanifest1 = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 0)
+ end
+ bigmanifest2 = bigmanifest1.gsub '.txt', '.txt2'
+ smallmanifest1 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small1.txt\n"
+ smallmanifest2 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small2.txt\n"
+ totalsize = bigmanifest1.length + bigmanifest2.length +
+ smallmanifest1.length + smallmanifest2.length
+ parts = time_block "create (total #{totalsize>>20}MiB)" do
+ use_token :active do
+ {
+ big1: Collection.create(manifest_text: bigmanifest1),
+ big2: Collection.create(manifest_text: bigmanifest2),
+ small1: Collection.create(manifest_text: smallmanifest1),
+ small2: Collection.create(manifest_text: smallmanifest2),
+ }
+ end
+ end
+ time_block 'combine' do
+ post :combine_selected_files_into_collection, {
+ selection: [parts[:big1].uuid,
+ parts[:big2].uuid,
+ parts[:small1].uuid + '/small1.txt',
+ parts[:small2].uuid + '/small2.txt',
+ ],
+ format: :html
+ }, session_for(:active)
+ end
+ assert_response :redirect
+ end
+
+ [:json, :html].each do |format|
+ test "show collection with big manifest (#{format})" do
+ bigmanifest = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 30,
+ bytes_per_block: 0)
+ end
+ @controller = CollectionsController.new
+ c = time_block "create (manifest size #{bigmanifest.length>>20}MiB)" do
+ use_token :active do
+ Collection.create(manifest_text: bigmanifest)
+ end
+ end
+ time_block 'show' do
+ get :show, {id: c.uuid, format: format}, session_for(:active)
+ end
+ assert_response :success
+ end
+ end
+end
diff --git a/services/api/test/helpers/manifest_examples.rb b/services/api/test/helpers/manifest_examples.rb
new file mode 100644
index 0000000..08712eb
--- /dev/null
+++ b/services/api/test/helpers/manifest_examples.rb
@@ -0,0 +1,31 @@
+module ManifestExamples
+ def make_manifest opts={}
+ opts = {
+ bytes_per_block: 1,
+ blocks_per_file: 1,
+ files_per_stream: 1,
+ streams: 1,
+ }.merge(opts)
+ datablip = "x" * opts[:bytes_per_block]
+ locator = Blob.sign_locator(Digest::MD5.hexdigest(datablip) +
+ '+' + datablip.length.to_s,
+ api_token: opts[:api_token])
+ filesize = datablip.length * opts[:blocks_per_file]
+ txt = ''
+ (1..opts[:streams]).each do |s|
+ streamtoken = "./stream#{s}"
+ streamsize = 0
+ blocktokens = []
+ filetokens = []
+ (1..opts[:files_per_stream]).each do |f|
+ filetokens << " #{streamsize}:#{filesize}:file#{f}.txt"
+ (1..opts[:blocks_per_file]).each do |b|
+ blocktokens << locator
+ end
+ streamsize += filesize
+ end
+ txt << ([streamtoken] + blocktokens + filetokens).join(' ') + "\n"
+ end
+ txt
+ end
+end
diff --git a/services/api/test/helpers/time_block.rb b/services/api/test/helpers/time_block.rb
new file mode 100644
index 0000000..a3b03ff
--- /dev/null
+++ b/services/api/test/helpers/time_block.rb
@@ -0,0 +1,11 @@
+class ActiveSupport::TestCase
+ def time_block label
+ t0 = Time.now
+ begin
+ yield
+ ensure
+ t1 = Time.now
+ $stderr.puts "#{t1 - t0}s #{label}"
+ end
+ end
+end
diff --git a/services/api/test/integration/collections_performance_test.rb b/services/api/test/integration/collections_performance_test.rb
new file mode 100644
index 0000000..7b790b7
--- /dev/null
+++ b/services/api/test/integration/collections_performance_test.rb
@@ -0,0 +1,40 @@
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class CollectionsApiPerformanceTest < ActionDispatch::IntegrationTest
+ include ManifestExamples
+
+ test "crud cycle for a collection with a big manifest" do
+ bigmanifest = time_block 'make example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 10,
+ bytes_per_block: 2**26,
+ api_token: api_token(:active))
+ end
+ json = time_block "JSON encode #{bigmanifest.length>>20}MiB manifest" do
+ Oj.dump({manifest_text: bigmanifest})
+ end
+ time_block 'create' do
+ post '/arvados/v1/collections', {collection: json}, auth(:active)
+ assert_response :success
+ end
+ uuid = json_response['uuid']
+ time_block 'read' do
+ get '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ assert_response :success
+ end
+ time_block 'list' do
+ get '/arvados/v1/collections', {select: ['manifest_text'], filters: [['uuid', '=', uuid]].to_json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'update' do
+ put '/arvados/v1/collections/' + uuid, {collection: json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'delete' do
+ delete '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ end
+ end
+end
diff --git a/services/api/test/unit/collection_performance_test.rb b/services/api/test/unit/collection_performance_test.rb
new file mode 100644
index 0000000..075b4c5
--- /dev/null
+++ b/services/api/test/unit/collection_performance_test.rb
@@ -0,0 +1,61 @@
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class CollectionModelPerformanceTest < ActiveSupport::TestCase
+ include ManifestExamples
+
+ setup do
+ # The Collection model needs to have a current token, not just a
+ # current user, to sign & verify manifests:
+ Thread.current[:api_client_authorization] =
+ api_client_authorizations(:active)
+ end
+
+ teardown do
+ Thread.current[:api_client_authorization] = nil
+ end
+
+ # "crrud" == "create read render update delete", not a typo
+ test "crrud cycle for a collection with a big manifest)" do
+ bigmanifest = time_block 'make example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 2**26,
+ api_token: api_token(:active))
+ end
+ act_as_user users(:active) do
+ c = time_block "new (manifest_text is #{bigmanifest.length>>20}MiB)" do
+ Collection.new manifest_text: bigmanifest.dup
+ end
+ time_block 'check signatures' do
+ c.check_signatures
+ end
+ time_block 'check signatures + save' do
+ c.save!
+ end
+ c = time_block 'read' do
+ Collection.find_by_uuid(c.uuid)
+ end
+ time_block 'sign' do
+ c.signed_manifest_text
+ end
+ time_block 'sign + render' do
+ resp = c.as_api_response(nil)
+ end
+ loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3',
+ api_token: api_token(:active))
+ # Note Collection's strip_manifest_text method has now removed
+ # the signatures from c.manifest_text, so we have to start from
+ # bigmanifest again here instead of just appending with "+=".
+ c.manifest_text = bigmanifest.dup + ". #{loc} 0:3:foo.txt\n"
+ time_block 'update' do
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ end
+ end
+end
commit 00d96755d50310f05ca7cf263bdb879a01ef5074
Author: Tom Clegg <tom at curoverse.com>
Date: Mon May 25 10:35:02 2015 -0400
6087: Use HTTPClient's compression feature (instead of adding the
Content-Encoding header ourselves). Rename config knob to describe
purpose instead of implementation.
diff --git a/apps/workbench/app/models/arvados_api_client.rb b/apps/workbench/app/models/arvados_api_client.rb
index aa3269a..17c5ec6 100644
--- a/apps/workbench/app/models/arvados_api_client.rb
+++ b/apps/workbench/app/models/arvados_api_client.rb
@@ -91,6 +91,9 @@ class ArvadosApiClient
# Use system CA certificates
@api_client.ssl_config.add_trust_ca('/etc/ssl/certs')
end
+ if Rails.configuration.api_response_compression
+ @api_client.transparent_gzip_decompression = true
+ end
end
end
@@ -130,9 +133,6 @@ class ArvadosApiClient
end
header = {"Accept" => "application/json"}
- if Rails.configuration.include_accept_encoding_header_in_api_requests
- header["Accept-Encoding"] = "gzip, deflate"
- end
profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]} #{query[:filters]} #{query[:order]}" }
msg = @client_mtx.synchronize do
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 07d79a0..cae7b19 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -209,5 +209,5 @@ common:
# in the directory where your API server is running.
anonymous_user_token: false
- # Enable response payload compression in Arvados API requests.
- include_accept_encoding_header_in_api_requests: true
+ # Ask Arvados API server to compress its response payloads.
+ api_response_compression: true
commit 249f231c21384d6fb3ebe391591a93098b839f58
Author: Tom Clegg <tom at curoverse.com>
Date: Mon May 25 10:23:53 2015 -0400
6087: Compute portable_data_hash only once during check_signatures.
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 015efed..f7a715a 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -51,7 +51,8 @@ class Collection < ArvadosModel
# subsequent passes without checking any signatures. This is
# important because the signatures have probably been stripped off
# by the time we get to a second validation pass!
- return true if @signatures_checked and @signatures_checked == compute_pdh
+ computed_pdh = compute_pdh
+ return true if @signatures_checked and @signatures_checked == computed_pdh
if self.manifest_text_changed?
# Check permissions on the collection manifest.
@@ -87,7 +88,7 @@ class Collection < ArvadosModel
end
end
end
- @signatures_checked = compute_pdh
+ @signatures_checked = computed_pdh
end
def strip_manifest_text
commit bd76ea1ee2b8ef0cd7a29ba2a02d2e77ce239e99
Author: Tom Clegg <tom at curoverse.com>
Date: Mon May 25 10:19:31 2015 -0400
6087: Use app-configured key by default for blob signing and verification.
diff --git a/services/api/app/models/blob.rb b/services/api/app/models/blob.rb
index 7ae13ef..56cdfb8 100644
--- a/services/api/app/models/blob.rb
+++ b/services/api/app/models/blob.rb
@@ -28,8 +28,8 @@ class Blob
# Blob.sign_locator: return a signed and timestamped blob locator.
#
# The 'opts' argument should include:
- # [required] :key - the Arvados server-side blobstore key
- # [required] :api_token - user's API token
+ # [required] :api_token - API token (signatures only work for this token)
+ # [optional] :key - the Arvados server-side blobstore key
# [optional] :ttl - number of seconds before signature should expire
# [optional] :expire - unix timestamp when signature should expire
#
@@ -44,14 +44,16 @@ class Blob
end
timestamp = opts[:expire]
else
- timestamp = db_current_time.to_i + (opts[:ttl] || 1209600)
+ timestamp = db_current_time.to_i +
+ (opts[:ttl] || Rails.configuration.blob_signature_ttl)
end
timestamp_hex = timestamp.to_s(16)
# => "53163cb4"
# Generate a signature.
signature =
- generate_signature opts[:key], blob_hash, opts[:api_token], timestamp_hex
+ generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+ blob_hash, opts[:api_token], timestamp_hex)
blob_locator + '+A' + signature + '@' + timestamp_hex
end
@@ -96,7 +98,8 @@ class Blob
end
my_signature =
- generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+ generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+ blob_hash, opts[:api_token], timestamp)
if my_signature != given_signature
raise Blob::InvalidSignatureError.new 'Signature is invalid.'
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 7f93e20..015efed 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -59,7 +59,6 @@ class Collection < ArvadosModel
# which will return 403 Permission denied to the client.
api_token = current_api_client_authorization.andand.api_token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: api_token,
now: db_current_time.to_i,
}
@@ -194,7 +193,6 @@ class Collection < ArvadosModel
def self.sign_manifest manifest, token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: token,
expire: db_current_time.to_i + Rails.configuration.blob_signature_ttl,
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list