[ARVADOS] updated: 46b43471b3c39ac187f8f81a211b74842b360ba4
git at public.curoverse.com
git at public.curoverse.com
Fri May 22 00:09:47 EDT 2015
Summary of changes:
apps/workbench/app/models/arvados_api_client.rb | 12 ++++--
apps/workbench/config/application.default.yml | 3 --
.../test/controllers/big_collection_test.rb | 49 ++++++++++++++++++++++
apps/workbench/test/helpers/time_block.rb | 1 +
apps/workbench/test/unit/big_collection_test.rb | 41 ++++++++++++++++++
services/api/app/models/collection.rb | 5 ++-
services/api/test/helpers/time_block.rb | 11 +++++
.../integration/collections_performance_test.rb | 34 +++++++++++++++
.../api/test/unit/collection_performance_test.rb | 37 ++++++++++++++++
9 files changed, 184 insertions(+), 9 deletions(-)
create mode 100644 apps/workbench/test/controllers/big_collection_test.rb
create mode 120000 apps/workbench/test/helpers/time_block.rb
create mode 100644 apps/workbench/test/unit/big_collection_test.rb
create mode 100644 services/api/test/helpers/time_block.rb
create mode 100644 services/api/test/integration/collections_performance_test.rb
create mode 100644 services/api/test/unit/collection_performance_test.rb
via 46b43471b3c39ac187f8f81a211b74842b360ba4 (commit)
from 1a2d8a738cf017c7b3f3234a1ea1aa90abb8162e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 46b43471b3c39ac187f8f81a211b74842b360ba4
Author: Tom Clegg <tom at curoverse.com>
Date: Fri May 22 00:08:59 2015 -0400
6087: Add big-manifest tests, with some finer-grained performance numbers on stderr.
diff --git a/apps/workbench/app/models/arvados_api_client.rb b/apps/workbench/app/models/arvados_api_client.rb
index aa3269a..dd62c67f 100644
--- a/apps/workbench/app/models/arvados_api_client.rb
+++ b/apps/workbench/app/models/arvados_api_client.rb
@@ -91,6 +91,7 @@ class ArvadosApiClient
# Use system CA certificates
@api_client.ssl_config.add_trust_ca('/etc/ssl/certs')
end
+ @api_client.transparent_gzip_decompression = true
end
end
@@ -130,11 +131,8 @@ class ArvadosApiClient
end
header = {"Accept" => "application/json"}
- if Rails.configuration.include_accept_encoding_header_in_api_requests
- header["Accept-Encoding"] = "gzip, deflate"
- end
- profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]} #{query[:filters]} #{query[:order]}" }
+ profile_checkpoint { "Prepare request #{query["_method"] or "POST"} #{url} #{query[:uuid]} #{query.inspect[0,256]}" }
msg = @client_mtx.synchronize do
begin
@api_client.post(url, query, header: header)
@@ -143,6 +141,12 @@ class ArvadosApiClient
end
end
profile_checkpoint 'API transaction'
+ if @@profiling_enabled
+ if msg.headers['X-Runtime']
+ Rails.logger.info "API Server: #{msg.headers['X-Runtime'].to_f} runtime"
+ end
+ Rails.logger.info "Content encoding #{msg.headers['Content-Encoding'].inspect}, transfer encoding #{msg.headers['Transfer-Encoding'].inspect}, message content size #{msg.content.size}"
+ end
begin
resp = Oj.load(msg.content, :symbol_keys => true)
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 4061ee8..57fdf02 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -208,6 +208,3 @@ common:
# the token by running "bundle exec ./script/get_anonymous_user_token.rb"
# in the directory where your API server is running.
anonymous_user_token: false
-
- # Enable response payload compression in Arvados API requests.
- include_accept_encoding_header_in_api_requests: true
diff --git a/apps/workbench/test/controllers/big_collection_test.rb b/apps/workbench/test/controllers/big_collection_test.rb
new file mode 100644
index 0000000..92a5b04
--- /dev/null
+++ b/apps/workbench/test/controllers/big_collection_test.rb
@@ -0,0 +1,49 @@
+require 'test_helper'
+require 'helpers/time_block'
+
+class BigCollectionsControllerTest < ActionController::TestCase
+ Bigmanifest1 = '. ' + ('d41d8cd98f00b204e9800998ecf8427e+0 ' * 100000) + '0:0:big1.txt' + "\n"
+ Bigmanifest2 = '. ' + ('d41d8cd98f00b204e9800998ecf8427e+0 ' * 100000) + '0:0:big2.txt' + "\n"
+ Smallmanifest1 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small1.txt\n"
+ Smallmanifest2 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small2.txt\n"
+
+ test "combine two big (#{Bigmanifest1.length>>20}MiB + #{Bigmanifest1.length>>20}MiB) and two small collections" do
+ @controller = ActionsController.new
+ parts = time_block 'create' do
+ use_token :active do
+ {
+ big1: Collection.create(manifest_text: Bigmanifest1),
+ big2: Collection.create(manifest_text: Bigmanifest2),
+ small1: Collection.create(manifest_text: Smallmanifest1),
+ small2: Collection.create(manifest_text: Smallmanifest2),
+ }
+ end
+ end
+ time_block 'combine' do
+ post :combine_selected_files_into_collection, {
+ selection: [parts[:big1].uuid + '/big1.txt',
+ parts[:big2].uuid + '/big2.txt',
+ parts[:small1].uuid + '/small1.txt',
+ parts[:small2].uuid + '/small2.txt',
+ ],
+ format: :html
+ }, session_for(:active)
+ end
+ assert_response :redirect
+ end
+
+ [:json, :html].each do |format|
+ test "show a big (#{Bigmanifest1.length>>20}MiB manifest) collection (#{format})" do
+ @controller = CollectionsController.new
+ c = time_block 'create' do
+ use_token :active do
+ Collection.create(manifest_text: Bigmanifest1)
+ end
+ end
+ time_block 'show' do
+ get :show, {id: c.uuid, format: format}, session_for(:active)
+ end
+ assert_response :success
+ end
+ end
+end
diff --git a/apps/workbench/test/helpers/time_block.rb b/apps/workbench/test/helpers/time_block.rb
new file mode 120000
index 0000000..afb43e7
--- /dev/null
+++ b/apps/workbench/test/helpers/time_block.rb
@@ -0,0 +1 @@
+../../../../services/api/test/helpers/time_block.rb
\ No newline at end of file
diff --git a/apps/workbench/test/unit/big_collection_test.rb b/apps/workbench/test/unit/big_collection_test.rb
new file mode 100644
index 0000000..2135812
--- /dev/null
+++ b/apps/workbench/test/unit/big_collection_test.rb
@@ -0,0 +1,41 @@
+require 'test_helper'
+require 'helpers/time_block'
+
+class BigCollectionTest < ActiveSupport::TestCase
+ Bigmanifest = '. ' + ('d41d8cd98f00b204e9800998ecf8427e+0 ' * 100000) + '0:0:inefficient.txt' + "\n"
+
+ test "crud cycle for a big collection (#{Bigmanifest.length>>20}MB manifest)" do
+ use_token :active
+ c = time_block 'new' do
+ Collection.new manifest_text: Bigmanifest
+ end
+ time_block 'create' do
+ c.save!
+ end
+ time_block 'read' do
+ Collection.find c.uuid
+ end
+ time_block 'read(cached)' do
+ Collection.find c.uuid
+ end
+ time_block 'list' do
+ list = Collection.filter [['uuid','=',c.uuid]]
+ assert_equal 1, list.count
+ assert_equal c.uuid, list.first.uuid
+ end
+ time_block 'update' do
+ c.manifest_text += ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:empty.txt\n"
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ time_block 'read(404)' do
+ assert_empty Collection.filter([['uuid','=',c.uuid]])
+ end
+ end
+
+ test "fail" do
+ assert_equal true, false
+ end
+end
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 7f93e20..56b61f0 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -51,7 +51,8 @@ class Collection < ArvadosModel
# subsequent passes without checking any signatures. This is
# important because the signatures have probably been stripped off
# by the time we get to a second validation pass!
- return true if @signatures_checked and @signatures_checked == compute_pdh
+ computed_pdh = compute_pdh
+ return true if @signatures_checked and @signatures_checked == computed_pdh
if self.manifest_text_changed?
# Check permissions on the collection manifest.
@@ -88,7 +89,7 @@ class Collection < ArvadosModel
end
end
end
- @signatures_checked = compute_pdh
+ @signatures_checked = computed_pdh
end
def strip_manifest_text
diff --git a/services/api/test/helpers/time_block.rb b/services/api/test/helpers/time_block.rb
new file mode 100644
index 0000000..a3b03ff
--- /dev/null
+++ b/services/api/test/helpers/time_block.rb
@@ -0,0 +1,11 @@
+class ActiveSupport::TestCase
+ def time_block label
+ t0 = Time.now
+ begin
+ yield
+ ensure
+ t1 = Time.now
+ $stderr.puts "#{t1 - t0}s #{label}"
+ end
+ end
+end
diff --git a/services/api/test/integration/collections_performance_test.rb b/services/api/test/integration/collections_performance_test.rb
new file mode 100644
index 0000000..5059c9f
--- /dev/null
+++ b/services/api/test/integration/collections_performance_test.rb
@@ -0,0 +1,34 @@
+require 'test_helper'
+require 'helpers/time_block'
+
+class CollectionsApiPerformanceTest < ActionDispatch::IntegrationTest
+ Bigmanifest = '. ' + ('d41d8cd98f00b204e9800998ecf8427e+0 ' * 100000) + '0:0:inefficient.txt' + "\n"
+
+ test "crud cycle for a big collection (#{Bigmanifest.length>>20}MB manifest)" do
+ json = Oj.dump({manifest_text: Bigmanifest})
+ time_block 'create' do
+ post '/arvados/v1/collections', {collection: json}, auth(:active)
+ assert_response :success
+ end
+ uuid = json_response['uuid']
+ time_block 'read' do
+ get '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ assert_response :success
+ end
+ time_block 'list' do
+ get '/arvados/v1/collections', {select: ['manifest_text'], filters: [['uuid', '=', uuid]].to_json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'update' do
+ put '/arvados/v1/collections/' + uuid, {collection: json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'delete' do
+ delete '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ end
+ end
+
+ test 'fail' do
+ assert_equal true, false
+ end
+end
diff --git a/services/api/test/unit/collection_performance_test.rb b/services/api/test/unit/collection_performance_test.rb
new file mode 100644
index 0000000..41b5450
--- /dev/null
+++ b/services/api/test/unit/collection_performance_test.rb
@@ -0,0 +1,37 @@
+require 'test_helper'
+require 'helpers/time_block'
+
+class CollectionModelPerformanceTest < ActiveSupport::TestCase
+ Bigmanifest = '. ' + ('d41d8cd98f00b204e9800998ecf8427e+0 ' * 100000) + '0:0:inefficient.txt' + "\n"
+
+ # "create read render update delete", not a typo
+ test "crrud cycle for a big collection (#{Bigmanifest.length>>20}MB manifest)" do
+ act_as_user users(:active) do
+ c = time_block 'new' do
+ Collection.new manifest_text: Bigmanifest
+ end
+ time_block 'check signatures' do
+ c.check_signatures
+ end
+ time_block 'check signatures + save' do
+ c.save!
+ end
+ c = time_block 'read' do
+ Collection.find_by_uuid(c.uuid)
+ end
+ time_block 'sign' do
+ c.signed_manifest_text
+ end
+ time_block 'sign + render' do
+ resp = c.as_api_response(nil)
+ end
+ c.manifest_text += ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:empty.txt\n"
+ time_block 'update' do
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ end
+ end
+end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list