[ARVADOS] created: 2.1.0-1256-ga1cf45928
Git user
git at public.arvados.org
Thu Aug 26 20:02:45 UTC 2021
at a1cf45928a5d4fe4e351889b91d6124b35b60960 (commit)
commit a1cf45928a5d4fe4e351889b91d6124b35b60960
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 26 16:02:10 2021 -0400
17217: Update RailsAPI tests to not expect signatures.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 440ac6401..13b0261d9 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -69,12 +69,12 @@ class Arvados::V1::CollectionsController < ApplicationController
include_old_versions: params[:include_old_versions],
}
- # It matters which Collection object we pick because we use it to get signed_manifest_text,
- # the value of which is affected by the value of trash_at.
+ # It matters which Collection object we pick because blob
+ # signatures depend on the value of trash_at.
#
- # From postgres doc: "By default, null values sort as if larger than any non-null
- # value; that is, NULLS FIRST is the default for DESC order, and
- # NULLS LAST otherwise."
+ # From postgres doc: "By default, null values sort as if larger
+ # than any non-null value; that is, NULLS FIRST is the default
+ # for DESC order, and NULLS LAST otherwise."
#
# "trash_at desc" sorts null first, then latest to earliest, so
# it will select the Collection object with the longest
@@ -84,7 +84,7 @@ class Arvados::V1::CollectionsController < ApplicationController
@object = {
uuid: c.portable_data_hash,
portable_data_hash: c.portable_data_hash,
- manifest_text: c.signed_manifest_text,
+ manifest_text: c.manifest_text,
}
end
else
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 2560ac2c9..a98cde444 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -44,8 +44,8 @@ class Collection < ArvadosModel
t.add :description
t.add :properties
t.add :portable_data_hash
- t.add :signed_manifest_text, as: :manifest_text
t.add :manifest_text, as: :unsigned_manifest_text
+ t.add :manifest_text, as: :manifest_text
t.add :replication_desired
t.add :replication_confirmed
t.add :replication_confirmed_at
@@ -71,15 +71,11 @@ class Collection < ArvadosModel
def self.attributes_required_columns
super.merge(
- # If we don't list manifest_text explicitly, the
- # params[:select] code gets confused by the way we
- # expose signed_manifest_text as manifest_text in the
- # API response, and never let clients select the
- # manifest_text column.
- #
- # We need trash_at and is_trashed to determine the
- # correct timestamp in signed_manifest_text.
- 'manifest_text' => ['manifest_text', 'trash_at', 'is_trashed'],
+ # If we don't list unsigned_manifest_text explicitly,
+ # the params[:select] code gets confused by the way we
+ # expose manifest_text as unsigned_manifest_text in
+ # the API response, and never let clients select the
+ # unsigned_manifest_text column.
'unsigned_manifest_text' => ['manifest_text'],
'name' => ['name'],
)
@@ -406,7 +402,7 @@ class Collection < ArvadosModel
end
end
- def signed_manifest_text
+ def signed_manifest_text_only_for_tests
if !has_attribute? :manifest_text
return nil
elsif is_trashed
@@ -415,12 +411,22 @@ class Collection < ArvadosModel
token = Thread.current[:token]
exp = [db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i,
trash_at].compact.map(&:to_i).min
- self.class.sign_manifest manifest_text, token, exp
+ self.class.sign_manifest_only_for_tests manifest_text, token, exp
end
end
- def self.sign_manifest manifest, token, exp=nil
- return manifest
+ def self.sign_manifest_only_for_tests manifest, token, exp=nil
+ if exp.nil?
+ exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
+ end
+ signing_opts = {
+ api_token: token,
+ expire: exp,
+ }
+ m = munge_manifest_locators(manifest) do |match|
+ Blob.sign_locator(match[0], signing_opts)
+ end
+ return m
end
def self.munge_manifest_locators manifest
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 1ca2dd1dc..2c9470d97 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -32,8 +32,7 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
end
end
- def assert_unsigned_manifest resp, label=''
- txt = resp['unsigned_manifest_text']
+ def assert_unsigned_manifest txt, label=''
assert_not_nil(txt, "#{label} unsigned_manifest_text was nil")
locs = 0
txt.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
@@ -66,24 +65,16 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
"past version not included on index")
end
- test "collections.get returns signed locators, and no unsigned_manifest_text" do
+ test "collections.get returns unsigned locators, and no unsigned_manifest_text" do
permit_unsigned_manifests
authorize_with :active
get :show, params: {id: collections(:foo_file).uuid}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file'
+ assert_unsigned_manifest json_response["manifest_text"], 'foo_file'
refute_includes json_response, 'unsigned_manifest_text'
end
['v1token', 'v2token'].each do |token_method|
- test "correct signatures are given for #{token_method}" do
- token = api_client_authorizations(:active).send(token_method)
- authorize_with_token token
- get :show, params: {id: collections(:foo_file).uuid}
- assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file', token: token
- end
-
test "signatures with #{token_method} are accepted" do
token = api_client_authorizations(:active).send(token_method)
signed = Blob.sign_locator(
@@ -98,11 +89,11 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
},
}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'updated', token: token
+ assert_unsigned_manifest json_response['manifest_text'], 'updated'
end
end
- test "index with manifest_text selected returns signed locators" do
+ test "index with manifest_text selected returns unsigned locators" do
columns = %w(uuid owner_uuid manifest_text)
authorize_with :active
get :index, params: {select: columns}
@@ -112,7 +103,7 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], columns,
"Collections index did not respect selected columns")
- assert_signed_manifest coll['manifest_text'], coll['uuid']
+ assert_unsigned_manifest coll['manifest_text'], coll['uuid']
end
end
@@ -125,7 +116,8 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], ['unsigned_manifest_text'],
"Collections index did not respect selected columns")
- locs += assert_unsigned_manifest coll, coll['uuid']
+ assert_nil coll['manifest_text']
+ locs += assert_unsigned_manifest coll['unsigned_manifest_text'], coll['uuid']
end
assert_operator locs, :>, 0, "no locators found in any manifests"
end
@@ -338,7 +330,7 @@ EOS
assert_not_nil assigns(:object)
resp = assigns(:object)
assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
- assert_signed_manifest resp[:manifest_text]
+ assert_unsigned_manifest resp[:manifest_text]
# The manifest in the response will have had permission hints added.
# Remove any permission hints in the response before comparing it to the source.
@@ -388,7 +380,7 @@ EOS
authorize_with :active
manifest_text = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo.txt\n"
if !unsigned
- manifest_text = Collection.sign_manifest manifest_text, api_token(:active)
+ manifest_text = Collection.sign_manifest_only_for_tests manifest_text, api_token(:active)
end
post :create, params: {
collection: {
@@ -594,10 +586,10 @@ EOS
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
@@ -641,10 +633,10 @@ EOS
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
@@ -746,50 +738,6 @@ EOS
assert_equal manifest_text, stripped_manifest
end
- test "multiple signed locators per line" do
- permit_unsigned_manifests
- authorize_with :active
- locators = %w(
- d41d8cd98f00b204e9800998ecf8427e+0
- acbd18db4cc2f85cedef654fccc4a4d8+3
- ea10d51bcf88862dbcc36eb292017dfd+45)
-
- signing_opts = {
- key: Rails.configuration.Collections.BlobSigningKey,
- api_token: api_token(:active),
- }
-
- unsigned_manifest = [".", *locators, "0:0:foo.txt\n"].join(" ")
- manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
- '+' +
- unsigned_manifest.length.to_s
-
- signed_locators = locators.map { |loc| Blob.sign_locator loc, signing_opts }
- signed_manifest = [".", *signed_locators, "0:0:foo.txt\n"].join(" ")
-
- post :create, params: {
- collection: {
- manifest_text: signed_manifest,
- portable_data_hash: manifest_uuid,
- }
- }
- assert_response :success
- assert_not_nil assigns(:object)
- resp = JSON.parse(@response.body)
- assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
- # Each line is of the form "path locator locator ... 0:0:file.txt"
- # entry.split[1..-2] will yield just the tokens in the middle of the line
- returned_locator_count = 0
- resp['manifest_text'].lines.each do |entry|
- entry.split[1..-2].each do |tok|
- returned_locator_count += 1
- assert Blob.verify_signature tok, signing_opts
- end
- end
- assert_equal locators.count, returned_locator_count
- end
-
test 'Reject manifest with unsigned blob' do
permit_unsigned_manifests false
authorize_with :active
diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb
index 070e964e5..baca6e278 100644
--- a/services/api/test/integration/collections_api_test.rb
+++ b/services/api/test/integration/collections_api_test.rb
@@ -224,7 +224,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
test "create collection, update manifest, and search with filename" do
# create collection
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
@@ -241,7 +241,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
search_using_filter 'my_test_file.txt', 1
# update the collection's manifest text
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
put "/arvados/v1/collections/#{created['uuid']}",
params: {
format: :json,
@@ -375,7 +375,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
test "create and get collection with properties" do
# create collection to be searched for
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
@@ -401,7 +401,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
test "create collection and update it with json encoded hash properties" do
# create collection to be searched for
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
@@ -431,7 +431,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
Rails.configuration.Collections.CollectionVersioning = true
Rails.configuration.Collections.PreserveVersionIfIdle = -1 # Disable auto versioning
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
diff --git a/services/api/test/unit/collection_performance_test.rb b/services/api/test/unit/collection_performance_test.rb
index 4efc947dd..bbae49f4a 100644
--- a/services/api/test/unit/collection_performance_test.rb
+++ b/services/api/test/unit/collection_performance_test.rb
@@ -42,10 +42,7 @@ class CollectionModelPerformanceTest < ActiveSupport::TestCase
c = time_block 'read' do
Collection.find_by_uuid(c.uuid)
end
- time_block 'sign' do
- c.signed_manifest_text
- end
- time_block 'sign + render' do
+ time_block 'render' do
c.as_api_response(nil)
end
loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3',
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 8b8edbc15..de0f1d360 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -856,7 +856,7 @@ class CollectionTest < ActiveSupport::TestCase
test "clear replication_confirmed* when introducing a new block in manifest" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text)
+ assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text_only_for_tests)
assert_nil c.replication_confirmed
assert_nil c.replication_confirmed_at
end
@@ -865,7 +865,7 @@ class CollectionTest < ActiveSupport::TestCase
test "don't clear replication_confirmed* when just renaming a file" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- new_manifest = c.signed_manifest_text.sub(':bar', ':foo')
+ new_manifest = c.signed_manifest_text_only_for_tests.sub(':bar', ':foo')
assert c.update_attributes(manifest_text: new_manifest)
assert_equal 2, c.replication_confirmed
assert_not_nil c.replication_confirmed_at
@@ -875,13 +875,13 @@ class CollectionTest < ActiveSupport::TestCase
test "don't clear replication_confirmed* when just deleting a data block" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- new_manifest = c.signed_manifest_text
+ new_manifest = c.signed_manifest_text_only_for_tests
new_manifest.sub!(/ \S+:bar/, '')
new_manifest.sub!(/ acbd\S+/, '')
# Confirm that we did just remove a block from the manifest (if
# not, this test would pass without testing the relevant case):
- assert_operator new_manifest.length+40, :<, c.signed_manifest_text.length
+ assert_operator new_manifest.length+40, :<, c.signed_manifest_text_only_for_tests.length
assert c.update_attributes(manifest_text: new_manifest)
assert_equal 2, c.replication_confirmed
@@ -895,7 +895,7 @@ class CollectionTest < ActiveSupport::TestCase
c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n", name: 'foo')
c.update_attributes! trash_at: (t0 + 1.hours)
c.reload
- sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+ sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
assert_operator sig_exp.to_i, :<=, (t0 + 1.hours).to_i
end
end
@@ -905,7 +905,7 @@ class CollectionTest < ActiveSupport::TestCase
c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n",
name: 'foo',
trash_at: db_current_time + 1.years)
- sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+ sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
expect_max_sig_exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
assert_operator c.trash_at.to_i, :>, expect_max_sig_exp
assert_operator sig_exp.to_i, :<=, expect_max_sig_exp
commit 33ff1f418e43f72602df13814045a50207c7cfed
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 26 14:36:18 2021 -0400
17217: Test signature expiry times.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 02f16084f..34c2fa23c 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,6 +6,9 @@ package localdb
import (
"context"
+ "regexp"
+ "strconv"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
@@ -51,6 +54,7 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
c.Check(err, check.IsNil)
c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
c.Check(err, check.IsNil)
@@ -78,6 +82,28 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
}
+ // early trash date causes lower signature TTL
+ trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
+ "trash_at": time.Now().UTC().Add(time.Hour),
+ }})
+ c.Assert(err, check.IsNil)
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
+
+ // distant future trash date does not cause higher signature TTL
+ trashed, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
+ "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
+ }})
+ c.Assert(err, check.IsNil)
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
+
// Make sure groups/contents doesn't return manifest_text with
// collections (if it did, we'd need to sign it).
gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
@@ -92,6 +118,16 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
}
}
+func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
+ m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
+ c.Assert(m, check.HasLen, 2)
+ sigexp, err := strconv.ParseInt(m[1], 16, 64)
+ c.Assert(err, check.IsNil)
+ expectedExp := time.Now().Add(expectedTTL).Unix()
+ c.Check(sigexp > expectedExp-60, check.Equals, true)
+ c.Check(sigexp <= expectedExp, check.Equals, true)
+}
+
func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
s.localdb.cluster.Collections.BlobSigning = false
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
commit 32433d2885cc9f0f671964e871c51ce2c77df0f1
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 26 00:10:52 2021 -0400
17217: Skip signatures when BlobSigning=false.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
index 529310519..414c92679 100644
--- a/lib/controller/localdb/collection.go
+++ b/lib/controller/localdb/collection.go
@@ -49,7 +49,7 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions)
}
func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
- if coll.IsTrashed || coll.ManifestText == "" {
+ if coll.IsTrashed || coll.ManifestText == "" || !conn.cluster.Collections.BlobSigning {
return
}
var token string
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 78a1f4cb6..02f16084f 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -91,3 +91,12 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
}
}
+
+func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
+ s.localdb.cluster.Collections.BlobSigning = false
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
+}
commit 7c66aef044ec7bf48aad64cd82308f48ef00c143
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 26 00:04:30 2021 -0400
17217: Add signatures test for groups/contents.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index d75ca3c40..78a1f4cb6 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -77,4 +77,17 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
c.Check(lresp.Items[0].ManifestText, check.Equals, "")
c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
}
+
+ // Make sure groups/contents doesn't return manifest_text with
+ // collections (if it did, we'd need to sign it).
+ gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
+ Limit: -1,
+ Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
+ Select: []string{"uuid", "manifest_text"},
+ })
+ c.Check(err, check.IsNil)
+ if c.Check(gresp.Items, check.HasLen, 1) {
+ c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
+ c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
+ }
}
commit 9ed0a21813117caf5c6bae73c09a0d725564743b
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 26 00:04:26 2021 -0400
17217: Update tests, remove unused code.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/fed_collections.go b/lib/controller/fed_collections.go
deleted file mode 100644
index a0a123129..000000000
--- a/lib/controller/fed_collections.go
+++ /dev/null
@@ -1,312 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
- "bufio"
- "bytes"
- "context"
- "crypto/md5"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "strings"
- "sync"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- "git.arvados.org/arvados.git/sdk/go/keepclient"
-)
-
-func rewriteSignatures(clusterID string, expectHash string,
- resp *http.Response, requestError error) (newResponse *http.Response, err error) {
-
- if requestError != nil {
- return resp, requestError
- }
-
- if resp.StatusCode != http.StatusOK {
- return resp, nil
- }
-
- originalBody := resp.Body
- defer originalBody.Close()
-
- var col arvados.Collection
- err = json.NewDecoder(resp.Body).Decode(&col)
- if err != nil {
- return nil, err
- }
-
- // rewriting signatures will make manifest text 5-10% bigger so calculate
- // capacity accordingly
- updatedManifest := bytes.NewBuffer(make([]byte, 0, int(float64(len(col.ManifestText))*1.1)))
-
- hasher := md5.New()
- mw := io.MultiWriter(hasher, updatedManifest)
- sz := 0
-
- scanner := bufio.NewScanner(strings.NewReader(col.ManifestText))
- scanner.Buffer(make([]byte, 1048576), len(col.ManifestText))
- for scanner.Scan() {
- line := scanner.Text()
- tokens := strings.Split(line, " ")
- if len(tokens) < 3 {
- return nil, fmt.Errorf("Invalid stream (<3 tokens): %q", line)
- }
-
- n, err := mw.Write([]byte(tokens[0]))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- for _, token := range tokens[1:] {
- n, err = mw.Write([]byte(" "))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
-
- m := keepclient.SignedLocatorRe.FindStringSubmatch(token)
- if m != nil {
- // Rewrite the block signature to be a remote signature
- _, err = fmt.Fprintf(updatedManifest, "%s%s%s+R%s-%s%s", m[1], m[2], m[3], clusterID, m[5][2:], m[8])
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
-
- // for hash checking, ignore signatures
- n, err = fmt.Fprintf(hasher, "%s%s", m[1], m[2])
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- } else {
- n, err = mw.Write([]byte(token))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- }
- }
- n, err = mw.Write([]byte("\n"))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- }
-
- // Check that expected hash is consistent with
- // portable_data_hash field of the returned record
- if expectHash == "" {
- expectHash = col.PortableDataHash
- } else if expectHash != col.PortableDataHash {
- return nil, fmt.Errorf("portable_data_hash %q on returned record did not match expected hash %q ", expectHash, col.PortableDataHash)
- }
-
- // Certify that the computed hash of the manifest_text matches our expectation
- sum := hasher.Sum(nil)
- computedHash := fmt.Sprintf("%x+%v", sum, sz)
- if computedHash != expectHash {
- return nil, fmt.Errorf("Computed manifest_text hash %q did not match expected hash %q", computedHash, expectHash)
- }
-
- col.ManifestText = updatedManifest.String()
-
- newbody, err := json.Marshal(col)
- if err != nil {
- return nil, err
- }
-
- buf := bytes.NewBuffer(newbody)
- resp.Body = ioutil.NopCloser(buf)
- resp.ContentLength = int64(buf.Len())
- resp.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
- return resp, nil
-}
-
-func filterLocalClusterResponse(resp *http.Response, requestError error) (newResponse *http.Response, err error) {
- if requestError != nil {
- return resp, requestError
- }
-
- if resp.StatusCode == http.StatusNotFound {
- // Suppress returning this result, because we want to
- // search the federation.
- return nil, nil
- }
- return resp, nil
-}
-
-type searchRemoteClusterForPDH struct {
- pdh string
- remoteID string
- mtx *sync.Mutex
- sentResponse *bool
- sharedContext *context.Context
- cancelFunc func()
- errors *[]string
- statusCode *int
-}
-
-func fetchRemoteCollectionByUUID(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "GET" {
- // Only handle GET requests right now
- return false
- }
-
- if uuid != "" {
- // Collection UUID GET request
- *clusterID = uuid[0:5]
- if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID {
- // request for remote collection by uuid
- resp, err := h.handler.remoteClusterRequest(*clusterID, req)
- newResponse, err := rewriteSignatures(*clusterID, "", resp, err)
- h.handler.proxy.ForwardResponse(w, newResponse, err)
- return true
- }
- }
-
- return false
-}
-
-func fetchRemoteCollectionByPDH(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "GET" {
- // Only handle GET requests right now
- return false
- }
-
- m := collectionsByPDHRe.FindStringSubmatch(req.URL.Path)
- if len(m) != 2 {
- return false
- }
-
- // Request for collection by PDH. Search the federation.
-
- // First, query the local cluster.
- resp, err := h.handler.localClusterRequest(req)
- newResp, err := filterLocalClusterResponse(resp, err)
- if newResp != nil || err != nil {
- h.handler.proxy.ForwardResponse(w, newResp, err)
- return true
- }
-
- // Create a goroutine for each cluster in the
- // RemoteClusters map. The first valid result gets
- // returned to the client. When that happens, all
- // other outstanding requests are cancelled
- sharedContext, cancelFunc := context.WithCancel(req.Context())
- defer cancelFunc()
-
- req = req.WithContext(sharedContext)
- wg := sync.WaitGroup{}
- pdh := m[1]
- success := make(chan *http.Response)
- errorChan := make(chan error, len(h.handler.Cluster.RemoteClusters))
-
- acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
-
- for remoteID := range h.handler.Cluster.RemoteClusters {
- if remoteID == h.handler.Cluster.ClusterID {
- // No need to query local cluster again
- continue
- }
- if remoteID == "*" {
- // This isn't a real remote cluster: it just sets defaults for unlisted remotes.
- continue
- }
-
- wg.Add(1)
- go func(remote string) {
- defer wg.Done()
- acquire()
- defer release()
- select {
- case <-sharedContext.Done():
- return
- default:
- }
-
- resp, err := h.handler.remoteClusterRequest(remote, req)
- wasSuccess := false
- defer func() {
- if resp != nil && !wasSuccess {
- resp.Body.Close()
- }
- }()
- if err != nil {
- errorChan <- err
- return
- }
- if resp.StatusCode != http.StatusOK {
- errorChan <- HTTPError{resp.Status, resp.StatusCode}
- return
- }
- select {
- case <-sharedContext.Done():
- return
- default:
- }
-
- newResponse, err := rewriteSignatures(remote, pdh, resp, nil)
- if err != nil {
- errorChan <- err
- return
- }
- select {
- case <-sharedContext.Done():
- case success <- newResponse:
- wasSuccess = true
- }
- }(remoteID)
- }
- go func() {
- wg.Wait()
- cancelFunc()
- }()
-
- errorCode := http.StatusNotFound
-
- for {
- select {
- case newResp = <-success:
- h.handler.proxy.ForwardResponse(w, newResp, nil)
- return true
- case <-sharedContext.Done():
- var errors []string
- for len(errorChan) > 0 {
- err := <-errorChan
- if httperr, ok := err.(HTTPError); !ok || httperr.Code != http.StatusNotFound {
- errorCode = http.StatusBadGateway
- }
- errors = append(errors, err.Error())
- }
- httpserver.Errors(w, errors, errorCode)
- return true
- }
- }
-
- // shouldn't ever get here
- return true
-}
diff --git a/lib/controller/fed_containers.go b/lib/controller/fed_containers.go
deleted file mode 100644
index fd4f0521b..000000000
--- a/lib/controller/fed_containers.go
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "net/http"
- "strings"
-
- "git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
-)
-
-func remoteContainerRequestCreate(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "POST" || uuid != "" || remainder != "" {
- return false
- }
-
- // First make sure supplied token is valid.
- creds := auth.NewCredentials()
- creds.LoadTokensFromHTTPRequest(req)
-
- currentUser, ok, err := h.handler.validateAPItoken(req, creds.Tokens[0])
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusInternalServerError)
- return true
- } else if !ok {
- httpserver.Error(w, "invalid API token", http.StatusForbidden)
- return true
- }
-
- if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID {
- // Submitting container request to local cluster. No
- // need to set a runtime_token (rails api will create
- // one when the container runs) or do a remote cluster
- // request.
- return false
- }
-
- if req.Header.Get("Content-Type") != "application/json" {
- httpserver.Error(w, "Expected Content-Type: application/json, got "+req.Header.Get("Content-Type"), http.StatusBadRequest)
- return true
- }
-
- originalBody := req.Body
- defer originalBody.Close()
- var request map[string]interface{}
- err = json.NewDecoder(req.Body).Decode(&request)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusBadRequest)
- return true
- }
-
- crString, ok := request["container_request"].(string)
- if ok {
- var crJSON map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJSON)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusBadRequest)
- return true
- }
-
- request["container_request"] = crJSON
- }
-
- containerRequest, ok := request["container_request"].(map[string]interface{})
- if !ok {
- // Use toplevel object as the container_request object
- containerRequest = request
- }
-
- // If runtime_token is not set, create a new token
- if _, ok := containerRequest["runtime_token"]; !ok {
- if len(currentUser.Authorization.Scopes) != 1 || currentUser.Authorization.Scopes[0] != "all" {
- httpserver.Error(w, "Token scope is not [all]", http.StatusForbidden)
- return true
- }
-
- if strings.HasPrefix(currentUser.Authorization.UUID, h.handler.Cluster.ClusterID) {
- // Local user, submitting to a remote cluster.
- // Create a new time-limited token.
- newtok, err := h.handler.createAPItoken(req, currentUser.UUID, nil)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusForbidden)
- return true
- }
- containerRequest["runtime_token"] = newtok.TokenV2()
- } else {
- // Remote user. Container request will use the
- // current token, minus the trailing portion
- // (optional container uuid).
- sp := strings.Split(creds.Tokens[0], "/")
- if len(sp) >= 3 {
- containerRequest["runtime_token"] = strings.Join(sp[0:3], "/")
- } else {
- containerRequest["runtime_token"] = creds.Tokens[0]
- }
- }
- }
-
- newbody, err := json.Marshal(request)
- buf := bytes.NewBuffer(newbody)
- req.Body = ioutil.NopCloser(buf)
- req.ContentLength = int64(buf.Len())
- req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
- resp, err := h.handler.remoteClusterRequest(*clusterID, req)
- h.handler.proxy.ForwardResponse(w, resp, err)
- return true
-}
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index 419d8b010..144d41c21 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -94,20 +94,12 @@ func (h *Handler) setupProxyRemoteCluster(next http.Handler) http.Handler {
wfHandler := &genericFederatedRequestHandler{next, h, wfRe, nil}
containersHandler := &genericFederatedRequestHandler{next, h, containersRe, nil}
- containerRequestsHandler := &genericFederatedRequestHandler{next, h, containerRequestsRe,
- []federatedRequestDelegate{remoteContainerRequestCreate}}
- collectionsRequestsHandler := &genericFederatedRequestHandler{next, h, collectionsRe,
- []federatedRequestDelegate{fetchRemoteCollectionByUUID, fetchRemoteCollectionByPDH}}
linksRequestsHandler := &genericFederatedRequestHandler{next, h, linksRe, nil}
mux.Handle("/arvados/v1/workflows", wfHandler)
mux.Handle("/arvados/v1/workflows/", wfHandler)
mux.Handle("/arvados/v1/containers", containersHandler)
mux.Handle("/arvados/v1/containers/", containersHandler)
- mux.Handle("/arvados/v1/container_requests", containerRequestsHandler)
- mux.Handle("/arvados/v1/container_requests/", containerRequestsHandler)
- mux.Handle("/arvados/v1/collections", collectionsRequestsHandler)
- mux.Handle("/arvados/v1/collections/", collectionsRequestsHandler)
mux.Handle("/arvados/v1/links", linksRequestsHandler)
mux.Handle("/arvados/v1/links/", linksRequestsHandler)
mux.Handle("/", next)
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index f4cadd821..4c0ffec60 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -64,6 +64,8 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
cluster.API.MaxItemsPerResponse = 1000
cluster.API.MaxRequestAmplification = 4
cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+ cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+ cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
@@ -695,7 +697,6 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
s.testHandler.Cluster.ClusterID = "zzzzz"
s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
- s.testHandler.Cluster.Collections.BlobSigningTTL = arvados.Duration(336 * time.Hour) // For some reason, this was set to 0h
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index e3558c3f4..54f8bdbeb 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -9,6 +9,7 @@ import (
"net/http"
"os"
"path/filepath"
+ "time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
@@ -39,6 +40,8 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
PostgreSQL: integrationTestCluster().PostgreSQL,
}}
handler.Cluster.TLS.Insecure = true
+ handler.Cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+ handler.Cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
commit 8e75f57bd693d0ceb1aab86ba0e84cb19b4d155a
Author: Tom Clegg <tom at curii.com>
Date: Tue Aug 24 16:07:43 2021 -0400
17217: Remove crufty test util funcs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 9fee66e1d..2975e3b3d 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -33,7 +33,6 @@ func (s *Suite) TearDownSuite(c *check.C) {
}
func (s *Suite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
// Get the various arvados, arvadosclient, and keep client objects
diff --git a/lib/recovercollection/cmd_test.go b/lib/recovercollection/cmd_test.go
index 7b3c8e1b4..f891e5567 100644
--- a/lib/recovercollection/cmd_test.go
+++ b/lib/recovercollection/cmd_test.go
@@ -27,7 +27,6 @@ var _ = check.Suite(&Suite{})
type Suite struct{}
func (*Suite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
}
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index 5b01db5c4..8f70c5ee2 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -5,53 +5,40 @@
package arvadostest
import (
- "bufio"
- "bytes"
+ "crypto/tls"
"fmt"
"io/ioutil"
"log"
+ "net/http"
"os"
"os/exec"
"path"
"strconv"
"strings"
+
+ "gopkg.in/check.v1"
)
var authSettings = make(map[string]string)
-// ResetEnv resets test env
+// ResetEnv resets ARVADOS_* env vars to whatever they were the first
+// time this func was called.
+//
+// Call it from your SetUpTest or SetUpSuite func if your tests modify
+// env vars.
func ResetEnv() {
- for k, v := range authSettings {
- os.Setenv(k, v)
- }
-}
-
-// APIHost returns the address:port of the current test server.
-func APIHost() string {
- h := authSettings["ARVADOS_API_HOST"]
- if h == "" {
- log.Fatal("arvadostest.APIHost() was called but authSettings is not populated")
- }
- return h
-}
-
-// ParseAuthSettings parses auth settings from given input
-func ParseAuthSettings(authScript []byte) {
- scanner := bufio.NewScanner(bytes.NewReader(authScript))
- for scanner.Scan() {
- line := scanner.Text()
- if 0 != strings.Index(line, "export ") {
- log.Printf("Ignoring: %v", line)
- continue
+ if len(authSettings) == 0 {
+ for _, e := range os.Environ() {
+ e := strings.SplitN(e, "=", 2)
+ if len(e) == 2 {
+ authSettings[e[0]] = e[1]
+ }
}
- toks := strings.SplitN(strings.Replace(line, "export ", "", 1), "=", 2)
- if len(toks) == 2 {
- authSettings[toks[0]] = toks[1]
- } else {
- log.Fatalf("Could not parse: %v", line)
+ } else {
+ for k, v := range authSettings {
+ os.Setenv(k, v)
}
}
- log.Printf("authSettings: %v", authSettings)
}
var pythonTestDir string
@@ -80,34 +67,17 @@ func chdirToPythonTests() {
}
}
-// StartAPI starts test API server
-func StartAPI() {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
- chdirToPythonTests()
-
- cmd := exec.Command("python", "run_test_server.py", "start", "--auth", "admin")
- cmd.Stdin = nil
- cmd.Stderr = os.Stderr
-
- authScript, err := cmd.Output()
- if err != nil {
- log.Fatalf("%+v: %s", cmd.Args, err)
- }
- ParseAuthSettings(authScript)
- ResetEnv()
-}
-
-// StopAPI stops test API server
-func StopAPI() {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
- chdirToPythonTests()
-
- cmd := exec.Command("python", "run_test_server.py", "stop")
- bgRun(cmd)
- // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
- cmd.Wait()
+func ResetDB(c *check.C) {
+ hc := http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }}
+ req, err := http.NewRequest("POST", "https://"+os.Getenv("ARVADOS_TEST_API_HOST")+"/database/reset", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+AdminToken)
+ resp, err := hc.Do(req)
+ c.Assert(err, check.IsNil)
+ defer resp.Body.Close()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
}
// StartKeep starts the given number of keep servers,
diff --git a/sdk/go/dispatch/dispatch_test.go b/sdk/go/dispatch/dispatch_test.go
index 4b115229b..2a9d84639 100644
--- a/sdk/go/dispatch/dispatch_test.go
+++ b/sdk/go/dispatch/dispatch_test.go
@@ -18,14 +18,6 @@ var _ = Suite(&suite{})
type suite struct{}
-func (s *suite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
-func (s *suite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func (s *suite) TestTrackContainer(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 62268fa46..cddf03bc3 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -52,13 +52,11 @@ func pythonDir() string {
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) SetUpTest(c *C) {
diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go
index 4e1a47dcb..688b256dc 100644
--- a/services/arv-git-httpd/auth_handler_test.go
+++ b/services/arv-git-httpd/auth_handler_test.go
@@ -26,14 +26,6 @@ type AuthHandlerSuite struct {
cluster *arvados.Cluster
}
-func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
-}
-
-func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
- arvadostest.StopAPI()
-}
-
func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
repoRoot, err := filepath.Abs("../api/tmp/git/test")
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index 7da85ee74..93a46e224 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -33,14 +33,6 @@ type IntegrationSuite struct {
cluster *arvados.Cluster
}
-func (s *IntegrationSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
-}
-
-func (s *IntegrationSuite) TearDownSuite(c *check.C) {
- arvadostest.StopAPI()
-}
-
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 92b8d2adc..7e8c42c25 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -39,15 +39,10 @@ var initialArgs []string
func (s *TestSuite) SetUpSuite(c *C) {
initialArgs = os.Args
- arvadostest.StartAPI()
runningCmds = make(map[string]*exec.Cmd)
logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})
}
-func (s *TestSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func (s *TestSuite) SetUpTest(c *C) {
args := []string{"crunch-dispatch-local"}
os.Args = args
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index e7a89db23..cf83257da 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -42,7 +42,8 @@ type IntegrationSuite struct {
}
func (s *IntegrationSuite) SetUpTest(c *C) {
- arvadostest.StartAPI()
+ arvadostest.ResetEnv()
+ arvadostest.ResetDB(c)
os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token)
s.disp = Dispatcher{}
s.disp.cluster = &arvados.Cluster{}
@@ -52,7 +53,7 @@ func (s *IntegrationSuite) SetUpTest(c *C) {
func (s *IntegrationSuite) TearDownTest(c *C) {
arvadostest.ResetEnv()
- arvadostest.StopAPI()
+ arvadostest.ResetDB(c)
}
type slurmFake struct {
diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index 52e614915..a6cc32810 100644
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -38,7 +38,6 @@ func (s *integrationSuite) SetUpSuite(c *check.C) {
c.Skip("-short")
}
arvadostest.ResetEnv()
- arvadostest.StartAPI()
arvadostest.StartKeep(4, true)
arv, err := arvadosclient.MakeArvadosClient()
@@ -62,7 +61,6 @@ func (s *integrationSuite) TearDownSuite(c *check.C) {
c.Skip("-short")
}
arvadostest.StopKeep(4)
- arvadostest.StopAPI()
}
func (s *integrationSuite) SetUpTest(c *check.C) {
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index a65a48892..21d767208 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -410,7 +410,7 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
+ arvadostest.ResetDB(c)
arvadostest.StartKeep(2, true)
arv, err := arvadosclient.MakeArvadosClient()
@@ -426,7 +426,6 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
func (s *IntegrationSuite) TearDownSuite(c *check.C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *IntegrationSuite) SetUpTest(c *check.C) {
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 4bdb42020..052109bf2 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -76,7 +76,6 @@ func closeListener() {
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
@@ -86,11 +85,9 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) {
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
// config.yml defines 4 keepstores
arvadostest.StartKeep(4, false)
}
@@ -101,11 +98,9 @@ func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(4)
- arvadostest.StopAPI()
}
func (s *NoKeepServerSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
// We need API to have some keep services listed, but the
// services themselves should be unresponsive.
arvadostest.StartKeep(2, false)
@@ -116,10 +111,6 @@ func (s *NoKeepServerSuite) SetUpTest(c *C) {
arvadostest.ResetEnv()
}
-func (s *NoKeepServerSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, Equals, nil)
diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index ad7ec15e7..eb7fe5fd6 100644
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -58,7 +58,6 @@ func (s *HandlerSuite) TestPullWorkerIntegration_GetNonExistingLocator(c *check.
}
pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false)
- defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
s.performPullWorkerIntegrationTest(testData, pullRequest, c)
@@ -76,7 +75,6 @@ func (s *HandlerSuite) TestPullWorkerIntegration_GetExistingLocator(c *check.C)
}
pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true)
- defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
s.performPullWorkerIntegrationTest(testData, pullRequest, c)
diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go
index 9f409e6af..e6519fb37 100644
--- a/tools/keep-block-check/keep-block-check_test.go
+++ b/tools/keep-block-check/keep-block-check_test.go
@@ -43,12 +43,7 @@ var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
arvadostest.ResetEnv()
}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 38968ae12..45ed3f67f 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -43,12 +43,7 @@ var _ = Suite(&DoMainTestSuite{})
type ServerRequiredSuite struct{}
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
arvadostest.ResetEnv()
}
commit bdd74e4dff9d9c5c1a329cfbed2e2e08336c8c51
Author: Tom Clegg <tom at curii.com>
Date: Tue Aug 24 16:07:34 2021 -0400
17217: Sign manifests in controller instead of RailsAPI.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
new file mode 100644
index 000000000..529310519
--- /dev/null
+++ b/lib/controller/localdb/collection.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+)
+
+// CollectionGet defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionGet(ctx context.Context, opts arvados.GetOptions) (arvados.Collection, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionGet(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ conn.signCollection(ctx, &resp)
+ return resp, nil
+}
+
+// CollectionList defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) (arvados.CollectionList, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionList(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ for i := range resp.Items {
+ conn.signCollection(ctx, &resp.Items[i])
+ }
+ return resp, nil
+}
+
+func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
+ if coll.IsTrashed || coll.ManifestText == "" {
+ return
+ }
+ var token string
+ if creds, ok := auth.FromContext(ctx); ok && len(creds.Tokens) > 0 {
+ token = creds.Tokens[0]
+ }
+ if token == "" {
+ return
+ }
+ ttl := conn.cluster.Collections.BlobSigningTTL.Duration()
+ exp := time.Now().Add(ttl)
+ if coll.TrashAt != nil && !coll.TrashAt.IsZero() && coll.TrashAt.Before(exp) {
+ exp = *coll.TrashAt
+ }
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
+}
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
new file mode 100644
index 000000000..d75ca3c40
--- /dev/null
+++ b/lib/controller/localdb/collection_test.go
@@ -0,0 +1,80 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CollectionSuite{})
+
+type CollectionSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *CollectionSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *CollectionSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *CollectionSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *CollectionSuite) TestSignatures(c *check.C) {
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+
+ lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
+ c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+ }
+
+ lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+ }
+
+ lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
+ }
+}
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 1bbe8cc66..2560ac2c9 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -420,17 +420,7 @@ class Collection < ArvadosModel
end
def self.sign_manifest manifest, token, exp=nil
- if exp.nil?
- exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
- end
- signing_opts = {
- api_token: token,
- expire: exp,
- }
- m = munge_manifest_locators(manifest) do |match|
- Blob.sign_locator(match[0], signing_opts)
- end
- return m
+ return manifest
end
def self.munge_manifest_locators manifest
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list