[ARVADOS] created: 2.1.0-1256-g9b46e402b

Git user git at public.arvados.org
Fri Aug 27 20:04:29 UTC 2021


        at  9b46e402be4366b591ce9c73e2afb24bef0a3dd5 (commit)


commit 9b46e402be4366b591ce9c73e2afb24bef0a3dd5
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 16:02:10 2021 -0400

    17217: Update RailsAPI tests to not expect signatures.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 440ac6401..13b0261d9 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -69,12 +69,12 @@ class Arvados::V1::CollectionsController < ApplicationController
         include_old_versions: params[:include_old_versions],
       }
 
-      # It matters which Collection object we pick because we use it to get signed_manifest_text,
-      # the value of which is affected by the value of trash_at.
+      # It matters which Collection object we pick because blob
+      # signatures depend on the value of trash_at.
       #
-      # From postgres doc: "By default, null values sort as if larger than any non-null
-      # value; that is, NULLS FIRST is the default for DESC order, and
-      # NULLS LAST otherwise."
+      # From postgres doc: "By default, null values sort as if larger
+      # than any non-null value; that is, NULLS FIRST is the default
+      # for DESC order, and NULLS LAST otherwise."
       #
       # "trash_at desc" sorts null first, then latest to earliest, so
       # it will select the Collection object with the longest
@@ -84,7 +84,7 @@ class Arvados::V1::CollectionsController < ApplicationController
         @object = {
           uuid: c.portable_data_hash,
           portable_data_hash: c.portable_data_hash,
-          manifest_text: c.signed_manifest_text,
+          manifest_text: c.manifest_text,
         }
       end
     else
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 2560ac2c9..a98cde444 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -44,8 +44,8 @@ class Collection < ArvadosModel
     t.add :description
     t.add :properties
     t.add :portable_data_hash
-    t.add :signed_manifest_text, as: :manifest_text
     t.add :manifest_text, as: :unsigned_manifest_text
+    t.add :manifest_text, as: :manifest_text
     t.add :replication_desired
     t.add :replication_confirmed
     t.add :replication_confirmed_at
@@ -71,15 +71,11 @@ class Collection < ArvadosModel
 
   def self.attributes_required_columns
     super.merge(
-                # If we don't list manifest_text explicitly, the
-                # params[:select] code gets confused by the way we
-                # expose signed_manifest_text as manifest_text in the
-                # API response, and never let clients select the
-                # manifest_text column.
-                #
-                # We need trash_at and is_trashed to determine the
-                # correct timestamp in signed_manifest_text.
-                'manifest_text' => ['manifest_text', 'trash_at', 'is_trashed'],
+                # If we don't list unsigned_manifest_text explicitly,
+                # the params[:select] code gets confused by the way we
+                # expose manifest_text as unsigned_manifest_text in
+                # the API response, and never let clients select the
+                # unsigned_manifest_text column.
                 'unsigned_manifest_text' => ['manifest_text'],
                 'name' => ['name'],
                 )
@@ -406,7 +402,7 @@ class Collection < ArvadosModel
     end
   end
 
-  def signed_manifest_text
+  def signed_manifest_text_only_for_tests
     if !has_attribute? :manifest_text
       return nil
     elsif is_trashed
@@ -415,12 +411,22 @@ class Collection < ArvadosModel
       token = Thread.current[:token]
       exp = [db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i,
              trash_at].compact.map(&:to_i).min
-      self.class.sign_manifest manifest_text, token, exp
+      self.class.sign_manifest_only_for_tests manifest_text, token, exp
     end
   end
 
-  def self.sign_manifest manifest, token, exp=nil
-    return manifest
+  def self.sign_manifest_only_for_tests manifest, token, exp=nil
+    if exp.nil?
+      exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
+    end
+    signing_opts = {
+      api_token: token,
+      expire: exp,
+    }
+    m = munge_manifest_locators(manifest) do |match|
+      Blob.sign_locator(match[0], signing_opts)
+    end
+    return m
   end
 
   def self.munge_manifest_locators manifest
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 1ca2dd1dc..2c9470d97 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -32,8 +32,7 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
     end
   end
 
-  def assert_unsigned_manifest resp, label=''
-    txt = resp['unsigned_manifest_text']
+  def assert_unsigned_manifest txt, label=''
     assert_not_nil(txt, "#{label} unsigned_manifest_text was nil")
     locs = 0
     txt.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
@@ -66,24 +65,16 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
            "past version not included on index")
   end
 
-  test "collections.get returns signed locators, and no unsigned_manifest_text" do
+  test "collections.get returns unsigned locators, and no unsigned_manifest_text" do
     permit_unsigned_manifests
     authorize_with :active
     get :show, params: {id: collections(:foo_file).uuid}
     assert_response :success
-    assert_signed_manifest json_response['manifest_text'], 'foo_file'
+    assert_unsigned_manifest json_response["manifest_text"], 'foo_file'
     refute_includes json_response, 'unsigned_manifest_text'
   end
 
   ['v1token', 'v2token'].each do |token_method|
-    test "correct signatures are given for #{token_method}" do
-      token = api_client_authorizations(:active).send(token_method)
-      authorize_with_token token
-      get :show, params: {id: collections(:foo_file).uuid}
-      assert_response :success
-      assert_signed_manifest json_response['manifest_text'], 'foo_file', token: token
-    end
-
     test "signatures with #{token_method} are accepted" do
       token = api_client_authorizations(:active).send(token_method)
       signed = Blob.sign_locator(
@@ -98,11 +89,11 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
             },
           }
       assert_response :success
-      assert_signed_manifest json_response['manifest_text'], 'updated', token: token
+      assert_unsigned_manifest json_response['manifest_text'], 'updated'
     end
   end
 
-  test "index with manifest_text selected returns signed locators" do
+  test "index with manifest_text selected returns unsigned locators" do
     columns = %w(uuid owner_uuid manifest_text)
     authorize_with :active
     get :index, params: {select: columns}
@@ -112,7 +103,7 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
     json_response["items"].each do |coll|
       assert_equal(coll.keys - ['kind'], columns,
                    "Collections index did not respect selected columns")
-      assert_signed_manifest coll['manifest_text'], coll['uuid']
+      assert_unsigned_manifest coll['manifest_text'], coll['uuid']
     end
   end
 
@@ -125,7 +116,8 @@ class Arvados::V1::CollectionsControllerTest < ActionController::TestCase
     json_response["items"].each do |coll|
       assert_equal(coll.keys - ['kind'], ['unsigned_manifest_text'],
                    "Collections index did not respect selected columns")
-      locs += assert_unsigned_manifest coll, coll['uuid']
+      assert_nil coll['manifest_text']
+      locs += assert_unsigned_manifest coll['unsigned_manifest_text'], coll['uuid']
     end
     assert_operator locs, :>, 0, "no locators found in any manifests"
   end
@@ -338,7 +330,7 @@ EOS
       assert_not_nil assigns(:object)
       resp = assigns(:object)
       assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
-      assert_signed_manifest resp[:manifest_text]
+      assert_unsigned_manifest resp[:manifest_text]
 
       # The manifest in the response will have had permission hints added.
       # Remove any permission hints in the response before comparing it to the source.
@@ -388,7 +380,7 @@ EOS
       authorize_with :active
       manifest_text = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo.txt\n"
       if !unsigned
-        manifest_text = Collection.sign_manifest manifest_text, api_token(:active)
+        manifest_text = Collection.sign_manifest_only_for_tests manifest_text, api_token(:active)
       end
       post :create, params: {
         collection: {
@@ -594,10 +586,10 @@ EOS
       assert_not_nil assigns(:object)
       resp = JSON.parse(@response.body)
       assert_equal manifest_uuid, resp['portable_data_hash']
-      # All of the locators in the output must be signed.
+      # All of the signatures in the output must be valid.
       resp['manifest_text'].lines.each do |entry|
         m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
-        if m
+        if m && m[0].index('+A')
           assert Blob.verify_signature m[0], signing_opts
         end
       end
@@ -641,10 +633,10 @@ EOS
     assert_not_nil assigns(:object)
     resp = JSON.parse(@response.body)
     assert_equal manifest_uuid, resp['portable_data_hash']
-    # All of the locators in the output must be signed.
+    # All of the signatures in the output must be valid.
     resp['manifest_text'].lines.each do |entry|
       m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
-      if m
+      if m && m[0].index('+A')
         assert Blob.verify_signature m[0], signing_opts
       end
     end
@@ -746,50 +738,6 @@ EOS
     assert_equal manifest_text, stripped_manifest
   end
 
-  test "multiple signed locators per line" do
-    permit_unsigned_manifests
-    authorize_with :active
-    locators = %w(
-      d41d8cd98f00b204e9800998ecf8427e+0
-      acbd18db4cc2f85cedef654fccc4a4d8+3
-      ea10d51bcf88862dbcc36eb292017dfd+45)
-
-    signing_opts = {
-      key: Rails.configuration.Collections.BlobSigningKey,
-      api_token: api_token(:active),
-    }
-
-    unsigned_manifest = [".", *locators, "0:0:foo.txt\n"].join(" ")
-    manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
-      '+' +
-      unsigned_manifest.length.to_s
-
-    signed_locators = locators.map { |loc| Blob.sign_locator loc, signing_opts }
-    signed_manifest = [".", *signed_locators, "0:0:foo.txt\n"].join(" ")
-
-    post :create, params: {
-      collection: {
-        manifest_text: signed_manifest,
-        portable_data_hash: manifest_uuid,
-      }
-    }
-    assert_response :success
-    assert_not_nil assigns(:object)
-    resp = JSON.parse(@response.body)
-    assert_equal manifest_uuid, resp['portable_data_hash']
-    # All of the locators in the output must be signed.
-    # Each line is of the form "path locator locator ... 0:0:file.txt"
-    # entry.split[1..-2] will yield just the tokens in the middle of the line
-    returned_locator_count = 0
-    resp['manifest_text'].lines.each do |entry|
-      entry.split[1..-2].each do |tok|
-        returned_locator_count += 1
-        assert Blob.verify_signature tok, signing_opts
-      end
-    end
-    assert_equal locators.count, returned_locator_count
-  end
-
   test 'Reject manifest with unsigned blob' do
     permit_unsigned_manifests false
     authorize_with :active
diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb
index 070e964e5..baca6e278 100644
--- a/services/api/test/integration/collections_api_test.rb
+++ b/services/api/test/integration/collections_api_test.rb
@@ -224,7 +224,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
 
   test "create collection, update manifest, and search with filename" do
     # create collection
-    signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+    signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
     post "/arvados/v1/collections",
       params: {
         format: :json,
@@ -241,7 +241,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
     search_using_filter 'my_test_file.txt', 1
 
     # update the collection's manifest text
-    signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
+    signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
     put "/arvados/v1/collections/#{created['uuid']}",
       params: {
         format: :json,
@@ -375,7 +375,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
 
   test "create and get collection with properties" do
     # create collection to be searched for
-    signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+    signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
     post "/arvados/v1/collections",
       params: {
         format: :json,
@@ -401,7 +401,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
 
   test "create collection and update it with json encoded hash properties" do
     # create collection to be searched for
-    signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+    signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
     post "/arvados/v1/collections",
       params: {
         format: :json,
@@ -431,7 +431,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
     Rails.configuration.Collections.CollectionVersioning = true
     Rails.configuration.Collections.PreserveVersionIfIdle = -1 # Disable auto versioning
 
-    signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+    signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
     post "/arvados/v1/collections",
       params: {
         format: :json,
diff --git a/services/api/test/unit/collection_performance_test.rb b/services/api/test/unit/collection_performance_test.rb
index 4efc947dd..bbae49f4a 100644
--- a/services/api/test/unit/collection_performance_test.rb
+++ b/services/api/test/unit/collection_performance_test.rb
@@ -42,10 +42,7 @@ class CollectionModelPerformanceTest < ActiveSupport::TestCase
       c = time_block 'read' do
         Collection.find_by_uuid(c.uuid)
       end
-      time_block 'sign' do
-        c.signed_manifest_text
-      end
-      time_block 'sign + render' do
+      time_block 'render' do
         c.as_api_response(nil)
       end
       loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3',
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 8b8edbc15..de0f1d360 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -856,7 +856,7 @@ class CollectionTest < ActiveSupport::TestCase
   test "clear replication_confirmed* when introducing a new block in manifest" do
     c = collections(:replication_desired_2_confirmed_2)
     act_as_user users(:active) do
-      assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text)
+      assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text_only_for_tests)
       assert_nil c.replication_confirmed
       assert_nil c.replication_confirmed_at
     end
@@ -865,7 +865,7 @@ class CollectionTest < ActiveSupport::TestCase
   test "don't clear replication_confirmed* when just renaming a file" do
     c = collections(:replication_desired_2_confirmed_2)
     act_as_user users(:active) do
-      new_manifest = c.signed_manifest_text.sub(':bar', ':foo')
+      new_manifest = c.signed_manifest_text_only_for_tests.sub(':bar', ':foo')
       assert c.update_attributes(manifest_text: new_manifest)
       assert_equal 2, c.replication_confirmed
       assert_not_nil c.replication_confirmed_at
@@ -875,13 +875,13 @@ class CollectionTest < ActiveSupport::TestCase
   test "don't clear replication_confirmed* when just deleting a data block" do
     c = collections(:replication_desired_2_confirmed_2)
     act_as_user users(:active) do
-      new_manifest = c.signed_manifest_text
+      new_manifest = c.signed_manifest_text_only_for_tests
       new_manifest.sub!(/ \S+:bar/, '')
       new_manifest.sub!(/ acbd\S+/, '')
 
       # Confirm that we did just remove a block from the manifest (if
       # not, this test would pass without testing the relevant case):
-      assert_operator new_manifest.length+40, :<, c.signed_manifest_text.length
+      assert_operator new_manifest.length+40, :<, c.signed_manifest_text_only_for_tests.length
 
       assert c.update_attributes(manifest_text: new_manifest)
       assert_equal 2, c.replication_confirmed
@@ -895,7 +895,7 @@ class CollectionTest < ActiveSupport::TestCase
       c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n", name: 'foo')
       c.update_attributes! trash_at: (t0 + 1.hours)
       c.reload
-      sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+      sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
       assert_operator sig_exp.to_i, :<=, (t0 + 1.hours).to_i
     end
   end
@@ -905,7 +905,7 @@ class CollectionTest < ActiveSupport::TestCase
       c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n",
                              name: 'foo',
                              trash_at: db_current_time + 1.years)
-      sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+      sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
       expect_max_sig_exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
       assert_operator c.trash_at.to_i, :>, expect_max_sig_exp
       assert_operator sig_exp.to_i, :<=, expect_max_sig_exp

commit 138e5417d2d11fcf404a63ca6e08815bfc0cca67
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 14:36:18 2021 -0400

    17217: Test signature expiry times.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 02f16084f..34c2fa23c 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,6 +6,9 @@ package localdb
 
 import (
 	"context"
+	"regexp"
+	"strconv"
+	"time"
 
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/controller/rpc"
@@ -51,6 +54,7 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
 	resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
 	c.Check(err, check.IsNil)
 	c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+	s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
 
 	resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
 	c.Check(err, check.IsNil)
@@ -78,6 +82,28 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
 		c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 	}
 
+	// early trash date causes lower signature TTL
+	trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
+			"trash_at":      time.Now().UTC().Add(time.Hour),
+		}})
+	c.Assert(err, check.IsNil)
+	resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+	c.Assert(err, check.IsNil)
+	s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
+
+	// distant future trash date does not cause higher signature TTL
+	trashed, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
+			"trash_at":      time.Now().UTC().Add(time.Hour * 24 * 365),
+		}})
+	c.Assert(err, check.IsNil)
+	resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+	c.Assert(err, check.IsNil)
+	s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
+
 	// Make sure groups/contents doesn't return manifest_text with
 	// collections (if it did, we'd need to sign it).
 	gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
@@ -92,6 +118,16 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
 	}
 }
 
+func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
+	m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
+	c.Assert(m, check.HasLen, 2)
+	sigexp, err := strconv.ParseInt(m[1], 16, 64)
+	c.Assert(err, check.IsNil)
+	expectedExp := time.Now().Add(expectedTTL).Unix()
+	c.Check(sigexp > expectedExp-60, check.Equals, true)
+	c.Check(sigexp <= expectedExp, check.Equals, true)
+}
+
 func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
 	s.localdb.cluster.Collections.BlobSigning = false
 	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})

commit 2ab371465ac93bdf83d3fc423e361c50c54855d8
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 00:10:52 2021 -0400

    17217: Skip signatures when BlobSigning=false.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 4c0ffec60..6d74ab65f 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -64,6 +64,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 	cluster.API.MaxItemsPerResponse = 1000
 	cluster.API.MaxRequestAmplification = 4
 	cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+	cluster.Collections.BlobSigning = true
 	cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
 	cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
index 529310519..414c92679 100644
--- a/lib/controller/localdb/collection.go
+++ b/lib/controller/localdb/collection.go
@@ -49,7 +49,7 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions)
 }
 
 func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
-	if coll.IsTrashed || coll.ManifestText == "" {
+	if coll.IsTrashed || coll.ManifestText == "" || !conn.cluster.Collections.BlobSigning {
 		return
 	}
 	var token string
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index 78a1f4cb6..02f16084f 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -91,3 +91,12 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
 		c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
 	}
 }
+
+func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
+	s.localdb.cluster.Collections.BlobSigning = false
+	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+	resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+	c.Check(err, check.IsNil)
+	c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
+}
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index 54f8bdbeb..051f35571 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -40,6 +40,7 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
 		PostgreSQL: integrationTestCluster().PostgreSQL,
 	}}
 	handler.Cluster.TLS.Insecure = true
+	handler.Cluster.Collections.BlobSigning = true
 	handler.Cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
 	handler.Cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
 	arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))

commit 7c66aef044ec7bf48aad64cd82308f48ef00c143
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 00:04:30 2021 -0400

    17217: Add signatures test for groups/contents.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index d75ca3c40..78a1f4cb6 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -77,4 +77,17 @@ func (s *CollectionSuite) TestSignatures(c *check.C) {
 		c.Check(lresp.Items[0].ManifestText, check.Equals, "")
 		c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
 	}
+
+	// Make sure groups/contents doesn't return manifest_text with
+	// collections (if it did, we'd need to sign it).
+	gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
+		Limit:   -1,
+		Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
+		Select:  []string{"uuid", "manifest_text"},
+	})
+	c.Check(err, check.IsNil)
+	if c.Check(gresp.Items, check.HasLen, 1) {
+		c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
+		c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
+	}
 }

commit 9ed0a21813117caf5c6bae73c09a0d725564743b
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 00:04:26 2021 -0400

    17217: Update tests, remove unused code.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/controller/fed_collections.go b/lib/controller/fed_collections.go
deleted file mode 100644
index a0a123129..000000000
--- a/lib/controller/fed_collections.go
+++ /dev/null
@@ -1,312 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"sync"
-
-	"git.arvados.org/arvados.git/sdk/go/arvados"
-	"git.arvados.org/arvados.git/sdk/go/httpserver"
-	"git.arvados.org/arvados.git/sdk/go/keepclient"
-)
-
-func rewriteSignatures(clusterID string, expectHash string,
-	resp *http.Response, requestError error) (newResponse *http.Response, err error) {
-
-	if requestError != nil {
-		return resp, requestError
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return resp, nil
-	}
-
-	originalBody := resp.Body
-	defer originalBody.Close()
-
-	var col arvados.Collection
-	err = json.NewDecoder(resp.Body).Decode(&col)
-	if err != nil {
-		return nil, err
-	}
-
-	// rewriting signatures will make manifest text 5-10% bigger so calculate
-	// capacity accordingly
-	updatedManifest := bytes.NewBuffer(make([]byte, 0, int(float64(len(col.ManifestText))*1.1)))
-
-	hasher := md5.New()
-	mw := io.MultiWriter(hasher, updatedManifest)
-	sz := 0
-
-	scanner := bufio.NewScanner(strings.NewReader(col.ManifestText))
-	scanner.Buffer(make([]byte, 1048576), len(col.ManifestText))
-	for scanner.Scan() {
-		line := scanner.Text()
-		tokens := strings.Split(line, " ")
-		if len(tokens) < 3 {
-			return nil, fmt.Errorf("Invalid stream (<3 tokens): %q", line)
-		}
-
-		n, err := mw.Write([]byte(tokens[0]))
-		if err != nil {
-			return nil, fmt.Errorf("Error updating manifest: %v", err)
-		}
-		sz += n
-		for _, token := range tokens[1:] {
-			n, err = mw.Write([]byte(" "))
-			if err != nil {
-				return nil, fmt.Errorf("Error updating manifest: %v", err)
-			}
-			sz += n
-
-			m := keepclient.SignedLocatorRe.FindStringSubmatch(token)
-			if m != nil {
-				// Rewrite the block signature to be a remote signature
-				_, err = fmt.Fprintf(updatedManifest, "%s%s%s+R%s-%s%s", m[1], m[2], m[3], clusterID, m[5][2:], m[8])
-				if err != nil {
-					return nil, fmt.Errorf("Error updating manifest: %v", err)
-				}
-
-				// for hash checking, ignore signatures
-				n, err = fmt.Fprintf(hasher, "%s%s", m[1], m[2])
-				if err != nil {
-					return nil, fmt.Errorf("Error updating manifest: %v", err)
-				}
-				sz += n
-			} else {
-				n, err = mw.Write([]byte(token))
-				if err != nil {
-					return nil, fmt.Errorf("Error updating manifest: %v", err)
-				}
-				sz += n
-			}
-		}
-		n, err = mw.Write([]byte("\n"))
-		if err != nil {
-			return nil, fmt.Errorf("Error updating manifest: %v", err)
-		}
-		sz += n
-	}
-
-	// Check that expected hash is consistent with
-	// portable_data_hash field of the returned record
-	if expectHash == "" {
-		expectHash = col.PortableDataHash
-	} else if expectHash != col.PortableDataHash {
-		return nil, fmt.Errorf("portable_data_hash %q on returned record did not match expected hash %q ", expectHash, col.PortableDataHash)
-	}
-
-	// Certify that the computed hash of the manifest_text matches our expectation
-	sum := hasher.Sum(nil)
-	computedHash := fmt.Sprintf("%x+%v", sum, sz)
-	if computedHash != expectHash {
-		return nil, fmt.Errorf("Computed manifest_text hash %q did not match expected hash %q", computedHash, expectHash)
-	}
-
-	col.ManifestText = updatedManifest.String()
-
-	newbody, err := json.Marshal(col)
-	if err != nil {
-		return nil, err
-	}
-
-	buf := bytes.NewBuffer(newbody)
-	resp.Body = ioutil.NopCloser(buf)
-	resp.ContentLength = int64(buf.Len())
-	resp.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
-	return resp, nil
-}
-
-func filterLocalClusterResponse(resp *http.Response, requestError error) (newResponse *http.Response, err error) {
-	if requestError != nil {
-		return resp, requestError
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		// Suppress returning this result, because we want to
-		// search the federation.
-		return nil, nil
-	}
-	return resp, nil
-}
-
-type searchRemoteClusterForPDH struct {
-	pdh           string
-	remoteID      string
-	mtx           *sync.Mutex
-	sentResponse  *bool
-	sharedContext *context.Context
-	cancelFunc    func()
-	errors        *[]string
-	statusCode    *int
-}
-
-func fetchRemoteCollectionByUUID(
-	h *genericFederatedRequestHandler,
-	effectiveMethod string,
-	clusterID *string,
-	uuid string,
-	remainder string,
-	w http.ResponseWriter,
-	req *http.Request) bool {
-
-	if effectiveMethod != "GET" {
-		// Only handle GET requests right now
-		return false
-	}
-
-	if uuid != "" {
-		// Collection UUID GET request
-		*clusterID = uuid[0:5]
-		if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID {
-			// request for remote collection by uuid
-			resp, err := h.handler.remoteClusterRequest(*clusterID, req)
-			newResponse, err := rewriteSignatures(*clusterID, "", resp, err)
-			h.handler.proxy.ForwardResponse(w, newResponse, err)
-			return true
-		}
-	}
-
-	return false
-}
-
-func fetchRemoteCollectionByPDH(
-	h *genericFederatedRequestHandler,
-	effectiveMethod string,
-	clusterID *string,
-	uuid string,
-	remainder string,
-	w http.ResponseWriter,
-	req *http.Request) bool {
-
-	if effectiveMethod != "GET" {
-		// Only handle GET requests right now
-		return false
-	}
-
-	m := collectionsByPDHRe.FindStringSubmatch(req.URL.Path)
-	if len(m) != 2 {
-		return false
-	}
-
-	// Request for collection by PDH.  Search the federation.
-
-	// First, query the local cluster.
-	resp, err := h.handler.localClusterRequest(req)
-	newResp, err := filterLocalClusterResponse(resp, err)
-	if newResp != nil || err != nil {
-		h.handler.proxy.ForwardResponse(w, newResp, err)
-		return true
-	}
-
-	// Create a goroutine for each cluster in the
-	// RemoteClusters map.  The first valid result gets
-	// returned to the client.  When that happens, all
-	// other outstanding requests are cancelled
-	sharedContext, cancelFunc := context.WithCancel(req.Context())
-	defer cancelFunc()
-
-	req = req.WithContext(sharedContext)
-	wg := sync.WaitGroup{}
-	pdh := m[1]
-	success := make(chan *http.Response)
-	errorChan := make(chan error, len(h.handler.Cluster.RemoteClusters))
-
-	acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
-
-	for remoteID := range h.handler.Cluster.RemoteClusters {
-		if remoteID == h.handler.Cluster.ClusterID {
-			// No need to query local cluster again
-			continue
-		}
-		if remoteID == "*" {
-			// This isn't a real remote cluster: it just sets defaults for unlisted remotes.
-			continue
-		}
-
-		wg.Add(1)
-		go func(remote string) {
-			defer wg.Done()
-			acquire()
-			defer release()
-			select {
-			case <-sharedContext.Done():
-				return
-			default:
-			}
-
-			resp, err := h.handler.remoteClusterRequest(remote, req)
-			wasSuccess := false
-			defer func() {
-				if resp != nil && !wasSuccess {
-					resp.Body.Close()
-				}
-			}()
-			if err != nil {
-				errorChan <- err
-				return
-			}
-			if resp.StatusCode != http.StatusOK {
-				errorChan <- HTTPError{resp.Status, resp.StatusCode}
-				return
-			}
-			select {
-			case <-sharedContext.Done():
-				return
-			default:
-			}
-
-			newResponse, err := rewriteSignatures(remote, pdh, resp, nil)
-			if err != nil {
-				errorChan <- err
-				return
-			}
-			select {
-			case <-sharedContext.Done():
-			case success <- newResponse:
-				wasSuccess = true
-			}
-		}(remoteID)
-	}
-	go func() {
-		wg.Wait()
-		cancelFunc()
-	}()
-
-	errorCode := http.StatusNotFound
-
-	for {
-		select {
-		case newResp = <-success:
-			h.handler.proxy.ForwardResponse(w, newResp, nil)
-			return true
-		case <-sharedContext.Done():
-			var errors []string
-			for len(errorChan) > 0 {
-				err := <-errorChan
-				if httperr, ok := err.(HTTPError); !ok || httperr.Code != http.StatusNotFound {
-					errorCode = http.StatusBadGateway
-				}
-				errors = append(errors, err.Error())
-			}
-			httpserver.Errors(w, errors, errorCode)
-			return true
-		}
-	}
-
-	// shouldn't ever get here
-	return true
-}
diff --git a/lib/controller/fed_containers.go b/lib/controller/fed_containers.go
deleted file mode 100644
index fd4f0521b..000000000
--- a/lib/controller/fed_containers.go
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-
-	"git.arvados.org/arvados.git/sdk/go/auth"
-	"git.arvados.org/arvados.git/sdk/go/httpserver"
-)
-
-func remoteContainerRequestCreate(
-	h *genericFederatedRequestHandler,
-	effectiveMethod string,
-	clusterID *string,
-	uuid string,
-	remainder string,
-	w http.ResponseWriter,
-	req *http.Request) bool {
-
-	if effectiveMethod != "POST" || uuid != "" || remainder != "" {
-		return false
-	}
-
-	// First make sure supplied token is valid.
-	creds := auth.NewCredentials()
-	creds.LoadTokensFromHTTPRequest(req)
-
-	currentUser, ok, err := h.handler.validateAPItoken(req, creds.Tokens[0])
-	if err != nil {
-		httpserver.Error(w, err.Error(), http.StatusInternalServerError)
-		return true
-	} else if !ok {
-		httpserver.Error(w, "invalid API token", http.StatusForbidden)
-		return true
-	}
-
-	if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID {
-		// Submitting container request to local cluster. No
-		// need to set a runtime_token (rails api will create
-		// one when the container runs) or do a remote cluster
-		// request.
-		return false
-	}
-
-	if req.Header.Get("Content-Type") != "application/json" {
-		httpserver.Error(w, "Expected Content-Type: application/json, got "+req.Header.Get("Content-Type"), http.StatusBadRequest)
-		return true
-	}
-
-	originalBody := req.Body
-	defer originalBody.Close()
-	var request map[string]interface{}
-	err = json.NewDecoder(req.Body).Decode(&request)
-	if err != nil {
-		httpserver.Error(w, err.Error(), http.StatusBadRequest)
-		return true
-	}
-
-	crString, ok := request["container_request"].(string)
-	if ok {
-		var crJSON map[string]interface{}
-		err := json.Unmarshal([]byte(crString), &crJSON)
-		if err != nil {
-			httpserver.Error(w, err.Error(), http.StatusBadRequest)
-			return true
-		}
-
-		request["container_request"] = crJSON
-	}
-
-	containerRequest, ok := request["container_request"].(map[string]interface{})
-	if !ok {
-		// Use toplevel object as the container_request object
-		containerRequest = request
-	}
-
-	// If runtime_token is not set, create a new token
-	if _, ok := containerRequest["runtime_token"]; !ok {
-		if len(currentUser.Authorization.Scopes) != 1 || currentUser.Authorization.Scopes[0] != "all" {
-			httpserver.Error(w, "Token scope is not [all]", http.StatusForbidden)
-			return true
-		}
-
-		if strings.HasPrefix(currentUser.Authorization.UUID, h.handler.Cluster.ClusterID) {
-			// Local user, submitting to a remote cluster.
-			// Create a new time-limited token.
-			newtok, err := h.handler.createAPItoken(req, currentUser.UUID, nil)
-			if err != nil {
-				httpserver.Error(w, err.Error(), http.StatusForbidden)
-				return true
-			}
-			containerRequest["runtime_token"] = newtok.TokenV2()
-		} else {
-			// Remote user. Container request will use the
-			// current token, minus the trailing portion
-			// (optional container uuid).
-			sp := strings.Split(creds.Tokens[0], "/")
-			if len(sp) >= 3 {
-				containerRequest["runtime_token"] = strings.Join(sp[0:3], "/")
-			} else {
-				containerRequest["runtime_token"] = creds.Tokens[0]
-			}
-		}
-	}
-
-	newbody, err := json.Marshal(request)
-	buf := bytes.NewBuffer(newbody)
-	req.Body = ioutil.NopCloser(buf)
-	req.ContentLength = int64(buf.Len())
-	req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
-	resp, err := h.handler.remoteClusterRequest(*clusterID, req)
-	h.handler.proxy.ForwardResponse(w, resp, err)
-	return true
-}
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index 419d8b010..144d41c21 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -94,20 +94,12 @@ func (h *Handler) setupProxyRemoteCluster(next http.Handler) http.Handler {
 
 	wfHandler := &genericFederatedRequestHandler{next, h, wfRe, nil}
 	containersHandler := &genericFederatedRequestHandler{next, h, containersRe, nil}
-	containerRequestsHandler := &genericFederatedRequestHandler{next, h, containerRequestsRe,
-		[]federatedRequestDelegate{remoteContainerRequestCreate}}
-	collectionsRequestsHandler := &genericFederatedRequestHandler{next, h, collectionsRe,
-		[]federatedRequestDelegate{fetchRemoteCollectionByUUID, fetchRemoteCollectionByPDH}}
 	linksRequestsHandler := &genericFederatedRequestHandler{next, h, linksRe, nil}
 
 	mux.Handle("/arvados/v1/workflows", wfHandler)
 	mux.Handle("/arvados/v1/workflows/", wfHandler)
 	mux.Handle("/arvados/v1/containers", containersHandler)
 	mux.Handle("/arvados/v1/containers/", containersHandler)
-	mux.Handle("/arvados/v1/container_requests", containerRequestsHandler)
-	mux.Handle("/arvados/v1/container_requests/", containerRequestsHandler)
-	mux.Handle("/arvados/v1/collections", collectionsRequestsHandler)
-	mux.Handle("/arvados/v1/collections/", collectionsRequestsHandler)
 	mux.Handle("/arvados/v1/links", linksRequestsHandler)
 	mux.Handle("/arvados/v1/links/", linksRequestsHandler)
 	mux.Handle("/", next)
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index f4cadd821..4c0ffec60 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -64,6 +64,8 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 	cluster.API.MaxItemsPerResponse = 1000
 	cluster.API.MaxRequestAmplification = 4
 	cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+	cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+	cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
 	arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
 	s.testHandler = &Handler{Cluster: cluster}
@@ -695,7 +697,6 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
 	s.testHandler.Cluster.ClusterID = "zzzzz"
 	s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
 	s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
-	s.testHandler.Cluster.Collections.BlobSigningTTL = arvados.Duration(336 * time.Hour) // For some reason, this was set to 0h
 
 	resp := s.testRequest(req).Result()
 	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index e3558c3f4..54f8bdbeb 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
@@ -39,6 +40,8 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
 		PostgreSQL: integrationTestCluster().PostgreSQL,
 	}}
 	handler.Cluster.TLS.Insecure = true
+	handler.Cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+	handler.Cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
 	arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
 

commit 8e75f57bd693d0ceb1aab86ba0e84cb19b4d155a
Author: Tom Clegg <tom at curii.com>
Date:   Tue Aug 24 16:07:43 2021 -0400

    17217: Remove crufty test util funcs.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 9fee66e1d..2975e3b3d 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -33,7 +33,6 @@ func (s *Suite) TearDownSuite(c *check.C) {
 }
 
 func (s *Suite) SetUpSuite(c *check.C) {
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(2, true)
 
 	// Get the various arvados, arvadosclient, and keep client objects
diff --git a/lib/recovercollection/cmd_test.go b/lib/recovercollection/cmd_test.go
index 7b3c8e1b4..f891e5567 100644
--- a/lib/recovercollection/cmd_test.go
+++ b/lib/recovercollection/cmd_test.go
@@ -27,7 +27,6 @@ var _ = check.Suite(&Suite{})
 type Suite struct{}
 
 func (*Suite) SetUpSuite(c *check.C) {
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(2, true)
 }
 
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index 5b01db5c4..8f70c5ee2 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -5,53 +5,40 @@
 package arvadostest
 
 import (
-	"bufio"
-	"bytes"
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"path"
 	"strconv"
 	"strings"
+
+	"gopkg.in/check.v1"
 )
 
 var authSettings = make(map[string]string)
 
-// ResetEnv resets test env
+// ResetEnv resets ARVADOS_* env vars to whatever they were the first
+// time this func was called.
+//
+// Call it from your SetUpTest or SetUpSuite func if your tests modify
+// env vars.
 func ResetEnv() {
-	for k, v := range authSettings {
-		os.Setenv(k, v)
-	}
-}
-
-// APIHost returns the address:port of the current test server.
-func APIHost() string {
-	h := authSettings["ARVADOS_API_HOST"]
-	if h == "" {
-		log.Fatal("arvadostest.APIHost() was called but authSettings is not populated")
-	}
-	return h
-}
-
-// ParseAuthSettings parses auth settings from given input
-func ParseAuthSettings(authScript []byte) {
-	scanner := bufio.NewScanner(bytes.NewReader(authScript))
-	for scanner.Scan() {
-		line := scanner.Text()
-		if 0 != strings.Index(line, "export ") {
-			log.Printf("Ignoring: %v", line)
-			continue
+	if len(authSettings) == 0 {
+		for _, e := range os.Environ() {
+			e := strings.SplitN(e, "=", 2)
+			if len(e) == 2 {
+				authSettings[e[0]] = e[1]
+			}
 		}
-		toks := strings.SplitN(strings.Replace(line, "export ", "", 1), "=", 2)
-		if len(toks) == 2 {
-			authSettings[toks[0]] = toks[1]
-		} else {
-			log.Fatalf("Could not parse: %v", line)
+	} else {
+		for k, v := range authSettings {
+			os.Setenv(k, v)
 		}
 	}
-	log.Printf("authSettings: %v", authSettings)
 }
 
 var pythonTestDir string
@@ -80,34 +67,17 @@ func chdirToPythonTests() {
 	}
 }
 
-// StartAPI starts test API server
-func StartAPI() {
-	cwd, _ := os.Getwd()
-	defer os.Chdir(cwd)
-	chdirToPythonTests()
-
-	cmd := exec.Command("python", "run_test_server.py", "start", "--auth", "admin")
-	cmd.Stdin = nil
-	cmd.Stderr = os.Stderr
-
-	authScript, err := cmd.Output()
-	if err != nil {
-		log.Fatalf("%+v: %s", cmd.Args, err)
-	}
-	ParseAuthSettings(authScript)
-	ResetEnv()
-}
-
-// StopAPI stops test API server
-func StopAPI() {
-	cwd, _ := os.Getwd()
-	defer os.Chdir(cwd)
-	chdirToPythonTests()
-
-	cmd := exec.Command("python", "run_test_server.py", "stop")
-	bgRun(cmd)
-	// Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
-	cmd.Wait()
+func ResetDB(c *check.C) {
+	hc := http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}}
+	req, err := http.NewRequest("POST", "https://"+os.Getenv("ARVADOS_TEST_API_HOST")+"/database/reset", nil)
+	c.Assert(err, check.IsNil)
+	req.Header.Set("Authorization", "Bearer "+AdminToken)
+	resp, err := hc.Do(req)
+	c.Assert(err, check.IsNil)
+	defer resp.Body.Close()
+	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 }
 
 // StartKeep starts the given number of keep servers,
diff --git a/sdk/go/dispatch/dispatch_test.go b/sdk/go/dispatch/dispatch_test.go
index 4b115229b..2a9d84639 100644
--- a/sdk/go/dispatch/dispatch_test.go
+++ b/sdk/go/dispatch/dispatch_test.go
@@ -18,14 +18,6 @@ var _ = Suite(&suite{})
 
 type suite struct{}
 
-func (s *suite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
-}
-
-func (s *suite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
-}
-
 func (s *suite) TestTrackContainer(c *C) {
 	arv, err := arvadosclient.MakeArvadosClient()
 	c.Assert(err, Equals, nil)
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 62268fa46..cddf03bc3 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -52,13 +52,11 @@ func pythonDir() string {
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(2, false)
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
 	arvadostest.StopKeep(2)
-	arvadostest.StopAPI()
 }
 
 func (s *ServerRequiredSuite) SetUpTest(c *C) {
diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go
index 4e1a47dcb..688b256dc 100644
--- a/services/arv-git-httpd/auth_handler_test.go
+++ b/services/arv-git-httpd/auth_handler_test.go
@@ -26,14 +26,6 @@ type AuthHandlerSuite struct {
 	cluster *arvados.Cluster
 }
 
-func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
-	arvadostest.StartAPI()
-}
-
-func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
-	arvadostest.StopAPI()
-}
-
 func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
 	arvadostest.ResetEnv()
 	repoRoot, err := filepath.Abs("../api/tmp/git/test")
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index 7da85ee74..93a46e224 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -33,14 +33,6 @@ type IntegrationSuite struct {
 	cluster     *arvados.Cluster
 }
 
-func (s *IntegrationSuite) SetUpSuite(c *check.C) {
-	arvadostest.StartAPI()
-}
-
-func (s *IntegrationSuite) TearDownSuite(c *check.C) {
-	arvadostest.StopAPI()
-}
-
 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 	arvadostest.ResetEnv()
 
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 92b8d2adc..7e8c42c25 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -39,15 +39,10 @@ var initialArgs []string
 
 func (s *TestSuite) SetUpSuite(c *C) {
 	initialArgs = os.Args
-	arvadostest.StartAPI()
 	runningCmds = make(map[string]*exec.Cmd)
 	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})
 }
 
-func (s *TestSuite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
-}
-
 func (s *TestSuite) SetUpTest(c *C) {
 	args := []string{"crunch-dispatch-local"}
 	os.Args = args
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index e7a89db23..cf83257da 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -42,7 +42,8 @@ type IntegrationSuite struct {
 }
 
 func (s *IntegrationSuite) SetUpTest(c *C) {
-	arvadostest.StartAPI()
+	arvadostest.ResetEnv()
+	arvadostest.ResetDB(c)
 	os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token)
 	s.disp = Dispatcher{}
 	s.disp.cluster = &arvados.Cluster{}
@@ -52,7 +53,7 @@ func (s *IntegrationSuite) SetUpTest(c *C) {
 
 func (s *IntegrationSuite) TearDownTest(c *C) {
 	arvadostest.ResetEnv()
-	arvadostest.StopAPI()
+	arvadostest.ResetDB(c)
 }
 
 type slurmFake struct {
diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index 52e614915..a6cc32810 100644
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -38,7 +38,6 @@ func (s *integrationSuite) SetUpSuite(c *check.C) {
 		c.Skip("-short")
 	}
 	arvadostest.ResetEnv()
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(4, true)
 
 	arv, err := arvadosclient.MakeArvadosClient()
@@ -62,7 +61,6 @@ func (s *integrationSuite) TearDownSuite(c *check.C) {
 		c.Skip("-short")
 	}
 	arvadostest.StopKeep(4)
-	arvadostest.StopAPI()
 }
 
 func (s *integrationSuite) SetUpTest(c *check.C) {
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index a65a48892..21d767208 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -410,7 +410,7 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 }
 
 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
-	arvadostest.StartAPI()
+	arvadostest.ResetDB(c)
 	arvadostest.StartKeep(2, true)
 
 	arv, err := arvadosclient.MakeArvadosClient()
@@ -426,7 +426,6 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 
 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 	arvadostest.StopKeep(2)
-	arvadostest.StopAPI()
 }
 
 func (s *IntegrationSuite) SetUpTest(c *check.C) {
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 4bdb42020..052109bf2 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -76,7 +76,6 @@ func closeListener() {
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(2, false)
 }
 
@@ -86,11 +85,9 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) {
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
 	arvadostest.StopKeep(2)
-	arvadostest.StopAPI()
 }
 
 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
 	// config.yml defines 4 keepstores
 	arvadostest.StartKeep(4, false)
 }
@@ -101,11 +98,9 @@ func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
 
 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
 	arvadostest.StopKeep(4)
-	arvadostest.StopAPI()
 }
 
 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
 	// We need API to have some keep services listed, but the
 	// services themselves should be unresponsive.
 	arvadostest.StartKeep(2, false)
@@ -116,10 +111,6 @@ func (s *NoKeepServerSuite) SetUpTest(c *C) {
 	arvadostest.ResetEnv()
 }
 
-func (s *NoKeepServerSuite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
-}
-
 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
 	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 	c.Assert(err, Equals, nil)
diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index ad7ec15e7..eb7fe5fd6 100644
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -58,7 +58,6 @@ func (s *HandlerSuite) TestPullWorkerIntegration_GetNonExistingLocator(c *check.
 	}
 
 	pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false)
-	defer arvadostest.StopAPI()
 	defer arvadostest.StopKeep(2)
 
 	s.performPullWorkerIntegrationTest(testData, pullRequest, c)
@@ -76,7 +75,6 @@ func (s *HandlerSuite) TestPullWorkerIntegration_GetExistingLocator(c *check.C)
 	}
 
 	pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true)
-	defer arvadostest.StopAPI()
 	defer arvadostest.StopKeep(2)
 
 	s.performPullWorkerIntegrationTest(testData, pullRequest, c)
diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go
index 9f409e6af..e6519fb37 100644
--- a/tools/keep-block-check/keep-block-check_test.go
+++ b/tools/keep-block-check/keep-block-check_test.go
@@ -43,12 +43,7 @@ var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
 
 var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
 
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
-}
-
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
 	arvadostest.ResetEnv()
 }
 
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 38968ae12..45ed3f67f 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -43,12 +43,7 @@ var _ = Suite(&DoMainTestSuite{})
 
 type ServerRequiredSuite struct{}
 
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
-}
-
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
 	arvadostest.ResetEnv()
 }
 

commit bdd74e4dff9d9c5c1a329cfbed2e2e08336c8c51
Author: Tom Clegg <tom at curii.com>
Date:   Tue Aug 24 16:07:34 2021 -0400

    17217: Sign manifests in controller instead of RailsAPI.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
new file mode 100644
index 000000000..529310519
--- /dev/null
+++ b/lib/controller/localdb/collection.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"context"
+	"time"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/auth"
+)
+
+// CollectionGet defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionGet(ctx context.Context, opts arvados.GetOptions) (arvados.Collection, error) {
+	if len(opts.Select) > 0 {
+		// We need to know IsTrashed and TrashAt to implement
+		// signing properly, even if the caller doesn't want
+		// them.
+		opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+	}
+	resp, err := conn.railsProxy.CollectionGet(ctx, opts)
+	if err != nil {
+		return resp, err
+	}
+	conn.signCollection(ctx, &resp)
+	return resp, nil
+}
+
+// CollectionList defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) (arvados.CollectionList, error) {
+	if len(opts.Select) > 0 {
+		// We need to know IsTrashed and TrashAt to implement
+		// signing properly, even if the caller doesn't want
+		// them.
+		opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+	}
+	resp, err := conn.railsProxy.CollectionList(ctx, opts)
+	if err != nil {
+		return resp, err
+	}
+	for i := range resp.Items {
+		conn.signCollection(ctx, &resp.Items[i])
+	}
+	return resp, nil
+}
+
+func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
+	if coll.IsTrashed || coll.ManifestText == "" {
+		return
+	}
+	var token string
+	if creds, ok := auth.FromContext(ctx); ok && len(creds.Tokens) > 0 {
+		token = creds.Tokens[0]
+	}
+	if token == "" {
+		return
+	}
+	ttl := conn.cluster.Collections.BlobSigningTTL.Duration()
+	exp := time.Now().Add(ttl)
+	if coll.TrashAt != nil && !coll.TrashAt.IsZero() && coll.TrashAt.Before(exp) {
+		exp = *coll.TrashAt
+	}
+	coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
+}
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
new file mode 100644
index 000000000..d75ca3c40
--- /dev/null
+++ b/lib/controller/localdb/collection_test.go
@@ -0,0 +1,80 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"context"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/lib/controller/rpc"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CollectionSuite{})
+
+type CollectionSuite struct {
+	cluster  *arvados.Cluster
+	localdb  *Conn
+	railsSpy *arvadostest.Proxy
+}
+
+func (s *CollectionSuite) TearDownSuite(c *check.C) {
+	// Undo any changes/additions to the user database so they
+	// don't affect subsequent tests.
+	arvadostest.ResetEnv()
+	c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *CollectionSuite) SetUpTest(c *check.C) {
+	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+	c.Assert(err, check.IsNil)
+	s.cluster, err = cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	s.localdb = NewConn(s.cluster)
+	s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+	*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *CollectionSuite) TearDownTest(c *check.C) {
+	s.railsSpy.Close()
+}
+
+func (s *CollectionSuite) TestSignatures(c *check.C) {
+	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+	resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+	c.Check(err, check.IsNil)
+	c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+
+	resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
+	c.Check(err, check.IsNil)
+	c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+
+	lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
+	c.Check(err, check.IsNil)
+	if c.Check(lresp.Items, check.HasLen, 1) {
+		c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
+		c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+		c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+	}
+
+	lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
+	c.Check(err, check.IsNil)
+	if c.Check(lresp.Items, check.HasLen, 1) {
+		c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+		c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+	}
+
+	lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
+	c.Check(err, check.IsNil)
+	if c.Check(lresp.Items, check.HasLen, 1) {
+		c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+		c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
+	}
+}
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 1bbe8cc66..2560ac2c9 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -420,17 +420,7 @@ class Collection < ArvadosModel
   end
 
   def self.sign_manifest manifest, token, exp=nil
-    if exp.nil?
-      exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
-    end
-    signing_opts = {
-      api_token: token,
-      expire: exp,
-    }
-    m = munge_manifest_locators(manifest) do |match|
-      Blob.sign_locator(match[0], signing_opts)
-    end
-    return m
+    return manifest
   end
 
   def self.munge_manifest_locators manifest

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list