[ARVADOS] created: 1.3.0-2555-gf53557462
Git user
git at public.arvados.org
Thu May 14 16:27:36 UTC 2020
at f5355746202a7d486e36d8440bb8c725bb9ca56c (commit)
commit f5355746202a7d486e36d8440bb8c725bb9ca56c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu May 14 12:25:18 2020 -0400
16312: Use V4 signatures for all AWS regions. Add config override.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 23d712043..dff4fc7e3 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -40,6 +40,10 @@ h2(#master). development master (as of 2020-02-07)
None in current development master.
+h3. S3 signatures
+
+Keepstore now uses "V4 signatures":https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html by default for S3 requests. If you are using Amazon S3, no action is needed; all regions support V4 signatures. If you are using a different S3-compatible service that does not support V4 signatures, add @V2Signature: true@ to your volume driver parameters to preserve the old behavior. See "configuring S3 object storage":{{site.baseurl}}/install/configure-s3-object-storage.html.
+
h2(#v2_0_0). v2.0.0 (2020-02-07)
"Upgrading from 1.4":#v1_4_1
diff --git a/doc/install/configure-s3-object-storage.html.textile.liquid b/doc/install/configure-s3-object-storage.html.textile.liquid
index e953f660f..b960ac1fd 100644
--- a/doc/install/configure-s3-object-storage.html.textile.liquid
+++ b/doc/install/configure-s3-object-storage.html.textile.liquid
@@ -59,6 +59,11 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
# declaration.
LocationConstraint: false
+ # Use V2 signatures instead of the default V4. Amazon S3
+ # supports V4 signatures in all regions, but this option
+ # might be needed for other S3-compatible services.
+ V2Signature: false
+
# Requested page size for "list bucket contents" requests.
IndexPageSize: 1000
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 12f4bd9de..ebe39e5b2 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -951,6 +951,7 @@ Clusters:
Region: us-east-1a
Bucket: aaaaa
LocationConstraint: false
+ V2Signature: false
IndexPageSize: 1000
ConnectTimeout: 1m
ReadTimeout: 10m
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 69de3f05e..880a91ee6 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -236,12 +236,14 @@ type Volume struct {
}
type S3VolumeDriverParameters struct {
+ IAMRole string
AccessKey string
SecretKey string
Endpoint string
Region string
Bucket string
LocationConstraint bool
+ V2Signature bool
IndexPageSize int
ConnectTimeout Duration
ReadTimeout Duration
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 80aa5ec3b..96f2e7db3 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -129,20 +129,9 @@ func s3regions() (okList []string) {
// S3Volume implements Volume using an S3 bucket.
type S3Volume struct {
- AccessKey string
- SecretKey string
- AuthToken string // populated automatically when IAMRole is used
- AuthExpiration time.Time // populated automatically when IAMRole is used
- IAMRole string
- Endpoint string
- Region string
- Bucket string
- LocationConstraint bool
- IndexPageSize int
- ConnectTimeout arvados.Duration
- ReadTimeout arvados.Duration
- RaceWindow arvados.Duration
- UnsafeDelete bool
+ arvados.S3VolumeDriverParameters
+ AuthToken string // populated automatically when IAMRole is used
+ AuthExpiration time.Time // populated automatically when IAMRole is used
cluster *arvados.Cluster
volume arvados.Volume
@@ -188,8 +177,7 @@ func (v *S3Volume) bootstrapIAMCredentials() error {
func (v *S3Volume) newS3Client() *s3.S3 {
auth := aws.NewAuth(v.AccessKey, v.SecretKey, v.AuthToken, v.AuthExpiration)
client := s3.New(*auth, v.region)
- if v.region.EC2Endpoint.Signer == aws.V4Signature {
- // Currently affects only eu-central-1
+ if !v.V2Signature {
client.Signature = aws.V4Signature
}
client.ConnectTimeout = time.Duration(v.ConnectTimeout)
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 2c5cdf5b9..5c642a942 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -122,13 +122,15 @@ func (s *StubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
w.WriteHeader(http.StatusNotFound)
}))
deadv := &S3Volume{
- IAMRole: s.metadata.URL + "/fake-metadata/test-role",
- Endpoint: "http://localhost:12345",
- Region: "test-region-1",
- Bucket: "test-bucket-name",
- cluster: s.cluster,
- logger: ctxlog.TestLogger(c),
- metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ IAMRole: s.metadata.URL + "/fake-metadata/test-role",
+ Endpoint: "http://localhost:12345",
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
+ },
+ cluster: s.cluster,
+ logger: ctxlog.TestLogger(c),
+ metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err := deadv.check()
c.Check(err, check.ErrorMatches, `.*/fake-metadata/test-role.*`)
@@ -468,19 +470,21 @@ func (s *StubbedS3Suite) newTestableVolume(c *check.C, cluster *arvados.Cluster,
v := &TestableS3Volume{
S3Volume: &S3Volume{
- AccessKey: accessKey,
- SecretKey: secretKey,
- IAMRole: iamRole,
- Bucket: TestBucketName,
- Endpoint: endpoint,
- Region: "test-region-1",
- LocationConstraint: true,
- UnsafeDelete: true,
- IndexPageSize: 1000,
- cluster: cluster,
- volume: volume,
- logger: ctxlog.TestLogger(c),
- metrics: metrics,
+ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+ IAMRole: iamRole,
+ AccessKey: accessKey,
+ SecretKey: secretKey,
+ Bucket: TestBucketName,
+ Endpoint: endpoint,
+ Region: "test-region-1",
+ LocationConstraint: true,
+ UnsafeDelete: true,
+ IndexPageSize: 1000,
+ },
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(c),
+ metrics: metrics,
},
c: c,
server: srv,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list