[arvados] created: 2.7.0-6568-g032096aeb2
git repository hosting
git at public.arvados.org
Tue May 14 14:57:41 UTC 2024
at 032096aeb25689e2e69b3eca4f1d5ff1b6221b84 (commit)
commit 032096aeb25689e2e69b3eca4f1d5ff1b6221b84
Author: Tom Clegg <tom at curii.com>
Date: Tue May 14 10:11:44 2024 -0400
21705: Update adal to unretracted version.
go: warning: github.com/Azure/go-autorest/autorest/adal at v0.9.17: retracted by module author: retracted due to token refresh errors
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/go.mod b/go.mod
index 29b4a3a50d..c69e9a0f6c 100644
--- a/go.mod
+++ b/go.mod
@@ -14,6 +14,7 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.27.12
github.com/aws/aws-sdk-go-v2/credentials v1.17.12
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16
+ github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
github.com/aws/smithy-go v1.20.2
github.com/bmatcuk/doublestar/v4 v4.6.1
@@ -60,7 +61,7 @@ require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
- github.com/Azure/go-autorest/autorest/adal v0.9.17 // indirect
+ github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
github.com/Azure/go-autorest/autorest/azure/cli v0.4.4 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect
@@ -75,7 +76,6 @@ require (
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
- github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
@@ -95,7 +95,7 @@ require (
github.com/gliderlabs/ssh v0.2.2 // indirect
github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
- github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
+ github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/s2a-go v0.1.4 // indirect
diff --git a/go.sum b/go.sum
index 46783b3324..c842f5426a 100644
--- a/go.sum
+++ b/go.sum
@@ -19,6 +19,8 @@ github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4Uw
github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
github.com/Azure/go-autorest/autorest/adal v0.9.17 h1:esOPl2dhcz9P3jqBSJ8tPGEj2EqzPPT6zfyuloiogKY=
github.com/Azure/go-autorest/autorest/adal v0.9.17/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
+github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
+github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.9 h1:Y2CgdzitFDsdMwYMzf9LIZWrrTFysqbRc7b94XVVJ78=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.9/go.mod h1:hg3/1yw0Bq87O3KvvnJoAh34/0zbP7SFizX/qN5JvjU=
github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
@@ -41,6 +43,8 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -173,6 +177,7 @@ github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7
github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -181,6 +186,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
@@ -259,6 +266,8 @@ github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0 h1:+XMEfFS2
github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d h1:9dIJ/sx3yapvuq3kvTSVQ6UVS2HxfOB4MCwWiH8JcvQ=
github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 h1:xXn0nBttYwok7DhU4RxqaADEpQn7fEMt5kKc3yoj/n0=
@@ -282,11 +291,14 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9 h1:ZivaaKmjs9q90zi6I4gTLW6tbVGtlBjellr3hMYaly0=
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9/go.mod h1:np1wUFZ6tyoke22qDJZY40URn9Ae51gX7ljIWXN5TJs=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
@@ -338,6 +350,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/xanzy/ssh-agent v0.1.0 h1:lOhdXLxtmYjaHc76ZtNmJWPg948y/RnT+3N3cvKWFzY=
github.com/xanzy/ssh-agent v0.1.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -354,6 +367,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
@@ -512,6 +526,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
commit ae6e366c1a680dbd998741fdc2693ec19f8a9a35
Author: Tom Clegg <tom at curii.com>
Date: Mon May 13 18:29:07 2024 -0400
21705: Migrate keep-web s3 tests to aws-sdk-go-v2.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3aws_test.go b/services/keep-web/s3aws_test.go
index 145d987bf6..f22b9c11b8 100644
--- a/services/keep-web/s3aws_test.go
+++ b/services/keep-web/s3aws_test.go
@@ -7,14 +7,13 @@ package keepweb
import (
"bytes"
"context"
+ "errors"
"io/ioutil"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"github.com/aws/aws-sdk-go-v2/aws"
- "github.com/aws/aws-sdk-go-v2/aws/defaults"
- "github.com/aws/aws-sdk-go-v2/aws/ec2metadata"
- "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
- "github.com/aws/aws-sdk-go-v2/aws/endpoints"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
check "gopkg.in/check.v1"
)
@@ -23,29 +22,40 @@ func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
- cfg := defaults.Config()
- cfg.Credentials = aws.NewChainProvider([]aws.CredentialsProvider{
- aws.NewStaticCredentialsProvider(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, ""),
- ec2rolecreds.New(ec2metadata.New(cfg)),
- })
- cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
- if service == "s3" {
- return aws.Endpoint{
- URL: s.testServer.URL,
- SigningRegion: "custom-signing-region",
- }, nil
- }
- return endpoints.NewDefaultResolver().ResolveEndpoint(service, region)
+ cfg, err := config.LoadDefaultConfig(context.TODO(),
+ func(o *config.LoadOptions) error {
+ o.Credentials = credentials.StaticCredentialsProvider{
+ Value: aws.Credentials{
+ AccessKeyID: arvadostest.ActiveTokenUUID,
+ SecretAccessKey: arvadostest.ActiveToken,
+ Source: "test suite configuration",
+ },
+ }
+ o.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service == "S3" {
+ return aws.Endpoint{
+ URL: s.testServer.URL,
+ HostnameImmutable: true,
+ SigningRegion: "test-region",
+ Source: aws.EndpointSourceCustom,
+ }, nil
+ }
+ // else, use default
+ return aws.Endpoint{}, &aws.EndpointNotFoundError{Err: errors.New("endpoint not overridden")}
+ })
+ return nil
+ })
+ c.Assert(err, check.IsNil)
+ client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+ o.Region = "test-region"
+ o.UsePathStyle = true
})
- client := s3.New(cfg)
- client.ForcePathStyle = true
- listreq := client.ListObjectsV2Request(&s3.ListObjectsV2Input{
+ resp, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
Bucket: aws.String(arvadostest.FooCollection),
- MaxKeys: aws.Int64(100),
+ MaxKeys: aws.Int32(100),
Prefix: aws.String(""),
ContinuationToken: nil,
})
- resp, err := listreq.Send(context.Background())
c.Assert(err, check.IsNil)
c.Check(resp.Contents, check.HasLen, 1)
for _, key := range resp.Contents {
@@ -56,20 +66,18 @@ func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) {
for i := range p {
p[i] = byte('a')
}
- putreq := client.PutObjectRequest(&s3.PutObjectInput{
+ _, err = client.PutObject(context.Background(), &s3.PutObjectInput{
Body: bytes.NewReader(p),
Bucket: aws.String(stage.collbucket.Name),
ContentType: aws.String("application/octet-stream"),
Key: aws.String("aaaa"),
})
- _, err = putreq.Send(context.Background())
c.Assert(err, check.IsNil)
- getreq := client.GetObjectRequest(&s3.GetObjectInput{
+ getresp, err := client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: aws.String(stage.collbucket.Name),
Key: aws.String("aaaa"),
})
- getresp, err := getreq.Send(context.Background())
c.Assert(err, check.IsNil)
getdata, err := ioutil.ReadAll(getresp.Body)
c.Assert(err, check.IsNil)
commit a2ce965f61f0650ca7eef4c41db5b0b3016f8861
Author: Tom Clegg <tom at curii.com>
Date: Mon May 13 17:27:20 2024 -0400
21705: Migrate keepstore s3 driver to latest aws-sdk-go-v2.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/go.mod b/go.mod
index bcbcb55abd..29b4a3a50d 100644
--- a/go.mod
+++ b/go.mod
@@ -9,8 +9,13 @@ require (
github.com/Azure/go-autorest/autorest/azure/auth v0.5.9
github.com/Azure/go-autorest/autorest/to v0.4.0
github.com/arvados/cgofuse v1.2.0-arvados1
- github.com/aws/aws-sdk-go v1.44.174
+ github.com/aws/aws-sdk-go v1.44.256
github.com/aws/aws-sdk-go-v2 v1.26.1
+ github.com/aws/aws-sdk-go-v2/config v1.27.12
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.12
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
+ github.com/aws/smithy-go v1.20.2
github.com/bmatcuk/doublestar/v4 v4.6.1
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
github.com/coreos/go-oidc/v3 v3.5.0
@@ -30,7 +35,7 @@ require (
github.com/imdario/mergo v0.3.12
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
github.com/jmoiron/sqlx v1.2.0
- github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc
+ github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d
github.com/julienschmidt/httprouter v1.3.0
github.com/lib/pq v1.10.2
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9
@@ -64,19 +69,20 @@ require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
- github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
- github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
- github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
- github.com/aws/smithy-go v1.20.2 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -114,13 +120,13 @@ require (
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 // indirect
github.com/sergi/go-diff v1.0.0 // indirect
- github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect
+ github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
- golang.org/x/tools v0.6.0 // indirect
+ golang.org/x/tools v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
diff --git a/go.sum b/go.sum
index 16e5a9cbab..46783b3324 100644
--- a/go.sum
+++ b/go.sum
@@ -48,39 +48,65 @@ github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQ
github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
+github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2 h1:q9L4OHOFwd3zNFlEADucYTBfKasWpz4NwjaGjJlYhqs=
+github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b h1:hK0t0aJTTXI64lpXln2A1SripqOym+GVNTnwsLes39Y=
github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.44.174 h1:9lR4a6MKQW/t6YCG0ZKAt1GAkjdEPP8sWch/pfcuR0c=
github.com/aws/aws-sdk-go v1.44.174/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4=
+github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.23.0 h1:+E1q1LLSfHSDn/DzOtdJOX+pLZE2HiNV2yO5AjZINwM=
github.com/aws/aws-sdk-go-v2 v0.23.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/config v1.27.12 h1:vq88mBaZI4NGLXk8ierArwSILmYHDJZGJOeAc/pzEVQ=
+github.com/aws/aws-sdk-go-v2/config v1.27.12/go.mod h1:IOrsf4IiN68+CgzyuyGUYTpCrtUQTbbMEAtR/MR/4ZU=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.12 h1:PVbKQ0KjDosI5+nEdRMU8ygEQDmkJTSHBqPjEX30lqc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.12/go.mod h1:jlWtGFRtKsqc5zqerHZYmKmRkUXo3KPM14YJ13ZEjwE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16 h1:n4k5rgvy0M748NadpDlGLOQ/KCVbNHCQsfI895wLrt0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16/go.mod h1:Rd5rkCdq44ZC5rS4CbF3Wc8FiWo7f9brp7qeIwWsJaU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 h1:o5cTaeunSpfXiLTIBx5xo2enQmiChtu1IBbzXnfU9Hs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.6/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 h1:Ciiz/plN+Z+pPO1G0W2zJoYIIl0KtKzY0LJ78NXYTws=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 h1:et3Ta53gotFR4ERLXXHIHl/Uuk1qYpP5uU7cvNql8ns=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.7/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -229,6 +255,10 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc h1:JJPhSHowepOF2+ElJVyb9jgt5ZyBkPMkPuhS0uODSFs=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
+github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0 h1:+XMEfFS2W506TvrEY5ViKNEIqmX6mxLmBmx98/dBNAg=
+github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
+github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d h1:9dIJ/sx3yapvuq3kvTSVQ6UVS2HxfOB4MCwWiH8JcvQ=
+github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 h1:xXn0nBttYwok7DhU4RxqaADEpQn7fEMt5kKc3yoj/n0=
@@ -286,6 +316,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0=
github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -311,6 +343,7 @@ github.com/xanzy/ssh-agent v0.1.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnW
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
@@ -332,6 +365,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -353,6 +387,7 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
@@ -376,6 +411,7 @@ golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -388,6 +424,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
@@ -399,6 +436,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
@@ -423,12 +461,15 @@ golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8i
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
+golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 2e2e97a974..49b94a72a1 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -13,6 +13,7 @@ import (
"errors"
"fmt"
"io"
+ "net/url"
"os"
"regexp"
"strings"
@@ -22,13 +23,13 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/aws/aws-sdk-go-v2/aws"
- "github.com/aws/aws-sdk-go-v2/aws/awserr"
- "github.com/aws/aws-sdk-go-v2/aws/defaults"
- "github.com/aws/aws-sdk-go-v2/aws/ec2metadata"
- "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
- "github.com/aws/aws-sdk-go-v2/aws/endpoints"
+ v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/aws/smithy-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -49,9 +50,14 @@ const (
)
var (
- errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false")
- s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
- s3AWSZeroTime time.Time
+ errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false")
+ s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
+ s3AWSZeroTime time.Time
+ defaultEndpointResolverV2 = s3.NewDefaultEndpointResolverV2()
+
+ // Returned by an aws.EndpointResolverWithOptions to indicate
+ // that the default resolver should be used.
+ errEndpointNotOverridden = &aws.EndpointNotFoundError{Err: errors.New("endpoint not overridden")}
)
// s3Volume implements Volume using an S3 bucket.
@@ -68,6 +74,9 @@ type s3Volume struct {
bucket *s3Bucket
region string
startOnce sync.Once
+
+ overrideEndpoint *aws.Endpoint
+ usePathStyle bool // used by test suite
}
// s3bucket wraps s3.bucket and counts I/O and API usage stats. The
@@ -114,12 +123,15 @@ func news3Volume(params newVolumeParams) (volume, error) {
}
func (v *s3Volume) translateError(err error) error {
- if _, ok := err.(*aws.RequestCanceledError); ok {
+ if cerr := (interface{ CanceledError() bool })(nil); errors.As(err, &cerr) && cerr.CanceledError() {
+ // *aws.RequestCanceledError and *smithy.CanceledError
+ // implement this interface.
return context.Canceled
- } else if aerr, ok := err.(awserr.Error); ok {
- if aerr.Code() == "NotFound" {
- return os.ErrNotExist
- } else if aerr.Code() == "NoSuchKey" {
+ }
+ var aerr smithy.APIError
+ if errors.As(err, &aerr) {
+ switch aerr.ErrorCode() {
+ case "NotFound", "NoSuchKey":
return os.ErrNotExist
}
}
@@ -140,20 +152,17 @@ func (v *s3Volume) safeCopy(dst, src string) error {
Key: aws.String(dst),
}
- req := v.bucket.svc.CopyObjectRequest(input)
- resp, err := req.Send(context.Background())
+ resp, err := v.bucket.svc.CopyObject(context.Background(), input)
err = v.translateError(err)
if os.IsNotExist(err) {
return err
} else if err != nil {
return fmt.Errorf("PutCopy(%q ← %q): %s", dst, v.bucket.bucket+"/"+src, err)
- }
-
- if resp.CopyObjectResult.LastModified == nil {
- return fmt.Errorf("PutCopy succeeded but did not return a timestamp: %q: %s", resp.CopyObjectResult.LastModified, err)
- } else if time.Now().Sub(*resp.CopyObjectResult.LastModified) > maxClockSkew {
- return fmt.Errorf("PutCopy succeeded but returned an old timestamp: %q: %s", resp.CopyObjectResult.LastModified, resp.CopyObjectResult.LastModified)
+ } else if resp.CopyObjectResult.LastModified == nil {
+ return fmt.Errorf("PutCopy(%q ← %q): succeeded but did not return a timestamp", dst, v.bucket.bucket+"/"+src)
+ } else if skew := time.Now().UTC().Sub(*resp.CopyObjectResult.LastModified); skew > maxClockSkew {
+ return fmt.Errorf("PutCopy succeeded but returned old timestamp %s (skew %v > max %v, now %s)", resp.CopyObjectResult.LastModified, skew, maxClockSkew, time.Now())
}
return nil
}
@@ -173,28 +182,18 @@ func (v *s3Volume) check(ec2metadataHostname string) error {
return errors.New("DriverParameters: V2Signature is not supported")
}
- defaultResolver := endpoints.NewDefaultResolver()
-
- cfg := defaults.Config()
-
if v.Endpoint == "" && v.Region == "" {
return fmt.Errorf("AWS region or endpoint must be specified")
- } else if v.Endpoint != "" || ec2metadataHostname != "" {
- myCustomResolver := func(service, region string) (aws.Endpoint, error) {
- if v.Endpoint != "" && service == "s3" {
- return aws.Endpoint{
- URL: v.Endpoint,
- SigningRegion: region,
- }, nil
- } else if service == "ec2metadata" && ec2metadataHostname != "" {
- return aws.Endpoint{
- URL: ec2metadataHostname,
- }, nil
- } else {
- return defaultResolver.ResolveEndpoint(service, region)
- }
+ } else if v.Endpoint != "" {
+ _, err := url.Parse(v.Endpoint)
+ if err != nil {
+ return fmt.Errorf("error parsing custom S3 endpoint %q: %w", v.Endpoint, err)
+ }
+ v.overrideEndpoint = &aws.Endpoint{
+ URL: v.Endpoint,
+ HostnameImmutable: true,
+ Source: aws.EndpointSourceCustom,
}
- cfg.EndpointResolver = aws.EndpointResolverFunc(myCustomResolver)
}
if v.Region == "" {
// Endpoint is already specified (otherwise we would
@@ -203,7 +202,6 @@ func (v *s3Volume) check(ec2metadataHostname string) error {
// SignatureVersions.
v.Region = "us-east-1"
}
- cfg.Region = v.Region
// Zero timeouts mean "wait forever", which is a bad
// default. Default to long timeouts instead.
@@ -214,33 +212,65 @@ func (v *s3Volume) check(ec2metadataHostname string) error {
v.ReadTimeout = s3DefaultReadTimeout
}
- creds := aws.NewChainProvider(
- []aws.CredentialsProvider{
- aws.NewStaticCredentialsProvider(v.AccessKeyID, v.SecretAccessKey, v.AuthToken),
- ec2rolecreds.New(ec2metadata.New(cfg), func(opts *ec2rolecreds.ProviderOptions) {
- // (from aws-sdk-go-v2 comments)
- // "allow the credentials to trigger
- // refreshing prior to the credentials
- // actually expiring. This is
- // beneficial so race conditions with
- // expiring credentials do not cause
- // request to fail unexpectedly due to
- // ExpiredTokenException exceptions."
- //
- // (from
- // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
- // "We make new credentials available
- // at least five minutes before the
- // expiration of the old credentials."
- opts.ExpiryWindow = 5 * time.Minute
- }),
- })
-
- cfg.Credentials = creds
+ cfg, err := config.LoadDefaultConfig(context.TODO(),
+ config.WithRegion(v.Region),
+ config.WithCredentialsCacheOptions(func(o *aws.CredentialsCacheOptions) {
+ // (from aws-sdk-go-v2 comments) "allow the
+ // credentials to trigger refreshing prior to
+ // the credentials actually expiring. This is
+ // beneficial so race conditions with expiring
+ // credentials do not cause request to fail
+ // unexpectedly due to ExpiredTokenException
+ // exceptions."
+ //
+ // (from
+ // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
+ // "We make new credentials available at least
+ // five minutes before the expiration of the
+ // old credentials."
+ o.ExpiryWindow = 5 * time.Minute
+ }),
+ func(o *config.LoadOptions) error {
+ if v.AccessKeyID == "" && v.SecretAccessKey == "" {
+ // Use default sdk behavior (IAM / IMDS)
+ return nil
+ }
+ v.logger.Debug("using static credentials")
+ o.Credentials = credentials.StaticCredentialsProvider{
+ Value: aws.Credentials{
+ AccessKeyID: v.AccessKeyID,
+ SecretAccessKey: v.SecretAccessKey,
+ Source: "Arvados configuration",
+ },
+ }
+ return nil
+ },
+ func(o *config.LoadOptions) error {
+ if ec2metadataHostname != "" {
+ o.EC2IMDSEndpoint = ec2metadataHostname
+ }
+ if v.overrideEndpoint != nil {
+ o.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service == "S3" {
+ return *v.overrideEndpoint, nil
+ }
+ return aws.Endpoint{}, errEndpointNotOverridden // use default resolver
+ })
+ }
+ return nil
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("error loading aws client config: %w", err)
+ }
v.bucket = &s3Bucket{
bucket: v.Bucket,
- svc: s3.New(cfg),
+ svc: s3.NewFromConfig(cfg, func(o *s3.Options) {
+ if v.usePathStyle {
+ o.UsePathStyle = true
+ }
+ }),
}
// Set up prometheus metrics
@@ -263,7 +293,7 @@ func (v *s3Volume) EmptyTrash() {
// Define "ready to delete" as "...when EmptyTrash started".
startT := time.Now()
- emptyOneKey := func(trash *s3.Object) {
+ emptyOneKey := func(trash *types.Object) {
key := strings.TrimPrefix(*trash.Key, "trash/")
loc, isblk := v.isKeepBlock(key)
if !isblk {
@@ -337,7 +367,7 @@ func (v *s3Volume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan *s3.Object, v.cluster.Collections.BlobDeleteConcurrency)
+ todo := make(chan *types.Object, v.cluster.Collections.BlobDeleteConcurrency)
for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
@@ -411,8 +441,7 @@ func (v *s3Volume) head(key string) (result *s3.HeadObjectOutput, err error) {
Key: aws.String(key),
}
- req := v.bucket.svc.HeadObjectRequest(input)
- res, err := req.Send(context.TODO())
+ res, err := v.bucket.svc.HeadObject(context.Background(), input)
v.bucket.stats.TickOps("head")
v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.HeadOps)
@@ -421,8 +450,7 @@ func (v *s3Volume) head(key string) (result *s3.HeadObjectOutput, err error) {
if err != nil {
return nil, v.translateError(err)
}
- result = res.HeadObjectOutput
- return
+ return res, nil
}
// BlockRead reads a Keep block that has been stored as a block blob
@@ -459,11 +487,11 @@ func (v *s3Volume) BlockRead(ctx context.Context, hash string, w io.WriterAt) er
}
func (v *s3Volume) readWorker(ctx context.Context, key string, dst io.WriterAt) error {
- downloader := s3manager.NewDownloaderWithClient(v.bucket.svc, func(u *s3manager.Downloader) {
+ downloader := manager.NewDownloader(v.bucket.svc, func(u *manager.Downloader) {
u.PartSize = s3downloaderPartSize
u.Concurrency = s3downloaderReadConcurrency
})
- count, err := downloader.DownloadWithContext(ctx, dst, &s3.GetObjectInput{
+ count, err := downloader.Download(ctx, dst, &s3.GetObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
})
@@ -481,7 +509,7 @@ func (v *s3Volume) writeObject(ctx context.Context, key string, r io.Reader) err
r = bytes.NewReader(nil)
}
- uploadInput := s3manager.UploadInput{
+ uploadInput := s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: r,
@@ -501,20 +529,15 @@ func (v *s3Volume) writeObject(ctx context.Context, key string, r io.Reader) err
// throughput, better than higher concurrency (10 or 13) by ~5%.
// Defining u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(64 * 1024 * 1024)
// is detrimental to throughput (minus ~15%).
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) {
+ uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) {
u.PartSize = s3uploaderPartSize
u.Concurrency = s3uploaderWriteConcurrency
})
- // Unlike the goamz S3 driver, we don't need to precompute ContentSHA256:
- // the aws-sdk-go v2 SDK uses a ReadSeeker to avoid having to copy the
- // block, so there is no extra memory use to be concerned about. See
- // makeSha256Reader in aws/signer/v4/v4.go. In fact, we explicitly disable
- // calculating the Sha-256 because we don't need it; we already use md5sum
- // hashes that match the name of the block.
- _, err := uploader.UploadWithContext(ctx, &uploadInput, s3manager.WithUploaderRequestOptions(func(r *aws.Request) {
- r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
- }))
+ _, err := uploader.Upload(ctx, &uploadInput,
+ // Avoid precomputing SHA256 before sending.
+ manager.WithUploaderRequestOptions(s3.WithAPIOptions(v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)),
+ )
v.bucket.stats.TickOps("put")
v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.PutOps)
@@ -544,13 +567,13 @@ type s3awsLister struct {
PageSize int
Stats *s3awsbucketStats
ContinuationToken string
- buf []s3.Object
+ buf []types.Object
err error
}
// First fetches the first page and returns the first item. It returns
// nil if the response is the empty set or an error occurs.
-func (lister *s3awsLister) First() *s3.Object {
+func (lister *s3awsLister) First() *types.Object {
lister.getPage()
return lister.pop()
}
@@ -558,7 +581,7 @@ func (lister *s3awsLister) First() *s3.Object {
// Next returns the next item, fetching the next page if necessary. It
// returns nil if the last available item has already been fetched, or
// an error occurs.
-func (lister *s3awsLister) Next() *s3.Object {
+func (lister *s3awsLister) Next() *types.Object {
if len(lister.buf) == 0 && lister.ContinuationToken != "" {
lister.getPage()
}
@@ -578,22 +601,22 @@ func (lister *s3awsLister) getPage() {
if lister.ContinuationToken == "" {
input = &s3.ListObjectsV2Input{
Bucket: aws.String(lister.Bucket.bucket),
- MaxKeys: aws.Int64(int64(lister.PageSize)),
+ MaxKeys: aws.Int32(int32(lister.PageSize)),
Prefix: aws.String(lister.Prefix),
}
} else {
input = &s3.ListObjectsV2Input{
Bucket: aws.String(lister.Bucket.bucket),
- MaxKeys: aws.Int64(int64(lister.PageSize)),
+ MaxKeys: aws.Int32(int32(lister.PageSize)),
Prefix: aws.String(lister.Prefix),
ContinuationToken: &lister.ContinuationToken,
}
}
- req := lister.Bucket.svc.ListObjectsV2Request(input)
- resp, err := req.Send(context.Background())
+ resp, err := lister.Bucket.svc.ListObjectsV2(context.Background(), input)
if err != nil {
- if aerr, ok := err.(awserr.Error); ok {
+ var aerr smithy.APIError
+ if errors.As(err, &aerr) {
lister.err = aerr
} else {
lister.err = err
@@ -606,7 +629,7 @@ func (lister *s3awsLister) getPage() {
} else {
lister.ContinuationToken = ""
}
- lister.buf = make([]s3.Object, 0, len(resp.Contents))
+ lister.buf = make([]types.Object, 0, len(resp.Contents))
for _, key := range resp.Contents {
if !strings.HasPrefix(*key.Key, lister.Prefix) {
lister.Logger.Warnf("s3awsLister: S3 Bucket.List(prefix=%q) returned key %q", lister.Prefix, *key.Key)
@@ -616,7 +639,7 @@ func (lister *s3awsLister) getPage() {
}
}
-func (lister *s3awsLister) pop() (k *s3.Object) {
+func (lister *s3awsLister) pop() (k *types.Object) {
if len(lister.buf) > 0 {
k = &lister.buf[0]
lister.buf = lister.buf[1:]
@@ -774,8 +797,7 @@ func (b *s3Bucket) Del(path string) error {
Bucket: aws.String(b.bucket),
Key: aws.String(path),
}
- req := b.svc.DeleteObjectRequest(input)
- _, err := req.Send(context.Background())
+ _, err := b.svc.DeleteObject(context.Background(), input)
b.stats.TickOps("delete")
b.stats.Tick(&b.stats.Ops, &b.stats.DelOps)
b.stats.TickErr(err)
@@ -833,12 +855,11 @@ func (s *s3awsbucketStats) TickErr(err error) {
return
}
errType := fmt.Sprintf("%T", err)
- if aerr, ok := err.(awserr.Error); ok {
- if reqErr, ok := err.(awserr.RequestFailure); ok {
- // A service error occurred
- errType = errType + fmt.Sprintf(" %d %s", reqErr.StatusCode(), aerr.Code())
+ if aerr := smithy.APIError(nil); errors.As(err, &aerr) {
+ if rerr := interface{ HTTPStatusCode() int }(nil); errors.As(err, &rerr) {
+ errType = errType + fmt.Sprintf(" %d %s", rerr.HTTPStatusCode(), aerr.ErrorCode())
} else {
- errType = errType + fmt.Sprintf(" 000 %s", aerr.Code())
+ errType = errType + fmt.Sprintf(" 000 %s", aerr.ErrorCode())
}
}
s.statsTicker.TickErr(err, errType)
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index fb68e1c057..50010b3bef 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -15,14 +15,15 @@ import (
"net/http/httptest"
"os"
"strings"
+ "sync/atomic"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
@@ -31,22 +32,18 @@ import (
check "gopkg.in/check.v1"
)
-const (
- s3TestBucketName = "testbucket"
-)
-
-type s3AWSFakeClock struct {
+type s3fakeClock struct {
now *time.Time
}
-func (c *s3AWSFakeClock) Now() time.Time {
+func (c *s3fakeClock) Now() time.Time {
if c.now == nil {
return time.Now().UTC()
}
return c.now.UTC()
}
-func (c *s3AWSFakeClock) Since(t time.Time) time.Duration {
+func (c *s3fakeClock) Since(t time.Time) time.Duration {
return c.Now().Sub(t)
}
@@ -55,14 +52,16 @@ var _ = check.Suite(&stubbedS3Suite{})
var srv httptest.Server
type stubbedS3Suite struct {
- s3server *httptest.Server
- metadata *httptest.Server
- cluster *arvados.Cluster
- volumes []*testableS3Volume
+ s3server *httptest.Server
+ s3fakeClock *s3fakeClock
+ metadata *httptest.Server
+ cluster *arvados.Cluster
+ volumes []*testableS3Volume
}
func (s *stubbedS3Suite) SetUpTest(c *check.C) {
s.s3server = nil
+ s.s3fakeClock = &s3fakeClock{}
s.metadata = nil
s.cluster = testCluster(c)
s.cluster.Volumes = map[string]arvados.Volume{
@@ -71,6 +70,12 @@ func (s *stubbedS3Suite) SetUpTest(c *check.C) {
}
}
+func (s *stubbedS3Suite) TearDownTest(c *check.C) {
+ if s.s3server != nil {
+ s.s3server.Close()
+ }
+}
+
func (s *stubbedS3Suite) TestGeneric(c *check.C) {
DoGenericVolumeTests(c, false, func(t TB, params newVolumeParams) TestableVolume {
// Use a negative raceWindow so s3test's 1-second
@@ -145,9 +150,9 @@ func (s *stubbedS3Suite) TestSignature(c *check.C) {
logger: ctxlog.TestLogger(c),
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
- err := vol.check("")
// Our test S3 server uses the older 'Path Style'
- vol.bucket.svc.ForcePathStyle = true
+ vol.usePathStyle = true
+ err := vol.check("")
c.Check(err, check.IsNil)
err = vol.BlockWrite(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
@@ -156,20 +161,37 @@ func (s *stubbedS3Suite) TestSignature(c *check.C) {
}
func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
+ var reqHeader http.Header
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reqHeader = r.Header
+ }))
+ defer stub.Close()
+
+ retrievedMetadata := false
s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ retrievedMetadata = true
upd := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
exp := time.Now().UTC().Add(time.Hour).Format(time.RFC3339)
- // Literal example from
- // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
- // but with updated timestamps
- io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`)
+ c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path)
+ switch {
+ case r.URL.Path == "/latest/meta-data/iam/security-credentials/":
+ io.WriteString(w, "testcredential\n")
+ case r.URL.Path == "/latest/api/token",
+ r.URL.Path == "/latest/meta-data/iam/security-credentials/testcredential":
+ // Literal example from
+ // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
+ // but with updated timestamps
+ io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
}))
defer s.metadata.Close()
v := &s3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
IAMRole: s.metadata.URL + "/latest/api/token",
- Endpoint: "http://localhost:12345",
+ Endpoint: stub.URL,
Region: "test-region-1",
Bucket: "test-bucket-name",
},
@@ -179,18 +201,21 @@ func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
}
err := v.check(s.metadata.URL + "/latest")
c.Check(err, check.IsNil)
- creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
+ resp, err := v.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{})
c.Check(err, check.IsNil)
- c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
- c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+ c.Check(resp.Buckets, check.HasLen, 0)
+ c.Check(retrievedMetadata, check.Equals, true)
+ c.Check(reqHeader.Get("Authorization"), check.Matches, `AWS4-HMAC-SHA256 Credential=ASIAIOSFODNN7EXAMPLE/\d+/test-region-1/s3/aws4_request, SignedHeaders=.*`)
+ retrievedMetadata = false
s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ retrievedMetadata = true
+ c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}))
deadv := &s3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
- IAMRole: s.metadata.URL + "/fake-metadata/test-role",
- Endpoint: "http://localhost:12345",
+ Endpoint: "http://localhost:9",
Region: "test-region-1",
Bucket: "test-bucket-name",
},
@@ -200,9 +225,10 @@ func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
}
err = deadv.check(s.metadata.URL + "/latest")
c.Check(err, check.IsNil)
- _, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
- c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`)
+ _, err = deadv.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{})
+ c.Check(err, check.ErrorMatches, `(?s).*failed to refresh cached credentials, no EC2 IMDS role found.*`)
c.Check(err, check.ErrorMatches, `(?s).*404.*`)
+ c.Check(retrievedMetadata, check.Equals, true)
}
func (s *stubbedS3Suite) TestStats(c *check.C) {
@@ -224,7 +250,7 @@ func (s *stubbedS3Suite) TestStats(c *check.C) {
err := v.BlockRead(context.Background(), loc, brdiscard)
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
- c.Check(stats(), check.Matches, `.*"s3.requestFailure 404 NoSuchKey[^"]*":[^0].*`)
+ c.Check(stats(), check.Matches, `.*"\*smithy.OperationError 404 NoSuchKey":[^0].*`)
c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
err = v.BlockWrite(context.Background(), loc, []byte("foo"))
@@ -334,9 +360,9 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) {
if t == none {
return
}
- v.serverClock.now = &t
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc)
- _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{
+ s.s3fakeClock.now = &t
+ uploader := manager.NewUploader(v.bucket.svc)
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: bytes.NewReader(data),
@@ -344,7 +370,7 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) {
if err != nil {
panic(err)
}
- v.serverClock.now = nil
+ s.s3fakeClock.now = nil
_, err = v.head(key)
if err != nil {
panic(err)
@@ -473,14 +499,14 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) {
putS3Obj(scenario.dataT, key, blk)
putS3Obj(scenario.recentT, "recent/"+key, nil)
putS3Obj(scenario.trashT, "trash/"+key, blk)
- v.serverClock.now = &t0
+ v.s3fakeClock.now = &t0
return loc, blk
}
// Check canGet
loc, blk := setupScenario()
err := v.BlockRead(context.Background(), loc, brdiscard)
- c.Check(err == nil, check.Equals, scenario.canGet)
+ c.Check(err == nil, check.Equals, scenario.canGet, check.Commentf("err was %+v", err))
if err != nil {
c.Check(os.IsNotExist(err), check.Equals, true)
}
@@ -538,47 +564,49 @@ type testableS3Volume struct {
*s3Volume
server *httptest.Server
c *check.C
- serverClock *s3AWSFakeClock
+ s3fakeClock *s3fakeClock
}
-type LogrusLog struct {
- log *logrus.FieldLogger
+type gofakes3logger struct {
+ logrus.FieldLogger
}
-func (l LogrusLog) Print(level gofakes3.LogLevel, v ...interface{}) {
+func (l gofakes3logger) Print(level gofakes3.LogLevel, v ...interface{}) {
switch level {
case gofakes3.LogErr:
- (*l.log).Errorln(v...)
+ l.Errorln(v...)
case gofakes3.LogWarn:
- (*l.log).Warnln(v...)
+ l.Warnln(v...)
case gofakes3.LogInfo:
- (*l.log).Infoln(v...)
+ l.Infoln(v...)
default:
panic("unknown level")
}
}
-func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume {
-
- clock := &s3AWSFakeClock{}
- // fake s3
- backend := s3mem.New(s3mem.WithTimeSource(clock))
+var testBucketSerial atomic.Int64
- // To enable GoFakeS3 debug logging, pass logger to gofakes3.WithLogger()
- /* logger := new(LogrusLog)
- ctxLogger := ctxlog.FromContext(context.Background())
- logger.log = &ctxLogger */
- faker := gofakes3.New(backend, gofakes3.WithTimeSource(clock), gofakes3.WithLogger(nil), gofakes3.WithTimeSkewLimit(0))
- srv := httptest.NewServer(faker.Server())
-
- endpoint := srv.URL
- if s.s3server != nil {
- endpoint = s.s3server.URL
+func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume {
+ if params.Logger == nil {
+ params.Logger = ctxlog.TestLogger(c)
}
+ if s.s3server == nil {
+ backend := s3mem.New(s3mem.WithTimeSource(s.s3fakeClock))
+ logger := ctxlog.TestLogger(c)
+ faker := gofakes3.New(backend,
+ gofakes3.WithTimeSource(s.s3fakeClock),
+ gofakes3.WithLogger(gofakes3logger{FieldLogger: logger}),
+ gofakes3.WithTimeSkewLimit(0))
+ s.s3server = httptest.NewServer(faker.Server())
+ }
+ endpoint := s.s3server.URL
+ bucketName := fmt.Sprintf("testbucket%d", testBucketSerial.Add(1))
- iamRole, accessKey, secretKey := "", "xxx", "xxx"
+ var metadataURL, iamRole, accessKey, secretKey string
if s.metadata != nil {
- iamRole, accessKey, secretKey = s.metadata.URL+"/fake-metadata/test-role", "", ""
+ metadataURL, iamRole = s.metadata.URL, s.metadata.URL+"/fake-metadata/test-role"
+ } else {
+ accessKey, secretKey = "xxx", "xxx"
}
v := &testableS3Volume{
@@ -587,32 +615,29 @@ func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, r
IAMRole: iamRole,
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
- Bucket: s3TestBucketName,
+ Bucket: bucketName,
Endpoint: endpoint,
Region: "test-region-1",
LocationConstraint: true,
UnsafeDelete: true,
IndexPageSize: 1000,
},
- cluster: params.Cluster,
- volume: params.ConfigVolume,
- logger: params.Logger,
- metrics: params.MetricsVecs,
- bufferPool: params.BufferPool,
+ cluster: params.Cluster,
+ volume: params.ConfigVolume,
+ logger: params.Logger,
+ metrics: params.MetricsVecs,
+ bufferPool: params.BufferPool,
+ usePathStyle: true,
},
c: c,
- server: srv,
- serverClock: clock,
+ s3fakeClock: s.s3fakeClock,
}
- c.Assert(v.s3Volume.check(""), check.IsNil)
- // Our test S3 server uses the older 'Path Style'
- v.s3Volume.bucket.svc.ForcePathStyle = true
+ c.Assert(v.s3Volume.check(metadataURL), check.IsNil)
// Create the testbucket
input := &s3.CreateBucketInput{
- Bucket: aws.String(s3TestBucketName),
+ Bucket: aws.String(bucketName),
}
- req := v.s3Volume.bucket.svc.CreateBucketRequest(input)
- _, err := req.Send(context.Background())
+ _, err := v.s3Volume.bucket.svc.CreateBucket(context.Background(), input)
c.Assert(err, check.IsNil)
// We couldn't set RaceWindow until now because check()
// rejects negative values.
@@ -624,12 +649,12 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e
key := v.key(loc)
r := newCountingReader(bytes.NewReader(block), v.bucket.stats.TickOutBytes)
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) {
+ uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) {
u.PartSize = 5 * 1024 * 1024
u.Concurrency = 13
})
- _, err := uploader.Upload(&s3manager.UploadInput{
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: r,
@@ -639,7 +664,7 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e
}
empty := bytes.NewReader([]byte{})
- _, err = uploader.Upload(&s3manager.UploadInput{
+ _, err = uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String("recent/" + key),
Body: empty,
@@ -651,11 +676,11 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e
// there are no other operations happening on the same s3test server
// while we do this.
func (v *testableS3Volume) TouchWithDate(loc string, lastPut time.Time) {
- v.serverClock.now = &lastPut
+ v.s3fakeClock.now = &lastPut
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc)
+ uploader := manager.NewUploader(v.bucket.svc)
empty := bytes.NewReader([]byte{})
- _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String("recent/" + v.key(loc)),
Body: empty,
@@ -664,11 +689,10 @@ func (v *testableS3Volume) TouchWithDate(loc string, lastPut time.Time) {
panic(err)
}
- v.serverClock.now = nil
+ v.s3fakeClock.now = nil
}
func (v *testableS3Volume) Teardown() {
- v.server.Close()
}
func (v *testableS3Volume) ReadWriteOperationLabelValues() (r, w string) {
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index 16084058b7..08911f3848 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -271,7 +271,7 @@ func (s *genericVolumeSuite) testPutAndTouch(t TB, factory TestableVolumeFactory
// Make sure v.Mtime() agrees the above Utime really worked.
if t0, err := v.Mtime(TestHash); err != nil || t0.IsZero() || !t0.Before(threshold) {
- t.Errorf("Setting mtime failed: %v, %v", t0, err)
+ t.Errorf("Setting mtime failed: threshold %v, t0 %v, err %v", threshold.UTC(), t0.UTC(), err)
}
// Write the same block again.
commit 59e3049bb85b4f3044a7f8a229c74cef1fd0854c
Author: Tom Clegg <tom at curii.com>
Date: Fri May 10 15:25:36 2024 -0400
21705: Upgrade go-retryablehttp and remove workaround.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/go.mod b/go.mod
index f224b8ec06..bcbcb55abd 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/gorilla/mux v1.8.0
- github.com/hashicorp/go-retryablehttp v0.7.2
+ github.com/hashicorp/go-retryablehttp v0.7.6
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87
github.com/imdario/mergo v0.3.12
@@ -41,7 +41,7 @@ require (
golang.org/x/crypto v0.22.0
golang.org/x/net v0.24.0
golang.org/x/oauth2 v0.11.0
- golang.org/x/sys v0.19.0
+ golang.org/x/sys v0.20.0
google.golang.org/api v0.126.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
gopkg.in/square/go-jose.v2 v2.5.1
diff --git a/go.sum b/go.sum
index 23dfa819ea..16e5a9cbab 100644
--- a/go.sum
+++ b/go.sum
@@ -205,8 +205,11 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.6 h1:TwRYfx2z2C4cLbXmT8I5PgP/xmuqASDyiVuGYfs9GZM=
+github.com/hashicorp/go-retryablehttp v0.7.6/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I=
@@ -389,6 +392,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 7bc3d5bc42..9fd294be2c 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -248,8 +248,6 @@ var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
var nopCancelFunc context.CancelFunc = func() {}
-var reqErrorRe = regexp.MustCompile(`net/http: invalid header `)
-
// Do augments (*http.Client)Do(): adds Authorization and X-Request-Id
// headers, delays in order to comply with rate-limiting restrictions,
// and retries failed requests when appropriate.
@@ -307,14 +305,6 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
if c.Timeout == 0 {
return false, nil
}
- // This check can be removed when
- // https://github.com/hashicorp/go-retryablehttp/pull/210
- // (or equivalent) is merged and we update go.mod.
- // Until then, it is needed to pass
- // TestNonRetryableStdlibError.
- if respErr != nil && reqErrorRe.MatchString(respErr.Error()) {
- return false, nil
- }
retrying, err := retryablehttp.DefaultRetryPolicy(ctx, resp, respErr)
if retrying {
lastResp, lastRespBody, lastErr = resp, nil, respErr
commit 95b54ae0abb0089258181f3e7b8aa9b9cc4b4021
Author: Tom Clegg <tom at curii.com>
Date: Thu May 2 16:45:28 2024 -0400
21705: Migrate lib/cloud/ec2 from aws-sdk-go to aws-sdk-go-v2.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/go.mod b/go.mod
index aaad05dab2..f224b8ec06 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
github.com/Azure/go-autorest/autorest/to v0.4.0
github.com/arvados/cgofuse v1.2.0-arvados1
github.com/aws/aws-sdk-go v1.44.174
- github.com/aws/aws-sdk-go-v2 v0.23.0
+ github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/bmatcuk/doublestar/v4 v4.6.1
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
github.com/coreos/go-oidc/v3 v3.5.0
@@ -64,6 +64,19 @@ require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
+ github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+ github.com/aws/smithy-go v1.20.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
diff --git a/go.sum b/go.sum
index d585789817..23dfa819ea 100644
--- a/go.sum
+++ b/go.sum
@@ -55,6 +55,34 @@ github.com/aws/aws-sdk-go v1.44.174 h1:9lR4a6MKQW/t6YCG0ZKAt1GAkjdEPP8sWch/pfcuR
github.com/aws/aws-sdk-go v1.44.174/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.23.0 h1:+E1q1LLSfHSDn/DzOtdJOX+pLZE2HiNV2yO5AjZINwM=
github.com/aws/aws-sdk-go-v2 v0.23.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
+github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 6251f18df0..5eb1afd3b5 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -5,14 +5,17 @@
package ec2
import (
+ "context"
"crypto/md5"
"crypto/rsa"
"crypto/sha1"
"crypto/x509"
"encoding/base64"
"encoding/json"
+ "errors"
"fmt"
"math/big"
+ "os"
"regexp"
"strconv"
"strings"
@@ -22,14 +25,12 @@ import (
"git.arvados.org/arvados.git/lib/cloud"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/awserr"
- "github.com/aws/aws-sdk-go/aws/credentials"
- "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
- "github.com/aws/aws-sdk-go/aws/ec2metadata"
- "github.com/aws/aws-sdk-go/aws/request"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/aws/retry"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/aws/smithy-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
@@ -50,7 +51,7 @@ type ec2InstanceSetConfig struct {
SecurityGroupIDs arvados.StringSet
SubnetID sliceOrSingleString
AdminUsername string
- EBSVolumeType string
+ EBSVolumeType types.VolumeType
EBSPrice float64
IAMInstanceProfile string
SpotPriceUpdateInterval arvados.Duration
@@ -90,14 +91,14 @@ func (ss *sliceOrSingleString) UnmarshalJSON(data []byte) error {
}
type ec2Interface interface {
- DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error)
- ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error)
- RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error)
- DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error)
- DescribeInstanceStatusPages(input *ec2.DescribeInstanceStatusInput, fn func(*ec2.DescribeInstanceStatusOutput, bool) bool) error
- DescribeSpotPriceHistoryPages(input *ec2.DescribeSpotPriceHistoryInput, fn func(*ec2.DescribeSpotPriceHistoryOutput, bool) bool) error
- CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error)
- TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error)
+ DescribeKeyPairs(context.Context, *ec2.DescribeKeyPairsInput, ...func(*ec2.Options)) (*ec2.DescribeKeyPairsOutput, error)
+ ImportKeyPair(context.Context, *ec2.ImportKeyPairInput, ...func(*ec2.Options)) (*ec2.ImportKeyPairOutput, error)
+ RunInstances(context.Context, *ec2.RunInstancesInput, ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error)
+ DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
+ DescribeInstanceStatus(context.Context, *ec2.DescribeInstanceStatusInput, ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error)
+ DescribeSpotPriceHistory(context.Context, *ec2.DescribeSpotPriceHistoryInput, ...func(*ec2.Options)) (*ec2.DescribeSpotPriceHistoryOutput, error)
+ CreateTags(context.Context, *ec2.CreateTagsInput, ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error)
+ TerminateInstances(context.Context, *ec2.TerminateInstancesInput, ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error)
}
type ec2InstanceSet struct {
@@ -129,19 +130,22 @@ func newEC2InstanceSet(config json.RawMessage, instanceSetID cloud.InstanceSetID
return nil, err
}
- sess, err := session.NewSession()
+ if len(instanceSet.ec2config.AccessKeyID)+len(instanceSet.ec2config.SecretAccessKey) > 0 {
+ // AWS SDK will use credentials in environment vars if
+ // present.
+ os.Setenv("AWS_ACCESS_KEY_ID", instanceSet.ec2config.AccessKeyID)
+ os.Setenv("AWS_SECRET_ACCESS_KEY", instanceSet.ec2config.SecretAccessKey)
+ } else {
+ os.Unsetenv("AWS_ACCESS_KEY_ID")
+ os.Unsetenv("AWS_SECRET_ACCESS_KEY")
+ }
+ awsConfig, err := awsconfig.LoadDefaultConfig(context.TODO(),
+ awsconfig.WithRegion(instanceSet.ec2config.Region))
if err != nil {
return nil, err
}
- // First try any static credentials, fall back to an IAM instance profile/role
- creds := credentials.NewChainCredentials(
- []credentials.Provider{
- &credentials.StaticProvider{Value: credentials.Value{AccessKeyID: instanceSet.ec2config.AccessKeyID, SecretAccessKey: instanceSet.ec2config.SecretAccessKey}},
- &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess)},
- })
- awsConfig := aws.NewConfig().WithCredentials(creds).WithRegion(instanceSet.ec2config.Region)
- instanceSet.client = ec2.New(session.Must(session.NewSession(awsConfig)))
+ instanceSet.client = ec2.NewFromConfig(awsConfig)
instanceSet.keys = make(map[string]string)
if instanceSet.ec2config.EBSVolumeType == "" {
instanceSet.ec2config.EBSVolumeType = "gp2"
@@ -219,9 +223,9 @@ func (instanceSet *ec2InstanceSet) Create(
initCommand cloud.InitCommand,
publicKey ssh.PublicKey) (cloud.Instance, error) {
- ec2tags := []*ec2.Tag{}
+ ec2tags := []types.Tag{}
for k, v := range newTags {
- ec2tags = append(ec2tags, &ec2.Tag{
+ ec2tags = append(ec2tags, types.Tag{
Key: aws.String(k),
Value: aws.String(v),
})
@@ -234,29 +238,28 @@ func (instanceSet *ec2InstanceSet) Create(
rii := ec2.RunInstancesInput{
ImageId: aws.String(string(imageID)),
- InstanceType: &instanceType.ProviderType,
- MaxCount: aws.Int64(1),
- MinCount: aws.Int64(1),
-
- NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
- {
- AssociatePublicIpAddress: aws.Bool(false),
- DeleteOnTermination: aws.Bool(true),
- DeviceIndex: aws.Int64(0),
- Groups: aws.StringSlice(groups),
- }},
+ InstanceType: types.InstanceType(instanceType.ProviderType),
+ MaxCount: aws.Int32(1),
+ MinCount: aws.Int32(1),
+
+ NetworkInterfaces: []types.InstanceNetworkInterfaceSpecification{{
+ AssociatePublicIpAddress: aws.Bool(false),
+ DeleteOnTermination: aws.Bool(true),
+ DeviceIndex: aws.Int32(0),
+ Groups: groups,
+ }},
DisableApiTermination: aws.Bool(false),
- InstanceInitiatedShutdownBehavior: aws.String("terminate"),
- TagSpecifications: []*ec2.TagSpecification{
+ InstanceInitiatedShutdownBehavior: types.ShutdownBehaviorTerminate,
+ TagSpecifications: []types.TagSpecification{
{
- ResourceType: aws.String("instance"),
+ ResourceType: types.ResourceTypeInstance,
Tags: ec2tags,
}},
- MetadataOptions: &ec2.InstanceMetadataOptionsRequest{
+ MetadataOptions: &types.InstanceMetadataOptionsRequest{
// Require IMDSv2, as described at
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-IMDS-new-instances.html
- HttpEndpoint: aws.String(ec2.InstanceMetadataEndpointStateEnabled),
- HttpTokens: aws.String(ec2.HttpTokensStateRequired),
+ HttpEndpoint: types.InstanceMetadataEndpointStateEnabled,
+ HttpTokens: types.HttpTokensStateRequired,
},
UserData: aws.String(base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n"))),
}
@@ -270,31 +273,31 @@ func (instanceSet *ec2InstanceSet) Create(
}
if instanceType.AddedScratch > 0 {
- rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{
+ rii.BlockDeviceMappings = []types.BlockDeviceMapping{{
DeviceName: aws.String("/dev/xvdt"),
- Ebs: &ec2.EbsBlockDevice{
+ Ebs: &types.EbsBlockDevice{
DeleteOnTermination: aws.Bool(true),
- VolumeSize: aws.Int64((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30),
- VolumeType: &instanceSet.ec2config.EBSVolumeType,
+ VolumeSize: aws.Int32(int32((int64(instanceType.AddedScratch) + (1<<30 - 1)) >> 30)),
+ VolumeType: instanceSet.ec2config.EBSVolumeType,
}}}
}
if instanceType.Preemptible {
- rii.InstanceMarketOptions = &ec2.InstanceMarketOptionsRequest{
- MarketType: aws.String("spot"),
- SpotOptions: &ec2.SpotMarketOptions{
- InstanceInterruptionBehavior: aws.String("terminate"),
+ rii.InstanceMarketOptions = &types.InstanceMarketOptionsRequest{
+ MarketType: types.MarketTypeSpot,
+ SpotOptions: &types.SpotMarketOptions{
+ InstanceInterruptionBehavior: types.InstanceInterruptionBehaviorTerminate,
MaxPrice: aws.String(fmt.Sprintf("%v", instanceType.Price)),
}}
}
if instanceSet.ec2config.IAMInstanceProfile != "" {
- rii.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{
+ rii.IamInstanceProfile = &types.IamInstanceProfileSpecification{
Name: aws.String(instanceSet.ec2config.IAMInstanceProfile),
}
}
- var rsv *ec2.Reservation
+ var rsv *ec2.RunInstancesOutput
var errToReturn error
subnets := instanceSet.ec2config.SubnetID
currentSubnetIDIndex := int(atomic.LoadInt32(&instanceSet.currentSubnetIDIndex))
@@ -307,7 +310,7 @@ func (instanceSet *ec2InstanceSet) Create(
rii.NetworkInterfaces[0].SubnetId = aws.String(trySubnet)
}
var err error
- rsv, err = instanceSet.client.RunInstances(&rii)
+ rsv, err = instanceSet.client.RunInstances(context.TODO(), &rii)
instanceSet.mInstanceStarts.WithLabelValues(trySubnet, boolLabelValue[err == nil]).Add(1)
if !isErrorCapacity(errToReturn) || isErrorCapacity(err) {
// We want to return the last capacity error,
@@ -353,10 +356,10 @@ func (instanceSet *ec2InstanceSet) getKeyName(publicKey ssh.PublicKey) (string,
if keyname, ok := instanceSet.keys[md5keyFingerprint]; ok {
return keyname, nil
}
- keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{
- Filters: []*ec2.Filter{{
+ keyout, err := instanceSet.client.DescribeKeyPairs(context.TODO(), &ec2.DescribeKeyPairsInput{
+ Filters: []types.Filter{{
Name: aws.String("fingerprint"),
- Values: []*string{&md5keyFingerprint, &sha1keyFingerprint},
+ Values: []string{md5keyFingerprint, sha1keyFingerprint},
}},
})
if err != nil {
@@ -366,7 +369,7 @@ func (instanceSet *ec2InstanceSet) getKeyName(publicKey ssh.PublicKey) (string,
return *(keyout.KeyPairs[0].KeyName), nil
}
keyname := "arvados-dispatch-keypair-" + md5keyFingerprint
- _, err = instanceSet.client.ImportKeyPair(&ec2.ImportKeyPairInput{
+ _, err = instanceSet.client.ImportKeyPair(context.TODO(), &ec2.ImportKeyPairInput{
KeyName: &keyname,
PublicKeyMaterial: ssh.MarshalAuthorizedKey(publicKey),
})
@@ -378,17 +381,17 @@ func (instanceSet *ec2InstanceSet) getKeyName(publicKey ssh.PublicKey) (string,
}
func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances []cloud.Instance, err error) {
- var filters []*ec2.Filter
+ var filters []types.Filter
for k, v := range tags {
- filters = append(filters, &ec2.Filter{
+ filters = append(filters, types.Filter{
Name: aws.String("tag:" + k),
- Values: []*string{aws.String(v)},
+ Values: []string{v},
})
}
needAZs := false
dii := &ec2.DescribeInstancesInput{Filters: filters}
for {
- dio, err := instanceSet.client.DescribeInstances(dii)
+ dio, err := instanceSet.client.DescribeInstances(context.TODO(), dii)
err = wrapError(err, &instanceSet.throttleDelayInstances)
if err != nil {
return nil, err
@@ -396,12 +399,15 @@ func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances
for _, rsv := range dio.Reservations {
for _, inst := range rsv.Instances {
- if *inst.State.Name != "shutting-down" && *inst.State.Name != "terminated" {
+ switch inst.State.Name {
+ case types.InstanceStateNameShuttingDown:
+ case types.InstanceStateNameTerminated:
+ default:
instances = append(instances, &ec2Instance{
provider: instanceSet,
instance: inst,
})
- if aws.StringValue(inst.InstanceLifecycle) == "spot" {
+ if inst.InstanceLifecycle == types.InstanceLifecycleTypeSpot {
needAZs = true
}
}
@@ -414,16 +420,20 @@ func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances
}
if needAZs && instanceSet.ec2config.SpotPriceUpdateInterval > 0 {
az := map[string]string{}
- err := instanceSet.client.DescribeInstanceStatusPages(&ec2.DescribeInstanceStatusInput{
- IncludeAllInstances: aws.Bool(true),
- }, func(page *ec2.DescribeInstanceStatusOutput, lastPage bool) bool {
+ disi := &ec2.DescribeInstanceStatusInput{IncludeAllInstances: aws.Bool(true)}
+ for {
+ page, err := instanceSet.client.DescribeInstanceStatus(context.TODO(), disi)
+ if err != nil {
+ instanceSet.logger.Warnf("error getting instance statuses: %s", err)
+ break
+ }
for _, ent := range page.InstanceStatuses {
az[*ent.InstanceId] = *ent.AvailabilityZone
}
- return true
- })
- if err != nil {
- instanceSet.logger.Warnf("error getting instance statuses: %s", err)
+ if page.NextToken == nil {
+ break
+ }
+ disi.NextToken = page.NextToken
}
for _, inst := range instances {
inst := inst.(*ec2Instance)
@@ -475,28 +485,28 @@ func (instanceSet *ec2InstanceSet) updateSpotPrices(instances []cloud.Instance)
updateTime := time.Now()
staleTime := updateTime.Add(-instanceSet.ec2config.SpotPriceUpdateInterval.Duration())
needUpdate := false
- allTypes := map[string]bool{}
+ allTypes := map[types.InstanceType]bool{}
for _, inst := range instances {
ec2inst := inst.(*ec2Instance).instance
- if aws.StringValue(ec2inst.InstanceLifecycle) == "spot" {
+ if ec2inst.InstanceLifecycle == types.InstanceLifecycleTypeSpot {
pk := priceKey{
- instanceType: *ec2inst.InstanceType,
+ instanceType: string(ec2inst.InstanceType),
spot: true,
availabilityZone: inst.(*ec2Instance).availabilityZone,
}
if instanceSet.pricesUpdated[pk].Before(staleTime) {
needUpdate = true
}
- allTypes[*ec2inst.InstanceType] = true
+ allTypes[ec2inst.InstanceType] = true
}
}
if !needUpdate {
return
}
- var typeFilterValues []*string
+ var typeFilterValues []string
for instanceType := range allTypes {
- typeFilterValues = append(typeFilterValues, aws.String(instanceType))
+ typeFilterValues = append(typeFilterValues, string(instanceType))
}
// Get 3x update interval worth of pricing data. (Ideally the
// AWS API would tell us "we have shown you all of the price
@@ -507,14 +517,19 @@ func (instanceSet *ec2InstanceSet) updateSpotPrices(instances []cloud.Instance)
// row.
dsphi := &ec2.DescribeSpotPriceHistoryInput{
StartTime: aws.Time(updateTime.Add(-3 * instanceSet.ec2config.SpotPriceUpdateInterval.Duration())),
- Filters: []*ec2.Filter{
- &ec2.Filter{Name: aws.String("instance-type"), Values: typeFilterValues},
- &ec2.Filter{Name: aws.String("product-description"), Values: []*string{aws.String("Linux/UNIX")}},
+ Filters: []types.Filter{
+ types.Filter{Name: aws.String("instance-type"), Values: typeFilterValues},
+ types.Filter{Name: aws.String("product-description"), Values: []string{"Linux/UNIX"}},
},
}
- err := instanceSet.client.DescribeSpotPriceHistoryPages(dsphi, func(page *ec2.DescribeSpotPriceHistoryOutput, lastPage bool) bool {
+ for {
+ page, err := instanceSet.client.DescribeSpotPriceHistory(context.TODO(), dsphi)
+ if err != nil {
+ instanceSet.logger.Warnf("error retrieving spot instance prices: %s", err)
+ break
+ }
for _, ent := range page.SpotPriceHistory {
- if ent.InstanceType == nil || ent.SpotPrice == nil || ent.Timestamp == nil {
+ if ent.InstanceType == "" || ent.SpotPrice == nil || ent.Timestamp == nil {
// bogus record?
continue
}
@@ -524,7 +539,7 @@ func (instanceSet *ec2InstanceSet) updateSpotPrices(instances []cloud.Instance)
continue
}
pk := priceKey{
- instanceType: *ent.InstanceType,
+ instanceType: string(ent.InstanceType),
spot: true,
availabilityZone: *ent.AvailabilityZone,
}
@@ -534,10 +549,10 @@ func (instanceSet *ec2InstanceSet) updateSpotPrices(instances []cloud.Instance)
})
instanceSet.pricesUpdated[pk] = updateTime
}
- return true
- })
- if err != nil {
- instanceSet.logger.Warnf("error retrieving spot instance prices: %s", err)
+ if page.NextToken == nil {
+ break
+ }
+ dsphi.NextToken = page.NextToken
}
expiredTime := updateTime.Add(-64 * instanceSet.ec2config.SpotPriceUpdateInterval.Duration())
@@ -557,7 +572,7 @@ func (instanceSet *ec2InstanceSet) Stop() {
type ec2Instance struct {
provider *ec2InstanceSet
- instance *ec2.Instance
+ instance types.Instance
availabilityZone string // sometimes available for spot instances
}
@@ -570,20 +585,20 @@ func (inst *ec2Instance) String() string {
}
func (inst *ec2Instance) ProviderType() string {
- return *inst.instance.InstanceType
+ return string(inst.instance.InstanceType)
}
func (inst *ec2Instance) SetTags(newTags cloud.InstanceTags) error {
- var ec2tags []*ec2.Tag
+ var ec2tags []types.Tag
for k, v := range newTags {
- ec2tags = append(ec2tags, &ec2.Tag{
+ ec2tags = append(ec2tags, types.Tag{
Key: aws.String(k),
Value: aws.String(v),
})
}
- _, err := inst.provider.client.CreateTags(&ec2.CreateTagsInput{
- Resources: []*string{inst.instance.InstanceId},
+ _, err := inst.provider.client.CreateTags(context.TODO(), &ec2.CreateTagsInput{
+ Resources: []string{*inst.instance.InstanceId},
Tags: ec2tags,
})
@@ -601,8 +616,8 @@ func (inst *ec2Instance) Tags() cloud.InstanceTags {
}
func (inst *ec2Instance) Destroy() error {
- _, err := inst.provider.client.TerminateInstances(&ec2.TerminateInstancesInput{
- InstanceIds: []*string{inst.instance.InstanceId},
+ _, err := inst.provider.client.TerminateInstances(context.TODO(), &ec2.TerminateInstancesInput{
+ InstanceIds: []string{*inst.instance.InstanceId},
})
return err
}
@@ -653,8 +668,8 @@ func (inst *ec2Instance) PriceHistory(instType arvados.InstanceType) []cloud.Ins
// inst.provider.prices only for spot instances, so if
// spot==false here, we will return no data.
pk := priceKey{
- instanceType: *inst.instance.InstanceType,
- spot: aws.StringValue(inst.instance.InstanceLifecycle) == "spot",
+ instanceType: string(inst.instance.InstanceType),
+ spot: inst.instance.InstanceLifecycle == types.InstanceLifecycleTypeSpot,
availabilityZone: inst.availabilityZone,
}
var prices []cloud.InstancePrice
@@ -706,8 +721,9 @@ var isCodeQuota = map[string]bool{
//
// Returns false if error is nil.
func isErrorQuota(err error) bool {
- if aerr, ok := err.(awserr.Error); ok && aerr != nil {
- if _, ok := isCodeQuota[aerr.Code()]; ok {
+ var aerr smithy.APIError
+ if errors.As(err, &aerr) {
+ if _, ok := isCodeQuota[aerr.ErrorCode()]; ok {
return true
}
}
@@ -719,11 +735,11 @@ var reSubnetSpecificInvalidParameterMessage = regexp.MustCompile(`(?ms).*( subne
// isErrorSubnetSpecific returns true if the problem encountered by
// RunInstances might be avoided by trying a different subnet.
func isErrorSubnetSpecific(err error) bool {
- aerr, ok := err.(awserr.Error)
- if !ok {
+ var aerr smithy.APIError
+ if !errors.As(err, &aerr) {
return false
}
- code := aerr.Code()
+ code := aerr.ErrorCode()
return strings.Contains(code, "Subnet") ||
code == "InsufficientInstanceCapacity" ||
code == "InsufficientVolumeCapacity" ||
@@ -732,7 +748,7 @@ func isErrorSubnetSpecific(err error) bool {
// we look for substrings in code/message instead of
// only using specific codes here.
(strings.Contains(code, "InvalidParameter") &&
- reSubnetSpecificInvalidParameterMessage.MatchString(aerr.Message()))
+ reSubnetSpecificInvalidParameterMessage.MatchString(aerr.ErrorMessage()))
}
// isErrorCapacity returns true if the error indicates lack of
@@ -740,13 +756,13 @@ func isErrorSubnetSpecific(err error) bool {
// type -- i.e., retrying with a different instance type might
// succeed.
func isErrorCapacity(err error) bool {
- aerr, ok := err.(awserr.Error)
- if !ok {
+ var aerr smithy.APIError
+ if !errors.As(err, &aerr) {
return false
}
- code := aerr.Code()
+ code := aerr.ErrorCode()
return code == "InsufficientInstanceCapacity" ||
- (code == "Unsupported" && strings.Contains(aerr.Message(), "requested instance type"))
+ (code == "Unsupported" && strings.Contains(aerr.ErrorMessage(), "requested instance type"))
}
type ec2QuotaError struct {
@@ -757,8 +773,17 @@ func (er *ec2QuotaError) IsQuotaError() bool {
return true
}
+func isThrottleError(err error) bool {
+ var aerr smithy.APIError
+ if !errors.As(err, &aerr) {
+ return false
+ }
+ _, is := retry.DefaultThrottleErrorCodes[aerr.ErrorCode()]
+ return is
+}
+
func wrapError(err error, throttleValue *atomic.Value) error {
- if request.IsErrorThrottle(err) {
+ if isThrottleError(err) {
// Back off exponentially until an upstream call
// either succeeds or returns a non-throttle error.
d, _ := throttleValue.Load().(time.Duration)
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 5e6cf2c82b..709094d6db 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -23,10 +23,10 @@
package ec2
import (
+ "context"
"encoding/json"
"errors"
"flag"
- "fmt"
"sync/atomic"
"testing"
"time"
@@ -37,9 +37,10 @@ import (
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/config"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/awserr"
- "github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/ec2"
+ "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/aws/smithy-go"
"github.com/ghodss/yaml"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
@@ -101,17 +102,17 @@ type ec2stub struct {
subnetErrorOnRunInstances map[string]error
}
-func (e *ec2stub) ImportKeyPair(input *ec2.ImportKeyPairInput) (*ec2.ImportKeyPairOutput, error) {
+func (e *ec2stub) ImportKeyPair(ctx context.Context, input *ec2.ImportKeyPairInput, _ ...func(*ec2.Options)) (*ec2.ImportKeyPairOutput, error) {
e.importKeyPairCalls = append(e.importKeyPairCalls, input)
return nil, nil
}
-func (e *ec2stub) DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
+func (e *ec2stub) DescribeKeyPairs(ctx context.Context, input *ec2.DescribeKeyPairsInput, _ ...func(*ec2.Options)) (*ec2.DescribeKeyPairsOutput, error) {
e.describeKeyPairsCalls = append(e.describeKeyPairsCalls, input)
return &ec2.DescribeKeyPairsOutput{}, nil
}
-func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) {
+func (e *ec2stub) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, _ ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error) {
e.runInstancesCalls = append(e.runInstancesCalls, input)
if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0].SubnetId != nil {
err := e.subnetErrorOnRunInstances[*input.NetworkInterfaces[0].SubnetId]
@@ -119,98 +120,90 @@ func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation,
return nil, err
}
}
- return &ec2.Reservation{Instances: []*ec2.Instance{{
+ return &ec2.RunInstancesOutput{Instances: []types.Instance{{
InstanceId: aws.String("i-123"),
- InstanceType: aws.String("t2.micro"),
+ InstanceType: types.InstanceTypeT2Micro,
Tags: input.TagSpecifications[0].Tags,
}}}, nil
}
-func (e *ec2stub) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
+func (e *ec2stub) DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
return &ec2.DescribeInstancesOutput{
- Reservations: []*ec2.Reservation{{
- Instances: []*ec2.Instance{{
+ Reservations: []types.Reservation{{
+ Instances: []types.Instance{{
InstanceId: aws.String("i-123"),
- InstanceLifecycle: aws.String("spot"),
- InstanceType: aws.String("t2.micro"),
+ InstanceLifecycle: types.InstanceLifecycleTypeSpot,
+ InstanceType: types.InstanceTypeT2Micro,
PrivateIpAddress: aws.String("10.1.2.3"),
- State: &ec2.InstanceState{Name: aws.String("running"), Code: aws.Int64(16)},
+ State: &types.InstanceState{Name: types.InstanceStateNameRunning, Code: aws.Int32(16)},
}, {
InstanceId: aws.String("i-124"),
- InstanceLifecycle: aws.String("spot"),
- InstanceType: aws.String("t2.micro"),
+ InstanceLifecycle: types.InstanceLifecycleTypeSpot,
+ InstanceType: types.InstanceTypeT2Micro,
PrivateIpAddress: aws.String("10.1.2.4"),
- State: &ec2.InstanceState{Name: aws.String("running"), Code: aws.Int64(16)},
+ State: &types.InstanceState{Name: types.InstanceStateNameRunning, Code: aws.Int32(16)},
}},
}},
}, nil
}
-func (e *ec2stub) DescribeInstanceStatusPages(input *ec2.DescribeInstanceStatusInput, fn func(*ec2.DescribeInstanceStatusOutput, bool) bool) error {
- fn(&ec2.DescribeInstanceStatusOutput{
- InstanceStatuses: []*ec2.InstanceStatus{{
+func (e *ec2stub) DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStatusInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error) {
+ return &ec2.DescribeInstanceStatusOutput{
+ InstanceStatuses: []types.InstanceStatus{{
InstanceId: aws.String("i-123"),
AvailabilityZone: aws.String("aa-east-1a"),
}, {
InstanceId: aws.String("i-124"),
AvailabilityZone: aws.String("aa-east-1a"),
}},
- }, true)
- return nil
+ }, nil
}
-func (e *ec2stub) DescribeSpotPriceHistoryPages(input *ec2.DescribeSpotPriceHistoryInput, fn func(*ec2.DescribeSpotPriceHistoryOutput, bool) bool) error {
- if !fn(&ec2.DescribeSpotPriceHistoryOutput{
- SpotPriceHistory: []*ec2.SpotPrice{
- &ec2.SpotPrice{
- InstanceType: aws.String("t2.micro"),
- AvailabilityZone: aws.String("aa-east-1a"),
- SpotPrice: aws.String("0.005"),
- Timestamp: aws.Time(e.reftime.Add(-9 * time.Minute)),
+func (e *ec2stub) DescribeSpotPriceHistory(ctx context.Context, input *ec2.DescribeSpotPriceHistoryInput, _ ...func(*ec2.Options)) (*ec2.DescribeSpotPriceHistoryOutput, error) {
+ if input.NextToken == nil {
+ return &ec2.DescribeSpotPriceHistoryOutput{
+ SpotPriceHistory: []types.SpotPrice{
+ types.SpotPrice{
+ InstanceType: types.InstanceTypeT2Micro,
+ AvailabilityZone: aws.String("aa-east-1a"),
+ SpotPrice: aws.String("0.005"),
+ Timestamp: aws.Time(e.reftime.Add(-9 * time.Minute)),
+ },
+ types.SpotPrice{
+ InstanceType: types.InstanceTypeT2Micro,
+ AvailabilityZone: aws.String("aa-east-1a"),
+ SpotPrice: aws.String("0.015"),
+ Timestamp: aws.Time(e.reftime.Add(-5 * time.Minute)),
+ },
},
- &ec2.SpotPrice{
- InstanceType: aws.String("t2.micro"),
- AvailabilityZone: aws.String("aa-east-1a"),
- SpotPrice: aws.String("0.015"),
- Timestamp: aws.Time(e.reftime.Add(-5 * time.Minute)),
+ NextToken: aws.String("stubnexttoken"),
+ }, nil
+ } else {
+ return &ec2.DescribeSpotPriceHistoryOutput{
+ SpotPriceHistory: []types.SpotPrice{
+ types.SpotPrice{
+ InstanceType: types.InstanceTypeT2Micro,
+ AvailabilityZone: aws.String("aa-east-1a"),
+ SpotPrice: aws.String("0.01"),
+ Timestamp: aws.Time(e.reftime.Add(-2 * time.Minute)),
+ },
},
- },
- }, false) {
- return nil
+ }, nil
}
- fn(&ec2.DescribeSpotPriceHistoryOutput{
- SpotPriceHistory: []*ec2.SpotPrice{
- &ec2.SpotPrice{
- InstanceType: aws.String("t2.micro"),
- AvailabilityZone: aws.String("aa-east-1a"),
- SpotPrice: aws.String("0.01"),
- Timestamp: aws.Time(e.reftime.Add(-2 * time.Minute)),
- },
- },
- }, true)
- return nil
}
-func (e *ec2stub) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
+func (e *ec2stub) CreateTags(ctx context.Context, input *ec2.CreateTagsInput, _ ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error) {
return nil, nil
}
-func (e *ec2stub) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) {
+func (e *ec2stub) TerminateInstances(ctx context.Context, input *ec2.TerminateInstancesInput, _ ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error) {
return nil, nil
}
-type ec2stubError struct {
- code string
- message string
-}
-
-func (err *ec2stubError) Code() string { return err.code }
-func (err *ec2stubError) Message() string { return err.message }
-func (err *ec2stubError) Error() string { return fmt.Sprintf("%s: %s", err.code, err.message) }
-func (err *ec2stubError) OrigErr() error { return errors.New("stub OrigErr") }
+type ec2stubError = smithy.GenericAPIError
-// Ensure ec2stubError satisfies the aws.Error interface
-var _ = awserr.Error(&ec2stubError{})
+// Ensure ec2stubError satisfies the smithy.APIError interface
+var _ = smithy.APIError(&ec2stubError{})
func GetInstanceSet(c *check.C, conf string) (*ec2InstanceSet, cloud.ImageID, arvados.Cluster, *prometheus.Registry) {
reg := prometheus.NewRegistry()
@@ -280,8 +273,8 @@ func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
runcalls := ap.client.(*ec2stub).runInstancesCalls
if c.Check(runcalls, check.HasLen, 1) {
- c.Check(runcalls[0].MetadataOptions.HttpEndpoint, check.DeepEquals, aws.String("enabled"))
- c.Check(runcalls[0].MetadataOptions.HttpTokens, check.DeepEquals, aws.String("required"))
+ c.Check(runcalls[0].MetadataOptions.HttpEndpoint, check.DeepEquals, types.InstanceMetadataEndpointStateEnabled)
+ c.Check(runcalls[0].MetadataOptions.HttpTokens, check.DeepEquals, types.HttpTokensStateRequired)
}
}
}
@@ -333,8 +326,8 @@ func (*EC2InstanceSetSuite) TestCreateFailoverSecondSubnet(c *check.C) {
ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-good"]}`)
ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
"subnet-full": &ec2stubError{
- code: "InsufficientFreeAddressesInSubnet",
- message: "subnet is full",
+ Code: "InsufficientFreeAddressesInSubnet",
+ Message: "subnet is full",
},
}
inst, err := ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
@@ -368,49 +361,49 @@ func (*EC2InstanceSetSuite) TestIsErrorSubnetSpecific(c *check.C) {
c.Check(isErrorSubnetSpecific(errors.New("misc error")), check.Equals, false)
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InsufficientInstanceCapacity",
+ Code: "InsufficientInstanceCapacity",
}), check.Equals, true)
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InsufficientVolumeCapacity",
+ Code: "InsufficientVolumeCapacity",
}), check.Equals, true)
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InsufficientFreeAddressesInSubnet",
- message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+ Code: "InsufficientFreeAddressesInSubnet",
+ Message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
}), check.Equals, true)
// #21603: (Sometimes?) EC2 returns code InvalidParameterValue
// even though the code "InsufficientFreeAddressesInSubnet"
// seems like it must be meant for exactly this error.
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InvalidParameterValue",
- message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+ Code: "InvalidParameterValue",
+ Message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
}), check.Equals, true)
// Similarly, AWS docs
// (https://repost.aws/knowledge-center/vpc-insufficient-ip-errors)
// suggest the following code/message combinations also exist.
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "Client.InvalidParameterValue",
- message: "There aren't sufficient free Ipv4 addresses or prefixes",
+ Code: "Client.InvalidParameterValue",
+ Message: "There aren't sufficient free Ipv4 addresses or prefixes",
}), check.Equals, true)
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InvalidParameterValue",
- message: "There aren't sufficient free Ipv4 addresses or prefixes",
+ Code: "InvalidParameterValue",
+ Message: "There aren't sufficient free Ipv4 addresses or prefixes",
}), check.Equals, true)
// Meanwhile, other AWS docs
// (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html)
// suggest Client.InvalidParameterValue is not a real code but
// ClientInvalidParameterValue is.
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "ClientInvalidParameterValue",
- message: "There aren't sufficient free Ipv4 addresses or prefixes",
+ Code: "ClientInvalidParameterValue",
+ Message: "There aren't sufficient free Ipv4 addresses or prefixes",
}), check.Equals, true)
c.Check(isErrorSubnetSpecific(&ec2stubError{
- code: "InvalidParameterValue",
- message: "Some other invalid parameter error",
+ Code: "InvalidParameterValue",
+ Message: "Some other invalid parameter error",
}), check.Equals, false)
}
@@ -423,12 +416,12 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
"subnet-full": &ec2stubError{
- code: "InsufficientFreeAddressesInSubnet",
- message: "subnet is full",
+ Code: "InsufficientFreeAddressesInSubnet",
+ Message: "subnet is full",
},
"subnet-broken": &ec2stubError{
- code: "InvalidSubnetId.NotFound",
- message: "bogus subnet id",
+ Code: "InvalidSubnetId.NotFound",
+ Message: "bogus subnet id",
},
}
_, err := ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
@@ -464,12 +457,12 @@ func (*EC2InstanceSetSuite) TestCreateOneSubnetFailingCapacity(c *check.C) {
ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
"subnet-full": &ec2stubError{
- code: "InsufficientFreeAddressesInSubnet",
- message: "subnet is full",
+ Code: "InsufficientFreeAddressesInSubnet",
+ Message: "subnet is full",
},
"subnet-broken": &ec2stubError{
- code: "InsufficientInstanceCapacity",
- message: "insufficient capacity",
+ Code: "InsufficientInstanceCapacity",
+ Message: "insufficient capacity",
},
}
for i := 0; i < 3; i++ {
@@ -560,7 +553,7 @@ func (*EC2InstanceSetSuite) TestInstancePriceHistory(c *check.C) {
running := 0
for _, inst := range instances {
ec2i := inst.(*ec2Instance).instance
- if *ec2i.InstanceLifecycle == "spot" && *ec2i.State.Code&16 != 0 {
+ if ec2i.InstanceLifecycle == types.InstanceLifecycleTypeSpot && *ec2i.State.Code&16 != 0 {
running++
}
}
@@ -591,12 +584,12 @@ func (*EC2InstanceSetSuite) TestInstancePriceHistory(c *check.C) {
}
func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
- retryError := awserr.New("Throttling", "", nil)
+ retryError := &ec2stubError{Code: "Throttling"}
wrapped := wrapError(retryError, &atomic.Value{})
_, ok := wrapped.(cloud.RateLimitError)
c.Check(ok, check.Equals, true)
- quotaError := awserr.New("InstanceLimitExceeded", "", nil)
+ quotaError := &ec2stubError{Code: "InstanceLimitExceeded"}
wrapped = wrapError(quotaError, nil)
_, ok = wrapped.(cloud.QuotaError)
c.Check(ok, check.Equals, true)
@@ -608,7 +601,7 @@ func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
{"InsufficientInstanceCapacity", ""},
{"Unsupported", "Your requested instance type (t3.micro) is not supported in your requested Availability Zone (us-east-1e). Please retry your request by not specifying an Availability Zone or choosing us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f."},
} {
- capacityError := awserr.New(trial.code, trial.msg, nil)
+ capacityError := &ec2stubError{Code: trial.code, Message: trial.msg}
wrapped = wrapError(capacityError, nil)
caperr, ok := wrapped.(cloud.CapacityError)
c.Check(ok, check.Equals, true)
commit 36d748c6ab1265f1b58e2304cff04e84fea958cb
Author: Tom Clegg <tom at curii.com>
Date: Thu May 2 14:25:29 2024 -0400
21705: Update go runtime to 1.22.2.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index b27a14a501..01327e5daf 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -31,7 +31,7 @@ import (
var Command cmd.Handler = &installCommand{}
-const goversion = "1.20.6"
+const goversion = "1.22.2"
const (
defaultRubyVersion = "3.2.2"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list