[ARVADOS] created: 2.1.0-912-g66d3cb88d

[Git user]

        at  66d3cb88d07eed627903b6db0b1cffb7491d4e34 (commit)


commit 66d3cb88d07eed627903b6db0b1cffb7491d4e34
Author: Ward Vandewege <ward at curii.com>
Date:   Tue Jun 8 17:37:13 2021 -0400

    17776: Basic throttling for VcpuLimitExceeded, MaxSpotInstanceCountExceeded,
    InsufficientInstanceCapacity.
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 071c95006..fb7afdda4 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -20,6 +20,7 @@ import (
 	"git.arvados.org/arvados.git/lib/cloud"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
@@ -349,8 +350,28 @@ func (err rateLimitError) EarliestRetry() time.Time {
 	return err.earliestRetry
 }
 
+var capacityCodes = map[string]struct{}{
+	"InsufficientInstanceCapacity": {},
+	"VcpuLimitExceeded":            {},
+	"MaxSpotInstanceCountExceeded": {},
+}
+
+// IsErrorCapacity returns whether the error is to be throttled based on its code.
+// Returns false if error is nil.
+func IsErrorCapacity(err error) bool {
+	if aerr, ok := err.(awserr.Error); ok && aerr != nil {
+		return isCodeCapacity(aerr.Code())
+	}
+	return false
+}
+
+func isCodeCapacity(code string) bool {
+	_, ok := capacityCodes[code]
+	return ok
+}
+
 func wrapError(err error, throttleValue *atomic.Value) error {
-	if request.IsErrorThrottle(err) {
+	if request.IsErrorThrottle(err) || IsErrorCapacity(err) {
 		// Back off exponentially until an upstream call
 		// either succeeds or returns a non-throttle error.
 		d, _ := throttleValue.Load().(time.Duration)

-----------------------------------------------------------------------


hooks/post-receive
--