[ARVADOS] created: 2.1.0-2108-g7b2c2c415

[Git user]

        at  7b2c2c415aa67936389998a5c93d52891e5644f4 (commit)


commit 7b2c2c415aa67936389998a5c93d52891e5644f4
Author: Tom Clegg <tom at curii.com>
Date:   Sun Mar 20 02:37:31 2022 -0400

    18596: Add PreemptiblePriceFactor config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 0a8f55244..c9232a76d 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -907,6 +907,13 @@ Clusters:
       # configured, and has no effect on top-level containers.
       AlwaysUsePreemptibleInstances: true
 
+      # Automatically add a preemptible variant for every
+      # non-preemptible entry in InstanceTypes below. The maximum bid
+      # price for the preemptible variant will be the non-preemptible
+      # price multiplied by PreemptiblePriceFactor. If 0, preemptible
+      # variants are not added automatically.
+      PreemptiblePriceFactor: 0
+
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # cloud dispatcher for executing containers on worker VMs.
       # Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
diff --git a/lib/config/export.go b/lib/config/export.go
index 4e903a8b3..67b7c3fa0 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -130,6 +130,7 @@ var whitelist = map[string]bool{
 	"Containers.MaxDispatchAttempts":           false,
 	"Containers.MaxRetryAttempts":              true,
 	"Containers.MinRetryPeriod":                true,
+	"Containers.PreemptiblePriceFactor":        false,
 	"Containers.ReserveExtraRAM":               true,
 	"Containers.RuntimeEngine":                 true,
 	"Containers.ShellAccess":                   true,
diff --git a/lib/config/load.go b/lib/config/load.go
index 8d498af17..f5d42c491 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -285,6 +285,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 		}
 	}
 
+	// Preprocess/automate some configs
+	for id, cc := range cfg.Clusters {
+		ldr.autofillPreemptible(&cc)
+
+		if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 {
+			// V2 token, strip it to just a secret
+			tmp := strings.Split(cc.Users.AnonymousUserToken, "/")
+			cc.Users.AnonymousUserToken = tmp[2]
+		}
+
+		cfg.Clusters[id] = cc
+	}
+
 	// Check for known mistakes
 	for id, cc := range cfg.Clusters {
 		for remote := range cc.RemoteClusters {
@@ -316,11 +329,6 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 				return nil, err
 			}
 		}
-		if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 {
-			// V2 token, strip it to just a secret
-			tmp := strings.Split(cc.Users.AnonymousUserToken, "/")
-			cc.Users.AnonymousUserToken = tmp[2]
-		}
 	}
 	return &cfg, nil
 }
@@ -527,3 +535,17 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi
 		}
 	}
 }
+
+func (ldr *Loader) autofillPreemptible(cc *arvados.Cluster) {
+	if factor := cc.Containers.PreemptiblePriceFactor; factor > 0 {
+		for name, it := range cc.InstanceTypes {
+			if !it.Preemptible {
+				it.Preemptible = true
+				it.Price = it.Price * factor
+				it.Name = name + ".preemptible"
+				cc.InstanceTypes[it.Name] = it
+			}
+		}
+	}
+
+}
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 1ede805b0..3d8045d31 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -695,3 +695,38 @@ Clusters:
 	_, err = ldr.Load()
 	c.Assert(err, check.ErrorMatches, `there is no default storage class.*`)
 }
+
+func (s *LoadSuite) TestPreemptiblePriceFactor(c *check.C) {
+	yaml := `
+Clusters:
+ z1111:
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+ z2222:
+  Containers:
+   PreemptiblePriceFactor: 0.5
+  InstanceTypes:
+   Type1:
+    RAM: 12345M
+    VCPUs: 8
+    Price: 1.23
+`
+	cfg, err := testLoader(c, yaml, nil).Load()
+	c.Assert(err, check.IsNil)
+	cc, err := cfg.GetCluster("z1111")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+	c.Check(cc.InstanceTypes, check.HasLen, 1)
+
+	cc, err = cfg.GetCluster("z2222")
+	c.Assert(err, check.IsNil)
+	c.Check(cc.InstanceTypes["Type1"].Preemptible, check.Equals, false)
+	c.Check(cc.InstanceTypes["Type1"].Price, check.Equals, 1.23)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].Preemptible, check.Equals, true)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23/2)
+	c.Check(cc.InstanceTypes["Type1.preemptible"].ProviderType, check.Equals, "Type1")
+	c.Check(cc.InstanceTypes, check.HasLen, 2)
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b8c8269f1..6242f6cd5 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -445,6 +445,7 @@ type ContainersConfig struct {
 	StaleLockTimeout              Duration
 	SupportedDockerImageFormats   StringSet
 	AlwaysUsePreemptibleInstances bool
+	PreemptiblePriceFactor        float64
 	RuntimeEngine                 string
 	LocalKeepBlobBuffersPerVCPU   int
 	LocalKeepLogsToContainerLog   string

-----------------------------------------------------------------------


hooks/post-receive
--