[ARVADOS] updated: 2.1.0-2112-g75556b4f8
Git user
git at public.arvados.org
Tue Mar 22 19:26:43 UTC 2022
Summary of changes:
doc/admin/spot-instances.html.textile.liquid | 25 +++++++++++++++---
lib/config/config.default.yml | 4 ++-
lib/config/load.go | 8 ++++--
lib/config/load_test.go | 38 +++++++++++++++++++++++++---
4 files changed, 65 insertions(+), 10 deletions(-)
via 75556b4f8a694bbb3835ec0387fd6d2c361a561b (commit)
via e0176be9d01ac0f39775ad28cd30cf264d3273c3 (commit)
via f08a2e4a1fd2c65798574e29c34fdff5cb8ef366 (commit)
via b4f172a329980127db895becda91543fd65b6b39 (commit)
from 7b2c2c415aa67936389998a5c93d52891e5644f4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 75556b4f8a694bbb3835ec0387fd6d2c361a561b
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 22 15:24:10 2022 -0400
18596: Warn about PreemptiblePriceFactor / InstanceTypes collision.
In case of a collision, use the explicit type, not the automatic one.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index f5d42c491..de43b9d2e 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -287,7 +287,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
// Preprocess/automate some configs
for id, cc := range cfg.Clusters {
- ldr.autofillPreemptible(&cc)
+ ldr.autofillPreemptible("Clusters."+id, &cc)
if strings.Count(cc.Users.AnonymousUserToken, "/") == 3 {
// V2 token, strip it to just a secret
@@ -536,13 +536,17 @@ func (ldr *Loader) logExtraKeys(expected, supplied map[string]interface{}, prefi
}
}
-func (ldr *Loader) autofillPreemptible(cc *arvados.Cluster) {
+func (ldr *Loader) autofillPreemptible(label string, cc *arvados.Cluster) {
if factor := cc.Containers.PreemptiblePriceFactor; factor > 0 {
for name, it := range cc.InstanceTypes {
if !it.Preemptible {
it.Preemptible = true
it.Price = it.Price * factor
it.Name = name + ".preemptible"
+ if it2, exists := cc.InstanceTypes[it.Name]; exists && it2 != it {
+ ldr.Logger.Warnf("%s.InstanceTypes[%s]: already exists, so not automatically adding a preemptible variant of %s", label, it.Name, name)
+ continue
+ }
cc.InstanceTypes[it.Name] = it
}
}
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 3d8045d31..5270dcccc 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -305,8 +305,6 @@ func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
var logbuf bytes.Buffer
- logger := logrus.New()
- logger.Out = &logbuf
cfg, err := testLoader(c, `
Clusters:
zzzzz:
@@ -713,8 +711,33 @@ Clusters:
RAM: 12345M
VCPUs: 8
Price: 1.23
+ z3333:
+ Containers:
+ PreemptiblePriceFactor: 0.5
+ InstanceTypes:
+ Type1:
+ RAM: 12345M
+ VCPUs: 8
+ Price: 1.23
+ Type1.preemptible: # higher price than the auto-added variant would use -- should generate warning
+ ProviderType: Type1
+ RAM: 12345M
+ VCPUs: 8
+ Price: 1.23
+ Preemptible: true
+ Type2:
+ RAM: 23456M
+ VCPUs: 16
+ Price: 2.46
+ Type2.preemptible: # identical to the auto-added variant -- so no warning
+ ProviderType: Type2
+ RAM: 23456M
+ VCPUs: 16
+ Price: 1.23
+ Preemptible: true
`
- cfg, err := testLoader(c, yaml, nil).Load()
+ var logbuf bytes.Buffer
+ cfg, err := testLoader(c, yaml, &logbuf).Load()
c.Assert(err, check.IsNil)
cc, err := cfg.GetCluster("z1111")
c.Assert(err, check.IsNil)
@@ -729,4 +752,13 @@ Clusters:
c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23/2)
c.Check(cc.InstanceTypes["Type1.preemptible"].ProviderType, check.Equals, "Type1")
c.Check(cc.InstanceTypes, check.HasLen, 2)
+
+ cc, err = cfg.GetCluster("z3333")
+ c.Assert(err, check.IsNil)
+ // Don't overwrite the explicitly configured preemptible variant
+ c.Check(cc.InstanceTypes["Type1.preemptible"].Price, check.Equals, 1.23)
+ c.Check(cc.InstanceTypes, check.HasLen, 4)
+ c.Check(logbuf.String(), check.Matches, `(?ms).*Clusters\.z3333\.InstanceTypes\[Type1\.preemptible\]: already exists, so not automatically adding a preemptible variant of Type1.*`)
+ c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*Type2\.preemptible.*`)
+ c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*(z1111|z2222)[^\n]*InstanceTypes.*`)
}
commit e0176be9d01ac0f39775ad28cd30cf264d3273c3
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 22 15:23:52 2022 -0400
18596: Mention that 1.0 is a reasonable PreemptiblePriceFactor.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index b72faa237..318e9ab1d 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -912,6 +912,8 @@ Clusters:
# price for the preemptible variant will be the non-preemptible
# price multiplied by PreemptiblePriceFactor. If 0, preemptible
# variants are not added automatically.
+ #
+ # A price factor of 1.0 is a reasonable starting point.
PreemptiblePriceFactor: 0
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
commit f08a2e4a1fd2c65798574e29c34fdff5cb8ef366
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 22 15:22:00 2022 -0400
18596: Change AlwaysUsePreemptibleInstances default to false.
Admin docs were already updated to say the default is false, which
makes sense now that users can request preemptible instances via
arvados-cwl-runner (#18180) -- now the default really is false.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index c9232a76d..b72faa237 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -905,7 +905,7 @@ Clusters:
#
# This flag is ignored if no preemptible instance types are
# configured, and has no effect on top-level containers.
- AlwaysUsePreemptibleInstances: true
+ AlwaysUsePreemptibleInstances: false
# Automatically add a preemptible variant for every
# non-preemptible entry in InstanceTypes below. The maximum bid
commit b4f172a329980127db895becda91543fd65b6b39
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 22 15:21:06 2022 -0400
18596: Explain PreemptiblePriceFactor in admin docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/spot-instances.html.textile.liquid b/doc/admin/spot-instances.html.textile.liquid
index 3837f30d6..703e70fb8 100644
--- a/doc/admin/spot-instances.html.textile.liquid
+++ b/doc/admin/spot-instances.html.textile.liquid
@@ -16,31 +16,48 @@ Currently Arvados supports preemptible instances using AWS and Azure spot instan
h2. Configuration
-Add entries to @InstanceTypes@ that have @Preemptible: true at . Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example:
+First, configure some @InstanceTypes@ that have @Preemptible: true at . For a preemptible instance, @Price@ determines the maximum bid price; the actual price paid is dynamic and will likely be lower.
+
+Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. To do this automatically, use @PreemptiblePriceFactor@ to enable a preemptible version of each listed type, using the given factor to set the maximum bid price relative to the non-preemptible price. Alternatively, you can configure preemptible instance types explicitly. For example, the following two configurations are equivalent:
<pre>
Clusters:
ClusterID:
+ Containers:
+ PreemptiblePriceFactor: 0.8
InstanceTypes:
m4.large:
- Preemptible: false
ProviderType: m4.large
VCPUs: 2
RAM: 8GiB
AddedScratch: 32GB
Price: 0.1
- m4.large.spot:
- Preemptible: true
+</pre>
+
+<pre>
+Clusters:
+ ClusterID:
+ InstanceTypes:
+ m4.large:
ProviderType: m4.large
VCPUs: 2
RAM: 8GiB
AddedScratch: 32GB
Price: 0.1
+ m4.large.preemptible:
+ Preemptible: true
+ ProviderType: m4.large
+ VCPUs: 2
+ RAM: 8GiB
+ AddedScratch: 32GB
+ Price: 0.08
</pre>
Next, you can choose to enable automatic use of preemptible instances:
<pre>
+Clusters:
+ ClusterID:
Containers:
AlwaysUsePreemptibleInstances: true
</pre>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list