[ARVADOS] created: 2.1.0-1165-gbad73626c
Git user
git at public.arvados.org
Thu Aug 5 15:04:47 UTC 2021
at bad73626c4208fb95ac8e3d9503fc4482f936cb3 (commit)
commit bad73626c4208fb95ac8e3d9503fc4482f936cb3
Author: Tom Clegg <tom at curii.com>
Date: Thu Aug 5 11:04:37 2021 -0400
17967: Use StorageClasses.*.Default instead of ["default"].
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index ad0480771..d8e673bd3 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -913,7 +913,7 @@ class ArvPutUploadJob(object):
self._local_collection = arvados.collection.Collection(
self._state['manifest'],
replication_desired=self.replication_desired,
- storage_classes_desired=(self.storage_classes or ['default']),
+ storage_classes_desired=self.storage_classes,
put_threads=self.put_threads,
api_client=self._api_client,
num_retries=self.num_retries)
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 4e7b64cf5..5edca82a0 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -17,7 +17,7 @@ class Collection < ArvadosModel
# Posgresql JSONB columns should NOT be declared as serialized, Rails 5
# already know how to properly treat them.
attribute :properties, :jsonbHash, default: {}
- attribute :storage_classes_desired, :jsonbArray, default: ["default"]
+ attribute :storage_classes_desired, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
attribute :storage_classes_confirmed, :jsonbArray, default: []
before_validation :default_empty_manifest
@@ -630,7 +630,7 @@ class Collection < ArvadosModel
# validation on empty desired storage classes return an error.
def default_storage_classes
if self.storage_classes_desired.nil? || self.storage_classes_desired.empty?
- self.storage_classes_desired = ["default"]
+ self.storage_classes_desired = Rails.configuration.DefaultStorageClasses
end
self.storage_classes_confirmed ||= []
end
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index af058494b..a880b65ac 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -22,7 +22,7 @@ class Container < ArvadosModel
attribute :secret_mounts, :jsonbHash, default: {}
attribute :runtime_status, :jsonbHash, default: {}
attribute :runtime_auth_scopes, :jsonbArray, default: []
- attribute :output_storage_classes, :jsonbArray, default: ["default"]
+ attribute :output_storage_classes, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
serialize :environment, Hash
serialize :mounts, Hash
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index 1de71102c..f603d4dd7 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -23,7 +23,7 @@ class ContainerRequest < ArvadosModel
# already know how to properly treat them.
attribute :properties, :jsonbHash, default: {}
attribute :secret_mounts, :jsonbHash, default: {}
- attribute :output_storage_classes, :jsonbArray, default: ["default"]
+ attribute :output_storage_classes, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
serialize :environment, Hash
serialize :mounts, Hash
diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb
index a6f1730e8..1b3c96a8a 100644
--- a/services/api/config/arvados_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -170,6 +170,7 @@ arvcfg.declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
ConfigLoader.set_cfg cfg, "RemoteClusters", h
}
arvcfg.declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+arvcfg.declare_config "StorageClasses", Hash
dbcfg = ConfigLoader.new
@@ -237,6 +238,17 @@ if $arvados_config["Collections"]["DefaultTrashLifetime"] < 86400.seconds then
raise "default_trash_lifetime is %d, must be at least 86400" % Rails.configuration.Collections.DefaultTrashLifetime
end
+default_storage_classes = []
+$arvados_config["StorageClasses"].each do |cls, cfg|
+ if cfg["Default"]
+ default_storage_classes << cls
+ end
+end
+if default_storage_classes.length == 0
+ default_storage_classes = ["default"]
+end
+$arvados_config["DefaultStorageClasses"] = default_storage_classes.sort
+
#
# Special case for test database where there's no database.yml,
# because the Arvados config.yml doesn't have a concept of multiple
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index e69d941b1..bb590e13b 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -538,10 +538,6 @@ func (bal *Balancer) setupLookupTables() {
// effectively read-only.
mnt.ReadOnly = mnt.ReadOnly || srv.ReadOnly
- if len(mnt.StorageClasses) == 0 {
- bal.mountsByClass["default"][mnt] = true
- continue
- }
for class := range mnt.StorageClasses {
if mbc := bal.mountsByClass[class]; mbc == nil {
bal.classes = append(bal.classes, class)
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 18a8bdcf4..4e2c6803c 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -87,20 +87,24 @@ var stubServices = []arvados.KeepService{
var stubMounts = map[string][]arvados.KeepMount{
"keep0.zzzzz.arvadosapi.com:25107": {{
- UUID: "zzzzz-ivpuk-000000000000000",
- DeviceID: "keep0-vol0",
+ UUID: "zzzzz-ivpuk-000000000000000",
+ DeviceID: "keep0-vol0",
+ StorageClasses: map[string]bool{"default": true},
}},
"keep1.zzzzz.arvadosapi.com:25107": {{
- UUID: "zzzzz-ivpuk-100000000000000",
- DeviceID: "keep1-vol0",
+ UUID: "zzzzz-ivpuk-100000000000000",
+ DeviceID: "keep1-vol0",
+ StorageClasses: map[string]bool{"default": true},
}},
"keep2.zzzzz.arvadosapi.com:25107": {{
- UUID: "zzzzz-ivpuk-200000000000000",
- DeviceID: "keep2-vol0",
+ UUID: "zzzzz-ivpuk-200000000000000",
+ DeviceID: "keep2-vol0",
+ StorageClasses: map[string]bool{"default": true},
}},
"keep3.zzzzz.arvadosapi.com:25107": {{
- UUID: "zzzzz-ivpuk-300000000000000",
- DeviceID: "keep3-vol0",
+ UUID: "zzzzz-ivpuk-300000000000000",
+ DeviceID: "keep3-vol0",
+ StorageClasses: map[string]bool{"default": true},
}},
}
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 5bc66dbf3..c529ac150 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -85,7 +85,8 @@ func (bal *balancerSuite) SetUpTest(c *check.C) {
}
srv.mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
- UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+ UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+ StorageClasses: map[string]bool{"default": true},
},
KeepService: srv,
}}
@@ -166,10 +167,11 @@ func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i)
srv.mounts = append(srv.mounts, &KeepMount{
KeepMount: arvados.KeepMount{
- DeviceID: fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
- UUID: fmt.Sprintf("zzzzz-mount-%015x", i<<16),
- ReadOnly: readonly,
- Replication: 1,
+ DeviceID: fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
+ UUID: fmt.Sprintf("zzzzz-mount-%015x", i<<16),
+ ReadOnly: readonly,
+ Replication: 1,
+ StorageClasses: map[string]bool{"default": true},
},
KeepService: srv,
})
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index a60d17d57..2b469a13e 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -252,6 +252,13 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
for i, sc := range wantStorageClasses {
wantStorageClasses[i] = strings.TrimSpace(sc)
}
+ } else {
+ // none specified -- use configured default
+ for class, cfg := range rtr.cluster.StorageClasses {
+ if cfg.Default {
+ wantStorageClasses = append(wantStorageClasses, class)
+ }
+ }
}
buf, err := getBufferWithContext(ctx, bufs, int(req.ContentLength))
commit fe5c9050c56be6828acead0e3c09c2195cde5b99
Author: Tom Clegg <tom at curii.com>
Date: Wed Aug 4 23:21:45 2021 -0400
17967: Read from volumes with high-priority storage classes first.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index db64449e4..00ef11b6e 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -320,6 +320,54 @@ func (s *HandlerSuite) TestPutAndDeleteSkipReadonlyVolumes(c *check.C) {
}
}
+func (s *HandlerSuite) TestReadsOrderedByStorageClassPriority(c *check.C) {
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-111111111111111": {
+ Driver: "mock",
+ Replication: 1,
+ StorageClasses: map[string]bool{"class1": true}},
+ "zzzzz-nyw5e-222222222222222": {
+ Driver: "mock",
+ Replication: 1,
+ StorageClasses: map[string]bool{"class2": true, "class3": true}},
+ }
+
+ for _, trial := range []struct {
+ priority1 int // priority of class1, thus vol1
+ priority2 int // priority of class2
+ priority3 int // priority of class3 (vol2 priority will be max(priority2, priority3))
+ get1 int // expected number of "get" ops on vol1
+ get2 int // expected number of "get" ops on vol2
+ }{
+ {100, 50, 50, 1, 0}, // class1 has higher priority => try vol1 first, no need to try vol2
+ {100, 100, 100, 1, 0}, // same priority, vol1 is first lexicographically => try vol1 first and succeed
+ {66, 99, 33, 1, 1}, // class2 has higher priority => try vol2 first, then try vol1
+ {66, 33, 99, 1, 1}, // class3 has highest priority => vol2 has highest => try vol2 first, then try vol1
+ } {
+ c.Logf("%+v", trial)
+ s.cluster.StorageClasses = map[string]arvados.StorageClassConfig{
+ "class1": {Priority: trial.priority1},
+ "class2": {Priority: trial.priority2},
+ "class3": {Priority: trial.priority3},
+ }
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ IssueRequest(s.handler,
+ &RequestTester{
+ method: "PUT",
+ uri: "/" + TestHash,
+ requestBody: TestBlock,
+ storageClasses: "class1",
+ })
+ IssueRequest(s.handler,
+ &RequestTester{
+ method: "GET",
+ uri: "/" + TestHash,
+ })
+ c.Check(s.handler.volmgr.mountMap["zzzzz-nyw5e-111111111111111"].Volume.(*MockVolume).CallCount("Get"), check.Equals, trial.get1)
+ c.Check(s.handler.volmgr.mountMap["zzzzz-nyw5e-222222222222222"].Volume.(*MockVolume).CallCount("Get"), check.Equals, trial.get2)
+ }
+}
+
// Test TOUCH requests.
func (s *HandlerSuite) TestTouchHandler(c *check.C) {
c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 26e6b7318..9bfc6ca3e 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"math/big"
+ "sort"
"sync/atomic"
"time"
@@ -343,6 +344,27 @@ func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, my
vm.writables = append(vm.writables, mnt)
}
}
+ // pri(i): return highest priority of any storage class
+ // offered by vm.readables[i]
+ pri := func(i int) int {
+ any, best := false, 0
+ for class := range vm.readables[i].KeepMount.StorageClasses {
+ if p := cluster.StorageClasses[class].Priority; !any || best < p {
+ best = p
+ any = true
+ }
+ }
+ return best
+ }
+ // sort vm.readables, first by highest priority of any offered
+ // storage class (highest->lowest), then by volume UUID
+ sort.Slice(vm.readables, func(i, j int) bool {
+ if pi, pj := pri(i), pri(j); pi != pj {
+ return pi > pj
+ } else {
+ return vm.readables[i].KeepMount.UUID < vm.readables[j].KeepMount.UUID
+ }
+ })
return vm, nil
}
commit 444dfb847a5d6eda3a84ef5f4e508703d0634a91
Author: Tom Clegg <tom at curii.com>
Date: Wed Aug 4 16:57:35 2021 -0400
17967: Add StorageClasses config section.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 66f508b5a..eb05c22fd 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1228,6 +1228,26 @@ Clusters:
Price: 0.1
Preemptible: false
+ StorageClasses:
+
+ # If you use multiple storage classes, specify them here, using
+ # the storage class name as the key (in place of "SAMPLE" in
+ # this sample entry).
+ SAMPLE:
+
+ # Priority determines the order volumes should be searched
+ # when reading data, in cases where a keepstore server has
+ # access to multiple volumes with different storage classes.
+ Priority: 0
+
+ # Default determines which storage class(es) should be used
+ # when a user/client writes data or saves a new collection
+ # without specifying storage classes.
+ #
+ # If any StorageClasses are configured, at least one of them
+ # must have Default: true.
+ Default: true
+
Volumes:
SAMPLE:
# AccessViaHosts specifies which keepstore processes can read
@@ -1251,7 +1271,9 @@ Clusters:
ReadOnly: false
Replication: 1
StorageClasses:
- default: true
+ # If you have configured storage classes (see StorageClasses
+ # section above), add an entry here for each storage class
+ # satisfied by this volume.
SAMPLE: true
Driver: S3
DriverParameters:
diff --git a/lib/config/export.go b/lib/config/export.go
index bbc5ea6c5..2a3d0e173 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -205,6 +205,10 @@ var whitelist = map[string]bool{
"Services.*": true,
"Services.*.ExternalURL": true,
"Services.*.InternalURLs": false,
+ "StorageClasses": true,
+ "StorageClasses.*": true,
+ "StorageClasses.*.Default": true,
+ "StorageClasses.*.Priority": true,
"SystemLogs": false,
"SystemRootToken": false,
"TLS": false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index ee2308413..6fe0c73c5 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -1234,6 +1234,26 @@ Clusters:
Price: 0.1
Preemptible: false
+ StorageClasses:
+
+ # If you use multiple storage classes, specify them here, using
+ # the storage class name as the key (in place of "SAMPLE" in
+ # this sample entry).
+ SAMPLE:
+
+ # Priority determines the order volumes should be searched
+ # when reading data, in cases where a keepstore server has
+ # access to multiple volumes with different storage classes.
+ Priority: 0
+
+ # Default determines which storage class(es) should be used
+ # when a user/client writes data or saves a new collection
+ # without specifying storage classes.
+ #
+ # If any StorageClasses are configured, at least one of them
+ # must have Default: true.
+ Default: true
+
Volumes:
SAMPLE:
# AccessViaHosts specifies which keepstore processes can read
@@ -1257,7 +1277,9 @@ Clusters:
ReadOnly: false
Replication: 1
StorageClasses:
- default: true
+ # If you have configured storage classes (see StorageClasses
+ # section above), add an entry here for each storage class
+ # satisfied by this volume.
SAMPLE: true
Driver: S3
DriverParameters:
diff --git a/lib/config/load.go b/lib/config/load.go
index 169b252a0..248960beb 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -269,6 +269,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
ldr.loadOldKeepBalanceConfig,
)
}
+ loadFuncs = append(loadFuncs, ldr.setImplicitStorageClasses)
for _, f := range loadFuncs {
err = f(&cfg)
if err != nil {
@@ -296,6 +297,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
ldr.checkEmptyKeepstores(cc),
ldr.checkUnlistedKeepstores(cc),
+ ldr.checkStorageClasses(cc),
// TODO: check non-empty Rendezvous on
// services other than Keepstore
} {
@@ -336,6 +338,57 @@ func (ldr *Loader) checkToken(label, token string) error {
return nil
}
+func (ldr *Loader) setImplicitStorageClasses(cfg *arvados.Config) error {
+cluster:
+ for id, cc := range cfg.Clusters {
+ if len(cc.StorageClasses) > 0 {
+ continue cluster
+ }
+ for _, vol := range cc.Volumes {
+ if len(vol.StorageClasses) > 0 {
+ continue cluster
+ }
+ }
+ // No explicit StorageClasses config info at all; fill
+ // in implicit defaults.
+ for id, vol := range cc.Volumes {
+ vol.StorageClasses = map[string]bool{"default": true}
+ cc.Volumes[id] = vol
+ }
+ cc.StorageClasses = map[string]arvados.StorageClassConfig{"default": {Default: true}}
+ cfg.Clusters[id] = cc
+ }
+ return nil
+}
+
+func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
+ classOnVolume := map[string]bool{}
+ for volid, vol := range cc.Volumes {
+ if len(vol.StorageClasses) == 0 {
+ return fmt.Errorf("%s: volume has no StorageClasses listed", volid)
+ }
+ for classid := range vol.StorageClasses {
+ if _, ok := cc.StorageClasses[classid]; !ok {
+ return fmt.Errorf("%s: volume refers to storage class %q that is not defined in StorageClasses", volid, classid)
+ }
+ classOnVolume[classid] = true
+ }
+ }
+ haveDefault := false
+ for classid, sc := range cc.StorageClasses {
+ if !classOnVolume[classid] && len(cc.Volumes) > 0 {
+ ldr.Logger.Warnf("there are no volumes providing storage class %q", classid)
+ }
+ if sc.Default {
+ haveDefault = true
+ }
+ }
+ if !haveDefault {
+ return fmt.Errorf("there is no default storage class (at least one entry in StorageClasses must have Default: true)")
+ }
+ return nil
+}
+
func checkKeyConflict(label string, m map[string]string) error {
saw := map[string]bool{}
for k := range m {
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 396faca48..d4896c39c 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -29,6 +29,8 @@ func Test(t *testing.T) {
var _ = check.Suite(&LoadSuite{})
+var emptyConfigYAML = `Clusters: {"z1111": {}}`
+
// Return a new Loader that reads cluster config from configdata
// (instead of the usual default /etc/arvados/config.yml), and logs to
// logdst or (if that's nil) c.Log.
@@ -59,7 +61,7 @@ func (s *LoadSuite) TestEmpty(c *check.C) {
}
func (s *LoadSuite) TestNoConfigs(c *check.C) {
- cfg, err := testLoader(c, `Clusters: {"z1111": {}}`, nil).Load()
+ cfg, err := testLoader(c, emptyConfigYAML, nil).Load()
c.Assert(err, check.IsNil)
c.Assert(cfg.Clusters, check.HasLen, 1)
cc, err := cfg.GetCluster("z1111")
@@ -79,7 +81,7 @@ func (s *LoadSuite) TestMungeLegacyConfigArgs(c *check.C) {
f, err = ioutil.TempFile("", "")
c.Check(err, check.IsNil)
defer os.Remove(f.Name())
- io.WriteString(f, "Clusters: {aaaaa: {}}\n")
+ io.WriteString(f, emptyConfigYAML)
newfile := f.Name()
for _, trial := range []struct {
@@ -562,11 +564,122 @@ func (s *LoadSuite) TestListKeys(c *check.C) {
c.Errorf("Should have produced an error")
}
- var logbuf bytes.Buffer
- loader := testLoader(c, string(DefaultYAML), &logbuf)
+ loader := testLoader(c, string(DefaultYAML), nil)
cfg, err := loader.Load()
c.Assert(err, check.IsNil)
if err := checkListKeys("", cfg); err != nil {
c.Error(err)
}
}
+
+func (s *LoadSuite) TestImplicitStorageClasses(c *check.C) {
+ // If StorageClasses and Volumes.*.StorageClasses are all
+ // empty, there is a default storage class named "default".
+ ldr := testLoader(c, `{"Clusters":{"z1111":{}}}`, nil)
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(cc.StorageClasses, check.HasLen, 1)
+ c.Check(cc.StorageClasses["default"].Default, check.Equals, true)
+ c.Check(cc.StorageClasses["default"].Priority, check.Equals, 0)
+
+ // The implicit "default" storage class is used by all
+ // volumes.
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ Volumes:
+ z: {}`, nil)
+ cfg, err = ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc, err = cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(cc.StorageClasses, check.HasLen, 1)
+ c.Check(cc.StorageClasses["default"].Default, check.Equals, true)
+ c.Check(cc.StorageClasses["default"].Priority, check.Equals, 0)
+ c.Check(cc.Volumes["z"].StorageClasses["default"], check.Equals, true)
+
+ // The "default" storage class isn't implicit if any classes
+ // are configured explicitly.
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ StorageClasses:
+ local:
+ Default: true
+ Priority: 111
+ Volumes:
+ z:
+ StorageClasses:
+ local: true`, nil)
+ cfg, err = ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc, err = cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(cc.StorageClasses, check.HasLen, 1)
+ c.Check(cc.StorageClasses["local"].Default, check.Equals, true)
+ c.Check(cc.StorageClasses["local"].Priority, check.Equals, 111)
+
+ // It is an error for a volume to refer to a storage class
+ // that isn't listed in StorageClasses.
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ StorageClasses:
+ local:
+ Default: true
+ Priority: 111
+ Volumes:
+ z:
+ StorageClasses:
+ nx: true`, nil)
+ _, err = ldr.Load()
+ c.Assert(err, check.ErrorMatches, `z: volume refers to storage class "nx" that is not defined.*`)
+
+ // It is an error for a volume to refer to a storage class
+ // that isn't listed in StorageClasses ... even if it's
+ // "default", which would exist implicitly if it weren't
+ // referenced explicitly by a volume.
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ Volumes:
+ z:
+ StorageClasses:
+ default: true`, nil)
+ _, err = ldr.Load()
+ c.Assert(err, check.ErrorMatches, `z: volume refers to storage class "default" that is not defined.*`)
+
+ // If the "default" storage class is configured explicitly, it
+ // is not used implicitly by any volumes, even if it's the
+ // only storage class.
+ var logbuf bytes.Buffer
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ StorageClasses:
+ default:
+ Default: true
+ Priority: 111
+ Volumes:
+ z: {}`, &logbuf)
+ _, err = ldr.Load()
+ c.Assert(err, check.ErrorMatches, `z: volume has no StorageClasses listed`)
+
+ // If StorageClasses are configured explicitly, there must be
+ // at least one with Default: true. (Calling one "default" is
+ // not sufficient.)
+ ldr = testLoader(c, `
+Clusters:
+ z1111:
+ StorageClasses:
+ default:
+ Priority: 111
+ Volumes:
+ z:
+ StorageClasses:
+ default: true`, nil)
+ _, err = ldr.Load()
+ c.Assert(err, check.ErrorMatches, `there is no default storage class.*`)
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 9e7eb521e..cc1de1be4 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -238,8 +238,9 @@ type Cluster struct {
PreferDomainForUsername string
UserSetupMailText string
}
- Volumes map[string]Volume
- Workbench struct {
+ StorageClasses map[string]StorageClassConfig
+ Volumes map[string]Volume
+ Workbench struct {
ActivationContactLink string
APIClientConnectTimeout Duration
APIClientReceiveTimeout Duration
@@ -281,6 +282,11 @@ type Cluster struct {
}
}
+type StorageClassConfig struct {
+ Default bool
+ Priority int
+}
+
type Volume struct {
AccessViaHosts map[URL]VolumeAccess
ReadOnly bool
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list