[ARVADOS] created: 1.3.0-718-gb728f8600
Git user
git at public.curoverse.com
Thu Apr 18 21:06:55 UTC 2019
at b728f8600ad4e1e1a99803c0bd2876690ebfe8db (commit)
commit b728f8600ad4e1e1a99803c0bd2876690ebfe8db
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Apr 18 17:05:57 2019 -0400
15003: Move keys from RequestLimits to API section.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 0a60c25b5..39dcb4fe6 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -28,6 +28,19 @@ func (s *CommandSuite) TestEmptyInput(c *check.C) {
c.Check(stderr.String(), check.Matches, `config does not define any clusters\n`)
}
+func (s *CommandSuite) TestLogDeprecatedKeys(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ RequestLimits:
+ MaxItemsPerResponse: 1234
+`
+ code := DumpCommand.RunCommand("arvados dump-config", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Check(code, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, `(?ms).*overriding Clusters.z1234.API.MaxItemsPerResponse .* = 1234.*`)
+}
+
func (s *CommandSuite) TestUnknownKey(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
@@ -38,6 +51,7 @@ Clusters:
`
code := DumpCommand.RunCommand("arvados dump-config", nil, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
+ c.Check(stderr.String(), check.Equals, "")
c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n z1234:\n.*`)
c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index a24a9055f..32cf983ee 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -76,7 +76,7 @@ Clusters:
# All parameters here are passed to the PG client library in a connection string;
# see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
Host: ""
- Port: 0
+ Port: "0"
User: ""
Password: ""
DBName: ""
@@ -115,6 +115,10 @@ Clusters:
# update on the permission view in the future, if not already scheduled.
AsyncPermissionsUpdateInterval: 20
+ # Maximum number of concurrent outgoing requests to make while
+ # serving a single incoming multi-cluster (federated) request.
+ MaxRequestAmplification: 4
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
diff --git a/lib/config/load.go b/lib/config/load.go
index 159dd65dc..5a690b03c 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -6,6 +6,7 @@ package config
import (
"bytes"
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -15,16 +16,25 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/ghodss/yaml"
+ "github.com/imdario/mergo"
)
type logger interface {
Warnf(string, ...interface{})
}
+type deprRequestLimits struct {
+ MaxItemsPerResponse *int
+ MultiClusterRequestConcurrency *int
+}
+
+type deprCluster struct {
+ RequestLimits deprRequestLimits
+ NodeProfiles map[string]arvados.NodeProfile
+}
+
type deprecatedConfig struct {
- Clusters map[string]struct {
- NodeProfiles map[string]arvados.NodeProfile
- }
+ Clusters map[string]deprCluster
}
func LoadFile(path string, log logger) (*arvados.Config, error) {
@@ -57,18 +67,41 @@ func Load(rdr io.Reader, log logger) (*arvados.Config, error) {
if len(dummy.Clusters) == 0 {
return nil, errors.New("config does not define any clusters")
}
+
+ // We can't merge deep structs here; instead, we unmarshal the
+ // default & loaded config files into generic maps, merge
+ // those, and then json-encode+decode the result into the
+ // config struct type.
+ var merged map[string]interface{}
for id := range dummy.Clusters {
- err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte("xxxxx"), []byte(id), -1), &cfg)
+ var src map[string]interface{}
+ err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte(" xxxxx:"), []byte(" "+id+":"), -1), &src)
if err != nil {
return nil, fmt.Errorf("loading defaults for %s: %s", id, err)
}
+ mergo.Merge(&merged, src, mergo.WithOverride)
}
- err = yaml.Unmarshal(buf, &cfg)
+ var src map[string]interface{}
+ err = yaml.Unmarshal(buf, &src)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("loading config data: %s", err)
+ }
+ mergo.Merge(&merged, src, mergo.WithOverride)
+
+ var errEnc error
+ pr, pw := io.Pipe()
+ go func() {
+ errEnc = json.NewEncoder(pw).Encode(merged)
+ pw.Close()
+ }()
+ err = json.NewDecoder(pr).Decode(&cfg)
+ if errEnc != nil {
+ err = errEnc
+ }
+ if err != nil {
+ return nil, fmt.Errorf("transcoding config data: %s", err)
}
- // Check for deprecated config values, and apply them to cfg.
var dc deprecatedConfig
err = yaml.Unmarshal(buf, &dc)
if err != nil {
@@ -98,6 +131,14 @@ func applyDeprecatedConfig(cfg *arvados.Config, dc *deprecatedConfig, log logger
applyDeprecatedNodeProfile(hostname, np.DispatchCloud, &cluster.Services.DispatchCloud)
}
}
+ if dst, n := &cluster.API.MaxItemsPerResponse, dcluster.RequestLimits.MaxItemsPerResponse; n != nil && *n != *dst {
+ log.Warnf("overriding Clusters.%s.API.MaxItemsPerResponse with deprecated config RequestLimits.MultiClusterRequestConcurrency = %d", id, *n)
+ *dst = *n
+ }
+ if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
+ log.Warnf("overriding Clusters.%s.API.MaxRequestAmplification with deprecated config RequestLimits.MultiClusterRequestConcurrency = %d", id, *n)
+ *dst = *n
+ }
cfg.Clusters[id] = cluster
}
return nil
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index f00ce33fd..277ff423a 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -38,6 +38,8 @@ func (s *LoadSuite) TestNoConfigs(c *check.C) {
cc, err := cfg.GetCluster("z1111")
c.Assert(err, check.IsNil)
c.Check(cc.ClusterID, check.Equals, "z1111")
+ c.Check(cc.API.MaxRequestAmplification, check.Equals, 4)
+ c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
}
func (s *LoadSuite) TestMultipleClusters(c *check.C) {
@@ -91,14 +93,46 @@ Clusters:
`)
}
+func (s *LoadSuite) TestMovedKeys(c *check.C) {
+ s.checkEquivalent(c, `# config has old keys only
+Clusters:
+ zzzzz:
+ RequestLimits:
+ MultiClusterRequestConcurrency: 3
+ MaxItemsPerResponse: 999
+`, `
+Clusters:
+ zzzzz:
+ API:
+ MaxRequestAmplification: 3
+ MaxItemsPerResponse: 999
+`)
+ s.checkEquivalent(c, `# config has both old and new keys; old values win
+Clusters:
+ zzzzz:
+ RequestLimits:
+ MultiClusterRequestConcurrency: 0
+ MaxItemsPerResponse: 555
+ API:
+ MaxRequestAmplification: 3
+ MaxItemsPerResponse: 999
+`, `
+Clusters:
+ zzzzz:
+ API:
+ MaxRequestAmplification: 0
+ MaxItemsPerResponse: 555
+`)
+}
+
func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
got, err := Load(bytes.NewBufferString(goty), ctxlog.TestLogger(c))
c.Assert(err, check.IsNil)
expected, err := Load(bytes.NewBufferString(expectedy), ctxlog.TestLogger(c))
c.Assert(err, check.IsNil)
if !c.Check(got, check.DeepEquals, expected) {
- cmd := exec.Command("diff", "-u", "--label", "got", "--label", "expected", "/dev/fd/3", "/dev/fd/4")
- for _, obj := range []interface{}{got, expected} {
+ cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
+ for _, obj := range []interface{}{expected, got} {
y, _ := yaml.Marshal(obj)
pr, pw, err := os.Pipe()
c.Assert(err, check.IsNil)
diff --git a/lib/controller/fed_collections.go b/lib/controller/fed_collections.go
index ab49e39d1..07daf2f90 100644
--- a/lib/controller/fed_collections.go
+++ b/lib/controller/fed_collections.go
@@ -217,17 +217,15 @@ func fetchRemoteCollectionByPDH(
// returned to the client. When that happens, all
// other outstanding requests are cancelled
sharedContext, cancelFunc := context.WithCancel(req.Context())
+ defer cancelFunc()
+
req = req.WithContext(sharedContext)
wg := sync.WaitGroup{}
pdh := m[1]
success := make(chan *http.Response)
errorChan := make(chan error, len(h.handler.Cluster.RemoteClusters))
- // use channel as a semaphore to limit the number of concurrent
- // requests at a time
- sem := make(chan bool, h.handler.Cluster.RequestLimits.GetMultiClusterRequestConcurrency())
-
- defer cancelFunc()
+ acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
for remoteID := range h.handler.Cluster.RemoteClusters {
if remoteID == h.handler.Cluster.ClusterID {
@@ -238,9 +236,8 @@ func fetchRemoteCollectionByPDH(
wg.Add(1)
go func(remote string) {
defer wg.Done()
- // blocks until it can put a value into the
- // channel (which has a max queue capacity)
- sem <- true
+ acquire()
+ defer release()
select {
case <-sharedContext.Done():
return
@@ -278,7 +275,6 @@ func fetchRemoteCollectionByPDH(
case success <- newResponse:
wasSuccess = true
}
- <-sem
}(remoteID)
}
go func() {
diff --git a/lib/controller/fed_generic.go b/lib/controller/fed_generic.go
index 9c8b1614b..fd2fbc226 100644
--- a/lib/controller/fed_generic.go
+++ b/lib/controller/fed_generic.go
@@ -175,9 +175,9 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response
httpserver.Error(w, "Federated multi-object may not provide 'limit', 'offset' or 'order'.", http.StatusBadRequest)
return true
}
- if expectCount > h.handler.Cluster.RequestLimits.GetMaxItemsPerResponse() {
+ if max := h.handler.Cluster.API.MaxItemsPerResponse; expectCount > max {
httpserver.Error(w, fmt.Sprintf("Federated multi-object request for %v objects which is more than max page size %v.",
- expectCount, h.handler.Cluster.RequestLimits.GetMaxItemsPerResponse()), http.StatusBadRequest)
+ expectCount, max), http.StatusBadRequest)
return true
}
if req.Form.Get("select") != "" {
@@ -203,10 +203,7 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response
// Perform concurrent requests to each cluster
- // use channel as a semaphore to limit the number of concurrent
- // requests at a time
- sem := make(chan bool, h.handler.Cluster.RequestLimits.GetMultiClusterRequestConcurrency())
- defer close(sem)
+ acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
wg := sync.WaitGroup{}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
@@ -220,23 +217,20 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response
// Nothing to query
continue
}
-
- // blocks until it can put a value into the
- // channel (which has a max queue capacity)
- sem <- true
+ acquire()
wg.Add(1)
go func(k string, v []string) {
+ defer release()
+ defer wg.Done()
rp, kn, err := h.remoteQueryUUIDs(w, req, k, v)
mtx.Lock()
+ defer mtx.Unlock()
if err == nil {
completeResponses = append(completeResponses, rp...)
kind = kn
} else {
errors = append(errors, err)
}
- mtx.Unlock()
- wg.Done()
- <-sem
}(k, v)
}
wg.Wait()
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 62916acd2..c4aa33c15 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -64,9 +64,9 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
NodeProfiles: map[string]arvados.NodeProfile{
"*": nodeProfile,
},
- RequestLimits: arvados.RequestLimits{
- MaxItemsPerResponse: 1000,
- MultiClusterRequestConcurrency: 4,
+ API: arvados.API{
+ MaxItemsPerResponse: 1000,
+ MaxRequestAmplification: 4,
},
}, NodeProfile: &nodeProfile}
s.testServer = newServerFromIntegrationTestEnv(c)
@@ -850,7 +850,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
}
func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
- s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
+ s.testHandler.Cluster.API.MaxItemsPerResponse = 1
req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
arvadostest.QueuedContainerUUID))),
diff --git a/lib/controller/semaphore.go b/lib/controller/semaphore.go
new file mode 100644
index 000000000..ff607bbb5
--- /dev/null
+++ b/lib/controller/semaphore.go
@@ -0,0 +1,14 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+func semaphore(max int) (acquire, release func()) {
+ if max > 0 {
+ ch := make(chan bool, max)
+ return func() { ch <- true }, func() { <-ch }
+ } else {
+ return func() {}, func() {}
+ }
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 32763acd1..610a3d288 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -51,9 +51,9 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
}
}
-type RequestLimits struct {
- MaxItemsPerResponse int
- MultiClusterRequestConcurrency int
+type API struct {
+ MaxItemsPerResponse int
+ MaxRequestAmplification int
}
type Cluster struct {
@@ -68,7 +68,7 @@ type Cluster struct {
HTTPRequestTimeout Duration
RemoteClusters map[string]RemoteCluster
PostgreSQL PostgreSQL
- RequestLimits RequestLimits
+ API API
Logging Logging
TLS TLS
}
@@ -332,20 +332,6 @@ func (np *NodeProfile) ServicePorts() map[ServiceName]string {
}
}
-func (h RequestLimits) GetMultiClusterRequestConcurrency() int {
- if h.MultiClusterRequestConcurrency == 0 {
- return 4
- }
- return h.MultiClusterRequestConcurrency
-}
-
-func (h RequestLimits) GetMaxItemsPerResponse() int {
- if h.MaxItemsPerResponse == 0 {
- return 1000
- }
- return h.MaxItemsPerResponse
-}
-
type SystemServiceInstance struct {
Listen string
TLS bool
commit 7277ba40c037ac4e9c31d2781de31ab50179a226
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Apr 16 16:12:12 2019 -0400
15003: Use config lib for arvados-server.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/load.go b/lib/config/load.go
index e16a6e4b7..159dd65dc 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -27,6 +27,15 @@ type deprecatedConfig struct {
}
}
+func LoadFile(path string, log logger) (*arvados.Config, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return Load(f, log)
+}
+
func Load(rdr io.Reader, log logger) (*arvados.Config, error) {
var cfg arvados.Config
buf, err := ioutil.ReadAll(rdr)
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index e853da943..4b7341d72 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -15,6 +15,7 @@ import (
"os"
"git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -69,7 +70,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
} else if err != nil {
return 2
}
- cfg, err := arvados.GetConfig(*configFile)
+ cfg, err := config.LoadFile(*configFile, log)
if err != nil {
return 1
}
commit 3a93489fe9ed1269ec2ac6491b4729822cfa853b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Apr 16 15:33:33 2019 -0400
15003: Add 'arvados-server dump-config' command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index cd15d25dd..cad540c91 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -8,6 +8,7 @@ import (
"os"
"git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/lib/controller"
"git.curoverse.com/arvados.git/lib/dispatchcloud"
)
@@ -21,6 +22,7 @@ var (
"controller": controller.Command,
"dispatch-cloud": dispatchcloud.Command,
+ "dump-config": config.DumpCommand,
})
)
diff --git a/lib/config/cmd.go b/lib/config/cmd.go
new file mode 100644
index 000000000..004121a35
--- /dev/null
+++ b/lib/config/cmd.go
@@ -0,0 +1,45 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "fmt"
+ "io"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/ghodss/yaml"
+)
+
+var DumpCommand cmd.Handler = dumpCommand{}
+
+type dumpCommand struct{}
+
+func (dumpCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ defer func() {
+ if err != nil {
+ fmt.Fprintf(stderr, "%s\n", err)
+ }
+ }()
+ if len(args) != 0 {
+ err = fmt.Errorf("usage: %s <config-src.yaml >config-min.json", prog)
+ return 2
+ }
+ log := ctxlog.New(stderr, "text", "info")
+ cfg, err := Load(stdin, log)
+ if err != nil {
+ return 1
+ }
+ out, err := yaml.Marshal(cfg)
+ if err != nil {
+ return 1
+ }
+ _, err = stdout.Write(out)
+ if err != nil {
+ return 1
+ }
+ return 0
+}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
new file mode 100644
index 000000000..0a60c25b5
--- /dev/null
+++ b/lib/config/cmd_test.go
@@ -0,0 +1,44 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (s *CommandSuite) TestBadArg(c *check.C) {
+ var stderr bytes.Buffer
+ code := DumpCommand.RunCommand("arvados dump-config", []string{"-badarg"}, bytes.NewBuffer(nil), bytes.NewBuffer(nil), &stderr)
+ c.Check(code, check.Equals, 2)
+ c.Check(stderr.String(), check.Matches, `(?ms)usage: .*`)
+}
+
+func (s *CommandSuite) TestEmptyInput(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ code := DumpCommand.RunCommand("arvados dump-config", nil, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(code, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `config does not define any clusters\n`)
+}
+
+func (s *CommandSuite) TestUnknownKey(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ in := `
+Clusters:
+ z1234:
+ UnknownKey: foobar
+ ManagementToken: secret
+`
+ code := DumpCommand.RunCommand("arvados dump-config", nil, bytes.NewBufferString(in), &stdout, &stderr)
+ c.Check(code, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n z1234:\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
+ c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
+}
commit cb7665df39395ba353a1090c2d325dac4bd16344
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Apr 16 14:05:53 2019 -0400
15003: Load config over defaults.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/load.go b/lib/config/load.go
new file mode 100644
index 000000000..e16a6e4b7
--- /dev/null
+++ b/lib/config/load.go
@@ -0,0 +1,113 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/ghodss/yaml"
+)
+
+type logger interface {
+ Warnf(string, ...interface{})
+}
+
+type deprecatedConfig struct {
+ Clusters map[string]struct {
+ NodeProfiles map[string]arvados.NodeProfile
+ }
+}
+
+func Load(rdr io.Reader, log logger) (*arvados.Config, error) {
+ var cfg arvados.Config
+ buf, err := ioutil.ReadAll(rdr)
+ if err != nil {
+ return nil, err
+ }
+
+ // Load the config into a dummy map to get the cluster ID
+ // keys, discarding the values; then set up defaults for each
+ // cluster ID; then load the real config on top of the
+ // defaults.
+ var dummy struct {
+ Clusters map[string]struct{}
+ }
+ err = yaml.Unmarshal(buf, &dummy)
+ if err != nil {
+ return nil, err
+ }
+ if len(dummy.Clusters) == 0 {
+ return nil, errors.New("config does not define any clusters")
+ }
+ for id := range dummy.Clusters {
+ err = yaml.Unmarshal(bytes.Replace(DefaultYAML, []byte("xxxxx"), []byte(id), -1), &cfg)
+ if err != nil {
+ return nil, fmt.Errorf("loading defaults for %s: %s", id, err)
+ }
+ }
+ err = yaml.Unmarshal(buf, &cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ // Check for deprecated config values, and apply them to cfg.
+ var dc deprecatedConfig
+ err = yaml.Unmarshal(buf, &dc)
+ if err != nil {
+ return nil, err
+ }
+ err = applyDeprecatedConfig(&cfg, &dc, log)
+ if err != nil {
+ return nil, err
+ }
+ return &cfg, nil
+}
+
+func applyDeprecatedConfig(cfg *arvados.Config, dc *deprecatedConfig, log logger) error {
+ hostname, err := os.Hostname()
+ if err != nil {
+ return err
+ }
+ for id, dcluster := range dc.Clusters {
+ cluster, ok := cfg.Clusters[id]
+ if !ok {
+ return fmt.Errorf("can't load legacy config %q that is not present in current config", id)
+ }
+ for name, np := range dcluster.NodeProfiles {
+ if name == "*" || name == os.Getenv("ARVADOS_NODE_PROFILE") || name == hostname {
+ applyDeprecatedNodeProfile(hostname, np.RailsAPI, &cluster.Services.RailsAPI)
+ applyDeprecatedNodeProfile(hostname, np.Controller, &cluster.Services.Controller)
+ applyDeprecatedNodeProfile(hostname, np.DispatchCloud, &cluster.Services.DispatchCloud)
+ }
+ }
+ cfg.Clusters[id] = cluster
+ }
+ return nil
+}
+
+func applyDeprecatedNodeProfile(hostname string, ssi arvados.SystemServiceInstance, svc *arvados.Service) {
+ scheme := "https"
+ if !ssi.TLS {
+ scheme = "http"
+ }
+ if svc.InternalURLs == nil {
+ svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
+ }
+ host := ssi.Listen
+ if host == "" {
+ return
+ }
+ if strings.HasPrefix(host, ":") {
+ host = hostname + host
+ }
+ svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+}
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
new file mode 100644
index 000000000..f00ce33fd
--- /dev/null
+++ b/lib/config/load_test.go
@@ -0,0 +1,116 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "os/exec"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/ghodss/yaml"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&LoadSuite{})
+
+type LoadSuite struct{}
+
+func (s *LoadSuite) TestEmpty(c *check.C) {
+ cfg, err := Load(&bytes.Buffer{}, ctxlog.TestLogger(c))
+ c.Check(cfg, check.IsNil)
+ c.Assert(err, check.ErrorMatches, `config does not define any clusters`)
+}
+
+func (s *LoadSuite) TestNoConfigs(c *check.C) {
+ cfg, err := Load(bytes.NewBufferString(`Clusters: {"z1111": {}}`), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ c.Assert(cfg.Clusters, check.HasLen, 1)
+ cc, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(cc.ClusterID, check.Equals, "z1111")
+}
+
+func (s *LoadSuite) TestMultipleClusters(c *check.C) {
+ cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"z1111":{},"z2222":{}}}`), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ c1, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
+ c.Check(c1.ClusterID, check.Equals, "z1111")
+ c2, err := cfg.GetCluster("z2222")
+ c.Assert(err, check.IsNil)
+ c.Check(c2.ClusterID, check.Equals, "z2222")
+}
+
+func (s *LoadSuite) TestNodeProfilesToServices(c *check.C) {
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ s.checkEquivalent(c, `
+Clusters:
+ z1111:
+ NodeProfiles:
+ "*":
+ arvados-dispatch-cloud:
+ listen: ":9006"
+ arvados-controller:
+ listen: ":9004"
+ `+hostname+`:
+ arvados-api-server:
+ listen: ":8000"
+`, `
+Clusters:
+ z1111:
+ Services:
+ RailsAPI:
+ InternalURLs:
+ "http://`+hostname+`:8000": {}
+ Controller:
+ InternalURLs:
+ "http://`+hostname+`:9004": {}
+ DispatchCloud:
+ InternalURLs:
+ "http://`+hostname+`:9006": {}
+ NodeProfiles:
+ "*":
+ arvados-dispatch-cloud:
+ listen: ":9006"
+ arvados-controller:
+ listen: ":9004"
+ `+hostname+`:
+ arvados-api-server:
+ listen: ":8000"
+`)
+}
+
+func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
+ got, err := Load(bytes.NewBufferString(goty), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ expected, err := Load(bytes.NewBufferString(expectedy), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ if !c.Check(got, check.DeepEquals, expected) {
+ cmd := exec.Command("diff", "-u", "--label", "got", "--label", "expected", "/dev/fd/3", "/dev/fd/4")
+ for _, obj := range []interface{}{got, expected} {
+ y, _ := yaml.Marshal(obj)
+ pr, pw, err := os.Pipe()
+ c.Assert(err, check.IsNil)
+ defer pr.Close()
+ go func() {
+ io.Copy(pw, bytes.NewBuffer(y))
+ pw.Close()
+ }()
+ cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
+ }
+ diff, err := cmd.CombinedOutput()
+ c.Log(string(diff))
+ c.Check(err, check.IsNil)
+ }
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 2965d5ecb..32763acd1 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -105,6 +105,10 @@ func (su *URL) UnmarshalText(text []byte) error {
return err
}
+func (su URL) MarshalText() ([]byte, error) {
+ return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
+}
+
type ServiceInstance struct{}
type Logging struct {
commit b03de60c4ed9098b60f4f453d7668074cb82d178
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 12 16:04:37 2019 -0400
15003: Generate config.DefaultYAML from source file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index dbf13940b..d7ecc2fcc 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -986,6 +986,7 @@ gostuff=(
lib/cloud
lib/cloud/azure
lib/cloud/ec2
+ lib/config
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
diff --git a/lib/config/generate.go b/lib/config/generate.go
new file mode 100644
index 000000000..c192d7bb2
--- /dev/null
+++ b/lib/config/generate.go
@@ -0,0 +1,72 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// +build ignore
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+)
+
+func main() {
+ err := generate()
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func generate() error {
+ outfn := "generated_config.go"
+ tmpfile, err := ioutil.TempFile(".", "."+outfn+".")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmpfile.Name())
+
+ gofmt := exec.Command("gofmt", "-s")
+ gofmt.Stdout = tmpfile
+ gofmt.Stderr = os.Stderr
+ w, err := gofmt.StdinPipe()
+ if err != nil {
+ return err
+ }
+ gofmt.Start()
+
+ // copyright header: same as this file
+ cmd := exec.Command("head", "-n", "4", "generate.go")
+ cmd.Stdout = w
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ if err != nil {
+ return err
+ }
+
+ data, err := ioutil.ReadFile("config.default.yml")
+ if err != nil {
+ return err
+ }
+ _, err = fmt.Fprintf(w, "package config\nvar DefaultYAML = []byte(`%s`)", bytes.Replace(data, []byte{'`'}, []byte("`+\"`\"+`"), -1))
+ if err != nil {
+ return err
+ }
+ err = w.Close()
+ if err != nil {
+ return err
+ }
+ err = gofmt.Wait()
+ if err != nil {
+ return err
+ }
+ err = tmpfile.Close()
+ if err != nil {
+ return err
+ }
+ return os.Rename(tmpfile.Name(), outfn)
+}
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
new file mode 100644
index 000000000..a24a9055f
--- /dev/null
+++ b/lib/config/generated_config.go
@@ -0,0 +1,457 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+var DefaultYAML = []byte(`# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Do not use this file for site configuration. Create
+# /etc/arvados/config.yml instead.
+#
+# The order of precedence (highest to lowest):
+# 1. Legacy component-specific config files (deprecated)
+# 2. /etc/arvados/config.yml
+# 3. config.default.yml
+
+Clusters:
+ xxxxx:
+ SystemRootToken: ""
+
+ # Token to be included in all healthcheck requests. Disabled by default.
+ # Server expects request header of the format "Authorization: Bearer xxx"
+ ManagementToken: ""
+
+ Services:
+ RailsAPI:
+ InternalURLs: {}
+ GitHTTP:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepstore:
+ InternalURLs: {}
+ Controller:
+ InternalURLs: {}
+ ExternalURL: ""
+ Websocket:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepbalance:
+ InternalURLs: {}
+ GitHTTP:
+ InternalURLs: {}
+ ExternalURL: ""
+ GitSSH:
+ ExternalURL: ""
+ DispatchCloud:
+ InternalURLs: {}
+ SSO:
+ ExternalURL: ""
+ Keepproxy:
+ InternalURLs: {}
+ ExternalURL: ""
+ WebDAV:
+ InternalURLs: {}
+ ExternalURL: ""
+ WebDAVDownload:
+ InternalURLs: {}
+ ExternalURL: ""
+ Keepstore:
+ InternalURLs: {}
+ Composer:
+ ExternalURL: ""
+ WebShell:
+ ExternalURL: ""
+ Workbench1:
+ InternalURLs: {}
+ ExternalURL: ""
+ Workbench2:
+ ExternalURL: ""
+ PostgreSQL:
+ # max concurrent connections per arvados server daemon
+ ConnectionPool: 32
+ Connection:
+ # All parameters here are passed to the PG client library in a connection string;
+ # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+ Host: ""
+ Port: 0
+ User: ""
+ Password: ""
+ DBName: ""
+ API:
+ # Maximum size (in bytes) allowed for a single API request. This
+ # limit is published in the discovery document for use by clients.
+ # Note: You must separately configure the upstream web server or
+ # proxy to actually enforce the desired maximum request size on the
+ # server side.
+ MaxRequestSize: 134217728
+
+ # Limit the number of bytes read from the database during an index
+ # request (by retrieving and returning fewer rows than would
+ # normally be returned in a single response).
+ # Note 1: This setting never reduces the number of returned rows to
+ # zero, no matter how big the first data row is.
+ # Note 2: Currently, this is only checked against a specific set of
+ # columns that tend to get large (collections.manifest_text,
+ # containers.mounts, workflows.definition). Other fields (e.g.,
+ # "properties" hashes) are not counted against this limit.
+ MaxIndexDatabaseRead: 134217728
+
+ # Maximum number of items to return when responding to a APIs that
+ # can return partial result sets using limit and offset parameters
+ # (e.g., *.index, groups.contents). If a request specifies a "limit"
+ # parameter higher than this value, this value is used instead.
+ MaxItemsPerResponse: 1000
+
+ # API methods to disable. Disabled methods are not listed in the
+ # discovery document, and respond 404 to all requests.
+ # Example: ["jobs.create", "pipeline_instances.create"]
+ DisabledAPIs: []
+
+ # Interval (seconds) between asynchronous permission view updates. Any
+ # permission-updating API called with the 'async' parameter schedules a an
+ # update on the permission view in the future, if not already scheduled.
+ AsyncPermissionsUpdateInterval: 20
+
+ Users:
+ # Config parameters to automatically setup new users. If enabled,
+ # this users will be able to self-activate. Enable this if you want
+ # to run an open instance where anyone can create an account and use
+ # the system without requiring manual approval.
+ #
+ # The params auto_setup_new_users_with_* are meaningful only when auto_setup_new_users is turned on.
+ # auto_setup_name_blacklist is a list of usernames to be blacklisted for auto setup.
+ AutoSetupNewUsers: false
+ AutoSetupNewUsersWithVmUUID: ""
+ AutoSetupNewUsersWithRepository: false
+ AutoSetupUsernameBlacklist: [arvados, git, gitolite, gitolite-admin, root, syslog]
+
+ # When new_users_are_active is set to true, new users will be active
+ # immediately. This skips the "self-activate" step which enforces
+ # user agreements. Should only be enabled for development.
+ NewUsersAreActive: false
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses OAuth2 against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ AutoAdminUserWithEmail: ""
+
+ # If auto_admin_first_user is set to true, the first user to log in when no
+ # other admin users exist will automatically become an admin user.
+ AutoAdminFirstUser: false
+
+ # Email address to notify whenever a user creates a profile for the
+ # first time
+ UserProfileNotificationAddress: ""
+ AdminNotifierEmailFrom: arvados at example.com
+ EmailSubjectPrefix: "[ARVADOS] "
+ UserNotifierEmailFrom: arvados at example.com
+ NewUserNotificationRecipients: []
+ NewInactiveUserNotificationRecipients: []
+
+ AuditLogs:
+ # Time to keep audit logs, in seconds. (An audit log is a row added
+ # to the "logs" table in the PostgreSQL database each time an
+ # Arvados object is created, modified, or deleted.)
+ #
+ # Currently, websocket event notifications rely on audit logs, so
+ # this should not be set lower than 600 (5 minutes).
+ MaxAge: 1209600
+
+ # Maximum number of log rows to delete in a single SQL transaction.
+ #
+ # If max_audit_log_delete_batch is 0, log entries will never be
+ # deleted by Arvados. Cleanup can be done by an external process
+ # without affecting any Arvados system processes, as long as very
+ # recent (<5 minutes old) logs are not deleted.
+ #
+ # 100000 is a reasonable batch size for most sites.
+ MaxDeleteBatch: 0
+
+ # Attributes to suppress in events and audit logs. Notably,
+ # specifying ["manifest_text"] here typically makes the database
+ # smaller and faster.
+ #
+ # Warning: Using any non-empty value here can have undesirable side
+ # effects for any client or component that relies on event logs.
+ # Use at your own risk.
+ UnloggedAttributes: []
+
+ SystemLogs:
+ # Maximum characters of (JSON-encoded) query parameters to include
+ # in each request log entry. When params exceed this size, they will
+ # be JSON-encoded, truncated to this size, and logged as
+ # params_truncated.
+ MaxRequestLogParamsSize: 2000
+
+ Collections:
+ # Allow clients to create collections by providing a manifest with
+ # unsigned data blob locators. IMPORTANT: This effectively disables
+ # access controls for data stored in Keep: a client who knows a hash
+ # can write a manifest that references the hash, pass it to
+ # collections.create (which will create a permission link), use
+ # collections.get to obtain a signature for that data locator, and
+ # use that signed locator to retrieve the data from Keep. Therefore,
+ # do not turn this on if your users expect to keep data private from
+ # one another!
+ BlobSigning: true
+
+ # blob_signing_key is a string of alphanumeric characters used to
+ # generate permission signatures for Keep locators. It must be
+ # identical to the permission key given to Keep. IMPORTANT: This is
+ # a site secret. It should be at least 50 characters.
+ #
+ # Modifying blob_signing_key will invalidate all existing
+ # signatures, which can cause programs to fail (e.g., arv-put,
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
+ # no such processes are running.
+ BlobSigningKey: ""
+
+ # Default replication level for collections. This is used when a
+ # collection's replication_desired attribute is nil.
+ DefaultReplication: 2
+
+ # Lifetime (in seconds) of blob permission signatures generated by
+ # the API server. This determines how long a client can take (after
+ # retrieving a collection record) to retrieve the collection data
+ # from Keep. If the client needs more time than that (assuming the
+ # collection still has the same content and the relevant user/token
+ # still has permission) the client can retrieve the collection again
+ # to get fresh signatures.
+ #
+ # This must be exactly equal to the -blob-signature-ttl flag used by
+ # keepstore servers. Otherwise, reading data blocks and saving
+ # collections will fail with HTTP 403 permission errors.
+ #
+ # Modifying blob_signature_ttl invalidates existing signatures; see
+ # blob_signing_key note above.
+ #
+ # The default is 2 weeks.
+ BlobSigningTTL: 1209600
+
+ # Default lifetime for ephemeral collections: 2 weeks. This must not
+ # be less than blob_signature_ttl.
+ DefaultTrashLifetime: 1209600
+
+ # Interval (seconds) between trash sweeps. During a trash sweep,
+ # collections are marked as trash if their trash_at time has
+ # arrived, and deleted if their delete_at time has arrived.
+ TrashSweepInterval: 60
+
+ # If true, enable collection versioning.
+ # When a collection's preserve_version field is true or the current version
+ # is older than the amount of seconds defined on preserve_version_if_idle,
+ # a snapshot of the collection's previous state is created and linked to
+ # the current collection.
+ CollectionVersioning: false
+
+ # 0 = auto-create a new version on every update.
+ # -1 = never auto-create new versions.
+ # > 0 = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1
+
+ Login:
+ # These settings are provided by your OAuth2 provider (e.g.,
+ # sso-provider).
+ ProviderAppSecret: ""
+ ProviderAppID: ""
+
+ Git:
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ Repositories: /var/lib/arvados/git/repositories
+
+ TLS:
+ Insecure: false
+
+ Containers:
+ # List of supported Docker Registry image formats that compute nodes
+ # are able to use. ` + "`" + `arv keep docker` + "`" + ` will error out if a user tries
+ # to store an image with an unsupported format. Use an empty array
+ # to skip the compatibility check (and display a warning message to
+ # that effect).
+ #
+ # Example for sites running docker < 1.10: ["v1"]
+ # Example for sites running docker >= 1.10: ["v2"]
+ # Example for disabling check: []
+ SupportedDockerImageFormats: ["v2"]
+
+ # Include details about job reuse decisions in the server log. This
+ # causes additional database queries to run, so it should not be
+ # enabled unless you expect to examine the resulting logs for
+ # troubleshooting purposes.
+ LogReuseDecisions: false
+
+ # Default value for keep_cache_ram of a container's runtime_constraints.
+ DefaultKeepCacheRAM: 268435456
+
+ # Number of times a container can be unlocked before being
+ # automatically cancelled.
+ MaxDispatchAttempts: 5
+
+ # Default value for container_count_max for container requests. This is the
+ # number of times Arvados will create a new container to satisfy a container
+ # request. If a container is cancelled it will retry a new container if
+ # container_count < container_count_max on any container requests associated
+ # with the cancelled container.
+ MaxRetryAttempts: 3
+
+ # The maximum number of compute nodes that can be in use simultaneously
+ # If this limit is reduced, any existing nodes with slot number >= new limit
+ # will not be counted against the new limit. In other words, the new limit
+ # won't be strictly enforced until those nodes with higher slot numbers
+ # go down.
+ MaxComputeVMs: 64
+
+ # Preemptible instance support (e.g. AWS Spot Instances)
+ # When true, child containers will get created with the preemptible
+ # scheduling parameter parameter set.
+ UsePreemptibleInstances: false
+
+ # Include details about job reuse decisions in the server log. This
+ # causes additional database queries to run, so it should not be
+ # enabled unless you expect to examine the resulting logs for
+ # troubleshooting purposes.
+ LogReuseDecisions: false
+
+ Logging:
+ # When you run the db:delete_old_container_logs task, it will find
+ # containers that have been finished for at least this many seconds,
+ # and delete their stdout, stderr, arv-mount, crunch-run, and
+ # crunchstat logs from the logs table.
+ MaxAge: 720h
+
+ # These two settings control how frequently log events are flushed to the
+ # database. Log lines are buffered until either crunch_log_bytes_per_event
+ # has been reached or crunch_log_seconds_between_events has elapsed since
+ # the last flush.
+ LogBytesPerEvent: 4096
+ LogSecondsBetweenEvents: 1
+
+ # The sample period for throttling logs, in seconds.
+ LogThrottlePeriod: 60
+
+ # Maximum number of bytes that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ LogThrottleBytes: 65536
+
+ # Maximum number of lines that job can log over crunch_log_throttle_period
+ # before being silenced until the end of the period.
+ LogThrottleLines: 1024
+
+ # Maximum bytes that may be logged by a single job. Log bytes that are
+ # silenced by throttling are not counted against this total.
+ LimitLogBytesPerJob: 67108864
+
+ LogPartialLineThrottlePeriod: 5
+
+ # Container logs are written to Keep and saved in a collection,
+ # which is updated periodically while the container runs. This
+ # value sets the interval (given in seconds) between collection
+ # updates.
+ LogUpdatePeriod: 1800
+
+ # The log collection is also updated when the specified amount of
+ # log data (given in bytes) is produced in less than one update
+ # period.
+ LogUpdateSize: 33554432
+
+ SLURM:
+ Managed:
+ # Path to dns server configuration directory
+ # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
+ # files or touch restart.txt (see below).
+ DNSServerConfDir: ""
+
+ # Template file for the dns server host snippets. See
+ # unbound.template in this directory for an example. If false, do
+ # not write any config files.
+ DNSServerConfTemplate: ""
+
+ # String to write to {dns_server_conf_dir}/restart.txt (with a
+ # trailing newline) after updating local data. If false, do not
+ # open or write the restart.txt file.
+ DNSServerReloadCommand: ""
+
+ # Command to run after each DNS update. Template variables will be
+ # substituted; see the "unbound" example below. If false, do not run
+ # a command.
+ DNSServerUpdateCommand: ""
+
+ ComputeNodeDomain: ""
+ ComputeNodeNameservers:
+ - 192.168.1.1
+
+ # Hostname to assign to a compute node when it sends a "ping" and the
+ # hostname in its Node record is nil.
+ # During bootstrapping, the "ping" script is expected to notice the
+ # hostname given in the ping response, and update its unix hostname
+ # accordingly.
+ # If false, leave the hostname alone (this is appropriate if your compute
+ # nodes' hostnames are already assigned by some other mechanism).
+ #
+ # One way or another, the hostnames of your node records should agree
+ # with your DNS records and your /etc/slurm-llnl/slurm.conf files.
+ #
+ # Example for compute0000, compute0001, ....:
+ # assign_node_hostname: compute%<slot_number>04d
+ # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
+ AssignNodeHostname: "compute%<slot_number>d"
+
+ JobsAPI:
+ # Enable the legacy Jobs API. This value must be a string.
+ # 'auto' -- (default) enable the Jobs API only if it has been used before
+ # (i.e., there are job records in the database)
+ # 'true' -- enable the Jobs API despite lack of existing records.
+ # 'false' -- disable the Jobs API despite presence of existing records.
+ Enable: 'auto'
+
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ GitInternalDir: /var/lib/arvados/internal.git
+
+ # Docker image to be used when none found in runtime_constraints of a job
+ DefaultDockerImage: ""
+
+ # none or slurm_immediate
+ CrunchJobWrapper: none
+
+ # username, or false = do not set uid when running jobs.
+ CrunchJobUser: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ CrunchRefreshTrigger: /tmp/crunch_refresh_trigger
+
+ # Control job reuse behavior when two completed jobs match the
+ # search criteria and have different outputs.
+ #
+ # If true, in case of a conflict, reuse the earliest job (this is
+ # similar to container reuse behavior).
+ #
+ # If false, in case of a conflict, do not reuse any completed job,
+ # but do reuse an already-running job if available (this is the
+ # original job reuse behavior, and is still the default).
+ ReuseJobIfOutputsDiffer: false
+
+ Mail:
+ MailchimpAPIKey: ""
+ MailchimpListID: ""
+ SendUserSetupNotificationEmail: ""
+ IssueReporterEmailFrom: ""
+ IssueReporterEmailTo: ""
+ SupportEmailAddress: ""
+ EmailFrom: ""
+ RemoteClusters:
+ "*":
+ Proxy: false
+ ActivateUsers: false
+`)
diff --git a/lib/config/noop.go b/lib/config/noop.go
new file mode 100644
index 000000000..71bdba710
--- /dev/null
+++ b/lib/config/noop.go
@@ -0,0 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+//go:generate go run generate.go
diff --git a/lib/config/uptodate_test.go b/lib/config/uptodate_test.go
new file mode 100644
index 000000000..10551f81e
--- /dev/null
+++ b/lib/config/uptodate_test.go
@@ -0,0 +1,22 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "io/ioutil"
+ "testing"
+)
+
+func TestUpToDate(t *testing.T) {
+ src := "config.default.yml"
+ srcdata, err := ioutil.ReadFile(src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(srcdata, DefaultYAML) {
+ t.Fatalf("content of %s differs from DefaultYAML -- you need to run 'go generate' and commit", src)
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list