[ARVADOS] updated: 1.3.0-1334-gd66a6a910
Git user
git at public.curoverse.com
Mon Jul 22 18:15:01 UTC 2019
Summary of changes:
lib/config/config.default.yml | 6 +++-
lib/config/deprecated.go | 19 +++++------
lib/config/export.go | 2 +-
lib/config/generated_config.go | 6 +++-
lib/config/load.go | 38 +++++++++++++++-------
sdk/go/arvados/config.go | 2 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 1 +
services/ws/main.go | 1 +
services/ws/router.go | 2 +-
services/ws/server_test.go | 3 +-
10 files changed, 52 insertions(+), 28 deletions(-)
via d66a6a910fc37cd4452a62ac53052224c376b573 (commit)
via 67fe327427f9949b2c2db7021e4112a4685f7ac5 (commit)
from 7388e2f812805496e67930fdd316bb19657cb9fa (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d66a6a910fc37cd4452a62ac53052224c376b573
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Jul 22 14:14:39 2019 -0400
14717: Rename WebsocketKeepaliveTimeout to SendTimeout and add a comment
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 1a1f1c5ac..595b05f8c 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -198,7 +198,11 @@ Clusters:
# Maximum wall clock time to spend handling an incoming request.
RequestTimeout: 5m
- WebsocketKeepaliveTimeout: 60s
+ # Websocket will send a periodic empty event after 'SendTimeout'
+ # if there is no other activity to maintain the connection /
+ # detect dropped connections.
+ SendTimeout: 60s
+
WebsocketClientEventQueue: 64
WebsocketServerEventQueue: 4
diff --git a/lib/config/export.go b/lib/config/export.go
index 1cb84a2a1..25d1b7a2c 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -65,7 +65,7 @@ var whitelist = map[string]bool{
"API.RailsSessionSecretToken": false,
"API.RequestTimeout": true,
"API.WebsocketClientEventQueue": false,
- "API.WebsocketKeepaliveTimeout": true,
+ "API.SendTimeout": true,
"API.WebsocketServerEventQueue": false,
"AuditLogs": false,
"AuditLogs.MaxAge": false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 7bcb38441..7704a3fae 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -204,7 +204,11 @@ Clusters:
# Maximum wall clock time to spend handling an incoming request.
RequestTimeout: 5m
- WebsocketKeepaliveTimeout: 60s
+ # Websocket will send a periodic empty event after 'SendTimeout'
+ # if there is no other activity to maintain the connection /
+ # detect dropped connections.
+ SendTimeout: 60s
+
WebsocketClientEventQueue: 64
WebsocketServerEventQueue: 4
commit 67fe327427f9949b2c2db7021e4112a4685f7ac5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Jul 22 13:59:36 2019 -0400
14717: Fix fallback behavior for component config vs main config
For backwards compatability, we need to be able to start with only a
component config and not the main config, and it is a fatal error if
it doesn't exist.
However, if there is a main config, and then it is okay if the
component config doesn't exist.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 4526706a3..845e5113f 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -127,10 +127,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
}
// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config, required bool) error {
var oc oldKeepstoreConfig
err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
- if os.IsNotExist(err) && ldr.KeepstorePath == defaultKeepstoreConfigPath {
+ if os.IsNotExist(err) && !required {
return nil
} else if err != nil {
return err
@@ -197,10 +197,10 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
}
// update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config, required bool) error {
var oc oldCrunchDispatchSlurmConfig
err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
- if os.IsNotExist(err) && ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath {
+ if os.IsNotExist(err) && !required {
return nil
} else if err != nil {
return err
@@ -259,13 +259,13 @@ type oldWsConfig struct {
ManagementToken *string
}
-const defaultWebsocketsConfigPath = "/etc/arvados/ws/ws.yml"
+const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
// update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config, required bool) error {
var oc oldWsConfig
- err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketsPath, &oc)
- if os.IsNotExist(err) && ldr.WebsocketsPath == defaultWebsocketsConfigPath {
+ err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
+ if os.IsNotExist(err) && !required {
return nil
} else if err != nil {
return err
@@ -277,7 +277,6 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
}
loadOldClientConfig(cluster, oc.Client)
- fmt.Printf("Clllllllllllient %v %v", *oc.Client, cluster.Services.Controller.ExternalURL)
if oc.Postgres != nil {
cluster.PostgreSQL.Connection = *oc.Postgres
@@ -295,7 +294,7 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
cluster.SystemLogs.Format = *oc.LogFormat
}
if oc.PingTimeout != nil {
- cluster.API.WebsocketKeepaliveTimeout = *oc.PingTimeout
+ cluster.API.SendTimeout = *oc.PingTimeout
}
if oc.ClientEventQueue != nil {
cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue
diff --git a/lib/config/load.go b/lib/config/load.go
index 63b6ac7d9..f9ee6989d 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -31,7 +31,13 @@ type Loader struct {
Path string
KeepstorePath string
CrunchDispatchSlurmPath string
- WebsocketsPath string
+ WebsocketPath string
+
+ // Legacy config file for the current component (will be the
+ // same as one of the above files). If set, not being able to
+ // load the 'main' config.yml will not be a fatal error, but
+ // the the legacy file will be required instead.
+ LegacyComponentConfig string
configdata []byte
}
@@ -60,7 +66,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
- flagset.StringVar(&ldr.WebsocketsPath, "legacy-ws-config", defaultWebsocketsConfigPath, "Legacy arvados-ws configuration `file`")
+ flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
}
// MungeLegacyConfigArgs checks args for a -config flag whose argument
@@ -134,20 +140,19 @@ func (ldr *Loader) loadBytes(path string) ([]byte, error) {
return ioutil.ReadAll(f)
}
-func (ldr *Loader) LoadDefaults() (*arvados.Config, error) {
- ldr.configdata = []byte(`Clusters: {zzzzz: {}}`)
- defer func() { ldr.configdata = nil }()
- return ldr.Load()
-}
-
func (ldr *Loader) Load() (*arvados.Config, error) {
if ldr.configdata == nil {
buf, err := ldr.loadBytes(ldr.Path)
if err != nil {
- return nil, err
+ if ldr.LegacyComponentConfig != "" && os.IsNotExist(err) && !ldr.SkipDeprecated {
+ buf = []byte(`Clusters: {zzzzz: {}}`)
+ } else {
+ return nil, err
+ }
}
ldr.configdata = buf
}
+ noConfigLoaded := bytes.Compare(ldr.configdata, []byte(`Clusters: {zzzzz: {}}`)) == 0
// Load the config into a dummy map to get the cluster ID
// keys, discarding the values; then set up defaults for each
@@ -213,10 +218,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
if err != nil {
return nil, err
}
+ // legacy file is required when either:
+ // * a non-default location was specified
+ // * no primary config was loaded, and this is the
+ // legacy config file for the current component
for _, err := range []error{
- ldr.loadOldKeepstoreConfig(&cfg),
- ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
- ldr.loadOldWebsocketsConfig(&cfg),
+ ldr.loadOldKeepstoreConfig(&cfg, (ldr.KeepstorePath != defaultKeepstoreConfigPath) ||
+ (noConfigLoaded && ldr.LegacyComponentConfig == ldr.KeepstorePath)),
+
+ ldr.loadOldCrunchDispatchSlurmConfig(&cfg, (ldr.CrunchDispatchSlurmPath != defaultCrunchDispatchSlurmConfigPath) ||
+ (noConfigLoaded && ldr.LegacyComponentConfig == ldr.CrunchDispatchSlurmPath)),
+
+ ldr.loadOldWebsocketConfig(&cfg, (ldr.WebsocketPath != defaultWebsocketConfigPath) ||
+ (noConfigLoaded && ldr.LegacyComponentConfig == ldr.WebsocketPath)),
} {
if err != nil {
return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 9072b7319..d0ab58ae4 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -76,7 +76,7 @@ type Cluster struct {
MaxRequestSize int
RailsSessionSecretToken string
RequestTimeout Duration
- WebsocketKeepaliveTimeout Duration
+ SendTimeout Duration
WebsocketClientEventQueue int
WebsocketServerEventQueue int
}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1a7ad6fac..75e6146f5 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -109,6 +109,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
disp.logger.Printf("crunch-dispatch-slurm %s started", version)
+ loader.LegacyComponentConfig = loader.CrunchDispatchSlurmPath
cfg, err := loader.Load()
if err != nil {
return err
diff --git a/services/ws/main.go b/services/ws/main.go
index 0556c77d6..2ea1a987d 100644
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -36,6 +36,7 @@ func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
return nil
}
+ loader.LegacyComponentConfig = loader.WebsocketPath
cfg, err := loader.Load()
if err != nil {
log.Fatal(err)
diff --git a/services/ws/router.go b/services/ws/router.go
index 5a5b7c53b..14dc63ec3 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -54,7 +54,7 @@ type debugStatuser interface {
func (rtr *router) setup() {
rtr.handler = &handler{
- PingTimeout: time.Duration(rtr.cluster.API.WebsocketKeepaliveTimeout),
+ PingTimeout: time.Duration(rtr.cluster.API.SendTimeout),
QueueSize: rtr.cluster.API.WebsocketClientEventQueue,
}
rtr.mux = http.NewServeMux()
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 097889c98..ca57f9faa 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -36,7 +36,8 @@ func (s *serverSuite) SetUpTest(c *check.C) {
func (*serverSuite) testConfig() (*arvados.Cluster, error) {
ldr := config.NewLoader(nil, nil)
- cfg, err := ldr.LoadDefaults()
+ ldr.LegacyComponentConfig = "ws-test"
+ cfg, err := ldr.Load()
if err != nil {
return nil, err
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list