[ARVADOS] created: 1.3.0-1022-g69b4828b2

Git user git at public.curoverse.com
Thu Jun 6 17:45:50 UTC 2019


        at  69b4828b22fbfdf26d1355e99bad5cd1858d64da (commit)


commit 69b4828b22fbfdf26d1355e99bad5cd1858d64da
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Jun 6 13:44:23 2019 -0400

    15000: Ensure all config defaults are captured in config struct.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 31a754493..fdcb6cc76 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -59,7 +59,7 @@ Clusters:
 `
 	code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
 	c.Check(code, check.Equals, 1)
-	c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
+	c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
 }
 
 func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 94cd8fcbf..a7b53489c 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -35,11 +35,13 @@ Clusters:
         InternalURLs: {}
         ExternalURL: ""
       GitSSH:
+        InternalURLs: {}
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
         ExternalURL: "-"
       SSO:
+        InternalURLs: {}
         ExternalURL: ""
       Keepproxy:
         InternalURLs: {}
@@ -54,13 +56,16 @@ Clusters:
         InternalURLs: {}
         ExternalURL: "-"
       Composer:
+        InternalURLs: {}
         ExternalURL: ""
       WebShell:
+        InternalURLs: {}
         ExternalURL: ""
       Workbench1:
         InternalURLs: {}
         ExternalURL: ""
       Workbench2:
+        InternalURLs: {}
         ExternalURL: ""
       Nodemanager:
         InternalURLs: {}
@@ -113,7 +118,7 @@ Clusters:
       # Interval (seconds) between asynchronous permission view updates. Any
       # permission-updating API called with the 'async' parameter schedules a an
       # update on the permission view in the future, if not already scheduled.
-      AsyncPermissionsUpdateInterval: 20
+      AsyncPermissionsUpdateInterval: 20s
 
       # Maximum number of concurrent outgoing requests to make while
       # serving a single incoming multi-cluster (federated) request.
@@ -260,7 +265,7 @@ Clusters:
       # Interval (seconds) between trash sweeps. During a trash sweep,
       # collections are marked as trash if their trash_at time has
       # arrived, and deleted if their delete_at time has arrived.
-      TrashSweepInterval: 60
+      TrashSweepInterval: 60s
 
       # If true, enable collection versioning.
       # When a collection's preserve_version field is true or the current version
@@ -269,10 +274,10 @@ Clusters:
       # the current collection.
       CollectionVersioning: false
 
-      #   0 = auto-create a new version on every update.
-      #  -1 = never auto-create new versions.
-      # > 0 = auto-create a new version when older than the specified number of seconds.
-      PreserveVersionIfIdle: -1
+      #   0s = auto-create a new version on every update.
+      #  -1s = never auto-create new versions.
+      # > 0s = auto-create a new version when older than the specified number of seconds.
+      PreserveVersionIfIdle: -1s
 
     Login:
       # These settings are provided by your OAuth2 provider (e.g.,
@@ -336,12 +341,6 @@ Clusters:
       # scheduling parameter parameter set.
       UsePreemptibleInstances: false
 
-      # Include details about job reuse decisions in the server log. This
-      # causes additional database queries to run, so it should not be
-      # enabled unless you expect to examine the resulting logs for
-      # troubleshooting purposes.
-      LogReuseDecisions: false
-
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # (experimental) cloud dispatcher for executing containers on
       # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -366,8 +365,8 @@ Clusters:
         LogBytesPerEvent: 4096
         LogSecondsBetweenEvents: 1
 
-        # The sample period for throttling logs, in seconds.
-        LogThrottlePeriod: 60
+        # The sample period for throttling logs.
+        LogThrottlePeriod: 60s
 
         # Maximum number of bytes that job can log over crunch_log_throttle_period
         # before being silenced until the end of the period.
@@ -381,18 +380,18 @@ Clusters:
         # silenced by throttling are not counted against this total.
         LimitLogBytesPerJob: 67108864
 
-        LogPartialLineThrottlePeriod: 5
+        LogPartialLineThrottlePeriod: 5s
 
-        # Container logs are written to Keep and saved in a collection,
-        # which is updated periodically while the container runs.  This
-        # value sets the interval (given in seconds) between collection
-        # updates.
-        LogUpdatePeriod: 1800
+        # Container logs are written to Keep and saved in a
+        # collection, which is updated periodically while the
+        # container runs.  This value sets the interval between
+        # collection updates.
+        LogUpdatePeriod: 30m
 
         # The log collection is also updated when the specified amount of
         # log data (given in bytes) is produced in less than one update
         # period.
-        LogUpdateSize: 33554432
+        LogUpdateSize: 32MiB
 
       SLURM:
         Managed:
@@ -613,8 +612,20 @@ Clusters:
         Insecure: false
         ActivateUsers: false
       SAMPLE:
+        # API endpoint host or host:port; default is {id}.arvadosapi.com
         Host: sample.arvadosapi.com
+
+        # Perform a proxy request when a local client requests an
+        # object belonging to this remote.
         Proxy: false
+
+        # Default "https". Can be set to "http" for testing.
         Scheme: https
+
+        # Disable TLS verify. Can be set to true for testing.
         Insecure: false
+
+        # When users present tokens issued by this remote cluster, and
+        # their accounts are active on the remote cluster, activate
+        # them on this cluster too.
         ActivateUsers: false
diff --git a/lib/config/export.go b/lib/config/export.go
index 98263c2b7..3c485e2b4 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -55,62 +55,143 @@ func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
 // exists.
 var whitelist = map[string]bool{
 	// | sort -t'"' -k2,2
-	"API":                                      true,
-	"API.MaxItemsPerResponse":                  true,
-	"API.MaxRequestAmplification":              true,
-	"API.RequestTimeout":                       true,
-	"Containers":                               true,
-	"Containers.CloudVMs":                      true,
-	"Containers.CloudVMs.BootProbeCommand":     true,
-	"Containers.CloudVMs.Driver":               true,
-	"Containers.CloudVMs.DriverParameters":     false,
-	"Containers.CloudVMs.Enable":               true,
-	"Containers.CloudVMs.ImageID":              true,
-	"Containers.CloudVMs.MaxCloudOpsPerSecond": true,
-	"Containers.CloudVMs.MaxProbesPerSecond":   true,
-	"Containers.CloudVMs.PollInterval":         true,
-	"Containers.CloudVMs.ProbeInterval":        true,
-	"Containers.CloudVMs.ResourceTags":         true,
-	"Containers.CloudVMs.ResourceTags.*":       true,
-	"Containers.CloudVMs.SSHPort":              true,
-	"Containers.CloudVMs.SyncInterval":         true,
-	"Containers.CloudVMs.TagKeyPrefix":         true,
-	"Containers.CloudVMs.TimeoutBooting":       true,
-	"Containers.CloudVMs.TimeoutIdle":          true,
-	"Containers.CloudVMs.TimeoutProbe":         true,
-	"Containers.CloudVMs.TimeoutShutdown":      true,
-	"Containers.CloudVMs.TimeoutSignal":        true,
-	"Containers.CloudVMs.TimeoutTERM":          true,
-	"Containers.DispatchPrivateKey":            true,
-	"Containers.StaleLockTimeout":              true,
-	"InstanceTypes":                            true,
-	"InstanceTypes.*":                          true,
-	"InstanceTypes.*.*":                        true,
-	"ManagementToken":                          false,
-	"PostgreSQL":                               true,
-	"PostgreSQL.Connection":                    false,
-	"PostgreSQL.ConnectionPool":                true,
-	"RemoteClusters":                           true,
-	"RemoteClusters.*":                         true,
-	"RemoteClusters.*.Host":                    true,
-	"RemoteClusters.*.Insecure":                true,
-	"RemoteClusters.*.Proxy":                   true,
-	"RemoteClusters.*.Scheme":                  true,
-	"Services":                                 true,
-	"Services.*":                               true,
-	"Services.*.ExternalURL":                   true,
-	"Services.*.InternalURLs":                  true,
-	"Services.*.InternalURLs.*":                true,
-	"Services.*.InternalURLs.*.*":              true,
-	"SystemLogs":                               true,
-	"SystemLogs.Format":                        true,
-	"SystemLogs.LogLevel":                      true,
-	"SystemLogs.MaxRequestLogParamsSize":       true,
-	"SystemRootToken":                          false,
-	"TLS":                                      true,
-	"TLS.Certificate":                          true,
-	"TLS.Insecure":                             true,
-	"TLS.Key":                                  false,
+	"API":                                             true,
+	"API.AsyncPermissionsUpdateInterval":              true,
+	"API.DisabledAPIs":                                true,
+	"API.MaxIndexDatabaseRead":                        true,
+	"API.MaxItemsPerResponse":                         true,
+	"API.MaxRequestAmplification":                     true,
+	"API.MaxRequestSize":                              true,
+	"API.RailsSessionSecretToken":                     false,
+	"API.RequestTimeout":                              true,
+	"AuditLogs":                                       true,
+	"AuditLogs.MaxAge":                                true,
+	"AuditLogs.MaxDeleteBatch":                        true,
+	"AuditLogs.UnloggedAttributes":                    true,
+	"Collections":                                     true,
+	"Collections.BlobSigning":                         true,
+	"Collections.BlobSigningKey":                      false,
+	"Collections.BlobSigningTTL":                      true,
+	"Collections.CollectionVersioning":                true,
+	"Collections.DefaultReplication":                  true,
+	"Collections.DefaultTrashLifetime":                true,
+	"Collections.PreserveVersionIfIdle":               true,
+	"Collections.TrashSweepInterval":                  true,
+	"Containers":                                      true,
+	"Containers.CloudVMs":                             true,
+	"Containers.CloudVMs.BootProbeCommand":            true,
+	"Containers.CloudVMs.Driver":                      true,
+	"Containers.CloudVMs.DriverParameters":            false,
+	"Containers.CloudVMs.Enable":                      true,
+	"Containers.CloudVMs.ImageID":                     true,
+	"Containers.CloudVMs.MaxCloudOpsPerSecond":        true,
+	"Containers.CloudVMs.MaxProbesPerSecond":          true,
+	"Containers.CloudVMs.PollInterval":                true,
+	"Containers.CloudVMs.ProbeInterval":               true,
+	"Containers.CloudVMs.ResourceTags":                true,
+	"Containers.CloudVMs.ResourceTags.*":              true,
+	"Containers.CloudVMs.SSHPort":                     true,
+	"Containers.CloudVMs.SyncInterval":                true,
+	"Containers.CloudVMs.TagKeyPrefix":                true,
+	"Containers.CloudVMs.TimeoutBooting":              true,
+	"Containers.CloudVMs.TimeoutIdle":                 true,
+	"Containers.CloudVMs.TimeoutProbe":                true,
+	"Containers.CloudVMs.TimeoutShutdown":             true,
+	"Containers.CloudVMs.TimeoutSignal":               true,
+	"Containers.CloudVMs.TimeoutTERM":                 true,
+	"Containers.DefaultKeepCacheRAM":                  true,
+	"Containers.DispatchPrivateKey":                   true,
+	"Containers.JobsAPI":                              true,
+	"Containers.JobsAPI.CrunchJobUser":                true,
+	"Containers.JobsAPI.CrunchJobWrapper":             true,
+	"Containers.JobsAPI.CrunchRefreshTrigger":         true,
+	"Containers.JobsAPI.DefaultDockerImage":           true,
+	"Containers.JobsAPI.Enable":                       true,
+	"Containers.JobsAPI.GitInternalDir":               true,
+	"Containers.JobsAPI.ReuseJobIfOutputsDiffer":      true,
+	"Containers.Logging":                              true,
+	"Containers.Logging.LimitLogBytesPerJob":          true,
+	"Containers.Logging.LogBytesPerEvent":             true,
+	"Containers.Logging.LogPartialLineThrottlePeriod": true,
+	"Containers.Logging.LogSecondsBetweenEvents":      true,
+	"Containers.Logging.LogThrottleBytes":             true,
+	"Containers.Logging.LogThrottleLines":             true,
+	"Containers.Logging.LogThrottlePeriod":            true,
+	"Containers.Logging.LogUpdatePeriod":              true,
+	"Containers.Logging.LogUpdateSize":                true,
+	"Containers.Logging.MaxAge":                       true,
+	"Containers.LogReuseDecisions":                    true,
+	"Containers.MaxComputeVMs":                        true,
+	"Containers.MaxDispatchAttempts":                  true,
+	"Containers.MaxRetryAttempts":                     true,
+	"Containers.SLURM":                                true,
+	"Containers.SLURM.Managed":                        true,
+	"Containers.SLURM.Managed.AssignNodeHostname":     true,
+	"Containers.SLURM.Managed.ComputeNodeDomain":      false,
+	"Containers.SLURM.Managed.ComputeNodeNameservers": false,
+	"Containers.SLURM.Managed.DNSServerConfDir":       true,
+	"Containers.SLURM.Managed.DNSServerConfTemplate":  true,
+	"Containers.SLURM.Managed.DNSServerReloadCommand": false,
+	"Containers.SLURM.Managed.DNSServerUpdateCommand": false,
+	"Containers.StaleLockTimeout":                     true,
+	"Containers.SupportedDockerImageFormats":          true,
+	"Containers.UsePreemptibleInstances":              true,
+	"Git":                                             true,
+	"Git.Repositories":                                true,
+	"InstanceTypes":                                   true,
+	"InstanceTypes.*":                                 true,
+	"InstanceTypes.*.*":                               true,
+	"Login":                                           true,
+	"Login.ProviderAppID":                             false,
+	"Login.ProviderAppSecret":                         false,
+	"Mail":                                            true,
+	"Mail.EmailFrom":                                  true,
+	"Mail.IssueReporterEmailFrom":                     true,
+	"Mail.IssueReporterEmailTo":                       true,
+	"Mail.MailchimpAPIKey":                            false,
+	"Mail.MailchimpListID":                            false,
+	"Mail.SendUserSetupNotificationEmail":             true,
+	"Mail.SupportEmailAddress":                        true,
+	"ManagementToken":                                 false,
+	"PostgreSQL":                                      true,
+	"PostgreSQL.Connection":                           false,
+	"PostgreSQL.ConnectionPool":                       true,
+	"RemoteClusters":                                  true,
+	"RemoteClusters.*":                                true,
+	"RemoteClusters.*.ActivateUsers":                  true,
+	"RemoteClusters.*.Host":                           true,
+	"RemoteClusters.*.Insecure":                       true,
+	"RemoteClusters.*.Proxy":                          true,
+	"RemoteClusters.*.Scheme":                         true,
+	"Services":                                        true,
+	"Services.*":                                      true,
+	"Services.*.ExternalURL":                          true,
+	"Services.*.InternalURLs":                         true,
+	"Services.*.InternalURLs.*":                       true,
+	"Services.*.InternalURLs.*.*":                     true,
+	"SystemLogs":                                      true,
+	"SystemLogs.Format":                               true,
+	"SystemLogs.LogLevel":                             true,
+	"SystemLogs.MaxRequestLogParamsSize":              true,
+	"SystemRootToken":                                 false,
+	"TLS":                                             true,
+	"TLS.Certificate":                                 true,
+	"TLS.Insecure":                                    true,
+	"TLS.Key":                                         false,
+	"Users":                                           true,
+	"Users.AdminNotifierEmailFrom":                    true,
+	"Users.AutoAdminFirstUser":                        false,
+	"Users.AutoAdminUserWithEmail":                    false,
+	"Users.AutoSetupNewUsers":                         true,
+	"Users.AutoSetupNewUsersWithRepository":           true,
+	"Users.AutoSetupNewUsersWithVmUUID":               true,
+	"Users.AutoSetupUsernameBlacklist":                false,
+	"Users.EmailSubjectPrefix":                        true,
+	"Users.NewInactiveUserNotificationRecipients":     false,
+	"Users.NewUserNotificationRecipients":             false,
+	"Users.NewUsersAreActive":                         true,
+	"Users.UserNotifierEmailFrom":                     true,
+	"Users.UserProfileNotificationAddress":            true,
 }
 
 func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 3492615e9..f9cd7bcd1 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -41,11 +41,13 @@ Clusters:
         InternalURLs: {}
         ExternalURL: ""
       GitSSH:
+        InternalURLs: {}
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
         ExternalURL: "-"
       SSO:
+        InternalURLs: {}
         ExternalURL: ""
       Keepproxy:
         InternalURLs: {}
@@ -60,13 +62,16 @@ Clusters:
         InternalURLs: {}
         ExternalURL: "-"
       Composer:
+        InternalURLs: {}
         ExternalURL: ""
       WebShell:
+        InternalURLs: {}
         ExternalURL: ""
       Workbench1:
         InternalURLs: {}
         ExternalURL: ""
       Workbench2:
+        InternalURLs: {}
         ExternalURL: ""
       Nodemanager:
         InternalURLs: {}
@@ -119,7 +124,7 @@ Clusters:
       # Interval (seconds) between asynchronous permission view updates. Any
       # permission-updating API called with the 'async' parameter schedules a an
       # update on the permission view in the future, if not already scheduled.
-      AsyncPermissionsUpdateInterval: 20
+      AsyncPermissionsUpdateInterval: 20s
 
       # Maximum number of concurrent outgoing requests to make while
       # serving a single incoming multi-cluster (federated) request.
@@ -266,7 +271,7 @@ Clusters:
       # Interval (seconds) between trash sweeps. During a trash sweep,
       # collections are marked as trash if their trash_at time has
       # arrived, and deleted if their delete_at time has arrived.
-      TrashSweepInterval: 60
+      TrashSweepInterval: 60s
 
       # If true, enable collection versioning.
       # When a collection's preserve_version field is true or the current version
@@ -275,10 +280,10 @@ Clusters:
       # the current collection.
       CollectionVersioning: false
 
-      #   0 = auto-create a new version on every update.
-      #  -1 = never auto-create new versions.
-      # > 0 = auto-create a new version when older than the specified number of seconds.
-      PreserveVersionIfIdle: -1
+      #   0s = auto-create a new version on every update.
+      #  -1s = never auto-create new versions.
+      # > 0s = auto-create a new version when older than the specified number of seconds.
+      PreserveVersionIfIdle: -1s
 
     Login:
       # These settings are provided by your OAuth2 provider (e.g.,
@@ -342,12 +347,6 @@ Clusters:
       # scheduling parameter parameter set.
       UsePreemptibleInstances: false
 
-      # Include details about job reuse decisions in the server log. This
-      # causes additional database queries to run, so it should not be
-      # enabled unless you expect to examine the resulting logs for
-      # troubleshooting purposes.
-      LogReuseDecisions: false
-
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # (experimental) cloud dispatcher for executing containers on
       # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -372,8 +371,8 @@ Clusters:
         LogBytesPerEvent: 4096
         LogSecondsBetweenEvents: 1
 
-        # The sample period for throttling logs, in seconds.
-        LogThrottlePeriod: 60
+        # The sample period for throttling logs.
+        LogThrottlePeriod: 60s
 
         # Maximum number of bytes that job can log over crunch_log_throttle_period
         # before being silenced until the end of the period.
@@ -387,18 +386,18 @@ Clusters:
         # silenced by throttling are not counted against this total.
         LimitLogBytesPerJob: 67108864
 
-        LogPartialLineThrottlePeriod: 5
+        LogPartialLineThrottlePeriod: 5s
 
-        # Container logs are written to Keep and saved in a collection,
-        # which is updated periodically while the container runs.  This
-        # value sets the interval (given in seconds) between collection
-        # updates.
-        LogUpdatePeriod: 1800
+        # Container logs are written to Keep and saved in a
+        # collection, which is updated periodically while the
+        # container runs.  This value sets the interval between
+        # collection updates.
+        LogUpdatePeriod: 30m
 
         # The log collection is also updated when the specified amount of
         # log data (given in bytes) is produced in less than one update
         # period.
-        LogUpdateSize: 33554432
+        LogUpdateSize: 32MiB
 
       SLURM:
         Managed:
@@ -619,9 +618,21 @@ Clusters:
         Insecure: false
         ActivateUsers: false
       SAMPLE:
+        # API endpoint host or host:port; default is {id}.arvadosapi.com
         Host: sample.arvadosapi.com
+
+        # Perform a proxy request when a local client requests an
+        # object belonging to this remote.
         Proxy: false
+
+        # Default "https". Can be set to "http" for testing.
         Scheme: https
+
+        # Disable TLS verify. Can be set to true for testing.
         Insecure: false
+
+        # When users present tokens issued by this remote cluster, and
+        # their accounts are active on the remote cluster, activate
+        # them on this cluster too.
         ActivateUsers: false
 `)
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 6ce81bb5f..6b014476b 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -97,6 +97,24 @@ Clusters:
 	c.Check(logs, check.HasLen, 2)
 }
 
+func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
+	var logbuf bytes.Buffer
+	logger := logrus.New()
+	logger.Out = &logbuf
+	var supplied map[string]interface{}
+	yaml.Unmarshal(DefaultYAML, &supplied)
+	cfg, err := Load(bytes.NewBuffer(DefaultYAML), logger)
+	c.Assert(err, check.IsNil)
+	var loaded map[string]interface{}
+	buf, err := yaml.Marshal(cfg)
+	c.Assert(err, check.IsNil)
+	err = yaml.Unmarshal(buf, &loaded)
+	c.Assert(err, check.IsNil)
+
+	logExtraKeys(logger, loaded, supplied, "")
+	c.Check(logbuf.String(), check.Equals, "")
+}
+
 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
 	var logbuf bytes.Buffer
 	logger := logrus.New()
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 1c859cfc5..7d8e7a433 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -57,12 +57,10 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 	cluster := &arvados.Cluster{
 		ClusterID:  "zhome",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		TLS:        arvados.TLS{Insecure: true},
-		API: arvados.API{
-			MaxItemsPerResponse:     1000,
-			MaxRequestAmplification: 4,
-		},
 	}
+	cluster.TLS.Insecure = true
+	cluster.API.MaxItemsPerResponse = 1000
+	cluster.API.MaxRequestAmplification = 4
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
 	arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
 	s.testHandler = &Handler{Cluster: cluster}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 06a5a8411..cb69f2f6b 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -42,8 +42,8 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
 	s.cluster = &arvados.Cluster{
 		ClusterID:  "zzzzz",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		TLS:        arvados.TLS{Insecure: true},
 	}
+	s.cluster.TLS.Insecure = true
 	arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
 	s.handler = newHandler(s.ctx, s.cluster, "")
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index a398af97b..ae7f138b1 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -36,8 +36,8 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
 	handler := &Handler{Cluster: &arvados.Cluster{
 		ClusterID:  "zzzzz",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		TLS:        arvados.TLS{Insecure: true},
 	}}
+	handler.Cluster.TLS.Insecure = true
 	arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
 
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index d96bf2517..fb01f25f8 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -50,12 +50,6 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
 	}
 }
 
-type API struct {
-	MaxItemsPerResponse     int
-	MaxRequestAmplification int
-	RequestTimeout          Duration
-}
-
 type Cluster struct {
 	ClusterID       string `json:"-"`
 	ManagementToken string
@@ -65,28 +59,98 @@ type Cluster struct {
 	Containers      ContainersConfig
 	RemoteClusters  map[string]RemoteCluster
 	PostgreSQL      PostgreSQL
-	API             API
-	SystemLogs      SystemLogs
-	TLS             TLS
+
+	API struct {
+		AsyncPermissionsUpdateInterval Duration
+		DisabledAPIs                   []string
+		MaxIndexDatabaseRead           int
+		MaxItemsPerResponse            int
+		MaxRequestAmplification        int
+		MaxRequestSize                 int
+		RailsSessionSecretToken        string
+		RequestTimeout                 Duration
+	}
+	AuditLogs struct {
+		MaxAge             Duration
+		MaxDeleteBatch     int
+		UnloggedAttributes []string
+	}
+	Collections struct {
+		BlobSigning           bool
+		BlobSigningKey        string
+		DefaultReplication    int
+		BlobSigningTTL        Duration
+		DefaultTrashLifetime  Duration
+		TrashSweepInterval    Duration
+		CollectionVersioning  bool
+		PreserveVersionIfIdle Duration
+	}
+	Git struct {
+		Repositories string
+	}
+	Login struct {
+		ProviderAppSecret string
+		ProviderAppID     string
+	}
+	Mail struct {
+		MailchimpAPIKey                string
+		MailchimpListID                string
+		SendUserSetupNotificationEmail string
+		IssueReporterEmailFrom         string
+		IssueReporterEmailTo           string
+		SupportEmailAddress            string
+		EmailFrom                      string
+	}
+	SystemLogs struct {
+		LogLevel                string
+		Format                  string
+		MaxRequestLogParamsSize int
+	}
+	TLS struct {
+		Certificate string
+		Key         string
+		Insecure    bool
+	}
+	Users struct {
+		AdminNotifierEmailFrom                string
+		AutoAdminFirstUser                    bool
+		AutoAdminUserWithEmail                string
+		AutoSetupNewUsers                     bool
+		AutoSetupNewUsersWithRepository       bool
+		AutoSetupNewUsersWithVmUUID           string
+		AutoSetupUsernameBlacklist            []string
+		EmailSubjectPrefix                    string
+		NewInactiveUserNotificationRecipients []string
+		NewUserNotificationRecipients         []string
+		NewUsersAreActive                     bool
+		UserNotifierEmailFrom                 string
+		UserProfileNotificationAddress        string
+	}
 }
 
 type Services struct {
-	Controller    Service
-	DispatchCloud Service
-	Health        Service
-	Keepbalance   Service
-	Keepproxy     Service
-	Keepstore     Service
-	Nodemanager   Service
-	RailsAPI      Service
-	WebDAV        Service
-	Websocket     Service
-	Workbench1    Service
-	Workbench2    Service
+	Composer       Service
+	Controller     Service
+	DispatchCloud  Service
+	GitHTTP        Service
+	GitSSH         Service
+	Health         Service
+	Keepbalance    Service
+	Keepproxy      Service
+	Keepstore      Service
+	Nodemanager    Service
+	RailsAPI       Service
+	SSO            Service
+	WebDAVDownload Service
+	WebDAV         Service
+	WebShell       Service
+	Websocket      Service
+	Workbench1     Service
+	Workbench2     Service
 }
 
 type Service struct {
-	InternalURLs map[URL]ServiceInstance `json:",omitempty"`
+	InternalURLs map[URL]ServiceInstance
 	ExternalURL  URL
 }
 
@@ -109,12 +173,6 @@ func (su URL) MarshalText() ([]byte, error) {
 
 type ServiceInstance struct{}
 
-type SystemLogs struct {
-	LogLevel                string
-	Format                  string
-	MaxRequestLogParamsSize int
-}
-
 type PostgreSQL struct {
 	Connection     PostgreSQLConnection
 	ConnectionPool int
@@ -123,15 +181,11 @@ type PostgreSQL struct {
 type PostgreSQLConnection map[string]string
 
 type RemoteCluster struct {
-	// API endpoint host or host:port; default is {id}.arvadosapi.com
-	Host string
-	// Perform a proxy request when a local client requests an
-	// object belonging to this remote.
-	Proxy bool
-	// Scheme, default "https". Can be set to "http" for testing.
-	Scheme string
-	// Disable TLS verify. Can be set to true for testing.
-	Insecure bool
+	Host          string
+	Proxy         bool
+	Scheme        string
+	Insecure      bool
+	ActivateUsers bool
 }
 
 type InstanceType struct {
@@ -147,9 +201,49 @@ type InstanceType struct {
 }
 
 type ContainersConfig struct {
-	CloudVMs           CloudVMsConfig
-	DispatchPrivateKey string
-	StaleLockTimeout   Duration
+	CloudVMs                    CloudVMsConfig
+	DefaultKeepCacheRAM         ByteSize
+	DispatchPrivateKey          string
+	LogReuseDecisions           bool
+	MaxComputeVMs               int
+	MaxDispatchAttempts         int
+	MaxRetryAttempts            int
+	StaleLockTimeout            Duration
+	SupportedDockerImageFormats []string
+	UsePreemptibleInstances     bool
+
+	JobsAPI struct {
+		Enable                  string
+		GitInternalDir          string
+		DefaultDockerImage      string
+		CrunchJobWrapper        string
+		CrunchJobUser           string
+		CrunchRefreshTrigger    string
+		ReuseJobIfOutputsDiffer bool
+	}
+	Logging struct {
+		MaxAge                       Duration
+		LogBytesPerEvent             int
+		LogSecondsBetweenEvents      int
+		LogThrottlePeriod            Duration
+		LogThrottleBytes             int
+		LogThrottleLines             int
+		LimitLogBytesPerJob          int
+		LogPartialLineThrottlePeriod Duration
+		LogUpdatePeriod              Duration
+		LogUpdateSize                ByteSize
+	}
+	SLURM struct {
+		Managed struct {
+			DNSServerConfDir       string
+			DNSServerConfTemplate  string
+			DNSServerReloadCommand string
+			DNSServerUpdateCommand string
+			ComputeNodeDomain      string
+			ComputeNodeNameservers []string
+			AssignNodeHostname     string
+		}
+	}
 }
 
 type CloudVMsConfig struct {
@@ -269,9 +363,3 @@ func (svcs Services) Map() map[ServiceName]Service {
 		ServiceNameKeepstore:     svcs.Keepstore,
 	}
 }
-
-type TLS struct {
-	Certificate string
-	Key         string
-	Insecure    bool
-}

commit 6e4478ae076e50d463df398a7bd1433d86bd596a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Jun 6 10:54:26 2019 -0400

    15000: Export safe parts of cluster config at /arvados/v1/config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/config/cmd.go b/lib/config/cmd.go
index 41a1d7d21..39df4ec17 100644
--- a/lib/config/cmd.go
+++ b/lib/config/cmd.go
@@ -12,12 +12,11 @@ import (
 	"os"
 	"os/exec"
 
-	"git.curoverse.com/arvados.git/lib/cmd"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"github.com/ghodss/yaml"
 )
 
-var DumpCommand cmd.Handler = dumpCommand{}
+var DumpCommand dumpCommand
 
 type dumpCommand struct{}
 
@@ -48,7 +47,7 @@ func (dumpCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
 	return 0
 }
 
-var CheckCommand cmd.Handler = checkCommand{}
+var CheckCommand checkCommand
 
 type checkCommand struct{}
 
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index e4d838f85..31a754493 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -7,11 +7,18 @@ package config
 import (
 	"bytes"
 
+	"git.curoverse.com/arvados.git/lib/cmd"
 	check "gopkg.in/check.v1"
 )
 
 var _ = check.Suite(&CommandSuite{})
 
+var (
+	// Commands must satisfy cmd.Handler interface
+	_ cmd.Handler = dumpCommand{}
+	_ cmd.Handler = checkCommand{}
+)
+
 type CommandSuite struct{}
 
 func (s *CommandSuite) TestBadArg(c *check.C) {
diff --git a/lib/config/export.go b/lib/config/export.go
new file mode 100644
index 000000000..98263c2b7
--- /dev/null
+++ b/lib/config/export.go
@@ -0,0 +1,144 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+// ExportJSON writes a JSON object with the safe (non-secret) portions
+// of the cluster config to w.
+func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
+	buf, err := json.Marshal(cluster)
+	if err != nil {
+		return err
+	}
+	var m map[string]interface{}
+	err = json.Unmarshal(buf, &m)
+	if err != nil {
+		return err
+	}
+	err = redactUnsafe(m, "", "")
+	if err != nil {
+		return err
+	}
+	return json.NewEncoder(w).Encode(m)
+}
+
+// whitelist classifies configs as safe/unsafe to reveal to
+// unauthenticated clients.
+//
+// Every config entry must either be listed explicitly here along with
+// all of its parent keys (e.g., "API" + "API.RequestTimeout"), or
+// have an ancestor listed as false (e.g.,
+// "PostgreSQL.Connection.password" has an ancestor
+// "PostgreSQL.Connection" with a false value). Otherwise, it is a bug
+// which should be caught by tests.
+//
+// Example: API.RequestTimeout is safe because whitelist["API"] == and
+// whitelist["API.RequestTimeout"] == true.
+//
+// Example: PostgreSQL.Connection.password is not safe because
+// whitelist["PostgreSQL.Connection"] == false.
+//
+// Example: PostgreSQL.BadKey would cause an error because
+// whitelist["PostgreSQL"] isn't false, and neither
+// whitelist["PostgreSQL.BadKey"] nor whitelist["PostgreSQL.*"]
+// exists.
+var whitelist = map[string]bool{
+	// | sort -t'"' -k2,2
+	"API":                                      true,
+	"API.MaxItemsPerResponse":                  true,
+	"API.MaxRequestAmplification":              true,
+	"API.RequestTimeout":                       true,
+	"Containers":                               true,
+	"Containers.CloudVMs":                      true,
+	"Containers.CloudVMs.BootProbeCommand":     true,
+	"Containers.CloudVMs.Driver":               true,
+	"Containers.CloudVMs.DriverParameters":     false,
+	"Containers.CloudVMs.Enable":               true,
+	"Containers.CloudVMs.ImageID":              true,
+	"Containers.CloudVMs.MaxCloudOpsPerSecond": true,
+	"Containers.CloudVMs.MaxProbesPerSecond":   true,
+	"Containers.CloudVMs.PollInterval":         true,
+	"Containers.CloudVMs.ProbeInterval":        true,
+	"Containers.CloudVMs.ResourceTags":         true,
+	"Containers.CloudVMs.ResourceTags.*":       true,
+	"Containers.CloudVMs.SSHPort":              true,
+	"Containers.CloudVMs.SyncInterval":         true,
+	"Containers.CloudVMs.TagKeyPrefix":         true,
+	"Containers.CloudVMs.TimeoutBooting":       true,
+	"Containers.CloudVMs.TimeoutIdle":          true,
+	"Containers.CloudVMs.TimeoutProbe":         true,
+	"Containers.CloudVMs.TimeoutShutdown":      true,
+	"Containers.CloudVMs.TimeoutSignal":        true,
+	"Containers.CloudVMs.TimeoutTERM":          true,
+	"Containers.DispatchPrivateKey":            true,
+	"Containers.StaleLockTimeout":              true,
+	"InstanceTypes":                            true,
+	"InstanceTypes.*":                          true,
+	"InstanceTypes.*.*":                        true,
+	"ManagementToken":                          false,
+	"PostgreSQL":                               true,
+	"PostgreSQL.Connection":                    false,
+	"PostgreSQL.ConnectionPool":                true,
+	"RemoteClusters":                           true,
+	"RemoteClusters.*":                         true,
+	"RemoteClusters.*.Host":                    true,
+	"RemoteClusters.*.Insecure":                true,
+	"RemoteClusters.*.Proxy":                   true,
+	"RemoteClusters.*.Scheme":                  true,
+	"Services":                                 true,
+	"Services.*":                               true,
+	"Services.*.ExternalURL":                   true,
+	"Services.*.InternalURLs":                  true,
+	"Services.*.InternalURLs.*":                true,
+	"Services.*.InternalURLs.*.*":              true,
+	"SystemLogs":                               true,
+	"SystemLogs.Format":                        true,
+	"SystemLogs.LogLevel":                      true,
+	"SystemLogs.MaxRequestLogParamsSize":       true,
+	"SystemRootToken":                          false,
+	"TLS":                                      true,
+	"TLS.Certificate":                          true,
+	"TLS.Insecure":                             true,
+	"TLS.Key":                                  false,
+}
+
+func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
+	var errs []string
+	for k, v := range m {
+		lookupKey := k
+		safe, ok := whitelist[lookupPrefix+k]
+		if !ok {
+			lookupKey = "*"
+			safe, ok = whitelist[lookupPrefix+"*"]
+		}
+		if !ok {
+			errs = append(errs, fmt.Sprintf("config bug: key %q not in whitelist map", lookupPrefix+k))
+			continue
+		}
+		if !safe {
+			delete(m, k)
+			continue
+		}
+		if v, ok := v.(map[string]interface{}); ok {
+			err := redactUnsafe(v, mPrefix+k+".", lookupPrefix+lookupKey+".")
+			if err != nil {
+				errs = append(errs, err.Error())
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return errors.New(strings.Join(errs, "\n"))
+	}
+	return nil
+}
diff --git a/lib/config/export_test.go b/lib/config/export_test.go
new file mode 100644
index 000000000..581e54cdc
--- /dev/null
+++ b/lib/config/export_test.go
@@ -0,0 +1,37 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ExportSuite{})
+
+type ExportSuite struct{}
+
+func (s *ExportSuite) TestExport(c *check.C) {
+	confdata := bytes.Replace(DefaultYAML, []byte("SAMPLE"), []byte("testkey"), -1)
+	cfg, err := Load(bytes.NewBuffer(confdata), ctxlog.TestLogger(c))
+	c.Assert(err, check.IsNil)
+	cluster := cfg.Clusters["xxxxx"]
+	cluster.ManagementToken = "abcdefg"
+
+	var exported bytes.Buffer
+	err = ExportJSON(&exported, &cluster)
+	c.Check(err, check.IsNil)
+	if err != nil {
+		c.Logf("If all the new keys are safe, add these to whitelist in export.go:")
+		for _, k := range regexp.MustCompile(`"[^"]*"`).FindAllString(err.Error(), -1) {
+			c.Logf("\t%q: true,", strings.Replace(k, `"`, "", -1))
+		}
+	}
+	c.Check(exported.String(), check.Not(check.Matches), `(?ms).*abcdefg.*`)
+}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 2c3ce1d4f..12faacdd4 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -5,16 +5,19 @@
 package controller
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/lib/config"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/health"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -73,6 +76,18 @@ func (h *Handler) setup() {
 		Prefix: "/_health/",
 		Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
 	})
+
+	mux.Handle("/arvados/v1/config", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		err := config.ExportJSON(&buf, h.Cluster)
+		if err != nil {
+			httpserver.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		io.Copy(w, &buf)
+	}))
+
 	hs := http.NotFoundHandler()
 	hs = prepend(hs, h.proxyRailsAPI)
 	hs = h.setupProxyRemoteCluster(hs)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index a1efaacdd..06a5a8411 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -53,6 +53,23 @@ func (s *HandlerSuite) TearDownTest(c *check.C) {
 	s.cancel()
 }
 
+func (s *HandlerSuite) TestConfigExport(c *check.C) {
+	s.cluster.Containers.CloudVMs.PollInterval = arvados.Duration(23 * time.Second)
+	req := httptest.NewRequest("GET", "/arvados/v1/config", nil)
+	resp := httptest.NewRecorder()
+	s.handler.ServeHTTP(resp, req)
+	c.Check(resp.Code, check.Equals, http.StatusOK)
+	var cluster arvados.Cluster
+	c.Log(resp.Body.String())
+	err := json.Unmarshal(resp.Body.Bytes(), &cluster)
+	c.Check(err, check.IsNil)
+	c.Check(cluster.ManagementToken, check.Equals, "")
+	c.Check(cluster.SystemRootToken, check.Equals, "")
+	c.Check(cluster.TLS.Insecure, check.Equals, true)
+	c.Check(cluster.Services.RailsAPI, check.DeepEquals, s.cluster.Services.RailsAPI)
+	c.Check(cluster.Containers.CloudVMs.PollInterval, check.Equals, arvados.Duration(23*time.Second))
+}
+
 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 	req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 	resp := httptest.NewRecorder()

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list