[ARVADOS] created: 1.3.0-1046-g68211a7ea
Git user
git at public.curoverse.com
Fri Jun 7 16:56:39 UTC 2019
at 68211a7ead5f3cbe1a90b1b7769118a2b3543211 (commit)
commit 68211a7ead5f3cbe1a90b1b7769118a2b3543211
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jun 6 17:08:23 2019 -0400
15000: Add Workbench configs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index a7b53489c..b79840abd 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -629,3 +629,53 @@ Clusters:
# their accounts are active on the remote cluster, activate
# them on this cluster too.
ActivateUsers: false
+
+ Workbench:
+ # Workbench1 configs
+ Theme: default
+ ActivationContactLink: mailto:info at arvados.org
+ ArvadosDocsite: https://doc.arvados.org
+ ArvadosPublicDataDocURL: https://playground.arvados.org/projects/public
+ ShowUserAgreementInline: false
+ SecretToken: ""
+ SecretKeyBase: ""
+ RepositoryCache: /var/www/arvados-workbench/current/tmp/git
+ UserProfileFormFields:
+ SAMPLE:
+ Type: text
+ FormFieldTitle: ""
+ FormFieldDescription: ""
+ Required: true
+ UserProfileFormMessage: 'Welcome to Arvados. All <span style="color:red">required fields</span> must be completed before you can proceed.'
+ ApplicationMimetypesWithViewIcon:
+ cwl: {}
+ fasta: {}
+ go: {}
+ javascript: {}
+ json: {}
+ pdf: {}
+ python: {}
+ x-python: {}
+ r: {}
+ rtf: {}
+ sam: {}
+ x-sh: {}
+ vnd.realvnc.bed: {}
+ xml: {}
+ xsl: {}
+ LogViewerMaxBytes: 1M
+ EnablePublicProjectsPage: true
+ EnableGettingStartedPopup: false
+ APIResponseCompression: true
+ APIClientConnectTimeout: 2m
+ APIClientReceiveTimeout: 5m
+ RunningJobLogRecordsToFetch: 2000
+ ShowRecentCollectionsOnDashboard: true
+ ShowUserNotifications: true
+ MultiSiteSearch: false
+ Repositories: true
+ SiteName: Arvados Workbench
+
+ # Workbench2 configs
+ VocabularyURL: ""
+ FileViewersConfigURL: ""
diff --git a/lib/config/export.go b/lib/config/export.go
index 3c485e2b4..fca6b93a8 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -55,143 +55,175 @@ func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
// exists.
var whitelist = map[string]bool{
// | sort -t'"' -k2,2
- "API": true,
- "API.AsyncPermissionsUpdateInterval": true,
- "API.DisabledAPIs": true,
- "API.MaxIndexDatabaseRead": true,
- "API.MaxItemsPerResponse": true,
- "API.MaxRequestAmplification": true,
- "API.MaxRequestSize": true,
- "API.RailsSessionSecretToken": false,
- "API.RequestTimeout": true,
- "AuditLogs": true,
- "AuditLogs.MaxAge": true,
- "AuditLogs.MaxDeleteBatch": true,
- "AuditLogs.UnloggedAttributes": true,
- "Collections": true,
- "Collections.BlobSigning": true,
- "Collections.BlobSigningKey": false,
- "Collections.BlobSigningTTL": true,
- "Collections.CollectionVersioning": true,
- "Collections.DefaultReplication": true,
- "Collections.DefaultTrashLifetime": true,
- "Collections.PreserveVersionIfIdle": true,
- "Collections.TrashSweepInterval": true,
- "Containers": true,
- "Containers.CloudVMs": true,
- "Containers.CloudVMs.BootProbeCommand": true,
- "Containers.CloudVMs.Driver": true,
- "Containers.CloudVMs.DriverParameters": false,
- "Containers.CloudVMs.Enable": true,
- "Containers.CloudVMs.ImageID": true,
- "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
- "Containers.CloudVMs.MaxProbesPerSecond": true,
- "Containers.CloudVMs.PollInterval": true,
- "Containers.CloudVMs.ProbeInterval": true,
- "Containers.CloudVMs.ResourceTags": true,
- "Containers.CloudVMs.ResourceTags.*": true,
- "Containers.CloudVMs.SSHPort": true,
- "Containers.CloudVMs.SyncInterval": true,
- "Containers.CloudVMs.TagKeyPrefix": true,
- "Containers.CloudVMs.TimeoutBooting": true,
- "Containers.CloudVMs.TimeoutIdle": true,
- "Containers.CloudVMs.TimeoutProbe": true,
- "Containers.CloudVMs.TimeoutShutdown": true,
- "Containers.CloudVMs.TimeoutSignal": true,
- "Containers.CloudVMs.TimeoutTERM": true,
- "Containers.DefaultKeepCacheRAM": true,
- "Containers.DispatchPrivateKey": true,
- "Containers.JobsAPI": true,
- "Containers.JobsAPI.CrunchJobUser": true,
- "Containers.JobsAPI.CrunchJobWrapper": true,
- "Containers.JobsAPI.CrunchRefreshTrigger": true,
- "Containers.JobsAPI.DefaultDockerImage": true,
- "Containers.JobsAPI.Enable": true,
- "Containers.JobsAPI.GitInternalDir": true,
- "Containers.JobsAPI.ReuseJobIfOutputsDiffer": true,
- "Containers.Logging": true,
- "Containers.Logging.LimitLogBytesPerJob": true,
- "Containers.Logging.LogBytesPerEvent": true,
- "Containers.Logging.LogPartialLineThrottlePeriod": true,
- "Containers.Logging.LogSecondsBetweenEvents": true,
- "Containers.Logging.LogThrottleBytes": true,
- "Containers.Logging.LogThrottleLines": true,
- "Containers.Logging.LogThrottlePeriod": true,
- "Containers.Logging.LogUpdatePeriod": true,
- "Containers.Logging.LogUpdateSize": true,
- "Containers.Logging.MaxAge": true,
- "Containers.LogReuseDecisions": true,
- "Containers.MaxComputeVMs": true,
- "Containers.MaxDispatchAttempts": true,
- "Containers.MaxRetryAttempts": true,
- "Containers.SLURM": true,
- "Containers.SLURM.Managed": true,
- "Containers.SLURM.Managed.AssignNodeHostname": true,
- "Containers.SLURM.Managed.ComputeNodeDomain": false,
- "Containers.SLURM.Managed.ComputeNodeNameservers": false,
- "Containers.SLURM.Managed.DNSServerConfDir": true,
- "Containers.SLURM.Managed.DNSServerConfTemplate": true,
- "Containers.SLURM.Managed.DNSServerReloadCommand": false,
- "Containers.SLURM.Managed.DNSServerUpdateCommand": false,
- "Containers.StaleLockTimeout": true,
- "Containers.SupportedDockerImageFormats": true,
- "Containers.UsePreemptibleInstances": true,
- "Git": true,
- "Git.Repositories": true,
- "InstanceTypes": true,
- "InstanceTypes.*": true,
- "InstanceTypes.*.*": true,
- "Login": true,
- "Login.ProviderAppID": false,
- "Login.ProviderAppSecret": false,
- "Mail": true,
- "Mail.EmailFrom": true,
- "Mail.IssueReporterEmailFrom": true,
- "Mail.IssueReporterEmailTo": true,
- "Mail.MailchimpAPIKey": false,
- "Mail.MailchimpListID": false,
- "Mail.SendUserSetupNotificationEmail": true,
- "Mail.SupportEmailAddress": true,
- "ManagementToken": false,
- "PostgreSQL": true,
- "PostgreSQL.Connection": false,
- "PostgreSQL.ConnectionPool": true,
- "RemoteClusters": true,
- "RemoteClusters.*": true,
- "RemoteClusters.*.ActivateUsers": true,
- "RemoteClusters.*.Host": true,
- "RemoteClusters.*.Insecure": true,
- "RemoteClusters.*.Proxy": true,
- "RemoteClusters.*.Scheme": true,
- "Services": true,
- "Services.*": true,
- "Services.*.ExternalURL": true,
- "Services.*.InternalURLs": true,
- "Services.*.InternalURLs.*": true,
- "Services.*.InternalURLs.*.*": true,
- "SystemLogs": true,
- "SystemLogs.Format": true,
- "SystemLogs.LogLevel": true,
- "SystemLogs.MaxRequestLogParamsSize": true,
- "SystemRootToken": false,
- "TLS": true,
- "TLS.Certificate": true,
- "TLS.Insecure": true,
- "TLS.Key": false,
- "Users": true,
- "Users.AdminNotifierEmailFrom": true,
- "Users.AutoAdminFirstUser": false,
- "Users.AutoAdminUserWithEmail": false,
- "Users.AutoSetupNewUsers": true,
- "Users.AutoSetupNewUsersWithRepository": true,
- "Users.AutoSetupNewUsersWithVmUUID": true,
- "Users.AutoSetupUsernameBlacklist": false,
- "Users.EmailSubjectPrefix": true,
- "Users.NewInactiveUserNotificationRecipients": false,
- "Users.NewUserNotificationRecipients": false,
- "Users.NewUsersAreActive": true,
- "Users.UserNotifierEmailFrom": true,
- "Users.UserProfileNotificationAddress": true,
+ "API": true,
+ "API.AsyncPermissionsUpdateInterval": true,
+ "API.DisabledAPIs": true,
+ "API.MaxIndexDatabaseRead": true,
+ "API.MaxItemsPerResponse": true,
+ "API.MaxRequestAmplification": true,
+ "API.MaxRequestSize": true,
+ "API.RailsSessionSecretToken": false,
+ "API.RequestTimeout": true,
+ "AuditLogs": true,
+ "AuditLogs.MaxAge": true,
+ "AuditLogs.MaxDeleteBatch": true,
+ "AuditLogs.UnloggedAttributes": true,
+ "Collections": true,
+ "Collections.BlobSigning": true,
+ "Collections.BlobSigningKey": false,
+ "Collections.BlobSigningTTL": true,
+ "Collections.CollectionVersioning": true,
+ "Collections.DefaultReplication": true,
+ "Collections.DefaultTrashLifetime": true,
+ "Collections.PreserveVersionIfIdle": true,
+ "Collections.TrashSweepInterval": true,
+ "Containers": true,
+ "Containers.CloudVMs": true,
+ "Containers.CloudVMs.BootProbeCommand": true,
+ "Containers.CloudVMs.Driver": true,
+ "Containers.CloudVMs.DriverParameters": false,
+ "Containers.CloudVMs.Enable": true,
+ "Containers.CloudVMs.ImageID": true,
+ "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
+ "Containers.CloudVMs.MaxProbesPerSecond": true,
+ "Containers.CloudVMs.PollInterval": true,
+ "Containers.CloudVMs.ProbeInterval": true,
+ "Containers.CloudVMs.ResourceTags": true,
+ "Containers.CloudVMs.ResourceTags.*": true,
+ "Containers.CloudVMs.SSHPort": true,
+ "Containers.CloudVMs.SyncInterval": true,
+ "Containers.CloudVMs.TagKeyPrefix": true,
+ "Containers.CloudVMs.TimeoutBooting": true,
+ "Containers.CloudVMs.TimeoutIdle": true,
+ "Containers.CloudVMs.TimeoutProbe": true,
+ "Containers.CloudVMs.TimeoutShutdown": true,
+ "Containers.CloudVMs.TimeoutSignal": true,
+ "Containers.CloudVMs.TimeoutTERM": true,
+ "Containers.DefaultKeepCacheRAM": true,
+ "Containers.DispatchPrivateKey": true,
+ "Containers.JobsAPI": true,
+ "Containers.JobsAPI.CrunchJobUser": true,
+ "Containers.JobsAPI.CrunchJobWrapper": true,
+ "Containers.JobsAPI.CrunchRefreshTrigger": true,
+ "Containers.JobsAPI.DefaultDockerImage": true,
+ "Containers.JobsAPI.Enable": true,
+ "Containers.JobsAPI.GitInternalDir": true,
+ "Containers.JobsAPI.ReuseJobIfOutputsDiffer": true,
+ "Containers.Logging": true,
+ "Containers.Logging.LimitLogBytesPerJob": true,
+ "Containers.Logging.LogBytesPerEvent": true,
+ "Containers.Logging.LogPartialLineThrottlePeriod": true,
+ "Containers.Logging.LogSecondsBetweenEvents": true,
+ "Containers.Logging.LogThrottleBytes": true,
+ "Containers.Logging.LogThrottleLines": true,
+ "Containers.Logging.LogThrottlePeriod": true,
+ "Containers.Logging.LogUpdatePeriod": true,
+ "Containers.Logging.LogUpdateSize": true,
+ "Containers.Logging.MaxAge": true,
+ "Containers.LogReuseDecisions": true,
+ "Containers.MaxComputeVMs": true,
+ "Containers.MaxDispatchAttempts": true,
+ "Containers.MaxRetryAttempts": true,
+ "Containers.SLURM": true,
+ "Containers.SLURM.Managed": true,
+ "Containers.SLURM.Managed.AssignNodeHostname": true,
+ "Containers.SLURM.Managed.ComputeNodeDomain": false,
+ "Containers.SLURM.Managed.ComputeNodeNameservers": false,
+ "Containers.SLURM.Managed.DNSServerConfDir": true,
+ "Containers.SLURM.Managed.DNSServerConfTemplate": true,
+ "Containers.SLURM.Managed.DNSServerReloadCommand": false,
+ "Containers.SLURM.Managed.DNSServerUpdateCommand": false,
+ "Containers.StaleLockTimeout": true,
+ "Containers.SupportedDockerImageFormats": true,
+ "Containers.UsePreemptibleInstances": true,
+ "Git": true,
+ "Git.Repositories": true,
+ "InstanceTypes": true,
+ "InstanceTypes.*": true,
+ "InstanceTypes.*.*": true,
+ "Login": true,
+ "Login.ProviderAppID": false,
+ "Login.ProviderAppSecret": false,
+ "Mail": true,
+ "Mail.EmailFrom": true,
+ "Mail.IssueReporterEmailFrom": true,
+ "Mail.IssueReporterEmailTo": true,
+ "Mail.MailchimpAPIKey": false,
+ "Mail.MailchimpListID": false,
+ "Mail.SendUserSetupNotificationEmail": true,
+ "Mail.SupportEmailAddress": true,
+ "ManagementToken": false,
+ "PostgreSQL": true,
+ "PostgreSQL.Connection": false,
+ "PostgreSQL.ConnectionPool": true,
+ "RemoteClusters": true,
+ "RemoteClusters.*": true,
+ "RemoteClusters.*.ActivateUsers": true,
+ "RemoteClusters.*.Host": true,
+ "RemoteClusters.*.Insecure": true,
+ "RemoteClusters.*.Proxy": true,
+ "RemoteClusters.*.Scheme": true,
+ "Services": true,
+ "Services.*": true,
+ "Services.*.ExternalURL": true,
+ "Services.*.InternalURLs": true,
+ "Services.*.InternalURLs.*": true,
+ "Services.*.InternalURLs.*.*": true,
+ "SystemLogs": true,
+ "SystemLogs.Format": true,
+ "SystemLogs.LogLevel": true,
+ "SystemLogs.MaxRequestLogParamsSize": true,
+ "SystemRootToken": false,
+ "TLS": true,
+ "TLS.Certificate": true,
+ "TLS.Insecure": true,
+ "TLS.Key": false,
+ "Users": true,
+ "Users.AdminNotifierEmailFrom": true,
+ "Users.AutoAdminFirstUser": false,
+ "Users.AutoAdminUserWithEmail": false,
+ "Users.AutoSetupNewUsers": true,
+ "Users.AutoSetupNewUsersWithRepository": true,
+ "Users.AutoSetupNewUsersWithVmUUID": true,
+ "Users.AutoSetupUsernameBlacklist": false,
+ "Users.EmailSubjectPrefix": true,
+ "Users.NewInactiveUserNotificationRecipients": false,
+ "Users.NewUserNotificationRecipients": false,
+ "Users.NewUsersAreActive": true,
+ "Users.UserNotifierEmailFrom": true,
+ "Users.UserProfileNotificationAddress": true,
+ "Workbench": true,
+ "Workbench.ActivationContactLink": true,
+ "Workbench.APIClientConnectTimeout": true,
+ "Workbench.APIClientReceiveTimeout": true,
+ "Workbench.APIResponseCompression": true,
+ "Workbench.ApplicationMimetypesWithViewIcon": true,
+ "Workbench.ApplicationMimetypesWithViewIcon.*": true,
+ "Workbench.ArvadosDocsite": true,
+ "Workbench.ArvadosPublicDataDocURL": true,
+ "Workbench.EnableGettingStartedPopup": true,
+ "Workbench.EnablePublicProjectsPage": true,
+ "Workbench.FileViewersConfigURL": true,
+ "Workbench.LogViewerMaxBytes": true,
+ "Workbench.MultiSiteSearch": true,
+ "Workbench.Repositories": true,
+ "Workbench.RepositoryCache": true,
+ "Workbench.RunningJobLogRecordsToFetch": true,
+ "Workbench.SecretKeyBase": false,
+ "Workbench.SecretToken": false,
+ "Workbench.ShowRecentCollectionsOnDashboard": true,
+ "Workbench.ShowUserAgreementInline": true,
+ "Workbench.ShowUserNotifications": true,
+ "Workbench.SiteName": true,
+ "Workbench.Theme": true,
+ "Workbench.UserProfileFormFields": true,
+ "Workbench.UserProfileFormFields.*": true,
+ "Workbench.UserProfileFormFields.*.Type": true,
+ "Workbench.UserProfileFormFields.*.FormFieldTitle": true,
+ "Workbench.UserProfileFormFields.*.FormFieldDescription": true,
+ "Workbench.UserProfileFormFields.*.Required": true,
+ "Workbench.UserProfileFormMessage": true,
+ "Workbench.VocabularyURL": true,
}
func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index f9cd7bcd1..2e891911f 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -635,4 +635,54 @@ Clusters:
# their accounts are active on the remote cluster, activate
# them on this cluster too.
ActivateUsers: false
+
+ Workbench:
+ # Workbench1 configs
+ Theme: default
+ ActivationContactLink: mailto:info at arvados.org
+ ArvadosDocsite: https://doc.arvados.org
+ ArvadosPublicDataDocURL: https://playground.arvados.org/projects/public
+ ShowUserAgreementInline: false
+ SecretToken: ""
+ SecretKeyBase: ""
+ RepositoryCache: /var/www/arvados-workbench/current/tmp/git
+ UserProfileFormFields:
+ SAMPLE:
+ Type: text
+ FormFieldTitle: ""
+ FormFieldDescription: ""
+ Required: true
+ UserProfileFormMessage: 'Welcome to Arvados. All <span style="color:red">required fields</span> must be completed before you can proceed.'
+ ApplicationMimetypesWithViewIcon:
+ cwl: {}
+ fasta: {}
+ go: {}
+ javascript: {}
+ json: {}
+ pdf: {}
+ python: {}
+ x-python: {}
+ r: {}
+ rtf: {}
+ sam: {}
+ x-sh: {}
+ vnd.realvnc.bed: {}
+ xml: {}
+ xsl: {}
+ LogViewerMaxBytes: 1M
+ EnablePublicProjectsPage: true
+ EnableGettingStartedPopup: false
+ APIResponseCompression: true
+ APIClientConnectTimeout: 2m
+ APIClientReceiveTimeout: 5m
+ RunningJobLogRecordsToFetch: 2000
+ ShowRecentCollectionsOnDashboard: true
+ ShowUserNotifications: true
+ MultiSiteSearch: false
+ Repositories: true
+ SiteName: Arvados Workbench
+
+ # Workbench2 configs
+ VocabularyURL: ""
+ FileViewersConfigURL: ""
`)
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index fb01f25f8..adee06723 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -126,6 +126,38 @@ type Cluster struct {
UserNotifierEmailFrom string
UserProfileNotificationAddress string
}
+ Workbench struct {
+ ActivationContactLink string
+ APIClientConnectTimeout Duration
+ APIClientReceiveTimeout Duration
+ APIResponseCompression bool
+ ApplicationMimetypesWithViewIcon map[string]struct{}
+ ArvadosDocsite string
+ ArvadosPublicDataDocURL string
+ EnableGettingStartedPopup bool
+ EnablePublicProjectsPage bool
+ FileViewersConfigURL string
+ LogViewerMaxBytes ByteSize
+ MultiSiteSearch bool
+ Repositories bool
+ RepositoryCache string
+ RunningJobLogRecordsToFetch int
+ SecretKeyBase string
+ SecretToken string
+ ShowRecentCollectionsOnDashboard bool
+ ShowUserAgreementInline bool
+ ShowUserNotifications bool
+ SiteName string
+ Theme string
+ UserProfileFormFields map[string]struct {
+ Type string
+ FormFieldTitle string
+ FormFieldDescription string
+ Required bool
+ }
+ UserProfileFormMessage string
+ VocabularyURL string
+ }
}
type Services struct {
commit 1034b2ada9bc1a63b63e094085d2851eb8846ae5
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jun 6 16:03:10 2019 -0400
15000: Accept byte size suffixes (KB, KiB, ...) for numeric configs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
index 6aeb6fce2..522aa73b0 100644
--- a/services/api/lib/config_loader.rb
+++ b/services/api/lib/config_loader.rb
@@ -134,6 +134,31 @@ class ConfigLoader
cfg[k] = URI(cfg[k])
end
+ if cfgtype == Integer && cfg[k].is_a?(String)
+ v = cfg[k].sub(/B\s*$/, '')
+ if mt = /(-?\d*\.?\d+)\s*([KMGTPE]i?)$/.match(v)
+ if mt[1].index('.')
+ v = mt[1].to_f
+ else
+ v = mt[1].to_i
+ end
+ cfg[k] = v * {
+ 'K' => 1000,
+ 'Ki' => 1 << 10,
+ 'M' => 1000000,
+ 'Mi' => 1 << 20,
+ "G" => 1000000000,
+ "Gi" => 1 << 30,
+ "T" => 1000000000000,
+ "Ti" => 1 << 40,
+ "P" => 1000000000000000,
+ "Pi" => 1 << 50,
+ "E" => 1000000000000000000,
+ "Ei" => 1 << 60,
+ }[mt[2]]
+ end
+ end
+
if !cfg[k].is_a? cfgtype
raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
end
commit 28ac7d4d181a0a9678099e579310d463c5057f06
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jun 6 16:02:35 2019 -0400
15000: Accept negative durations in config, fix crashing error log.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
index 90b6d9ddc..6aeb6fce2 100644
--- a/services/api/lib/config_loader.rb
+++ b/services/api/lib/config_loader.rb
@@ -126,7 +126,7 @@ class ConfigLoader
if cfg[k].is_a? Integer
cfg[k] = cfg[k].seconds
elsif cfg[k].is_a? String
- cfg[k] = ConfigLoader.parse_duration cfg[k]
+ cfg[k] = ConfigLoader.parse_duration(cfg[k], cfgkey: cfgkey)
end
end
@@ -155,13 +155,13 @@ class ConfigLoader
end
end
- def self.parse_duration durstr
- duration_re = /(\d+(\.\d+)?)(s|m|h)/
+ def self.parse_duration durstr, cfgkey:
+ duration_re = /-?(\d+(\.\d+)?)(s|m|h)/
dursec = 0
while durstr != ""
mt = duration_re.match durstr
if !mt
- raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+ raise "#{cfgkey} not a valid duration: '#{durstr}', accepted suffixes are s, m, h"
end
multiplier = {s: 1, m: 60, h: 3600}
dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
commit e12c7f0bcb0f5cbe80a0274d1758c0dec57aa6e4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jun 6 13:44:23 2019 -0400
15000: Ensure all config defaults are captured in config struct.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 31a754493..fdcb6cc76 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -59,7 +59,7 @@ Clusters:
`
code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 1)
- c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
+ c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
}
func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 94cd8fcbf..a7b53489c 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -35,11 +35,13 @@ Clusters:
InternalURLs: {}
ExternalURL: ""
GitSSH:
+ InternalURLs: {}
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
ExternalURL: "-"
SSO:
+ InternalURLs: {}
ExternalURL: ""
Keepproxy:
InternalURLs: {}
@@ -54,13 +56,16 @@ Clusters:
InternalURLs: {}
ExternalURL: "-"
Composer:
+ InternalURLs: {}
ExternalURL: ""
WebShell:
+ InternalURLs: {}
ExternalURL: ""
Workbench1:
InternalURLs: {}
ExternalURL: ""
Workbench2:
+ InternalURLs: {}
ExternalURL: ""
Nodemanager:
InternalURLs: {}
@@ -113,7 +118,7 @@ Clusters:
# Interval (seconds) between asynchronous permission view updates. Any
# permission-updating API called with the 'async' parameter schedules a an
# update on the permission view in the future, if not already scheduled.
- AsyncPermissionsUpdateInterval: 20
+ AsyncPermissionsUpdateInterval: 20s
# Maximum number of concurrent outgoing requests to make while
# serving a single incoming multi-cluster (federated) request.
@@ -260,7 +265,7 @@ Clusters:
# Interval (seconds) between trash sweeps. During a trash sweep,
# collections are marked as trash if their trash_at time has
# arrived, and deleted if their delete_at time has arrived.
- TrashSweepInterval: 60
+ TrashSweepInterval: 60s
# If true, enable collection versioning.
# When a collection's preserve_version field is true or the current version
@@ -269,10 +274,10 @@ Clusters:
# the current collection.
CollectionVersioning: false
- # 0 = auto-create a new version on every update.
- # -1 = never auto-create new versions.
- # > 0 = auto-create a new version when older than the specified number of seconds.
- PreserveVersionIfIdle: -1
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
Login:
# These settings are provided by your OAuth2 provider (e.g.,
@@ -336,12 +341,6 @@ Clusters:
# scheduling parameter parameter set.
UsePreemptibleInstances: false
- # Include details about job reuse decisions in the server log. This
- # causes additional database queries to run, so it should not be
- # enabled unless you expect to examine the resulting logs for
- # troubleshooting purposes.
- LogReuseDecisions: false
-
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -366,8 +365,8 @@ Clusters:
LogBytesPerEvent: 4096
LogSecondsBetweenEvents: 1
- # The sample period for throttling logs, in seconds.
- LogThrottlePeriod: 60
+ # The sample period for throttling logs.
+ LogThrottlePeriod: 60s
# Maximum number of bytes that job can log over crunch_log_throttle_period
# before being silenced until the end of the period.
@@ -381,18 +380,18 @@ Clusters:
# silenced by throttling are not counted against this total.
LimitLogBytesPerJob: 67108864
- LogPartialLineThrottlePeriod: 5
+ LogPartialLineThrottlePeriod: 5s
- # Container logs are written to Keep and saved in a collection,
- # which is updated periodically while the container runs. This
- # value sets the interval (given in seconds) between collection
- # updates.
- LogUpdatePeriod: 1800
+ # Container logs are written to Keep and saved in a
+ # collection, which is updated periodically while the
+ # container runs. This value sets the interval between
+ # collection updates.
+ LogUpdatePeriod: 30m
# The log collection is also updated when the specified amount of
# log data (given in bytes) is produced in less than one update
# period.
- LogUpdateSize: 33554432
+ LogUpdateSize: 32MiB
SLURM:
Managed:
@@ -613,8 +612,20 @@ Clusters:
Insecure: false
ActivateUsers: false
SAMPLE:
+ # API endpoint host or host:port; default is {id}.arvadosapi.com
Host: sample.arvadosapi.com
+
+ # Perform a proxy request when a local client requests an
+ # object belonging to this remote.
Proxy: false
+
+ # Default "https". Can be set to "http" for testing.
Scheme: https
+
+ # Disable TLS verify. Can be set to true for testing.
Insecure: false
+
+ # When users present tokens issued by this remote cluster, and
+ # their accounts are active on the remote cluster, activate
+ # them on this cluster too.
ActivateUsers: false
diff --git a/lib/config/export.go b/lib/config/export.go
index 98263c2b7..3c485e2b4 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -55,62 +55,143 @@ func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
// exists.
var whitelist = map[string]bool{
// | sort -t'"' -k2,2
- "API": true,
- "API.MaxItemsPerResponse": true,
- "API.MaxRequestAmplification": true,
- "API.RequestTimeout": true,
- "Containers": true,
- "Containers.CloudVMs": true,
- "Containers.CloudVMs.BootProbeCommand": true,
- "Containers.CloudVMs.Driver": true,
- "Containers.CloudVMs.DriverParameters": false,
- "Containers.CloudVMs.Enable": true,
- "Containers.CloudVMs.ImageID": true,
- "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
- "Containers.CloudVMs.MaxProbesPerSecond": true,
- "Containers.CloudVMs.PollInterval": true,
- "Containers.CloudVMs.ProbeInterval": true,
- "Containers.CloudVMs.ResourceTags": true,
- "Containers.CloudVMs.ResourceTags.*": true,
- "Containers.CloudVMs.SSHPort": true,
- "Containers.CloudVMs.SyncInterval": true,
- "Containers.CloudVMs.TagKeyPrefix": true,
- "Containers.CloudVMs.TimeoutBooting": true,
- "Containers.CloudVMs.TimeoutIdle": true,
- "Containers.CloudVMs.TimeoutProbe": true,
- "Containers.CloudVMs.TimeoutShutdown": true,
- "Containers.CloudVMs.TimeoutSignal": true,
- "Containers.CloudVMs.TimeoutTERM": true,
- "Containers.DispatchPrivateKey": true,
- "Containers.StaleLockTimeout": true,
- "InstanceTypes": true,
- "InstanceTypes.*": true,
- "InstanceTypes.*.*": true,
- "ManagementToken": false,
- "PostgreSQL": true,
- "PostgreSQL.Connection": false,
- "PostgreSQL.ConnectionPool": true,
- "RemoteClusters": true,
- "RemoteClusters.*": true,
- "RemoteClusters.*.Host": true,
- "RemoteClusters.*.Insecure": true,
- "RemoteClusters.*.Proxy": true,
- "RemoteClusters.*.Scheme": true,
- "Services": true,
- "Services.*": true,
- "Services.*.ExternalURL": true,
- "Services.*.InternalURLs": true,
- "Services.*.InternalURLs.*": true,
- "Services.*.InternalURLs.*.*": true,
- "SystemLogs": true,
- "SystemLogs.Format": true,
- "SystemLogs.LogLevel": true,
- "SystemLogs.MaxRequestLogParamsSize": true,
- "SystemRootToken": false,
- "TLS": true,
- "TLS.Certificate": true,
- "TLS.Insecure": true,
- "TLS.Key": false,
+ "API": true,
+ "API.AsyncPermissionsUpdateInterval": true,
+ "API.DisabledAPIs": true,
+ "API.MaxIndexDatabaseRead": true,
+ "API.MaxItemsPerResponse": true,
+ "API.MaxRequestAmplification": true,
+ "API.MaxRequestSize": true,
+ "API.RailsSessionSecretToken": false,
+ "API.RequestTimeout": true,
+ "AuditLogs": true,
+ "AuditLogs.MaxAge": true,
+ "AuditLogs.MaxDeleteBatch": true,
+ "AuditLogs.UnloggedAttributes": true,
+ "Collections": true,
+ "Collections.BlobSigning": true,
+ "Collections.BlobSigningKey": false,
+ "Collections.BlobSigningTTL": true,
+ "Collections.CollectionVersioning": true,
+ "Collections.DefaultReplication": true,
+ "Collections.DefaultTrashLifetime": true,
+ "Collections.PreserveVersionIfIdle": true,
+ "Collections.TrashSweepInterval": true,
+ "Containers": true,
+ "Containers.CloudVMs": true,
+ "Containers.CloudVMs.BootProbeCommand": true,
+ "Containers.CloudVMs.Driver": true,
+ "Containers.CloudVMs.DriverParameters": false,
+ "Containers.CloudVMs.Enable": true,
+ "Containers.CloudVMs.ImageID": true,
+ "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
+ "Containers.CloudVMs.MaxProbesPerSecond": true,
+ "Containers.CloudVMs.PollInterval": true,
+ "Containers.CloudVMs.ProbeInterval": true,
+ "Containers.CloudVMs.ResourceTags": true,
+ "Containers.CloudVMs.ResourceTags.*": true,
+ "Containers.CloudVMs.SSHPort": true,
+ "Containers.CloudVMs.SyncInterval": true,
+ "Containers.CloudVMs.TagKeyPrefix": true,
+ "Containers.CloudVMs.TimeoutBooting": true,
+ "Containers.CloudVMs.TimeoutIdle": true,
+ "Containers.CloudVMs.TimeoutProbe": true,
+ "Containers.CloudVMs.TimeoutShutdown": true,
+ "Containers.CloudVMs.TimeoutSignal": true,
+ "Containers.CloudVMs.TimeoutTERM": true,
+ "Containers.DefaultKeepCacheRAM": true,
+ "Containers.DispatchPrivateKey": true,
+ "Containers.JobsAPI": true,
+ "Containers.JobsAPI.CrunchJobUser": true,
+ "Containers.JobsAPI.CrunchJobWrapper": true,
+ "Containers.JobsAPI.CrunchRefreshTrigger": true,
+ "Containers.JobsAPI.DefaultDockerImage": true,
+ "Containers.JobsAPI.Enable": true,
+ "Containers.JobsAPI.GitInternalDir": true,
+ "Containers.JobsAPI.ReuseJobIfOutputsDiffer": true,
+ "Containers.Logging": true,
+ "Containers.Logging.LimitLogBytesPerJob": true,
+ "Containers.Logging.LogBytesPerEvent": true,
+ "Containers.Logging.LogPartialLineThrottlePeriod": true,
+ "Containers.Logging.LogSecondsBetweenEvents": true,
+ "Containers.Logging.LogThrottleBytes": true,
+ "Containers.Logging.LogThrottleLines": true,
+ "Containers.Logging.LogThrottlePeriod": true,
+ "Containers.Logging.LogUpdatePeriod": true,
+ "Containers.Logging.LogUpdateSize": true,
+ "Containers.Logging.MaxAge": true,
+ "Containers.LogReuseDecisions": true,
+ "Containers.MaxComputeVMs": true,
+ "Containers.MaxDispatchAttempts": true,
+ "Containers.MaxRetryAttempts": true,
+ "Containers.SLURM": true,
+ "Containers.SLURM.Managed": true,
+ "Containers.SLURM.Managed.AssignNodeHostname": true,
+ "Containers.SLURM.Managed.ComputeNodeDomain": false,
+ "Containers.SLURM.Managed.ComputeNodeNameservers": false,
+ "Containers.SLURM.Managed.DNSServerConfDir": true,
+ "Containers.SLURM.Managed.DNSServerConfTemplate": true,
+ "Containers.SLURM.Managed.DNSServerReloadCommand": false,
+ "Containers.SLURM.Managed.DNSServerUpdateCommand": false,
+ "Containers.StaleLockTimeout": true,
+ "Containers.SupportedDockerImageFormats": true,
+ "Containers.UsePreemptibleInstances": true,
+ "Git": true,
+ "Git.Repositories": true,
+ "InstanceTypes": true,
+ "InstanceTypes.*": true,
+ "InstanceTypes.*.*": true,
+ "Login": true,
+ "Login.ProviderAppID": false,
+ "Login.ProviderAppSecret": false,
+ "Mail": true,
+ "Mail.EmailFrom": true,
+ "Mail.IssueReporterEmailFrom": true,
+ "Mail.IssueReporterEmailTo": true,
+ "Mail.MailchimpAPIKey": false,
+ "Mail.MailchimpListID": false,
+ "Mail.SendUserSetupNotificationEmail": true,
+ "Mail.SupportEmailAddress": true,
+ "ManagementToken": false,
+ "PostgreSQL": true,
+ "PostgreSQL.Connection": false,
+ "PostgreSQL.ConnectionPool": true,
+ "RemoteClusters": true,
+ "RemoteClusters.*": true,
+ "RemoteClusters.*.ActivateUsers": true,
+ "RemoteClusters.*.Host": true,
+ "RemoteClusters.*.Insecure": true,
+ "RemoteClusters.*.Proxy": true,
+ "RemoteClusters.*.Scheme": true,
+ "Services": true,
+ "Services.*": true,
+ "Services.*.ExternalURL": true,
+ "Services.*.InternalURLs": true,
+ "Services.*.InternalURLs.*": true,
+ "Services.*.InternalURLs.*.*": true,
+ "SystemLogs": true,
+ "SystemLogs.Format": true,
+ "SystemLogs.LogLevel": true,
+ "SystemLogs.MaxRequestLogParamsSize": true,
+ "SystemRootToken": false,
+ "TLS": true,
+ "TLS.Certificate": true,
+ "TLS.Insecure": true,
+ "TLS.Key": false,
+ "Users": true,
+ "Users.AdminNotifierEmailFrom": true,
+ "Users.AutoAdminFirstUser": false,
+ "Users.AutoAdminUserWithEmail": false,
+ "Users.AutoSetupNewUsers": true,
+ "Users.AutoSetupNewUsersWithRepository": true,
+ "Users.AutoSetupNewUsersWithVmUUID": true,
+ "Users.AutoSetupUsernameBlacklist": false,
+ "Users.EmailSubjectPrefix": true,
+ "Users.NewInactiveUserNotificationRecipients": false,
+ "Users.NewUserNotificationRecipients": false,
+ "Users.NewUsersAreActive": true,
+ "Users.UserNotifierEmailFrom": true,
+ "Users.UserProfileNotificationAddress": true,
}
func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 3492615e9..f9cd7bcd1 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -41,11 +41,13 @@ Clusters:
InternalURLs: {}
ExternalURL: ""
GitSSH:
+ InternalURLs: {}
ExternalURL: ""
DispatchCloud:
InternalURLs: {}
ExternalURL: "-"
SSO:
+ InternalURLs: {}
ExternalURL: ""
Keepproxy:
InternalURLs: {}
@@ -60,13 +62,16 @@ Clusters:
InternalURLs: {}
ExternalURL: "-"
Composer:
+ InternalURLs: {}
ExternalURL: ""
WebShell:
+ InternalURLs: {}
ExternalURL: ""
Workbench1:
InternalURLs: {}
ExternalURL: ""
Workbench2:
+ InternalURLs: {}
ExternalURL: ""
Nodemanager:
InternalURLs: {}
@@ -119,7 +124,7 @@ Clusters:
# Interval (seconds) between asynchronous permission view updates. Any
# permission-updating API called with the 'async' parameter schedules a an
# update on the permission view in the future, if not already scheduled.
- AsyncPermissionsUpdateInterval: 20
+ AsyncPermissionsUpdateInterval: 20s
# Maximum number of concurrent outgoing requests to make while
# serving a single incoming multi-cluster (federated) request.
@@ -266,7 +271,7 @@ Clusters:
# Interval (seconds) between trash sweeps. During a trash sweep,
# collections are marked as trash if their trash_at time has
# arrived, and deleted if their delete_at time has arrived.
- TrashSweepInterval: 60
+ TrashSweepInterval: 60s
# If true, enable collection versioning.
# When a collection's preserve_version field is true or the current version
@@ -275,10 +280,10 @@ Clusters:
# the current collection.
CollectionVersioning: false
- # 0 = auto-create a new version on every update.
- # -1 = never auto-create new versions.
- # > 0 = auto-create a new version when older than the specified number of seconds.
- PreserveVersionIfIdle: -1
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
Login:
# These settings are provided by your OAuth2 provider (e.g.,
@@ -342,12 +347,6 @@ Clusters:
# scheduling parameter parameter set.
UsePreemptibleInstances: false
- # Include details about job reuse decisions in the server log. This
- # causes additional database queries to run, so it should not be
- # enabled unless you expect to examine the resulting logs for
- # troubleshooting purposes.
- LogReuseDecisions: false
-
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
# (experimental) cloud dispatcher for executing containers on
# worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -372,8 +371,8 @@ Clusters:
LogBytesPerEvent: 4096
LogSecondsBetweenEvents: 1
- # The sample period for throttling logs, in seconds.
- LogThrottlePeriod: 60
+ # The sample period for throttling logs.
+ LogThrottlePeriod: 60s
# Maximum number of bytes that job can log over crunch_log_throttle_period
# before being silenced until the end of the period.
@@ -387,18 +386,18 @@ Clusters:
# silenced by throttling are not counted against this total.
LimitLogBytesPerJob: 67108864
- LogPartialLineThrottlePeriod: 5
+ LogPartialLineThrottlePeriod: 5s
- # Container logs are written to Keep and saved in a collection,
- # which is updated periodically while the container runs. This
- # value sets the interval (given in seconds) between collection
- # updates.
- LogUpdatePeriod: 1800
+ # Container logs are written to Keep and saved in a
+ # collection, which is updated periodically while the
+ # container runs. This value sets the interval between
+ # collection updates.
+ LogUpdatePeriod: 30m
# The log collection is also updated when the specified amount of
# log data (given in bytes) is produced in less than one update
# period.
- LogUpdateSize: 33554432
+ LogUpdateSize: 32MiB
SLURM:
Managed:
@@ -619,9 +618,21 @@ Clusters:
Insecure: false
ActivateUsers: false
SAMPLE:
+ # API endpoint host or host:port; default is {id}.arvadosapi.com
Host: sample.arvadosapi.com
+
+ # Perform a proxy request when a local client requests an
+ # object belonging to this remote.
Proxy: false
+
+ # Default "https". Can be set to "http" for testing.
Scheme: https
+
+ # Disable TLS verify. Can be set to true for testing.
Insecure: false
+
+ # When users present tokens issued by this remote cluster, and
+ # their accounts are active on the remote cluster, activate
+ # them on this cluster too.
ActivateUsers: false
`)
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 6ce81bb5f..6b014476b 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -97,6 +97,24 @@ Clusters:
c.Check(logs, check.HasLen, 2)
}
+func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
+ var logbuf bytes.Buffer
+ logger := logrus.New()
+ logger.Out = &logbuf
+ var supplied map[string]interface{}
+ yaml.Unmarshal(DefaultYAML, &supplied)
+ cfg, err := Load(bytes.NewBuffer(DefaultYAML), logger)
+ c.Assert(err, check.IsNil)
+ var loaded map[string]interface{}
+ buf, err := yaml.Marshal(cfg)
+ c.Assert(err, check.IsNil)
+ err = yaml.Unmarshal(buf, &loaded)
+ c.Assert(err, check.IsNil)
+
+ logExtraKeys(logger, loaded, supplied, "")
+ c.Check(logbuf.String(), check.Equals, "")
+}
+
func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
var logbuf bytes.Buffer
logger := logrus.New()
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 1c859cfc5..7d8e7a433 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -57,12 +57,10 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
cluster := &arvados.Cluster{
ClusterID: "zhome",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
- API: arvados.API{
- MaxItemsPerResponse: 1000,
- MaxRequestAmplification: 4,
- },
}
+ cluster.TLS.Insecure = true
+ cluster.API.MaxItemsPerResponse = 1000
+ cluster.API.MaxRequestAmplification = 4
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 06a5a8411..cb69f2f6b 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -42,8 +42,8 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
s.cluster = &arvados.Cluster{
ClusterID: "zzzzz",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
}
+ s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
s.handler = newHandler(s.ctx, s.cluster, "")
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index a398af97b..ae7f138b1 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -36,8 +36,8 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
handler := &Handler{Cluster: &arvados.Cluster{
ClusterID: "zzzzz",
PostgreSQL: integrationTestCluster().PostgreSQL,
- TLS: arvados.TLS{Insecure: true},
}}
+ handler.Cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index d96bf2517..fb01f25f8 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -50,12 +50,6 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
}
}
-type API struct {
- MaxItemsPerResponse int
- MaxRequestAmplification int
- RequestTimeout Duration
-}
-
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
@@ -65,28 +59,98 @@ type Cluster struct {
Containers ContainersConfig
RemoteClusters map[string]RemoteCluster
PostgreSQL PostgreSQL
- API API
- SystemLogs SystemLogs
- TLS TLS
+
+ API struct {
+ AsyncPermissionsUpdateInterval Duration
+ DisabledAPIs []string
+ MaxIndexDatabaseRead int
+ MaxItemsPerResponse int
+ MaxRequestAmplification int
+ MaxRequestSize int
+ RailsSessionSecretToken string
+ RequestTimeout Duration
+ }
+ AuditLogs struct {
+ MaxAge Duration
+ MaxDeleteBatch int
+ UnloggedAttributes []string
+ }
+ Collections struct {
+ BlobSigning bool
+ BlobSigningKey string
+ DefaultReplication int
+ BlobSigningTTL Duration
+ DefaultTrashLifetime Duration
+ TrashSweepInterval Duration
+ CollectionVersioning bool
+ PreserveVersionIfIdle Duration
+ }
+ Git struct {
+ Repositories string
+ }
+ Login struct {
+ ProviderAppSecret string
+ ProviderAppID string
+ }
+ Mail struct {
+ MailchimpAPIKey string
+ MailchimpListID string
+ SendUserSetupNotificationEmail string
+ IssueReporterEmailFrom string
+ IssueReporterEmailTo string
+ SupportEmailAddress string
+ EmailFrom string
+ }
+ SystemLogs struct {
+ LogLevel string
+ Format string
+ MaxRequestLogParamsSize int
+ }
+ TLS struct {
+ Certificate string
+ Key string
+ Insecure bool
+ }
+ Users struct {
+ AdminNotifierEmailFrom string
+ AutoAdminFirstUser bool
+ AutoAdminUserWithEmail string
+ AutoSetupNewUsers bool
+ AutoSetupNewUsersWithRepository bool
+ AutoSetupNewUsersWithVmUUID string
+ AutoSetupUsernameBlacklist []string
+ EmailSubjectPrefix string
+ NewInactiveUserNotificationRecipients []string
+ NewUserNotificationRecipients []string
+ NewUsersAreActive bool
+ UserNotifierEmailFrom string
+ UserProfileNotificationAddress string
+ }
}
type Services struct {
- Controller Service
- DispatchCloud Service
- Health Service
- Keepbalance Service
- Keepproxy Service
- Keepstore Service
- Nodemanager Service
- RailsAPI Service
- WebDAV Service
- Websocket Service
- Workbench1 Service
- Workbench2 Service
+ Composer Service
+ Controller Service
+ DispatchCloud Service
+ GitHTTP Service
+ GitSSH Service
+ Health Service
+ Keepbalance Service
+ Keepproxy Service
+ Keepstore Service
+ Nodemanager Service
+ RailsAPI Service
+ SSO Service
+ WebDAVDownload Service
+ WebDAV Service
+ WebShell Service
+ Websocket Service
+ Workbench1 Service
+ Workbench2 Service
}
type Service struct {
- InternalURLs map[URL]ServiceInstance `json:",omitempty"`
+ InternalURLs map[URL]ServiceInstance
ExternalURL URL
}
@@ -109,12 +173,6 @@ func (su URL) MarshalText() ([]byte, error) {
type ServiceInstance struct{}
-type SystemLogs struct {
- LogLevel string
- Format string
- MaxRequestLogParamsSize int
-}
-
type PostgreSQL struct {
Connection PostgreSQLConnection
ConnectionPool int
@@ -123,15 +181,11 @@ type PostgreSQL struct {
type PostgreSQLConnection map[string]string
type RemoteCluster struct {
- // API endpoint host or host:port; default is {id}.arvadosapi.com
- Host string
- // Perform a proxy request when a local client requests an
- // object belonging to this remote.
- Proxy bool
- // Scheme, default "https". Can be set to "http" for testing.
- Scheme string
- // Disable TLS verify. Can be set to true for testing.
- Insecure bool
+ Host string
+ Proxy bool
+ Scheme string
+ Insecure bool
+ ActivateUsers bool
}
type InstanceType struct {
@@ -147,9 +201,49 @@ type InstanceType struct {
}
type ContainersConfig struct {
- CloudVMs CloudVMsConfig
- DispatchPrivateKey string
- StaleLockTimeout Duration
+ CloudVMs CloudVMsConfig
+ DefaultKeepCacheRAM ByteSize
+ DispatchPrivateKey string
+ LogReuseDecisions bool
+ MaxComputeVMs int
+ MaxDispatchAttempts int
+ MaxRetryAttempts int
+ StaleLockTimeout Duration
+ SupportedDockerImageFormats []string
+ UsePreemptibleInstances bool
+
+ JobsAPI struct {
+ Enable string
+ GitInternalDir string
+ DefaultDockerImage string
+ CrunchJobWrapper string
+ CrunchJobUser string
+ CrunchRefreshTrigger string
+ ReuseJobIfOutputsDiffer bool
+ }
+ Logging struct {
+ MaxAge Duration
+ LogBytesPerEvent int
+ LogSecondsBetweenEvents int
+ LogThrottlePeriod Duration
+ LogThrottleBytes int
+ LogThrottleLines int
+ LimitLogBytesPerJob int
+ LogPartialLineThrottlePeriod Duration
+ LogUpdatePeriod Duration
+ LogUpdateSize ByteSize
+ }
+ SLURM struct {
+ Managed struct {
+ DNSServerConfDir string
+ DNSServerConfTemplate string
+ DNSServerReloadCommand string
+ DNSServerUpdateCommand string
+ ComputeNodeDomain string
+ ComputeNodeNameservers []string
+ AssignNodeHostname string
+ }
+ }
}
type CloudVMsConfig struct {
@@ -269,9 +363,3 @@ func (svcs Services) Map() map[ServiceName]Service {
ServiceNameKeepstore: svcs.Keepstore,
}
}
-
-type TLS struct {
- Certificate string
- Key string
- Insecure bool
-}
commit 29bfdcc937816027c162071cc7044eb8e695ab4c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Jun 6 10:54:26 2019 -0400
15000: Export safe parts of cluster config at /arvados/v1/config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/cmd.go b/lib/config/cmd.go
index 41a1d7d21..39df4ec17 100644
--- a/lib/config/cmd.go
+++ b/lib/config/cmd.go
@@ -12,12 +12,11 @@ import (
"os"
"os/exec"
- "git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/ghodss/yaml"
)
-var DumpCommand cmd.Handler = dumpCommand{}
+var DumpCommand dumpCommand
type dumpCommand struct{}
@@ -48,7 +47,7 @@ func (dumpCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
return 0
}
-var CheckCommand cmd.Handler = checkCommand{}
+var CheckCommand checkCommand
type checkCommand struct{}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index e4d838f85..31a754493 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -7,11 +7,18 @@ package config
import (
"bytes"
+ "git.curoverse.com/arvados.git/lib/cmd"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&CommandSuite{})
+var (
+ // Commands must satisfy cmd.Handler interface
+ _ cmd.Handler = dumpCommand{}
+ _ cmd.Handler = checkCommand{}
+)
+
type CommandSuite struct{}
func (s *CommandSuite) TestBadArg(c *check.C) {
diff --git a/lib/config/export.go b/lib/config/export.go
new file mode 100644
index 000000000..98263c2b7
--- /dev/null
+++ b/lib/config/export.go
@@ -0,0 +1,144 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+// ExportJSON writes a JSON object with the safe (non-secret) portions
+// of the cluster config to w.
+func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
+ buf, err := json.Marshal(cluster)
+ if err != nil {
+ return err
+ }
+ var m map[string]interface{}
+ err = json.Unmarshal(buf, &m)
+ if err != nil {
+ return err
+ }
+ err = redactUnsafe(m, "", "")
+ if err != nil {
+ return err
+ }
+ return json.NewEncoder(w).Encode(m)
+}
+
+// whitelist classifies configs as safe/unsafe to reveal to
+// unauthenticated clients.
+//
+// Every config entry must either be listed explicitly here along with
+// all of its parent keys (e.g., "API" + "API.RequestTimeout"), or
+// have an ancestor listed as false (e.g.,
+// "PostgreSQL.Connection.password" has an ancestor
+// "PostgreSQL.Connection" with a false value). Otherwise, it is a bug
+// which should be caught by tests.
+//
+// Example: API.RequestTimeout is safe because whitelist["API"] == and
+// whitelist["API.RequestTimeout"] == true.
+//
+// Example: PostgreSQL.Connection.password is not safe because
+// whitelist["PostgreSQL.Connection"] == false.
+//
+// Example: PostgreSQL.BadKey would cause an error because
+// whitelist["PostgreSQL"] isn't false, and neither
+// whitelist["PostgreSQL.BadKey"] nor whitelist["PostgreSQL.*"]
+// exists.
+var whitelist = map[string]bool{
+ // | sort -t'"' -k2,2
+ "API": true,
+ "API.MaxItemsPerResponse": true,
+ "API.MaxRequestAmplification": true,
+ "API.RequestTimeout": true,
+ "Containers": true,
+ "Containers.CloudVMs": true,
+ "Containers.CloudVMs.BootProbeCommand": true,
+ "Containers.CloudVMs.Driver": true,
+ "Containers.CloudVMs.DriverParameters": false,
+ "Containers.CloudVMs.Enable": true,
+ "Containers.CloudVMs.ImageID": true,
+ "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
+ "Containers.CloudVMs.MaxProbesPerSecond": true,
+ "Containers.CloudVMs.PollInterval": true,
+ "Containers.CloudVMs.ProbeInterval": true,
+ "Containers.CloudVMs.ResourceTags": true,
+ "Containers.CloudVMs.ResourceTags.*": true,
+ "Containers.CloudVMs.SSHPort": true,
+ "Containers.CloudVMs.SyncInterval": true,
+ "Containers.CloudVMs.TagKeyPrefix": true,
+ "Containers.CloudVMs.TimeoutBooting": true,
+ "Containers.CloudVMs.TimeoutIdle": true,
+ "Containers.CloudVMs.TimeoutProbe": true,
+ "Containers.CloudVMs.TimeoutShutdown": true,
+ "Containers.CloudVMs.TimeoutSignal": true,
+ "Containers.CloudVMs.TimeoutTERM": true,
+ "Containers.DispatchPrivateKey": true,
+ "Containers.StaleLockTimeout": true,
+ "InstanceTypes": true,
+ "InstanceTypes.*": true,
+ "InstanceTypes.*.*": true,
+ "ManagementToken": false,
+ "PostgreSQL": true,
+ "PostgreSQL.Connection": false,
+ "PostgreSQL.ConnectionPool": true,
+ "RemoteClusters": true,
+ "RemoteClusters.*": true,
+ "RemoteClusters.*.Host": true,
+ "RemoteClusters.*.Insecure": true,
+ "RemoteClusters.*.Proxy": true,
+ "RemoteClusters.*.Scheme": true,
+ "Services": true,
+ "Services.*": true,
+ "Services.*.ExternalURL": true,
+ "Services.*.InternalURLs": true,
+ "Services.*.InternalURLs.*": true,
+ "Services.*.InternalURLs.*.*": true,
+ "SystemLogs": true,
+ "SystemLogs.Format": true,
+ "SystemLogs.LogLevel": true,
+ "SystemLogs.MaxRequestLogParamsSize": true,
+ "SystemRootToken": false,
+ "TLS": true,
+ "TLS.Certificate": true,
+ "TLS.Insecure": true,
+ "TLS.Key": false,
+}
+
+func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
+ var errs []string
+ for k, v := range m {
+ lookupKey := k
+ safe, ok := whitelist[lookupPrefix+k]
+ if !ok {
+ lookupKey = "*"
+ safe, ok = whitelist[lookupPrefix+"*"]
+ }
+ if !ok {
+ errs = append(errs, fmt.Sprintf("config bug: key %q not in whitelist map", lookupPrefix+k))
+ continue
+ }
+ if !safe {
+ delete(m, k)
+ continue
+ }
+ if v, ok := v.(map[string]interface{}); ok {
+ err := redactUnsafe(v, mPrefix+k+".", lookupPrefix+lookupKey+".")
+ if err != nil {
+ errs = append(errs, err.Error())
+ }
+ }
+ }
+ if len(errs) > 0 {
+ return errors.New(strings.Join(errs, "\n"))
+ }
+ return nil
+}
diff --git a/lib/config/export_test.go b/lib/config/export_test.go
new file mode 100644
index 000000000..581e54cdc
--- /dev/null
+++ b/lib/config/export_test.go
@@ -0,0 +1,37 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "regexp"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ExportSuite{})
+
+type ExportSuite struct{}
+
+func (s *ExportSuite) TestExport(c *check.C) {
+ confdata := bytes.Replace(DefaultYAML, []byte("SAMPLE"), []byte("testkey"), -1)
+ cfg, err := Load(bytes.NewBuffer(confdata), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ cluster := cfg.Clusters["xxxxx"]
+ cluster.ManagementToken = "abcdefg"
+
+ var exported bytes.Buffer
+ err = ExportJSON(&exported, &cluster)
+ c.Check(err, check.IsNil)
+ if err != nil {
+ c.Logf("If all the new keys are safe, add these to whitelist in export.go:")
+ for _, k := range regexp.MustCompile(`"[^"]*"`).FindAllString(err.Error(), -1) {
+ c.Logf("\t%q: true,", strings.Replace(k, `"`, "", -1))
+ }
+ }
+ c.Check(exported.String(), check.Not(check.Matches), `(?ms).*abcdefg.*`)
+}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 2c3ce1d4f..12faacdd4 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -5,16 +5,19 @@
package controller
import (
+ "bytes"
"context"
"database/sql"
"errors"
"fmt"
+ "io"
"net/http"
"net/url"
"strings"
"sync"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -73,6 +76,18 @@ func (h *Handler) setup() {
Prefix: "/_health/",
Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
})
+
+ mux.Handle("/arvados/v1/config", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buf bytes.Buffer
+ err := config.ExportJSON(&buf, h.Cluster)
+ if err != nil {
+ httpserver.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ io.Copy(w, &buf)
+ }))
+
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
hs = h.setupProxyRemoteCluster(hs)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index a1efaacdd..06a5a8411 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -53,6 +53,23 @@ func (s *HandlerSuite) TearDownTest(c *check.C) {
s.cancel()
}
+func (s *HandlerSuite) TestConfigExport(c *check.C) {
+ s.cluster.Containers.CloudVMs.PollInterval = arvados.Duration(23 * time.Second)
+ req := httptest.NewRequest("GET", "/arvados/v1/config", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ var cluster arvados.Cluster
+ c.Log(resp.Body.String())
+ err := json.Unmarshal(resp.Body.Bytes(), &cluster)
+ c.Check(err, check.IsNil)
+ c.Check(cluster.ManagementToken, check.Equals, "")
+ c.Check(cluster.SystemRootToken, check.Equals, "")
+ c.Check(cluster.TLS.Insecure, check.Equals, true)
+ c.Check(cluster.Services.RailsAPI, check.DeepEquals, s.cluster.Services.RailsAPI)
+ c.Check(cluster.Containers.CloudVMs.PollInterval, check.Equals, arvados.Duration(23*time.Second))
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list