[ARVADOS] updated: 1.3.0-551-g42218d31e
Git user
git at public.curoverse.com
Mon Mar 25 13:14:50 UTC 2019
Summary of changes:
lib/config/config.defaults.yml | 4 +
.../controllers/arvados/v1/schema_controller.rb | 49 ++++----
services/api/app/controllers/static_controller.rb | 2 +-
.../api/app/models/api_client_authorization.rb | 6 +-
services/api/app/models/collection.rb | 2 +-
services/api/app/models/node.rb | 14 ++-
services/api/app/models/repository.rb | 7 +-
services/api/app/models/user.rb | 2 +-
.../admin_notifier/new_inactive_user.text.erb | 4 +-
.../api/app/views/admin_notifier/new_user.text.erb | 4 +-
.../views/user_notifier/account_is_setup.text.erb | 4 +-
services/api/config/initializers/load_config.rb | 114 ++++++------------
services/api/lib/config_loader.rb | 134 +++++++++++++++++++++
services/api/lib/tasks/config_dump.rake | 4 +-
services/api/lib/tasks/delete_old_job_logs.rake | 2 +-
15 files changed, 220 insertions(+), 132 deletions(-)
create mode 100644 services/api/lib/config_loader.rb
via 42218d31e9ec529f77c9226ae131679cb7880647 (commit)
from feea5d82fbbe6e6360d67f8294b65eeeee661979 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 42218d31e9ec529f77c9226ae131679cb7880647
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Mar 22 17:34:12 2019 -0400
13996: More config migrations, refactor some code into config_loader.rb
ActiveSupport::Duration is serialized properly when dumping config.
Non-empty string checks to use !.empty?
Because some configuration parameters could previously be either false
or a string, which are now just an empty string, but empty strings in
Ruby are truthy.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/config/config.defaults.yml b/lib/config/config.defaults.yml
index c22e2df1e..58c1c86dd 100644
--- a/lib/config/config.defaults.yml
+++ b/lib/config/config.defaults.yml
@@ -433,3 +433,7 @@ Clusters:
IssueReporterEmailTo: ""
SupportEmailAddress: ""
EmailFrom: ""
+ RemoteClusters:
+ "*":
+ Proxy: false
+ ActivateUsers: false
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 6a658561d..5f52ef1df 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -46,34 +46,27 @@ class Arvados::V1::SchemaController < ApplicationController
rootUrl: root_url,
servicePath: "arvados/v1/",
batchPath: "batch",
- uuidPrefix: Rails.application.config.uuid_prefix,
- defaultTrashLifetime: Rails.application.config.default_trash_lifetime,
- blobSignatureTtl: Rails.application.config.blob_signature_ttl,
- maxRequestSize: Rails.application.config.max_request_size,
- maxItemsPerResponse: Rails.application.config.max_items_per_response,
- dockerImageFormats: Rails.application.config.docker_image_formats,
- crunchLogBytesPerEvent: Rails.application.config.crunch_log_bytes_per_event,
- crunchLogSecondsBetweenEvents: Rails.application.config.crunch_log_seconds_between_events,
- crunchLogThrottlePeriod: Rails.application.config.crunch_log_throttle_period,
- crunchLogThrottleBytes: Rails.application.config.crunch_log_throttle_bytes,
- crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
- crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
- crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
- crunchLogUpdatePeriod: Rails.application.config.crunch_log_update_period,
- crunchLogUpdateSize: Rails.application.config.crunch_log_update_size,
- remoteHosts: Rails.configuration.remote_hosts,
- remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
- websocketUrl: Rails.application.config.websocket_address,
- workbenchUrl: Rails.application.config.workbench_address,
- keepWebServiceUrl: Rails.application.config.keep_web_service_url,
- gitUrl: case Rails.application.config.git_repo_https_base
- when false
- ''
- when true
- 'https://git.%s.arvadosapi.com/' % Rails.configuration.ClusterID
- else
- Rails.application.config.git_repo_https_base
- end,
+ uuidPrefix: Rails.application.config.ClusterID,
+ defaultTrashLifetime: Rails.application.config.Collections["DefaultTrashLifetime"],
+ blobSignatureTtl: Rails.application.config.Collections["BlobSigningTTL"],
+ maxRequestSize: Rails.application.config.API["MaxRequestSize"],
+ maxItemsPerResponse: Rails.application.API["MaxItemsPerResponse"],
+ dockerImageFormats: Rails.application.config.Containers["SupportedDockerImageFormats"],
+ crunchLogBytesPerEvent: Rails.application.config.Containers["Logging"]["LogBytesPerEvent"],
+ crunchLogSecondsBetweenEvents: Rails.application.config.Containers["Logging"]["LogSecondsBetweenEvents"],
+ crunchLogThrottlePeriod: Rails.application.config.Containers["Logging"]["LogThrottlePeriod"],
+ crunchLogThrottleBytes: Rails.application.config.Containers["Logging"]["LogThrottleBytes"],
+ crunchLogThrottleLines: Rails.application.config.Containers["Logging"]["LogThrottleLines"],
+ crunchLimitLogBytesPerJob: Rails.application.config.Containers["Logging"]["LimitLogBytesPerJob"],
+ crunchLogPartialLineThrottlePeriod: Rails.application.config.Containers["Logging"]["LogPartialLineThrottlePeriod"],
+ crunchLogUpdatePeriod: Rails.application.config.Containers["Logging"]["LogUpdatePeriod"],
+ crunchLogUpdateSize: Rails.application.config.Containers["Logging"]["LogUpdateSize"],
+ remoteHosts: Rails.application.config.RemoteClusters.map {|k,v| v.Host},
+ remoteHostsViaDNS: Rails.application.config.RemoteClusters["*"]["Proxy"],
+ websocketUrl: Rails.application.config.Services["Websocket"]["ExternalURL"],
+ workbenchUrl: Rails.application.config.Services["Workbench1"]["ExternalURL"],
+ keepWebServiceUrl: Rails.application.config.Services["WebDAV"]["ExternalURL"],
+ gitUrl: Rails.application.config.Services["GitHTTP"]["ExternalURL"],
parameters: {
alt: {
type: "string",
diff --git a/services/api/app/controllers/static_controller.rb b/services/api/app/controllers/static_controller.rb
index f0f7a34b5..8965670db 100644
--- a/services/api/app/controllers/static_controller.rb
+++ b/services/api/app/controllers/static_controller.rb
@@ -12,7 +12,7 @@ class StaticController < ApplicationController
def home
respond_to do |f|
f.html do
- if Rails.configuration.Services["Workbench1"]["ExternalURL"]
+ if !Rails.configuration.Services["Workbench1"]["ExternalURL"].empty?
redirect_to Rails.configuration.Services["Workbench1"]["ExternalURL"]
else
render_not_found "Oops, this is an API endpoint. You probably want to point your browser to an Arvados Workbench site instead."
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index fc8ae5282..db99c0bc1 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -87,8 +87,8 @@ class ApiClientAuthorization < ArvadosModel
end
def self.remote_host(uuid_prefix:)
- Rails.configuration.remote_hosts[uuid_prefix] ||
- (Rails.configuration.remote_hosts_via_dns &&
+ Rails.configuration.RemoteClusters[uuid_prefix]["Host"] ||
+ (Rails.configuration.RemoteClusters["*"]["Proxy"] &&
uuid_prefix+".arvadosapi.com")
end
@@ -188,7 +188,7 @@ class ApiClientAuthorization < ArvadosModel
end
if Rails.configuration.Users["NewUsersAreActive"] ||
- Rails.configuration.auto_activate_users_from.include?(remote_user['uuid'][0..4])
+ Rails.configuration.RemoteClusters[remote_user['uuid'][0..4]].andand["ActivateUsers"]
# Update is_active to whatever it is at the remote end
user.is_active = remote_user['is_active']
elsif !remote_user['is_active']
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 9e9ae5635..d84d326c7 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -501,7 +501,7 @@ class Collection < ArvadosModel
if loc = Keep::Locator.parse(search_term)
loc.strip_hints!
coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1)
- if coll_match.any? or Rails.configuration.remote_hosts.length == 0
+ if coll_match.any? or Rails.configuration.RemoteClusters.length > 1
return get_compatible_images(readers, pattern, coll_match)
else
# Allow bare pdh that doesn't exist in the local database so
diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index d9aca4c76..e3a4b8b41 100644
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -195,7 +195,8 @@ class Node < ArvadosModel
ptr_domain: ptr_domain,
}
- if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"]
+ if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+ !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"].empty?)
tmpfile = nil
begin
begin
@@ -223,7 +224,7 @@ class Node < ArvadosModel
end
end
- if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"]
+ if !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"].empty?
cmd = Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"] % template_vars
if not system cmd
logger.error "dns_server_update_command #{cmd.inspect} failed: #{$?}"
@@ -231,7 +232,8 @@ class Node < ArvadosModel
end
end
- if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerReloadCommand"]
+ if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+ !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerReloadCommand"].empty?)
restartfile = File.join(Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"], 'restart.txt')
begin
File.open(restartfile, 'w') do |f|
@@ -257,9 +259,9 @@ class Node < ArvadosModel
# At startup, make sure all DNS entries exist. Otherwise, slurmctld
# will refuse to start.
- if (Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and
- Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"] and
- Rails.configuration.Containers["SLURM"]["Managed"]["AssignNodeHostname"])
+ if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+ !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"].empty? and
+ !Rails.configuration.Containers["SLURM"]["Managed"]["AssignNodeHostname"].empty?)
(0..Rails.configuration.Containers["MaxComputeVMs"]-1).each do |slot_number|
hostname = hostname_for_slot(slot_number)
diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb
index bde9d51d2..ac89ecf6b 100644
--- a/services/api/app/models/repository.rb
+++ b/services/api/app/models/repository.rb
@@ -98,16 +98,15 @@ class Repository < ArvadosModel
end
def ssh_clone_url
- _clone_url :git_repo_ssh_base, 'git at git.%s.arvadosapi.com:'
+ _clone_url Rails.configuration.Services["GitSSH"]["ExternalURL"], 'git at git.%s.arvadosapi.com:'
end
def https_clone_url
- _clone_url :git_repo_https_base, 'https://git.%s.arvadosapi.com/'
+ _clone_url Rails.configuration.Services["GitHTTP"]["ExternalURL"], 'https://git.%s.arvadosapi.com/'
end
def _clone_url config_var, default_base_fmt
- configured_base = Rails.configuration.send config_var
- return nil if configured_base == false
+ configured_base = config_var
prefix = new_record? ? Rails.configuration.ClusterID : uuid[0,5]
if prefix == Rails.configuration.ClusterID and configured_base != true
base = configured_base
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 137b0e093..8735dbabf 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -364,7 +364,7 @@ class User < ArvadosModel
def check_auto_admin
return if self.uuid.end_with?('anonymouspublic')
if (User.where("email = ?",self.email).where(:is_admin => true).count == 0 and
- Rails.configuration.Users["AutoAdminUserWithEmail"] and self.email == Rails.configuration.Users["AutoAdminUserWithEmail"]) or
+ !Rails.configuration.Users["AutoAdminUserWithEmail"].empty? and self.email == Rails.configuration.Users["AutoAdminUserWithEmail"]) or
(User.where("uuid not like '%-000000000000000'").where(:is_admin => true).count == 0 and
Rails.configuration.Users["AutoAdminFirstUser"])
self.is_admin = true
diff --git a/services/api/app/views/admin_notifier/new_inactive_user.text.erb b/services/api/app/views/admin_notifier/new_inactive_user.text.erb
index 097412c25..fa8652b8a 100644
--- a/services/api/app/views/admin_notifier/new_inactive_user.text.erb
+++ b/services/api/app/views/admin_notifier/new_inactive_user.text.erb
@@ -7,10 +7,10 @@ A new user landed on the inactive user page:
<%= @user.full_name %> <<%= @user.email %>>
-<% if Rails.configuration.workbench_address -%>
+<% if Rails.configuration.Services["Workbench1"]["ExternalURL"] -%>
Please see workbench for more information:
- <%= Rails.configuration.workbench_address %>
+ <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %>
<% end -%>
Thanks,
diff --git a/services/api/app/views/admin_notifier/new_user.text.erb b/services/api/app/views/admin_notifier/new_user.text.erb
index 20a36afcb..a96dfdba9 100644
--- a/services/api/app/views/admin_notifier/new_user.text.erb
+++ b/services/api/app/views/admin_notifier/new_user.text.erb
@@ -14,10 +14,10 @@ A new user has been created<%=add_to_message%>:
This user is <%= @user.is_active ? '' : 'NOT ' %>active.
-<% if Rails.configuration.workbench_address -%>
+<% if Rails.configuration.Services["Workbench1"]["ExternalURL"] -%>
Please see workbench for more information:
- <%= Rails.configuration.workbench_address %>
+ <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %>
<% end -%>
Thanks,
diff --git a/services/api/app/views/user_notifier/account_is_setup.text.erb b/services/api/app/views/user_notifier/account_is_setup.text.erb
index ca7082774..417698c38 100644
--- a/services/api/app/views/user_notifier/account_is_setup.text.erb
+++ b/services/api/app/views/user_notifier/account_is_setup.text.erb
@@ -8,9 +8,9 @@ SPDX-License-Identifier: AGPL-3.0 %>
Hi there,
<% end -%>
-Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.workbench_address %>at
+Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services["Workbench1"]["ExternalURL"] %>at
- <%= Rails.configuration.workbench_address %><%= "/" if !Rails.configuration.workbench_address.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
+ <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
for connection instructions.
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
index 80198d2db..53439cec8 100644
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/initializers/load_config.rb
@@ -2,6 +2,8 @@
#
# SPDX-License-Identifier: AGPL-3.0
+require 'config_loader'
+
begin
# If secret_token.rb exists here, we need to load it first.
require_relative 'secret_token.rb'
@@ -39,37 +41,8 @@ $arvados_config = {}
end
end
-def set_cfg cfg, k, v
- # "foo.bar: baz" --> { config.foo.bar = baz }
- ks = k.split '.'
- k = ks.pop
- ks.each do |kk|
- cfg = cfg[kk]
- if cfg.nil?
- break
- end
- end
- if !cfg.nil?
- cfg[k] = v
- end
-end
-
-$config_migrate_map = {}
-$config_types = {}
-def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
- if migrate_from
- $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
- set_cfg cfg, assign_to, v
- }
- end
- $config_types[assign_to] = configtype
-end
-
-module Boolean; end
-class TrueClass; include Boolean; end
-class FalseClass; include Boolean; end
-
-declare_config "ClusterID", String, :uuid_prefix
+declare_config "ClusterID", NonemptyString, :uuid_prefix
+declare_config "ManagementToken", NonemptyString, :ManagementToken
declare_config "Git.Repositories", String, :git_repositories_dir
declare_config "API.DisabledAPIs", Array, :disable_api_methods
declare_config "API.MaxRequestSize", Integer, :max_request_size
@@ -89,10 +62,10 @@ declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
-declare_config "Login.ProviderAppSecret", String, :sso_app_secret
-declare_config "Login.ProviderAppID", String, :sso_app_id
+declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
+declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
declare_config "TLS.Insecure", Boolean, :sso_insecure
-declare_config "Services.SSO.ExternalURL", String, :sso_provider_url
+declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
@@ -102,7 +75,7 @@ declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :def
declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
-declare_config "Collections.BlobSigningKey", String, :blob_signing_key
+declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
@@ -129,9 +102,7 @@ declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_s
declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
-declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) {
- set_cfg cfg, "Containers.JobsAPI.Enable", if v.is_a? Boolean then v.to_s else v end
-}
+declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
@@ -145,6 +116,20 @@ declare_config "Services.Websocket.ExternalURL", String, :websocket_address
declare_config "Services.WebDAV.ExternalURL", String, :keep_web_service_url
declare_config "Services.GitHTTP.ExternalURL", String, :git_repo_https_base
declare_config "Services.GitSSH.ExternalURL", String, :git_repo_ssh_base
+declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
+ h = {}
+ v.each do |clusterid, host|
+ h[clusterid] = {
+ "Host" => host,
+ "Proxy" => true,
+ "Scheme" => "https",
+ "Insecure" => false,
+ "ActivateUsers" => false
+ }
+ end
+ set_cfg cfg, "RemoteClusters", h
+}
+declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
application_config = {}
%w(application.default application).each do |cfgfile|
@@ -159,53 +144,18 @@ application_config = {}
end
end
-application_config.each do |k, v|
- if $config_migrate_map[k.to_sym]
- $config_migrate_map[k.to_sym].call $arvados_config, k, v
- else
- set_cfg $arvados_config, k, v
- end
-end
-
-duration_re = /(\d+(\.\d+)?)(ms|s|m|h)/
-
-$config_types.each do |cfgkey, cfgtype|
- cfg = $arvados_config
- k = cfgkey
- ks = k.split '.'
- k = ks.pop
- ks.each do |kk|
- cfg = cfg[kk]
- if cfg.nil?
- break
- end
- end
-
- if cfg.nil?
- raise "missing #{cfgkey}"
- end
+remainders = migrate_config application_config, $arvados_config
- if cfgtype == String and !cfg[k]
- cfg[k] = ""
- end
- if cfgtype == ActiveSupport::Duration
- if cfg[k].is_a? Integer
- cfg[k] = cfg[k].seconds
- elsif cfg[k].is_a? String
- mt = duration_re.match cfg[k]
- if !mt
- raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are ms, s, m, h"
- end
- multiplier = {ms: 0.001, s: 1, m: 60, h: 3600}
- cfg[k] = (Float(mt[1]) * multiplier[mt[3].to_sym]).seconds
+if application_config[:auto_activate_users_from]
+ application_config[:auto_activate_users_from].each do |cluster|
+ if $arvados_config.RemoteClusters[cluster]
+ $arvados_config.RemoteClusters[cluster]["ActivateUsers"] = true
end
end
-
- if !cfg[k].is_a? cfgtype
- raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
- end
end
+coercion_and_check $arvados_config
+
Server::Application.configure do
nils = []
$arvados_config.each do |k, v|
@@ -223,6 +173,10 @@ Server::Application.configure do
cfg.send "#{k}=", v
end
end
+ remainders.each do |k, v|
+ config.send "#{k}=", v
+ end
+
if !nils.empty?
raise <<EOS
Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
new file mode 100644
index 000000000..6bfbdd7a8
--- /dev/null
+++ b/services/api/lib/config_loader.rb
@@ -0,0 +1,134 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+module Psych
+ module Visitors
+ class YAMLTree < Psych::Visitors::Visitor
+ def visit_ActiveSupport_Duration o
+ seconds = o.to_i
+ outstr = ""
+ if seconds / 3600 > 0
+ outstr += "#{seconds / 3600}h"
+ seconds = seconds % 3600
+ end
+ if seconds / 60 > 0
+ outstr += "#{seconds / 60}m"
+ seconds = seconds % 60
+ end
+ if seconds > 0
+ outstr += "#{seconds}s"
+ end
+ if outstr == ""
+ outstr = "0s"
+ end
+ @emitter.scalar outstr, nil, nil, true, false, Nodes::Scalar::ANY
+ end
+ end
+ end
+end
+
+def set_cfg cfg, k, v
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg[kk]
+ if cfg.nil?
+ break
+ end
+ end
+ if !cfg.nil?
+ cfg[k] = v
+ end
+end
+
+$config_migrate_map = {}
+$config_types = {}
+def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
+ if migrate_from
+ $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
+ set_cfg cfg, assign_to, v
+ }
+ end
+ $config_types[assign_to] = configtype
+end
+
+module Boolean; end
+class TrueClass; include Boolean; end
+class FalseClass; include Boolean; end
+
+class NonemptyString < String
+end
+
+def parse_duration durstr
+ duration_re = /(\d+(\.\d+)?)(s|m|h)/
+ dursec = 0
+ while durstr != ""
+ mt = duration_re.match durstr
+ if !mt
+ raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+ end
+ multiplier = {s: 1, m: 60, h: 3600}
+ dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
+ durstr = durstr[mt[0].length..-1]
+ end
+ return dursec.seconds
+end
+
+def migrate_config from_config, to_config
+ remainders = {}
+ from_config.each do |k, v|
+ if $config_migrate_map[k.to_sym]
+ $config_migrate_map[k.to_sym].call to_config, k, v
+ else
+ remainders[k] = v
+ end
+ end
+ remainders
+end
+
+def coercion_and_check check_cfg
+ $config_types.each do |cfgkey, cfgtype|
+ cfg = check_cfg
+ k = cfgkey
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg[kk]
+ if cfg.nil?
+ break
+ end
+ end
+
+ if cfg.nil?
+ raise "missing #{cfgkey}"
+ end
+
+ if cfgtype == String and !cfg[k]
+ cfg[k] = ""
+ end
+
+ if cfgtype == NonemptyString
+ if (!cfg[k] || cfg[k] == "")
+ raise "#{cfgkey} cannot be empty"
+ end
+ if cfg[k].is_a? String
+ next
+ end
+ end
+
+ if cfgtype == ActiveSupport::Duration
+ if cfg[k].is_a? Integer
+ cfg[k] = cfg[k].seconds
+ elsif cfg[k].is_a? String
+ cfg[k] = parse_duration cfg[k]
+ end
+ end
+
+ if !cfg[k].is_a? cfgtype
+ raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
+ end
+ end
+
+end
diff --git a/services/api/lib/tasks/config_dump.rake b/services/api/lib/tasks/config_dump.rake
index 4532225a3..bc6deb4bb 100644
--- a/services/api/lib/tasks/config_dump.rake
+++ b/services/api/lib/tasks/config_dump.rake
@@ -5,6 +5,8 @@
namespace :config do
desc 'Show site configuration'
task dump: :environment do
- puts $arvados_config.to_yaml
+ cfg = { "Clusters" => {}}
+ cfg["Clusters"][$arvados_config["ClusterID"]] = $arvados_config.select {|k,v| k != "ClusterID"}
+ puts cfg.to_yaml
end
end
diff --git a/services/api/lib/tasks/delete_old_job_logs.rake b/services/api/lib/tasks/delete_old_job_logs.rake
index 327f663b2..6f195e954 100644
--- a/services/api/lib/tasks/delete_old_job_logs.rake
+++ b/services/api/lib/tasks/delete_old_job_logs.rake
@@ -9,7 +9,7 @@
namespace :db do
desc "Remove old job stderr entries from the logs table"
task delete_old_job_logs: :environment do
- delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Rails.configuration.Containers["Logging"]["MaxAge"]} seconds')"
+ delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Containers["Logging"]["MaxAge"]} seconds')"
ActiveRecord::Base.connection.execute(delete_sql)
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list