[ARVADOS] updated: 1.3.0-551-g42218d31e

Git user git at public.curoverse.com
Mon Mar 25 13:14:50 UTC 2019


Summary of changes:
 lib/config/config.defaults.yml                     |   4 +
 .../controllers/arvados/v1/schema_controller.rb    |  49 ++++----
 services/api/app/controllers/static_controller.rb  |   2 +-
 .../api/app/models/api_client_authorization.rb     |   6 +-
 services/api/app/models/collection.rb              |   2 +-
 services/api/app/models/node.rb                    |  14 ++-
 services/api/app/models/repository.rb              |   7 +-
 services/api/app/models/user.rb                    |   2 +-
 .../admin_notifier/new_inactive_user.text.erb      |   4 +-
 .../api/app/views/admin_notifier/new_user.text.erb |   4 +-
 .../views/user_notifier/account_is_setup.text.erb  |   4 +-
 services/api/config/initializers/load_config.rb    | 114 ++++++------------
 services/api/lib/config_loader.rb                  | 134 +++++++++++++++++++++
 services/api/lib/tasks/config_dump.rake            |   4 +-
 services/api/lib/tasks/delete_old_job_logs.rake    |   2 +-
 15 files changed, 220 insertions(+), 132 deletions(-)
 create mode 100644 services/api/lib/config_loader.rb

       via  42218d31e9ec529f77c9226ae131679cb7880647 (commit)
      from  feea5d82fbbe6e6360d67f8294b65eeeee661979 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 42218d31e9ec529f77c9226ae131679cb7880647
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Mar 22 17:34:12 2019 -0400

    13996: More config migrations, refactor some code into config_loader.rb
    
    ActiveSupport::Duration is serialized properly when dumping config.
    
    Non-empty string checks to use !.empty?
    
    Because some configuration parameters could previously be either false
    or a string, which are now just an empty string, but empty strings in
    Ruby are truthy.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/lib/config/config.defaults.yml b/lib/config/config.defaults.yml
index c22e2df1e..58c1c86dd 100644
--- a/lib/config/config.defaults.yml
+++ b/lib/config/config.defaults.yml
@@ -433,3 +433,7 @@ Clusters:
       IssueReporterEmailTo: ""
       SupportEmailAddress: ""
       EmailFrom: ""
+    RemoteClusters:
+      "*":
+        Proxy: false
+        ActivateUsers: false
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 6a658561d..5f52ef1df 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -46,34 +46,27 @@ class Arvados::V1::SchemaController < ApplicationController
         rootUrl: root_url,
         servicePath: "arvados/v1/",
         batchPath: "batch",
-        uuidPrefix: Rails.application.config.uuid_prefix,
-        defaultTrashLifetime: Rails.application.config.default_trash_lifetime,
-        blobSignatureTtl: Rails.application.config.blob_signature_ttl,
-        maxRequestSize: Rails.application.config.max_request_size,
-        maxItemsPerResponse: Rails.application.config.max_items_per_response,
-        dockerImageFormats: Rails.application.config.docker_image_formats,
-        crunchLogBytesPerEvent: Rails.application.config.crunch_log_bytes_per_event,
-        crunchLogSecondsBetweenEvents: Rails.application.config.crunch_log_seconds_between_events,
-        crunchLogThrottlePeriod: Rails.application.config.crunch_log_throttle_period,
-        crunchLogThrottleBytes: Rails.application.config.crunch_log_throttle_bytes,
-        crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
-        crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
-        crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
-        crunchLogUpdatePeriod: Rails.application.config.crunch_log_update_period,
-        crunchLogUpdateSize: Rails.application.config.crunch_log_update_size,
-        remoteHosts: Rails.configuration.remote_hosts,
-        remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
-        websocketUrl: Rails.application.config.websocket_address,
-        workbenchUrl: Rails.application.config.workbench_address,
-        keepWebServiceUrl: Rails.application.config.keep_web_service_url,
-        gitUrl: case Rails.application.config.git_repo_https_base
-                when false
-                  ''
-                when true
-                  'https://git.%s.arvadosapi.com/' % Rails.configuration.ClusterID
-                else
-                  Rails.application.config.git_repo_https_base
-                end,
+        uuidPrefix: Rails.application.config.ClusterID,
+        defaultTrashLifetime: Rails.application.config.Collections["DefaultTrashLifetime"],
+        blobSignatureTtl: Rails.application.config.Collections["BlobSigningTTL"],
+        maxRequestSize: Rails.application.config.API["MaxRequestSize"],
+        maxItemsPerResponse: Rails.application.API["MaxItemsPerResponse"],
+        dockerImageFormats: Rails.application.config.Containers["SupportedDockerImageFormats"],
+        crunchLogBytesPerEvent: Rails.application.config.Containers["Logging"]["LogBytesPerEvent"],
+        crunchLogSecondsBetweenEvents: Rails.application.config.Containers["Logging"]["LogSecondsBetweenEvents"],
+        crunchLogThrottlePeriod: Rails.application.config.Containers["Logging"]["LogThrottlePeriod"],
+        crunchLogThrottleBytes: Rails.application.config.Containers["Logging"]["LogThrottleBytes"],
+        crunchLogThrottleLines: Rails.application.config.Containers["Logging"]["LogThrottleLines"],
+        crunchLimitLogBytesPerJob: Rails.application.config.Containers["Logging"]["LimitLogBytesPerJob"],
+        crunchLogPartialLineThrottlePeriod: Rails.application.config.Containers["Logging"]["LogPartialLineThrottlePeriod"],
+        crunchLogUpdatePeriod: Rails.application.config.Containers["Logging"]["LogUpdatePeriod"],
+        crunchLogUpdateSize: Rails.application.config.Containers["Logging"]["LogUpdateSize"],
+        remoteHosts: Rails.application.config.RemoteClusters.map {|k,v| v.Host},
+        remoteHostsViaDNS: Rails.application.config.RemoteClusters["*"]["Proxy"],
+        websocketUrl: Rails.application.config.Services["Websocket"]["ExternalURL"],
+        workbenchUrl: Rails.application.config.Services["Workbench1"]["ExternalURL"],
+        keepWebServiceUrl: Rails.application.config.Services["WebDAV"]["ExternalURL"],
+        gitUrl: Rails.application.config.Services["GitHTTP"]["ExternalURL"],
         parameters: {
           alt: {
             type: "string",
diff --git a/services/api/app/controllers/static_controller.rb b/services/api/app/controllers/static_controller.rb
index f0f7a34b5..8965670db 100644
--- a/services/api/app/controllers/static_controller.rb
+++ b/services/api/app/controllers/static_controller.rb
@@ -12,7 +12,7 @@ class StaticController < ApplicationController
   def home
     respond_to do |f|
       f.html do
-        if Rails.configuration.Services["Workbench1"]["ExternalURL"]
+        if !Rails.configuration.Services["Workbench1"]["ExternalURL"].empty?
           redirect_to Rails.configuration.Services["Workbench1"]["ExternalURL"]
         else
           render_not_found "Oops, this is an API endpoint. You probably want to point your browser to an Arvados Workbench site instead."
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index fc8ae5282..db99c0bc1 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -87,8 +87,8 @@ class ApiClientAuthorization < ArvadosModel
   end
 
   def self.remote_host(uuid_prefix:)
-    Rails.configuration.remote_hosts[uuid_prefix] ||
-      (Rails.configuration.remote_hosts_via_dns &&
+    Rails.configuration.RemoteClusters[uuid_prefix]["Host"] ||
+      (Rails.configuration.RemoteClusters["*"]["Proxy"] &&
        uuid_prefix+".arvadosapi.com")
   end
 
@@ -188,7 +188,7 @@ class ApiClientAuthorization < ArvadosModel
         end
 
         if Rails.configuration.Users["NewUsersAreActive"] ||
-           Rails.configuration.auto_activate_users_from.include?(remote_user['uuid'][0..4])
+           Rails.configuration.RemoteClusters[remote_user['uuid'][0..4]].andand["ActivateUsers"]
           # Update is_active to whatever it is at the remote end
           user.is_active = remote_user['is_active']
         elsif !remote_user['is_active']
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 9e9ae5635..d84d326c7 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -501,7 +501,7 @@ class Collection < ArvadosModel
     if loc = Keep::Locator.parse(search_term)
       loc.strip_hints!
       coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1)
-      if coll_match.any? or Rails.configuration.remote_hosts.length == 0
+      if coll_match.any? or Rails.configuration.RemoteClusters.length > 1
         return get_compatible_images(readers, pattern, coll_match)
       else
         # Allow bare pdh that doesn't exist in the local database so
diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index d9aca4c76..e3a4b8b41 100644
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -195,7 +195,8 @@ class Node < ArvadosModel
       ptr_domain: ptr_domain,
     }
 
-    if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"]
+    if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+        !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"].empty?)
       tmpfile = nil
       begin
         begin
@@ -223,7 +224,7 @@ class Node < ArvadosModel
       end
     end
 
-    if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"]
+    if !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"].empty?
       cmd = Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerUpdateCommand"] % template_vars
       if not system cmd
         logger.error "dns_server_update_command #{cmd.inspect} failed: #{$?}"
@@ -231,7 +232,8 @@ class Node < ArvadosModel
       end
     end
 
-    if Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerReloadCommand"]
+    if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+        !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerReloadCommand"].empty?)
       restartfile = File.join(Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"], 'restart.txt')
       begin
         File.open(restartfile, 'w') do |f|
@@ -257,9 +259,9 @@ class Node < ArvadosModel
 
   # At startup, make sure all DNS entries exist.  Otherwise, slurmctld
   # will refuse to start.
-  if (Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"] and
-      Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"] and
-      Rails.configuration.Containers["SLURM"]["Managed"]["AssignNodeHostname"])
+  if (!Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfDir"].empty? and
+      !Rails.configuration.Containers["SLURM"]["Managed"]["DNSServerConfTemplate"].empty? and
+      !Rails.configuration.Containers["SLURM"]["Managed"]["AssignNodeHostname"].empty?)
 
     (0..Rails.configuration.Containers["MaxComputeVMs"]-1).each do |slot_number|
       hostname = hostname_for_slot(slot_number)
diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb
index bde9d51d2..ac89ecf6b 100644
--- a/services/api/app/models/repository.rb
+++ b/services/api/app/models/repository.rb
@@ -98,16 +98,15 @@ class Repository < ArvadosModel
   end
 
   def ssh_clone_url
-    _clone_url :git_repo_ssh_base, 'git at git.%s.arvadosapi.com:'
+    _clone_url Rails.configuration.Services["GitSSH"]["ExternalURL"], 'git at git.%s.arvadosapi.com:'
   end
 
   def https_clone_url
-    _clone_url :git_repo_https_base, 'https://git.%s.arvadosapi.com/'
+    _clone_url Rails.configuration.Services["GitHTTP"]["ExternalURL"], 'https://git.%s.arvadosapi.com/'
   end
 
   def _clone_url config_var, default_base_fmt
-    configured_base = Rails.configuration.send config_var
-    return nil if configured_base == false
+    configured_base = config_var
     prefix = new_record? ? Rails.configuration.ClusterID : uuid[0,5]
     if prefix == Rails.configuration.ClusterID and configured_base != true
       base = configured_base
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 137b0e093..8735dbabf 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -364,7 +364,7 @@ class User < ArvadosModel
   def check_auto_admin
     return if self.uuid.end_with?('anonymouspublic')
     if (User.where("email = ?",self.email).where(:is_admin => true).count == 0 and
-        Rails.configuration.Users["AutoAdminUserWithEmail"] and self.email == Rails.configuration.Users["AutoAdminUserWithEmail"]) or
+        !Rails.configuration.Users["AutoAdminUserWithEmail"].empty? and self.email == Rails.configuration.Users["AutoAdminUserWithEmail"]) or
        (User.where("uuid not like '%-000000000000000'").where(:is_admin => true).count == 0 and
         Rails.configuration.Users["AutoAdminFirstUser"])
       self.is_admin = true
diff --git a/services/api/app/views/admin_notifier/new_inactive_user.text.erb b/services/api/app/views/admin_notifier/new_inactive_user.text.erb
index 097412c25..fa8652b8a 100644
--- a/services/api/app/views/admin_notifier/new_inactive_user.text.erb
+++ b/services/api/app/views/admin_notifier/new_inactive_user.text.erb
@@ -7,10 +7,10 @@ A new user landed on the inactive user page:
 
   <%= @user.full_name %> <<%= @user.email %>>
 
-<% if Rails.configuration.workbench_address -%>
+<% if Rails.configuration.Services["Workbench1"]["ExternalURL"] -%>
 Please see workbench for more information:
 
-  <%= Rails.configuration.workbench_address %>
+  <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %>
 
 <% end -%>
 Thanks,
diff --git a/services/api/app/views/admin_notifier/new_user.text.erb b/services/api/app/views/admin_notifier/new_user.text.erb
index 20a36afcb..a96dfdba9 100644
--- a/services/api/app/views/admin_notifier/new_user.text.erb
+++ b/services/api/app/views/admin_notifier/new_user.text.erb
@@ -14,10 +14,10 @@ A new user has been created<%=add_to_message%>:
 
 This user is <%= @user.is_active ? '' : 'NOT ' %>active.
 
-<% if Rails.configuration.workbench_address -%>
+<% if Rails.configuration.Services["Workbench1"]["ExternalURL"] -%>
 Please see workbench for more information:
 
-  <%= Rails.configuration.workbench_address %>
+  <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %>
 
 <% end -%>
 Thanks,
diff --git a/services/api/app/views/user_notifier/account_is_setup.text.erb b/services/api/app/views/user_notifier/account_is_setup.text.erb
index ca7082774..417698c38 100644
--- a/services/api/app/views/user_notifier/account_is_setup.text.erb
+++ b/services/api/app/views/user_notifier/account_is_setup.text.erb
@@ -8,9 +8,9 @@ SPDX-License-Identifier: AGPL-3.0 %>
 Hi there,
 <% end -%>
 
-Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.workbench_address %>at
+Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services["Workbench1"]["ExternalURL"] %>at
 
-  <%= Rails.configuration.workbench_address %><%= "/" if !Rails.configuration.workbench_address.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
+  <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
 
 for connection instructions.
 
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
index 80198d2db..53439cec8 100644
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/initializers/load_config.rb
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+require 'config_loader'
+
 begin
   # If secret_token.rb exists here, we need to load it first.
   require_relative 'secret_token.rb'
@@ -39,37 +41,8 @@ $arvados_config = {}
   end
 end
 
-def set_cfg cfg, k, v
-  # "foo.bar: baz" --> { config.foo.bar = baz }
-  ks = k.split '.'
-  k = ks.pop
-  ks.each do |kk|
-    cfg = cfg[kk]
-    if cfg.nil?
-      break
-    end
-  end
-  if !cfg.nil?
-    cfg[k] = v
-  end
-end
-
-$config_migrate_map = {}
-$config_types = {}
-def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
-  if migrate_from
-    $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
-      set_cfg cfg, assign_to, v
-    }
-  end
-  $config_types[assign_to] = configtype
-end
-
-module Boolean; end
-class TrueClass; include Boolean; end
-class FalseClass; include Boolean; end
-
-declare_config "ClusterID", String, :uuid_prefix
+declare_config "ClusterID", NonemptyString, :uuid_prefix
+declare_config "ManagementToken", NonemptyString, :ManagementToken
 declare_config "Git.Repositories", String, :git_repositories_dir
 declare_config "API.DisabledAPIs", Array, :disable_api_methods
 declare_config "API.MaxRequestSize", Integer, :max_request_size
@@ -89,10 +62,10 @@ declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
 declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
 declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
 declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
-declare_config "Login.ProviderAppSecret", String, :sso_app_secret
-declare_config "Login.ProviderAppID", String, :sso_app_id
+declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
+declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
 declare_config "TLS.Insecure", Boolean, :sso_insecure
-declare_config "Services.SSO.ExternalURL", String, :sso_provider_url
+declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
 declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
 declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
 declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
@@ -102,7 +75,7 @@ declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :def
 declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
 declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
 declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
-declare_config "Collections.BlobSigningKey", String, :blob_signing_key
+declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
 declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
 declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
 declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
@@ -129,9 +102,7 @@ declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_s
 declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
 declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
 declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
-declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) {
-  set_cfg cfg, "Containers.JobsAPI.Enable", if v.is_a? Boolean then v.to_s else v end
-}
+declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
 declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
 declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
 declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
@@ -145,6 +116,20 @@ declare_config "Services.Websocket.ExternalURL", String, :websocket_address
 declare_config "Services.WebDAV.ExternalURL", String, :keep_web_service_url
 declare_config "Services.GitHTTP.ExternalURL", String, :git_repo_https_base
 declare_config "Services.GitSSH.ExternalURL", String, :git_repo_ssh_base
+declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
+  h = {}
+  v.each do |clusterid, host|
+    h[clusterid] = {
+      "Host" => host,
+      "Proxy" => true,
+      "Scheme" => "https",
+      "Insecure" => false,
+      "ActivateUsers" => false
+    }
+  end
+  set_cfg cfg, "RemoteClusters", h
+}
+declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
 
 application_config = {}
 %w(application.default application).each do |cfgfile|
@@ -159,53 +144,18 @@ application_config = {}
   end
 end
 
-application_config.each do |k, v|
-  if $config_migrate_map[k.to_sym]
-    $config_migrate_map[k.to_sym].call $arvados_config, k, v
-  else
-    set_cfg $arvados_config, k, v
-  end
-end
-
-duration_re = /(\d+(\.\d+)?)(ms|s|m|h)/
-
-$config_types.each do |cfgkey, cfgtype|
-  cfg = $arvados_config
-  k = cfgkey
-  ks = k.split '.'
-  k = ks.pop
-  ks.each do |kk|
-    cfg = cfg[kk]
-    if cfg.nil?
-      break
-    end
-  end
-
-  if cfg.nil?
-    raise "missing #{cfgkey}"
-  end
+remainders = migrate_config application_config, $arvados_config
 
-  if cfgtype == String and !cfg[k]
-    cfg[k] = ""
-  end
-  if cfgtype == ActiveSupport::Duration
-    if cfg[k].is_a? Integer
-      cfg[k] = cfg[k].seconds
-    elsif cfg[k].is_a? String
-      mt = duration_re.match cfg[k]
-      if !mt
-        raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are ms, s, m, h"
-      end
-      multiplier = {ms: 0.001, s: 1, m: 60, h: 3600}
-      cfg[k] = (Float(mt[1]) * multiplier[mt[3].to_sym]).seconds
+if application_config[:auto_activate_users_from]
+  application_config[:auto_activate_users_from].each do |cluster|
+    if $arvados_config.RemoteClusters[cluster]
+      $arvados_config.RemoteClusters[cluster]["ActivateUsers"] = true
     end
   end
-
-  if !cfg[k].is_a? cfgtype
-    raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
-  end
 end
 
+coercion_and_check $arvados_config
+
 Server::Application.configure do
   nils = []
   $arvados_config.each do |k, v|
@@ -223,6 +173,10 @@ Server::Application.configure do
       cfg.send "#{k}=", v
     end
   end
+  remainders.each do |k, v|
+    config.send "#{k}=", v
+  end
+
   if !nils.empty?
     raise <<EOS
 Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
new file mode 100644
index 000000000..6bfbdd7a8
--- /dev/null
+++ b/services/api/lib/config_loader.rb
@@ -0,0 +1,134 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+module Psych
+  module Visitors
+    class YAMLTree < Psych::Visitors::Visitor
+      def visit_ActiveSupport_Duration o
+        seconds = o.to_i
+        outstr = ""
+        if seconds / 3600 > 0
+          outstr += "#{seconds / 3600}h"
+          seconds = seconds % 3600
+        end
+        if seconds / 60 > 0
+          outstr += "#{seconds / 60}m"
+          seconds = seconds % 60
+        end
+        if seconds > 0
+          outstr += "#{seconds}s"
+        end
+        if outstr == ""
+          outstr = "0s"
+        end
+        @emitter.scalar outstr, nil, nil, true, false, Nodes::Scalar::ANY
+      end
+    end
+  end
+end
+
+def set_cfg cfg, k, v
+  # "foo.bar: baz" --> { config.foo.bar = baz }
+  ks = k.split '.'
+  k = ks.pop
+  ks.each do |kk|
+    cfg = cfg[kk]
+    if cfg.nil?
+      break
+    end
+  end
+  if !cfg.nil?
+    cfg[k] = v
+  end
+end
+
+$config_migrate_map = {}
+$config_types = {}
+def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
+  if migrate_from
+    $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
+      set_cfg cfg, assign_to, v
+    }
+  end
+  $config_types[assign_to] = configtype
+end
+
+module Boolean; end
+class TrueClass; include Boolean; end
+class FalseClass; include Boolean; end
+
+class NonemptyString < String
+end
+
+def parse_duration durstr
+  duration_re = /(\d+(\.\d+)?)(s|m|h)/
+  dursec = 0
+  while durstr != ""
+    mt = duration_re.match durstr
+    if !mt
+      raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+    end
+    multiplier = {s: 1, m: 60, h: 3600}
+    dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
+    durstr = durstr[mt[0].length..-1]
+  end
+  return dursec.seconds
+end
+
+def migrate_config from_config, to_config
+  remainders = {}
+  from_config.each do |k, v|
+    if $config_migrate_map[k.to_sym]
+      $config_migrate_map[k.to_sym].call to_config, k, v
+    else
+      remainders[k] = v
+    end
+  end
+  remainders
+end
+
+def coercion_and_check check_cfg
+  $config_types.each do |cfgkey, cfgtype|
+    cfg = check_cfg
+    k = cfgkey
+    ks = k.split '.'
+    k = ks.pop
+    ks.each do |kk|
+      cfg = cfg[kk]
+      if cfg.nil?
+        break
+      end
+    end
+
+    if cfg.nil?
+      raise "missing #{cfgkey}"
+    end
+
+    if cfgtype == String and !cfg[k]
+      cfg[k] = ""
+    end
+
+    if cfgtype == NonemptyString
+      if (!cfg[k] || cfg[k] == "")
+        raise "#{cfgkey} cannot be empty"
+      end
+      if cfg[k].is_a? String
+        next
+      end
+    end
+
+    if cfgtype == ActiveSupport::Duration
+      if cfg[k].is_a? Integer
+        cfg[k] = cfg[k].seconds
+      elsif cfg[k].is_a? String
+        cfg[k] = parse_duration cfg[k]
+      end
+    end
+
+    if !cfg[k].is_a? cfgtype
+      raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
+    end
+  end
+
+end
diff --git a/services/api/lib/tasks/config_dump.rake b/services/api/lib/tasks/config_dump.rake
index 4532225a3..bc6deb4bb 100644
--- a/services/api/lib/tasks/config_dump.rake
+++ b/services/api/lib/tasks/config_dump.rake
@@ -5,6 +5,8 @@
 namespace :config do
   desc 'Show site configuration'
   task dump: :environment do
-    puts $arvados_config.to_yaml
+    cfg = { "Clusters" => {}}
+    cfg["Clusters"][$arvados_config["ClusterID"]] = $arvados_config.select {|k,v| k != "ClusterID"}
+    puts cfg.to_yaml
   end
 end
diff --git a/services/api/lib/tasks/delete_old_job_logs.rake b/services/api/lib/tasks/delete_old_job_logs.rake
index 327f663b2..6f195e954 100644
--- a/services/api/lib/tasks/delete_old_job_logs.rake
+++ b/services/api/lib/tasks/delete_old_job_logs.rake
@@ -9,7 +9,7 @@
 namespace :db do
   desc "Remove old job stderr entries from the logs table"
   task delete_old_job_logs: :environment do
-    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Rails.configuration.Containers["Logging"]["MaxAge"]} seconds')"
+    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Containers["Logging"]["MaxAge"]} seconds')"
 
     ActiveRecord::Base.connection.execute(delete_sql)
   end

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list