[ARVADOS] updated: 1.3.0-621-g1eec70126
Git user
git at public.curoverse.com
Wed Apr 3 20:42:39 UTC 2019
Summary of changes:
lib/config/config.defaults.yml | 11 +
services/api/config/arvados_config.rb | 224 +++++++++++++++++++++
services/api/config/environment.rb | 1 +
.../api/config/initializers/legacy_jobs_api.rb | 1 -
services/api/config/initializers/load_config.rb | 167 ---------------
.../api/config/initializers/preload_all_models.rb | 1 -
services/api/lib/config_loader.rb | 193 +++++++++---------
services/api/lib/tasks/config_diff.rake | 32 ---
.../arvados/v1/repositories_controller_test.rb | 2 +-
services/api/test/test_helper.rb | 4 +-
services/api/test/unit/container_request_test.rb | 1 -
11 files changed, 339 insertions(+), 298 deletions(-)
create mode 100644 services/api/config/arvados_config.rb
delete mode 100644 services/api/config/initializers/load_config.rb
delete mode 100644 services/api/lib/tasks/config_diff.rake
via 1eec7012611b33bfa2db0bbd067f45d5250639e4 (commit)
via 950feaf1c33cd43918a9703dd7b65ec079ad24d7 (commit)
from 9476b723aa5498015bc415c0709263dfdea612eb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1eec7012611b33bfa2db0bbd067f45d5250639e4
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Apr 3 16:41:58 2019 -0400
13996: Can now use database info from config.yml
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/arvados_config.rb
similarity index 85%
rename from services/api/config/initializers/load_config.rb
rename to services/api/config/arvados_config.rb
index 8bed5c655..b0ac7c0be 100644
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -2,6 +2,19 @@
#
# SPDX-License-Identifier: AGPL-3.0
+#
+# Load Arvados configuration from /etc/arvados/config.yml, using defaults from config.defaults.yml
+#
+# Existing application.yml is migrated into the new config structure.
+# Keys in the legacy application.yml take precedence.
+#
+# Use "bundle exec config:dump" to get the complete active configuration
+#
+# Use "bundle exec config:migrate" to migrate application.yml and
+# database.yml to config.yml. After adding the output of
+# config:migrate to /etc/arvados/config.yml, you will be able to
+# delete application.yml and database.yml.
+
require 'config_loader'
begin
@@ -36,7 +49,7 @@ $arvados_config = {}
if confs
clusters = confs["Clusters"].first
$arvados_config["ClusterID"] = clusters[0]
- $arvados_config.merge!(clusters[1])
+ $arvados_config.deep_merge!(clusters[1])
end
end
end
@@ -154,17 +167,17 @@ application_config = {}
confs = YAML.load(yaml, deserialize_symbols: true)
# Ignore empty YAML file:
next if confs == false
- application_config.merge!(confs['common'] || {})
- application_config.merge!(confs[::Rails.env.to_s] || {})
+ application_config.deep_merge!(confs['common'] || {})
+ application_config.deep_merge!(confs[::Rails.env.to_s] || {})
end
end
db_config = {}
-path = "#{::Rails.root.to_s}/config/database.ymlx"
+path = "#{::Rails.root.to_s}/config/database.yml"
if File.exist? path
yaml = ERB.new(IO.read path).result(binding)
confs = YAML.load(yaml, deserialize_symbols: true)
- db_config.merge!(confs[::Rails.env.to_s] || {})
+ db_config.deep_merge!(confs[::Rails.env.to_s] || {})
end
$remaining_config = arvcfg.migrate_config(application_config, $arvados_config)
@@ -183,22 +196,29 @@ end
arvcfg.coercion_and_check $arvados_config
dbcfg.coercion_and_check $arvados_config
+#
+# Special case for test database, because the Arvados config.yml
+# doesn't have a concept of multiple rails environments.
+#
+if ::Rails.env.to_s == "test"
+ $arvados_config["PostgreSQL"]["Connection"]["DBName"] = "arvados_test"
+end
+
+dbhost = $arvados_config["PostgreSQL"]["Connection"]["Host"]
+if $arvados_config["PostgreSQL"]["Connection"]["Post"] != 0
+ dbhost += ":#{$arvados_config["PostgreSQL"]["Connection"]["Post"]}"
+end
+
+#
+# If DATABASE_URL is set, then ActiveRecord won't error out if database.yml doesn't exist.
+#
+# For config migration, we've previously populated the PostgreSQL
+# section of the config from database.yml
+#
+ENV["DATABASE_URL"] = "postgresql://#{$arvados_config["PostgreSQL"]["Connection"]["User"]}:#{$arvados_config["PostgreSQL"]["Connection"]["Password"]}@#{dbhost}/#{$arvados_config["PostgreSQL"]["Connection"]["DBName"]}?template=#{$arvados_config["PostgreSQL"]["Connection"]["Template"]}&encoding=#{$arvados_config["PostgreSQL"]["Connection"]["client_encoding"]}&pool=#{$arvados_config["PostgreSQL"]["ConnectionPool"]}"
+
Server::Application.configure do
ConfigLoader.copy_into_config $arvados_config, config
ConfigLoader.copy_into_config $remaining_config, config
config.secret_key_base = config.secret_token
-
- dbcfg = {}
- dbcfg[::Rails.env.to_s] = {
- adapter: 'postgresql',
- template: $arvados_config["PostgreSQL"]["Connection"]["Template"],
- encoding: $arvados_config["PostgreSQL"]["Connection"]["Encoding"],
- database: $arvados_config["PostgreSQL"]["Connection"]["DBName"],
- username: $arvados_config["PostgreSQL"]["Connection"]["User"],
- password: $arvados_config["PostgreSQL"]["Connection"]["Password"],
- host: $arvados_config["PostgreSQL"]["Connection"]["Host"],
- port: $arvados_config["PostgreSQL"]["Connection"]["Port"],
- pool: $arvados_config["PostgreSQL"]["ConnectionPool"]
- }
- Rails.application.config.database_configuration = dbcfg
end
diff --git a/services/api/config/environment.rb b/services/api/config/environment.rb
index e24eee035..62717b6c9 100644
--- a/services/api/config/environment.rb
+++ b/services/api/config/environment.rb
@@ -5,6 +5,7 @@
# Load the rails application
require File.expand_path('../application', __FILE__)
require 'josh_id'
+require_relative 'arvados_config'
# Initialize the rails application
Server::Application.initialize!
diff --git a/services/api/config/initializers/legacy_jobs_api.rb b/services/api/config/initializers/legacy_jobs_api.rb
index 9ea6b2884..8f3b3cb5f 100644
--- a/services/api/config/initializers/legacy_jobs_api.rb
+++ b/services/api/config/initializers/legacy_jobs_api.rb
@@ -5,7 +5,6 @@
# Config must be done before we files; otherwise they
# won't be able to use Rails.configuration.* to initialize their
# classes.
-require_relative 'load_config.rb'
require 'enable_jobs_api'
diff --git a/services/api/config/initializers/preload_all_models.rb b/services/api/config/initializers/preload_all_models.rb
index 0ab2b032a..713c61fd7 100644
--- a/services/api/config/initializers/preload_all_models.rb
+++ b/services/api/config/initializers/preload_all_models.rb
@@ -7,7 +7,6 @@
# Config must be done before we load model class files; otherwise they
# won't be able to use Rails.configuration.* to initialize their
# classes.
-require_relative 'load_config.rb'
if Rails.env == 'development'
Dir.foreach("#{Rails.root}/app/models") do |model_file|
diff --git a/services/api/lib/tasks/config_diff.rake b/services/api/lib/tasks/config_diff.rake
deleted file mode 100644
index 0fd6fdd46..000000000
--- a/services/api/lib/tasks/config_diff.rake
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-def diff_hash base, final
- diffed = {}
- base.each do |k,v|
- bk = base[k]
- fk = final[k]
- if bk.is_a? Hash
- d = diff_hash bk, fk
- if d.length > 0
- diffed[k] = d
- end
- else
- if bk.to_s != fk.to_s
- diffed[k] = fk
- end
- end
- end
- diffed
-end
-
-namespace :config do
- desc 'Print configuration loaded from legacy application.yml as new Arvados configuration structure'
- task diff: :environment do
- diffed = diff_hash $base_arvados_config, $arvados_config
- cfg = { "Clusters" => {}}
- cfg["Clusters"][$arvados_config["ClusterID"]] = diffed.select {|k,v| k != "ClusterID"}
- puts cfg.to_yaml
- end
-end
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index c8dcfef23..6f0233306 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -525,7 +525,6 @@ class ContainerRequestTest < ActiveSupport::TestCase
'ENOEXIST',
'arvados/apitestfixture:ENOEXIST',
].each do |img|
- puts "RC", Rails.configuration.RemoteClusters
test "container_image_for_container(#{img.inspect}) => 422" do
set_user_from_auth :active
assert_raises(ArvadosModel::UnresolvableContainerError) do
commit 950feaf1c33cd43918a9703dd7b65ec079ad24d7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Apr 2 11:28:23 2019 -0400
13996: Refactor, create ConfigLoader class
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/config/config.defaults.yml b/lib/config/config.defaults.yml
index 58c1c86dd..3da6dc803 100644
--- a/lib/config/config.defaults.yml
+++ b/lib/config/config.defaults.yml
@@ -63,6 +63,17 @@ Clusters:
ExternalURL: ""
Workbench2:
ExternalURL: ""
+ PostgreSQL:
+ # max concurrent connections per arvados server daemon
+ ConnectionPool: 32
+ Connection:
+ # All parameters here are passed to the PG client library in a connection string;
+ # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+ Host: ""
+ Port: 0
+ User: ""
+ Password: ""
+ DBName: ""
API:
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
index 96f6a7f04..8bed5c655 100644
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/initializers/load_config.rb
@@ -43,82 +43,84 @@ end
$base_arvados_config = $arvados_config.deep_dup
-declare_config "ClusterID", NonemptyString, :uuid_prefix
-declare_config "ManagementToken", String, :ManagementToken
-declare_config "Git.Repositories", String, :git_repositories_dir
-declare_config "API.DisabledAPIs", Array, :disable_api_methods
-declare_config "API.MaxRequestSize", Integer, :max_request_size
-declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
-declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
-declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
-declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
-declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
-declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
-declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
-declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
-declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
-declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
-declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
-declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
-declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
-declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
-declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
-declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
-declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
-declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
-declare_config "TLS.Insecure", Boolean, :sso_insecure
-declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
-declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
-declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
-declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
-declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
-declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
-declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
-declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
-declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
-declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
-declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
-declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
-declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
-declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
-declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
-declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
-declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
-declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
-declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
-declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
-declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
-declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
-declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
-declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
-declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
-declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
-declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
-declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
-declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
-declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
-declare_config "Containers.SLURM.Managed.DNSServerConfDir", Pathname, :dns_server_conf_dir
-declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", Pathname, :dns_server_conf_template
-declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
-declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
-declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
-declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
-declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
-declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
-declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
-declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
-declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
-declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
-declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
-declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
-declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
-declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
-declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
-declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
-declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
-declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
-declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
-declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
+arvcfg = ConfigLoader.new
+
+arvcfg.declare_config "ClusterID", NonemptyString, :uuid_prefix
+arvcfg.declare_config "ManagementToken", String, :ManagementToken
+arvcfg.declare_config "Git.Repositories", String, :git_repositories_dir
+arvcfg.declare_config "API.DisabledAPIs", Array, :disable_api_methods
+arvcfg.declare_config "API.MaxRequestSize", Integer, :max_request_size
+arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
+arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
+arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
+arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
+arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
+arvcfg.declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
+arvcfg.declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
+arvcfg.declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
+arvcfg.declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
+arvcfg.declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
+arvcfg.declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
+arvcfg.declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
+arvcfg.declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
+arvcfg.declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
+arvcfg.declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
+arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
+arvcfg.declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
+arvcfg.declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
+arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure
+arvcfg.declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
+arvcfg.declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
+arvcfg.declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
+arvcfg.declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
+arvcfg.declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
+arvcfg.declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
+arvcfg.declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
+arvcfg.declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
+arvcfg.declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
+arvcfg.declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
+arvcfg.declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
+arvcfg.declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
+arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
+arvcfg.declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
+arvcfg.declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
+arvcfg.declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
+arvcfg.declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
+arvcfg.declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
+arvcfg.declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
+arvcfg.declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
+arvcfg.declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
+arvcfg.declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
+arvcfg.declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
+arvcfg.declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
+arvcfg.declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
+arvcfg.declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
+arvcfg.declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
+arvcfg.declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
+arvcfg.declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
+arvcfg.declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerConfDir", Pathname, :dns_server_conf_dir
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", Pathname, :dns_server_conf_template
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
+arvcfg.declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
+arvcfg.declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
+arvcfg.declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
+arvcfg.declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
+arvcfg.declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
+arvcfg.declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
+arvcfg.declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
+arvcfg.declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
+arvcfg.declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
+arvcfg.declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
+arvcfg.declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
+arvcfg.declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
+arvcfg.declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
+arvcfg.declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
+arvcfg.declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
+arvcfg.declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
+arvcfg.declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
+arvcfg.declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
h = {}
v.each do |clusterid, host|
h[clusterid] = {
@@ -129,9 +131,20 @@ declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
"ActivateUsers" => false
}
end
- set_cfg cfg, "RemoteClusters", h
+ ConfigLoader.set_cfg cfg, "RemoteClusters", h
}
-declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+arvcfg.declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+
+dbcfg = ConfigLoader.new
+
+dbcfg.declare_config "PostgreSQL.ConnectionPool", Integer, :pool
+dbcfg.declare_config "PostgreSQL.Connection.Host", String, :host
+dbcfg.declare_config "PostgreSQL.Connection.Port", Integer, :port
+dbcfg.declare_config "PostgreSQL.Connection.User", String, :username
+dbcfg.declare_config "PostgreSQL.Connection.Password", String, :password
+dbcfg.declare_config "PostgreSQL.Connection.DBName", String, :database
+dbcfg.declare_config "PostgreSQL.Connection.Template", String, :template
+dbcfg.declare_config "PostgreSQL.Connection.Encoding", String, :encoding
application_config = {}
%w(application.default application).each do |cfgfile|
@@ -146,7 +159,16 @@ application_config = {}
end
end
-$remaining_config = migrate_config application_config, $arvados_config
+db_config = {}
+path = "#{::Rails.root.to_s}/config/database.ymlx"
+if File.exist? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml, deserialize_symbols: true)
+ db_config.merge!(confs[::Rails.env.to_s] || {})
+end
+
+$remaining_config = arvcfg.migrate_config(application_config, $arvados_config)
+dbcfg.migrate_config(db_config, $arvados_config)
if application_config[:auto_activate_users_from]
application_config[:auto_activate_users_from].each do |cluster|
@@ -158,10 +180,25 @@ end
# Checks for wrongly typed configuration items, and essential items
# that can't be empty
-coercion_and_check $arvados_config
+arvcfg.coercion_and_check $arvados_config
+dbcfg.coercion_and_check $arvados_config
Server::Application.configure do
- copy_into_config $arvados_config, config
- copy_into_config $remaining_config, config
+ ConfigLoader.copy_into_config $arvados_config, config
+ ConfigLoader.copy_into_config $remaining_config, config
config.secret_key_base = config.secret_token
+
+ dbcfg = {}
+ dbcfg[::Rails.env.to_s] = {
+ adapter: 'postgresql',
+ template: $arvados_config["PostgreSQL"]["Connection"]["Template"],
+ encoding: $arvados_config["PostgreSQL"]["Connection"]["Encoding"],
+ database: $arvados_config["PostgreSQL"]["Connection"]["DBName"],
+ username: $arvados_config["PostgreSQL"]["Connection"]["User"],
+ password: $arvados_config["PostgreSQL"]["Connection"]["Password"],
+ host: $arvados_config["PostgreSQL"]["Connection"]["Host"],
+ port: $arvados_config["PostgreSQL"]["Connection"]["Port"],
+ pool: $arvados_config["PostgreSQL"]["ConnectionPool"]
+ }
+ Rails.application.config.database_configuration = dbcfg
end
diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
index 8b31a62c4..2d1ddd8b8 100644
--- a/services/api/lib/config_loader.rb
+++ b/services/api/lib/config_loader.rb
@@ -40,31 +40,6 @@ module Psych
end
end
-def set_cfg cfg, k, v
- # "foo.bar = baz" --> { cfg["foo"]["bar"] = baz }
- ks = k.split '.'
- k = ks.pop
- ks.each do |kk|
- cfg = cfg[kk]
- if cfg.nil?
- break
- end
- end
- if !cfg.nil?
- cfg[k] = v
- end
-end
-
-$config_migrate_map = {}
-$config_types = {}
-def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
- if migrate_from
- $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
- set_cfg cfg, assign_to, v
- }
- end
- $config_types[assign_to] = configtype
-end
module Boolean; end
class TrueClass; include Boolean; end
@@ -73,100 +48,132 @@ class FalseClass; include Boolean; end
class NonemptyString < String
end
-def parse_duration durstr
- duration_re = /(\d+(\.\d+)?)(s|m|h)/
- dursec = 0
- while durstr != ""
- mt = duration_re.match durstr
- if !mt
- raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+class ConfigLoader
+ def initialize
+ @config_migrate_map = {}
+ @config_types = {}
+ end
+
+ def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
+ if migrate_from
+ @config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
+ ConfigLoader.set_cfg cfg, assign_to, v
+ }
end
- multiplier = {s: 1, m: 60, h: 3600}
- dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
- durstr = durstr[mt[0].length..-1]
+ @config_types[assign_to] = configtype
end
- return dursec.seconds
-end
-def migrate_config from_config, to_config
- remainders = {}
- from_config.each do |k, v|
- if $config_migrate_map[k.to_sym]
- $config_migrate_map[k.to_sym].call to_config, k, v
- else
- remainders[k] = v
+
+ def migrate_config from_config, to_config
+ remainders = {}
+ from_config.each do |k, v|
+ if @config_migrate_map[k.to_sym]
+ @config_migrate_map[k.to_sym].call to_config, k, v
+ else
+ remainders[k] = v
+ end
end
+ remainders
end
- remainders
-end
-def coercion_and_check check_cfg
- $config_types.each do |cfgkey, cfgtype|
- cfg = check_cfg
- k = cfgkey
- ks = k.split '.'
- k = ks.pop
- ks.each do |kk|
- cfg = cfg[kk]
+ def coercion_and_check check_cfg
+ @config_types.each do |cfgkey, cfgtype|
+ cfg = check_cfg
+ k = cfgkey
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg[kk]
+ if cfg.nil?
+ break
+ end
+ end
+
if cfg.nil?
- break
+ raise "missing #{cfgkey}"
end
- end
- if cfg.nil?
- raise "missing #{cfgkey}"
- end
+ if cfgtype == String and !cfg[k]
+ cfg[k] = ""
+ end
- if cfgtype == String and !cfg[k]
- cfg[k] = ""
- end
+ if cfgtype == String and cfg[k].is_a? Symbol
+ cfg[k] = cfg[k].to_s
+ end
- if cfgtype == String and cfg[k].is_a? Symbol
- cfg[k] = cfg[k].to_s
- end
+ if cfgtype == Pathname and cfg[k].is_a? String
- if cfgtype == Pathname and cfg[k].is_a? String
+ if cfg[k] == ""
+ cfg[k] = Pathname.new("")
+ else
+ cfg[k] = Pathname.new(cfg[k])
+ if !cfg[k].exist?
+ raise "#{cfgkey} path #{cfg[k]} does not exist"
+ end
+ end
+ end
- if cfg[k] == ""
- cfg[k] = Pathname.new("")
- else
- cfg[k] = Pathname.new(cfg[k])
- if !cfg[k].exist?
- raise "#{cfgkey} path #{cfg[k]} does not exist"
+ if cfgtype == NonemptyString
+ if (!cfg[k] || cfg[k] == "")
+ raise "#{cfgkey} cannot be empty"
+ end
+ if cfg[k].is_a? String
+ next
end
end
- end
- if cfgtype == NonemptyString
- if (!cfg[k] || cfg[k] == "")
- raise "#{cfgkey} cannot be empty"
+ if cfgtype == ActiveSupport::Duration
+ if cfg[k].is_a? Integer
+ cfg[k] = cfg[k].seconds
+ elsif cfg[k].is_a? String
+ cfg[k] = ConfigLoader.parse_duration cfg[k]
+ end
end
- if cfg[k].is_a? String
- next
+
+ if cfgtype == URI
+ cfg[k] = URI(cfg[k])
end
- end
- if cfgtype == ActiveSupport::Duration
- if cfg[k].is_a? Integer
- cfg[k] = cfg[k].seconds
- elsif cfg[k].is_a? String
- cfg[k] = parse_duration cfg[k]
+ if !cfg[k].is_a? cfgtype
+ raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
end
end
+ end
- if cfgtype == URI
- cfg[k] = URI(cfg[k])
+ def self.set_cfg cfg, k, v
+ # "foo.bar = baz" --> { cfg["foo"]["bar"] = baz }
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg[kk]
+ if cfg.nil?
+ break
+ end
end
-
- if !cfg[k].is_a? cfgtype
- raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
+ if !cfg.nil?
+ cfg[k] = v
end
end
-end
+ def self.parse_duration durstr
+ duration_re = /(\d+(\.\d+)?)(s|m|h)/
+ dursec = 0
+ while durstr != ""
+ mt = duration_re.match durstr
+ if !mt
+ raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+ end
+ multiplier = {s: 1, m: 60, h: 3600}
+ dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
+ durstr = durstr[mt[0].length..-1]
+ end
+ return dursec.seconds
+ end
-def copy_into_config src, dst
- src.each do |k, v|
- dst.send "#{k}=", Marshal.load(Marshal.dump v)
+ def self.copy_into_config src, dst
+ src.each do |k, v|
+ dst.send "#{k}=", Marshal.load(Marshal.dump v)
+ end
end
+
end
diff --git a/services/api/test/functional/arvados/v1/repositories_controller_test.rb b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
index f54a6a0a9..faa5921dd 100644
--- a/services/api/test/functional/arvados/v1/repositories_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
@@ -208,7 +208,7 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
{cfg: "GitHTTP", cfgval: false, refute: /^http/ },
].each do |expect|
test "set #{expect[:cfg]} to #{expect[:cfgval]}" do
- set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval]
+ ConfigLoader.set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval]
authorize_with :active
get :index
assert_response :success
diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb
index 9c2504fd7..fa62c1ec4 100644
--- a/services/api/test/test_helper.rb
+++ b/services/api/test/test_helper.rb
@@ -99,8 +99,8 @@ class ActiveSupport::TestCase
def restore_configuration
# Restore configuration settings changed during tests
- copy_into_config $arvados_config, Rails.configuration
- copy_into_config $remaining_config, Rails.configuration
+ ConfigLoader.copy_into_config $arvados_config, Rails.configuration
+ ConfigLoader.copy_into_config $remaining_config, Rails.configuration
end
def set_user_from_auth(auth_name)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list