[ARVADOS] created: 74aa6539fccff9b135f3edec58801976b9a51538
git at public.curoverse.com
git at public.curoverse.com
Wed Mar 5 13:16:33 EST 2014
at 74aa6539fccff9b135f3edec58801976b9a51538 (commit)
commit 74aa6539fccff9b135f3edec58801976b9a51538
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Mar 5 04:35:45 2014 -0500
Read configuration from config.defaults.yml and config.yml.
Refuse to start if any configs are set to nil in config.default.yml
and not overridden in config/environments/*.rb or config.yml.
refs #2076
diff --git a/apps/workbench/.gitignore b/apps/workbench/.gitignore
index 89efb81..1c551bc 100644
--- a/apps/workbench/.gitignore
+++ b/apps/workbench/.gitignore
@@ -22,6 +22,7 @@
/config/environments/development.rb
/config/environments/test.rb
/config/environments/production.rb
+/config/config.yml
/config/piwik.yml
diff --git a/apps/workbench/config/config.defaults.yml b/apps/workbench/config/config.defaults.yml
new file mode 100644
index 0000000..9394943
--- /dev/null
+++ b/apps/workbench/config/config.defaults.yml
@@ -0,0 +1,65 @@
+# Do not use this file for site configuration. Create config.yml
+# instead (see config.yml.example).
+
+development:
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+ profiling_enabled: true
+ site_name: Workbench:dev
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+ i18n.fallbacks: true
+ active_support.deprecation: :notify
+ profiling_enabled: false
+
+ arvados_insecure_https: false
+
+ data_import_dir: /data/arvados-workbench-upload/data
+ data_export_dir: /data/arvados-workbench-download/data
+
+ site_name: Arvados Workbench
+
+test:
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_record.mass_assignment_sanitizer: :strict
+ active_support.deprecation: :stderr
+ profiling_enabled: false
+
+ site_name: Workbench:test
+
+common:
+ data_import_dir: /tmp/arvados-workbench-upload
+ data_export_dir: /tmp/arvados-workbench-download
+ arvados_login_base: https://arvados.local/login
+ arvados_v1_base: https://arvados.local/arvados/v1
+ arvados_insecure_https: true
+ activation_contact_link: mailto:info at arvados.org
+ arvados_docsite: http://doc.arvados.org
+ arvados_theme: default
+ show_user_agreement_inline: false
+ secret_token: ~
diff --git a/apps/workbench/config/config.yml.example b/apps/workbench/config/config.yml.example
new file mode 100644
index 0000000..45c8a28
--- /dev/null
+++ b/apps/workbench/config/config.yml.example
@@ -0,0 +1,20 @@
+# Copy this file to config.yml and edit to suit.
+#
+# Consult config.defaults.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in config.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in config.yml called "common"
+# 4. Section in config.defaults.yml corresponding to RAILS_ENV
+# 5. Section in config.defaults.yml called "common"
+
+common:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ # You probably also want to point to your API server.
+ arvados_login_base: https://arvados.local:3000/login
+ arvados_v1_base: https://arvados.local:3000/arvados/v1
+ arvados_insecure_https: true
diff --git a/apps/workbench/config/initializers/load_config.rb b/apps/workbench/config/initializers/load_config.rb
new file mode 100644
index 0000000..7f2b1cc
--- /dev/null
+++ b/apps/workbench/config/initializers/load_config.rb
@@ -0,0 +1,44 @@
+conf = {}
+%w(config.defaults config).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ conf.merge!(confs['common'] || {})
+ conf.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+ArvadosWorkbench::Application.configure do
+ nils = []
+ conf.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in config/config.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
diff --git a/doc/install/install-api-server.html.md.liquid b/doc/install/install-api-server.html.md.liquid
index a16e50f..582feb9 100644
--- a/doc/install/install-api-server.html.md.liquid
+++ b/doc/install/install-api-server.html.md.liquid
@@ -5,18 +5,20 @@ title: Install the API server
navorder: 1
...
-{% include 'alert_stub' %}
-
# API server setup
-## Prerequisites
+## Prerequisites:
-1. A GNU/linux (virtual) machine
+1. A GNU/Linux (virtual) machine
2. A domain name for your api server
+3. Ruby >= 2.0.0
+4. Bundler
## Download the source tree
-Please follow the instructions on the [Download page](https://arvados.org/projects/arvados/wiki/Download) in the wiki.
+ git clone https://github.com/curoverse/arvados.git
+
+See also: [Downloading the source code](https://arvados.org/projects/arvados/wiki/Download) on the Arvados wiki.
## Configure the API server
@@ -27,61 +29,58 @@ First install the gems:
Next, configure the database:
- cp config/database.yml.sample config/database.yml
+ cp -i config/database.yml.sample config/database.yml
-Edit database.yml to your liking and make sure the database and db user exist.
+Edit `database.yml` to your liking and make sure the database and db user exist.
Then set up the database:
RAILS_ENV=production rake db:setup
Then set up omniauth:
- cp config/initializers/omniauth.rb.example config/initializers/omniauth.rb
+ cp -i config/initializers/omniauth.rb.example config/initializers/omniauth.rb
-Edit config/initializers/omniauth.rb. Choose an *APP_SECRET* and *APP_ID*. Also set
-*CUSTOM_PROVIDER_URL*.
+Edit `config/initializers/omniauth.rb`. Choose an *APP_SECRET* and
+*APP_ID*. Also set *CUSTOM_PROVIDER_URL*.
Make sure your Omniauth provider knows about your *APP_ID* and *APP_SECRET*
combination.
-You also need to update config/initializers/secret_token.rb. Generate a new secret with
+You also need to update `config/initializers/secret_token.rb`.
+Generate a new secret with
rake secret
-and put it in config/initializers/secret_token.rb:
+and put it in `config/initializers/secret_token.rb`:
Server::Application.config.secret_token = 'your-new-secret-here'
-Finally, edit the main configuration:
+Edit the main configuration:
- cp config/environments/production.rb.example config/environments/production.rb
+ cp -i config/config.yml.example config/config.yml
-First, you want to make sure that
+First, you want to make sure that
- config.uuid_prefix
+ uuid_prefix
-is set to a unique 5-digit hex string. You can replace the 'cfi-aws-0' string
-with a string of your choice to make that happen.
+is set to a unique 5-character alphanumeric string. An example is
+given that generates a 5-character string based on a hash of your
+hostname.
-The *config.uuid_prefix* string is a unique identifier for your API server. It
-also serves as the first part of the hostname for your API server, for instance
+The `uuid_prefix` is a unique identifier for your API server. It also
+serves as the first part of the hostname for your API server, for
+instance
{{ site.arvados_api_host }}
-You should use your own domain instead of arvadosapi.com
-
-Second, unless you are running on AWS, you will want to change the definition of
-
- config.compute_node_nameservers
+For a development site, use your own domain instead of arvadosapi.com.
-If you know your nameservers and they are fixed, you can hardcode them, and
-make sure to remove the code that tries to look them up from the AWS metadata:
+You will also want to change `compute_node_nameservers` to suit your
+environment.
- config.compute_node_nameservers = ['1.2.3.4','2.3.4.5','3.4.5.6']
- #require 'net/http'
- #config.compute_node_nameservers = ['local', 'public'].collect do |iface|
- # Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- #end << '172.16.0.23'
+Consult `config.defaults.yml` for a full list of configuration
+options. Always put your local configuration in `config.yml` instead
+of editing `config.defaults.yml`.
## Apache/Passenger
diff --git a/services/api/.gitignore b/services/api/.gitignore
index 80ba000..0e95cf8 100644
--- a/services/api/.gitignore
+++ b/services/api/.gitignore
@@ -18,6 +18,7 @@
/config/api.clinicalfuture.com.*
/config/database.yml
/config/initializers/omniauth.rb
+/config/config.yml
# asset cache
/public/assets/
diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index 459535b..805e1cc 100644
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -8,13 +8,7 @@ class Node < ArvadosModel
MAX_SLOTS = 64
- @@confdir = if Rails.configuration.respond_to? :dnsmasq_conf_dir
- Rails.configuration.dnsmasq_conf_dir
- elsif File.exists? '/etc/dnsmasq.d/.'
- '/etc/dnsmasq.d'
- else
- nil
- end
+ @@confdir = Rails.configuration.dnsmasq_conf_dir
@@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
@@nameservers = Rails.configuration.compute_node_nameservers
@@ -127,8 +121,8 @@ class Node < ArvadosModel
def start!(ping_url_method)
ensure_permission_to_update
ping_url = ping_url_method.call({ uuid: self.uuid, ping_secret: self.info[:ping_secret] })
- if (Rails.configuration.compute_node_ec2run_args rescue false) and
- (Rails.configuration.compute_node_ami rescue false)
+ if (Rails.configuration.compute_node_ec2run_args and
+ Rails.configuration.compute_node_ami)
ec2_args = ["--user-data '#{ping_url}'",
"-t c1.xlarge -n 1",
Rails.configuration.compute_node_ec2run_args,
diff --git a/services/api/config/config.defaults.yml b/services/api/config/config.defaults.yml
new file mode 100644
index 0000000..d474988
--- /dev/null
+++ b/services/api/config/config.defaults.yml
@@ -0,0 +1,85 @@
+# Do not use this file for site configuration. Create config.yml
+# instead (see config.yml.example).
+
+development:
+ force_ssl: false
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ action_mailer.perform_deliveries: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+
+test:
+ force_ssl: false
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_support.deprecation: :stderr
+ active_record.mass_assignment_sanitizer: :strict
+
+common:
+ uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
+
+ git_repositories_dir: /var/cache/git
+
+ # :none or :slurm_immediate
+ crunch_job_wrapper: :none
+
+ # username, or false = do not set uid when running jobs.
+ crunch_job_user: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ crunch_refresh_trigger: /tmp/crunch_refresh_trigger
+
+ # Path to /etc/dnsmasq.d, or false = do not update dnsmasq data.
+ dnsmasq_conf_dir: false
+
+ # Set to AMI id (ami-123456) to auto-start nodes. See app/models/node.rb
+ compute_node_ami: false
+ compute_node_ec2run_args: -g arvados-compute
+ compute_node_spot_bid: 0.11
+
+ compute_node_domain: <%= `hostname --domain`.strip %>
+ compute_node_nameservers:
+ - 192.168.1.1
+ compute_node_ec2_tag_enable: false
+
+ accept_api_token: {}
+
+ new_users_are_active: false
+ admin_notifier_email_from: arvados at example.com
+ email_subject_prefix: "[ARVADOS] "
+
+ # Visitors to the API server will be redirected to the workbench
+ workbench_address: https://workbench.local:3001/
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses openid against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ auto_admin_user: ~
diff --git a/services/api/config/config.yml.example b/services/api/config/config.yml.example
new file mode 100644
index 0000000..15f2be3
--- /dev/null
+++ b/services/api/config/config.yml.example
@@ -0,0 +1,37 @@
+# Copy this file to config.yml and edit to suit.
+#
+# Consult config.defaults.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in config.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in config.yml called "common"
+# 4. Section in config.defaults.yml corresponding to RAILS_ENV
+# 5. Section in config.defaults.yml called "common"
+
+development:
+
+production:
+ uuid_prefix: bogus
+
+ # This is suitable for AWS; see common section below for a static example.
+ compute_node_nameservers: <%=
+ require 'net/http'
+ ['local', 'public'].collect do |iface|
+ Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+ end << '172.16.0.23'
+ %>
+ # You must customize these. See config.defaults.yml for information.
+ compute_node_ami: ~
+ compute_node_ec2_tag_enable: ~
+ compute_node_domain: ~
+ compute_node_spot_bid: ~
+
+test:
+ uuid_prefix: zzzzz
+
+common:
+ compute_node_nameservers:
+ - 192.168.0.1
+ - 172.16.0.1
diff --git a/services/api/config/database.yml.sample b/services/api/config/database.yml.sample
index 62edd84..8f54e66 100644
--- a/services/api/config/database.yml.sample
+++ b/services/api/config/database.yml.sample
@@ -1,5 +1,5 @@
development:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_development
username: arvados
@@ -7,7 +7,7 @@ development:
host: localhost
test:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_test
username: arvados
@@ -15,7 +15,7 @@ test:
host: localhost
production:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_production
username: arvados
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
new file mode 100644
index 0000000..f56bb2d
--- /dev/null
+++ b/services/api/config/initializers/load_config.rb
@@ -0,0 +1,44 @@
+conf = {}
+%w(config.defaults config).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ conf.merge!(confs['common'] || {})
+ conf.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+Server::Application.configure do
+ nils = []
+ conf.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in config/config.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list