[ARVADOS] updated: 1.3.0-3285-gf43a1f883
Git user
git at public.arvados.org
Fri Oct 16 17:33:07 UTC 2020
Summary of changes:
doc/install/install-using-salt.html.textile.liquid | 84 +++++++----
tools/salt-install/Vagrantfile | 8 +-
tools/salt-install/provision.sh | 123 ++++++++++++++--
tools/salt-install/single_host/arvados.sls | 162 +++++++++++++++++++++
tools/salt-install/single_host/locale.sls | 14 ++
.../single_host/nginx_api_configuration.sls | 28 ++++
.../single_host/nginx_controller_configuration.sls | 59 ++++++++
.../single_host/nginx_keepproxy_configuration.sls | 58 ++++++++
.../single_host/nginx_keepweb_configuration.sls | 58 ++++++++
tools/salt-install/single_host/nginx_passenger.sls | 24 +++
.../single_host/nginx_webshell_configuration.sls | 75 ++++++++++
.../single_host/nginx_websocket_configuration.sls | 59 ++++++++
.../single_host/nginx_workbench2_configuration.sls | 49 +++++++
.../single_host/nginx_workbench_configuration.sls | 74 ++++++++++
tools/salt-install/single_host/postgresql.sls | 42 ++++++
15 files changed, 869 insertions(+), 48 deletions(-)
create mode 100644 tools/salt-install/single_host/arvados.sls
create mode 100644 tools/salt-install/single_host/locale.sls
create mode 100644 tools/salt-install/single_host/nginx_api_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_controller_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_keepproxy_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_keepweb_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_passenger.sls
create mode 100644 tools/salt-install/single_host/nginx_webshell_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_websocket_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_workbench2_configuration.sls
create mode 100644 tools/salt-install/single_host/nginx_workbench_configuration.sls
create mode 100644 tools/salt-install/single_host/postgresql.sls
via f43a1f883619d2ec25439145b9b35fec1265b0cc (commit)
from 244e797aed6f895dcdfe72ed90b31f3fe4cf6f85 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f43a1f883619d2ec25439145b9b35fec1265b0cc
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Oct 16 13:25:31 2020 -0300
feat(provision): example configurations in salt
refs #16471 and #16992
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/doc/install/install-using-salt.html.textile.liquid b/doc/install/install-using-salt.html.textile.liquid
index d57658ab7..629b760ae 100644
--- a/doc/install/install-using-salt.html.textile.liquid
+++ b/doc/install/install-using-salt.html.textile.liquid
@@ -13,49 +13,39 @@ SPDX-License-Identifier: CC-BY-SA-3.0
# "Introduction":#introduction
# "Install Saltstack":#saltstack
# "Install dependencies":#dependencies
-# "Install Arvados":#arvados
+# "Install Arvados using Saltstack":#saltstack
+# "Single host install using the provision.sh script":#single_host
+# "Local testing Arvados in a Vagrant box":#vagrant
h2(#tlrl). For the impatient (TL;RL)
-We added a "script":https://github.com/arvados/arvados/tree/master/tools/salt-install/provision.sh
-under the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory.
+We added a "script":https://github.com/arvados/arvados/tree/master/tools/salt-install/provision.sh under the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory.
-Just edit the @provision.sh@ script, modify the variables at the top of the file to suit your needs
-(usually not required), copy the file to the host where you want to install Arvados and run it as root
+Just edit the @provision.sh@ script, modify the variables at the top of the file to suit your needs (usually not required), copy the file to the host where you want to install Arvados and run it as root
<notextile>
-<pre><code>
-bash ./provision.sh
+<pre><code>sudo bash ./provision.sh
</code></pre>
</notextile>
It will install all the main Arvados components to get you up and running.
-There is a companion @Vagrantfile@ to install Arvados in a local Vagrant virtual machine. If you have
-"Vagrant":https://www.vagrantup.com/ installed in your computer, you can just get Arvados installed
-running in a local virtual machine with the command
+There is a companion @Vagrantfile@ to install Arvados in a local Vagrant virtual machine. If you have "Vagrant":https://www.vagrantup.com/ installed in your computer, you can just get Arvados installed running in a local virtual machine with the command
<notextile>
-<pre><code>
-vagrant up
+<pre><code>vagrant up
</code></pre>
</notextile>
-If you want to run these scripts in another host or instance, just copy the @provision.sh@ file to the
-desired host and run it.
+If you want to run these scripts in another host or instance, just copy the directory to the destination host, edit the @provision.sh@ file and run it.
The rest of the document is a description of the individual steps the @provision.sh@ script perform.
h2(#introduction). Introduction
-To ease the installation of the various Arvados components, we have developed a
-"Saltstack":https://www.saltstack.com/ 's
-"arvados-formula":https://github.com/saltstack-formulas/arvados-formula which
-can help you get an Arvados cluster up and running.
+To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://github.com/saltstack-formulas/arvados-formula which can help you get an Arvados cluster up and running.
-Saltstack is a Python-based, open-source software for event-driven IT
-automation, remote task execution, and configuration management. It can be used
-in a master/minion setup or master-less.
+Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a master/minion setup or master-less.
h2(#saltstack). Install Saltsack
@@ -64,8 +54,7 @@ If you already have a Saltstack environment you can skip this section.
The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
<notextile>
-<pre><code>
-curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+<pre><code>curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
</code></pre>
</notextile>
@@ -74,8 +63,7 @@ For more information check "Saltstack's documentation":https://docs.saltstack.co
h2(#dependencies). Install dependencies
-Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be
-installed using their respective Saltstack formulas.
+Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be installed using their respective Saltstack formulas.
The formulas we use are:
@@ -84,15 +72,11 @@ The formulas we use are:
* "docker":https://github.com/saltstack-formulas/docker-formula.git
* "locale":https://github.com/saltstack-formulas/locale-formula.git
-There are example Salt pillar files for each of those formulas in the
-test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples
-directory. As they are, they allow you to get all the main Arvados components up and running
-in a single node.
+There are example Salt pillar files for each of those formulas in the test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples directory. As they are, they allow you to get all the main Arvados components up and running.
-h2(#arvados). Install Arvados
+h2(#saltstack). Install Arvados using Saltstack
-The Arvados formula we maintain is located in the Saltstack's community repository of
-formulas:
+The Arvados formula we maintain is located in the Saltstack's community repository of formulas:
* "arvados-formula":https://github.com/saltstack-formulas/arvados-formula.git
@@ -101,3 +85,39 @@ and we keep a @development@ version at our own repo
* "arvados-formula develpment":https://github.com/arvados/arvados-formula.git
This last one might break from time to time, as we try and add new features. Use with caution.
+
+As much as possible, we try to keep it up to date, with example pillars to help you deploy Arvados.
+
+For those familiar with Saltstack, the process to get it deployed is similar to any other formula:
+
+1. Fork/copy the formula to your Salt master host.
+2. Edit the Arvados, nginx, postgres, locale and docker pillars to match your desired configuration.
+3. Run a @state.apply@ to get it deployed.
+
+h2(#single_host). Single host install using the provision.sh script
+
+In case you want to run it on a single node and don't want to (or don't have) a Saltstack master/minion infrastructure in place, we provide a very simple script named @provision.sh@ which can be used to deploy Arvados, using the @arvados-formula@ in a master-less setup.
+
+Just
+
+1. edit the variables at the very beginning ot the file,
+2. run the script
+3. wait for it to finish
+
+If everything goes OK, you'll get some final lines stating something like:
+
+<notextile>
+<pre><code>arvados: Succeeded: 109 (changed=9)
+arvados: Failed: 0
+</code></pre>
+</notextile>
+
+h2(#vagrant). Local testing Arvados in a Vagrant box
+
+If you want to try Arvados in a local virtual machine, there's a companion @Vagrantfile@ to install it on a virtual machine in your computer using "Vagrant":https://www.vagrantup.com/.
+
+To get it running, simply install Vagrant in your computer, edit the @provision@ file as described in the previous section, and run
+<notextile>
+<pre><code>vagrant up
+</code></pre>
+</notextile>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 36f682761..93bb77d4f 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -13,7 +13,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.define "arvados" do |arv|
arv.vm.box = "bento/debian-10"
- arv.vm.hostname = "arv.local"
+ arv.vm.hostname = "arva2.arv.local"
# Networking
arv.vm.network "forwarded_port", guest: 8443, host: 8443
arv.vm.network "forwarded_port", guest: 25100, host: 25100
@@ -28,6 +28,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# arv.vm.synced_folder "salt_pillars", "/srv/pillars",
# create: true
arv.vm.provision "shell",
- path: "provision.sh"
+ path: "provision.sh",
+ args: [
+ "--vagrant",
+ "--ssl-port=8443"
+ ].join(" ")
end
end
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 228b50664..1ee13752b 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -15,11 +15,21 @@
CLUSTER="arva2"
DOMAIN="arv.local"
+# The example config you want to use. Currently, only "single_host" is
+# available
+CONFIG_DIR="single_host"
+
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to 'latest'
# in the desired repo
-# VERSION="2.0.4"
+VERSION="latest"
+
+# Host SSL port where you want to point your browser to access Arvados
+# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
+# You can point it to another port if desired
+# In Vagrant, make sure it matches what you set in the Vagrantfile
+# HOST_SSL_PORT=443
# This is a arvados-formula setting.
# If branch is set, the script will switch to it before running salt
@@ -29,6 +39,55 @@ RELEASE="production"
##########################################################
# Usually there's no need to modify things below this line
+set -o pipefail
+
+usage() {
+ echo >&2
+ echo >&2 "Usage: $0 [-h] [-h]"
+ echo >&2
+ echo >&2 "$0 options:"
+ echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
+ echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2
+}
+
+arguments() {
+ # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+ TEMP=`getopt -o hvp: \
+ --long help,vagrant,ssl-port: \
+ -n "$0" -- "$@"`
+
+ if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ # Note the quotes around `$TEMP': they are essential!
+ eval set -- "$TEMP"
+
+ while [ $# -ge 1 ]; do
+ case $1 in
+ -v | --vagrant)
+ VAGRANT="yes"
+ shift
+ ;;
+ -p | --ssl-port)
+ HOST_SSL_PORT=${2}
+ shift 2
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+ done
+}
+
+HOST_SSL_PORT=443
+
+arguments $@
+
# Salt's dir
## states
S_DIR="/srv/salt"
@@ -36,19 +95,17 @@ S_DIR="/srv/salt"
F_DIR="/srv/formulas"
##pillars
P_DIR="/srv/pillars"
-# In vagrant, we can use the shared dir
-# P_DIR="/vagrant/salt_pillars"
-sudo apt-get update
-sudo apt-get install -y curl git
+apt-get update
+apt-get install -y curl git
dpkg -l |grep salt-minion
if [ ${?} -eq 0 ]; then
echo "Salt already installed"
else
curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
- sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
- sudo /bin/systemctl disable salt-minion.service
+ sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+ /bin/systemctl disable salt-minion.service
fi
# Set salt to masterless mode
@@ -103,7 +160,8 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
for f in postgres arvados nginx docker locale; do
- git clone https://github.com/saltstack-formulas/${f}-formula.git
+ # git clone https://github.com/saltstack-formulas/${f}-formula.git
+ git clone https://github.com/netmanagers/${f}-formula.git
done
if [ "x${BRANCH}" != "x" ]; then
@@ -112,18 +170,55 @@ if [ "x${BRANCH}" != "x" ]; then
cd -
fi
-sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g; s/release: development/release: ${RELEASE}/g; s/# version: '2.0.4'/version: '${VERSION}'/g" \
- ${F_DIR}/arvados-formula/test/salt/pillar/arvados_dev.sls > ${P_DIR}/arvados.sls
+# sed "s/__DOMAIN__/${DOMAIN}/g; s/__CLUSTER__/${CLUSTER}/g; s/__RELEASE__/${RELEASE}/g; s/__VERSION__/${VERSION}/g" \
+# ${CONFIG_DIR}/arvados_dev.sls > ${P_DIR}/arvados.sls
+
+if [ "x${VAGRANT}" = "xyes" ]; then
+ SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+else
+ SOURCE_PILLARS_DIR="./${CONFIG_DIR}"
+fi
# Replace cluster and domain name in the example pillars
-for f in ${F_DIR}/arvados-formula/test/salt/pillar/examples/*; do
- sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
+for f in ${SOURCE_PILLARS_DIR}/*; do
+ # sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
+ sed "s/__DOMAIN__/${DOMAIN}/g;
+ s/__CLUSTER__/${CLUSTER}/g;
+ s/__RELEASE__/${RELEASE}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
+ s/__VERSION__/${VERSION}/g" \
${f} > ${P_DIR}/$(basename ${f})
done
-# Let's write a /etc/hosts file that points all the hosts to localhost
+# Let's write an /etc/hosts file that points all the hosts to localhost
echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ -e /root/.psqlrc ]; then
+ if ! ( grep 'pset pager off' /root/.psqlrc ); then
+ RESTORE_PSQL="yes"
+ cp /root/.psqlrc /root/.psqlrc.provision.backup
+ fi
+else
+ DELETE_PSQL="yes"
+fi
+
+echo '\pset pager off' >> /root/.psqlrc
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
# Now run the install
salt-call --local state.apply -l debug
+
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ "x${DELETE_PSQL}" = "xyes" ]; then
+ echo "Removing .psql file"
+ rm /root/.psqlrc
+fi
+
+if [ "x${RESTORE_PSQL}" = "xyes" ]; then
+ echo "Restroting .psql file"
+ mv -v /root/.psqlrc.provision.backup /root/.psqlrc
+fi
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
diff --git a/tools/salt-install/single_host/arvados.sls b/tools/salt-install/single_host/arvados.sls
new file mode 100644
index 000000000..9b1e41d46
--- /dev/null
+++ b/tools/salt-install/single_host/arvados.sls
@@ -0,0 +1,162 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The variables commented out are the default values that the formula uses.
+# The uncommented values are REQUIRED values. If you don't set them, running
+# this formula will fail.
+arvados:
+ ### GENERAL CONFIG
+ version: '__VERSION__'
+ ## It makes little sense to disable this flag, but you can, if you want :)
+ # use_upstream_repo: true
+
+ ## Repo URL is built with grains values. If desired, it can be completely
+ ## overwritten with the pillar parameter 'repo_url'
+ # repo:
+ # humanname: Arvados Official Repository
+
+ release: __RELEASE__
+
+ ## IMPORTANT!!!!!
+ ## api, workbench and shell require some gems, so you need to make sure ruby
+ ## and deps are installed in order to install and compile the gems.
+ ## We default to `false` in these two variables as it's expected you already
+ ## manage OS packages with some other tool and you don't want us messing up
+ ## with your setup.
+ ruby:
+ ## We set these to `true` here for testing purposes.
+ ## They both default to `false`.
+ manage_ruby: true
+ manage_gems_deps: true
+ # pkg: ruby
+ # gems_deps:
+ # - curl
+ # - g++
+ # - gcc
+ # - git
+ # - libcurl4
+ # - libcurl4-gnutls-dev
+ # - libpq-dev
+ # - libxml2
+ # - libxml2-dev
+ # - make
+ # - python3-dev
+ # - ruby-dev
+ # - zlib1g-dev
+
+ # config:
+ # file: /etc/arvados/config.yml
+ # user: root
+ ## IMPORTANT!!!!!
+ ## If you're intalling any of the rails apps (api, workbench), the group
+ ## should be set to that of the web server, usually `www-data`
+ # group: root
+ # mode: 640
+
+ ### ARVADOS CLUSTER CONFIG
+ cluster:
+ name: __CLUSTER__
+ domain: __DOMAIN__
+
+ database:
+ # max concurrent connections per arvados server daemon
+ # connection_pool_max: 32
+ name: arvados
+ host: 127.0.0.1
+ password: changeme_arvados
+ user: arvados
+ encoding: en_US.utf8
+ client_encoding: UTF8
+
+ tls:
+ # certificate: ''
+ # key: ''
+ # required to test with snakeoil certs
+ insecure: true
+
+ ### TOKENS
+ tokens:
+ system_root: changeme_system_root_token
+ management: changeme_management_token
+ rails_secret: changeme_rails_secret_token
+ anonymous_user: changeme_anonymous_user_token
+
+ ### KEYS
+ secrets:
+ blob_signing_key: changeme_blob_signing_key
+ workbench_secret_key: changeme_workbench_secret_key
+ dispatcher_access_key: changeme_dispatcher_access_key
+ dispatcher_secret_key: changeme_dispatcher_secret_key
+ keep_access_key: changeme_keep_access_key
+ keep_secret_key: changeme_keep_secret_key
+
+ Login:
+ Test:
+ Enable: true
+ javier:
+ User: javier at arva2.arv.local
+ Password: perico
+
+ AuditLogs:
+ Section_to_ignore:
+ - some_random_value
+
+ ### VOLUMES
+ ## This should usually match all your `keepstore` instances
+ Volumes:
+ # the volume name will be composed with
+ # <cluster>-nyw5e-<volume>
+ __CLUSTER__-nyw5e-000000000000000:
+ AccessViaHosts:
+ http://keep0.__CLUSTER__.__DOMAIN__:25107:
+ ReadOnly: false
+ Replication: 2
+ Driver: Directory
+ DriverParameters:
+ Root: /tmp
+
+ Users:
+ NewUsersAreActive: true
+ AutoAdminFirstUser: true
+ AutoSetupNewUsers: true
+ AutoSetupNewUsersWithRepository: true
+
+ Services:
+ Controller:
+ ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://127.0.0.2:8003: {}
+ DispatchCloud:
+ InternalURLs:
+ http://__CLUSTER__.__DOMAIN__:9006: {}
+ Keepbalance:
+ InternalURLs:
+ http://__CLUSTER__.__DOMAIN__:9005: {}
+ Keepproxy:
+ ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://127.0.0.2:25100: {}
+ Keepstore:
+ InternalURLs:
+ http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
+ RailsAPI:
+ InternalURLs:
+ http://127.0.0.2:8004: {}
+ WebDAV:
+ ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://127.0.0.2:9002: {}
+ WebDAVDownload:
+ ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ WebShell:
+ ExternalURL: https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ Websocket:
+ ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
+ InternalURLs:
+ http://127.0.0.2:8005: {}
+ Workbench1:
+ ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ Workbench2:
+ ExternalURL: https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
diff --git a/tools/salt-install/single_host/locale.sls b/tools/salt-install/single_host/locale.sls
new file mode 100644
index 000000000..17f53a288
--- /dev/null
+++ b/tools/salt-install/single_host/locale.sls
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+locale:
+ present:
+ - "en_US.UTF-8 UTF-8"
+ default:
+ # Note: On debian systems don't write the second 'UTF-8' here or you will
+ # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8
+ # Restart the minion after you corrected this!
+ name: 'en_US.UTF-8'
+ requires: 'en_US.UTF-8 UTF-8'
diff --git a/tools/salt-install/single_host/nginx_api_configuration.sls b/tools/salt-install/single_host/nginx_api_configuration.sls
new file mode 100644
index 000000000..db0bea126
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_api_configuration.sls
@@ -0,0 +1,28 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ arvados_api:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: '127.0.0.2:8004'
+ - server_name: api
+ - root: /var/www/arvados-api/current/public
+ - index: index.html index.htm
+ - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+ - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log
+ - passenger_enabled: 'on'
+ - client_max_body_size: 128m
diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls
new file mode 100644
index 000000000..2b2e7d591
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_controller_configuration.sls
@@ -0,0 +1,59 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ 'geo $external_client':
+ default: 1
+ '127.0.0.0/8': 0
+ upstream controller_upstream:
+ - server: '127.0.0.2:8003 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_controller_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: __CLUSTER__.__DOMAIN__
+ - listen:
+ - 80 default
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_controller_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: __CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://controller_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_set_header: 'X-External-Client $external_client'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
+ - client_max_body_size: 128m
diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
new file mode 100644
index 000000000..29cd0cb44
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream keepproxy_upstream:
+ - server: '127.0.0.2:25100 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_keepproxy_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_keepproxy_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://keepproxy_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
new file mode 100644
index 000000000..bd0a636b0
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream collections_downloads_upstream:
+ - server: '127.0.0.2:9002 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_collections_download_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ ### COLLECTIONS / DOWNLOAD
+ arvados_collections_download_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://collections_downloads_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_max_body_size: 0
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_passenger.sls b/tools/salt-install/single_host/nginx_passenger.sls
new file mode 100644
index 000000000..6ce75faa7
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_passenger.sls
@@ -0,0 +1,24 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ install_from_phusionpassenger: true
+ lookup:
+ passenger_package: libnginx-mod-http-passenger
+ passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+
+ ### SERVER
+ server:
+ config:
+ include: 'modules-enabled/*.conf'
+ worker_processes: 4
+
+ ### SITES
+ servers:
+ managed:
+ # Remove default webserver
+ default:
+ enabled: false
diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls
new file mode 100644
index 000000000..e33ddcea7
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls
@@ -0,0 +1,75 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ upstream webshell_upstream:
+ - server: '127.0.0.2:4200 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ arvados_webshell_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: webshell.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_webshell_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: webshell.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /shell.__CLUSTER__.__DOMAIN__:
+ - proxy_pass: 'http://webshell_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_ssl_session_reuse: 'off'
+
+ - "if ($request_method = 'OPTIONS')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+ - add_header: "'Access-Control-Max-Age' 1728000"
+ - add_header: "'Content-Type' 'text/plain charset=UTF-8'"
+ - add_header: "'Content-Length' 0"
+ - return: 204
+
+ - "if ($request_method = 'POST')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+ - "if ($request_method = 'GET')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
+
diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls
new file mode 100644
index 000000000..2241d3b8e
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls
@@ -0,0 +1,59 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream websocket_upstream:
+ - server: '127.0.0.2:8005 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_websocket_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_websocket_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://websocket_upstream'
+ - proxy_read_timeout: 600
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: 'Host $host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'Upgrade $http_upgrade'
+ - proxy_set_header: 'Connection "upgrade"'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
new file mode 100644
index 000000000..733397adf
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
@@ -0,0 +1,49 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench2_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench2_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - root: /var/www/arvados-workbench2/workbench2
+ - try_files: '$uri $uri/ /index.html'
+ - 'if (-f $document_root/maintenance.html)':
+ - return: 503
+ - location /config.json:
+ - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__"}' ~ "'" }}
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls
new file mode 100644
index 000000000..76fb13438
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls
@@ -0,0 +1,74 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ upstream workbench_upstream:
+ - server: '127.0.0.2:9000 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://workbench_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
+
+ arvados_workbench_upstream:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: '127.0.0.2:9000'
+ - server_name: workbench
+ - root: /var/www/arvados-workbench/current/public
+ - index: index.html index.htm
+ - passenger_enabled: 'on'
+ # yamllint disable-line rule:line-length
+ - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+ - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log
diff --git a/tools/salt-install/single_host/postgresql.sls b/tools/salt-install/single_host/postgresql.sls
new file mode 100644
index 000000000..56b0a42e8
--- /dev/null
+++ b/tools/salt-install/single_host/postgresql.sls
@@ -0,0 +1,42 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### POSTGRESQL
+postgres:
+ use_upstream_repo: false
+ pkgs_extra:
+ - postgresql-contrib
+ postgresconf: |-
+ listen_addresses = '*' # listen on all interfaces
+ acls:
+ - ['local', 'all', 'postgres', 'peer']
+ - ['local', 'all', 'all', 'peer']
+ - ['host', 'all', 'all', '127.0.0.1/32', 'md5']
+ - ['host', 'all', 'all', '::1/128', 'md5']
+ - ['host', 'arvados', 'arvados', '127.0.0.1/32']
+ users:
+ arvados:
+ ensure: present
+ password: changeme_arvados
+
+ # tablespaces:
+ # arvados_tablespace:
+ # directory: /path/to/some/tbspace/arvados_tbsp
+ # owner: arvados
+
+ databases:
+ arvados:
+ owner: arvados
+ template: template0
+ lc_ctype: en_US.utf8
+ lc_collate: en_US.utf8
+ # tablespace: arvados_tablespace
+ schemas:
+ public:
+ owner: arvados
+ extensions:
+ pg_trgm:
+ if_not_exists: true
+ schema: public
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list