[ARVADOS] created: 1.3.0-3280-g2501c7aa3
Git user
git at public.arvados.org
Wed Oct 7 21:39:57 UTC 2020
at 2501c7aa363e2b515e3d6c3934704930fd2a71b7 (commit)
commit 2501c7aa363e2b515e3d6c3934704930fd2a71b7
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Oct 7 18:01:09 2020 -0300
feat(salt): start salt install documentation
refs #16471
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/doc/install/install-using-salt.html.textile.liquid b/doc/install/install-using-salt.html.textile.liquid
new file mode 100644
index 000000000..294ef89be
--- /dev/null
+++ b/doc/install/install-using-salt.html.textile.liquid
@@ -0,0 +1,53 @@
+---
+layout: default
+navsection: installguide
+title: Install Arvados using Saltstack's arvados-formula
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "For the impatient (tl;rl)":#tlrl
+# "Introduction":#introduction
+# "Install dependencies":#dependencies
+
+h2(#tlrl). For the impatient (TL;RL)
+
+We added a "script"::https://github.com/arvados/arvados/tree/master/tools/salt-install/bin/provision.sh
+and example configuration files under the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install
+directory, which you can use to get all the Arvados components up and running
+in a single node (The whole directory can be used to create a local VM with all
+Arvados' components in it, using "Vagrant":https://www.vagrantup.com/ )
+
+Just edit the parameters in the "provision.sh" script and run
+
+<notextile>
+<pre><code>
+vagrant up
+</notextile>
+
+If you want to run these scripts in another host or instance, just copy the
+directory to a temporary directory, edit it and run
+
+<notextile>
+<pre><code>
+bash ./bin/provision.sh
+</notextile>
+
+h2(#introduction). Introduction
+
+To ease the installation of the various Arvados components, we have developed a
+"Saltstack":https://www.saltstack.com/ 's
+"arvados-formula":https://github.com/saltstack-formulas/arvados-formula which
+can help you get an Arvados cluster up and running.
+
+Saltstack is a Python-based, open-source software for event-driven IT
+automation, remote task execution, and configuration management. It can be used
+in a master/minion setup or master-less.
+
+h2(#dependencies). Install dependencies
+
+Arvados depends in a few applications (postgresql, nginx) that can also be installed using Salt.
+
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
new file mode 100644
index 000000000..d91336fd7
--- /dev/null
+++ b/tools/salt-install/Vagrantfile
@@ -0,0 +1,33 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Vagrantfile API/syntax version. Don"t touch unless you know what you"re doing!
+VAGRANTFILE_API_VERSION = "2".freeze
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ config.ssh.insert_key = false
+ config.ssh.forward_x11 = true
+
+ config.vm.define "arvados" do |arv|
+ arv.vm.box = "bento/debian-10"
+ arv.vm.hostname = "arv.local"
+ # Networking
+ arv.vm.network "forwarded_port", guest: 8443, host: 8443
+ arv.vm.network "forwarded_port", guest: 25101, host: 25101
+ arv.vm.network "forwarded_port", guest: 9002, host: 9002
+ arv.vm.network "forwarded_port", guest: 9000, host: 9000
+ arv.vm.network "forwarded_port", guest: 8900, host: 8900
+ arv.vm.network "forwarded_port", guest: 8002, host: 8002
+ arv.vm.network "forwarded_port", guest: 8001, host: 8001
+ arv.vm.network "forwarded_port", guest: 8000, host: 8000
+ arv.vm.network "forwarded_port", guest: 3001, host: 3001
+ # config.vm.network "private_network", ip: "192.168.33.10"
+ arv.vm.synced_folder "salt_pillars", "/srv/pillars",
+ create: true
+ arv.vm.provision "shell",
+ path: "bin/provision.sh"
+ end
+end
diff --git a/tools/salt-install/bin/provision.sh b/tools/salt-install/bin/provision.sh
new file mode 100755
index 000000000..c7bf81de5
--- /dev/null
+++ b/tools/salt-install/bin/provision.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# If you want to test arvados in a single host, you can run this script, which
+# will install it using salt masterless
+# This script is run by the Vagrant file when you run it with
+#
+# vagrant up
+
+# This could have been done with the Salt vagrant provisioner, but this script
+# can be used in environments other than vagrant.
+
+# If branch is set, the script will switch to it before running salt
+BRANCH="refactor-config-add-service"
+
+CLUSTER="arva2"
+DOMAIN="arv.local"
+
+# Salt's dir
+## states
+S_DIR="/srv/salt"
+## formulas
+F_DIR="/srv/formulas"
+##pillars
+P_DIR="/srv/pillars"
+# In vagrant, we can use the shared dir
+P_DIR="/vagrant/salt_pillars"
+
+sudo apt-get update
+sudo apt-get install -y curl git
+
+dpkg -l |grep salt-minion
+if [ ${?} -eq 0 ]; then
+ echo "Salt already installed"
+else
+ curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+ sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+ sudo /bin/systemctl disable salt-minion.service
+fi
+
+# Set salt to masterless mode
+cat > /etc/salt/minion << EOFSM
+file_client: local
+file_roots:
+ base:
+ - ${S_DIR}
+ - ${F_DIR}/*
+ - ${F_DIR}/*/test/salt/states
+
+pillar_roots:
+ base:
+ - ${P_DIR}
+EOFSM
+
+mkdir -p ${S_DIR}
+mkdir -p ${F_DIR}
+mkdir -p ${P_DIR}
+
+cat > ${S_DIR}/top.sls << EOFTSLS
+base:
+ '*':
+ - example_add_snakeoil_certs
+ - nginx.passenger
+ - postgres
+ - docker
+ - arvados
+EOFTSLS
+
+cat > ${P_DIR}/top.sls << EOFPSLS
+base:
+ '*':
+ - arvados
+ - nginx_api_configuration
+ - nginx_controller_configuration
+ - nginx_keepproxy_configuration
+ - nginx_keepweb_configuration
+ - nginx_passenger
+ - nginx_websocket_configuration
+ - nginx_workbench2_configuration
+ - nginx_workbench_configuration
+ - postgresql
+EOFPSLS
+
+
+# Get the formula and dependencies
+cd ${F_DIR} || exit 1
+for f in postgres arvados nginx docker; do
+ git clone https://github.com/netmanagers/${f}-formula.git
+done
+
+if [ "x${BRANCH}" != "x" ]; then
+ cd ${F_DIR}/arvados-formula
+ git checkout -t origin/${BRANCH}
+ cd -
+fi
+
+sed "s/example.net/${DOMAIN}/g; s/name: fixme/name: ${CLUSTER}/g" \
+ ${F_DIR}/arvados-formula/test/salt/pillar/arvados.sls > ${P_DIR}/arvados.sls
+
+# Replace domain name in the example pillars
+for f in ${F_DIR}/arvados-formula/test/salt/pillar/examples/*; do
+ sed "s/example.net/${DOMAIN}/g" ${f} > ${P_DIR}/$(basename ${f})
+done
+
+# # Copy arvados' pillar.example file to the pillars dir, so it's used
+# sed "s/example.net/${DOMAIN}/g" ${F_DIR}/arvados-formula/pillar.example > ${P_DIR}/arvados.sls
+#
+# # Replace domain name in the example pillars
+# for f in ${F_DIR}/arvados-formula/test/salt/pillar/examples/*; do
+# sed "s/example.net/${DOMAIN}/g" ${f} > ${P_DIR}/$(basename ${f})
+# done
+#
+# Let's write a /etc/hosts file that points all the hosts to localhost
+
+echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+
+# Now run the install
+salt-call --local state.apply -l debug
diff --git a/tools/salt-install/salt_pillars/arvados.sls b/tools/salt-install/salt_pillars/arvados.sls
new file mode 100644
index 000000000..1609e8341
--- /dev/null
+++ b/tools/salt-install/salt_pillars/arvados.sls
@@ -0,0 +1,284 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# The variables commented out are the default values that the formula uses.
+# The uncommented values are REQUIRED values. If you don't set them, running
+# this formula will fail.
+arvados:
+ ### GENERAL CONFIG
+ # version: '2.0.4'
+ ## It makes little sense to disable this flag, but you can, if you want :)
+ # use_upstream_repo: true
+
+ ## Repo URL is built with grains values. If desired, it can be completely
+ ## overwritten with the pillar parameter 'repo_url'
+ # repo:
+ # humanname: Arvados Official Repository
+
+ ## IMPORTANT!!!!!
+ ## api, workbench and shell require some gems, so you need to make sure ruby
+ ## and deps are installed in order to install and compile the gems.
+ ## We default to `false` in these two variables as it's expected you already
+ ## manage OS packages with some other tool and you don't want us messing up
+ ## with your setup.
+ ruby:
+ ## We set these to `true` here for testing purposes.
+ ## They both default to `false`.
+ manage_ruby: true
+ manage_gems_deps: true
+ # pkg: ruby
+ # gems_deps:
+ # - curl
+ # - g++
+ # - gcc
+ # - git
+ # - libcurl4
+ # - libcurl4-gnutls-dev
+ # - libpq-dev
+ # - libxml2
+ # - libxml2-dev
+ # - make
+ # - python3-dev
+ # - ruby-dev
+ # - zlib1g-dev
+
+ # config:
+ # file: /etc/arvados/config.yml
+ # user: root
+ ## IMPORTANT!!!!!
+ ## If you're intalling any of the rails apps (api, workbench), the group
+ ## should be set to that of the web server, usually `www-data`
+ # group: root
+ # mode: 640
+
+ ### ARVADOS CLUSTER CONFIG
+ cluster:
+ name: fixme
+ domain: arv.local
+
+ database:
+ # max concurrent connections per arvados server daemon
+ # connection_pool_max: 32
+ name: arvados
+ host: 127.0.0.1
+ password: changeme_arvados
+ user: arvados
+ encoding: en_US.utf8
+ client_encoding: UTF8
+
+ tls:
+ # certificate: ''
+ # key: ''
+ # required to test with snakeoil certs
+ insecure: true
+
+ ### TOKENS
+ tokens:
+ system_root: changeme_system_root_token
+ management: changeme_management_token
+ rails_secret: changeme_rails_secret_token
+ anonymous_user: changeme_anonymous_user_token
+
+ ### KEYS
+ secrets:
+ blob_signing_key: changeme_blob_signing_key
+ workbench_secret_key: changeme_workbench_secret_key
+ dispatcher_access_key: changeme_dispatcher_access_key
+ dispatcher_secret_key: changeme_dispatcher_secret_key
+ keep_access_key: changeme_keep_access_key
+ keep_secret_key: changeme_keep_secret_key
+
+ AuditLogs:
+ Section_to_ignore:
+ - some_random_value
+
+ ### VOLUMES
+ ## This should usually match all your `keepstore` instances
+ Volumes:
+ # the volume name will be composed with
+ # <cluster>-nyw5e-<volume>
+ fixme-nyw5e-000000000000000:
+ AccessViaHosts:
+ http://keep0.fixme.arv.local:25107:
+ ReadOnly: false
+ Replication: 2
+ Driver: Directory
+ DriverParameters:
+ Root: /tmp
+
+ Users:
+ NewUsersAreActive: true
+ AutoAdminFirstUser: true
+ AutoSetupNewUsers: true
+ AutoSetupNewUsersWithRepository: true
+
+ Services:
+ Controller:
+ ExternalURL: https://fixme.arv.local
+ InternalURLs:
+ http://localhost:8003: {}
+ DispatchCloud:
+ InternalURLs:
+ http://fixme.arv.local:9006: {}
+ Keepbalance:
+ InternalURLs:
+ http://fixme.arv.local:9005: {}
+ Keepproxy:
+ ExternalURL: https://keep.fixme.arv.local
+ InternalURLs:
+ http://localhost:25107: {}
+ Keepstore:
+ InternalURLs:
+ http://keep0.fixme.arv.local:25107: {}
+ RailsAPI:
+ InternalURLs:
+ http://localhost:8004: {}
+ WebDAV:
+ ExternalURL: https://collections.fixme.arv.local
+ InternalURLs:
+ http://localhost:9002: {}
+ WebDAVDownload:
+ ExternalURL: https://download.fixme.arv.local
+ Websocket:
+ ExternalURL: wss://ws.fixme.arv.local/websocket
+ InternalURLs:
+ http://localhost:8005: {}
+ Workbench1:
+ ExternalURL: https://workbench.fixme.arv.local
+ Workbench2:
+ ExternalURL: https://workbench2.fixme.arv.local
+
+# ### THESE ARE THE PACKAGES AND DAEMONS BASIC CONFIGS
+# #### API
+# api:
+# pkg:
+# name:
+# - arvados-api-server
+# - arvados-dispatch-cloud
+# gem:
+# name:
+# - arvados-cli
+# service:
+# name:
+# - nginx
+# port: 8004
+# #### CONTROLLER
+# controller:
+# pkg:
+# name: arvados-controller
+# service:
+# name: arvados-controller
+# port: 8003
+# #### DISPATCHER
+# dispatcher:
+# pkg:
+# name:
+# - crunch-dispatch-local
+# # - arvados-dispatch-cloud
+# # - crunch-dispatch-slurm
+# service:
+# name: crunch-dispatch-local
+# port: 9006
+# #### KEEPPROXY
+# keepproxy:
+# pkg:
+# name: keepproxy
+# service:
+# name: keepproxy
+# port: 25107
+# #### KEEPWEB
+# keepweb:
+# pkg:
+# name: keep-web
+# service:
+# name: keep-web
+# # webdav
+# port: 9002
+# #### KEEPSTORE
+# keepstore:
+# pkg:
+# name: keepstore
+# service:
+# name: keepstore
+# port: 25107
+# #### GIT-HTTPD
+# githttpd:
+# pkg:
+# name: arvados-git-httpd
+# service:
+# name: arvados-git-httpd
+# port: 9001
+# #### SHELL
+# shell:
+# pkg:
+# name:
+# - arvados-client
+# - arvados-src
+# - libpam-arvados
+# - python-arvados-fuse
+# - python3-arvados-python-client
+# - python3-arvados-cwl-runner
+# gem:
+# name:
+# - arvados-cli
+# - arvados-login-sync
+# #### WORKBENCH
+# workbench:
+# pkg:
+# name: arvados-workbench
+# service:
+# name: nginx
+# #### WORKBENCH2
+# workbench2:
+# pkg:
+# name: arvados-workbench2
+# service:
+# name: nginx
+# #### WEBSOCKET
+# websocket:
+# pkg:
+# name: arvados-ws
+# service:
+# name: arvados-ws
+# port: 8005
+# #### SSO
+# sso:
+# pkg:
+# name: arvados-sso
+# service:
+# name: arvados-sso
+# port: 8900
+
+# ## SALTSTACK FORMULAS TOFS configuration
+# https://template-formula.readthedocs.io/en/latest/TOFS_pattern.html
+# tofs:
+# # The files_switch key serves as a selector for alternative
+# # directories under the formula files directory. See TOFS pattern
+# # doc for more info.
+# # Note: Any value not evaluated by `config.get` will be used literally.
+# # This can be used to set custom paths, as many levels deep as required.
+# files_switch:
+# - any/path/can/be/used/here
+# - id
+# - roles
+# - osfinger
+# - os
+# - os_family
+# # All aspects of path/file resolution are customisable using the options below.
+# # This is unnecessary in most cases; there are sensible defaults.
+# # Default path: salt://< path_prefix >/< dirs.files >/< dirs.default >
+# # I.e.: salt://arvados/files/default
+# # path_prefix: template_alt
+# # dirs:
+# # files: files_alt
+# # default: default_alt
+# # The entries under `source_files` are prepended to the default source files
+# # given for the state
+# # source_files:
+# # arvados-config-file-file-managed:
+# # - 'example_alt.tmpl'
+# # - 'example_alt.tmpl.jinja'
diff --git a/tools/salt-install/salt_pillars/nginx_api_configuration.sls b/tools/salt-install/salt_pillars/nginx_api_configuration.sls
new file mode 100644
index 000000000..f2dca810a
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_api_configuration.sls
@@ -0,0 +1,30 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ arvados_api:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: '127.0.0.1:8004'
+ - server_name: api
+ - root: /var/www/arvados-api/current/public
+ - index: index.html index.htm
+ - access_log: {{ nginx_log }}/api.arv.local-upstream.access.log combined
+ - error_log: {{ nginx_log }}/api.arv.local-upstream.error.log
+ - passenger_enabled: 'on'
+ - client_max_body_size: 128m
diff --git a/tools/salt-install/salt_pillars/nginx_controller_configuration.sls b/tools/salt-install/salt_pillars/nginx_controller_configuration.sls
new file mode 100644
index 000000000..6e2aa6af6
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_controller_configuration.sls
@@ -0,0 +1,62 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ 'geo $external_client':
+ default: 1
+ '127.0.0.0/24': 0
+ upstream controller_upstream:
+ - server: 'localhost:8003 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_controller_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: arv.local api.arv.local
+ - listen:
+ - 80 default
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_controller:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://controller_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_set_header: 'X-External-Client $external_client'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/arv.local.access.log combined
+ - error_log: {{ nginx_log }}/arv.local.error.log
+ - client_max_body_size: 128m
diff --git a/tools/salt-install/salt_pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/salt_pillars/nginx_keepproxy_configuration.sls
new file mode 100644
index 000000000..6a39cc473
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_keepproxy_configuration.sls
@@ -0,0 +1,60 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream keepproxy_upstream:
+ - server: '127.0.0.1:25107 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_keep_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.arv.local
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_keepproxy:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://keepproxy_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/keepproxy.arv.local.access.log combined
+ - error_log: {{ nginx_log }}/keepproxy.arv.local.error.log
diff --git a/tools/salt-install/salt_pillars/nginx_keepweb_configuration.sls b/tools/salt-install/salt_pillars/nginx_keepweb_configuration.sls
new file mode 100644
index 000000000..b769c0305
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_keepweb_configuration.sls
@@ -0,0 +1,60 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream collections_downloads_upstream:
+ - server: '127.0.0.1:9002 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_collections_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.arv.local download.arv.local
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ ### COLLECTIONS / DOWNLOAD
+ arvados_collections_downloads:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.arv.local download.arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://collections_downloads_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_max_body_size: 0
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/collections.arv.local.access.log combined
+ - error_log: {{ nginx_log }}/collections.arv.local.error.log
diff --git a/tools/salt-install/salt_pillars/nginx_passenger.sls b/tools/salt-install/salt_pillars/nginx_passenger.sls
new file mode 100644
index 000000000..6c9372604
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_passenger.sls
@@ -0,0 +1,29 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+states:
+ - nginx.passenger
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### NGINX
+nginx:
+ install_from_phusionpassenger: true
+ lookup:
+ passenger_package: libnginx-mod-http-passenger
+ passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+
+ ### SERVER
+ server:
+ config:
+ include: 'modules-enabled/*.conf'
+ worker_processes: 4
+
+ ### SITES
+ servers:
+ managed:
+ # Remove default webserver
+ default:
+ enabled: false
diff --git a/tools/salt-install/salt_pillars/nginx_websocket_configuration.sls b/tools/salt-install/salt_pillars/nginx_websocket_configuration.sls
new file mode 100644
index 000000000..6db306a70
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_websocket_configuration.sls
@@ -0,0 +1,61 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream websocket_upstream:
+ - server: '127.0.0.1:8005 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_ws_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.arv.local
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_websocket:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://websocket_upstream'
+ - proxy_read_timeout: 600
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: 'Host $host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'Upgrade $http_upgrade'
+ - proxy_set_header: 'Connection "upgrade"'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/ws.arv.local.access.log combined
+ - error_log: {{ nginx_log }}/ws.arv.local.error.log
diff --git a/tools/salt-install/salt_pillars/nginx_workbench2_configuration.sls b/tools/salt-install/salt_pillars/nginx_workbench2_configuration.sls
new file mode 100644
index 000000000..d0c85347c
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_workbench2_configuration.sls
@@ -0,0 +1,49 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench2_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.arv.local
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench2:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - root: /var/www/arvados-workbench2/workbench2
+ - try_files: '$uri $uri/ /index.html'
+ - 'if (-f $document_root/maintenance.html)':
+ - return: 503
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/workbench2.arv.local.access.log combined
+ - error_log: {{ nginx_log }}/workbench2.arv.local.error.log
diff --git a/tools/salt-install/salt_pillars/nginx_workbench_configuration.sls b/tools/salt-install/salt_pillars/nginx_workbench_configuration.sls
new file mode 100644
index 000000000..58eb15cc7
--- /dev/null
+++ b/tools/salt-install/salt_pillars/nginx_workbench_configuration.sls
@@ -0,0 +1,75 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+{% set nginx_log = '/var/log/nginx' %}
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ upstream workbench_upstream:
+ - server: '127.0.0.1:9000 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.arv.local
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.arv.local
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://workbench_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ # - include: 'snippets/letsencrypt.conf'
+ - include: 'snippets/snakeoil.conf'
+ - access_log: {{ nginx_log }}/workbench.arv.local.access.log combined
+ - error_log: {{ nginx_log }}/workbench.arv.local.error.log
+
+ arvados_workbench_upstream:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: '127.0.0.1:9000'
+ - server_name: workbench
+ - root: /var/www/arvados-workbench/current/public
+ - index: index.html index.htm
+ # yamllint disable-line rule:line-length
+ - access_log: {{ nginx_log }}/workbench.arv.local-upstream.access.log combined
+ - error_log: {{ nginx_log }}/workbench.arv.local-upstream.error.log
diff --git a/tools/salt-install/salt_pillars/postgresql.sls b/tools/salt-install/salt_pillars/postgresql.sls
new file mode 100644
index 000000000..18c58e0dc
--- /dev/null
+++ b/tools/salt-install/salt_pillars/postgresql.sls
@@ -0,0 +1,45 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+states:
+ - postgres
+
+### POSTGRESQL
+postgres:
+ use_upstream_repo: false
+ pkgs_extra:
+ - postgresql-contrib
+ postgresconf: |-
+ listen_addresses = '*' # listen on all interfaces
+ acls:
+ - ['local', 'all', 'postgres', 'peer']
+ - ['local', 'all', 'all', 'peer']
+ - ['host', 'all', 'all', '127.0.0.1/32', 'md5']
+ - ['host', 'all', 'all', '::1/128', 'md5']
+ - ['host', 'arvados', 'arvados', '127.0.0.1/32']
+ users:
+ arvados:
+ ensure: present
+ password: changeme_arvados
+
+ # tablespaces:
+ # arvados_tablespace:
+ # directory: /path/to/some/tbspace/arvados_tbsp
+ # owner: arvados
+
+ databases:
+ arvados:
+ owner: arvados
+ template: template0
+ lc_ctype: en_US.utf8
+ lc_collate: en_US.utf8
+ # tablespace: arvados_tablespace
+ schemas:
+ public:
+ owner: arvados
+ extensions:
+ pg_trgm:
+ if_not_exists: true
+ schema: public
diff --git a/tools/salt-install/salt_pillars/top.sls b/tools/salt-install/salt_pillars/top.sls
new file mode 100644
index 000000000..48c26a124
--- /dev/null
+++ b/tools/salt-install/salt_pillars/top.sls
@@ -0,0 +1,17 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+base:
+ '*':
+ - arvados
+ - nginx_api_configuration
+ - nginx_controller_configuration
+ - nginx_keepproxy_configuration
+ - nginx_keepweb_configuration
+ - nginx_passenger
+ - nginx_websocket_configuration
+ - nginx_workbench2_configuration
+ - nginx_workbench_configuration
+ - postgresql
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list