[ARVADOS] updated: 2.2.1-60-g6dcfb2d9d
Git user
git at public.arvados.org
Mon Aug 30 17:26:00 UTC 2021
Summary of changes:
services/arv-git-httpd/server_test.go | 6 +-
tools/salt-install/Vagrantfile | 4 +-
.../multiple_hostnames/pillars/arvados.sls | 21 ++-
.../pillars/nginx_api_configuration.sls | 10 +-
.../pillars/nginx_controller_configuration.sls | 9 +-
.../pillars/nginx_keepproxy_configuration.sls | 9 +-
.../pillars/nginx_keepweb_configuration.sls | 9 +-
.../multiple_hostnames/pillars/nginx_passenger.sls | 56 ++++++-
.../pillars/nginx_webshell_configuration.sls | 42 +++++-
.../pillars/nginx_websocket_configuration.sls | 9 +-
.../pillars/nginx_workbench2_configuration.sls | 19 ++-
.../pillars/nginx_workbench_configuration.sls | 19 ++-
.../multiple_hostnames/pillars/postgresql.sls | 20 ++-
.../multiple_hostnames/states/snakeoil_certs.sls | 46 +++---
.../single_hostname/pillars/nginx_passenger.sls | 56 ++++++-
.../single_hostname/states/snakeoil_certs.sls | 43 +++---
.../local.params.example.multiple_hosts | 7 +-
...l.params.example.single_host_multiple_hostnames | 7 +-
...ocal.params.example.single_host_single_hostname | 7 +-
tools/salt-install/provision.sh | 167 ++++++++++++++++-----
tools/salt-install/tests/run-test.sh | 6 +-
21 files changed, 427 insertions(+), 145 deletions(-)
via 6dcfb2d9d117cacd900ec94bc044d78f56faddfa (commit)
via 5c6950c5011fd29d7ba69862bedd5cab4337ec7d (commit)
via d65a27583e999bb2825c28600571d58629443af8 (commit)
via 8a2323b01f935d443728999b1596188919811312 (commit)
via 7e71975f688e02c8253cc793124c7ffff00379ec (commit)
via c7010f7eb2ceba31f7704766d48dc6306a1834ad (commit)
via ae1f781252e07371f7d650219f6f247e40467182 (commit)
via 95b0cf6cac8e7bca802d25bb3fd3e84a0590635f (commit)
via 4c362a1d531762184d9eef775bf4fcd68f2e4950 (commit)
via 5cd4453b8602e8172665dbd172e9d2b3c1f4901a (commit)
via f8b2605671befef2f4b0e8c529478c623adee15f (commit)
via 326aece8fbc677c02b78e29f679819dae131bdc5 (commit)
via 629f622df7ab0f4ad1816c04bc5cf0ed7eeaa58c (commit)
via 94fe1571806a2203f56f58be7d5b90276d34115e (commit)
via 471acedb1968388ac42f1aa56773f22fdcb301f9 (commit)
via 23a110b5deee8ab930422f87a7af9645127270f6 (commit)
via bd72cd6e25e068c4c1418eb252965040504845b4 (commit)
via 636a9676a1edf3c80c07875c03f6347e060c2ba0 (commit)
via d9bdbbce2f455f2e10b757e76bb9475c9b6e6304 (commit)
from 23a073e813208a6a83d08c7e9cdf4305ae6e09d1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6dcfb2d9d117cacd900ec94bc044d78f56faddfa
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Aug 30 13:22:41 2021 -0400
Fix merge error no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 76d8273af..dec559537 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -164,13 +164,16 @@ WORKBENCH2_EXT_SSL_PORT=3001
# For a stable release, change RELEASE "production" and VERSION to the
# package version (including the iteration, e.g. X.Y.Z-1) of the
# release.
+# The "local.params.example.*" files already set "RELEASE=production"
+# to deploy production-ready packages
RELEASE="production"
VERSION="2.2.2-1"
# These are arvados-formula-related parameters
# An arvados-formula tag. For a stable release, this should be a
# branch name (e.g. X.Y-dev) or tag for the release.
-ARVADOS_TAG="2.2-dev"
+# ARVADOS_TAG="2.2.0"
+BRANCH="2.2-dev"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
@@ -277,17 +280,24 @@ mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-
echo "Cloning formulas"
rm -rf ${F_DIR}/* || exit 1
+git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
+( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
+( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
+
+git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
+( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
-git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
-git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
-git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
-# git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${NGINX_TAG}" https://github.com/netmanagers/nginx-formula.git
-git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
-git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
+git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
# If we want to try a specific branch of the formula
if [ "x${BRANCH}" != "x" ]; then
commit 5c6950c5011fd29d7ba69862bedd5cab4337ec7d
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 12 15:47:55 2021 -0300
Disable SELinux on Centos-7 platform
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ddeced207..76d8273af 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -237,6 +237,9 @@ else
case ${OS_ID} in
"centos")
+ echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
+ sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+ setenforce permissive
yum install -y curl git jq
;;
"debian"|"ubuntu")
commit d65a27583e999bb2825c28600571d58629443af8
Author: Javier Bértoli <jbertoli at curii.com>
Date: Mon Aug 9 19:19:45 2021 -0300
Fix nginx_passenger parameters for Centos-7 and Ubuntu-18.04
refs #17535
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6faafb5b5..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
#
# SPDX-License-Identifier: AGPL-3.0
-{%- if grains.os_family in ('RedHat',) %}
- {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
- {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
- {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
-{%- else %}
- {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
- {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
- {%- set passenger_ruby = '/usr/bin/ruby' %}
-{%- endif %}
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+ if grains.osfinger in ('CentOS Linux-7') else
+ 'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+ if grains.osfinger in ('CentOS Linux-7',) else
+ '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ '/usr/bin/ruby' %}
### NGINX
nginx:
commit 8a2323b01f935d443728999b1596188919811312
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 18:39:20 2021 -0300
Don't force the ruby version in the examples
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 12231fa8c..ccf6bac78 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
- pkg: ruby-2.7.2
## We set these to `true` here for testing purposes.
## They both default to `false`.
commit 7e71975f688e02c8253cc793124c7ffff00379ec
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:37:34 2021 -0300
Adapt the provision scripts to manage Centos 7 too
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index ccf6bac78..12231fa8c 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,6 +28,7 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
+ pkg: ruby-2.7.2
## We set these to `true` here for testing purposes.
## They both default to `false`.
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index a4d3c34f2..6faafb5b5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
#
# SPDX-License-Identifier: AGPL-3.0
-{%- set passenger_pkg = 'nginx-mod-http-passenger'
- if grains.osfinger in ('CentOS Linux-7') else
- 'libnginx-mod-http-passenger' %}
-{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
- if grains.osfinger in ('CentOS Linux-7',) else
- '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
-{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
- '/usr/bin/ruby' %}
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
+{%- else %}
+ {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/bin/ruby' %}
+{%- endif %}
### NGINX
nginx:
commit c7010f7eb2ceba31f7704766d48dc6306a1834ad
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:16:07 2021 -0300
Some more Centos required tweaks to the provision script.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 02b2ae7c6..ddeced207 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -232,23 +232,19 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
else
# Install a few dependency packages
# First, let's figure out the OS we're working on
- OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \")
echo "Detected distro: ${OS_ID}"
case ${OS_ID} in
- centos)
- PREINSTALL_CMD="/bin/true"
- INSTALL_CMD="yum install -y"
+ "centos")
+ yum install -y curl git jq
;;
- debian|ubuntu)
- PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
- INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+ "debian"|"ubuntu")
+ DEBIAN_FRONTEND=noninteractive apt update
+ DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
;;
esac
- ${PREINSTALL_CMD}
- ${INSTALL_CMD} curl git jq
-
if which salt-call; then
echo "Salt already installed"
else
commit ae1f781252e07371f7d650219f6f247e40467182
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jul 23 15:02:45 2021 -0300
17535: Modify provision script to run on Centos
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 76d8273af..02b2ae7c6 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -232,22 +232,23 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
else
# Install a few dependency packages
# First, let's figure out the OS we're working on
- OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \")
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
echo "Detected distro: ${OS_ID}"
case ${OS_ID} in
- "centos")
- echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
- sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
- setenforce permissive
- yum install -y curl git jq
+ centos)
+ PREINSTALL_CMD="/bin/true"
+ INSTALL_CMD="yum install -y"
;;
- "debian"|"ubuntu")
- DEBIAN_FRONTEND=noninteractive apt update
- DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
+ debian|ubuntu)
+ PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
+ INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
;;
esac
+ ${PREINSTALL_CMD}
+ ${INSTALL_CMD} curl git jq
+
if which salt-call; then
echo "Salt already installed"
else
commit 95b0cf6cac8e7bca802d25bb3fd3e84a0590635f
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 12 15:47:55 2021 -0300
Disable SELinux on Centos-7 platform
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ddeced207..76d8273af 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -237,6 +237,9 @@ else
case ${OS_ID} in
"centos")
+ echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
+ sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+ setenforce permissive
yum install -y curl git jq
;;
"debian"|"ubuntu")
commit 4c362a1d531762184d9eef775bf4fcd68f2e4950
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Aug 10 08:20:31 2021 -0300
Make Saltstack fail on first error
no issue #
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b48ce416e..ddeced207 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -256,6 +256,8 @@ else
# Set salt to masterless mode
cat > /etc/salt/minion << EOFSM
+failhard: "True"
+
file_client: local
file_roots:
base:
commit 5cd4453b8602e8172665dbd172e9d2b3c1f4901a
Author: Javier Bértoli <jbertoli at curii.com>
Date: Mon Aug 9 19:19:45 2021 -0300
Fix nginx_passenger parameters for Centos-7 and Ubuntu-18.04
refs #17535
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6faafb5b5..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
#
# SPDX-License-Identifier: AGPL-3.0
-{%- if grains.os_family in ('RedHat',) %}
- {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
- {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
- {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
-{%- else %}
- {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
- {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
- {%- set passenger_ruby = '/usr/bin/ruby' %}
-{%- endif %}
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+ if grains.osfinger in ('CentOS Linux-7') else
+ 'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+ if grains.osfinger in ('CentOS Linux-7',) else
+ '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ '/usr/bin/ruby' %}
### NGINX
nginx:
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
index 6ce75faa7..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
@@ -3,19 +3,69 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+ if grains.osfinger in ('CentOS Linux-7') else
+ 'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+ if grains.osfinger in ('CentOS Linux-7',) else
+ '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ '/usr/bin/ruby' %}
+
### NGINX
nginx:
install_from_phusionpassenger: true
lookup:
- passenger_package: libnginx-mod-http-passenger
- passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+ passenger_package: {{ passenger_pkg }}
+ ### PASSENGER
+ passenger:
+ passenger_ruby: {{ passenger_ruby }}
### SERVER
server:
config:
- include: 'modules-enabled/*.conf'
+ # This is required to get the passenger module loaded
+ # In Debian it can be done with this
+ # include: 'modules-enabled/*.conf'
+ load_module: {{ passenger_mod }}
+
worker_processes: 4
+ ### SNIPPETS
+ snippets:
+ # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+ ssl_hardening_default.conf:
+ - ssl_session_timeout: 1d
+ - ssl_session_cache: 'shared:arvadosSSL:10m'
+ - ssl_session_tickets: 'off'
+
+ # intermediate configuration
+ - ssl_protocols: TLSv1.2 TLSv1.3
+ - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ - ssl_prefer_server_ciphers: 'off'
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+ # OCSP stapling
+ # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+ # - ssl_stapling: 'on'
+ # - ssl_stapling_verify: 'on'
+
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+ # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+ # - ssl_dhparam: /path/to/dhparam
+
+ # replace with the IP address of your resolver
+ # - resolver: 127.0.0.1
+
+ arvados-snakeoil.conf:
+ - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem
+ - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key
+
### SITES
servers:
managed:
commit f8b2605671befef2f4b0e8c529478c623adee15f
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 18:39:20 2021 -0300
Don't force the ruby version in the examples
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 12231fa8c..ccf6bac78 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
- pkg: ruby-2.7.2
## We set these to `true` here for testing purposes.
## They both default to `false`.
commit 326aece8fbc677c02b78e29f679819dae131bdc5
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:37:34 2021 -0300
Adapt the provision scripts to manage Centos 7 too
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 3019a9fb1..a3463bfc5 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -35,7 +35,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
cp -vr /vagrant/tests /home/vagrant/tests;
sed 's#cluster_fixme_or_this_wont_work#harpo#g;
s#domain_fixme_or_this_wont_work#local#g;
- s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+ s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \
/vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
arv.vm.provision "shell",
@@ -78,7 +78,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
cp -vr /vagrant/tests /home/vagrant/tests;
sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
s#cluster_fixme_or_this_wont_work#zeppo#g;
- s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+ s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
s#domain_fixme_or_this_wont_work#local#g;' \
/vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
arv.vm.provision "shell",
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 23e007650..12231fa8c 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -26,6 +28,8 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
+ pkg: ruby-2.7.2
+
## We set these to `true` here for testing purposes.
## They both default to `false`.
manage_ruby: true
@@ -67,8 +71,15 @@ arvados:
host: 127.0.0.1
password: "__DATABASE_PASSWORD__"
user: __CLUSTER___arvados
- encoding: en_US.utf8
- client_encoding: UTF8
+ extra_conn_params:
+ client_encoding: UTF8
+ # Centos7 does not enable SSL by default, so we disable
+ # it here just for testing of the formula purposes only.
+ # You should not do this in production, and should
+ # configure Postgres certificates correctly
+ {%- if grains.os_family in ('RedHat',) %}
+ sslmode: disable
+ {%- endif %}
tls:
# certificate: ''
@@ -76,6 +87,13 @@ arvados:
# required to test with arvados-snakeoil certs
insecure: true
+ resources:
+ virtual_machines:
+ shell:
+ name: webshell
+ backend: 127.0.1.1
+ port: 4200
+
### TOKENS
tokens:
system_root: __SYSTEM_ROOT_TOKEN__
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
index b2f12c773..54087f6d6 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
@@ -3,17 +3,23 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
- arvados_api:
+ arvados_api.conf:
enabled: true
overwrite: true
config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
index 3adf0580a..195e9af82 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
@@ -20,7 +20,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_controller_default:
+ arvados_controller_default.conf:
enabled: true
overwrite: true
config:
@@ -33,9 +33,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_controller_ssl:
+ arvados_controller_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: __CLUSTER__.__DOMAIN__
@@ -52,7 +54,8 @@ nginx:
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- proxy_set_header: 'X-External-Client $external_client'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
- client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
index 2d8922df9..91179d4a8 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_keepproxy_default:
+ arvados_keepproxy_default.conf:
enabled: true
overwrite: true
config:
@@ -29,9 +29,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_keepproxy_ssl:
+ arvados_keepproxy_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: keep.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
index d180a3bad..9ea16bfb5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_collections_download_default:
+ arvados_collections_download_default.conf:
enabled: true
overwrite: true
config:
@@ -30,9 +30,11 @@ nginx:
- return: '301 https://$host$request_uri'
### COLLECTIONS / DOWNLOAD
- arvados_collections_download_ssl:
+ arvados_collections_download_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
- client_max_body_size: 0
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6ce75faa7..6faafb5b5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,19 +3,69 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
+{%- else %}
+ {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/bin/ruby' %}
+{%- endif %}
+
### NGINX
nginx:
install_from_phusionpassenger: true
lookup:
- passenger_package: libnginx-mod-http-passenger
- passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+ passenger_package: {{ passenger_pkg }}
+ ### PASSENGER
+ passenger:
+ passenger_ruby: {{ passenger_ruby }}
### SERVER
server:
config:
- include: 'modules-enabled/*.conf'
+ # This is required to get the passenger module loaded
+ # In Debian it can be done with this
+ # include: 'modules-enabled/*.conf'
+ load_module: {{ passenger_mod }}
+
worker_processes: 4
+ ### SNIPPETS
+ snippets:
+ # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+ ssl_hardening_default.conf:
+ - ssl_session_timeout: 1d
+ - ssl_session_cache: 'shared:arvadosSSL:10m'
+ - ssl_session_tickets: 'off'
+
+ # intermediate configuration
+ - ssl_protocols: TLSv1.2 TLSv1.3
+ - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ - ssl_prefer_server_ciphers: 'off'
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+ # OCSP stapling
+ # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+ # - ssl_stapling: 'on'
+ # - ssl_stapling_verify: 'on'
+
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+ # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+ # - ssl_dhparam: /path/to/dhparam
+
+ # replace with the IP address of your resolver
+ # - resolver: 127.0.0.1
+
+ arvados-snakeoil.conf:
+ - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem
+ - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key
+
### SITES
servers:
managed:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
index e75f04434..9b73ab4a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
@@ -3,6 +3,20 @@
#
# SPDX-License-Identifier: AGPL-3.0
+# This parameter will be used here to generate a list of upstreams and vhosts.
+# This dict is here for convenience and should be managed some other way, but the
+# different ways of orchestration that can be used for this are outside the scope
+# of this formula and their examples.
+# These upstreams should match those defined in `arvados:cluster:resources:virtual_machines`
+{% set webshell_virtual_machines = {
+ 'shell': {
+ 'name': 'webshell',
+ 'backend': '127.0.1.1',
+ 'port': 4200,
+ }
+}
+%}
+
### NGINX
nginx:
### SERVER
@@ -11,13 +25,20 @@ nginx:
### STREAMS
http:
- upstream webshell_upstream:
- - server: 'shell.internal:4200 fail_timeout=10s'
+ {%- for vm, params in webshell_virtual_machines.items() %}
+ {%- set vm_name = params.name | default(vm) %}
+ {%- set vm_backend = params.backend | default(vm_name) %}
+ {%- set vm_port = params.port | default(4200) %}
+
+ upstream {{ vm_name }}_upstream:
+ - server: '{{ vm_backend }}:{{ vm_port }} fail_timeout=10s'
+
+ {%- endfor %}
### SITES
servers:
managed:
- arvados_webshell_default:
+ arvados_webshell_default.conf:
enabled: true
overwrite: true
config:
@@ -30,17 +51,21 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_webshell_ssl:
+ arvados_webshell_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: webshell.__CLUSTER__.__DOMAIN__
- listen:
- __CONTROLLER_EXT_SSL_PORT__ http2 ssl
- index: index.html index.htm
- - location /shell.__CLUSTER__.__DOMAIN__:
- - proxy_pass: 'http://webshell_upstream'
+ {%- for vm, params in webshell_virtual_machines.items() %}
+ {%- set vm_name = params.name | default(vm) %}
+ - location /{{ vm_name }}:
+ - proxy_pass: 'http://{{ vm_name }}_upstream'
- proxy_read_timeout: 90
- proxy_connect_timeout: 90
- proxy_set_header: 'Host $http_host'
@@ -67,8 +92,9 @@ nginx:
- add_header: "'Access-Control-Allow-Origin' '*'"
- add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
- add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
-
- - include: 'snippets/arvados-snakeoil.conf'
+ {%- endfor %}
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
index 3a354ac29..bcd0457c9 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_websocket_default:
+ arvados_websocket_default.conf:
enabled: true
overwrite: true
config:
@@ -29,9 +29,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_websocket_ssl:
+ arvados_websocket_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: ws.__CLUSTER__.__DOMAIN__
@@ -53,6 +55,7 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
index 8fdd55399..44bd16fe3 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
@@ -1,12 +1,18 @@
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
@@ -14,7 +20,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_workbench2_default:
+ arvados_workbench2_default.conf:
enabled: true
overwrite: true
config:
@@ -27,9 +33,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_workbench2_ssl:
+ arvados_workbench2_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: workbench2.__CLUSTER__.__DOMAIN__
@@ -43,6 +51,7 @@ nginx:
- return: 503
- location /config.json:
- return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
index 649af10b6..6b7ab969f 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
@@ -3,10 +3,16 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
@@ -23,7 +29,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_workbench_default:
+ arvados_workbench_default.conf:
enabled: true
overwrite: true
config:
@@ -36,9 +42,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_workbench_ssl:
+ arvados_workbench_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: workbench.__CLUSTER__.__DOMAIN__
@@ -54,11 +62,12 @@ nginx:
- proxy_set_header: 'Host $http_host'
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
- arvados_workbench_upstream:
+ arvados_workbench_upstream.conf:
enabled: true
overwrite: true
config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
index 71e712cad..fda1545a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
@@ -5,11 +5,29 @@
### POSTGRESQL
postgres:
- use_upstream_repo: false
+ # Centos-7's postgres package is too old, so we need to force using upstream's
+ # This is not required in Debian's family as they already ship with PG +11
+ {%- if salt['grains.get']('os_family') == 'RedHat' %}
+ use_upstream_repo: true
+ version: '12'
+
+ pkgs_deps:
+ - libicu
+ - libxslt
+ - systemd-sysv
+
+ pkgs_extra:
+ - postgresql12-contrib
+
+ {%- else %}
pkgs_extra:
- postgresql-contrib
+ {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
+ #ssl = on
+ #ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+ #ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
acls:
- ['local', 'all', 'postgres', 'peer']
- ['local', 'all', 'all', 'peer']
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
index fb1473def..91617e4fa 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
{%- set curr_tpldir = tpldir %}
{%- set tpldir = 'arvados' %}
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+ - nginx.passenger
+ - nginx.config
+ - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
{%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
- ca-certificates
arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
- # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+ # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
# These dirs are not to CentOS-ish, but this is a helper script
@@ -121,6 +128,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
- require:
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+ # We need this before we can add the nginx's snippet
+ - require_in:
+ - file: nginx_snippet_arvados-snakeoil.conf
{%- if grains.get('os_family') == 'Debian' %}
arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -130,29 +140,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
- sls: postgres
arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
- cmd.run:
- - name: |
- chown root:ssl-cert {{ arvados_key_file }}
+ file.managed:
+ - name: {{ arvados_key_file }}
+ - owner: root
+ - group: ssl-cert
- require:
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
- file.managed:
- - name: /etc/nginx/snippets/arvados-snakeoil.conf
- - contents: |
- ssl_certificate {{ arvados_cert_file }};
- ssl_certificate_key {{ arvados_key_file }};
- - watch_in:
- - service: nginx_service
- - require:
- - pkg: passenger_install
- - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run
- require_in:
- - file: nginx_config
- - service: nginx_service
- - watch_in:
- - service: nginx_service
-
-
+ - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
index 130fb5e93..b6929fb88 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
{%- set curr_tpldir = tpldir %}
{%- set tpldir = 'arvados' %}
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+ - nginx.passenger
+ - nginx.config
+ - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
{%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
- ca-certificates
arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
- # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+ # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
# These dirs are not to CentOS-ish, but this is a helper script
@@ -124,6 +131,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
- require:
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+ # We need this before we can add the nginx's snippet
+ - require_in:
+ - file: nginx_snippet_arvados-snakeoil.conf
{%- if grains.get('os_family') == 'Debian' %}
arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -133,26 +143,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
- sls: postgres
arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
- cmd.run:
- - name: |
- chown root:ssl-cert {{ arvados_key_file }}
+ file.managed:
+ - name: {{ arvados_key_file }}
+ - owner: root
+ - group: ssl-cert
- require:
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
- file.managed:
- - name: /etc/nginx/snippets/arvados-snakeoil.conf
- - contents: |
- ssl_certificate {{ arvados_cert_file }};
- ssl_certificate_key {{ arvados_key_file }};
- - require:
- - pkg: nginx_install
- require_in:
- - file: nginx_config
- - service: nginx_service
- - watch_in:
- - service: nginx_service
-
-
+ - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index 17b7b8888..283c631ec 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -100,6 +100,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index ae54e7437..e23634e8c 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -72,6 +72,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index a35bd45bf..ae9804863 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -81,6 +81,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
commit 629f622df7ab0f4ad1816c04bc5cf0ed7eeaa58c
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:29:55 2021 -0300
The user's `api_client_authorization` token can't be obtained using filters
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index 53c51a2c5..020efa94e 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -55,13 +55,17 @@ echo "Activating user '__INITIAL_USER__'"
arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
echo "Getting the user API TOKEN"
-user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+user_api_token=$(arv api_client_authorization list | jq -r ".items[] | select( .owner_uuid == \"${user_uuid}\" ).api_token" | head -1)
if [ "x${user_api_token}" = "x" ]; then
+ echo "No existing token found for user '__INITIAL_USER__' (user_uuid: '${user_uuid}'). Creating token"
user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
fi
+echo "API TOKEN FOR user '__INITIAL_USER__': '${user_api_token}'."
+
# Change to the user's token and run the workflow
+echo "Switching to user '__INITIAL_USER__'"
export ARVADOS_API_TOKEN="${user_api_token}"
echo "Running test CWL workflow"
commit 94fe1571806a2203f56f58be7d5b90276d34115e
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:16:07 2021 -0300
Some more Centos required tweaks to the provision script.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index e9ed4c891..b48ce416e 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -175,7 +175,7 @@ ARVADOS_TAG="2.2-dev"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
NGINX_TAG="temp-fix-missing-statements-in-pillar"
-DOCKER_TAG="v1.0.0"
+DOCKER_TAG="v2.0.7"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
@@ -232,23 +232,19 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
else
# Install a few dependency packages
# First, let's figure out the OS we're working on
- OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \")
echo "Detected distro: ${OS_ID}"
case ${OS_ID} in
- centos)
- PREINSTALL_CMD="/bin/true"
- INSTALL_CMD="yum install -y"
+ "centos")
+ yum install -y curl git jq
;;
- debian|ubuntu)
- PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
- INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+ "debian"|"ubuntu")
+ DEBIAN_FRONTEND=noninteractive apt update
+ DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
;;
esac
- ${PREINSTALL_CMD}
- ${INSTALL_CMD} curl git jq
-
if which salt-call; then
echo "Salt already installed"
else
@@ -614,5 +610,10 @@ fi
# Test that the installation finished correctly
if [ "x${TEST}" = "xyes" ]; then
cd ${T_DIR}
- ./run-test.sh
+ # If we use RVM, we need to run this with it, or most ruby commands will fail
+ RVM_EXEC=""
+ if [ -x /usr/local/rvm/bin/rvm-exec ]; then
+ RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
+ fi
+ ${RVM_EXEC} ./run-test.sh
fi
commit 471acedb1968388ac42f1aa56773f22fdcb301f9
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu May 20 16:11:52 2021 -0300
feat(provision): moved the arvados-formula to Arvados.org
refs #17655
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index af8b41a47..e9ed4c891 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -276,24 +276,17 @@ mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
# Get the formula and dependencies
cd ${F_DIR} || exit 1
+
echo "Cloning formulas"
rm -rf ${F_DIR}/* || exit 1
-git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
-( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
-( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
-
-git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
-( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
-( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
-( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
-git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
+git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
+git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
+git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
+# git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
+git clone --branch "${NGINX_TAG}" https://github.com/netmanagers/nginx-formula.git
+git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
+git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
# If we want to try a specific branch of the formula
if [ "x${BRANCH}" != "x" ]; then
commit 23a110b5deee8ab930422f87a7af9645127270f6
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jul 23 15:02:45 2021 -0300
17535: Modify provision script to run on Centos
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 6ccf2b910..af8b41a47 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -11,6 +11,7 @@
# vagrant up
set -o pipefail
+set -x
# capture the directory that the script is running from
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -229,8 +230,24 @@ fi
if [ "${DUMP_CONFIG}" = "yes" ]; then
echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
else
- apt-get update
- apt-get install -y curl git jq
+ # Install a few dependency packages
+ # First, let's figure out the OS we're working on
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+ echo "Detected distro: ${OS_ID}"
+
+ case ${OS_ID} in
+ centos)
+ PREINSTALL_CMD="/bin/true"
+ INSTALL_CMD="yum install -y"
+ ;;
+ debian|ubuntu)
+ PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
+ INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+ ;;
+ esac
+
+ ${PREINSTALL_CMD}
+ ${INSTALL_CMD} curl git jq
if which salt-call; then
echo "Salt already installed"
commit bd72cd6e25e068c4c1418eb252965040504845b4
Author: Ward Vandewege <ward at curii.com>
Date: Thu Jul 1 14:00:01 2021 -0400
The bleeding edge branch is now called 'main'.
refs #17817
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index cba82fe3f..386205d37 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -39,7 +39,7 @@ func (s *GitSuite) TestPathVariants(c *check.C) {
func (s *GitSuite) TestReadonly(c *check.C) {
err := s.RunGit(c, spectatorToken, "fetch", "active/foo.git")
c.Assert(err, check.Equals, nil)
- err = s.RunGit(c, spectatorToken, "push", "active/foo.git", "master:newbranchfail")
+ err = s.RunGit(c, spectatorToken, "push", "active/foo.git", "main:newbranchfail")
c.Assert(err, check.ErrorMatches, `.*HTTP (code = )?403.*`)
_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranchfail")
c.Assert(err, check.FitsTypeOf, &os.PathError{})
@@ -48,7 +48,7 @@ func (s *GitSuite) TestReadonly(c *check.C) {
func (s *GitSuite) TestReadwrite(c *check.C) {
err := s.RunGit(c, activeToken, "fetch", "active/foo.git")
c.Assert(err, check.Equals, nil)
- err = s.RunGit(c, activeToken, "push", "active/foo.git", "master:newbranch")
+ err = s.RunGit(c, activeToken, "push", "active/foo.git", "main:newbranch")
c.Assert(err, check.Equals, nil)
_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranch")
c.Assert(err, check.Equals, nil)
@@ -104,7 +104,7 @@ func (s *GitSuite) makeArvadosRepo(c *check.C) {
msg, err := exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git").CombinedOutput()
c.Log(string(msg))
c.Assert(err, check.Equals, nil)
- msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:master").CombinedOutput()
+ msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:main").CombinedOutput()
c.Log(string(msg))
c.Assert(err, check.Equals, nil)
}
commit 636a9676a1edf3c80c07875c03f6347e060c2ba0
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jun 17 16:50:16 2021 -0300
17821: Add a parameter to provision.sh to dump the configuration
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index f5e40ff15..17b7b8888 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -82,6 +82,7 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
@@ -90,13 +91,13 @@ RELEASE="production"
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index 6dd47722c..ae54e7437 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -54,6 +54,7 @@ USE_LETSENCRYPT="no"
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
@@ -62,13 +63,13 @@ RELEASE="production"
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index fda42a9c7..a35bd45bf 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -63,6 +63,7 @@ USE_LETSENCRYPT="no"
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+# These are ARVADOS-related settings.
# Which release of Arvados repo you want to use
RELEASE="production"
# Which version of Arvados you want to install. Defaults to latest stable
@@ -71,13 +72,13 @@ RELEASE="production"
# This is an arvados-formula setting.
# If branch is set, the script will switch to it before running salt
# Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
##########################################################
# Usually there's no need to modify things below this line
# Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
# DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b92c98f11..6ccf2b910 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -21,7 +21,6 @@ usage() {
echo >&2
echo >&2 "${0} options:"
echo >&2 " -d, --debug Run salt installation in debug mode"
- echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
echo >&2 " -c <local.params>, --config <local.params> Path to the local.params config file"
echo >&2 " -t, --test Test installation running a CWL workflow"
echo >&2 " -r, --roles List of Arvados roles to apply to the host, comma separated"
@@ -39,17 +38,35 @@ usage() {
echo >&2 " workbench2"
echo >&2 " Defaults to applying them all"
echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " --dump-config <dest_dir> Dumps the pillars and states to a directory"
+ echo >&2 " This parameter does not perform any installation at all. It's"
+ echo >&2 " intended to give you a parsed sot of configuration files so"
+ echo >&2 " you can inspect them or use them in you Saltstack infrastructure."
+ echo >&2 " It"
+ echo >&2 " - parses the pillar and states templates,"
+ echo >&2 " - downloads the helper formulas with their desired versions,"
+ echo >&2 " - prepares the 'top.sls' files both for pillars and states"
+ echo >&2 " for the selected role/s"
+ echo >&2 " - writes the resulting files into <dest_dir>"
echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
echo >&2
}
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+ if ! which getopt > /dev/null; then
+ echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
+ exit 1
+ fi
+
TEMP=$(getopt -o c:dhp:r:tv \
- --long config:,debug,help,ssl-port:,roles:,test,vagrant \
+ --long config:,debug,dump-config:,help,roles:,test,vagrant \
-n "${0}" -- "${@}")
- if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ if [ ${?} != 0 ];
+ then echo "Please check the parameters you entered and re-run again"
+ exit 1
+ fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
@@ -62,9 +79,23 @@ arguments() {
-d | --debug)
LOG_LEVEL="debug"
shift
+ set -x
;;
- -p | --ssl-port)
- CONTROLLER_EXT_SSL_PORT=${2}
+ --dump-config)
+ if [[ ${2} = /* ]]; then
+ DUMP_SALT_CONFIG_DIR=${2}
+ else
+ DUMP_SALT_CONFIG_DIR=${PWD}/${2}
+ fi
+ ## states
+ S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
+ ## formulas
+ F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
+ ## pillars
+ P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
+ ## tests
+ T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
+ DUMP_CONFIG="yes"
shift 2
;;
-r | --roles)
@@ -102,6 +133,7 @@ arguments() {
CONFIG_FILE="${SCRIPT_DIR}/local.params"
CONFIG_DIR="local_config_dir"
+DUMP_CONFIG="no"
LOG_LEVEL="info"
CONTROLLER_EXT_SSL_PORT=443
TESTS_DIR="tests"
@@ -127,44 +159,51 @@ WEBSOCKET_EXT_SSL_PORT=8002
WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
+## These are ARVADOS-related parameters
# For a stable release, change RELEASE "production" and VERSION to the
# package version (including the iteration, e.g. X.Y.Z-1) of the
# release.
RELEASE="production"
VERSION="2.2.2-1"
-# The arvados-formula version. For a stable release, this should be a
+# These are arvados-formula-related parameters
+# An arvados-formula tag. For a stable release, this should be a
# branch name (e.g. X.Y-dev) or tag for the release.
ARVADOS_TAG="2.2-dev"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
-NGINX_TAG="v2.7.4"
+NGINX_TAG="temp-fix-missing-statements-in-pillar"
DOCKER_TAG="v1.0.0"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
# Salt's dir
+DUMP_SALT_CONFIG_DIR=""
## states
S_DIR="/srv/salt"
## formulas
F_DIR="/srv/formulas"
-##pillars
+## pillars
P_DIR="/srv/pillars"
+## tests
+T_DIR="/tmp/cluster_tests"
arguments ${@}
if [ -s ${CONFIG_FILE} ]; then
source ${CONFIG_FILE}
else
- echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in"
+ echo >&2 "You don't seem to have a config file with initial values."
+ echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
echo >&2 " * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
echo >&2 " * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
exit 1
fi
if [ ! -d ${CONFIG_DIR} ]; then
- echo >&2 "Please create a '${CONFIG_DIR}' with initial values, as described in"
+ echo >&2 "You don't seem to have a config directory with pillars and states."
+ echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
echo >&2 " * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
echo >&2 " * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
exit 1
@@ -176,7 +215,7 @@ if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
exit 1
fi
-if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
+if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
exit 1
@@ -187,20 +226,23 @@ if [ "x${HOSTNAME_EXT}" = "x" ] ; then
HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
fi
-apt-get update
-apt-get install -y curl git jq
-
-if which salt-call; then
- echo "Salt already installed"
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+ echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
else
- curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
- sh /tmp/bootstrap_salt.sh -XdfP -x python3
- /bin/systemctl stop salt-minion.service
- /bin/systemctl disable salt-minion.service
-fi
+ apt-get update
+ apt-get install -y curl git jq
+
+ if which salt-call; then
+ echo "Salt already installed"
+ else
+ curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+ sh /tmp/bootstrap_salt.sh -XdfP -x python3
+ /bin/systemctl stop salt-minion.service
+ /bin/systemctl disable salt-minion.service
+ fi
-# Set salt to masterless mode
-cat > /etc/salt/minion << EOFSM
+ # Set salt to masterless mode
+ cat > /etc/salt/minion << EOFSM
file_client: local
file_roots:
base:
@@ -211,23 +253,36 @@ pillar_roots:
base:
- ${P_DIR}
EOFSM
+fi
-mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
+mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
-git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
-git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
-git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
-git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
+echo "Cloning formulas"
+rm -rf ${F_DIR}/* || exit 1
+git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
+( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
+( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
+
+git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
+( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
+
+git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
# If we want to try a specific branch of the formula
if [ "x${BRANCH}" != "x" ]; then
- cd ${F_DIR}/arvados-formula || exit 1
- git checkout -t origin/"${BRANCH}" -b "${BRANCH}"
- cd -
+ ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
+elif [ "x${ARVADOS_TAG}" != "x" ]; then
+( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
fi
if [ "x${VAGRANT}" = "xyes" ]; then
@@ -242,6 +297,8 @@ fi
SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
+echo "Writing pillars and states"
+
# Replace variables (cluster, domain, etc) in the pillars, states and tests
# to ease deployment for newcomers
if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
@@ -293,7 +350,7 @@ if [ "x${TEST}" = "xyes" ] && [ ! -d "${SOURCE_TESTS_DIR}" ]; then
echo "You requested to run tests, but ${SOURCE_TESTS_DIR} does not exist or is not a directory. Exiting."
exit 1
fi
-mkdir -p /tmp/cluster_tests
+mkdir -p ${T_DIR}
# Replace cluster and domain name in the test files
for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
sed "s#__CLUSTER__#${CLUSTER}#g;
@@ -305,9 +362,9 @@ for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
s#__INITIAL_USER__#${INITIAL_USER}#g;
s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
- "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
+ "${f}" > ${T_DIR}/$(basename "${f}")
done
-chmod 755 /tmp/cluster_tests/run-test.sh
+chmod 755 ${T_DIR}/run-test.sh
# Replace helper state files that differ from the formula's examples
if [ -d "${SOURCE_STATES_DIR}" ]; then
@@ -499,6 +556,11 @@ else
done
fi
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+ # We won't run the rest of the script because we're just dumping the config
+ exit 0
+fi
+
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ -e /root/.psqlrc ]; then
if ! ( grep 'pset pager off' /root/.psqlrc ); then
@@ -541,6 +603,6 @@ fi
# Test that the installation finished correctly
if [ "x${TEST}" = "xyes" ]; then
- cd /tmp/cluster_tests
+ cd ${T_DIR}
./run-test.sh
fi
commit d9bdbbce2f455f2e10b757e76bb9475c9b6e6304
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jun 18 12:34:14 2021 -0300
17823: change references back to saltstack-formulas' nginx formula
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 2ab856d53..b92c98f11 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -139,7 +139,7 @@ ARVADOS_TAG="2.2-dev"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
-NGINX_TAG="temp-fix-missing-statements-in-pillar"
+NGINX_TAG="v2.7.4"
DOCKER_TAG="v1.0.0"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
@@ -219,8 +219,7 @@ cd ${F_DIR} || exit 1
git clone --branch "${ARVADOS_TAG}" https://git.arvados.org/arvados-formula.git
git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
-# git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${NGINX_TAG}" https://github.com/netmanagers/nginx-formula.git
+git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list