[ARVADOS] updated: 2.2.1-60-g6dcfb2d9d

Git user git at public.arvados.org
Mon Aug 30 17:26:00 UTC 2021


Summary of changes:
 services/arv-git-httpd/server_test.go              |   6 +-
 tools/salt-install/Vagrantfile                     |   4 +-
 .../multiple_hostnames/pillars/arvados.sls         |  21 ++-
 .../pillars/nginx_api_configuration.sls            |  10 +-
 .../pillars/nginx_controller_configuration.sls     |   9 +-
 .../pillars/nginx_keepproxy_configuration.sls      |   9 +-
 .../pillars/nginx_keepweb_configuration.sls        |   9 +-
 .../multiple_hostnames/pillars/nginx_passenger.sls |  56 ++++++-
 .../pillars/nginx_webshell_configuration.sls       |  42 +++++-
 .../pillars/nginx_websocket_configuration.sls      |   9 +-
 .../pillars/nginx_workbench2_configuration.sls     |  19 ++-
 .../pillars/nginx_workbench_configuration.sls      |  19 ++-
 .../multiple_hostnames/pillars/postgresql.sls      |  20 ++-
 .../multiple_hostnames/states/snakeoil_certs.sls   |  46 +++---
 .../single_hostname/pillars/nginx_passenger.sls    |  56 ++++++-
 .../single_hostname/states/snakeoil_certs.sls      |  43 +++---
 .../local.params.example.multiple_hosts            |   7 +-
 ...l.params.example.single_host_multiple_hostnames |   7 +-
 ...ocal.params.example.single_host_single_hostname |   7 +-
 tools/salt-install/provision.sh                    | 167 ++++++++++++++++-----
 tools/salt-install/tests/run-test.sh               |   6 +-
 21 files changed, 427 insertions(+), 145 deletions(-)

       via  6dcfb2d9d117cacd900ec94bc044d78f56faddfa (commit)
       via  5c6950c5011fd29d7ba69862bedd5cab4337ec7d (commit)
       via  d65a27583e999bb2825c28600571d58629443af8 (commit)
       via  8a2323b01f935d443728999b1596188919811312 (commit)
       via  7e71975f688e02c8253cc793124c7ffff00379ec (commit)
       via  c7010f7eb2ceba31f7704766d48dc6306a1834ad (commit)
       via  ae1f781252e07371f7d650219f6f247e40467182 (commit)
       via  95b0cf6cac8e7bca802d25bb3fd3e84a0590635f (commit)
       via  4c362a1d531762184d9eef775bf4fcd68f2e4950 (commit)
       via  5cd4453b8602e8172665dbd172e9d2b3c1f4901a (commit)
       via  f8b2605671befef2f4b0e8c529478c623adee15f (commit)
       via  326aece8fbc677c02b78e29f679819dae131bdc5 (commit)
       via  629f622df7ab0f4ad1816c04bc5cf0ed7eeaa58c (commit)
       via  94fe1571806a2203f56f58be7d5b90276d34115e (commit)
       via  471acedb1968388ac42f1aa56773f22fdcb301f9 (commit)
       via  23a110b5deee8ab930422f87a7af9645127270f6 (commit)
       via  bd72cd6e25e068c4c1418eb252965040504845b4 (commit)
       via  636a9676a1edf3c80c07875c03f6347e060c2ba0 (commit)
       via  d9bdbbce2f455f2e10b757e76bb9475c9b6e6304 (commit)
      from  23a073e813208a6a83d08c7e9cdf4305ae6e09d1 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 6dcfb2d9d117cacd900ec94bc044d78f56faddfa
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Mon Aug 30 13:22:41 2021 -0400

    Fix merge error no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 76d8273af..dec559537 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -164,13 +164,16 @@ WORKBENCH2_EXT_SSL_PORT=3001
 # For a stable release, change RELEASE "production" and VERSION to the
 # package version (including the iteration, e.g. X.Y.Z-1) of the
 # release.
+# The "local.params.example.*" files already set "RELEASE=production"
+# to deploy  production-ready packages
 RELEASE="production"
 VERSION="2.2.2-1"
 
 # These are arvados-formula-related parameters
 # An arvados-formula tag. For a stable release, this should be a
 # branch name (e.g. X.Y-dev) or tag for the release.
-ARVADOS_TAG="2.2-dev"
+# ARVADOS_TAG="2.2.0"
+BRANCH="2.2-dev"
 
 # Other formula versions we depend on
 POSTGRES_TAG="v0.41.6"
@@ -277,17 +280,24 @@ mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
 
 # Get the formula and dependencies
 cd ${F_DIR} || exit 1
-
 echo "Cloning formulas"
 rm -rf ${F_DIR}/* || exit 1
+git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
+( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
+( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
+
+git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
+( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
 
-git clone --branch "${ARVADOS_TAG}"     https://git.arvados.org/arvados-formula.git
-git clone --branch "${DOCKER_TAG}"      https://github.com/saltstack-formulas/docker-formula.git
-git clone --branch "${LOCALE_TAG}"      https://github.com/saltstack-formulas/locale-formula.git
-# git clone --branch "${NGINX_TAG}"       https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${NGINX_TAG}"       https://github.com/netmanagers/nginx-formula.git
-git clone --branch "${POSTGRES_TAG}"    https://github.com/saltstack-formulas/postgres-formula.git
-git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
+git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 
 # If we want to try a specific branch of the formula
 if [ "x${BRANCH}" != "x" ]; then

commit 5c6950c5011fd29d7ba69862bedd5cab4337ec7d
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 12 15:47:55 2021 -0300

    Disable SELinux on Centos-7 platform
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ddeced207..76d8273af 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -237,6 +237,9 @@ else
 
   case ${OS_ID} in
     "centos")
+      echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
+      sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+      setenforce permissive
       yum install -y  curl git jq
       ;;
     "debian"|"ubuntu")

commit d65a27583e999bb2825c28600571d58629443af8
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Mon Aug 9 19:19:45 2021 -0300

    Fix nginx_passenger parameters for Centos-7 and Ubuntu-18.04
    
    refs #17535
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6faafb5b5..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-{%- if grains.os_family in ('RedHat',) %}
-  {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
-  {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
-  {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
-{%- else %}
-  {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
-  {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
-  {%- set passenger_ruby = '/usr/bin/ruby' %}
-{%- endif %}
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+                          if grains.osfinger in ('CentOS Linux-7') else
+                        'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+                          if grains.osfinger in ('CentOS Linux-7',) else
+                        '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                         '/usr/bin/ruby' %}
 
 ### NGINX
 nginx:

commit 8a2323b01f935d443728999b1596188919811312
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 18:39:20 2021 -0300

    Don't force the ruby version in the examples
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 12231fa8c..ccf6bac78 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
   ## manage OS packages with some other tool and you don't want us messing up
   ## with your setup.
   ruby:
-    pkg: ruby-2.7.2
 
     ## We set these to `true` here for testing purposes.
     ## They both default to `false`.

commit 7e71975f688e02c8253cc793124c7ffff00379ec
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 12:37:34 2021 -0300

    Adapt the provision scripts to manage Centos 7 too
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index ccf6bac78..12231fa8c 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,6 +28,7 @@ arvados:
   ## manage OS packages with some other tool and you don't want us messing up
   ## with your setup.
   ruby:
+    pkg: ruby-2.7.2
 
     ## We set these to `true` here for testing purposes.
     ## They both default to `false`.
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index a4d3c34f2..6faafb5b5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-{%- set passenger_pkg = 'nginx-mod-http-passenger'
-                          if grains.osfinger in ('CentOS Linux-7') else
-                        'libnginx-mod-http-passenger' %}
-{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
-                          if grains.osfinger in ('CentOS Linux-7',) else
-                        '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
-{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
-                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
-                         '/usr/bin/ruby' %}
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
+  {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
+  {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
+{%- else %}
+  {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
+  {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+  {%- set passenger_ruby = '/usr/bin/ruby' %}
+{%- endif %}
 
 ### NGINX
 nginx:

commit c7010f7eb2ceba31f7704766d48dc6306a1834ad
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 12:16:07 2021 -0300

    Some more Centos required tweaks to the provision script.
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 02b2ae7c6..ddeced207 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -232,23 +232,19 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
 else
   # Install a few dependency packages
   # First, let's figure out the OS we're working on
-  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
   echo "Detected distro: ${OS_ID}"
 
   case ${OS_ID} in
-    centos)
-      PREINSTALL_CMD="/bin/true"
-      INSTALL_CMD="yum install -y"
+    "centos")
+      yum install -y  curl git jq
       ;;
-    debian|ubuntu)
-      PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
-      INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+    "debian"|"ubuntu")
+      DEBIAN_FRONTEND=noninteractive apt update
+      DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
       ;;
   esac
 
-  ${PREINSTALL_CMD}
-  ${INSTALL_CMD} curl git jq
-
   if which salt-call; then
     echo "Salt already installed"
   else

commit ae1f781252e07371f7d650219f6f247e40467182
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Fri Jul 23 15:02:45 2021 -0300

    17535: Modify provision script to run on Centos
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 76d8273af..02b2ae7c6 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -232,22 +232,23 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
 else
   # Install a few dependency packages
   # First, let's figure out the OS we're working on
-  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
+  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
   echo "Detected distro: ${OS_ID}"
 
   case ${OS_ID} in
-    "centos")
-      echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
-      sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
-      setenforce permissive
-      yum install -y  curl git jq
+    centos)
+      PREINSTALL_CMD="/bin/true"
+      INSTALL_CMD="yum install -y"
       ;;
-    "debian"|"ubuntu")
-      DEBIAN_FRONTEND=noninteractive apt update
-      DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
+    debian|ubuntu)
+      PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
+      INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
       ;;
   esac
 
+  ${PREINSTALL_CMD}
+  ${INSTALL_CMD} curl git jq
+
   if which salt-call; then
     echo "Salt already installed"
   else

commit 95b0cf6cac8e7bca802d25bb3fd3e84a0590635f
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 12 15:47:55 2021 -0300

    Disable SELinux on Centos-7 platform
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ddeced207..76d8273af 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -237,6 +237,9 @@ else
 
   case ${OS_ID} in
     "centos")
+      echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
+      sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+      setenforce permissive
       yum install -y  curl git jq
       ;;
     "debian"|"ubuntu")

commit 4c362a1d531762184d9eef775bf4fcd68f2e4950
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Tue Aug 10 08:20:31 2021 -0300

    Make Saltstack fail on first error
    
    no issue #
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b48ce416e..ddeced207 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -256,6 +256,8 @@ else
 
   # Set salt to masterless mode
   cat > /etc/salt/minion << EOFSM
+failhard: "True"
+
 file_client: local
 file_roots:
   base:

commit 5cd4453b8602e8172665dbd172e9d2b3c1f4901a
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Mon Aug 9 19:19:45 2021 -0300

    Fix nginx_passenger parameters for Centos-7 and Ubuntu-18.04
    
    refs #17535
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6faafb5b5..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,15 +3,15 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-{%- if grains.os_family in ('RedHat',) %}
-  {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
-  {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
-  {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
-{%- else %}
-  {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
-  {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
-  {%- set passenger_ruby = '/usr/bin/ruby' %}
-{%- endif %}
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+                          if grains.osfinger in ('CentOS Linux-7') else
+                        'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+                          if grains.osfinger in ('CentOS Linux-7',) else
+                        '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                         '/usr/bin/ruby' %}
 
 ### NGINX
 nginx:
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
index 6ce75faa7..a4d3c34f2 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
@@ -3,19 +3,69 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+                          if grains.osfinger in ('CentOS Linux-7') else
+                        'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+                          if grains.osfinger in ('CentOS Linux-7',) else
+                        '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                         '/usr/bin/ruby' %}
+
 ### NGINX
 nginx:
   install_from_phusionpassenger: true
   lookup:
-    passenger_package: libnginx-mod-http-passenger
-    passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+    passenger_package: {{ passenger_pkg }}
+  ### PASSENGER
+  passenger:
+    passenger_ruby: {{ passenger_ruby }}
 
   ### SERVER
   server:
     config:
-      include: 'modules-enabled/*.conf'
+      # This is required to get the passenger module loaded
+      # In Debian it can be done with this
+      # include: 'modules-enabled/*.conf'
+      load_module: {{ passenger_mod }}
+
       worker_processes: 4
 
+  ### SNIPPETS
+  snippets:
+    # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+    ssl_hardening_default.conf:
+      - ssl_session_timeout: 1d
+      - ssl_session_cache: 'shared:arvadosSSL:10m'
+      - ssl_session_tickets: 'off'
+
+      # intermediate configuration
+      - ssl_protocols: TLSv1.2 TLSv1.3
+      - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+      - ssl_prefer_server_ciphers: 'off'
+
+      # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+      - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+      # OCSP stapling
+      # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+      # - ssl_stapling: 'on'
+      # - ssl_stapling_verify: 'on'
+
+      # verify chain of trust of OCSP response using Root CA and Intermediate certs
+      # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+      # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+      # - ssl_dhparam: /path/to/dhparam
+
+      # replace with the IP address of your resolver
+      # - resolver: 127.0.0.1
+
+    arvados-snakeoil.conf:
+      - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem
+      - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key
+
   ### SITES
   servers:
     managed:

commit f8b2605671befef2f4b0e8c529478c623adee15f
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 18:39:20 2021 -0300

    Don't force the ruby version in the examples
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 12231fa8c..ccf6bac78 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
   ## manage OS packages with some other tool and you don't want us messing up
   ## with your setup.
   ruby:
-    pkg: ruby-2.7.2
 
     ## We set these to `true` here for testing purposes.
     ## They both default to `false`.

commit 326aece8fbc677c02b78e29f679819dae131bdc5
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 12:37:34 2021 -0300

    Adapt the provision scripts to manage Centos 7 too
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 3019a9fb1..a3463bfc5 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -35,7 +35,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                                cp -vr /vagrant/tests /home/vagrant/tests;
                                sed 's#cluster_fixme_or_this_wont_work#harpo#g;
                                     s#domain_fixme_or_this_wont_work#local#g;
-                                    s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+                                    s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
                                     s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \
                                     /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
      arv.vm.provision "shell",
@@ -78,7 +78,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                                cp -vr /vagrant/tests /home/vagrant/tests;
                                sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
                                     s#cluster_fixme_or_this_wont_work#zeppo#g;
-                                    s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+                                    s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
                                     s#domain_fixme_or_this_wont_work#local#g;' \
                                     /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
      arv.vm.provision "shell",
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 23e007650..12231fa8c 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
 ---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -26,6 +28,8 @@ arvados:
   ## manage OS packages with some other tool and you don't want us messing up
   ## with your setup.
   ruby:
+    pkg: ruby-2.7.2
+
     ## We set these to `true` here for testing purposes.
     ## They both default to `false`.
     manage_ruby: true
@@ -67,8 +71,15 @@ arvados:
       host: 127.0.0.1
       password: "__DATABASE_PASSWORD__"
       user: __CLUSTER___arvados
-      encoding: en_US.utf8
-      client_encoding: UTF8
+      extra_conn_params:
+        client_encoding: UTF8
+      # Centos7 does not enable SSL by default, so we disable
+      # it here just for testing of the formula purposes only.
+      # You should not do this in production, and should
+      # configure Postgres certificates correctly
+      {%- if grains.os_family in ('RedHat',) %}
+        sslmode: disable
+      {%- endif %}
 
     tls:
       # certificate: ''
@@ -76,6 +87,13 @@ arvados:
       # required to test with arvados-snakeoil certs
       insecure: true
 
+    resources:
+      virtual_machines:
+        shell:
+          name: webshell
+          backend: 127.0.1.1
+          port: 4200
+
     ### TOKENS
     tokens:
       system_root: __SYSTEM_ROOT_TOKEN__
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
index b2f12c773..54087f6d6 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
@@ -3,17 +3,23 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
+
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
   ### SITES
   servers:
     managed:
-      arvados_api:
+      arvados_api.conf:
         enabled: true
         overwrite: true
         config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
index 3adf0580a..195e9af82 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
@@ -20,7 +20,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_controller_default:
+      arvados_controller_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -33,9 +33,11 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_controller_ssl:
+      arvados_controller_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: __CLUSTER__.__DOMAIN__
@@ -52,7 +54,8 @@ nginx:
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
               - proxy_set_header: 'X-External-Client $external_client'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
             - client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
index 2d8922df9..91179d4a8 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
@@ -16,7 +16,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_keepproxy_default:
+      arvados_keepproxy_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -29,9 +29,11 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_keepproxy_ssl:
+      arvados_keepproxy_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: keep.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
             - client_max_body_size: 64M
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
index d180a3bad..9ea16bfb5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
@@ -16,7 +16,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_collections_download_default:
+      arvados_collections_download_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -30,9 +30,11 @@ nginx:
               - return: '301 https://$host$request_uri'
 
       ### COLLECTIONS / DOWNLOAD
-      arvados_collections_download_ssl:
+      arvados_collections_download_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
             - client_max_body_size: 0
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6ce75faa7..6faafb5b5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,19 +3,69 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
+  {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
+  {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
+{%- else %}
+  {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
+  {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+  {%- set passenger_ruby = '/usr/bin/ruby' %}
+{%- endif %}
+
 ### NGINX
 nginx:
   install_from_phusionpassenger: true
   lookup:
-    passenger_package: libnginx-mod-http-passenger
-    passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+    passenger_package: {{ passenger_pkg }}
+  ### PASSENGER
+  passenger:
+    passenger_ruby: {{ passenger_ruby }}
 
   ### SERVER
   server:
     config:
-      include: 'modules-enabled/*.conf'
+      # This is required to get the passenger module loaded
+      # In Debian it can be done with this
+      # include: 'modules-enabled/*.conf'
+      load_module: {{ passenger_mod }}
+
       worker_processes: 4
 
+  ### SNIPPETS
+  snippets:
+    # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+    ssl_hardening_default.conf:
+      - ssl_session_timeout: 1d
+      - ssl_session_cache: 'shared:arvadosSSL:10m'
+      - ssl_session_tickets: 'off'
+
+      # intermediate configuration
+      - ssl_protocols: TLSv1.2 TLSv1.3
+      - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+      - ssl_prefer_server_ciphers: 'off'
+
+      # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+      - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+      # OCSP stapling
+      # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+      # - ssl_stapling: 'on'
+      # - ssl_stapling_verify: 'on'
+
+      # verify chain of trust of OCSP response using Root CA and Intermediate certs
+      # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+      # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+      # - ssl_dhparam: /path/to/dhparam
+
+      # replace with the IP address of your resolver
+      # - resolver: 127.0.0.1
+
+    arvados-snakeoil.conf:
+      - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem
+      - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key
+
   ### SITES
   servers:
     managed:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
index e75f04434..9b73ab4a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
@@ -3,6 +3,20 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+# This parameter will be used here to generate a list of upstreams and vhosts.
+# This dict is here for convenience and should be managed some other way, but the
+# different ways of orchestration that can be used for this are outside the scope
+# of this formula and their examples.
+# These upstreams should match those defined in `arvados:cluster:resources:virtual_machines`
+{% set webshell_virtual_machines = {
+  'shell': {
+    'name': 'webshell',
+    'backend': '127.0.1.1',
+    'port': 4200,
+  }
+}
+%}
+
 ### NGINX
 nginx:
   ### SERVER
@@ -11,13 +25,20 @@ nginx:
 
       ### STREAMS
       http:
-        upstream webshell_upstream:
-          - server: 'shell.internal:4200 fail_timeout=10s'
+        {%- for vm, params in webshell_virtual_machines.items() %}
+          {%- set vm_name = params.name | default(vm) %}
+          {%- set vm_backend = params.backend | default(vm_name) %}
+          {%- set vm_port = params.port | default(4200) %}
+
+        upstream {{ vm_name }}_upstream:
+          - server: '{{ vm_backend }}:{{ vm_port }} fail_timeout=10s'
+
+        {%- endfor %}
 
   ### SITES
   servers:
     managed:
-      arvados_webshell_default:
+      arvados_webshell_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -30,17 +51,21 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_webshell_ssl:
+      arvados_webshell_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: webshell.__CLUSTER__.__DOMAIN__
             - listen:
               - __CONTROLLER_EXT_SSL_PORT__ http2 ssl
             - index: index.html index.htm
-            - location /shell.__CLUSTER__.__DOMAIN__:
-              - proxy_pass: 'http://webshell_upstream'
+            {%- for vm, params in webshell_virtual_machines.items() %}
+              {%- set vm_name = params.name | default(vm) %}
+            - location /{{ vm_name }}:
+              - proxy_pass: 'http://{{ vm_name }}_upstream'
               - proxy_read_timeout: 90
               - proxy_connect_timeout: 90
               - proxy_set_header: 'Host $http_host'
@@ -67,8 +92,9 @@ nginx:
                 - add_header: "'Access-Control-Allow-Origin' '*'"
                 - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
                 - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
-
-            - include: 'snippets/arvados-snakeoil.conf'
+            {%- endfor %}
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
 
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
index 3a354ac29..bcd0457c9 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
@@ -16,7 +16,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_websocket_default:
+      arvados_websocket_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -29,9 +29,11 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_websocket_ssl:
+      arvados_websocket_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: ws.__CLUSTER__.__DOMAIN__
@@ -53,6 +55,7 @@ nginx:
             - client_max_body_size: 64M
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
index 8fdd55399..44bd16fe3 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
@@ -1,12 +1,18 @@
 ---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
 
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
@@ -14,7 +20,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_workbench2_default:
+      arvados_workbench2_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -27,9 +33,11 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_workbench2_ssl:
+      arvados_workbench2_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: workbench2.__CLUSTER__.__DOMAIN__
@@ -43,6 +51,7 @@ nginx:
                 - return: 503
             - location /config.json:
               - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
index 649af10b6..6b7ab969f 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
@@ -3,10 +3,16 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
+
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
@@ -23,7 +29,7 @@ nginx:
   servers:
     managed:
       ### DEFAULT
-      arvados_workbench_default:
+      arvados_workbench_default.conf:
         enabled: true
         overwrite: true
         config:
@@ -36,9 +42,11 @@ nginx:
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_workbench_ssl:
+      arvados_workbench_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          file: nginx_snippet_arvados-snakeoil.conf
         config:
           - server:
             - server_name: workbench.__CLUSTER__.__DOMAIN__
@@ -54,11 +62,12 @@ nginx:
               - proxy_set_header: 'Host $http_host'
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - include: snippets/arvados-snakeoil.conf
             - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
 
-      arvados_workbench_upstream:
+      arvados_workbench_upstream.conf:
         enabled: true
         overwrite: true
         config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
index 71e712cad..fda1545a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
@@ -5,11 +5,29 @@
 
 ### POSTGRESQL
 postgres:
-  use_upstream_repo: false
+  # Centos-7's postgres package is too old, so we need to force using upstream's
+  # This is not required in Debian's family as they already ship with PG +11
+  {%- if salt['grains.get']('os_family') == 'RedHat' %}
+  use_upstream_repo: true
+  version: '12'
+
+  pkgs_deps:
+    - libicu
+    - libxslt
+    - systemd-sysv
+
+  pkgs_extra:
+    - postgresql12-contrib
+
+  {%- else %}
   pkgs_extra:
     - postgresql-contrib
+  {%- endif %}
   postgresconf: |-
     listen_addresses = '*'  # listen on all interfaces
+    #ssl = on
+    #ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+    #ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
   acls:
     - ['local', 'all', 'postgres', 'peer']
     - ['local', 'all', 'all', 'peer']
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
index fb1473def..91617e4fa 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
 
 {%- set curr_tpldir = tpldir %}
 {%- set tpldir = 'arvados' %}
 {%- from "arvados/map.jinja" import arvados with context %}
 {%- set tpldir = curr_tpldir %}
 
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+  - nginx.passenger
+  - nginx.config
+  - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
 {%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
 {%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
 {%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
 
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
       - ca-certificates
 
 arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
-  # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+  # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
   cmd.run:
     - name: |
         # These dirs are not to CentOS-ish, but this is a helper script
@@ -121,6 +128,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
     - require:
       - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
       - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+    # We need this before we can add the nginx's snippet
+    - require_in:
+      - file: nginx_snippet_arvados-snakeoil.conf
 
 {%- if grains.get('os_family') == 'Debian' %}
 arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -130,29 +140,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
       - sls: postgres
 
 arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
-  cmd.run:
-    - name: |
-        chown root:ssl-cert {{ arvados_key_file }}
+  file.managed:
+    - name: {{ arvados_key_file }}
+    - owner: root
+    - group: ssl-cert
     - require:
       - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
       - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
-  file.managed:
-    - name: /etc/nginx/snippets/arvados-snakeoil.conf
-    - contents: |
-        ssl_certificate {{ arvados_cert_file }};
-        ssl_certificate_key {{ arvados_key_file }};
-    - watch_in:
-      - service: nginx_service
-    - require:
-      - pkg: passenger_install
-      - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run
     - require_in:
-      - file: nginx_config
-      - service: nginx_service
-    - watch_in:
-      - service: nginx_service
-
-
+      - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
index 130fb5e93..b6929fb88 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
 
 {%- set curr_tpldir = tpldir %}
 {%- set tpldir = 'arvados' %}
 {%- from "arvados/map.jinja" import arvados with context %}
 {%- set tpldir = curr_tpldir %}
 
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+  - nginx.passenger
+  - nginx.config
+  - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
 {%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
 {%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
 {%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
 
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
       - ca-certificates
 
 arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
-  # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+  # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
   cmd.run:
     - name: |
         # These dirs are not to CentOS-ish, but this is a helper script
@@ -124,6 +131,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
     - require:
       - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
       - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+    # We need this before we can add the nginx's snippet
+    - require_in:
+      - file: nginx_snippet_arvados-snakeoil.conf
 
 {%- if grains.get('os_family') == 'Debian' %}
 arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -133,26 +143,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
       - sls: postgres
 
 arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
-  cmd.run:
-    - name: |
-        chown root:ssl-cert {{ arvados_key_file }}
+  file.managed:
+    - name: {{ arvados_key_file }}
+    - owner: root
+    - group: ssl-cert
     - require:
       - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
       - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
-  file.managed:
-    - name: /etc/nginx/snippets/arvados-snakeoil.conf
-    - contents: |
-        ssl_certificate {{ arvados_cert_file }};
-        ssl_certificate_key {{ arvados_key_file }};
-    - require:
-      - pkg: nginx_install
     - require_in:
-      - file: nginx_config
-      - service: nginx_service
-    - watch_in:
-      - service: nginx_service
-
-
+      - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index 17b7b8888..283c631ec 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -100,6 +100,6 @@ RELEASE="production"
 # ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index ae54e7437..e23634e8c 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -72,6 +72,6 @@ RELEASE="production"
 # ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index a35bd45bf..ae9804863 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -81,6 +81,6 @@ RELEASE="production"
 # ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"

commit 629f622df7ab0f4ad1816c04bc5cf0ed7eeaa58c
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 12:29:55 2021 -0300

    The user's `api_client_authorization` token can't be obtained using filters
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index 53c51a2c5..020efa94e 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -55,13 +55,17 @@ echo "Activating user '__INITIAL_USER__'"
 arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
 
 echo "Getting the user API TOKEN"
-user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+user_api_token=$(arv api_client_authorization list | jq -r ".items[] | select( .owner_uuid == \"${user_uuid}\" ).api_token" | head -1)
 
 if [ "x${user_api_token}" = "x" ]; then
+  echo "No existing token found for user '__INITIAL_USER__' (user_uuid: '${user_uuid}'). Creating token"
   user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
 fi
 
+echo "API TOKEN FOR user '__INITIAL_USER__': '${user_api_token}'."
+
 # Change to the user's token and run the workflow
+echo "Switching to user '__INITIAL_USER__'"
 export ARVADOS_API_TOKEN="${user_api_token}"
 
 echo "Running test CWL workflow"

commit 94fe1571806a2203f56f58be7d5b90276d34115e
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Aug 5 12:16:07 2021 -0300

    Some more Centos required tweaks to the provision script.
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index e9ed4c891..b48ce416e 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -175,7 +175,7 @@ ARVADOS_TAG="2.2-dev"
 # Other formula versions we depend on
 POSTGRES_TAG="v0.41.6"
 NGINX_TAG="temp-fix-missing-statements-in-pillar"
-DOCKER_TAG="v1.0.0"
+DOCKER_TAG="v2.0.7"
 LOCALE_TAG="v0.3.4"
 LETSENCRYPT_TAG="v2.1.0"
 
@@ -232,23 +232,19 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
 else
   # Install a few dependency packages
   # First, let's figure out the OS we're working on
-  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
   echo "Detected distro: ${OS_ID}"
 
   case ${OS_ID} in
-    centos)
-      PREINSTALL_CMD="/bin/true"
-      INSTALL_CMD="yum install -y"
+    "centos")
+      yum install -y  curl git jq
       ;;
-    debian|ubuntu)
-      PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
-      INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+    "debian"|"ubuntu")
+      DEBIAN_FRONTEND=noninteractive apt update
+      DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
       ;;
   esac
 
-  ${PREINSTALL_CMD}
-  ${INSTALL_CMD} curl git jq
-
   if which salt-call; then
     echo "Salt already installed"
   else
@@ -614,5 +610,10 @@ fi
 # Test that the installation finished correctly
 if [ "x${TEST}" = "xyes" ]; then
   cd ${T_DIR}
-  ./run-test.sh
+  # If we use RVM, we need to run this with it, or most ruby commands will fail
+  RVM_EXEC=""
+  if [ -x /usr/local/rvm/bin/rvm-exec ]; then
+    RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
+  fi
+  ${RVM_EXEC} ./run-test.sh
 fi

commit 471acedb1968388ac42f1aa56773f22fdcb301f9
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu May 20 16:11:52 2021 -0300

    feat(provision): moved the arvados-formula to Arvados.org
    
    refs #17655
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index af8b41a47..e9ed4c891 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -276,24 +276,17 @@ mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
 
 # Get the formula and dependencies
 cd ${F_DIR} || exit 1
+
 echo "Cloning formulas"
 rm -rf ${F_DIR}/* || exit 1
-git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
-( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
-( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
-
-git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
-( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
-( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
-
-git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
-( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
 
-git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
+git clone --branch "${ARVADOS_TAG}"     https://git.arvados.org/arvados-formula.git
+git clone --branch "${DOCKER_TAG}"      https://github.com/saltstack-formulas/docker-formula.git
+git clone --branch "${LOCALE_TAG}"      https://github.com/saltstack-formulas/locale-formula.git
+# git clone --branch "${NGINX_TAG}"       https://github.com/saltstack-formulas/nginx-formula.git
+git clone --branch "${NGINX_TAG}"       https://github.com/netmanagers/nginx-formula.git
+git clone --branch "${POSTGRES_TAG}"    https://github.com/saltstack-formulas/postgres-formula.git
+git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
 
 # If we want to try a specific branch of the formula
 if [ "x${BRANCH}" != "x" ]; then

commit 23a110b5deee8ab930422f87a7af9645127270f6
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Fri Jul 23 15:02:45 2021 -0300

    17535: Modify provision script to run on Centos
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 6ccf2b910..af8b41a47 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -11,6 +11,7 @@
 # vagrant up
 
 set -o pipefail
+set -x
 
 # capture the directory that the script is running from
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -229,8 +230,24 @@ fi
 if [ "${DUMP_CONFIG}" = "yes" ]; then
   echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
 else
-  apt-get update
-  apt-get install -y curl git jq
+  # Install a few dependency packages
+  # First, let's figure out the OS we're working on
+  OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+  echo "Detected distro: ${OS_ID}"
+
+  case ${OS_ID} in
+    centos)
+      PREINSTALL_CMD="/bin/true"
+      INSTALL_CMD="yum install -y"
+      ;;
+    debian|ubuntu)
+      PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
+      INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+      ;;
+  esac
+
+  ${PREINSTALL_CMD}
+  ${INSTALL_CMD} curl git jq
 
   if which salt-call; then
     echo "Salt already installed"

commit bd72cd6e25e068c4c1418eb252965040504845b4
Author: Ward Vandewege <ward at curii.com>
Date:   Thu Jul 1 14:00:01 2021 -0400

    The bleeding edge branch is now called 'main'.
    
    refs #17817
    
    Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>

diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index cba82fe3f..386205d37 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -39,7 +39,7 @@ func (s *GitSuite) TestPathVariants(c *check.C) {
 func (s *GitSuite) TestReadonly(c *check.C) {
 	err := s.RunGit(c, spectatorToken, "fetch", "active/foo.git")
 	c.Assert(err, check.Equals, nil)
-	err = s.RunGit(c, spectatorToken, "push", "active/foo.git", "master:newbranchfail")
+	err = s.RunGit(c, spectatorToken, "push", "active/foo.git", "main:newbranchfail")
 	c.Assert(err, check.ErrorMatches, `.*HTTP (code = )?403.*`)
 	_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranchfail")
 	c.Assert(err, check.FitsTypeOf, &os.PathError{})
@@ -48,7 +48,7 @@ func (s *GitSuite) TestReadonly(c *check.C) {
 func (s *GitSuite) TestReadwrite(c *check.C) {
 	err := s.RunGit(c, activeToken, "fetch", "active/foo.git")
 	c.Assert(err, check.Equals, nil)
-	err = s.RunGit(c, activeToken, "push", "active/foo.git", "master:newbranch")
+	err = s.RunGit(c, activeToken, "push", "active/foo.git", "main:newbranch")
 	c.Assert(err, check.Equals, nil)
 	_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranch")
 	c.Assert(err, check.Equals, nil)
@@ -104,7 +104,7 @@ func (s *GitSuite) makeArvadosRepo(c *check.C) {
 	msg, err := exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git").CombinedOutput()
 	c.Log(string(msg))
 	c.Assert(err, check.Equals, nil)
-	msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:master").CombinedOutput()
+	msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:main").CombinedOutput()
 	c.Log(string(msg))
 	c.Assert(err, check.Equals, nil)
 }

commit 636a9676a1edf3c80c07875c03f6347e060c2ba0
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Thu Jun 17 16:50:16 2021 -0300

    17821: Add a parameter to provision.sh to dump the configuration
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index f5e40ff15..17b7b8888 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -82,6 +82,7 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
 # Extra states to apply. If you use your own subdir, change this value accordingly
 # EXTRA_STATES_DIR="${CONFIG_DIR}/states"
 
+# These are ARVADOS-related settings.
 # Which release of Arvados repo you want to use
 RELEASE="production"
 # Which version of Arvados you want to install. Defaults to latest stable
@@ -90,13 +91,13 @@ RELEASE="production"
 # This is an arvados-formula setting.
 # If branch is set, the script will switch to it before running salt
 # Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
 
 ##########################################################
 # Usually there's no need to modify things below this line
 
 # Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
 # DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index 6dd47722c..ae54e7437 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -54,6 +54,7 @@ USE_LETSENCRYPT="no"
 # Extra states to apply. If you use your own subdir, change this value accordingly
 # EXTRA_STATES_DIR="${CONFIG_DIR}/states"
 
+# These are ARVADOS-related settings.
 # Which release of Arvados repo you want to use
 RELEASE="production"
 # Which version of Arvados you want to install. Defaults to latest stable
@@ -62,13 +63,13 @@ RELEASE="production"
 # This is an arvados-formula setting.
 # If branch is set, the script will switch to it before running salt
 # Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
 
 ##########################################################
 # Usually there's no need to modify things below this line
 
 # Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
 # DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index fda42a9c7..a35bd45bf 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -63,6 +63,7 @@ USE_LETSENCRYPT="no"
 # Extra states to apply. If you use your own subdir, change this value accordingly
 # EXTRA_STATES_DIR="${CONFIG_DIR}/states"
 
+# These are ARVADOS-related settings.
 # Which release of Arvados repo you want to use
 RELEASE="production"
 # Which version of Arvados you want to install. Defaults to latest stable
@@ -71,13 +72,13 @@ RELEASE="production"
 # This is an arvados-formula setting.
 # If branch is set, the script will switch to it before running salt
 # Usually not needed, only used for testing
-# BRANCH="master"
+# BRANCH="main"
 
 ##########################################################
 # Usually there's no need to modify things below this line
 
 # Formulas versions
-# ARVADOS_TAG="v1.1.4"
+# ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.41.6"
 # NGINX_TAG="temp-fix-missing-statements-in-pillar"
 # DOCKER_TAG="v1.0.0"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b92c98f11..6ccf2b910 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -21,7 +21,6 @@ usage() {
   echo >&2
   echo >&2 "${0} options:"
   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
-  echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
@@ -39,17 +38,35 @@ usage() {
   echo >&2 "                                                workbench2"
   echo >&2 "                                              Defaults to applying them all"
   echo >&2 "  -h, --help                                  Display this help and exit"
+  echo >&2 "  --dump-config <dest_dir>                    Dumps the pillars and states to a directory"
+  echo >&2 "                                              This parameter does not perform any installation at all. It's"
+  echo >&2 "                                              intended to give you a parsed sot of configuration files so"
+  echo >&2 "                                              you can inspect them or use them in you Saltstack infrastructure."
+  echo >&2 "                                              It"
+  echo >&2 "                                                - parses the pillar and states templates,"
+  echo >&2 "                                                - downloads the helper formulas with their desired versions,"
+  echo >&2 "                                                - prepares the 'top.sls' files both for pillars and states"
+  echo >&2 "                                                  for the selected role/s"
+  echo >&2 "                                                - writes the resulting files into <dest_dir>"
   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
   echo >&2
 }
 
 arguments() {
   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+  if ! which getopt > /dev/null; then
+    echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
+    exit 1
+  fi
+
   TEMP=$(getopt -o c:dhp:r:tv \
-    --long config:,debug,help,ssl-port:,roles:,test,vagrant \
+    --long config:,debug,dump-config:,help,roles:,test,vagrant \
     -n "${0}" -- "${@}")
 
-  if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+  if [ ${?} != 0 ];
+    then echo "Please check the parameters you entered and re-run again"
+    exit 1
+  fi
   # Note the quotes around `$TEMP': they are essential!
   eval set -- "$TEMP"
 
@@ -62,9 +79,23 @@ arguments() {
       -d | --debug)
         LOG_LEVEL="debug"
         shift
+        set -x
         ;;
-      -p | --ssl-port)
-        CONTROLLER_EXT_SSL_PORT=${2}
+      --dump-config)
+        if [[ ${2} = /* ]]; then
+          DUMP_SALT_CONFIG_DIR=${2}
+        else
+          DUMP_SALT_CONFIG_DIR=${PWD}/${2}
+        fi
+        ## states
+        S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
+        ## formulas
+        F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
+        ## pillars
+        P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
+        ## tests
+        T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
+        DUMP_CONFIG="yes"
         shift 2
         ;;
       -r | --roles)
@@ -102,6 +133,7 @@ arguments() {
 
 CONFIG_FILE="${SCRIPT_DIR}/local.params"
 CONFIG_DIR="local_config_dir"
+DUMP_CONFIG="no"
 LOG_LEVEL="info"
 CONTROLLER_EXT_SSL_PORT=443
 TESTS_DIR="tests"
@@ -127,44 +159,51 @@ WEBSOCKET_EXT_SSL_PORT=8002
 WORKBENCH1_EXT_SSL_PORT=443
 WORKBENCH2_EXT_SSL_PORT=3001
 
+## These are ARVADOS-related parameters
 # For a stable release, change RELEASE "production" and VERSION to the
 # package version (including the iteration, e.g. X.Y.Z-1) of the
 # release.
 RELEASE="production"
 VERSION="2.2.2-1"
 
-# The arvados-formula version.  For a stable release, this should be a
+# These are arvados-formula-related parameters
+# An arvados-formula tag. For a stable release, this should be a
 # branch name (e.g. X.Y-dev) or tag for the release.
 ARVADOS_TAG="2.2-dev"
 
 # Other formula versions we depend on
 POSTGRES_TAG="v0.41.6"
-NGINX_TAG="v2.7.4"
+NGINX_TAG="temp-fix-missing-statements-in-pillar"
 DOCKER_TAG="v1.0.0"
 LOCALE_TAG="v0.3.4"
 LETSENCRYPT_TAG="v2.1.0"
 
 # Salt's dir
+DUMP_SALT_CONFIG_DIR=""
 ## states
 S_DIR="/srv/salt"
 ## formulas
 F_DIR="/srv/formulas"
-##pillars
+## pillars
 P_DIR="/srv/pillars"
+## tests
+T_DIR="/tmp/cluster_tests"
 
 arguments ${@}
 
 if [ -s ${CONFIG_FILE} ]; then
   source ${CONFIG_FILE}
 else
-  echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in"
+  echo >&2 "You don't seem to have a config file with initial values."
+  echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
   exit 1
 fi
 
 if [ ! -d ${CONFIG_DIR} ]; then
-  echo >&2 "Please create a '${CONFIG_DIR}' with initial values, as described in"
+  echo >&2 "You don't seem to have a config directory with pillars and states."
+  echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
   exit 1
@@ -176,7 +215,7 @@ if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
   exit 1
 fi
 
-if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
+if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
   exit 1
@@ -187,20 +226,23 @@ if [ "x${HOSTNAME_EXT}" = "x" ] ; then
   HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
 fi
 
-apt-get update
-apt-get install -y curl git jq
-
-if which salt-call; then
-  echo "Salt already installed"
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+  echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
 else
-  curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
-  sh /tmp/bootstrap_salt.sh -XdfP -x python3
-  /bin/systemctl stop salt-minion.service
-  /bin/systemctl disable salt-minion.service
-fi
+  apt-get update
+  apt-get install -y curl git jq
+
+  if which salt-call; then
+    echo "Salt already installed"
+  else
+    curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+    sh /tmp/bootstrap_salt.sh -XdfP -x python3
+    /bin/systemctl stop salt-minion.service
+    /bin/systemctl disable salt-minion.service
+  fi
 
-# Set salt to masterless mode
-cat > /etc/salt/minion << EOFSM
+  # Set salt to masterless mode
+  cat > /etc/salt/minion << EOFSM
 file_client: local
 file_roots:
   base:
@@ -211,23 +253,36 @@ pillar_roots:
   base:
     - ${P_DIR}
 EOFSM
+fi
 
-mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
+mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
 
 # Get the formula and dependencies
 cd ${F_DIR} || exit 1
-git clone --branch "${ARVADOS_TAG}"     https://git.arvados.org/arvados-formula.git
-git clone --branch "${DOCKER_TAG}"      https://github.com/saltstack-formulas/docker-formula.git
-git clone --branch "${LOCALE_TAG}"      https://github.com/saltstack-formulas/locale-formula.git
-git clone --branch "${NGINX_TAG}"       https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${POSTGRES_TAG}"    https://github.com/saltstack-formulas/postgres-formula.git
-git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
+echo "Cloning formulas"
+rm -rf ${F_DIR}/* || exit 1
+git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
+( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
+( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
+
+git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
+
+git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
+( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
+
+git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 
 # If we want to try a specific branch of the formula
 if [ "x${BRANCH}" != "x" ]; then
-  cd ${F_DIR}/arvados-formula || exit 1
-  git checkout -t origin/"${BRANCH}" -b "${BRANCH}"
-  cd -
+  ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
+elif [ "x${ARVADOS_TAG}" != "x" ]; then
+( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
 fi
 
 if [ "x${VAGRANT}" = "xyes" ]; then
@@ -242,6 +297,8 @@ fi
 
 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 
+echo "Writing pillars and states"
+
 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 # to ease deployment for newcomers
 if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
@@ -293,7 +350,7 @@ if [ "x${TEST}" = "xyes" ] && [ ! -d "${SOURCE_TESTS_DIR}" ]; then
   echo "You requested to run tests, but ${SOURCE_TESTS_DIR} does not exist or is not a directory. Exiting."
   exit 1
 fi
-mkdir -p /tmp/cluster_tests
+mkdir -p ${T_DIR}
 # Replace cluster and domain name in the test files
 for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
   sed "s#__CLUSTER__#${CLUSTER}#g;
@@ -305,9 +362,9 @@ for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
        s#__INITIAL_USER__#${INITIAL_USER}#g;
        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
-  "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
+  "${f}" > ${T_DIR}/$(basename "${f}")
 done
-chmod 755 /tmp/cluster_tests/run-test.sh
+chmod 755 ${T_DIR}/run-test.sh
 
 # Replace helper state files that differ from the formula's examples
 if [ -d "${SOURCE_STATES_DIR}" ]; then
@@ -499,6 +556,11 @@ else
   done
 fi
 
+if [ "${DUMP_CONFIG}" = "yes" ]; then
+  # We won't run the rest of the script because we're just dumping the config
+  exit 0
+fi
+
 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 if [ -e /root/.psqlrc ]; then
   if ! ( grep 'pset pager off' /root/.psqlrc ); then
@@ -541,6 +603,6 @@ fi
 
 # Test that the installation finished correctly
 if [ "x${TEST}" = "xyes" ]; then
-  cd /tmp/cluster_tests
+  cd ${T_DIR}
   ./run-test.sh
 fi

commit d9bdbbce2f455f2e10b757e76bb9475c9b6e6304
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Fri Jun 18 12:34:14 2021 -0300

    17823: change references back to saltstack-formulas' nginx formula
    
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 2ab856d53..b92c98f11 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -139,7 +139,7 @@ ARVADOS_TAG="2.2-dev"
 
 # Other formula versions we depend on
 POSTGRES_TAG="v0.41.6"
-NGINX_TAG="temp-fix-missing-statements-in-pillar"
+NGINX_TAG="v2.7.4"
 DOCKER_TAG="v1.0.0"
 LOCALE_TAG="v0.3.4"
 LETSENCRYPT_TAG="v2.1.0"
@@ -219,8 +219,7 @@ cd ${F_DIR} || exit 1
 git clone --branch "${ARVADOS_TAG}"     https://git.arvados.org/arvados-formula.git
 git clone --branch "${DOCKER_TAG}"      https://github.com/saltstack-formulas/docker-formula.git
 git clone --branch "${LOCALE_TAG}"      https://github.com/saltstack-formulas/locale-formula.git
-# git clone --branch "${NGINX_TAG}"       https://github.com/saltstack-formulas/nginx-formula.git
-git clone --branch "${NGINX_TAG}"       https://github.com/netmanagers/nginx-formula.git
+git clone --branch "${NGINX_TAG}"       https://github.com/saltstack-formulas/nginx-formula.git
 git clone --branch "${POSTGRES_TAG}"    https://github.com/saltstack-formulas/postgres-formula.git
 git clone --branch "${LETSENCRYPT_TAG}" https://github.com/saltstack-formulas/letsencrypt-formula.git
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list