[ARVADOS] updated: 2.1.0-1841-g4f5540fdd
Git user
git at public.arvados.org
Fri Jan 21 19:57:38 UTC 2022
Summary of changes:
.../multi_host/aws/pillars/nginx_passenger.sls | 32 ++++++++++----
.../aws}/states/custom_certs.sls | 0
.../local.params.example.multiple_hosts | 13 +++---
...l.params.example.single_host_multiple_hostnames | 7 ++-
tools/salt-install/provision.sh | 51 +++++++++++++++++-----
5 files changed, 76 insertions(+), 27 deletions(-)
copy tools/salt-install/config_examples/{single_host/multiple_hostnames => multi_host/aws}/states/custom_certs.sls (100%)
via 4f5540fdd686522e73f2c4416bd11d1000f99004 (commit)
via 94942f7b2f35a775aea5b22d2be637022e6b4fb7 (commit)
via c598d0afc20deec62c6af58a8b0cb1da26d861c4 (commit)
via 2459f83fafbad9a8969324ff5aef11fc47cbd142 (commit)
via 729843af064ef140dc046b3009a9230626e23bf4 (commit)
from e3e6b374664f92aa6b48c8572e70f3ba5f723b4c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4f5540fdd686522e73f2c4416bd11d1000f99004
Merge: e3e6b3746 94942f7b2
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jan 21 16:56:04 2022 -0300
Merge branch '18658-fix-custom-certs-deployment-on-multi-host'
closes #18658
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
commit 94942f7b2f35a775aea5b22d2be637022e6b4fb7
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jan 21 16:07:00 2022 -0300
18658: address review comments.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index eb64bb622..c6f196ca9 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -79,10 +79,12 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
+# directory where the "provision.sh" script was copied in the destination host.
# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both downoad/collections):
+# keepweb, which is split in both download/collections):
# "controller"
# "websocket"
# "workbench"
@@ -90,10 +92,10 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# "webshell"
# "download" # Part of keepweb
# "collections" # Part of keepweb
-# "keep" # Keepproxy
+# "keepproxy" # Keepproxy
# Ie., 'keep', the script will lookup for
-# ${CUSTOM_CERTS_DIR}/keep.crt
-# ${CUSTOM_CERTS_DIR}/keep.key
+# ${CUSTOM_CERTS_DIR}/keepproxy.crt
+# ${CUSTOM_CERTS_DIR}/keepproxy.key
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index 6c9258a3c..11ebc119f 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -52,10 +52,12 @@ USE_LETSENCRYPT="no"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
+# directory where the "provision.sh" script was copied in the destination host.
# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both downoad/collections):
+# keepweb, which is split in both download/collections):
# "controller"
# "websocket"
# "workbench"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 6f1e625c9..83a538ee5 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -141,8 +141,19 @@ copy_custom_cert() {
cert_name=${2}
mkdir -p /srv/salt/certs
- cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
- cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+
+ if [ -f ${cert_dir}/${cert_name}.crt ]; then
+ cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
+ else
+ echo "${cert_dir}/${cert_name}.crt does not exist. Exiting"
+ exit 1
+ fi
+ if [ -f ${cert_dir}/${cert_name}.key ]; then
+ cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+ else
+ echo "${cert_dir}/${cert_name}.key does not exist. Exiting"
+ exit 1
+ fi
}
DEV_MODE="no"
commit c598d0afc20deec62c6af58a8b0cb1da26d861c4
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 20:17:26 2022 -0300
18658: ensure custom SSL certs are copied to the correct host dir
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index d0c4cc2bd..6f1e625c9 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -136,6 +136,15 @@ arguments() {
done
}
+copy_custom_cert() {
+ cert_dir=${1}
+ cert_name=${2}
+
+ mkdir -p /srv/salt/certs
+ cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
+ cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+}
+
DEV_MODE="no"
CONFIG_FILE="${SCRIPT_DIR}/local.params"
CONFIG_DIR="local_config_dir"
@@ -547,6 +556,17 @@ if [ -z "${ROLES}" ]; then
else
# If we add individual roles, make sure we add the repo first
echo " - arvados.repo" >> ${S_DIR}/top.sls
+ # We add the custom_certs state
+ grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+
+ # And we add the basic part for the certs pillar
+ if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+ # And add the certs in the custom_certs pillar
+ echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
+ echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
+ grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo " - extra_custom_certs" >> ${P_DIR}/top.sls
+ fi
+
for R in ${ROLES}; do
case "${R}" in
"database")
@@ -570,9 +590,8 @@ else
grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
# Use custom certs
- cp -v ${CUSTOM_CERTS_DIR}/controller.* "${F_DIR}/extra/extra/files/"
- # We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+ grep -q controller ${P_DIR}/extra_custom_certs.sls || echo " - controller" >> ${P_DIR}/extra_custom_certs.sls
fi
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
@@ -594,14 +613,11 @@ else
else
# Use custom certs, special case for keepweb
if [ ${R} = "keepweb" ]; then
- cp -v ${CUSTOM_CERTS_DIR}/download.* "${F_DIR}/extra/extra/files/"
- cp -v ${CUSTOM_CERTS_DIR}/collections.* "${F_DIR}/extra/extra/files/"
+ copy_custom_cert ${CUSTOM_CERTS_DIR} download
+ copy_custom_cert ${CUSTOM_CERTS_DIR} collections
else
- cp -v ${CUSTOM_CERTS_DIR}/${R}.* "${F_DIR}/extra/extra/files/"
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
fi
- # We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
-
fi
# webshell role is just a nginx vhost, so it has no state
if [ "${R}" != "webshell" ]; then
@@ -640,8 +656,6 @@ else
${P_DIR}/nginx_${R}_configuration.sls
fi
else
- grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
-
# As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
@@ -650,12 +664,14 @@ else
s#__CERT_PEM__#/etc/nginx/ssl/arvados-${kwsub}.pem#g;
s#__CERT_KEY__#/etc/nginx/ssl/arvados-${kwsub}.key#g" \
${P_DIR}/nginx_${kwsub}_configuration.sls
+ grep -q ${kwsub} ${P_DIR}/extra_custom_certs.sls || echo " - ${kwsub}" >> ${P_DIR}/extra_custom_certs.sls
done
else
sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g;
s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g;
s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \
${P_DIR}/nginx_${R}_configuration.sls
+ grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
fi
fi
;;
commit 2459f83fafbad9a8969324ff5aef11fc47cbd142
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 20:16:12 2022 -0300
18658: update multi-host's nginx pillar to use rvm ruby on Ubuntu-18.04
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
index a2df3ff09..28cc748da 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
@@ -3,12 +3,34 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+ if grains.osfinger in ('CentOS Linux-7') else
+ 'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+ if grains.osfinger in ('CentOS Linux-7',) else
+ '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ '/usr/bin/ruby' %}
+
### NGINX
nginx:
install_from_phusionpassenger: true
lookup:
- passenger_package: libnginx-mod-http-passenger
- passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+ passenger_package: {{ passenger_pkg }}
+ ### PASSENGER
+ passenger:
+ passenger_ruby: {{ passenger_ruby }}
+
+ ### SERVER
+ server:
+ config:
+ # This is required to get the passenger module loaded
+ # In Debian it can be done with this
+ # include: 'modules-enabled/*.conf'
+ load_module: {{ passenger_mod }}
+
+ worker_processes: 4
### SNIPPETS
snippets:
@@ -39,12 +61,6 @@ nginx:
# replace with the IP address of your resolver
# - resolver: 127.0.0.1
- ### SERVER
- server:
- config:
- include: 'modules-enabled/*.conf'
- worker_processes: 4
-
### SITES
servers:
managed:
commit 729843af064ef140dc046b3009a9230626e23bf4
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 12:22:16 2022 -0300
18658: add missing state to deploy custom certs in multi-host env
Also, set the full path to the certs dir if not configured
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls b/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls
new file mode 100644
index 000000000..371650339
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls
@@ -0,0 +1,31 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs') %}
+{%- set dest_cert_dir = '/etc/nginx/ssl' %}
+{%- set certs = salt['pillar.get']('extra_custom_certs', []) %}
+
+extra_custom_certs_file_directory_certs_dir:
+ file.directory:
+ - name: /etc/nginx/ssl
+ - require:
+ - pkg: nginx_install
+
+{%- for cert in certs %}
+ {%- set cert_file = 'arvados-' ~ cert ~ '.pem' %}
+ {#- set csr_file = 'arvados-' ~ cert ~ '.csr' #}
+ {%- set key_file = 'arvados-' ~ cert ~ '.key' %}
+ {% for c in [cert_file, key_file] %}
+extra_custom_certs_file_copy_{{ c }}:
+ file.copy:
+ - name: {{ dest_cert_dir }}/{{ c }}
+ - source: {{ orig_cert_dir }}/{{ c }}
+ - force: true
+ - user: root
+ - group: root
+ - unless: cmp {{ dest_cert_dir }}/{{ c }} {{ orig_cert_dir }}/{{ c }}
+ - require:
+ - file: extra_custom_certs_file_directory_certs_dir
+ {%- endfor %}
+{%- endfor %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index c770c8d74..eb64bb622 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -79,7 +79,8 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-CUSTOM_CERTS_DIR="./certs"
+# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
# keepweb, which is split in both downoad/collections):
# "controller"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index cf79fe244..6c9258a3c 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -52,7 +52,8 @@ USE_LETSENCRYPT="no"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-CUSTOM_CERTS_DIR="./certs"
+# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
# keepweb, which is split in both downoad/collections):
# "controller"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 537f087b6..d0c4cc2bd 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -166,7 +166,7 @@ WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
USE_LETSENCRYPT="no"
-CUSTOM_CERTS_DIR="./certs"
+CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
## These are ARVADOS-related parameters
# For a stable release, change RELEASE "production" and VERSION to the
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list