[ARVADOS] updated: 2.3.2-13-g696f86231
Git user
git at public.arvados.org
Fri Jan 21 20:07:29 UTC 2022
Summary of changes:
.../multi_host/aws/pillars/nginx_passenger.sls | 32 ++++++++++----
.../aws}/states/custom_certs.sls | 0
.../local.params.example.multiple_hosts | 13 +++---
...l.params.example.single_host_multiple_hostnames | 7 ++-
tools/salt-install/provision.sh | 51 +++++++++++++++++-----
5 files changed, 76 insertions(+), 27 deletions(-)
copy tools/salt-install/config_examples/{single_host/multiple_hostnames => multi_host/aws}/states/custom_certs.sls (100%)
via 696f8623133576ddb3fc61f00fbdcccfecdf4fb2 (commit)
via f985928367bd3638e68d613b232010b4f587f1e2 (commit)
via b287377976ce841cd99d9f8ef1c881722a0dc0b4 (commit)
via ed695eb01f9e63463685312962dbbbda293348f0 (commit)
via a8d04e784a6b4b76f15c4f063fcdca85625e96b1 (commit)
from 50cedb091f840e62821bd2bb0304f22d238691b1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 696f8623133576ddb3fc61f00fbdcccfecdf4fb2
Merge: 50cedb091 f98592836
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jan 21 17:05:30 2022 -0300
Merge branch '18658-fix-custom-certs-deployment-on-multi-host' into 2.3-release
closes #18658
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
commit f985928367bd3638e68d613b232010b4f587f1e2
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jan 21 16:07:00 2022 -0300
18658: address review comments.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index eb64bb622..c6f196ca9 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -79,10 +79,12 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
+# directory where the "provision.sh" script was copied in the destination host.
# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both downoad/collections):
+# keepweb, which is split in both download/collections):
# "controller"
# "websocket"
# "workbench"
@@ -90,10 +92,10 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# "webshell"
# "download" # Part of keepweb
# "collections" # Part of keepweb
-# "keep" # Keepproxy
+# "keepproxy" # Keepproxy
# Ie., 'keep', the script will lookup for
-# ${CUSTOM_CERTS_DIR}/keep.crt
-# ${CUSTOM_CERTS_DIR}/keep.key
+# ${CUSTOM_CERTS_DIR}/keepproxy.crt
+# ${CUSTOM_CERTS_DIR}/keepproxy.key
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index 6c9258a3c..11ebc119f 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -52,10 +52,12 @@ USE_LETSENCRYPT="no"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
+# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
+# directory where the "provision.sh" script was copied in the destination host.
# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both downoad/collections):
+# keepweb, which is split in both download/collections):
# "controller"
# "websocket"
# "workbench"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 11acf839e..b498bdf60 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -141,8 +141,19 @@ copy_custom_cert() {
cert_name=${2}
mkdir -p /srv/salt/certs
- cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
- cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+
+ if [ -f ${cert_dir}/${cert_name}.crt ]; then
+ cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
+ else
+ echo "${cert_dir}/${cert_name}.crt does not exist. Exiting"
+ exit 1
+ fi
+ if [ -f ${cert_dir}/${cert_name}.key ]; then
+ cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+ else
+ echo "${cert_dir}/${cert_name}.key does not exist. Exiting"
+ exit 1
+ fi
}
DEV_MODE="no"
commit b287377976ce841cd99d9f8ef1c881722a0dc0b4
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 20:17:26 2022 -0300
18658: ensure custom SSL certs are copied to the correct host dir
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index f5e3d4eb3..11acf839e 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -136,6 +136,15 @@ arguments() {
done
}
+copy_custom_cert() {
+ cert_dir=${1}
+ cert_name=${2}
+
+ mkdir -p /srv/salt/certs
+ cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
+ cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+}
+
DEV_MODE="no"
CONFIG_FILE="${SCRIPT_DIR}/local.params"
CONFIG_DIR="local_config_dir"
@@ -547,6 +556,17 @@ if [ -z "${ROLES}" ]; then
else
# If we add individual roles, make sure we add the repo first
echo " - arvados.repo" >> ${S_DIR}/top.sls
+ # We add the custom_certs state
+ grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+
+ # And we add the basic part for the certs pillar
+ if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+ # And add the certs in the custom_certs pillar
+ echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
+ echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
+ grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo " - extra_custom_certs" >> ${P_DIR}/top.sls
+ fi
+
for R in ${ROLES}; do
case "${R}" in
"database")
@@ -570,9 +590,8 @@ else
grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
# Use custom certs
- cp -v ${CUSTOM_CERTS_DIR}/controller.* "${F_DIR}/extra/extra/files/"
- # We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+ grep -q controller ${P_DIR}/extra_custom_certs.sls || echo " - controller" >> ${P_DIR}/extra_custom_certs.sls
fi
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
@@ -594,14 +613,11 @@ else
else
# Use custom certs, special case for keepweb
if [ ${R} = "keepweb" ]; then
- cp -v ${CUSTOM_CERTS_DIR}/download.* "${F_DIR}/extra/extra/files/"
- cp -v ${CUSTOM_CERTS_DIR}/collections.* "${F_DIR}/extra/extra/files/"
+ copy_custom_cert ${CUSTOM_CERTS_DIR} download
+ copy_custom_cert ${CUSTOM_CERTS_DIR} collections
else
- cp -v ${CUSTOM_CERTS_DIR}/${R}.* "${F_DIR}/extra/extra/files/"
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
fi
- # We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
-
fi
# webshell role is just a nginx vhost, so it has no state
if [ "${R}" != "webshell" ]; then
@@ -640,8 +656,6 @@ else
${P_DIR}/nginx_${R}_configuration.sls
fi
else
- grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
-
# As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
@@ -650,12 +664,14 @@ else
s#__CERT_PEM__#/etc/nginx/ssl/arvados-${kwsub}.pem#g;
s#__CERT_KEY__#/etc/nginx/ssl/arvados-${kwsub}.key#g" \
${P_DIR}/nginx_${kwsub}_configuration.sls
+ grep -q ${kwsub} ${P_DIR}/extra_custom_certs.sls || echo " - ${kwsub}" >> ${P_DIR}/extra_custom_certs.sls
done
else
sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g;
s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g;
s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \
${P_DIR}/nginx_${R}_configuration.sls
+ grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
fi
fi
;;
commit ed695eb01f9e63463685312962dbbbda293348f0
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 20:16:12 2022 -0300
18658: update multi-host's nginx pillar to use rvm ruby on Ubuntu-18.04
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
index a2df3ff09..28cc748da 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
@@ -3,12 +3,34 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- set passenger_pkg = 'nginx-mod-http-passenger'
+ if grains.osfinger in ('CentOS Linux-7') else
+ 'libnginx-mod-http-passenger' %}
+{%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so'
+ if grains.osfinger in ('CentOS Linux-7',) else
+ '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ '/usr/bin/ruby' %}
+
### NGINX
nginx:
install_from_phusionpassenger: true
lookup:
- passenger_package: libnginx-mod-http-passenger
- passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+ passenger_package: {{ passenger_pkg }}
+ ### PASSENGER
+ passenger:
+ passenger_ruby: {{ passenger_ruby }}
+
+ ### SERVER
+ server:
+ config:
+ # This is required to get the passenger module loaded
+ # In Debian it can be done with this
+ # include: 'modules-enabled/*.conf'
+ load_module: {{ passenger_mod }}
+
+ worker_processes: 4
### SNIPPETS
snippets:
@@ -39,12 +61,6 @@ nginx:
# replace with the IP address of your resolver
# - resolver: 127.0.0.1
- ### SERVER
- server:
- config:
- include: 'modules-enabled/*.conf'
- worker_processes: 4
-
### SITES
servers:
managed:
commit a8d04e784a6b4b76f15c4f063fcdca85625e96b1
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Jan 20 12:22:16 2022 -0300
18658: add missing state to deploy custom certs in multi-host env
Also, set the full path to the certs dir if not configured
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls b/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls
new file mode 100644
index 000000000..371650339
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/states/custom_certs.sls
@@ -0,0 +1,31 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs') %}
+{%- set dest_cert_dir = '/etc/nginx/ssl' %}
+{%- set certs = salt['pillar.get']('extra_custom_certs', []) %}
+
+extra_custom_certs_file_directory_certs_dir:
+ file.directory:
+ - name: /etc/nginx/ssl
+ - require:
+ - pkg: nginx_install
+
+{%- for cert in certs %}
+ {%- set cert_file = 'arvados-' ~ cert ~ '.pem' %}
+ {#- set csr_file = 'arvados-' ~ cert ~ '.csr' #}
+ {%- set key_file = 'arvados-' ~ cert ~ '.key' %}
+ {% for c in [cert_file, key_file] %}
+extra_custom_certs_file_copy_{{ c }}:
+ file.copy:
+ - name: {{ dest_cert_dir }}/{{ c }}
+ - source: {{ orig_cert_dir }}/{{ c }}
+ - force: true
+ - user: root
+ - group: root
+ - unless: cmp {{ dest_cert_dir }}/{{ c }} {{ orig_cert_dir }}/{{ c }}
+ - require:
+ - file: extra_custom_certs_file_directory_certs_dir
+ {%- endfor %}
+{%- endfor %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index c770c8d74..eb64bb622 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -79,7 +79,8 @@ LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-CUSTOM_CERTS_DIR="./certs"
+# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
# keepweb, which is split in both downoad/collections):
# "controller"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index cf79fe244..6c9258a3c 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -52,7 +52,8 @@ USE_LETSENCRYPT="no"
# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
# and copy the required certificates under the directory specified in the next line.
# The certs will be copied from this directory by the provision script.
-CUSTOM_CERTS_DIR="./certs"
+# Plese set it to the FULL PATH to the certs dir if you're going to use a different dir
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
# The script expects cert/key files with these basenames (matching the role except for
# keepweb, which is split in both downoad/collections):
# "controller"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index f0fbb4331..f5e3d4eb3 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -166,7 +166,7 @@ WORKBENCH1_EXT_SSL_PORT=443
WORKBENCH2_EXT_SSL_PORT=3001
USE_LETSENCRYPT="no"
-CUSTOM_CERTS_DIR="./certs"
+CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
## These are ARVADOS-related parameters
# For a stable release, change RELEASE "production" and VERSION to the
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list