[ARVADOS] created: 2.1.0-1092-g98099b854
Git user
git at public.arvados.org
Thu Aug 5 15:39:01 UTC 2021
at 98099b85467c6f8432a2a969ba4132de3132e256 (commit)
commit 98099b85467c6f8432a2a969ba4132de3132e256
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:37:34 2021 -0300
Adapt the provision scripts to manage Centos 7 too
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 3019a9fb1..a3463bfc5 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -35,7 +35,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
cp -vr /vagrant/tests /home/vagrant/tests;
sed 's#cluster_fixme_or_this_wont_work#harpo#g;
s#domain_fixme_or_this_wont_work#local#g;
- s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+ s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \
/vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
arv.vm.provision "shell",
@@ -78,7 +78,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
cp -vr /vagrant/tests /home/vagrant/tests;
sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
s#cluster_fixme_or_this_wont_work#zeppo#g;
- s/#\ BRANCH=\"master\"/\ BRANCH=\"master\"/g;
+ s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
s#domain_fixme_or_this_wont_work#local#g;' \
/vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
arv.vm.provision "shell",
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 23e007650..12231fa8c 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -26,6 +28,8 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
+ pkg: ruby-2.7.2
+
## We set these to `true` here for testing purposes.
## They both default to `false`.
manage_ruby: true
@@ -67,8 +71,15 @@ arvados:
host: 127.0.0.1
password: "__DATABASE_PASSWORD__"
user: __CLUSTER___arvados
- encoding: en_US.utf8
- client_encoding: UTF8
+ extra_conn_params:
+ client_encoding: UTF8
+ # Centos7 does not enable SSL by default, so we disable
+ # it here just for testing of the formula purposes only.
+ # You should not do this in production, and should
+ # configure Postgres certificates correctly
+ {%- if grains.os_family in ('RedHat',) %}
+ sslmode: disable
+ {%- endif %}
tls:
# certificate: ''
@@ -76,6 +87,13 @@ arvados:
# required to test with arvados-snakeoil certs
insecure: true
+ resources:
+ virtual_machines:
+ shell:
+ name: webshell
+ backend: 127.0.1.1
+ port: 4200
+
### TOKENS
tokens:
system_root: __SYSTEM_ROOT_TOKEN__
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
index b2f12c773..54087f6d6 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_api_configuration.sls
@@ -3,17 +3,23 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
- arvados_api:
+ arvados_api.conf:
enabled: true
overwrite: true
config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
index 3adf0580a..195e9af82 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_controller_configuration.sls
@@ -20,7 +20,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_controller_default:
+ arvados_controller_default.conf:
enabled: true
overwrite: true
config:
@@ -33,9 +33,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_controller_ssl:
+ arvados_controller_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: __CLUSTER__.__DOMAIN__
@@ -52,7 +54,8 @@ nginx:
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- proxy_set_header: 'X-External-Client $external_client'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
- client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
index 2d8922df9..91179d4a8 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepproxy_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_keepproxy_default:
+ arvados_keepproxy_default.conf:
enabled: true
overwrite: true
config:
@@ -29,9 +29,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_keepproxy_ssl:
+ arvados_keepproxy_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: keep.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
index d180a3bad..9ea16bfb5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_keepweb_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_collections_download_default:
+ arvados_collections_download_default.conf:
enabled: true
overwrite: true
config:
@@ -30,9 +30,11 @@ nginx:
- return: '301 https://$host$request_uri'
### COLLECTIONS / DOWNLOAD
- arvados_collections_download_ssl:
+ arvados_collections_download_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
@@ -52,6 +54,7 @@ nginx:
- client_max_body_size: 0
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 6ce75faa7..6faafb5b5 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -3,19 +3,69 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set passenger_pkg = 'nginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby' %}
+{%- else %}
+ {%- set passenger_pkg = 'libnginx-mod-http-passenger' %}
+ {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
+ {%- set passenger_ruby = '/usr/bin/ruby' %}
+{%- endif %}
+
### NGINX
nginx:
install_from_phusionpassenger: true
lookup:
- passenger_package: libnginx-mod-http-passenger
- passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+ passenger_package: {{ passenger_pkg }}
+ ### PASSENGER
+ passenger:
+ passenger_ruby: {{ passenger_ruby }}
### SERVER
server:
config:
- include: 'modules-enabled/*.conf'
+ # This is required to get the passenger module loaded
+ # In Debian it can be done with this
+ # include: 'modules-enabled/*.conf'
+ load_module: {{ passenger_mod }}
+
worker_processes: 4
+ ### SNIPPETS
+ snippets:
+ # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+ ssl_hardening_default.conf:
+ - ssl_session_timeout: 1d
+ - ssl_session_cache: 'shared:arvadosSSL:10m'
+ - ssl_session_tickets: 'off'
+
+ # intermediate configuration
+ - ssl_protocols: TLSv1.2 TLSv1.3
+ - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+ - ssl_prefer_server_ciphers: 'off'
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+ # OCSP stapling
+ # FIXME! Stapling does not work with self-signed certificates, so disabling for tests
+ # - ssl_stapling: 'on'
+ # - ssl_stapling_verify: 'on'
+
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+ # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+ # - ssl_dhparam: /path/to/dhparam
+
+ # replace with the IP address of your resolver
+ # - resolver: 127.0.0.1
+
+ arvados-snakeoil.conf:
+ - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem
+ - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key
+
### SITES
servers:
managed:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
index e75f04434..9b73ab4a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_webshell_configuration.sls
@@ -3,6 +3,20 @@
#
# SPDX-License-Identifier: AGPL-3.0
+# This parameter will be used here to generate a list of upstreams and vhosts.
+# This dict is here for convenience and should be managed some other way, but the
+# different ways of orchestration that can be used for this are outside the scope
+# of this formula and their examples.
+# These upstreams should match those defined in `arvados:cluster:resources:virtual_machines`
+{% set webshell_virtual_machines = {
+ 'shell': {
+ 'name': 'webshell',
+ 'backend': '127.0.1.1',
+ 'port': 4200,
+ }
+}
+%}
+
### NGINX
nginx:
### SERVER
@@ -11,13 +25,20 @@ nginx:
### STREAMS
http:
- upstream webshell_upstream:
- - server: 'shell.internal:4200 fail_timeout=10s'
+ {%- for vm, params in webshell_virtual_machines.items() %}
+ {%- set vm_name = params.name | default(vm) %}
+ {%- set vm_backend = params.backend | default(vm_name) %}
+ {%- set vm_port = params.port | default(4200) %}
+
+ upstream {{ vm_name }}_upstream:
+ - server: '{{ vm_backend }}:{{ vm_port }} fail_timeout=10s'
+
+ {%- endfor %}
### SITES
servers:
managed:
- arvados_webshell_default:
+ arvados_webshell_default.conf:
enabled: true
overwrite: true
config:
@@ -30,17 +51,21 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_webshell_ssl:
+ arvados_webshell_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: webshell.__CLUSTER__.__DOMAIN__
- listen:
- __CONTROLLER_EXT_SSL_PORT__ http2 ssl
- index: index.html index.htm
- - location /shell.__CLUSTER__.__DOMAIN__:
- - proxy_pass: 'http://webshell_upstream'
+ {%- for vm, params in webshell_virtual_machines.items() %}
+ {%- set vm_name = params.name | default(vm) %}
+ - location /{{ vm_name }}:
+ - proxy_pass: 'http://{{ vm_name }}_upstream'
- proxy_read_timeout: 90
- proxy_connect_timeout: 90
- proxy_set_header: 'Host $http_host'
@@ -67,8 +92,9 @@ nginx:
- add_header: "'Access-Control-Allow-Origin' '*'"
- add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
- add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
-
- - include: 'snippets/arvados-snakeoil.conf'
+ {%- endfor %}
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
index 3a354ac29..bcd0457c9 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_websocket_configuration.sls
@@ -16,7 +16,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_websocket_default:
+ arvados_websocket_default.conf:
enabled: true
overwrite: true
config:
@@ -29,9 +29,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_websocket_ssl:
+ arvados_websocket_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: ws.__CLUSTER__.__DOMAIN__
@@ -53,6 +55,7 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
index 8fdd55399..44bd16fe3 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench2_configuration.sls
@@ -1,12 +1,18 @@
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
@@ -14,7 +20,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_workbench2_default:
+ arvados_workbench2_default.conf:
enabled: true
overwrite: true
config:
@@ -27,9 +33,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_workbench2_ssl:
+ arvados_workbench2_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: workbench2.__CLUSTER__.__DOMAIN__
@@ -43,6 +51,7 @@ nginx:
- return: 503
- location /config.json:
- return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
index 649af10b6..6b7ab969f 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_workbench_configuration.sls
@@ -3,10 +3,16 @@
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
@@ -23,7 +29,7 @@ nginx:
servers:
managed:
### DEFAULT
- arvados_workbench_default:
+ arvados_workbench_default.conf:
enabled: true
overwrite: true
config:
@@ -36,9 +42,11 @@ nginx:
- location /:
- return: '301 https://$host$request_uri'
- arvados_workbench_ssl:
+ arvados_workbench_ssl.conf:
enabled: true
overwrite: true
+ requires:
+ file: nginx_snippet_arvados-snakeoil.conf
config:
- server:
- server_name: workbench.__CLUSTER__.__DOMAIN__
@@ -54,11 +62,12 @@ nginx:
- proxy_set_header: 'Host $http_host'
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- - include: 'snippets/arvados-snakeoil.conf'
+ - include: snippets/ssl_hardening_default.conf
+ - include: snippets/arvados-snakeoil.conf
- access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
- arvados_workbench_upstream:
+ arvados_workbench_upstream.conf:
enabled: true
overwrite: true
config:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
index 71e712cad..fda1545a0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/postgresql.sls
@@ -5,11 +5,29 @@
### POSTGRESQL
postgres:
- use_upstream_repo: false
+ # Centos-7's postgres package is too old, so we need to force using upstream's
+ # This is not required in Debian's family as they already ship with PG +11
+ {%- if salt['grains.get']('os_family') == 'RedHat' %}
+ use_upstream_repo: true
+ version: '12'
+
+ pkgs_deps:
+ - libicu
+ - libxslt
+ - systemd-sysv
+
+ pkgs_extra:
+ - postgresql12-contrib
+
+ {%- else %}
pkgs_extra:
- postgresql-contrib
+ {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
+ #ssl = on
+ #ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+ #ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
acls:
- ['local', 'all', 'postgres', 'peer']
- ['local', 'all', 'all', 'peer']
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
index fb1473def..91617e4fa 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
{%- set curr_tpldir = tpldir %}
{%- set tpldir = 'arvados' %}
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+ - nginx.passenger
+ - nginx.config
+ - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
{%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
- ca-certificates
arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
- # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+ # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
# These dirs are not to CentOS-ish, but this is a helper script
@@ -121,6 +128,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
- require:
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+ # We need this before we can add the nginx's snippet
+ - require_in:
+ - file: nginx_snippet_arvados-snakeoil.conf
{%- if grains.get('os_family') == 'Debian' %}
arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -130,29 +140,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
- sls: postgres
arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
- cmd.run:
- - name: |
- chown root:ssl-cert {{ arvados_key_file }}
+ file.managed:
+ - name: {{ arvados_key_file }}
+ - owner: root
+ - group: ssl-cert
- require:
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
- file.managed:
- - name: /etc/nginx/snippets/arvados-snakeoil.conf
- - contents: |
- ssl_certificate {{ arvados_cert_file }};
- ssl_certificate_key {{ arvados_key_file }};
- - watch_in:
- - service: nginx_service
- - require:
- - pkg: passenger_install
- - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run
- require_in:
- - file: nginx_config
- - service: nginx_service
- - watch_in:
- - service: nginx_service
-
-
+ - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
index 130fb5e93..b6929fb88 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
@@ -1,15 +1,22 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
{%- set curr_tpldir = tpldir %}
{%- set tpldir = 'arvados' %}
{%- from "arvados/map.jinja" import arvados with context %}
{%- set tpldir = curr_tpldir %}
-{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %}
+include:
+ - nginx.passenger
+ - nginx.config
+ - nginx.service
+
+# Debian uses different dirs for certs and keys, but being a Snake Oil example,
+# we'll keep it simple here.
+{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
{%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %}
+{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
@@ -30,7 +37,7 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_in
- ca-certificates
arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
- # Taken from https://github.com/arvados/arvados/blob/main/tools/arvbox/lib/arvbox/docker/service/certificate/run
+ # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
- name: |
# These dirs are not to CentOS-ish, but this is a helper script
@@ -124,6 +131,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
- require:
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+ # We need this before we can add the nginx's snippet
+ - require_in:
+ - file: nginx_snippet_arvados-snakeoil.conf
{%- if grains.get('os_family') == 'Debian' %}
arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
@@ -133,26 +143,13 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_instal
- sls: postgres
arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
- cmd.run:
- - name: |
- chown root:ssl-cert {{ arvados_key_file }}
+ file.managed:
+ - name: {{ arvados_key_file }}
+ - owner: root
+ - group: ssl-cert
- require:
- cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
- pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-{%- endif %}
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed:
- file.managed:
- - name: /etc/nginx/snippets/arvados-snakeoil.conf
- - contents: |
- ssl_certificate {{ arvados_cert_file }};
- ssl_certificate_key {{ arvados_key_file }};
- - require:
- - pkg: nginx_install
- require_in:
- - file: nginx_config
- - service: nginx_service
- - watch_in:
- - service: nginx_service
-
-
+ - file: nginx_snippet_arvados-snakeoil.conf
+{%- endif %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index 17b7b8888..283c631ec 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -100,6 +100,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index ae54e7437..e23634e8c 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -72,6 +72,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index a35bd45bf..ae9804863 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -81,6 +81,6 @@ RELEASE="production"
# ARVADOS_TAG="2.2.0"
# POSTGRES_TAG="v0.41.6"
# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v1.0.0"
+# DOCKER_TAG="v2.0.7"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
commit 210e68e9d1637d6f1699fad1f73c7694fd881b18
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:29:55 2021 -0300
The user's `api_client_authorization` token can't be obtained using filters
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index 53c51a2c5..020efa94e 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -55,13 +55,17 @@ echo "Activating user '__INITIAL_USER__'"
arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
echo "Getting the user API TOKEN"
-user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+user_api_token=$(arv api_client_authorization list | jq -r ".items[] | select( .owner_uuid == \"${user_uuid}\" ).api_token" | head -1)
if [ "x${user_api_token}" = "x" ]; then
+ echo "No existing token found for user '__INITIAL_USER__' (user_uuid: '${user_uuid}'). Creating token"
user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
fi
+echo "API TOKEN FOR user '__INITIAL_USER__': '${user_api_token}'."
+
# Change to the user's token and run the workflow
+echo "Switching to user '__INITIAL_USER__'"
export ARVADOS_API_TOKEN="${user_api_token}"
echo "Running test CWL workflow"
commit 10aa4dd324ddc13ad66945e0588198077b45dc6c
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Aug 5 12:16:07 2021 -0300
Some more Centos required tweaks to the provision script.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index e95fa2c1f..4790c9808 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -178,7 +178,7 @@ VERSION="latest"
# Other formula versions we depend on
POSTGRES_TAG="v0.41.6"
NGINX_TAG="temp-fix-missing-statements-in-pillar"
-DOCKER_TAG="v1.0.0"
+DOCKER_TAG="v2.0.7"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
@@ -235,23 +235,19 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
else
# Install a few dependency packages
# First, let's figure out the OS we're working on
- OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \")
echo "Detected distro: ${OS_ID}"
case ${OS_ID} in
- centos)
- PREINSTALL_CMD="/bin/true"
- INSTALL_CMD="yum install -y"
+ "centos")
+ yum install -y curl git jq
;;
- debian|ubuntu)
- PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
- INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+ "debian"|"ubuntu")
+ DEBIAN_FRONTEND=noninteractive apt update
+ DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
;;
esac
- ${PREINSTALL_CMD}
- ${INSTALL_CMD} curl git jq
-
if which salt-call; then
echo "Salt already installed"
else
@@ -624,5 +620,10 @@ fi
# Test that the installation finished correctly
if [ "x${TEST}" = "xyes" ]; then
cd ${T_DIR}
- ./run-test.sh
+ # If we use RVM, we need to run this with it, or most ruby commands will fail
+ RVM_EXEC=""
+ if [ -x /usr/local/rvm/bin/rvm-exec ]; then
+ RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
+ fi
+ ${RVM_EXEC} ./run-test.sh
fi
commit adead93f417f41e73dbbe4e02620b77ef6a5a3db
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Jul 23 15:02:45 2021 -0300
17535: Modify provision script to run on Centos
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 7ac120e5f..e95fa2c1f 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -11,6 +11,7 @@
# vagrant up
set -o pipefail
+set -x
# capture the directory that the script is running from
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -232,8 +233,24 @@ fi
if [ "${DUMP_CONFIG}" = "yes" ]; then
echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
else
- apt-get update
- apt-get install -y curl git jq
+ # Install a few dependency packages
+ # First, let's figure out the OS we're working on
+ OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d \")
+ echo "Detected distro: ${OS_ID}"
+
+ case ${OS_ID} in
+ centos)
+ PREINSTALL_CMD="/bin/true"
+ INSTALL_CMD="yum install -y"
+ ;;
+ debian|ubuntu)
+ PREINSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt update"
+ INSTALL_CMD="DEBIAN_FRONTEND=noninteractive apt install -y"
+ ;;
+ esac
+
+ ${PREINSTALL_CMD}
+ ${INSTALL_CMD} curl git jq
if which salt-call; then
echo "Salt already installed"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list