[ARVADOS] created: 2.1.0-2104-g972f9d0c4
Git user
git at public.arvados.org
Wed Mar 23 22:44:30 UTC 2022
at 972f9d0c4c32082e873bcbe29d5f7ac87b66af4a (commit)
commit 972f9d0c4c32082e873bcbe29d5f7ac87b66af4a
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Mar 23 19:39:09 2022 -0300
18631: add login-sync cron to provision script
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 1573b6862..b08ff8ffa 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -68,19 +68,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# Networking
# WEBUI PORT
- arv.vm.network "forwarded_port", guest: 9443, host: 9443
+ arv.vm.network "forwarded_port", guest: 443, host: 9443
# WORKBENCH1
- arv.vm.network "forwarded_port", guest: 9444, host: 9444
+ arv.vm.network "forwarded_port", guest: 8805, host: 9444
# WORKBENCH2
- arv.vm.network "forwarded_port", guest: 9445, host: 9445
+ arv.vm.network "forwarded_port", guest: 443, host: 9445
# KEEPPROXY
- arv.vm.network "forwarded_port", guest: 35101, host: 35101
+ arv.vm.network "forwarded_port", guest: 8801, host: 35101
# KEEPWEB
- arv.vm.network "forwarded_port", guest: 11002, host: 11002
+ arv.vm.network "forwarded_port", guest: 8802, host: 11002
# WEBSHELL
- arv.vm.network "forwarded_port", guest: 14202, host: 14202
+ arv.vm.network "forwarded_port", guest: 8803, host: 14202
# WEBSOCKET
- arv.vm.network "forwarded_port", guest: 18002, host: 18002
+ arv.vm.network "forwarded_port", guest: 8804, host: 18002
arv.vm.provision "shell",
inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
cp -vr /vagrant/tests /home/vagrant/tests;
diff --git a/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls b/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls
new file mode 100644
index 000000000..8e68b4131
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls
@@ -0,0 +1,100 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# scoped_token for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/v2.0/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- set sls_config_file = 'arvados.config.file' %}
+{# from "arvados/map.jinja" import arvados with context #}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+include:
+ - arvados
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+ pkg.installed:
+ - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+ {%- set vm_name = vm_params.name | default(vm) %}
+
+ # Check if any of the specified virtual_machines parameters corresponds to this instance
+ # It should be an error if we get more than one occurrence
+ {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+ vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+ grains['ipv4'] + grains['ipv6'] %}
+
+ {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+ %}
+
+# We need to use the UUID generated in the previous command to see if there's a
+# scoped token for it. There's no easy way to pass the value from a shellout
+# to another state, so we store it in a temp file and use that in the next
+# command. Flaky, mostly because the `unless` clause is just checking that
+# the file content is a token uuid :|
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+ - require:
+ - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run
+ - unless:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+ # There's no direct way to query the scoped_token for a given virtual_machine
+ # so we need to parse the api_client_authorization list through some jq
+ {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/vm_uuid_' ~ vm ~ ') && ' ~
+ 'arv api_client_authorization list | ' ~
+ '/usr/bin/jq -e \'.items[]| select(.scopes[] == "GET ' ~
+ '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins") | ' ~
+ '.api_token\' | head -1 | tee /tmp/scoped_token_' ~ vm ~ ' && ' ~
+ 'unset VM_UUID'
+ %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_scoped_token_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - name: {{ cmd_query_scoped_token_url }}
+ - require:
+ - cmd: extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run
+ - unless:
+ - test -s /tmp/scoped_token_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST
+ - value: {{ api_host }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_TOKEN
+ - value: __slot__:salt:cmd.run("cat /tmp/scoped_token_{{ vm }}")
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_VIRTUAL_MACHINE_UUID
+ - value: __slot__:salt:cmd.run("cat /tmp/scoped_token_{{ vm }}")
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+ cron.present:
+ - name: arvados-login-sync
+ - minute: '*/2'
+
+ {%- endif %}
+{%- endfor %}
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 2579c5ffb..d11e61bba 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
-
## We set these to `true` here for testing purposes.
## They both default to `false`.
manage_ruby: true
@@ -90,7 +89,7 @@ arvados:
virtual_machines:
shell:
name: webshell
- backend: 127.0.1.1
+ backend: 127.0.0.1
port: 4200
### TOKENS
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
index 8e4d66caf..21bb2c45a 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
@@ -88,8 +88,8 @@ arvados:
resources:
virtual_machines:
shell:
- name: webshell
- backend: 127.0.1.1
+ name: __HOSTNAME_EXT__
+ backend: 127.0.0.1
port: 4200
### TOKENS
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls
new file mode 100644
index 000000000..8e68b4131
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls
@@ -0,0 +1,100 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# scoped_token for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/v2.0/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- set sls_config_file = 'arvados.config.file' %}
+{# from "arvados/map.jinja" import arvados with context #}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+include:
+ - arvados
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+ pkg.installed:
+ - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+ {%- set vm_name = vm_params.name | default(vm) %}
+
+ # Check if any of the specified virtual_machines parameters corresponds to this instance
+ # It should be an error if we get more than one occurrence
+ {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+ vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+ grains['ipv4'] + grains['ipv6'] %}
+
+ {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+ %}
+
+# We need to use the UUID generated in the previous command to see if there's a
+# scoped token for it. There's no easy way to pass the value from a shellout
+# to another state, so we store it in a temp file and use that in the next
+# command. Flaky, mostly because the `unless` clause is just checking that
+# the file content is a token uuid :|
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+ - require:
+ - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run
+ - unless:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+ # There's no direct way to query the scoped_token for a given virtual_machine
+ # so we need to parse the api_client_authorization list through some jq
+ {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/vm_uuid_' ~ vm ~ ') && ' ~
+ 'arv api_client_authorization list | ' ~
+ '/usr/bin/jq -e \'.items[]| select(.scopes[] == "GET ' ~
+ '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins") | ' ~
+ '.api_token\' | head -1 | tee /tmp/scoped_token_' ~ vm ~ ' && ' ~
+ 'unset VM_UUID'
+ %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_scoped_token_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - name: {{ cmd_query_scoped_token_url }}
+ - require:
+ - cmd: extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run
+ - unless:
+ - test -s /tmp/scoped_token_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST
+ - value: {{ api_host }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_TOKEN
+ - value: __slot__:salt:cmd.run("cat /tmp/scoped_token_{{ vm }}")
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_VIRTUAL_MACHINE_UUID
+ - value: __slot__:salt:cmd.run("cat /tmp/scoped_token_{{ vm }}")
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+ cron.present:
+ - name: arvados-login-sync
+ - minute: '*/2'
+
+ {%- endif %}
+{%- endfor %}
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index c4ccfd126..5f6c389b7 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -514,7 +514,7 @@ if [ -d "${F_DIR}"/extra/extra ]; then
# Same when using self-signed certificates.
SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
fi
- for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_sudo_passwordless"); do
+ for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_"); do
echo " - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
done
# Use byo or self-signed certificates
@@ -544,10 +544,11 @@ if [ -z "${ROLES}" ]; then
grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
fi
- echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
echo " - postgres" >> ${S_DIR}/top.sls
echo " - docker.software" >> ${S_DIR}/top.sls
echo " - arvados" >> ${S_DIR}/top.sls
+ echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+ echo " - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
# Pillars
echo " - docker" >> ${P_DIR}/top.sls
@@ -755,6 +756,7 @@ else
"shell")
# States
echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+ echo " - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
grep -q "docker" ${S_DIR}/top.sls || echo " - docker.software" >> ${S_DIR}/top.sls
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list