[ARVADOS] updated: 2.1.0-2206-gb8e2072eb

Git user git at public.arvados.org
Wed Apr 6 21:00:58 UTC 2022


Summary of changes:
 tools/salt-install/Vagrantfile                     | 14 ++--
 .../multi_host/aws/pillars/arvados.sls             |  7 ++
 .../aws/states/shell_cron_add_login_sync.sls       | 86 ++++++++++++++++++++++
 .../multiple_hostnames/pillars/arvados.sls         |  3 +-
 .../single_hostname/pillars/arvados.sls            |  4 +-
 .../states/shell_cron_add_login_sync.sls           | 86 ++++++++++++++++++++++
 tools/salt-install/provision.sh                    |  6 +-
 7 files changed, 193 insertions(+), 13 deletions(-)
 create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls
 create mode 100644 tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls

       via  b8e2072eb1b9dc95d8312ff5eafa8381013b76d2 (commit)
      from  74057d05e57dc6833b330aeffffe966a698118ea (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b8e2072eb1b9dc95d8312ff5eafa8381013b76d2
Author: Javier Bértoli <jbertoli at curii.com>
Date:   Wed Apr 6 10:50:51 2022 -0300

    Merge branch '18631-shell-login-sync'
    
    closes #18631
    Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>

diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 1573b6862..b08ff8ffa 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -68,19 +68,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  
      # Networking
      # WEBUI PORT
-     arv.vm.network "forwarded_port", guest: 9443, host: 9443
+     arv.vm.network "forwarded_port", guest: 443, host: 9443
      # WORKBENCH1
-     arv.vm.network "forwarded_port", guest: 9444, host: 9444
+     arv.vm.network "forwarded_port", guest: 8805, host: 9444
      # WORKBENCH2
-     arv.vm.network "forwarded_port", guest: 9445, host: 9445
+     arv.vm.network "forwarded_port", guest: 443, host: 9445
      # KEEPPROXY
-     arv.vm.network "forwarded_port", guest: 35101, host: 35101
+     arv.vm.network "forwarded_port", guest: 8801, host: 35101
      # KEEPWEB
-     arv.vm.network "forwarded_port", guest: 11002, host: 11002
+     arv.vm.network "forwarded_port", guest: 8802, host: 11002
      # WEBSHELL
-     arv.vm.network "forwarded_port", guest: 14202, host: 14202
+     arv.vm.network "forwarded_port", guest: 8803, host: 14202
      # WEBSOCKET
-     arv.vm.network "forwarded_port", guest: 18002, host: 18002
+     arv.vm.network "forwarded_port", guest: 8804, host: 18002
      arv.vm.provision "shell",
                       inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
                                cp -vr /vagrant/tests /home/vagrant/tests;
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
index 8c14c56ed..b92c818b8 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
@@ -81,6 +81,13 @@ arvados:
       # required to test with arvados-snakeoil certs
       insecure: false
 
+    resources:
+      virtual_machines:
+        shell:
+          name: shell
+          backend: __SHELL_INT_IP__
+          port: 4200
+
     ### TOKENS
     tokens:
       system_root: __SYSTEM_ROOT_TOKEN__
diff --git a/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls b/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls
new file mode 100644
index 000000000..86c591e97
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/states/shell_cron_add_login_sync.sls
@@ -0,0 +1,86 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/main/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+  pkg.installed:
+    - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+  {%- set vm_name = vm_params.name | default(vm) %}
+
+  # Check if any of the specified virtual_machines parameters corresponds to this instance
+  # It should be an error if we get more than one occurrence
+  {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+         vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+                               grains['ipv4'] + grains['ipv6'] %}
+
+    # We need to query the VM UUID
+    {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+                                ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+    %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+  cmd.run:
+    - env:
+      - ARVADOS_API_TOKEN: {{ api_token }}
+      - ARVADOS_API_HOST: {{ api_host }}
+      - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }}
+    - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+    - unless:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+    - require:
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST
+    - value: {{ api_host }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_TOKEN
+    - value: {{ api_token }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST_INSECURE
+    - value: {{ arvados.cluster.tls.insecure | default(false) }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_VIRTUAL_MACHINE_UUID
+    - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}")
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+  cron.present:
+    - name: /usr/local/bin/arvados-login-sync
+    - minute: '*/2'
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+  {%- endif %}
+{%- endfor %}
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 2579c5ffb..d11e61bba 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -28,7 +28,6 @@ arvados:
   ## manage OS packages with some other tool and you don't want us messing up
   ## with your setup.
   ruby:
-
     ## We set these to `true` here for testing purposes.
     ## They both default to `false`.
     manage_ruby: true
@@ -90,7 +89,7 @@ arvados:
       virtual_machines:
         shell:
           name: webshell
-          backend: 127.0.1.1
+          backend: 127.0.0.1
           port: 4200
 
     ### TOKENS
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
index 8e4d66caf..21bb2c45a 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
@@ -88,8 +88,8 @@ arvados:
     resources:
       virtual_machines:
         shell:
-          name: webshell
-          backend: 127.0.1.1
+          name: __HOSTNAME_EXT__
+          backend: 127.0.0.1
           port: 4200
 
     ### TOKENS
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls
new file mode 100644
index 000000000..86c591e97
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/shell_cron_add_login_sync.sls
@@ -0,0 +1,86 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/main/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+  pkg.installed:
+    - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+  {%- set vm_name = vm_params.name | default(vm) %}
+
+  # Check if any of the specified virtual_machines parameters corresponds to this instance
+  # It should be an error if we get more than one occurrence
+  {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+         vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+                               grains['ipv4'] + grains['ipv6'] %}
+
+    # We need to query the VM UUID
+    {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+                                ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+    %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+  cmd.run:
+    - env:
+      - ARVADOS_API_TOKEN: {{ api_token }}
+      - ARVADOS_API_HOST: {{ api_host }}
+      - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }}
+    - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+    - unless:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+    - require:
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST
+    - value: {{ api_host }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_TOKEN
+    - value: {{ api_token }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST_INSECURE
+    - value: {{ arvados.cluster.tls.insecure | default(false) }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_VIRTUAL_MACHINE_UUID
+    - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}")
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+  cron.present:
+    - name: /usr/local/bin/arvados-login-sync
+    - minute: '*/2'
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+  {%- endif %}
+{%- endfor %}
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ebca93819..8cb87203f 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -518,7 +518,7 @@ if [ -d "${F_DIR}"/extra/extra ]; then
     # Same when using self-signed certificates.
     SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
   fi
-  for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_sudo_passwordless"); do
+  for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_"); do
   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
   done
   # Use byo or self-signed certificates
@@ -548,10 +548,11 @@ if [ -z "${ROLES}" ]; then
     grep -q "custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
   fi
 
-  echo "    - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
   echo "    - postgres" >> ${S_DIR}/top.sls
   echo "    - docker.software" >> ${S_DIR}/top.sls
   echo "    - arvados" >> ${S_DIR}/top.sls
+  echo "    - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+  echo "    - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
 
   # Pillars
   echo "    - docker" >> ${P_DIR}/top.sls
@@ -759,6 +760,7 @@ else
       "shell")
         # States
         echo "    - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+        echo "    - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
         # Pillars

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list