[arvados] updated: 2.5.0-322-g908d141b6
git repository hosting
git at public.arvados.org
Thu Mar 30 13:30:53 UTC 2023
Summary of changes:
doc/install/salt-multi-host.html.textile.liquid | 33 ++++++++++++++++++++--
.../multi_host/aws/pillars/grafana.sls | 12 +++++++-
tools/salt-install/installer.sh | 11 +++++++-
.../local.params.example.multiple_hosts | 3 +-
...l.params.example.single_host_multiple_hostnames | 7 +++++
...ocal.params.example.single_host_single_hostname | 7 +++++
tools/salt-install/terraform/aws/vpc/locals.tf | 3 +-
7 files changed, 68 insertions(+), 8 deletions(-)
via 908d141b6564f90c2ed9e0e6c9d7a4397a528c9f (commit)
via a263abb4c84bf639b5169749a992454d0948d3d3 (commit)
via 340caa63a2ef01224c1b69db7aa63da8ec20696b (commit)
from 247fd765bc1fbf559572d6e5590893c9190f086a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 908d141b6564f90c2ed9e0e6c9d7a4397a528c9f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Mar 30 09:27:35 2023 -0400
16379: Updates for grafana install
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index 1740c24af..2a8708fd7 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -460,7 +460,15 @@ You can monitor the health and performance of the system using the admin dashboa
https://grafana.@${CLUSTER}.${DOMAIN}@
-Use the username and password you gave in @local.conf@ to log in.
+To log in, use username "admin" and @${INITIAL_USER_PASSWORD}@ from @local.conf at .
+
+Once logged in, you will want to add the dashboards to the front page.
+
+# On the left icon bar, click on "Browse"
+# If the check box next to "Starred" is selected, click on it to de-select it
+# You should see a folder with "Arvados cluster overview", "Node exporter" and "Postgres exporter"
+# You can visit each dashboard and click on the star next to the title to "Mark as favorite"
+# They should now be linked on the front page.
h2(#post_install). After the installation
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/grafana.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/grafana.sls
index 1869dbdc2..1cdff39a6 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/grafana.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/grafana.sls
@@ -4,6 +4,17 @@
# SPDX-License-Identifier: AGPL-3.0
grafana:
+ pkg:
+ name: grafana
+ use_upstream_archive: false
+ use_upstream_repo: true
+ repo:
+ humanname: grafana_official
+ name: deb https://apt.grafana.com/ stable main
+ file: /etc/apt/sources.list.d/grafana.list
+ key_url: https://apt.grafana.com/gpg.key
+ require_in:
+ - pkg: grafana
config:
default:
instance_name: __CLUSTER__.__DOMAIN__
@@ -17,4 +28,3 @@ grafana:
http_port: 3000
domain: grafana.__CLUSTER__.__DOMAIN__
root_url: https://grafana.__CLUSTER__.__DOMAIN__
-
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index 20a04872c..ef682c319 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -133,6 +133,7 @@ MONITORING_EMAIL=${INITIAL_USER_EMAIL}
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
# CONFIG_DIR="local_config_dir"
+
# Extra states to apply. If you use your own subdir, change this value accordingly
# EXTRA_STATES_DIR="${CONFIG_DIR}/states"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index 56ecf9f92..b94d687e4 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -63,6 +63,13 @@ SSL_MODE="self-signed"
SSL_KEY_ENCRYPTED="no"
SSL_KEY_AWS_SECRET_NAME="${CLUSTER}-arvados-ssl-privkey-password"
+# Customize Prometheus & Grafana web UI access credentials
+MONITORING_USERNAME=${INITIAL_USER}
+MONITORING_PASSWORD=${INITIAL_USER_PASSWORD}
+MONITORING_EMAIL=${INITIAL_USER_EMAIL}
+# Sets the directory for Grafana dashboards
+# GRAFANA_DASHBOARDS_DIR="${SCRIPT_DIR}/local_config_dir/dashboards"
+
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
# CONFIG_DIR="local_config_dir"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index 54a78b619..42c1ebb72 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -73,6 +73,13 @@ SSL_MODE="self-signed"
SSL_KEY_ENCRYPTED="no"
SSL_KEY_AWS_SECRET_NAME="${CLUSTER}-arvados-ssl-privkey-password"
+# Customize Prometheus & Grafana web UI access credentials
+MONITORING_USERNAME=${INITIAL_USER}
+MONITORING_PASSWORD=${INITIAL_USER_PASSWORD}
+MONITORING_EMAIL=${INITIAL_USER_EMAIL}
+# Sets the directory for Grafana dashboards
+# GRAFANA_DASHBOARDS_DIR="${SCRIPT_DIR}/local_config_dir/dashboards"
+
# The directory to check for the config files (pillars, states) you want to use.
# There are a few examples under 'config_examples'.
# CONFIG_DIR="local_config_dir"
commit a263abb4c84bf639b5169749a992454d0948d3d3
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 28 18:07:43 2023 -0400
16379: Add prometheus and grafana to workbench aliases
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index ed6a20197..1740c24af 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -29,6 +29,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
## "Iterating on config changes":#iterating
## "Common problems and solutions":#common-problems
# "Initial user and login":#initial_user
+# "Monitoring and Metrics":#monitoring
# "After the installation":#post_install
h2(#introduction). Introduction
@@ -62,6 +63,7 @@ In the default configuration these are:
# @workbench2.${CLUSTER}.${DOMAIN}@
# @webshell.${CLUSTER}.${DOMAIN}@
# @shell.${CLUSTER}.${DOMAIN}@
+# @monitoring.${CLUSTER}.${DOMAIN}@
For more information, see "DNS entries and TLS certificates":install-manual-prerequisites.html#dnstls.
@@ -452,6 +454,14 @@ If you did *not* "configure a different authentication provider":#authentication
If you *did* configure a different authentication provider, the first user to log in will automatically be given Arvados admin privileges.
+h2(#monitoring). Monitoring and Metrics
+
+You can monitor the health and performance of the system using the admin dashboard:
+
+https://grafana.@${CLUSTER}.${DOMAIN}@
+
+Use the username and password you gave in @local.conf@ to log in.
+
h2(#post_install). After the installation
As part of the operation of @installer.sh@, it automatically creates a @git@ repository with your configuration templates. You should retain this repository but *be aware that it contains sensitive information* (passwords and tokens used by the Arvados services as well as cloud credentials if you used Terraform to create the infrastructure).
diff --git a/tools/salt-install/terraform/aws/vpc/locals.tf b/tools/salt-install/terraform/aws/vpc/locals.tf
index 289eb3e04..00e9d9494 100644
--- a/tools/salt-install/terraform/aws/vpc/locals.tf
+++ b/tools/salt-install/terraform/aws/vpc/locals.tf
@@ -21,7 +21,7 @@ locals {
}
aliases = {
controller: ["ws"]
- workbench: ["workbench2", "webshell", "keep", "download", "*.collections"]
+ workbench: ["workbench2", "webshell", "keep", "download", "prometheus", "grafana", "*.collections"]
}
cname_by_host = flatten([
for host, aliases in local.aliases : [
@@ -32,4 +32,3 @@ locals {
]
])
}
-
commit 340caa63a2ef01224c1b69db7aa63da8ec20696b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 28 16:38:25 2023 -0400
16379: doc updates
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index ae76c5b58..ed6a20197 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -95,7 +95,7 @@ The Terraform state files (that keep crucial infrastructure information from the
h4. Terraform code configuration
-Each section described above contain a @terraform.tfvars@ file with some configuration values that you should set before applying each configuration. You should set the cluster prefix and domain name in @vpc/terraform.tfvars@:
+Each section described above contain a @terraform.tfvars@ file with some configuration values that you should set before applying each configuration. You should set the cluster prefix and domain name in @terraform/vpc/terraform.tfvars@:
<pre><code>region_name = "us-east-1"
# cluster_name = "xarv1"
@@ -105,6 +105,15 @@ If you don't set the variables @vpc/terraform.tfvars@ file, you will be asked to
The @data-storage/terraform.tfvars@ and @services/terraform.tfvars@ let you configure the location of your ssh public key (default @~/.ssh/id_rsa.pub@) and the instance type to use (default @m5a.large@).
+h4. Set credentials
+
+You will need an AWS access key and secret key to create the infrastructure.
+
+<pre><code>
+$ export AWS_ACCESS_KEY_ID="anaccesskey"
+$ export AWS_SECRET_ACCESS_KEY="asecretkey"
+</code></pre>
+
h4. Create the infrastructure
Build the infrastructure by running @./installer.sh terraform at . The last stage will output the information needed to set up the cluster's domain and continue with the installer. for example:
@@ -253,8 +262,8 @@ This can be found wherever you choose to initialize the install files (@~/setup-
# Set @CLUSTER@ to the 5-character cluster identifier (e.g "xarv1")
# Set @DOMAIN@ to the base DNS domain of the environment, e.g. "example.com"
# Set the @*_INT_IP@ variables with the internal (private) IP addresses of each host. Since services share hosts, some hosts are the same. See "note about /etc/hosts":#etchosts
-# Edit @CLUSTER_INT_CIDR@, this should be the CIDR of the private network that Arvados is running on, e.g. the VPC.
-CIDR stands for "Classless Inter-Domain Routing" and describes which portion of the IP address that refers to the network. For example 192.168.3.0/24 means that the first 24 bits are the network (192.168.3) and the last 8 bits are a specific host on that network.
+# Edit @CLUSTER_INT_CIDR@, this should be the CIDR of the private network that Arvados is running on, e.g. the VPC. If you used terraform, this is emitted as @vpc_cidr at .
+_CIDR stands for "Classless Inter-Domain Routing" and describes which portion of the IP address that refers to the network. For example 192.168.3.0/24 means that the first 24 bits are the network (192.168.3) and the last 8 bits are a specific host on that network._
_AWS Specific: Go to the AWS console and into the VPC service, there is a column in this table view of the VPCs that gives the CIDR for the VPC (IPv4 CIDR)._
# Set @INITIAL_USER_EMAIL@ to your email address, as you will be the first admin user of the system.
# Set each @KEY@ / @TOKEN@ / @PASSWORD@ to a random string. You can use @installer.sh generate-tokens@
diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh
index 000ed3292..e5aff213e 100755
--- a/tools/salt-install/installer.sh
+++ b/tools/salt-install/installer.sh
@@ -202,11 +202,12 @@ case "$subcmd" in
if [[ -n "$TERRAFORM" ]] ; then
mkdir $SETUPDIR/terraform
cp -r $TERRAFORM/* $SETUPDIR/terraform/
- cp $TERRAFORM/.gitignore $SETUPDIR/terraform/
fi
cd $SETUPDIR
echo '*.log' > .gitignore
+ echo '**/.terraform' >> .gitignore
+ echo '**/.infracost' >> .gitignore
if [[ -n "$TERRAFORM" ]] ; then
git add terraform
@@ -235,6 +236,13 @@ case "$subcmd" in
(cd terraform/services && echo -n 'letsencrypt_iam_secret_access_key = ' && terraform output letsencrypt_iam_secret_access_key) 2>&1 | tee -a $logfile
;;
+ terraform-destroy)
+ logfile=terraform-$(date -Iseconds).log
+ (cd terraform/services && terraform destroy) 2>&1 | tee -a $logfile
+ (cd terraform/data-storage && terraform destroy) 2>&1 | tee -a $logfile
+ (cd terraform/vpc && terraform destroy) 2>&1 | tee -a $logfile
+ ;;
+
generate-tokens)
for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
echo ${i}=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 32 ; echo '')
@@ -341,6 +349,7 @@ case "$subcmd" in
echo ""
echo "initialize initialize the setup directory for configuration"
echo "terraform create cloud resources using terraform"
+ echo "terraform-destroy destroy cloud resources created by terraform"
echo "generate-tokens generate random values for tokens"
echo "deploy deploy the configuration from the setup directory"
echo "diagnostics check your install using diagnostics"
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index 2bf009be1..20a04872c 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -14,7 +14,7 @@ DOMAIN="domain_fixme_or_this_wont_work"
# For multi-node installs, the ssh log in for each node
# must be root or able to sudo
-DEPLOY_USER=root
+DEPLOY_USER=admin
# The mapping of nodes to roles
# installer.sh will log in to each of these nodes and then provision
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list