[arvados] created: 2.7.0-6363-g843ae5b166
git repository hosting
git at public.arvados.org
Fri Apr 5 19:20:13 UTC 2024
at 843ae5b166ff3c63a6aec061e0dd50f76872162f (commit)
commit 843ae5b166ff3c63a6aec061e0dd50f76872162f
Author: Brett Smith <brett.smith at curii.com>
Date: Thu Mar 7 14:36:01 2024 -0500
21383: Salt provisioner installs systemd-cron
See comments for rationale.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index cceedba259..c78f65e9ca 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -360,23 +360,24 @@ fi
if [ "${DUMP_CONFIG}" = "yes" ]; then
echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
else
- # Install a few dependency packages
- # First, let's figure out the OS we're working on
OS_IDS="$(. /etc/os-release && echo "${ID:-} ${ID_LIKE:-}")"
echo "Detected distro families: $OS_IDS"
+ # Several of our formulas use the cron module, which requires the crontab
+ # command. We install systemd-cron to ensure we have that.
+ # The rest of these packages are required by the rest of the script.
for OS_ID in $OS_IDS; do
case "$OS_ID" in
rhel)
echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
setenforce permissive
- yum install -y curl git jq
+ yum install -y curl git jq systemd-cron
break
;;
debian)
DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update
- DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
+ DEBIAN_FRONTEND=noninteractive apt install -y curl git jq systemd-cron
break
;;
esac
commit 2ed85c8f80ff3511559413c31e2e0c2a57fbb534
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Mar 6 09:31:25 2024 -0500
21383: Reconfigure Salt logrotate service
See comments for rationale.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/logrotate.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/logrotate.sls
new file mode 100644
index 0000000000..8c455e9c78
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/logrotate.sls
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The logrotate formula checks that an associated service is running.
+# The default it checks is cron.
+# All the distributions Arvados supports (Debian 11+/Ubuntu 20.04+)
+# have switched to a systemd timer, so check that instead.
+# Refer to logrotate-formula's documentation for details
+# https://github.com/salt-formulas/salt-formula-logrotate/blob/master/README.rst
+
+logrotate:
+ service: logrotate.timer
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/logrotate.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/logrotate.sls
new file mode 100644
index 0000000000..8c455e9c78
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/logrotate.sls
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The logrotate formula checks that an associated service is running.
+# The default it checks is cron.
+# All the distributions Arvados supports (Debian 11+/Ubuntu 20.04+)
+# have switched to a systemd timer, so check that instead.
+# Refer to logrotate-formula's documentation for details
+# https://github.com/salt-formulas/salt-formula-logrotate/blob/master/README.rst
+
+logrotate:
+ service: logrotate.timer
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/logrotate.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/logrotate.sls
new file mode 100644
index 0000000000..8c455e9c78
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/logrotate.sls
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The logrotate formula checks that an associated service is running.
+# The default it checks is cron.
+# All the distributions Arvados supports (Debian 11+/Ubuntu 20.04+)
+# have switched to a systemd timer, so check that instead.
+# Refer to logrotate-formula's documentation for details
+# https://github.com/salt-formulas/salt-formula-logrotate/blob/master/README.rst
+
+logrotate:
+ service: logrotate.timer
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index a32c48b529..cceedba259 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -618,6 +618,7 @@ if [ -z "${ROLES:-}" ]; then
# Pillars
echo " - docker" >> ${PILLARS_TOP}
echo " - nginx_api_configuration" >> ${PILLARS_TOP}
+ echo " - logrotate" >> ${PILLARS_TOP}
echo " - logrotate_api" >> ${PILLARS_TOP}
echo " - nginx_controller_configuration" >> ${PILLARS_TOP}
echo " - nginx_keepproxy_configuration" >> ${PILLARS_TOP}
@@ -853,6 +854,7 @@ else
grep -q "arvados.controller" ${STATES_TOP} || echo " - arvados.controller" >> ${STATES_TOP}
### Pillars ###
+ grep -q "logrotate" ${PILLARS_TOP} || echo " - logrotate" >> ${PILLARS_TOP}
grep -q "logrotate_api" ${PILLARS_TOP} || echo " - logrotate_api" >> ${PILLARS_TOP}
grep -q "aws_credentials" ${PILLARS_TOP} || echo " - aws_credentials" >> ${PILLARS_TOP}
grep -q "postgresql" ${PILLARS_TOP} || echo " - postgresql" >> ${PILLARS_TOP}
commit e3d03fe0cb80fe0e4787773f3b17532f43d944af
Author: Brett Smith <brett.smith at curii.com>
Date: Tue Mar 5 14:17:15 2024 -0500
21383: Remove docker-ce version pin
Now that we've removed support for older distros, all the distros we
currently support have the minimum version of docker-ce
available. Meanwhile, that older version is not available for some newer
distros we want to support. Remove the pin accordingly.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index f5bd38b4d5..c19febdc01 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -72,15 +72,12 @@ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes ins
python3-arvados-fuse \
arvados-docker-cleaner
-# We want Docker 20.10 or later so that we support glibc 2.33 and up in the container, cf.
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005906
-dockerversion=5:20.10.13~3-0
DOCKER_URL="https://download.docker.com/linux/$DISTRO_ID"
curl -fsSL "$DOCKER_URL/gpg" | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOCKER_URL/ $VERSION_CODENAME stable" | \
$SUDO tee /etc/apt/sources.list.d/docker.list
$SUDO apt-get update
-$SUDO apt-get -yq --no-install-recommends install docker-ce="${dockerversion}~${DISTRO_ID}-${VERSION_CODENAME}"
+$SUDO apt-get -yq --no-install-recommends install docker-ce
# Set a higher ulimit and the resolver (if set) for docker
$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
commit 96d84c9a62520b31d4f152aab61551059166d2ec
Author: Brett Smith <brett.smith at curii.com>
Date: Tue Mar 5 14:16:06 2024 -0500
21383: Standardize on os-release variables
This script was getting this information three different ways: by
reading /etc/os-release, calling lsb_release, and using its own internal
lookup tables. This is silly. Just read /etc/os-release and use those
variables throughout.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 370c3f3a3a..f5bd38b4d5 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -15,8 +15,8 @@ wait_for_apt_locks() {
done
}
-# $DIST should not have a dot if there is one in /etc/os-release (e.g. 18.04)
-DIST=$(. /etc/os-release; echo $ID$VERSION_ID | tr -d '.')
+. /etc/os-release
+DISTRO_ID="$ID"
# Run apt-get update
$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
@@ -36,9 +36,6 @@ if [[ ! -d /var/lib/cloud/scripts/per-boot ]]; then
mkdir -p /var/lib/cloud/scripts/per-boot
fi
-TMP_LSB=`/usr/bin/lsb_release -c -s`
-LSB_RELEASE_CODENAME=${TMP_LSB//[$'\t\r\n ']}
-
SET_RESOLVER=
if [ -n "$RESOLVER" ]; then
SET_RESOLVER="--dns ${RESOLVER}"
@@ -46,7 +43,7 @@ fi
# Add the arvados apt repository
echo "# apt.arvados.org" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
-echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
+echo "deb http://apt.arvados.org/$VERSION_CODENAME $VERSION_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
# Add the arvados signing key
cat /tmp/1078ECD7.asc | $SUDO apt-key add -
@@ -78,29 +75,12 @@ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes ins
# We want Docker 20.10 or later so that we support glibc 2.33 and up in the container, cf.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005906
dockerversion=5:20.10.13~3-0
-if [[ "$DIST" =~ ^debian ]]; then
- family="debian"
- if [ "$DIST" == "debian11" ]; then
- distro="bullseye"
- elif [ "$DIST" == "debian12" ]; then
- distro="bookworm"
- fi
-elif [[ "$DIST" =~ ^ubuntu ]]; then
- family="ubuntu"
- if [ "$DIST" == "ubuntu2004" ]; then
- distro="focal"
- elif [ "$DIST" == "ubuntu2204" ]; then
- distro="jammy"
- fi
-else
- echo "Unsupported distribution $DIST"
- exit 1
-fi
-curl -fsSL https://download.docker.com/linux/$family/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-echo deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/$family/ $distro stable | \
+DOCKER_URL="https://download.docker.com/linux/$DISTRO_ID"
+curl -fsSL "$DOCKER_URL/gpg" | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOCKER_URL/ $VERSION_CODENAME stable" | \
$SUDO tee /etc/apt/sources.list.d/docker.list
$SUDO apt-get update
-$SUDO apt-get -yq --no-install-recommends install docker-ce=${dockerversion}~${family}-${distro}
+$SUDO apt-get -yq --no-install-recommends install docker-ce="${dockerversion}~${DISTRO_ID}-${VERSION_CODENAME}"
# Set a higher ulimit and the resolver (if set) for docker
$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
@@ -173,7 +153,7 @@ $SUDO chown root:root /etc/cloud/cloud.cfg.d/07_compute_arvados_dispatch_cloud.c
if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
# We need a kernel and matching headers
- if [[ "$DIST" =~ ^debian ]]; then
+ if [[ "$DISTRO_ID" == debian ]]; then
$SUDO apt-get -y install linux-image-cloud-amd64 linux-headers-cloud-amd64
elif [ "$CLOUD" == "azure" ]; then
$SUDO apt-get -y install linux-image-azure linux-headers-azure
@@ -182,10 +162,11 @@ if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
fi
# Install CUDA
- $SUDO apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/7fa2af80.pub
- $SUDO apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/3bf863cc.pub
+ NVIDIA_URL="https://developer.download.nvidia.com/compute/cuda/repos/$(echo "$DISTRO_ID$VERSION_ID" | tr -d .)/x86_64"
+ $SUDO apt-key adv --fetch-keys "$NVIDIA_URL/7fa2af80.pub"
+ $SUDO apt-key adv --fetch-keys "$NVIDIA_URL/3bf863cc.pub"
$SUDO apt-get -y install software-properties-common
- $SUDO add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/ /"
+ $SUDO add-apt-repository "deb $NVIDIA_URL/ /"
$SUDO add-apt-repository contrib
$SUDO apt-get update
$SUDO apt-get -y install cuda
@@ -193,7 +174,7 @@ if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
# Install libnvidia-container, the tooling for Docker/Singularity
curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | \
$SUDO apt-key add -
- if [ "$DIST" == "debian11" ]; then
+ if [[ "$VERSION_CODENAME" == bullseye ]]; then
# As of 2021-12-16 libnvidia-container and friends are only available for
# Debian 10, not yet Debian 11. Install experimental rc1 package as per this
# workaround:
@@ -202,9 +183,7 @@ if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
$SUDO tee /etc/apt/sources.list.d/libnvidia-container.list
$SUDO sed -i -e '/experimental/ s/^#//g' /etc/apt/sources.list.d/libnvidia-container.list
else
- # here, $DIST should have a dot if there is one in /etc/os-release (e.g. 18.04)...
- DIST=$(. /etc/os-release; echo $ID$VERSION_ID)
- curl -s -L https://nvidia.github.io/libnvidia-container/$DIST/libnvidia-container.list | \
+ curl -s -L "https://nvidia.github.io/libnvidia-container/$DISTRO_ID$VERSION_ID/libnvidia-container.list" | \
$SUDO tee /etc/apt/sources.list.d/libnvidia-container.list
fi
commit e363c429eeeb5ff181c32df8d92fbf23c7a9434f
Author: Brett Smith <brett.smith at curii.com>
Date: Tue Jan 16 09:17:04 2024 -0500
21383: Update Salt installer formulas
I'm not aware of any specific reason we need this, but it's good
maintenance since I'm in here anyway.
letsencrypt 3.0.0 does introduce a backwards incompatible change because
it no longer installs the Apache plugin by default. Since we never
wanted that plugin and always specify the pkgs we want anyway, this
shouldn't affect us.
https://github.com/saltstack-formulas/letsencrypt-formula/blob/master/CHANGELOG.md#300-2021-07-02
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 143d21cb81..a32c48b529 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -291,8 +291,8 @@ POSTGRES_TAG="7529300c287b1c288af0f494ca668c2217bd1c5d"
POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git"
NGINX_TAG="v2.8.1"
DOCKER_TAG="v2.4.2"
-LOCALE_TAG="v0.3.4"
-LETSENCRYPT_TAG="v2.1.0"
+LOCALE_TAG="v0.3.5"
+LETSENCRYPT_TAG="v3.2.0"
LOGROTATE_TAG="v0.14.0"
PROMETHEUS_TAG="v5.6.5"
GRAFANA_TAG="v3.1.3"
commit 2564a9c0e4a702b8fed1527d458639e1142cc2a5
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Jan 15 15:29:34 2024 -0500
21383: Use postgres formula with support for Debian 12
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index f3704c4c19..143d21cb81 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -287,10 +287,8 @@ VERSION="latest"
SALT_VERSION="3006"
# Other formula versions we depend on
-#POSTGRES_TAG="v0.44.0"
-#POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git"
-POSTGRES_TAG="0.45.0-bugfix327"
-POSTGRES_URL="https://github.com/arvados/postgres-formula.git"
+POSTGRES_TAG="7529300c287b1c288af0f494ca668c2217bd1c5d"
+POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git"
NGINX_TAG="v2.8.1"
DOCKER_TAG="v2.4.2"
LOCALE_TAG="v0.3.4"
@@ -431,7 +429,7 @@ test -d nginx && ( cd nginx && git fetch ) \
echo "...postgres"
test -d postgres && ( cd postgres && git fetch ) \
|| git clone --quiet ${POSTGRES_URL} ${F_DIR}/postgres
-( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" )
+( cd postgres && git checkout --quiet "${POSTGRES_TAG}" )
echo "...prometheus"
test -d prometheus && ( cd prometheus && git fetch ) \
commit 2e61e3866f2ee58c0e3e6ee856948cd96b478165
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Jan 15 13:39:34 2024 -0500
21383: Upgrade Salt installer to Salt 3006
Salt 3006 is the first version to support Python 3.11, included with
Debian 12.
Because 3006 is the current stable, we have to update the bootstrap
invocation to match. I guess we'll have to make another change after it
becomes old-stable.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 8dd07020c3..f3704c4c19 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -284,7 +284,7 @@ VERSION="latest"
# We pin the salt version to avoid potential incompatibilities when a new
# stable version is released.
-SALT_VERSION="3004"
+SALT_VERSION="3006"
# Other formula versions we depend on
#POSTGRES_TAG="v0.44.0"
@@ -388,7 +388,7 @@ else
echo "Salt already installed"
else
curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
- sh /tmp/bootstrap_salt.sh -XdfP -x python3 old-stable ${SALT_VERSION}
+ sh /tmp/bootstrap_salt.sh -XdfP -x python3 stable ${SALT_VERSION}
/bin/systemctl stop salt-minion.service
/bin/systemctl disable salt-minion.service
fi
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list