[ARVADOS] updated: 2.1.0-1520-gc5e4fb583
Git user
git at public.arvados.org
Fri Oct 22 16:49:27 UTC 2021
Summary of changes:
doc/_config.yml | 1 -
.../_singularity_mksquashfs_configuration.liquid | 15 ++++++++
doc/architecture/singularity.html.textile.liquid | 2 +-
.../install-compute-node.html.textile.liquid | 15 +++++---
.../crunch2-slurm/install-test.html.textile.liquid | 2 +-
...install-compute-node-docker.html.textile.liquid | 2 +-
...ll-compute-node-singularity.html.textile.liquid | 35 ++++++++++++++----
doc/install/index.html.textile.liquid | 2 +-
doc/install/singularity.html.textile.liquid | 41 ----------------------
doc/user/topics/arv-docker.html.textile.liquid | 2 +-
tools/compute-images/arvados-images-aws.json | 3 +-
tools/compute-images/arvados-images-azure.json | 3 +-
tools/compute-images/build.sh | 12 ++++++-
tools/compute-images/scripts/base.sh | 5 +++
14 files changed, 79 insertions(+), 61 deletions(-)
create mode 100644 doc/_includes/_singularity_mksquashfs_configuration.liquid
delete mode 100644 doc/install/singularity.html.textile.liquid
via c5e4fb5838d2f447ae126159a71340b90cfea33c (commit)
via 3dcd4659f10710621f429066f11ab7b67cd6e878 (commit)
from 01698bea4703ce073425e2080c7cad83e2f873cc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c5e4fb5838d2f447ae126159a71340b90cfea33c
Author: Ward Vandewege <ward at curii.com>
Date: Fri Oct 22 12:47:17 2021 -0400
18289: add support to set `mksquashfs mem` for Singularity in our
compute node image builder for cloud setups.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/tools/compute-images/arvados-images-aws.json b/tools/compute-images/arvados-images-aws.json
index 4d757abfd..b1b4c909d 100644
--- a/tools/compute-images/arvados-images-aws.json
+++ b/tools/compute-images/arvados-images-aws.json
@@ -8,6 +8,7 @@
"aws_source_ami": "ami-04d70e069399af2e9",
"build_environment": "aws",
"public_key_file": "",
+ "mksquashfs_mem": "",
"reposuffix": "",
"resolver": "",
"ssh_user": "admin",
@@ -76,6 +77,6 @@
"type": "shell",
"execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
"script": "scripts/base.sh",
- "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+ "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}"]
}]
}
diff --git a/tools/compute-images/arvados-images-azure.json b/tools/compute-images/arvados-images-azure.json
index ec1d9b6a6..20f776d04 100644
--- a/tools/compute-images/arvados-images-azure.json
+++ b/tools/compute-images/arvados-images-azure.json
@@ -10,6 +10,7 @@
"location": "centralus",
"project_id": "",
"public_key_file": "",
+ "mksquashfs_mem": "",
"reposuffix": "",
"resolver": "",
"resource_group": null,
@@ -65,6 +66,6 @@
"type": "shell",
"execute_command": "sudo -S env {{ .Vars }} /bin/bash '{{ .Path }}'",
"script": "scripts/base.sh",
- "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}"]
+ "environment_vars": ["RESOLVER={{user `resolver`}}","REPOSUFFIX={{user `reposuffix`}}","MKSQUASHFS_MEM={{user `mksquashfs_mem`}}"]
}]
}
diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh
index a2dd2ed28..a714bafc1 100755
--- a/tools/compute-images/build.sh
+++ b/tools/compute-images/build.sh
@@ -55,6 +55,8 @@ Options:
Set this to "-dev" to track the unstable/dev Arvados repositories
--public-key-file (required)
Path to the public key file that a-d-c will use to log into the compute node
+ --mksquashfs-mem (default: 512M)
+ Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
--debug
Output debug information (default: false)
@@ -78,9 +80,10 @@ DEBUG=
SSH_USER=
AWS_DEFAULT_REGION=us-east-1
PUBLIC_KEY_FILE=
+MKSQUASHFS_MEM=512M
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,debug \
+ help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,resolver:,reposuffix:,public-key-file:,mksquashfs-mem:,debug \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -154,6 +157,9 @@ while [ $# -gt 0 ]; do
--public-key-file)
PUBLIC_KEY_FILE="$2"; shift
;;
+ --mksquashfs-mem)
+ MKSQUASHFS_MEM="$2"; shift
+ ;;
--debug)
# If you want to debug a build issue, add the -debug flag to the build
# command in question.
@@ -256,6 +262,10 @@ fi
if [[ "$PUBLIC_KEY_FILE" != "" ]]; then
EXTRA2+=" -var public_key_file=$PUBLIC_KEY_FILE"
fi
+if [[ "$MKSQUASHFS_MEM" != "" ]]; then
+ EXTRA2+=" -var mksquashfs_mem=$MKSQUASHFS_MEM"
+fi
+
echo
packer version
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 022f4a7e5..0ab51223b 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -89,6 +89,11 @@ make -C ./builddir
make -C ./builddir install
ln -sf /var/lib/arvados/bin/* /usr/local/bin/
+# set `mksquashfs mem` in the singularity config file if it is configured
+if [ "$MKSQUASHFS_MEM" != "" ]; then
+ echo "mksquashfs mem = ${MKSQUASHFS_MEM}" >> /var/lib/arvados/etc/singularity/singularity.conf
+fi
+
# Print singularity version installed
singularity --version
commit 3dcd4659f10710621f429066f11ab7b67cd6e878
Author: Ward Vandewege <ward at curii.com>
Date: Fri Oct 22 12:46:45 2021 -0400
18289: Address review comments (documentation).
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index 8cc4c398e..31db9c41d 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -261,7 +261,6 @@ navbar:
- Containers API (LSF):
- install/crunch2-lsf/install-dispatch.html.textile.liquid
- Additional configuration:
- - install/singularity.html.textile.liquid
- install/container-shell-access.html.textile.liquid
- External dependencies:
- install/install-postgresql.html.textile.liquid
diff --git a/doc/_includes/_singularity_mksquashfs_configuration.liquid b/doc/_includes/_singularity_mksquashfs_configuration.liquid
new file mode 100644
index 000000000..dc0c394ba
--- /dev/null
+++ b/doc/_includes/_singularity_mksquashfs_configuration.liquid
@@ -0,0 +1,15 @@
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+h2(#singularity_mksquashfs_configuration). Singularity mksquashfs configuration
+
+{% if show_docker_warning != nil %}
+{% include 'notebox_begin_warning' %}
+This section is only relevant when using Singularity. Skip this section when using Docker.
+{% include 'notebox_end' %}
+{% endif %}
+
+Docker images are converted on the fly by @mksquashfs@, which can consume a considerable amount of RAM. The RAM usage of mksquashfs can be restricted in @/etc/singularity/singularity.conf@ with a line like @mksquashfs mem = 512M at . The amount of memory made available for mksquashfs should be configured lower than the smallest amount of memory requested by a container on the cluster to avoid the conversion being killed for using too much memory.
diff --git a/doc/architecture/singularity.html.textile.liquid b/doc/architecture/singularity.html.textile.liquid
index a94af598b..9a82cd93d 100644
--- a/doc/architecture/singularity.html.textile.liquid
+++ b/doc/architecture/singularity.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados can be configured to use "Singularity":https://sylabs.io/singularity/ instead of Docker to execute containers on cloud nodes or a Slurm/LSF cluster. Singularity may be preferable due to its simpler installation and lack of long-running daemon process and special system users/groups. See the "Singularity page in the installation guide":{{ site.baseurl }}/install/singularity.html for configuration details.
+Arvados can be configured to use "Singularity":https://sylabs.io/singularity/ instead of Docker to execute containers on cloud nodes or a Slurm/LSF cluster. Singularity may be preferable due to its simpler installation and lack of long-running daemon process and special system users/groups. For on premises Slurm/LSF clusters, see the "Set up a compute node with Singularity":{{ site.baseurl }}/install/crunch2/install-compute-node-singularity.html page. For cloud compute clusters, see the "Build a cloud compute node image":{{ site.baseurl }}/install/crunch2-cloud/install-compute-node.html page.
h2. Design overview
diff --git a/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid b/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid
index 5ea72f5e7..7c922e28d 100644
--- a/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid
+++ b/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid
@@ -16,6 +16,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
# "Introduction":#introduction
# "Create an SSH keypair":#sshkeypair
# "The build script":#building
+# "Singularity mksquashfs configuration":#singularity_mksquashfs_configuration
# "Build an AWS image":#aws
# "Build an Azure image":#azure
@@ -54,6 +55,12 @@ foktmqOY8MyctzFgXBpGTxPliGjqo8OkrOyQP2g+FL7v+Km31Xs61P8=
</code></pre>
</notextile>
+{% assign show_docker_warning = true %}
+
+{% include 'singularity_mksquashfs_configuration' %}
+
+The desired amount of memory to make available for @mksquashfs@ can be configured in an argument to the build script, see the next section. It defaults to @512M at .
+
h2(#building). The build script
The necessary files are located in the @arvados/tools/compute-images@ directory in the source tree. A build script is provided to generate the image. The @--help@ argument lists all available options:
@@ -97,15 +104,15 @@ Options:
--azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS)
Azure SKU image to use
--ssh_user (default: packer)
- The user packer will use lo log into the image
- --domain (default: arvadosapi.com)
- The domain part of the FQDN for the cluster
- --resolver (default: 8.8.8.8)
+ The user packer will use to log into the image
+ --resolver (default: host's network provided)
The dns resolver for the machine
--reposuffix (default: unset)
Set this to "-dev" to track the unstable/dev Arvados repositories
--public-key-file (required)
Path to the public key file that a-d-c will use to log into the compute node
+ --mksquashfs-mem (default: 512M)
+ Only relevant when using Singularity. This is the amount of memory mksquashfs is allowed to use.
--debug
Output debug information (default: false)
</code></pre></notextile>
diff --git a/doc/install/crunch2-slurm/install-test.html.textile.liquid b/doc/install/crunch2-slurm/install-test.html.textile.liquid
index 786a71d3e..dc13c3c0f 100644
--- a/doc/install/crunch2-slurm/install-test.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-test.html.textile.liquid
@@ -26,7 +26,7 @@ If it works, this command should print @OK@ (it may also show some status messag
h2. Test the dispatcher
-Make sure all of your compute nodes are set up with "Docker":../crunch2/install-compute-node-singularity.html or "Singularity":../crunch2/install-compute-node-docker.html.
+Make sure all of your compute nodes are set up with "Docker":../crunch2/install-compute-node-docker.html or "Singularity":../crunch2/install-compute-node-singularity.html.
On the dispatch node, start monitoring the crunch-dispatch-slurm logs:
diff --git a/doc/install/crunch2/install-compute-node-docker.html.textile.liquid b/doc/install/crunch2/install-compute-node-docker.html.textile.liquid
index 7e8f1dea7..876bb6ae5 100644
--- a/doc/install/crunch2/install-compute-node-docker.html.textile.liquid
+++ b/doc/install/crunch2/install-compute-node-docker.html.textile.liquid
@@ -25,7 +25,7 @@ These instructions apply when Containers.RuntimeEngine is set to @docker@, refer
h2(#introduction). Introduction
-This page describes how to configure a compute node so that it can be used to run containers dispatched by Arvados, with Slurm on a static cluster. These steps must be performed on every compute node.
+This page describes how to configure a compute node so that it can be used to run containers dispatched by Arvados on a static cluster. These steps must be performed on every compute node.
h2(#docker). Set up Docker
diff --git a/doc/install/crunch2/install-compute-node-singularity.html.textile.liquid b/doc/install/crunch2/install-compute-node-singularity.html.textile.liquid
index 52b2612a5..09a3b4e3a 100644
--- a/doc/install/crunch2/install-compute-node-singularity.html.textile.liquid
+++ b/doc/install/crunch2/install-compute-node-singularity.html.textile.liquid
@@ -14,22 +14,43 @@ This page describes the requirements for a compute node in a Slurm or LSF cluste
{% include 'notebox_end' %}
{% include 'notebox_begin_warning' %}
-These instructions apply when Containers.RuntimeEngine is set to @singularity@, refer to "Set up a Slurm compute node with Docker":install-compute-node-docker.html when running @docker at .
+These instructions apply when Containers.RuntimeEngine is set to @singularity@, refer to "Set up a compute node with Docker":install-compute-node-docker.html when running @docker at .
{% include 'notebox_end' %}
# "Introduction":#introduction
+# "Install python-arvados-fuse and crunch-run and squashfs-tools":#install-packages
# "Set up Singularity":#singularity
-# "Update fuse.conf":#fuse
-# "Install'python-arvados-fuse and crunch-run":#install-packages
+# "Singularity mksquashfs configuration":#singularity_mksquashfs_configuration
h2(#introduction). Introduction
-This page describes how to configure a compute node so that it can be used to run containers dispatched by Arvados, with Slurm on a static cluster. These steps must be performed on every compute node.
+Please refer to the "Singularity":{{site.baseurl}}/architecture/singularity.html documentation in the Architecture section.
+
+This page describes how to configure a compute node so that it can be used to run containers dispatched by Arvados on a static cluster. These steps must be performed on every compute node.
+
+{% assign arvados_component = 'python-arvados-fuse crunch-run squashfs-tools' %}
+
+{% include 'install_packages' %}
h2(#singularity). Set up Singularity
-See "Singularity container runtime":../singularity.html
+Follow the "Singularity installation instructions":https://sylabs.io/guides/3.7/user-guide/quick_start.html. Make sure @singularity@ and @mksquashfs@ are working:
-{% assign arvados_component = 'python-arvados-fuse crunch-run' %}
+<notextile>
+<pre><code>$ <span class="userinput">singularity version</span>
+3.7.4
+$ <span class="userinput">mksquashfs -version</span>
+mksquashfs version 4.3-git (2014/06/09)
+[...]
+</code></pre>
+</notextile>
-{% include 'install_packages' %}
+Then update @Containers.RuntimeEngine@ in your cluster configuration:
+
+<notextile>
+<pre><code> # Container runtime: "docker" (default) or "singularity"
+ RuntimeEngine: singularity
+</code></pre>
+</notextile>
+
+{% include 'singularity_mksquashfs_configuration' %}
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 1b27ca6ed..2bd9710f7 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -10,7 +10,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
{% include 'notebox_begin' %}
-This section is about installing an Arvados cluster. If you are just looking to install Arvados client tools and libraries, "go to the SDK section.":{{site.baseurl}}/sdk
+This section is about installing an Arvados cluster. If you are just looking to install Arvados client tools and libraries, "go to the SDK section.":{{site.baseurl}}/sdk/
{% include 'notebox_end' %}
Arvados components run on GNU/Linux systems, and supports AWS, GCP and Azure cloud platforms as well as on-premises installs. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS. "Arvados is Free Software":{{site.baseurl}}/user/copying/copying.html and self-install installations are not limited in any way. Commercial support and development are also available from "Curii Corporation.":mailto:info at curii.com
diff --git a/doc/install/singularity.html.textile.liquid b/doc/install/singularity.html.textile.liquid
deleted file mode 100644
index dfe12f314..000000000
--- a/doc/install/singularity.html.textile.liquid
+++ /dev/null
@@ -1,41 +0,0 @@
----
-layout: default
-navsection: installguide
-title: Singularity container runtime
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-h2(#overview). Overview
-
-Please refer to the "Singularity":{{site.baseurl}}/architecture/singularity.html documentation in the Architecture section.
-
-h2(#configuration). Configuration
-
-To use singularity, first make sure "Singularity is installed":https://sylabs.io/guides/3.7/user-guide/quick_start.html on your cloud worker image or Slurm/LSF compute nodes as applicable. Note @squashfs-tools@ is required.
-
-<notextile>
-<pre><code>$ <span class="userinput">singularity version</span>
-3.7.4
-$ <span class="userinput">mksquashfs -version</span>
-mksquashfs version 4.3-git (2014/06/09)
-[...]
-</code></pre>
-</notextile>
-
-Then update @Containers.RuntimeEngine@ in your cluster configuration:
-
-<notextile>
-<pre><code> # Container runtime: "docker" (default) or "singularity"
- RuntimeEngine: singularity
-</code></pre>
-</notextile>
-
-Restart your dispatcher (@crunch-dispatch-slurm@, @arvados-dispatch-cloud@, or @arvados-dispatch-lsf@) after updating your configuration file.
-
-h2(#singularity_configuration). Singularity configuration
-
-Docker images are converted on the fly by @mksquashfs@, which can consume a considerable amount of RAM. The RAM usage of mksquashfs can be restricted in @/etc/singularity/singularity.conf@ with a line like @mksquashfs mem = 512M at . The amount of memory made available for mksquashfs should be configured lower than the smallest amount of memory requested by a container on the cluster to avoid the conversion being killed for using too much memory.
diff --git a/doc/user/topics/arv-docker.html.textile.liquid b/doc/user/topics/arv-docker.html.textile.liquid
index 8a97df6e1..391b4e779 100644
--- a/doc/user/topics/arv-docker.html.textile.liquid
+++ b/doc/user/topics/arv-docker.html.textile.liquid
@@ -11,7 +11,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
This page describes how to set up the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a workflow step will be run in using "Docker":https://www.docker.com/ or "Singularity":https://sylabs.io/singularity/. Docker and Singularity are tools for building and running containers that isolate applications from other applications running on the same node. For detailed information, see the "Docker User Guide":https://docs.docker.com/userguide/ and the "Introduction to Singularity":https://sylabs.io/guides/3.5/user-guide/introduction.html.
-Note that Arvados always works with Docker images, even when it is configured to use Singularity to run containers. There are some differences between the two runtimes that can affect your containers. See the "Singularity container runtime":{{site.baseurl}}/install/singularity.html page for details.
+Note that Arvados always works with Docker images, even when it is configured to use Singularity to run containers. There are some differences between the two runtimes that can affect your containers. See the "Singularity architecture":{{site.baseurl}}/architecture/singularity.html page for details.
This page describes:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list