[ARVADOS] updated: e009f03d9f3a620f375cf187291bd12c2bf1d74c
git at public.curoverse.com
git at public.curoverse.com
Wed Apr 8 09:47:21 EDT 2015
Summary of changes:
sdk/cli/bin/crunch-job | 34 ++++++++++++++++++++++++++++++----
1 file changed, 30 insertions(+), 4 deletions(-)
via e009f03d9f3a620f375cf187291bd12c2bf1d74c (commit)
via bae652c25653d21aece17e7137b375c0471b484e (commit)
via e1456a12e9b2daa5a7cae0daa7a952d71dd2dd1b (commit)
from 83759d40b868f7583cffcfa986bb78ff9c9f6c42 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e009f03d9f3a620f375cf187291bd12c2bf1d74c
Merge: 83759d4 bae652c
Author: Brett Smith <brett at curoverse.com>
Date: Wed Apr 8 09:47:09 2015 -0400
Merge branch '5642-crunch-job-swap-limits-wip'
Closes #5642, #5664.
commit bae652c25653d21aece17e7137b375c0471b484e
Author: Brett Smith <brett at curoverse.com>
Date: Wed Apr 8 09:46:51 2015 -0400
5642: crunch-job uses higher memory limit when running few tasks.
This gives a task all of the node's RAM when it's the only task
running this round, which is more in line with user expectations.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 912159a..a6ccebc 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -659,6 +659,10 @@ update_progress_stats();
THISROUND:
+my $tasks_this_level = 0;
+foreach my $id (@jobstep_todo) {
+ $tasks_this_level++ if ($jobstep[$id]->{level} == $level);
+}
for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
{
my $id = $jobstep_todo[$todo_ptr];
@@ -716,6 +720,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$ENV{"GZIP"} = "-n";
+ my $max_node_concurrent_tasks = $ENV{CRUNCH_NODE_SLOTS};
+ if ($tasks_this_level < $max_node_concurrent_tasks) {
+ $max_node_concurrent_tasks = $tasks_this_level;
+ }
+
my @srunargs = (
"srun",
"--nodelist=".$childnode->{name},
@@ -730,7 +739,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
# $command. No tool is expected to read these values directly.
.q{&& MEM=$(awk '($1 == "MemTotal:"){print $2}' </proc/meminfo) }
.q{&& SWAP=$(awk '($1 == "SwapTotal:"){print $2}' </proc/meminfo) }
- ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($ENV{CRUNCH_NODE_SLOTS} * 100) )) "
+ ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($max_node_concurrent_tasks * 100) )) "
."&& let SWAPLIMIT=\$MEMLIMIT+\$SWAP ";
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
commit e1456a12e9b2daa5a7cae0daa7a952d71dd2dd1b
Author: Brett Smith <brett at curoverse.com>
Date: Wed Apr 8 09:46:23 2015 -0400
5642: Explicitly make all swap available under Docker in crunch-job.
Without this, Docker 1.2 through 1.5 send subprocesses SIGKILL if they
exceed the memory limit. Refer to #5642 for an example.
--memory-swap is pretty new (newer than 1.3.3), so we don't want to
require it. At the same time, we don't want to impose any memory
limits if we can't use it, because killing subprocesses that exceed a
--memory limit is too strict. This commit arranges to use both
--memory and --memory-swap only if the latter is available.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 294696c..912159a 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -376,7 +376,7 @@ if (!defined $no_clear_tmp) {
# If this job requires a Docker image, install that.
my $docker_bin = "/usr/bin/docker.io";
-my ($docker_locator, $docker_stream, $docker_hash);
+my ($docker_locator, $docker_stream, $docker_hash, $docker_limitmem);
if ($docker_locator = $Job->{docker_image_locator}) {
($docker_stream, $docker_hash) = find_docker_image($docker_locator);
if (!$docker_hash)
@@ -408,6 +408,12 @@ fi
.exit_status_s($?));
}
+ # Determine whether this version of Docker supports memory+swap limits.
+ srun(["srun", "--nodelist=" . $node[0]],
+ ["/bin/sh", "-ec", "$docker_bin run --help | grep -qe --memory-swap="],
+ {fork => 1});
+ $docker_limitmem = ($? == 0);
+
if ($Job->{arvados_sdk_version}) {
# The job also specifies an Arvados SDK version. Add the SDKs to the
# tar file for the build script to install.
@@ -720,14 +726,25 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
."mkdir -p $ENV{CRUNCH_TMP} $ENV{JOB_WORK} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
."&& cd $ENV{CRUNCH_TMP} "
- ."&& MEM=\$(cat /proc/meminfo | grep MemTotal | sed 's/\\s\\s*/ /g' |cut -d' ' -f2) "
- ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($ENV{CRUNCH_NODE_SLOTS} * 100) )) ";
+ # These environment variables get used explicitly later in
+ # $command. No tool is expected to read these values directly.
+ .q{&& MEM=$(awk '($1 == "MemTotal:"){print $2}' </proc/meminfo) }
+ .q{&& SWAP=$(awk '($1 == "SwapTotal:"){print $2}' </proc/meminfo) }
+ ."&& MEMLIMIT=\$(( (\$MEM * 95) / ($ENV{CRUNCH_NODE_SLOTS} * 100) )) "
+ ."&& let SWAPLIMIT=\$MEMLIMIT+\$SWAP ";
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
{
my $cidfile = "$ENV{CRUNCH_TMP}/$Jobstep->{arvados_task}->{uuid}-$Jobstep->{failures}.cid";
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
- $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --attach=stdin -i --user=crunch --cidfile=$cidfile --sig-proxy --memory=\${MEMLIMIT}k ";
+ $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --attach=stdin -i --user=crunch --cidfile=$cidfile --sig-proxy ";
+ # We only set memory limits if Docker lets us limit both memory and swap.
+ # Memory limits alone have been supported longer, but subprocesses tend
+ # to get SIGKILL if they exceed that without any swap limit set.
+ # See #5642 for additional background.
+ if ($docker_limitmem) {
+ $command .= "--memory=\${MEMLIMIT}k --memory-swap=\${SWAPLIMIT}k ";
+ }
# Dynamically configure the container to use the host system as its
# DNS server. Get the host's global addresses from the ip command,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list