[ARVADOS] created: 22c51cdf55e895daf25fd8f8bced113f0b2be45e
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 23 13:19:46 EDT 2014
at 22c51cdf55e895daf25fd8f8bced113f0b2be45e (commit)
commit 22c51cdf55e895daf25fd8f8bced113f0b2be45e
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Oct 23 13:19:31 2014 -0400
3824: Use a docker volume for TASK_WORK, clean up env var and tmp dir handling, improve logging.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 9bad7ae..731dc91 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -357,7 +357,7 @@ if (!defined $no_clear_tmp) {
if ($cleanpid == 0)
{
srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
+ ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep $CRUNCH_TMP/task/*.keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src*']);
exit (1);
}
while (1)
@@ -547,12 +547,10 @@ else {
my @execargs = ("sh", "-c",
"mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
- # Note: this section is almost certainly unnecessary if we're
- # running tasks in docker containers.
my $installpid = fork();
if ($installpid == 0)
{
- srun (\@srunargs, \@execargs, {}, $build_script . $git_archive);
+ srun (undef, \@srunargs, \@execargs, {}, $build_script . $git_archive);
exit (1);
}
while (1)
@@ -588,7 +586,8 @@ fi
my $docker_pid = fork();
if ($docker_pid == 0)
{
- srun (["srun", "--nodelist=" . join(',', @node)],
+ srun (undef,
+ ["srun", "--nodelist=" . join(',', @node)],
["/bin/sh", "-ec", $docker_install_script]);
exit ($?);
}
@@ -694,7 +693,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
}
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
- $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/$id.$$";
+ $ENV{"TASK_WORK"} = $ENV{"CRUNCH_TMP"}."/task/$childslotname";
$ENV{"HOME"} = $ENV{"TASK_WORK"};
$ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}.".keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
@@ -723,33 +722,48 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
{
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
- $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --user=crunch --cidfile=$ENV{TASK_WORK}/docker.cid ";
+ my $cidfile = "$ENV{CRUNCH_TMP}/$ENV{TASK_UUID}.cid";
+ $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
+ $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --attach=stdin -i --user=crunch --cidfile=$cidfile ";
+
# Dynamically configure the container to use the host system as its
# DNS server. Get the host's global addresses from the ip command,
# and turn them into docker --dns options using gawk.
$command .=
q{$(ip -o address show scope global |
gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
- $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
+
+ # The source tree and $destdir directory (which we have installed
+ $command .= "--volume=\Q$ENV{CRUNCH_SRC}:$ENV{CRUNCH_SRC}:ro\E ";
+ $command .= "--volume=\Q$ENV{CRUNCH_INSTALL}:$ENV{CRUNCH_INSTALL}:ro\E ";
+
+ # For some reason we make arv-mount's mount point appear at
+ # /keep inside the container, instead of using the same path as
+ # the host and expecting the task to pay attention to
+ # $TASK_KEEPMOUNT like we do with everything else.
$command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
- $command .= "--env=\QHOME=/home/crunch\E ";
+ $ENV{TASK_KEEPMOUNT} = "/keep";
+
+ # TASK_WORK is a plain docker data volume: it starts out empty,
+ # is writable, and persists until no containers use it any
+ # more. We don't use --volumes-from to share it with other
+ # containers: it is only accessible to this task, and it goes
+ # away when this task stops.
+ $command .= "--volume=\Q$ENV{TASK_WORK}\E ";
+
+ # JOB_WORK is also a plain docker data volume for now. TODO:
+ # Share a single JOB_WORK volume across all task containers on a
+ # given worker node, and delete it when the job ends (and, in
+ # case that doesn't work, when the next job starts).
+ $command .= "--volume=\Q$ENV{JOB_WORK}\E ";
+
while (my ($env_key, $env_val) = each %ENV)
{
- if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
- if ($env_key eq "TASK_WORK") {
- $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
- }
- elsif ($env_key eq "TASK_KEEPMOUNT") {
- $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
- }
- else {
- $command .= "--env=\Q$env_key=$env_val\E ";
- }
+ if ($env_key =~ /^(ARVADOS|CRUNCH|JOB|TASK)_/) {
+ $command .= "--env=\Q$env_key=$env_val\E ";
}
}
- $command .= "--env=\QCRUNCH_NODE_SLOTS=$ENV{CRUNCH_NODE_SLOTS}\E ";
- $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
+ $command .= "--env=\QHOME=$ENV{HOME}\E ";
$command .= "\Q$docker_hash\E ";
$command .= "stdbuf --output=0 --error=0 ";
$command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
@@ -761,10 +775,9 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
}
my @execargs = ('bash', '-c', $command);
- srun (\@srunargs, \@execargs, undef, $build_script_to_send);
+ srun ($id, \@srunargs, \@execargs, undef, $build_script_to_send);
# exec() failed, we assume nothing happened.
- Log(undef, "srun() failed on build script");
- die;
+ die "srun() failed on build script\n";
}
close("writer");
if (!defined $childpid)
@@ -1555,11 +1568,11 @@ sub srun
my $opts = shift || {};
my $stdin = shift;
my $args = $have_slurm ? [@$srunargs, @$execargs] : $execargs;
- print STDERR (join (" ",
- map { / / ? "'$_'" : $_ }
- (@$args)),
- "\n")
- if $ENV{CRUNCH_DEBUG};
+ warn ("starting: " .
+ join (" ", map {
+ /^(.*TOKEN=)/ ? "\Q${1}XXXXX\E" : "\Q$_\E";
+ } @$args) .
+ "\n");
if (defined $stdin) {
my $child = open STDIN, "-|";
@@ -1692,7 +1705,7 @@ __DATA__
# checkout-and-build
use Fcntl ':flock';
-use File::Path qw( make_path );
+use File::Path qw( make_path remove_tree );
my $destdir = $ENV{"CRUNCH_SRC"};
my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
@@ -1700,12 +1713,18 @@ my $repo = $ENV{"CRUNCH_SRC_URL"};
my $task_work = $ENV{"TASK_WORK"};
for my $dir ($destdir, $task_work) {
- if ($dir) {
- make_path $dir;
- -e $dir or die "Failed to create temporary directory ($dir): $!";
- }
+ if ($dir) {
+ make_path $dir;
+ -e $dir or die "Failed to create temporary directory ($dir): $!";
+ }
}
+if ($task_work) {
+ remove_tree($task_work, {keep_root => 1});
+ chmod 01777, $task_work;
+}
+
+
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
flock L, LOCK_EX;
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
commit d501ef45757f78f5c1d2843eccf8016e8978345e
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Oct 22 10:43:38 2014 -0400
3824: Show stdout/stderr if shell cmd fails in install script.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 369bc3e..9bad7ae 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1718,6 +1718,7 @@ if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
}
unlink "$destdir.commit";
+open STDERR_ORIG, ">&STDERR";
open STDOUT, ">", "$destdir.log";
open STDERR, ">&STDOUT";
@@ -1772,8 +1773,12 @@ sub shell_or_die
if ($ENV{"DEBUG"}) {
print STDERR "@_\n";
}
- system (@_) == 0
- or die "@_ failed: $! exit 0x".sprintf("%x",$?);
+ if (system (@_) != 0) {
+ my $exitstatus = sprintf("exit %d signal %d", $? >> 8, $? & 0x7f);
+ open STDERR, ">&STDERR_ORIG";
+ system ("cat $destdir.log >&2");
+ die "@_ failed ($!): $exitstatus";
+ }
}
__DATA__
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list