[ARVADOS] updated: 945bebae56ec2fb5faef965cbf55fc8309e8df11
git at public.curoverse.com
git at public.curoverse.com
Wed Jul 2 10:55:22 EDT 2014
Summary of changes:
docker/bcbio-nextgen/Dockerfile | 20 ++++++++++----------
docker/jobs/Dockerfile | 6 +++---
sdk/cli/bin/crunch-job | 26 +++++++++++++-------------
3 files changed, 26 insertions(+), 26 deletions(-)
via 945bebae56ec2fb5faef965cbf55fc8309e8df11 (commit)
via 2d7610520e4d1ab289899821bf4d885b77a1e327 (commit)
from 497851073641c8486ece1a0e5066aeca6e51f7a8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 945bebae56ec2fb5faef965cbf55fc8309e8df11
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jul 2 10:54:48 2014 -0400
Use separate $HOME for commands run outside the docker
container (e.g. arv-mount) and commands inside the docker container because of
the user id mapping problem. Converted docker command line to use long form
for readability.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index b0d779b..dd1ba8f 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -622,7 +622,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
$ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/$id.$$";
- $ENV{"HOME"} = $ENV{"TASK_WORK"};
+ $ENV{"HOME"} = $ENV{"TASK_WORK"}.".home";
$ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}.".keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
$ENV{"CRUNCH_NODE_SLOTS"} = $slot[$childslot]->{node}->{ncpus};
@@ -639,7 +639,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
my $build_script_to_send = "";
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
- ."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
+ ."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} ${HOME}"
."&& chmod og+wrx $ENV{TASK_WORK}"
."&& cd $ENV{CRUNCH_TMP} ";
if ($build_script)
@@ -652,41 +652,41 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
if ($docker_hash)
{
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
- $command .= "$docker_bin run -i -a stdin -a stdout -a stderr --cidfile=$ENV{TASK_WORK}/docker.cid ";
+ $command .= "$docker_bin run --attach=stdout --attach=stderr --user=crunch --cidfile=$ENV{TASK_WORK}/docker.cid ";
# Dynamically configure the container to use the host system as its
# DNS server. Get the host's global addresses from the ip command,
# and turn them into docker --dns options using gawk.
$command .=
q{$(ip -o address show scope global |
gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
- $command .= "-v \Q$ENV{TASK_WORK}:/tmp/crunch-job:rw\E ";
- $command .= "-v \Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
- $command .= "-v \Q$ENV{TASK_KEEPMOUNT}:/mnt:ro\E ";
- $command .= "-e \QHOME=/tmp/crunch-job\E ";
+ $command .= "--volume=\Q$ENV{TASK_WORK}:/tmp/crunch-job:rw\E ";
+ $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
+ $command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
+ $command .= "--env=\QHOME=/home/crunch\E ";
while (my ($env_key, $env_val) = each %ENV)
{
if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
if ($env_key eq "TASK_WORK") {
- $command .= "-e \QTASK_WORK=/tmp/crunch-job\E ";
+ $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
}
elsif ($env_key eq "TASK_KEEPMOUNT") {
- $command .= "-e \QTASK_KEEPMOUNT=/mnt\E ";
+ $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
}
elsif ($env_key eq "CRUNCH_SRC") {
- $command .= "-e \QCRUNCH_SRC=/tmp/crunch-src\E ";
+ $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
}
else {
- $command .= "-e \Q$env_key=$env_val\E ";
+ $command .= "--env=\Q$env_key=$env_val\E ";
}
}
}
$command .= "\Q$docker_hash\E ";
- $command .= "stdbuf -o0 -e0 ";
+ $command .= "stdbuf --output=0 --error=0 ";
$command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
} else {
# Non-docker run
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
- $command .= "stdbuf -o0 -e0 ";
+ $command .= "stdbuf --output=0 --error=0 ";
$command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
}
commit 2d7610520e4d1ab289899821bf4d885b77a1e327
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jul 2 10:52:41 2014 -0400
Use 'install' to create /keep, /tmp/crunch-src, /tmp/crunch-job mountpoints in
the docker image. Changed some commands to use long command line parameters
for readability.
diff --git a/docker/bcbio-nextgen/Dockerfile b/docker/bcbio-nextgen/Dockerfile
index 436aa9c..e3e705b 100644
--- a/docker/bcbio-nextgen/Dockerfile
+++ b/docker/bcbio-nextgen/Dockerfile
@@ -3,28 +3,28 @@ MAINTAINER Peter Amstutz <peter.amstutz at curoverse.com>
USER root
-RUN apt-get remove -q -y ruby && \
+ADD generated/arvados.tar.gz /usr/src/arvados/
+ENV GEM_HOME /usr/local/rvm/gems/ruby-2.1.0
+ENV GEM_PATH /usr/local/rvm/gems/ruby-2.1.0:/usr/local/rvm/gems/ruby-2.1.0 at global
+ENV PATH /usr/local/rvm/gems/ruby-2.1.0/bin:/usr/local/rvm/gems/ruby-2.1.0 at global/bin:/usr/local/rvm/rubies/ruby-2.1.0/bin:/usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Install Ruby 2.1.0
+RUN apt-get remove --quiet --assume-yes ruby && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.1.0 && \
/bin/mkdir -p /usr/src/arvados
-ADD generated/arvados.tar.gz /usr/src/arvados/
-
# Install dependencies and set up system.
# The FUSE packages help ensure that we can install the Python SDK (arv-mount).
-RUN /usr/bin/apt-get update && \
- /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
+RUN /usr/bin/apt-get update && \
+ /usr/bin/apt-get install --quiet --assume-yes python-dev python-llfuse python-pip \
libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl \
fuse libattr1-dev libfuse-dev && \
/usr/sbin/adduser --disabled-password \
--gecos 'Crunch execution user' crunch && \
- /usr/bin/install -d -o crunch -g crunch -m 0700 /tmp/crunch-job && \
+ /usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job && \
/bin/ln -s /usr/src/arvados /usr/local/src/arvados
-ENV GEM_HOME /usr/local/rvm/gems/ruby-2.1.0
-ENV GEM_PATH /usr/local/rvm/gems/ruby-2.1.0:/usr/local/rvm/gems/ruby-2.1.0 at global
-ENV PATH /usr/local/rvm/gems/ruby-2.1.0/bin:/usr/local/rvm/gems/ruby-2.1.0 at global/bin:/usr/local/rvm/rubies/ruby-2.1.0/bin:/usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
# Install Arvados packages.
RUN gem update --system && \
find /usr/src/arvados/sdk -name '*.gem' -print0 | \
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 29c9d54..2cad65c 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -8,12 +8,12 @@ RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
fuse libattr1-dev libfuse-dev && \
/usr/sbin/adduser --disabled-password \
--gecos 'Crunch execution user' crunch && \
- /usr/bin/install -d -o crunch -g crunch -m 0700 /tmp/crunch-job && \
+ /usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job && \
/bin/ln -s /usr/src/arvados /usr/local/src/arvados
# Install Arvados packages.
-RUN find /usr/src/arvados/sdk -name '*.gem' -print0 | \
- xargs -0rn 1 gem install && \
+RUN (find /usr/src/arvados/sdk -name '*.gem' -print0 | \
+ xargs -0rn 1 gem install) && \
cd /usr/src/arvados/services/fuse && \
python setup.py install && \
cd /usr/src/arvados/sdk/python && \
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list