[ARVADOS] updated: 222ce386e36b3d146e718a5d2f64a95fb30996bb
git at public.curoverse.com
git at public.curoverse.com
Wed May 7 11:31:59 EDT 2014
Summary of changes:
doc/api/schema/Job.html.textile.liquid | 1 +
docker/build_tools/Makefile | 10 ++++++
docker/jobs/Dockerfile | 20 +++++++++++++
sdk/cli/bin/crunch-job | 49 +++++++++++++++++++++++++++++--
sdk/python/bin/arv-mount | 11 +++----
5 files changed, 81 insertions(+), 10 deletions(-)
create mode 100644 docker/jobs/Dockerfile
via 222ce386e36b3d146e718a5d2f64a95fb30996bb (commit)
via 26d69329e87e343b84a132a4754068aca66ad132 (commit)
via 036a701b45399e0114eed5b0e1ce6b478c614534 (commit)
via b4516edf0a7cd1b584442953a8c965fc65104906 (commit)
from 8fcba3783d9a79bb18a00d339f9b3222f1eccd7f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 222ce386e36b3d146e718a5d2f64a95fb30996bb
Merge: 8fcba37 26d6932
Author: Brett Smith <brett at curoverse.com>
Date: Wed May 7 11:32:18 2014 -0400
Merge branch '2492-docker-crunch-jobs'
Closes #2492.
commit 26d69329e87e343b84a132a4754068aca66ad132
Author: Brett Smith <brett at curoverse.com>
Date: Wed May 7 11:28:16 2014 -0400
crunch: Install Docker images on demand.
Ward requested this to simplify the installation and testing
procedure. This is sort of temporary scaffolding; we expect that in
the future Docker images will be stored in, and read from, Keep.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 515aaa6..14b1be9 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -501,6 +501,27 @@ if (!$have_slurm)
# If this job requires a Docker image, install that.
my $docker_bin = "/usr/bin/docker.io";
my $docker_image = $Job->{runtime_constraints}->{docker_image} || "";
+if ($docker_image) {
+ my $docker_pid = fork();
+ if ($docker_pid == 0)
+ {
+ srun (["srun", "--nodelist=" . join(' ', @node)],
+ [$docker_bin, 'pull', $docker_image]);
+ exit ($?);
+ }
+ while (1)
+ {
+ last if $docker_pid == waitpid (-1, WNOHANG);
+ freeze_if_want_freeze ($docker_pid);
+ select (undef, undef, undef, 0.1);
+ }
+ # If the Docker image was specified as a hash, pull will fail.
+ # Ignore that error. We'll see what happens when we try to run later.
+ if (($? != 0) && ($docker_image !~ /^[0-9a-fA-F]{5,}$/))
+ {
+ croak("Installing Docker image $docker_image returned exit code $?");
+ }
+}
foreach (qw (script script_version script_parameters runtime_constraints))
{
commit 036a701b45399e0114eed5b0e1ce6b478c614534
Author: Brett Smith <brett at curoverse.com>
Date: Wed May 7 11:24:21 2014 -0400
crunch-job: Support runtime Docker image.
This adds support for the allow_other mount option in arv-mount to
make the Keep mount available to the container.
diff --git a/doc/api/schema/Job.html.textile.liquid b/doc/api/schema/Job.html.textile.liquid
index 82f9242..097d0df 100644
--- a/doc/api/schema/Job.html.textile.liquid
+++ b/doc/api/schema/Job.html.textile.liquid
@@ -50,6 +50,7 @@ h3. Runtime constraints
table(table table-bordered table-condensed).
|_. Key|_. Type|_. Description|_. Implemented|
+|docker_image|string|The name of a Docker image that this Job needs to run. If specified, Crunch will create a Docker container from this image, and run the Job's script inside that. The Keep mount and work directories will be available as volumes inside this container. You may specify the image in any format that Docker accepts, such as "arvados/jobs" or a hash identifier. If you specify a name, Crunch will try to install the latest version using @docker.io pull at .|✓|
|min_nodes|integer||✓|
|max_nodes|integer|||
|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓|
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 48a6c9d..515aaa6 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -498,7 +498,9 @@ if (!$have_slurm)
must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
}
-
+# If this job requires a Docker image, install that.
+my $docker_bin = "/usr/bin/docker.io";
+my $docker_image = $Job->{runtime_constraints}->{docker_image} || "";
foreach (qw (script script_version script_parameters runtime_constraints))
{
@@ -603,7 +605,6 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'},
"--job-name=$job_id.$id.$$",
);
- my @execargs = qw(sh);
my $build_script_to_send = "";
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
@@ -615,8 +616,27 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$command .=
"&& perl -";
}
- $command .=
- "&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
+ if ($docker_image)
+ {
+ $command .= "$docker_bin run -i -a stdin -a stdout -a stderr ";
+ # Dynamically configure the container to use the host system as its
+ # DNS server. Get the host's global addresses from the ip command,
+ # and turn them into docker --dns options using gawk.
+ $command .=
+ q{$(ip -o address show scope global |
+ gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
+ foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT))
+ {
+ $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E ";
+ }
+ while (my ($env_key, $env_val) = each %ENV)
+ {
+ $command .= "-e \Q$env_key=$env_val\E ";
+ }
+ $command .= "$docker_image ";
+ }
+ $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
diff --git a/sdk/python/bin/arv-mount b/sdk/python/bin/arv-mount
index 5e773df..cc7e288 100755
--- a/sdk/python/bin/arv-mount
+++ b/sdk/python/bin/arv-mount
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from arvados.fuse import *
+from arvados.fuse import *
import arvados
import subprocess
import argparse
@@ -15,6 +15,8 @@ mountpoint before --exec, or mark the end of your --exec arguments
with "--".
""")
parser.add_argument('mountpoint', type=str, help="""Mount point.""")
+ parser.add_argument('--allow-other', action='store_true',
+ help="""Let other users read the mount""")
parser.add_argument('--collection', type=str, help="""Collection locator""")
parser.add_argument('--debug', action='store_true', help="""Debug mode""")
parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
@@ -35,12 +37,9 @@ with "--".
operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
# FUSE options, see mount.fuse(8)
- opts = []
+ opts = [optname for optname in ['allow_other', 'debug']
+ if getattr(args, optname)]
- # Enable FUSE debugging (logs each FUSE request)
- if args.debug:
- opts += ['debug']
-
# Initialize the fuse connection
llfuse.init(operations, args.mountpoint, opts)
commit b4516edf0a7cd1b584442953a8c965fc65104906
Author: Brett Smith <brett at curoverse.com>
Date: Fri Apr 4 12:00:57 2014 -0400
docker: Build base container for Crunch jobs.
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 9eac2ec..69db746 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -24,6 +24,8 @@ BUILD = build/.buildstamp
BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
+JOBS_DEPS = jobs/Dockerfile
+
API_DEPS = api/Dockerfile $(API_GENERATED)
DOC_DEPS = doc/Dockerfile doc/apache2_vhost
@@ -86,6 +88,10 @@ $(BUILD):
mkdir -p build
rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
--chmod=Da+rx,Fa+rX ../ build/
+ find build/ -name \*.gem -delete
+ cd build/sdk/python/ && ./build.sh
+ cd build/sdk/cli && gem build arvados-cli.gemspec
+ cd build/sdk/ruby && gem build arvados.gemspec
touch build/.buildstamp
$(BASE_GENERATED): config.yml $(BUILD)
@@ -125,6 +131,10 @@ doc-image: base-image $(BUILD) $(DOC_DEPS)
$(DOCKER_BUILD) -t arvados/doc doc
date >doc-image
+jobs-image: base-image $(BUILD) $(JOBS_DEPS)
+ $(DOCKER_BUILD) -t arvados/jobs jobs
+ date >jobs-image
+
workbench-image: passenger-image $(BUILD) $(WORKBENCH_DEPS)
mkdir -p workbench/generated
tar -czf workbench/generated/workbench.tar.gz -C build/apps workbench
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
new file mode 100644
index 0000000..28ef858
--- /dev/null
+++ b/docker/jobs/Dockerfile
@@ -0,0 +1,20 @@
+FROM arvados/base
+MAINTAINER Brett Smith <brett at curoverse.com>
+
+# Install dependencies and set up system.
+# The FUSE packages help ensure that we can install the Python SDK (arv-mount).
+RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
+ libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl \
+ fuse libattr1-dev libfuse-dev && \
+ /usr/sbin/adduser --disabled-password \
+ --gecos 'Crunch execution user' crunch && \
+ /usr/bin/install -d -o crunch -g crunch -m 0700 /tmp/crunch-job && \
+ /bin/ln -s /usr/src/arvados /usr/local/src/arvados
+
+# Install Arvados packages.
+RUN find /usr/src/arvados/sdk -name '*.gem' -print0 | \
+ xargs -0rn 1 gem install && \
+ cd /usr/src/arvados/sdk/python && \
+ python setup.py install
+
+USER crunch
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list