[ARVADOS] created: dfc93aac9c256d6ebb868aeb6c2107821e9fd041
Git user
git at public.curoverse.com
Tue Mar 29 09:48:02 EDT 2016
at dfc93aac9c256d6ebb868aeb6c2107821e9fd041 (commit)
commit dfc93aac9c256d6ebb868aeb6c2107821e9fd041
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Mar 29 09:38:16 2016 -0400
8815: Rely on system-provided crunchrunner. Also use arvados/jobs by default if no docker provided.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index ab0e48d..e3fd1fc 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -26,10 +26,6 @@ from arvados.api import OrderedJsonModel
logger = logging.getLogger('arvados.cwl-runner')
logger.setLevel(logging.INFO)
-crunchrunner_pdh = "ff6fc71e593081ef9733afacaeee15ea+140"
-crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
-certs_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/ca-certificates.crt"
-
tmpdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.tmpdir\)=(.*)")
outdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.outdir\)=(.*)")
keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.keep\)=(.*)")
@@ -152,6 +148,8 @@ class ArvadosJob(object):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if docker_req and kwargs.get("use_container") is not False:
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
+ else:
+ runtime_constraints["docker_image"] = "arvados/jobs"
resources = self.builder.resources
if resources is not None:
@@ -166,7 +164,7 @@ class ArvadosJob(object):
"repository": "arvados",
"script_version": "master",
"minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6",
- "script_parameters": {"tasks": [script_parameters], "crunchrunner": crunchrunner_pdh+"/crunchrunner"},
+ "script_parameters": {"tasks": [script_parameters]},
"runtime_constraints": runtime_constraints
}, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
@@ -398,22 +396,6 @@ class ArvCwlRunner(object):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
self.debug = args.debug
-
- try:
- self.api.collections().get(uuid=crunchrunner_pdh).execute()
- except arvados.errors.ApiError as e:
- import httplib2
- h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
- resp, content = h.request(crunchrunner_download, "GET")
- resp2, content2 = h.request(certs_download, "GET")
- with arvados.collection.Collection() as col:
- with col.open("crunchrunner", "w") as f:
- f.write(content)
- with col.open("ca-certificates.crt", "w") as f:
- f.write(content2)
-
- col.save_new("crunchrunner binary", ensure_unique_name=True)
-
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
commit 4727a3b6417bd254b3e8bca1bd19a77773493572
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Mar 29 09:23:51 2016 -0400
8815: Crunch-job bind mounts crunchrunner binary and certificates from host.
Updated arvbox to compile and install crunchrunner.
diff --git a/crunch_scripts/crunchrunner b/crunch_scripts/crunchrunner
index 71c10c9..f192c48 100755
--- a/crunch_scripts/crunchrunner
+++ b/crunch_scripts/crunchrunner
@@ -1,2 +1,2 @@
#!/bin/sh
-exec $TASK_KEEPMOUNT/$JOB_PARAMETER_CRUNCHRUNNER
+exec /usr/lib/crunchrunner/crunchrunner
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 7c2855c..88f9d56 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -109,6 +109,9 @@ unless (defined $ENV{"CRUNCH_TMP"}) {
}
}
+$ENV{"HOST_CRUNCHRUNNER_BIN"} ||= `which crunchrunner`;
+$ENV{"HOST_CERTS"} ||= "/etc/ssl/certs/ca-certificates.crt";
+
# Create the tmp directory if it does not exist
if ( ! -d $ENV{"CRUNCH_TMP"} ) {
make_path $ENV{"CRUNCH_TMP"} or die "Failed to create temporary working directory: " . $ENV{"CRUNCH_TMP"};
@@ -917,6 +920,11 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
# For now, use the same approach as TASK_WORK above.
$ENV{"JOB_WORK"} = "/tmp/crunch-job-work";
+ # Bind mount the crunchrunner binary and host TLS certificates file into
+ # the container.
+ $command .= "--volume=\Q$ENV{HOST_CRUNCHRUNNER_BIN}:/usr/lib/crunchrunner/crunchrunner\E ";
+ $command .= "--volume=\Q$ENV{HOST_CERTS}:/usr/lib/crunchrunner/ca-certificates.crt\E ";
+
while (my ($env_key, $env_val) = each %ENV)
{
if ($env_key =~ /^(ARVADOS|CRUNCH|JOB|TASK)_/) {
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 88726a4..e21db0d 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -39,10 +39,6 @@ if test -z "$ARVADOS_ROOT" ; then
ARVADOS_ROOT="$ARVBOX_DATA/arvados"
fi
-if test -z "$ARVADOS_DEV_ROOT" ; then
- ARVADOS_DEV_ROOT="$ARVBOX_DATA/arvados-dev"
-fi
-
if test -z "$SSO_ROOT" ; then
SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider"
fi
@@ -165,10 +161,6 @@ run() {
if test "$1" = test ; then
shift
- if ! test -d "$ARVADOS_DEV_ROOT" ; then
- git clone https://github.com/curoverse/arvados-dev.git "$ARVADOS_DEV_ROOT"
- fi
-
mkdir -p $VAR_DATA/test
docker run \
@@ -176,7 +168,6 @@ run() {
--name=$ARVBOX_CONTAINER \
--privileged \
"--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
- "--volume=$ARVADOS_DEV_ROOT:/usr/src/arvados-dev:rw" \
"--volume=$SSO_ROOT:/usr/src/sso:rw" \
"--volume=$PG_DATA:/var/lib/postgresql:rw" \
"--volume=$VAR_DATA:/var/lib/arvados:rw" \
@@ -203,7 +194,7 @@ run() {
docker exec -ti \
$ARVBOX_CONTAINER \
/usr/local/lib/arvbox/runsu.sh \
- /usr/src/arvados-dev/jenkins/run-tests.sh \
+ /usr/src/arvados/build/run-tests.sh \
--temp /var/lib/arvados/test \
WORKSPACE=/usr/src/arvados \
GEM_HOME=/var/lib/gems \
diff --git a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
index 178fec1..a3397c1 100755
--- a/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/crunch-setup.sh
@@ -12,7 +12,8 @@ export GOPATH=$PWD
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "/usr/src/arvados" "$GOPATH/src/git.curoverse.com/arvados.git"
flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/services/crunchstat"
-install bin/crunchstat /usr/local/bin
+flock /var/lib/arvados/gostuff.lock go get -t "git.curoverse.com/arvados.git/sdk/go/crunchrunner"
+install bin/crunchstat bin/crunchrunner /usr/local/bin
export ARVADOS_API_HOST=$localip:${services[api]}
export ARVADOS_API_HOST_INSECURE=1
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list