[ARVADOS] updated: 555c19acd8547135cb43fdd062c16d7f02344b7c
git at public.curoverse.com
git at public.curoverse.com
Fri Apr 3 16:29:13 EDT 2015
Summary of changes:
docker/build_tools/Makefile | 6 ++++++
docker/cwl-runner/Dockerfile | 13 +++++++++++++
sdk/python/arvados/commands/cwl_job.py | 29 +++++++++++++++++------------
sdk/python/arvados/commands/cwl_runner.py | 11 +++++++++--
sdk/python/arvados/commands/keepdocker.py | 12 ++++++------
sdk/python/gittaggers.py | 11 +++++++----
sdk/python/setup.py | 6 ++++--
7 files changed, 62 insertions(+), 26 deletions(-)
create mode 100644 docker/cwl-runner/Dockerfile
via 555c19acd8547135cb43fdd062c16d7f02344b7c (commit)
from 0ae2871e43fdfad53246a3ce3f25d2ad8e17aa32 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 555c19acd8547135cb43fdd062c16d7f02344b7c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Apr 3 16:32:44 2015 -0400
4685: Putting new job runner all together
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index 621aa04..fd1d639 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -234,6 +234,12 @@ arv-web-image: passenger-image $(ARV_WEB_DEPS)
$(DOCKER_BUILD) -t arvados/arv-web arv-web
date >arv-web-image
+cwl-runner-image: debian-arvados-image
+ rm -rf cwl-runner/sdk
+ tar -czf cwl-runner/generated/sdk.tar.gz -C ../sdk .
+ $(DOCKER_BUILD) -t arvados/cwl-runner cwl-runner
+ date >cwl-runner-image
+
# ============================================================
# The arvados/base image is the base Debian image plus packages
# that are dependencies for every Arvados service.
diff --git a/docker/cwl-runner/Dockerfile b/docker/cwl-runner/Dockerfile
new file mode 100644
index 0000000..6c47624
--- /dev/null
+++ b/docker/cwl-runner/Dockerfile
@@ -0,0 +1,13 @@
+# Arvados cwl runner image
+
+# Based on Debian Wheezy
+FROM arvados/debian:wheezy
+MAINTAINER Ward Vandewege <ward at curoverse.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install -qqy git python-setuptools
+
+ADD generated/sdk.tar.gz /usr/src/arvados/sdk
+
+RUN cd /usr/src/arvados/sdk/python && python setup.py install
diff --git a/sdk/python/arvados/commands/cwl_job.py b/sdk/python/arvados/commands/cwl_job.py
index 73e5ca7..b45bafb 100644
--- a/sdk/python/arvados/commands/cwl_job.py
+++ b/sdk/python/arvados/commands/cwl_job.py
@@ -162,7 +162,7 @@ class TaskEvents(object):
for slot in self.slots:
if self.slots[slot]["task"] is None:
task = self.task_queue[0]
- run_on_slot(self.resources, slot, task, task["uuid"])
+ run_on_slot(self.resources, slot, task["parameters"], task["uuid"])
del self.task_queue[0]
assigned = True
if not assigned:
@@ -180,7 +180,7 @@ class TaskEvents(object):
def cancel_tasks(self):
for slot in self.slots:
- if self.slots[slot]["task"].get("__subprocess") is not None:
+ if self.slots[slot]["task"] and self.slots[slot]["task"].get("__subprocess"):
self.slots[slot]["task"]["__subprocess"].terminate()
self.slots[slot]["task"]["__subprocess"].wait()
@@ -198,17 +198,17 @@ class TaskEvents(object):
if ev["properties"]["new_attributes"].get("success") is not None:
self.finish_task(ev["properties"]["new_attributes"])
-def run_executive(api, job, api_config):
+def run_executive(resources, api, job, api_config):
execution_script = Template(script_header + """
cd $tmpdir
git init
git config --local credential.$githttp/.helper '!tok(){ echo password=$ARVADOS_API_TOKEN; };tok'
git config --local credential.$githttp/.username none
-ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git fetch $githttp/$gitrepo $script_version
+ARVADOS_API_HOST=$ARVADOS_API_HOST ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE git fetch $githttp/$gitrepo
git checkout $script_version
docker run \
- --env=CRUNCH_JOB_UUID=$job_uuid \
+ --env=JOB_UUID=$job_uuid \
--env=ARVADOS_API_HOST=$ARVADOS_API_HOST \
--env=ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN \
--env=ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE \
@@ -222,7 +222,11 @@ docker run \
tmpdir = "/tmp/%s-%i" % (job["uuid"], random.randint(1, 100000))
- docker_hash = image_hash_in_collection(arvados.CollectionReader(job["docker_image_locator"], api_client=api))
+ docker_hash = arvados.commands.keepdocker.image_hash_in_collection(arvados.CollectionReader(job["docker_image_locator"], api_client=api))
+
+ repo = job["repository"]
+ if not repo.endswith(".git"):
+ repo += "/.git"
ex = execution_script.substitute(docker_hash=pipes.quote(docker_hash),
docker_locator=pipes.quote(job["docker_image_locator"]),
@@ -232,9 +236,10 @@ docker run \
ARVADOS_API_HOST_INSECURE=pipes.quote(api_config.get("ARVADOS_API_TOKEN", "0")),
TASK_CANCELED=TASK_CANCELED,
githttp=pipes.quote(api._rootDesc.get("gitHttpBase")),
- gitrepo=pipes.quote(job["repository"]),
+ gitrepo=pipes.quote(repo),
script=pipes.quote(job["script"]),
- script_version=pipes.quote(job["script_version"]))
+ script_version=pipes.quote(job["script_version"]),
+ job_uuid=job["uuid"])
if resources["have_slurm"]:
pass
@@ -292,15 +297,15 @@ def main(arguments=None):
if job_uuid:
ts = TaskEvents(api_config, resources, job_uuid)
- run_executive(api, job, api_config["ARVADOS_API_TOKEN"])
+ run_executive(resources, api, job, api_config)
# Set up signal handling
sig = SigHandler(ts)
# Forward terminate signals to the subprocesses.
- signal.signal(signal.SIGINT, self.sig)
- signal.signal(signal.SIGTERM, self.sig)
- signal.signal(signal.SIGQUIT, self.sig)
+ signal.signal(signal.SIGINT, sig.send_signal)
+ signal.signal(signal.SIGTERM, sig.send_signal)
+ signal.signal(signal.SIGQUIT, sig.send_signal)
ts.ws.run_forever()
else:
diff --git a/sdk/python/arvados/commands/cwl_runner.py b/sdk/python/arvados/commands/cwl_runner.py
index cb2b62f..3f17b60 100644
--- a/sdk/python/arvados/commands/cwl_runner.py
+++ b/sdk/python/arvados/commands/cwl_runner.py
@@ -3,5 +3,12 @@ import cwltool
# Whee!
-def main(arguments):
- return 0
+def main(arguments=None):
+ api = arvados.api('v1')
+ api.job_tasks().create(body={
+ 'job_uuid': os.environ["JOB_UUID"],
+ 'parameters': {
+ "docker_hash": "b0ae8c8d8f58d528530499841bd0bc05968f0443767132619eb3aad59d972c5a",
+ "command": ["ls", "/"],
+ "environment": {}
+ }}).execute()
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 1b01d48..3f943d7 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -239,14 +239,14 @@ def image_hash_in_collection(cr):
def load_image_from_collection(api_client, docker_image_locator):
cr = arvados.CollectionReader(docker_image_locator, api_client=api_client)
- docker_image = image_hash_in_collection(api_client, cr)
+ docker_image = image_hash_in_collection(cr)
if docker_image:
for d in docker_images():
- if d.hash == docker_image.group(1):
- print "Docker image '%s' is already loaded" % docker_image.group(1)
- return docker_image.group(1)
+ if d.hash == docker_image:
+ print "Docker image '%s' is already loaded" % docker_image
+ return docker_image
- with cr.open(docker_image.group(0)) as img:
+ with cr.open(docker_image+".tar") as img:
docker_load = subprocess.Popen(["docker", "load"], stdin=subprocess.PIPE)
data = img.read(64000)
n = len(data)
@@ -259,7 +259,7 @@ def load_image_from_collection(api_client, docker_image_locator):
if docker_load.returncode != 0:
raise arvados.errors.CommandFailedError("Failed to load image")
- return docker_image.group(1)
+ return docker_image
else:
raise arvados.errors.ArgumentError("Failed to find Docker image in collection %s" % docker_image_locator)
diff --git a/sdk/python/gittaggers.py b/sdk/python/gittaggers.py
index 9223443..ba80f60 100644
--- a/sdk/python/gittaggers.py
+++ b/sdk/python/gittaggers.py
@@ -9,10 +9,13 @@ class EggInfoFromGit(egg_info):
from source package), leave it alone.
"""
def git_timestamp_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', '.']).strip()
- return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
+ try:
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
+ except subprocess.CalledProcessError:
+ return time.strftime('.%Y%m%d%H%M%S', time.gmtime(0))
def tags(self):
if self.tag_build is None:
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 7aa6b32..75a23eb 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -33,8 +33,10 @@ setup(name='arvados-python-client',
'bin/arv-normalize',
'bin/arv-put',
'bin/arv-run',
- 'bin/arv-ws'
- ],
+ 'bin/arv-ws',
+ 'bin/arv-cwl-job',
+ 'bin/cwl-runner'
+ ],
install_requires=[
'python-gflags',
'google-api-python-client',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list