[ARVADOS] updated: defc3f65bfeed18d4ca0beb0e2a3aabd0db6cc75
Git user
git at public.curoverse.com
Sat Mar 12 08:51:30 EST 2016
Summary of changes:
crunch_scripts/cwl-runner | 38 +++++++-
docker/jobs/Dockerfile | 4 +-
sdk/cwl/arvados_cwl/__init__.py | 107 +++++++++++----------
sdk/cwl/setup.py | 2 +-
.../arvados/v1/repositories_controller.rb | 4 +-
.../arvados/v1/repositories_controller_test.rb | 4 +-
services/arv-git-httpd/gitolite_test.go | 14 ++-
services/datamanager/datamanager.go | 2 +-
services/datamanager/datamanager_test.go | 26 ++++-
services/nodemanager/arvnodeman/baseactor.py | 85 ++++++++++++++++
services/nodemanager/arvnodeman/clientactor.py | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 7 +-
services/nodemanager/arvnodeman/config.py | 4 +-
services/nodemanager/arvnodeman/daemon.py | 21 ++--
services/nodemanager/arvnodeman/fullstopactor.py | 17 ----
services/nodemanager/arvnodeman/launcher.py | 12 +--
services/nodemanager/arvnodeman/timedcallback.py | 2 +-
services/nodemanager/tests/test_daemon.py | 1 +
services/nodemanager/tests/test_failure.py | 29 ++----
services/nodemanager/tests/testutil.py | 5 +-
20 files changed, 257 insertions(+), 129 deletions(-)
create mode 100644 services/nodemanager/arvnodeman/baseactor.py
delete mode 100644 services/nodemanager/arvnodeman/fullstopactor.py
via defc3f65bfeed18d4ca0beb0e2a3aabd0db6cc75 (commit)
via a87af13d2d3461470b89b0811629a077a70c1938 (commit)
via 846a2728f3cfc80efc220a60536dea16b5f0fcbd (commit)
via 83cafb38e512275be28b1bdc1ac439adf8d9f553 (commit)
via 42d4746b7e446811c98c518db03a986a3a499ae1 (commit)
via 9032247f9281283eeebb68e4cb5bf586236ba4bd (commit)
via a59cf490d81c559b658dab9012a1f8d4327ca354 (commit)
via 071cbfb5b1b9328d0db5d4e4c07fd0c8d604c39e (commit)
via 648aaa04080e11b0a793d6100260770b878b24c0 (commit)
via c6df16d2af30e989bcfb04f6ef730cde658a9dc9 (commit)
via d54cd5298bb6e043205995c6e5d414a841d9c389 (commit)
via 2dbbaaefc6a4a46a7f17b9e7799fc455cd722113 (commit)
via 054f95044461c08fd5fb6cd983d1e8ea1dc62ea8 (commit)
via f6aee8a6a829c60015506d89a4e87eb9dc96a07a (commit)
via e5c99ebf68f31d630f2a35f7e4e79e93143a3607 (commit)
via e5ee153a56578d13a025cde47fd0c07e21fd975f (commit)
via 0e97dd2d9fe31a6c1cf73471e6e5ca1f33500850 (commit)
from bdfd1e9f8bc2a3933f6b0bc7afd5cfebd815fd90 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit defc3f65bfeed18d4ca0beb0e2a3aabd0db6cc75
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 11 17:35:09 2016 -0500
8654: Collect output files into output collection, use as output for task.
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 59f6817..d042426 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -9,7 +9,9 @@ import cwltool.main
import logging
import os
import json
+import argparse
from arvados.api import OrderedJsonModel
+from cwltool.process import adjustFiles
api = arvados.api("v1")
@@ -26,15 +28,40 @@ try:
t = cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True)
- outputObj = runner.arvExecutor(t, job_order_object, "", None)
+ np = argparse.Namespace()
+ np.project_uuid = arvados.current_job()["owner_uuid"]
+ np.enable_reuse = True
+ outputObj = runner.arvExecutor(t, job_order_object, "", np)
- c = arvados.collection.Collection()
- with c.open("cwl.output.json", "w") as f:
+ files = {}
+ def capture(path):
+ sp = path.split("/")
+ col = sp[0][5:]
+ if col not in files:
+ files[col] = set()
+ files[col].add("/".join(sp[1:]))
+ return path
+
+ adjustFiles(outputObj, capture)
+
+ final = arvados.collection.Collection()
+
+ for k,v in files.iteritems():
+ with arvados.collection.Collection(k) as c:
+ for f in c:
+ final.copy(f, f, c, True)
+
+ def makeRelative(path):
+ return "/".join(path.split("/")[1:])
+
+ adjustFiles(outputObj, makeRelative)
+
+ with final.open("cwl.output.json", "w") as f:
json.dump(outputObj, f, indent=4)
api.job_tasks().update(uuid=arvados.current_task()['uuid'],
body={
- 'output': c.save_new(create_collection_record=False),
+ 'output': final.save_new(create_collection_record=False),
'success': True,
'progress':1.0
}).execute()
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 1ba9537..f69591b 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -365,8 +365,7 @@ class ArvCwlRunner(object):
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
- if args:
- kwargs["enable_reuse"] = args.enable_reuse
+ kwargs["enable_reuse"] = args.enable_reuse
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
commit a87af13d2d3461470b89b0811629a077a70c1938
Merge: 846a272 83cafb3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 11 16:55:23 2016 -0500
Merge branch '8673-cwl-runner-project-uuid' into 8654-arv-jobs-cwl-runner
commit 846a2728f3cfc80efc220a60536dea16b5f0fcbd
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 11 16:54:59 2016 -0500
8654: Passes in local testing.
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 8cbf4d8..59f6817 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -8,6 +8,7 @@ from cwltool.process import shortname
import cwltool.main
import logging
import os
+import json
from arvados.api import OrderedJsonModel
api = arvados.api("v1")
@@ -28,8 +29,8 @@ try:
outputObj = runner.arvExecutor(t, job_order_object, "", None)
c = arvados.collection.Collection()
- with c.open("cwl.output.json") as f:
- json.dump(f, indent=4)
+ with c.open("cwl.output.json", "w") as f:
+ json.dump(outputObj, f, indent=4)
api.job_tasks().update(uuid=arvados.current_task()['uuid'],
body={
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 8366349..bbe7844 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -6,10 +6,8 @@ ENV DEBIAN_FRONTEND noninteractive
ADD apt.arvados.org.list /etc/apt/sources.list.d/
RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
-RUN apt-get update -q
-RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libcurl4-gnutls-dev nodejs
-RUN pip install arvados-cwl-runner
+RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8854971..67a324a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -362,7 +362,8 @@ class ArvCwlRunner(object):
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
- kwargs["enable_reuse"] = args.enable_reuse
+ if args:
+ kwargs["enable_reuse"] = args.enable_reuse
kwargs["outdir"] = "$(task.outdir)"
kwargs["tmpdir"] = "$(task.tmpdir)"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list