[ARVADOS] updated: 525e930dcb35b440424d53b09a9788fd8726b781

[Git user]

Summary of changes:
 crunch_scripts/cwl-runner          |  2 --
 sdk/cwl/arvados_cwl/__init__.py    | 28 ++++++++++++++++++++++------
 sdk/python/arvados/commands/run.py |  6 ++++++
 3 files changed, 28 insertions(+), 8 deletions(-)

       via  525e930dcb35b440424d53b09a9788fd8726b781 (commit)
       via  48a323ca58614b7e2e100fd69fdc8a441d5d0c80 (commit)
      from  a8af246b8c99b070fc445cda3d3a13bdf0885f24 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 525e930dcb35b440424d53b09a9788fd8726b781
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Mar 25 16:35:10 2016 -0400

    8654: Passes 100% CWL conformance tests using running cwl-runner in a crunch job!

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index eeb9d78..d4d61d4 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -18,8 +18,6 @@ api = arvados.api("v1")
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
-    print job_order_object
-
     def keeppath(v):
         if arvados.util.keep_locator_pattern.match(v):
             return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 61c065f..eab5d4a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -372,9 +372,12 @@ class RunnerJob(object):
 
         outputs = None
         try:
-            outc = arvados.collection.Collection(record["output"])
-            with outc.open("cwl.output.json") as f:
-                outputs = json.load(f)
+            try:
+                outc = arvados.collection.Collection(record["output"])
+                with outc.open("cwl.output.json") as f:
+                    outputs = json.load(f)
+            except Exception as e:
+                logger.error("While getting final output object: %s", e)
             self.arvrunner.output_callback(outputs, processStatus)
         finally:
             del self.arvrunner.jobs[record["uuid"]]
@@ -383,20 +386,22 @@ class ArvPathMapper(cwltool.pathmapper.PathMapper):
     def __init__(self, arvrunner, referenced_files, basedir,
                  collection_pattern, file_pattern, name=None, **kwargs):
         self._pathmap = arvrunner.get_uploaded()
-        uploadfiles = []
+        uploadfiles = set()
 
         pdh_path = re.compile(r'^keep:[0-9a-f]{32}\+\d+/.+')
 
         for src in referenced_files:
             if isinstance(src, basestring) and pdh_path.match(src):
                 self._pathmap[src] = (src, collection_pattern % src[5:])
+            if "#" in src:
+                src = src[:src.index("#")]
             if src not in self._pathmap:
                 ab = cwltool.pathmapper.abspath(src, basedir)
                 st = arvados.commands.run.statfile("", ab, fnPattern=file_pattern)
                 if kwargs.get("conformance_test"):
                     self._pathmap[src] = (src, ab)
                 elif isinstance(st, arvados.commands.run.UploadFile):
-                    uploadfiles.append((src, ab, st))
+                    uploadfiles.add((src, ab, st))
                 elif isinstance(st, arvados.commands.run.ArvFile):
                     self._pathmap[src] = (ab, st.fn)
                 else:
@@ -609,7 +614,7 @@ def versionstring():
     arvpkg = pkg_resources.require("arvados-python-client")
     arvcwlpkg = pkg_resources.require("arvados-cwl-runner")
 
-    return "%s\n%s %s\n%s %s\n%s %s" % (sys.argv[0],
+    return "%s %s, %s %s, %s %s" % (sys.argv[0],
                                         "arvados-cwl-runner", arvcwlpkg[0].version,
                                         "arvados-python-client", cwlpkg[0].version,
                                         "cwltool", arvpkg[0].version)
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index ad442d5..7a298b4 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -34,6 +34,12 @@ class ArvFile(object):
         self.prefix = prefix
         self.fn = fn
 
+    def __hash__(self):
+        return (self.prefix+self.fn).__hash__()
+
+    def __eq__(self, other):
+        return (self.prefix == other.prefix) and (self.fn == other.fn)
+
 class UploadFile(ArvFile):
     pass
 

commit 48a323ca58614b7e2e100fd69fdc8a441d5d0c80
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Mar 25 13:54:29 2016 -0400

    8654: --version reports versions for arvados-cwl-runner, arvados-python-client,
    and cwltool.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 2077f21..61c065f 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -21,6 +21,7 @@ import os
 import sys
 import functools
 import json
+import pkg_resources  # part of setuptools
 
 from cwltool.process import get_feature, adjustFiles, scandeps
 from arvados.api import OrderedJsonModel
@@ -603,6 +604,15 @@ class ArvCwlRunner(object):
 
             return self.final_output
 
+def versionstring():
+    cwlpkg = pkg_resources.require("cwltool")
+    arvpkg = pkg_resources.require("arvados-python-client")
+    arvcwlpkg = pkg_resources.require("arvados-cwl-runner")
+
+    return "%s\n%s %s\n%s %s\n%s %s" % (sys.argv[0],
+                                        "arvados-cwl-runner", arvcwlpkg[0].version,
+                                        "arvados-python-client", cwlpkg[0].version,
+                                        "cwltool", arvpkg[0].version)
 
 def main(args, stdout, stderr, api_client=None):
     args.insert(0, "--leave-outputs")
@@ -633,4 +643,5 @@ def main(args, stdout, stderr, api_client=None):
                              stderr=stderr,
                              executor=runner.arvExecutor,
                              makeTool=runner.arvMakeTool,
-                             parser=parser)
+                             parser=parser,
+                             versionfunc=versionstring)

-----------------------------------------------------------------------


hooks/post-receive
--