[ARVADOS] created: c1429bc60d45525469ee9f73d5215814f405d8d7

[Git user]

        at  c1429bc60d45525469ee9f73d5215814f405d8d7 (commit)


commit c1429bc60d45525469ee9f73d5215814f405d8d7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Sep 22 22:38:06 2016 -0400

    10088: Updated cwltool deduplicates dependencies to avoid creating separate
    collections for subdirectories.  Don't try to update file literals that arn't
    in scandeps.

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 40a64ff..8371554 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -6,7 +6,7 @@ import os
 import arvados.commands.run
 import arvados.collection
 
-from cwltool.pathmapper import PathMapper, MapperEnt, abspath
+from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
 from cwltool.workflow import WorkflowException
 
 logger = logging.getLogger('arvados.cwl-runner')
@@ -42,8 +42,11 @@ class ArvPathMapper(PathMapper):
                     uploadfiles.add((src, ab, st))
                 elif isinstance(st, arvados.commands.run.ArvFile):
                     self._pathmap[src] = MapperEnt(ab, st.fn, "File")
-                elif src.startswith("_:") and "contents" in srcobj:
-                    pass
+                elif src.startswith("_:"):
+                    if "contents" in srcobj:
+                        pass
+                    else:
+                        raise WorkflowException("File literal '%s' is missing contents" % src)
                 else:
                     raise WorkflowException("Input file path '%s' is invalid" % st)
             if "secondaryFiles" in srcobj:
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 155c6a0..bdd8077 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -69,25 +69,19 @@ def upload_dependencies(arvrunner, name, document_loader,
                   set(("$include", "$schemas", "location")),
                   loadref)
 
-    files = []
-    def visitFiles(path):
-        files.append(path)
-
-    adjustFileObjs(sc, visitFiles)
-    adjustDirObjs(sc, visitFiles)
-
-    normalizeFilesDirs(files)
+    normalizeFilesDirs(sc)
 
     if "id" in workflowobj:
-        files.append({"class": "File", "location": workflowobj["id"]})
+        sc.append({"class": "File", "location": workflowobj["id"]})
 
-    mapper = ArvPathMapper(arvrunner, files, "",
+    mapper = ArvPathMapper(arvrunner, sc, "",
                            "keep:%s",
                            "keep:%s/%s",
                            name=name)
 
     def setloc(p):
-        p["location"] = mapper.mapper(p["location"]).target
+        if not p["location"].startswith("_:"):
+            p["location"] = mapper.mapper(p["location"]).target
     adjustFileObjs(workflowobj, setloc)
     adjustDirObjs(workflowobj, setloc)
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 8735192..31841df 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -32,7 +32,7 @@ setup(name='arvados-cwl-runner',
       # Make sure to update arvados/build/run-build-packages.sh as well
       # when updating the cwltool version pin.
       install_requires=[
-          'cwltool==1.0.20160922135240',
+          'cwltool==1.0.20160923180109',
           'arvados-python-client>=0.1.20160826210445'
       ],
       data_files=[

-----------------------------------------------------------------------


hooks/post-receive
--