[ARVADOS] created: d1a7f9691ff2b9d77106f953c714b1455b228c64

[Git user]

        at  d1a7f9691ff2b9d77106f953c714b1455b228c64 (commit)


commit d1a7f9691ff2b9d77106f953c714b1455b228c64
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Jul 11 12:40:05 2017 -0400

    11948: Strip out default values which reference unavailable files.
    
    Passes CWL conformance test "Test warning instead of error when default path is not found"
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7f2c53a..695597f 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -426,12 +426,6 @@ class ArvCwlRunner(object):
             if self.work_api == "containers":
                 if tool.tool["class"] == "CommandLineTool" and kwargs.get("wait"):
                     kwargs["runnerjob"] = tool.tool["id"]
-                    upload_dependencies(self,
-                                        kwargs["name"],
-                                        tool.doc_loader,
-                                        tool.tool,
-                                        tool.tool["id"],
-                                        False)
                     runnerjob = tool.job(job_order,
                                          self.output_callback,
                                          **kwargs).next()
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 683f548..19ce393 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -18,7 +18,7 @@ from cwltool.draft2tool import CommandLineTool
 import cwltool.workflow
 from cwltool.process import get_feature, scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname
 from cwltool.load_tool import fetch_document
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 from cwltool.builder import substitute
 from cwltool.pack import pack
@@ -46,6 +46,17 @@ def trim_anonymous_location(obj):
     if obj.get("location", "").startswith("_:"):
         del obj["location"]
 
+def find_defaults(d, op):
+    if isinstance(d, list):
+        for i in d:
+            find_defaults(i, op)
+    if isinstance(d, dict):
+        if "default" in d:
+            op(d)
+        else:
+            for i in d.itervalues():
+                find_defaults(i, op)
+
 def upload_dependencies(arvrunner, name, document_loader,
                         workflowobj, uri, loadref_run, include_primary=True):
     """Upload the dependencies of the workflowobj document to Keep.
@@ -101,6 +112,28 @@ def upload_dependencies(arvrunner, name, document_loader,
         for s in workflowobj["$schemas"]:
             sc.append({"class": "File", "location": s})
 
+    def capture_default(obj):
+        remove = [False]
+        def add_default(f):
+            if "location" not in f and "path" in f:
+                f["location"] = f["path"]
+                del f["path"]
+            if not arvrunner.fs_access.exists(f["location"]):
+                # Remove from sc
+                i = 0
+                while i < len(sc):
+                    if sc[i]["location"] == f["location"]:
+                        del sc[i]
+                    else:
+                        i += 1
+                # Delete "default" from workflowobj
+                remove[0] = True
+        visit_class(obj["default"], ("File", "Directory"), add_default)
+        if remove[0]:
+            del obj["default"]
+
+    find_defaults(workflowobj, capture_default)
+
     mapper = ArvPathMapper(arvrunner, sc, "",
                            "keep:%s",
                            "keep:%s/%s",

commit e953b9ecd6815968d8af36981770fe4553209892
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Sat Jul 8 16:01:08 2017 -0400

    Update cwltool & schema-salad and fix for changes.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curoverse.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 52009da..7f2c53a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -756,6 +756,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
     arvargs.use_container = True
     arvargs.relax_path_checks = True
     arvargs.validate = None
+    arvargs.print_supported_versions = False
 
     make_fs_access = partial(CollectionFsAccess,
                            collection_cache=runner.collection_cache)
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 8ab86c1..35a068f 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -15,7 +15,7 @@ class ArvadosCommandTool(CommandLineTool):
         self.arvrunner = arvrunner
         self.work_api = kwargs["work_api"]
 
-    def makeJobRunner(self):
+    def makeJobRunner(self, use_container=True):
         if self.work_api == "containers":
             return ArvadosContainer(self.arvrunner)
         elif self.work_api == "jobs":
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 55987ca..375485e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20170525215327',
-          'schema-salad==2.5.20170428142041',
+          'cwltool==1.0.20170707200431',
+          'schema-salad==2.6.20170630075932',
           'typing==3.5.3.0',
           'ruamel.yaml==0.13.7',
           'arvados-python-client>=0.1.20170526013812',

-----------------------------------------------------------------------


hooks/post-receive
--