[ARVADOS] created: b810854ddeccc8fff3ac718e8085a4cafb7f6b86

[Git user]

        at  b810854ddeccc8fff3ac718e8085a4cafb7f6b86 (commit)


commit b810854ddeccc8fff3ac718e8085a4cafb7f6b86
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Sat Sep 17 21:26:36 2016 -0400

    10088: Bump cwltool dependency for bugfix to scandeps.

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index efc6056..6922b90 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -32,7 +32,7 @@ setup(name='arvados-cwl-runner',
       # Make sure to update arvados/build/run-build-packages.sh as well
       # when updating the cwltool version pin.
       install_requires=[
-          'cwltool==1.0.20160913171024',
+          'cwltool==1.0.20160918012352',
           'arvados-python-client>=0.1.20160826210445'
       ],
       data_files=[

commit d63f3ade27bb506da049164789679f9fed171005
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Sep 16 17:37:13 2016 -0400

    10088: upload_dependencies reads raw files to find actual dependencies

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index e17264b..40a64ff 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -58,6 +58,8 @@ class ArvPathMapper(PathMapper):
     def addentry(self, obj, c, path, subdirs):
         if obj["location"] in self._pathmap:
             src, srcpath = self.arvrunner.fs_access.get_collection(self._pathmap[obj["location"]].resolved)
+            if srcpath == "":
+                srcpath = "."
             c.copy(srcpath, path + "/" + obj["basename"], source_collection=src, overwrite=True)
             for l in obj.get("secondaryFiles", []):
                 self.addentry(l, c, path, subdirs)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 72f347e..570675b 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -4,6 +4,7 @@ from functools import partial
 import logging
 import json
 import re
+from cStringIO import StringIO
 
 import cwltool.draft2tool
 from cwltool.draft2tool import CommandLineTool
@@ -13,6 +14,7 @@ from cwltool.load_tool import fetch_document
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
 
 import arvados.collection
+import ruamel.yaml as yaml
 
 from .arvdocker import arv_docker_get_image
 from .pathmapper import ArvPathMapper
@@ -26,9 +28,16 @@ def upload_dependencies(arvrunner, name, document_loader,
     loaded = set()
     def loadref(b, u):
         joined = urlparse.urljoin(b, u)
-        if joined not in loaded:
-            loaded.add(joined)
-            return document_loader.fetch(urlparse.urljoin(b, u))
+        defrg, _ = urlparse.urldefrag(joined)
+        if defrg not in loaded:
+            loaded.add(defrg)
+            # Use fetch_text to get raw file (before preprocessing).
+            text = document_loader.fetch_text(defrg)
+            if isinstance(text, bytes):
+                textIO = StringIO(text.decode('utf-8'))
+            else:
+                textIO = StringIO(text)
+            return yaml.safe_load(textIO)
         else:
             return {}
 
@@ -37,9 +46,15 @@ def upload_dependencies(arvrunner, name, document_loader,
     else:
         loadref_fields = set(("$import",))
 
-    sc = scandeps(uri, workflowobj,
+    scanobj = workflowobj
+    if "id" in workflowobj:
+        # Need raw file content (before preprocessing) to ensure
+        # that external references in $include and $mixin are captured.
+        scanobj = loadref("", workflowobj["id"])
+
+    sc = scandeps(uri, scanobj,
                   loadref_fields,
-                  set(("$include", "$schemas", "path", "location")),
+                  set(("$include", "$schemas")),
                   loadref)
 
     files = []

-----------------------------------------------------------------------


hooks/post-receive
--