[ARVADOS] created: 1.3.0-848-gb26223d09

Git user git at public.curoverse.com
Thu May 2 21:30:21 UTC 2019


        at  b26223d09ef41fb824dd4501509cbaa12ba31d90 (commit)


commit b26223d09ef41fb824dd4501509cbaa12ba31d90
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu May 2 17:20:10 2019 -0400

    15028: Handling secondaryFiles on record fields
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4a8a22a4a..752cc32ba 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -5,6 +5,7 @@
 from future import standard_library
 standard_library.install_aliases()
 from future.utils import  viewvalues, viewitems
+from past.builtins import basestring
 
 import os
 import sys
@@ -106,36 +107,91 @@ def make_builder(joborder, hints, requirements, runtimeContext):
                  stagedir="",            # type: Text
                 )
 
+def search_schemadef(name, reqs):
+    for r in reqs:
+        if r["class"] == "SchemaDefRequirement":
+            for sd in r["types"]:
+                if sd["name"] == name:
+                    return sd
+    return None
+
+primitive_types_set = frozenset(("null", "boolean", "int", "long",
+                                 "float", "double", "string", "record",
+                                 "array", "enum"))
+
+def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discovered):
+    if isinstance(inputschema, Sequence) and not isinstance(inputschema, basestring):
+        # union type, collect all possible secondaryFiles
+        for i in inputschema:
+            set_secondary(fsaccess, builder, i, secondaryspec, primary, discovered)
+        return
+
+    if isinstance(inputschema, basestring):
+        sd = search_schemadef(inputschema, reversed(builder.hints+builder.requirements))
+        if sd:
+            inputschema = sd
+        else:
+            return
 
-def set_secondary(fsaccess, builder, inputschema, primary, discovered):
-    if isinstance(primary, Mapping) and primary.get("class") == "File":
-        if "secondaryFiles" not in primary:
-            primary["secondaryFiles"] = []
-            for i, sf in enumerate(inputschema["secondaryFiles"]):
-                pattern = builder.do_eval(sf["pattern"], context=primary)
-                if pattern is None:
-                    continue
-                sfpath = substitute(primary["location"], pattern)
-                required = builder.do_eval(sf["required"], context=primary)
-
-                if fsaccess.exists(sfpath):
-                    primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
-                elif required:
-                    raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
-                        "Required secondary file '%s' does not exist" % sfpath)
-
-            primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
-            if discovered is not None:
-                discovered[primary["location"]] = primary["secondaryFiles"]
-    elif isinstance(primary, Sequence):
-        for e in primary:
-            set_secondary(fsaccess, builder, inputschema, e, discovered)
+    if "secondaryFiles" in inputschema:
+        # set secondaryFiles, may be inherited by compound types.
+        secondaryspec = inputschema["secondaryFiles"]
+
+    if isinstance(inputschema["type"], Mapping):
+        # compound type (array or record)
+        set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
+
+    elif (inputschema["type"] == "record" and
+          isinstance(primary, Mapping)):
+        #
+        # record type, find secondary files associated with fields.
+        #
+        for f in inputschema["fields"]:
+            p = primary.get(shortname(f["name"]))
+            if p:
+                set_secondary(fsaccess, builder, f, secondaryspec, p, discovered)
+
+    elif (inputschema["type"] == "array" and
+          isinstance(primary, Sequence)):
+        #
+        # array type, find secondary files of elements
+        #
+        for p in primary:
+            set_secondary(fsaccess, builder, {"type": inputschema["items"]}, secondaryspec, p, discovered)
+
+    elif (inputschema["type"] == "File" and
+          secondaryspec and
+          isinstance(primary, Mapping) and
+          primary.get("class") == "File" and
+          "secondaryFiles" not in primary):
+        #
+        # Found a file, check for secondaryFiles
+        #
+        primary["secondaryFiles"] = []
+        for i, sf in enumerate(aslist(secondaryspec)):
+            pattern = builder.do_eval(sf["pattern"], context=primary)
+            if pattern is None:
+                continue
+            sfpath = substitute(primary["location"], pattern)
+            required = builder.do_eval(sf["required"], context=primary)
+
+            if fsaccess.exists(sfpath):
+                primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+            elif required:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Required secondary file '%s' does not exist" % sfpath)
+
+        primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+        if discovered is not None:
+            discovered[primary["location"]] = primary["secondaryFiles"]
+    elif inputschema["type"] not in primitive_types_set:
+        set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
 
 def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None):
     for inputschema in inputs:
         primary = job_order.get(shortname(inputschema["id"]))
-        if isinstance(primary, (Mapping, Sequence)) and inputschema.get("secondaryFiles"):
-            set_secondary(fsaccess, builder, inputschema, primary, discovered)
+        if isinstance(primary, (Mapping, Sequence)):
+            set_secondary(fsaccess, builder, inputschema, None, primary, discovered)
 
 collection_uuid_pattern = re.compile(r'^keep:([a-z0-9]{5}-4zz18-[a-z0-9]{15})(/.*)?$')
 collection_pdh_pattern = re.compile(r'^keep:([0-9a-f]{32}\+\d+)(/.*)?')
@@ -418,8 +474,8 @@ def upload_job_order(arvrunner, name, tool, job_order):
                      arvrunner.fs_access)
     # Need to create a builder object to evaluate expressions.
     builder = make_builder(builder_job_order,
-                           tool.tool.get("hints", []),
-                           tool.tool.get("requirements", []),
+                           tool.hints,
+                           tool.requirements,
                            ArvRuntimeContext())
     # Now update job_order with secondaryFiles
     discover_secondary_files(arvrunner.fs_access,
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index c9776eac1..3af3dfb4d 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190426195829',
+          'cwltool==1.0.20190502174223',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',

commit 3b2aea1d7daeebf6007c87b3b4a78dbb3e2ffbd1
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu May 2 15:25:40 2019 -0400

    15028: Support optional secondaryFiles
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 31e6be12b..4fc02a016 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from cwltool.command_line_tool import CommandLineTool, ExpressionTool
-from cwltool.builder import Builder
 from .arvjob import ArvadosJob
 from .arvcontainer import ArvadosContainer
 from .pathmapper import ArvPathMapper
+from .runner import make_builder
 from functools import partial
 from schema_salad.sourceline import SourceLine
 from cwltool.errors import WorkflowException
@@ -37,30 +37,6 @@ def set_cluster_target(tool, arvrunner, builder, runtimeContext):
 
     return runtimeContext
 
-def make_builder(joborder, hints, requirements, runtimeContext):
-    return Builder(
-                 job=joborder,
-                 files=[],               # type: List[Dict[Text, Text]]
-                 bindings=[],            # type: List[Dict[Text, Any]]
-                 schemaDefs={},          # type: Dict[Text, Dict[Text, Any]]
-                 names=None,               # type: Names
-                 requirements=requirements,        # type: List[Dict[Text, Any]]
-                 hints=hints,               # type: List[Dict[Text, Any]]
-                 resources={},           # type: Dict[str, int]
-                 mutation_manager=None,    # type: Optional[MutationManager]
-                 formatgraph=None,         # type: Optional[Graph]
-                 make_fs_access=None,      # type: Type[StdFsAccess]
-                 fs_access=None,           # type: StdFsAccess
-                 job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
-                 timeout=runtimeContext.eval_timeout,             # type: float
-                 debug=runtimeContext.debug,               # type: bool
-                 js_console=runtimeContext.js_console,          # type: bool
-                 force_docker_pull=runtimeContext.force_docker_pull,   # type: bool
-                 loadListing="",         # type: Text
-                 outdir="",              # type: Text
-                 tmpdir="",              # type: Text
-                 stagedir="",            # type: Text
-                )
 
 class ArvadosCommandTool(CommandLineTool):
     """Wrap cwltool CommandLineTool to override selected methods."""
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 325fccb2c..3c60ac9fd 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -22,9 +22,11 @@ from cwltool.context import LoadingContext
 import ruamel.yaml as yaml
 
 from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
-                     trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
+                     trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
+                     make_builder)
 from .pathmapper import ArvPathMapper, trim_listing
-from .arvtool import ArvadosCommandTool, set_cluster_target, make_builder
+from .arvtool import ArvadosCommandTool, set_cluster_target
+
 from .perf import Perf
 
 logger = logging.getLogger('arvados.cwl-runner')
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ed6bcd008..4a8a22a4a 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -13,8 +13,10 @@ import urllib.parse
 from functools import partial
 import logging
 import json
+import copy
 from collections import namedtuple
 from io import StringIO
+from typing import Mapping, Sequence
 
 if os.name == "posix" and sys.version_info[0] < 3:
     import subprocess32 as subprocess
@@ -25,13 +27,15 @@ from schema_salad.sourceline import SourceLine, cmap
 
 from cwltool.command_line_tool import CommandLineTool
 import cwltool.workflow
-from cwltool.process import scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname, Process
+from cwltool.process import (scandeps, UnsupportedRequirement, normalizeFilesDirs,
+                             shortname, Process, fill_in_defaults)
 from cwltool.load_tool import fetch_document
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 from cwltool.builder import substitute
 from cwltool.pack import pack
 from cwltool.update import INTERNAL_VERSION
+from cwltool.builder import Builder
 import schema_salad.validate as validate
 
 import arvados.collection
@@ -42,6 +46,7 @@ import arvados_cwl.arvdocker
 from .pathmapper import ArvPathMapper, trim_listing
 from ._version import __version__
 from . import done
+from . context import ArvRuntimeContext
 
 logger = logging.getLogger('arvados.cwl-runner')
 
@@ -76,20 +81,61 @@ def find_defaults(d, op):
             for i in viewvalues(d):
                 find_defaults(i, op)
 
-def setSecondary(t, fileobj, discovered):
-    if isinstance(fileobj, dict) and fileobj.get("class") == "File":
-        if "secondaryFiles" not in fileobj:
-            fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf["pattern"]), "class": "File"} for sf in t["secondaryFiles"]])
+def make_builder(joborder, hints, requirements, runtimeContext):
+    return Builder(
+                 job=joborder,
+                 files=[],               # type: List[Dict[Text, Text]]
+                 bindings=[],            # type: List[Dict[Text, Any]]
+                 schemaDefs={},          # type: Dict[Text, Dict[Text, Any]]
+                 names=None,               # type: Names
+                 requirements=requirements,        # type: List[Dict[Text, Any]]
+                 hints=hints,               # type: List[Dict[Text, Any]]
+                 resources={},           # type: Dict[str, int]
+                 mutation_manager=None,    # type: Optional[MutationManager]
+                 formatgraph=None,         # type: Optional[Graph]
+                 make_fs_access=None,      # type: Type[StdFsAccess]
+                 fs_access=None,           # type: StdFsAccess
+                 job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
+                 timeout=runtimeContext.eval_timeout,             # type: float
+                 debug=runtimeContext.debug,               # type: bool
+                 js_console=runtimeContext.js_console,          # type: bool
+                 force_docker_pull=runtimeContext.force_docker_pull,   # type: bool
+                 loadListing="",         # type: Text
+                 outdir="",              # type: Text
+                 tmpdir="",              # type: Text
+                 stagedir="",            # type: Text
+                )
+
+
+def set_secondary(fsaccess, builder, inputschema, primary, discovered):
+    if isinstance(primary, Mapping) and primary.get("class") == "File":
+        if "secondaryFiles" not in primary:
+            primary["secondaryFiles"] = []
+            for i, sf in enumerate(inputschema["secondaryFiles"]):
+                pattern = builder.do_eval(sf["pattern"], context=primary)
+                if pattern is None:
+                    continue
+                sfpath = substitute(primary["location"], pattern)
+                required = builder.do_eval(sf["required"], context=primary)
+
+                if fsaccess.exists(sfpath):
+                    primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+                elif required:
+                    raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                        "Required secondary file '%s' does not exist" % sfpath)
+
+            primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
             if discovered is not None:
-                discovered[fileobj["location"]] = fileobj["secondaryFiles"]
-    elif isinstance(fileobj, list):
-        for e in fileobj:
-            setSecondary(t, e, discovered)
+                discovered[primary["location"]] = primary["secondaryFiles"]
+    elif isinstance(primary, Sequence):
+        for e in primary:
+            set_secondary(fsaccess, builder, inputschema, e, discovered)
 
-def discover_secondary_files(inputs, job_order, discovered=None):
-    for t in inputs:
-        if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
-            setSecondary(t, job_order[shortname(t["id"])], discovered)
+def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None):
+    for inputschema in inputs:
+        primary = job_order.get(shortname(inputschema["id"]))
+        if isinstance(primary, (Mapping, Sequence)) and inputschema.get("secondaryFiles"):
+            set_secondary(fsaccess, builder, inputschema, primary, discovered)
 
 collection_uuid_pattern = re.compile(r'^keep:([a-z0-9]{5}-4zz18-[a-z0-9]{15})(/.*)?$')
 collection_pdh_pattern = re.compile(r'^keep:([0-9a-f]{32}\+\d+)(/.*)?')
@@ -220,8 +266,18 @@ def upload_dependencies(arvrunner, name, document_loader,
 
     discovered = {}
     def discover_default_secondary_files(obj):
-        discover_secondary_files(obj["inputs"],
-                                 {shortname(t["id"]): t["default"] for t in obj["inputs"] if "default" in t},
+        builder_job_order = {}
+        for t in obj["inputs"]:
+            builder_job_order[shortname(t["id"])] = t["default"] if "default" in t else None
+        # Need to create a builder object to evaluate expressions.
+        builder = make_builder(builder_job_order,
+                               obj.get("hints", []),
+                               obj.get("requirements", []),
+                               ArvRuntimeContext())
+        discover_secondary_files(arvrunner.fs_access,
+                                 builder,
+                                 obj["inputs"],
+                                 builder_job_order,
                                  discovered)
 
     visit_class(workflowobj, ("CommandLineTool", "Workflow"), discover_default_secondary_files)
@@ -355,7 +411,21 @@ def upload_job_order(arvrunner, name, tool, job_order):
     object with 'location' updated to the proper keep references.
     """
 
-    discover_secondary_files(tool.tool["inputs"], job_order)
+    # Make a copy of the job order and set defaults.
+    builder_job_order = copy.copy(job_order)
+    fill_in_defaults(tool.tool["inputs"],
+                     builder_job_order,
+                     arvrunner.fs_access)
+    # Need to create a builder object to evaluate expressions.
+    builder = make_builder(builder_job_order,
+                           tool.tool.get("hints", []),
+                           tool.tool.get("requirements", []),
+                           ArvRuntimeContext())
+    # Now update job_order with secondaryFiles
+    discover_secondary_files(arvrunner.fs_access,
+                             builder,
+                             tool.tool["inputs"],
+                             job_order)
 
     jobmapper = upload_dependencies(arvrunner,
                                     name,

commit e9a55e851052008f5bd026f4d406460f5a67253f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Apr 30 14:18:24 2019 -0400

    15028: Bump cwltool
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 2b273017e..c9776eac1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190426175817',
+          'cwltool==1.0.20190426195829',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',

commit e411f9827af41433ff65b8cd3c4fc20ce4ac0e24
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Apr 29 15:40:24 2019 -0400

    15028: Propagate --enable-dev and handle blank node ids
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 437640ac6..b0dff491f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -485,6 +485,9 @@ class RunnerContainer(Runner):
         if self.arvrunner.project_uuid:
             command.append("--project-uuid="+self.arvrunner.project_uuid)
 
+        if self.enable_dev:
+            command.append("--enable-dev")
+
         command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
 
         container_req["command"] = command
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index badf58938..9b9c2f161 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -365,7 +365,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
 
                 while keys:
                     page = keys[:pageSize]
-                    keys = keys[pageSize:]
                     try:
                         proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
                     except Exception:
@@ -381,6 +380,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                                 "new_attributes": p
                             }
                         })
+                    keys = keys[pageSize:]
+
                 finish_poll = time.time()
                 remain_wait = self.poll_interval - (finish_poll - begin_poll)
         except:
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 04fd1c443..ed6bcd008 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -131,7 +131,7 @@ def upload_dependencies(arvrunner, name, document_loader,
         loadref_fields = set(("$import",))
 
     scanobj = workflowobj
-    if "id" in workflowobj:
+    if "id" in workflowobj and not workflowobj["id"].startswith("_:"):
         # Need raw file content (before preprocessing) to ensure
         # that external references in $include and $mixin are captured.
         scanobj = loadref("", workflowobj["id"])
@@ -479,6 +479,7 @@ class Runner(Process):
         self.intermediate_output_ttl = intermediate_output_ttl
         self.priority = priority
         self.secret_store = secret_store
+        self.enable_dev = loadingContext.enable_dev
 
         self.submit_runner_cores = 1
         self.submit_runner_ram = 1024  # defaut 1 GiB

commit d63d6d34d8948c6ec252b62814bb26224ad1e2aa
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:53:20 2019 -0400

    15028: Fix discover secondaryFiles after internal v1.1 update
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index c5fd7e3cf..badf58938 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -550,6 +550,10 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if not runtimeContext.name:
             runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
 
+        # Upload local file references in the job order.
+        job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+                                     tool, job_order)
+
         submitting = (runtimeContext.update_workflow or
                       runtimeContext.create_workflow or
                       (runtimeContext.submit and not
@@ -579,10 +583,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         loadingContext.metadata = tool.metadata
         tool = load_tool(tool.tool, loadingContext)
 
-        # Upload local file references in the job order.
-        job_order = upload_job_order(self, "%s input" % runtimeContext.name,
-                                     tool, job_order)
-
         existing_uuid = runtimeContext.update_workflow
         if existing_uuid or runtimeContext.create_workflow:
             # Create a pipeline template or workflow record and exit.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3235f4763..04fd1c443 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -79,7 +79,7 @@ def find_defaults(d, op):
 def setSecondary(t, fileobj, discovered):
     if isinstance(fileobj, dict) and fileobj.get("class") == "File":
         if "secondaryFiles" not in fileobj:
-            fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
+            fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf["pattern"]), "class": "File"} for sf in t["secondaryFiles"]])
             if discovered is not None:
                 discovered[fileobj["location"]] = fileobj["secondaryFiles"]
     elif isinstance(fileobj, list):

commit 5127d81421ec9aa4d7f4c09dafebfbb22c57ce0a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:08:41 2019 -0400

    15028: Bump cwltool
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 37682bc9c..2b273017e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190425212529',
+          'cwltool==1.0.20190426175817',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',

commit 6a300a01b8bbbc0c603442add737da5613a81f46
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:07:05 2019 -0400

    15028: Set the correct loading context to reload tools
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 768783df9..c5fd7e3cf 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -574,6 +574,9 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         # ArvadosCommandTool) because tool document may have been
         # updated by upload_workflow_deps in ways that modify
         # inheritance of hints or requirements.
+        loadingContext.loader = tool.doc_loader
+        loadingContext.avsc_names = tool.doc_schema
+        loadingContext.metadata = tool.metadata
         tool = load_tool(tool.tool, loadingContext)
 
         # Upload local file references in the job order.
@@ -647,8 +650,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if runtimeContext.submit:
             # Submit a runner job to run the workflow for us.
             if self.work_api == "containers":
-                loadingContext.loader = tool.doc_loader
-                loadingContext.avsc_names = tool.doc_schema
                 if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
                     runtimeContext.runnerjob = tool.tool["id"]
                 else:

commit 8e5c34ec0233e75c2953b10c7794b89c8e4e63e1
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Apr 25 17:42:49 2019 -0400

    15028: Instead of suppressing update, reload old version for submit
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 95711762c..def720381 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -219,8 +219,11 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 def add_arv_hints():
     cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
     cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
-    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
-    use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
+    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
+    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
+    customschema = res.read()
+    use_custom_schema("v1.0", "http://arvados.org/cwl", customschema)
+    use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema)
     res.close()
     cwltool.process.supportedProcessRequirements.extend([
         "http://arvados.org/cwl#RunInSingleContainer",
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
similarity index 100%
copy from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
copy to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
similarity index 80%
rename from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
rename to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index dce1bd4d0..b9b9e6165 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -9,24 +9,6 @@ $namespaces:
 $graph:
 - $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
 
-- name: cwltool:LoadListingRequirement
-  type: record
-  extends: cwl:ProcessRequirement
-  inVocab: false
-  fields:
-    class:
-      type: string
-      doc: "Always 'LoadListingRequirement'"
-      jsonldPredicate:
-        "_id": "@type"
-        "_type": "@vocab"
-    loadListing:
-      type:
-        - "null"
-        - type: enum
-          name: LoadListingEnum
-          symbols: [no_listing, shallow_listing, deep_listing]
-
 - name: cwltool:Secrets
   type: record
   inVocab: false
@@ -47,28 +29,6 @@ $graph:
         "_type": "@id"
         refScope: 0
 
-- name: cwltool:TimeLimit
-  type: record
-  inVocab: false
-  extends: cwl:ProcessRequirement
-  doc: |
-    Set an upper limit on the execution time of a CommandLineTool or
-    ExpressionTool.  A tool execution which exceeds the time limit may
-    be preemptively terminated and considered failed.  May also be
-    used by batch systems to make scheduling decisions.
-  fields:
-    - name: class
-      type: string
-      doc: "Always 'TimeLimit'"
-      jsonldPredicate:
-        "_id": "@type"
-        "_type": "@vocab"
-    - name: timelimit
-      type: [long, string]
-      doc: |
-        The time limit, in seconds.  A time limit of zero means no
-        time limit.  Negative time limits are an error.
-
 - name: RunInSingleContainer
   type: record
   extends: cwl:ProcessRequirement
@@ -196,22 +156,6 @@ $graph:
         recommended minimum value for TTL is the expected duration of the
         entire the workflow.
 
-- name: ReuseRequirement
-  type: record
-  extends: cwl:ProcessRequirement
-  inVocab: false
-  doc: |
-    Enable/disable work reuse for current process.  Default true (work reuse enabled).
-  fields:
-    - name: class
-      type: string
-      doc: "Always 'arv:ReuseRequirement'"
-      jsonldPredicate:
-        _id: "@type"
-        _type: "@vocab"
-    - name: enableReuse
-      type: boolean
-
 - name: WorkflowRunnerResources
   type: record
   extends: cwl:ProcessRequirement
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index ab2078e15..11efc0c1c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -236,8 +236,11 @@ class ArvadosJob(JobBase):
         try:
             if record["state"] == "Complete":
                 processStatus = "success"
+                # we don't have the real exit code so fake it.
+                record["exit_code"] = 0
             else:
                 processStatus = "permanentFail"
+                record["exit_code"] = 1
 
             outputs = {}
             try:
@@ -272,8 +275,8 @@ class ArvadosJob(JobBase):
                         outputs = done.done(self, record, dirs["tmpdir"],
                                             dirs["outdir"], dirs["keep"])
             except WorkflowException as e:
-                # Only include a stack trace if in debug mode. 
-                # This is most likely a user workflow error and a stack trace may obfuscate more useful output. 
+                # Only include a stack trace if in debug mode.
+                # This is most likely a user workflow error and a stack trace may obfuscate more useful output.
                 logger.error("%s unable to collect output from %s:\n%s",
                              self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False))
                 processStatus = "permanentFail"
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 20cff6060..768783df9 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -188,7 +188,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         self.loadingContext.fetcher_constructor = self.fetcher_constructor
         self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
         self.loadingContext.construct_tool_object = self.arv_make_tool
-        self.loadingContext.do_update = False
 
         # Add a custom logging handler to the root logger for runtime status reporting
         # if running inside a container
@@ -551,21 +550,31 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if not runtimeContext.name:
             runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
 
-        # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
-        # Also uploads docker images.
-        merged_map = upload_workflow_deps(self, tool)
+        submitting = (runtimeContext.update_workflow or
+                      runtimeContext.create_workflow or
+                      (runtimeContext.submit and not
+                       (tool.tool["class"] == "CommandLineTool" and
+                        runtimeContext.wait and
+                        not runtimeContext.always_submit_runner)))
 
-        # Reload tool object which may have been updated by
-        # upload_workflow_deps
-        # Don't validate this time because it will just print redundant errors.
         loadingContext = self.loadingContext.copy()
-        loadingContext.loader = tool.doc_loader
-        loadingContext.avsc_names = tool.doc_schema
-        loadingContext.metadata = tool.metadata
         loadingContext.do_validate = False
+        loadingContext.do_update = False
+        if submitting:
+            # Document may have been auto-updated. Reload the original
+            # document with updating disabled because we want to
+            # submit the original document, not the auto-updated one.
+            tool = load_tool(tool.tool["id"], loadingContext)
+
+        # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
+        # Also uploads docker images.
+        merged_map = upload_workflow_deps(self, tool)
 
-        tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
-                                  loadingContext)
+        # Recreate process object (ArvadosWorkflow or
+        # ArvadosCommandTool) because tool document may have been
+        # updated by upload_workflow_deps in ways that modify
+        # inheritance of hints or requirements.
+        tool = load_tool(tool.tool, loadingContext)
 
         # Upload local file references in the job order.
         job_order = upload_job_order(self, "%s input" % runtimeContext.name,
@@ -638,6 +647,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if runtimeContext.submit:
             # Submit a runner job to run the workflow for us.
             if self.work_api == "containers":
+                loadingContext.loader = tool.doc_loader
+                loadingContext.avsc_names = tool.doc_schema
                 if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
                     runtimeContext.runnerjob = tool.tool["id"]
                 else:
@@ -673,11 +684,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                     "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
             logger.info("Pipeline instance %s", self.pipeline["uuid"])
 
-        if not isinstance(tool, Runner):
-            loadingContext.do_update = True
-            tool = load_tool(tool.doc_loader.idx[tool.tool["id"]],
-                             loadingContext)
-
         if runtimeContext.cwl_runner_job is not None:
             self.uuid = runtimeContext.cwl_runner_job.get('uuid')
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 79c5fc209..37682bc9c 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -25,7 +25,7 @@ setup(name='arvados-cwl-runner',
       download_url="https://github.com/curoverse/arvados.git",
       license='Apache 2.0',
       packages=find_packages(),
-      package_data={'arvados_cwl': ['arv-cwl-schema.yml']},
+      package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml']},
       scripts=[
           'bin/cwl-runner',
           'bin/arvados-cwl-runner',
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190423203253',
+          'cwltool==1.0.20190425212529',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2ddd06535..1f169cf23 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -513,7 +513,7 @@ class TestContainer(unittest.TestCase):
         self.assertFalse(api.collections().create.called)
         self.assertFalse(runner.runtime_status_error.called)
 
-        arvjob.collect_outputs.assert_called_with("keep:abc+123")
+        arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 

commit 2be795af227cbc9cd2e5449e6129cd9eb968b7f3
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Apr 23 18:01:55 2019 -0400

    15028: Fix reporting logs on container requests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b194f3dfc..437640ac6 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -527,6 +527,7 @@ class RunnerContainer(Runner):
             container = self.arvrunner.api.containers().get(
                 uuid=record["container_uuid"]
             ).execute(num_retries=self.arvrunner.num_retries)
+            container["log"] = record["log_uuid"]
         except Exception:
             logger.exception("%s while getting runner container", self.arvrunner.label(self))
             self.arvrunner.output_callback({}, "permanentFail")

commit 899e4ed186b3ad373388789269dace42df797cbb
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Apr 23 17:32:33 2019 -0400

    15028: Fixing 1.0 conformance tests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 2e4c45738..52df80f58 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -37,8 +37,12 @@ fi
 cd "$WORKSPACE"
 
 py=python
+pipcmd=pip
 if [[ -n "$PYCMD" ]] ; then
-    py="$PYCMD" ;
+    py="$PYCMD"
+    if [[ $py = python3 ]] ; then
+	pipcmd=pip3
+    fi
 fi
 
 (cd sdk/python && python setup.py sdist)
@@ -75,6 +79,6 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
     cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
 fi
 
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
 echo arv-keepdocker arvados/jobs $cwl_runner_version
 arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 9b26ad706..c0e3e0de2 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -52,7 +52,7 @@ def done(self, record, tmpdir, outdir, keepdir):
 def done_outputs(self, record, tmpdir, outdir, keepdir):
     self.builder.outdir = outdir
     self.builder.pathmapper.keepdir = keepdir
-    return self.collect_outputs("keep:" + record["output"])
+    return self.collect_outputs("keep:" + record["output"], record["exit_code"])
 
 crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[a-z0-9]{15} \d+ \d+ stderr crunchstat:")
 timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)")
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 2875dbc22..20cff6060 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -45,6 +45,7 @@ from ._version import __version__
 from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
 from cwltool.command_line_tool import compute_checksums
+from cwltool.load_tool import load_tool
 
 logger = logging.getLogger('arvados.cwl-runner')
 metrics = logging.getLogger('arvados.cwl-runner.metrics')
@@ -672,6 +673,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                     "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
             logger.info("Pipeline instance %s", self.pipeline["uuid"])
 
+        if not isinstance(tool, Runner):
+            loadingContext.do_update = True
+            tool = load_tool(tool.doc_loader.idx[tool.tool["id"]],
+                             loadingContext)
+
         if runtimeContext.cwl_runner_job is not None:
             self.uuid = runtimeContext.cwl_runner_job.get('uuid')
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 3ccf49f9e..79c5fc209 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190422193858',
+          'cwltool==1.0.20190423203253',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index f9e370794..dd067e977 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -13,22 +13,19 @@
 # (This dockerfile file must be located in the arvados/sdk/ directory because
 #  of the docker build root.)
 
-FROM debian:jessie
+FROM debian:9
 MAINTAINER Ward Vandewege <ward at curoverse.com>
 
 ENV DEBIAN_FRONTEND noninteractive
 
 ARG pythoncmd=python
+ARG pipcmd=pip
 
 RUN apt-get update -q && apt-get install -qy --no-install-recommends \
     git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
     libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
 
-RUN if [ "$pythoncmd" = "python3" ]; then \
-       pip3 install -U setuptools six requests ; \
-    else \
-       pip install -U setuptools six requests ; \
-    fi
+RUN $pipcmd install -U setuptools six requests
 
 ARG sdk
 ARG runner
@@ -40,10 +37,10 @@ ADD cwl/salad_dist/$salad /tmp/
 ADD cwl/cwltool_dist/$cwltool /tmp/
 ADD cwl/dist/$runner /tmp/
 
-RUN cd /tmp/arvados-python-client-* && $pythoncmd setup.py install
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pythoncmd setup.py install ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pythoncmd setup.py install ; fi
-RUN cd /tmp/arvados-cwl-runner-* && $pythoncmd setup.py install
+RUN cd /tmp/arvados-python-client-* && $pipcmd install .
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi
+RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
 
 # Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \

commit 2ce3493f606469f52fe4e9cba08dfdd079e9c83d
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 5 13:12:45 2019 -0400

    15028: Update cwltool/schema-salad deps, fix tests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8e6bff4f0..325fccb2c 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -288,6 +288,10 @@ class ArvadosWorkflow(Workflow):
                 adjustDirObjs(packed, keepmount)
                 self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
 
+        self.loadingContext = self.loadingContext.copy()
+        self.loadingContext.metadata = self.loadingContext.metadata.copy()
+        self.loadingContext.metadata["http://commonwl.org/cwltool#original_cwlVersion"] = "v1.0"
+
         wf_runner = cmap({
             "class": "CommandLineTool",
             "baseCommand": "cwltool",
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index eeb44dbd7..2875dbc22 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -187,6 +187,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         self.loadingContext.fetcher_constructor = self.fetcher_constructor
         self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
         self.loadingContext.construct_tool_object = self.arv_make_tool
+        self.loadingContext.do_update = False
 
         # Add a custom logging handler to the root logger for runtime status reporting
         # if running inside a container
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9385bde63..3235f4763 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -31,6 +31,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 from cwltool.builder import substitute
 from cwltool.pack import pack
+from cwltool.update import INTERNAL_VERSION
 import schema_salad.validate as validate
 
 import arvados.collection
@@ -451,6 +452,10 @@ class Runner(Process):
                  collection_cache_size=256,
                  collection_cache_is_default=True):
 
+        loadingContext = loadingContext.copy()
+        loadingContext.metadata = loadingContext.metadata.copy()
+        loadingContext.metadata["cwlVersion"] = INTERNAL_VERSION
+
         super(Runner, self).__init__(tool.tool, loadingContext)
 
         self.arvrunner = runner
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 1052fb0d7..3ccf49f9e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20181217162649',
-          'schema-salad==3.0.20181129082112',
+          'cwltool==1.0.20190422193858',
+          'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
           'arvados-python-client>=1.3.0.20190205182514',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 07d962bf9..2ddd06535 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -57,7 +57,7 @@ class CollectionMock(object):
 class TestContainer(unittest.TestCase):
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -66,7 +66,7 @@ class TestContainer(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": Loader({}),
-             "metadata": {"cwlVersion": "v1.0"}})
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "containers",
              "basedir": "",
@@ -400,7 +400,7 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({
             "inputs": [],
@@ -607,7 +607,7 @@ class TestContainer(unittest.TestCase):
             "portable_data_hash": "99999999999999999999999999999994+99",
             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({
             "inputs": [
@@ -697,7 +697,7 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         tool = cmap({"arguments": ["md5sum", "example.conf"],
                      "class": "CommandLineTool",
@@ -803,7 +803,7 @@ class TestContainer(unittest.TestCase):
             "class": "CommandLineTool",
             "hints": [
                 {
-                    "class": "http://commonwl.org/cwltool#TimeLimit",
+                    "class": "ToolTimeLimit",
                     "timelimit": 42
                 }
             ]
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 022d75be1..9cac68aa6 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -34,7 +34,7 @@ if not os.getenv('ARVADOS_DEBUG'):
 class TestJob(unittest.TestCase):
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -43,7 +43,7 @@ class TestJob(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": Loader({}),
-             "metadata": {"cwlVersion": "v1.0"},
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
              "makeTool": runner.arv_make_tool})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "jobs",
@@ -343,7 +343,7 @@ class TestJob(unittest.TestCase):
 
 class TestWorkflow(unittest.TestCase):
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -358,7 +358,7 @@ class TestWorkflow(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": document_loader,
-             "metadata": {"cwlVersion": "v1.0"},
+             "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
              "construct_tool_object": runner.arv_make_tool})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "jobs",
@@ -458,6 +458,7 @@ bytes(b'''{
     @mock.patch("arvados.collection.Collection")
     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
+        # TODO copy this over to test_container
         arvados_cwl.add_arv_hints()
 
         api = mock.MagicMock()
@@ -475,16 +476,15 @@ bytes(b'''{
         runner.num_retries = 0
 
         loadingContext, runtimeContext = self.helper(runner)
-
+        loadingContext.do_update = True
         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
-        metadata["cwlVersion"] = tool["cwlVersion"]
 
         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
 
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
         arvtool.formatgraph = None
         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
-        
+
         next(it).run(runtimeContext)
         next(it).run(runtimeContext)
 
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 9535f6ba2..5f92cee94 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -84,6 +84,7 @@ def stubs(func):
 
         stubs.api = mock.MagicMock()
         stubs.api._rootDesc = get_rootDesc()
+        stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 
         stubs.api.users().current().execute.return_value = {
             "uuid": stubs.fake_user_uuid,
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 2af115543..ecd36026f 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -6,7 +6,7 @@
   "$graph": [
     {
       "class": "Workflow",
-      "cwlVersion": "v1.0",
+      "cwlVersion": "v1.1.0-dev1",
       "hints": [],
       "id": "#main",
       "inputs": [
@@ -59,10 +59,10 @@
           "run": {
             "baseCommand": "sleep",
             "class": "CommandLineTool",
-            "id": "#main/sleep1/subtool",
+            "id": "#main/sleep1/run/subtool",
             "inputs": [
               {
-                "id": "#main/sleep1/subtool/sleeptime",
+                "id": "#main/sleep1/run/subtool/sleeptime",
                 "inputBinding": {
                   "position": 1
                 },
@@ -71,7 +71,7 @@
             ],
             "outputs": [
               {
-                "id": "#main/sleep1/subtool/out",
+                "id": "#main/sleep1/run/subtool/out",
                 "outputBinding": {
                   "outputEval": "out"
                 },
@@ -83,5 +83,5 @@
       ]
     }
   ],
-  "cwlVersion": "v1.0"
+  "cwlVersion": "v1.1.0-dev1"
 }
\ No newline at end of file

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list