[ARVADOS] created: 1.3.0-2771-g6563930ed
Git user
git at public.arvados.org
Mon Jul 13 17:50:55 UTC 2020
at 6563930ed602f47c1489b39b2eebfb54f9945e3c (commit)
commit 6563930ed602f47c1489b39b2eebfb54f9945e3c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Jul 13 11:20:10 2020 -0400
16353: Bump cwltool version
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 8540b6d24..b10f02d14 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -106,7 +106,7 @@ def make_builder(joborder, hints, requirements, runtimeContext, metadata):
outdir="", # type: Text
tmpdir="", # type: Text
stagedir="", # type: Text
- cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion")
+ cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion") or metadata.get("cwlVersion")
)
def search_schemadef(name, reqs):
@@ -173,7 +173,10 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
specs = []
primary["secondaryFiles"] = secondaryspec
for i, sf in enumerate(aslist(secondaryspec)):
- pattern = builder.do_eval(sf["pattern"], context=primary)
+ if builder.cwlVersion == "v1.0":
+ pattern = builder.do_eval(sf, context=primary)
+ else:
+ pattern = builder.do_eval(sf["pattern"], context=primary)
if pattern is None:
continue
if isinstance(pattern, list):
@@ -264,6 +267,8 @@ def upload_dependencies(arvrunner, name, document_loader,
# that external references in $include and $mixin are captured.
scanobj = loadref("", workflowobj["id"])
+ metadata = scanobj
+
sc_result = scandeps(uri, scanobj,
loadref_fields,
set(("$include", "$schemas", "location")),
@@ -356,7 +361,7 @@ def upload_dependencies(arvrunner, name, document_loader,
obj.get("hints", []),
obj.get("requirements", []),
ArvRuntimeContext(),
- {})
+ metadata)
discover_secondary_files(arvrunner.fs_access,
builder,
obj["inputs"],
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index fd41be86a..ea63ec485 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,7 +39,7 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200706173533',
+ 'cwltool==3.0.20200710214758',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
commit 44e035c4f9873171ae48025515102b513898a7c3
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jul 7 13:55:04 2020 -0400
16353: Initialize 'builder' with document version
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 704edaccb..a9361a85f 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -57,7 +57,7 @@ class ArvadosCommandTool(CommandLineTool):
"/keep/%s/%s")
def job(self, joborder, output_callback, runtimeContext):
- builder = make_builder(joborder, self.hints, self.requirements, runtimeContext)
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
if runtimeContext.work_api == "containers":
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ddd3c0076..97c5fafe7 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -141,7 +141,8 @@ class ArvadosWorkflowStep(WorkflowStep):
runtimeContext = runtimeContext.copy()
runtimeContext.toplevel = True # Preserve behavior for #13365
- builder = make_builder({shortname(k): v for k,v in viewitems(joborder)}, self.hints, self.requirements, runtimeContext)
+ builder = make_builder({shortname(k): v for k,v in viewitems(joborder)}, self.hints, self.requirements,
+ runtimeContext, self.metadata)
runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
@@ -161,7 +162,7 @@ class ArvadosWorkflow(Workflow):
def job(self, joborder, output_callback, runtimeContext):
- builder = make_builder(joborder, self.hints, self.requirements, runtimeContext)
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 71e499ebc..8540b6d24 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -83,7 +83,7 @@ def find_defaults(d, op):
for i in viewvalues(d):
find_defaults(i, op)
-def make_builder(joborder, hints, requirements, runtimeContext):
+def make_builder(joborder, hints, requirements, runtimeContext, metadata):
return Builder(
job=joborder,
files=[], # type: List[Dict[Text, Text]]
@@ -106,6 +106,7 @@ def make_builder(joborder, hints, requirements, runtimeContext):
outdir="", # type: Text
tmpdir="", # type: Text
stagedir="", # type: Text
+ cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion")
)
def search_schemadef(name, reqs):
@@ -354,7 +355,8 @@ def upload_dependencies(arvrunner, name, document_loader,
builder = make_builder(builder_job_order,
obj.get("hints", []),
obj.get("requirements", []),
- ArvRuntimeContext())
+ ArvRuntimeContext(),
+ {})
discover_secondary_files(arvrunner.fs_access,
builder,
obj["inputs"],
@@ -516,7 +518,8 @@ def upload_job_order(arvrunner, name, tool, job_order):
builder = make_builder(builder_job_order,
tool.hints,
tool.requirements,
- ArvRuntimeContext())
+ ArvRuntimeContext(),
+ tool.metadata)
# Now update job_order with secondaryFiles
discover_secondary_files(arvrunner.fs_access,
builder,
commit 5d5cc4f6b9272e3958721fc958453d649c919dcc
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jul 7 13:23:11 2020 -0400
16353: Bump schema salad and cwltool versions for v1.2.0-dev4 support
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 40ee67985..fd41be86a 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,8 +39,8 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200530110633',
- 'schema-salad==6.0.20200601095207',
+ 'cwltool==3.0.20200706173533',
+ 'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list