[ARVADOS] created: 1.1.4-248-g7ab351a
Git user
git at public.curoverse.com
Wed May 9 14:28:50 EDT 2018
at 7ab351a4f34e71e4351481eb2a5b41e5d06a52a7 (commit)
commit 7ab351a4f34e71e4351481eb2a5b41e5d06a52a7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed May 9 14:28:14 2018 -0400
12256: Bump cwltool version for workflow packing bugfix.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index c636bb8..48b1903 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180504184252',
+ 'cwltool==1.0.20180508202931',
'schema-salad==2.7.20180501211602',
'typing==3.5.3.0',
'ruamel.yaml >=0.13.7, <0.15',
commit 3b4da80a23136c0cec743ee4cdad9b22b51ac3e0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 4 16:44:29 2018 -0400
12256: Handle default references in RunInSingleContainer
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index ddc270d..8268300 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -24,7 +24,7 @@ class ArvadosCommandTool(CommandLineTool):
def makePathMapper(self, reffiles, stagedir, **kwargs):
# type: (List[Any], unicode, **Any) -> PathMapper
if self.work_api == "containers":
- return ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+ return ArvPathMapper(self.arvrunner, reffiles+kwargs.get("extra_reffiles", []), kwargs["basedir"],
"/keep/%s",
"/keep/%s/%s",
**kwargs)
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index bdc2e27..ba16b86 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -205,13 +205,17 @@ class ArvadosWorkflow(Workflow):
joborder_keepmount = copy.deepcopy(joborder)
reffiles = []
- visit_class(joborder_keepmount, ("File", "Directory"), lambda x: reffiles.append(x))
+ visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
+ reffiles2 = []
+ visit_class(packed, ("File", "Directory"), reffiles2.append)
- mapper = ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+ mapper = ArvPathMapper(self.arvrunner, reffiles+reffiles2, kwargs["basedir"],
"/keep/%s",
"/keep/%s/%s",
**kwargs)
+ kwargs["extra_reffiles"] = copy.deepcopy(reffiles2)
+
def keepmount(obj):
remove_redundant_fields(obj)
with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index fe1ef22..3ce08f6 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -217,13 +217,13 @@ def packed_workflow(arvrunner, tool, merged_map):
packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
tool.tool["id"], tool.metadata, rewrite_out=rewrites)
- rewrite_to_orig = {}
- for k,v in rewrites.items():
- rewrite_to_orig[v] = k
+ rewrite_to_orig = {v: k for k,v in rewrites.items()}
def visit(v, cur_id):
if isinstance(v, dict):
if v.get("class") in ("CommandLineTool", "Workflow"):
+ if "id" not in v:
+ raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
cur_id = rewrite_to_orig.get(v["id"], v["id"])
if "location" in v and not v["location"].startswith("keep:"):
v["location"] = merged_map[cur_id].resolved[v["location"]]
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 563015c..c636bb8 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180502225535',
+ 'cwltool==1.0.20180504184252',
'schema-salad==2.7.20180501211602',
'typing==3.5.3.0',
'ruamel.yaml >=0.13.7, <0.15',
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 172e6a9..9106426 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -199,7 +199,7 @@
output:
out: null
tool: wf-defaults/wf3.cwl
- doc: "Do not accept a directory literal without a basename
+ doc: "Do not accept a directory literal without a basename"
should_fail: true
- job: null
commit bd3e0a105044c0b61c3b07649c6c284f5f1a641b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu May 3 12:34:05 2018 -0400
12256: Add a bunch of test cases.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 1950867..172e6a9 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -182,3 +182,46 @@
output: {}
tool: secondaryFiles/example3.cwl
doc: Discover secondaryFiles on default values
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf1.cwl
+ doc: "Can have separate default parameters including directory and file inside same directory"
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf2.cwl
+ doc: "Can have a parameter default value that is a directory literal with a file literal"
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf3.cwl
+ doc: "Do not accept a directory literal without a basename
+ should_fail: true
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf4.cwl
+ doc: default in embedded subworkflow
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf5.cwl
+ doc: default in embedded subworkflow
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf6.cwl
+ doc: default in RunInSingleContainer step
+
+- job: null
+ output:
+ out: null
+ tool: wf-defaults/wf7.cwl
+ doc: workflow level default in RunInSingleContainer
diff --git a/sdk/cwl/tests/wf-defaults/default-dir1.cwl b/sdk/cwl/tests/wf-defaults/default-dir1.cwl
new file mode 100644
index 0000000..ed09e6e
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir1.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ inp1:
+ type: File
+ default:
+ class: File
+ location: inp1/hello.txt
+outputs: []
+arguments: [echo, $(inputs.inp1), $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir2.cwl b/sdk/cwl/tests/wf-defaults/default-dir2.cwl
new file mode 100644
index 0000000..c826464
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir2.cwl
@@ -0,0 +1,14 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ basename: inp2
+ listing:
+ - class: File
+ basename: "hello.txt"
+ contents: "hello world"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir3.cwl b/sdk/cwl/tests/wf-defaults/default-dir3.cwl
new file mode 100644
index 0000000..ab7b0a4
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir3.cwl
@@ -0,0 +1,12 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ listing:
+ - class: File
+ location: "inp1/hello.txt"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir4.cwl b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
new file mode 100644
index 0000000..cd57ff3
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
@@ -0,0 +1,20 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run:
+ class: CommandLineTool
+ inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ outputs: []
+ arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir5.cwl b/sdk/cwl/tests/wf-defaults/default-dir5.cwl
new file mode 100644
index 0000000..d4f667c
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir5.cwl
@@ -0,0 +1,21 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run:
+ id: stepid
+ class: CommandLineTool
+ inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+ outputs: []
+ arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir6.cwl b/sdk/cwl/tests/wf-defaults/default-dir6.cwl
new file mode 100644
index 0000000..597ea96
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir6.cwl
@@ -0,0 +1,11 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir6a.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir6a.cwl b/sdk/cwl/tests/wf-defaults/default-dir6a.cwl
new file mode 100644
index 0000000..76437a2
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir6a.cwl
@@ -0,0 +1,10 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir7.cwl b/sdk/cwl/tests/wf-defaults/default-dir7.cwl
new file mode 100644
index 0000000..4e6372b
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir7.cwl
@@ -0,0 +1,17 @@
+cwlVersion: v1.0
+class: Workflow
+inputs:
+ inp2:
+ type: Directory
+ default:
+ class: Directory
+ location: inp1
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+steps:
+ step1:
+ in:
+ inp2: inp2
+ out: []
+ run: default-dir7a.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir7a.cwl b/sdk/cwl/tests/wf-defaults/default-dir7a.cwl
new file mode 100644
index 0000000..df9009a
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir7a.cwl
@@ -0,0 +1,7 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp2:
+ type: Directory
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/inp1/hello.txt b/sdk/cwl/tests/wf-defaults/inp1/hello.txt
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/wf-defaults/wf1.cwl b/sdk/cwl/tests/wf-defaults/wf1.cwl
new file mode 100644
index 0000000..45faa89
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf1.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir1.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf2.cwl b/sdk/cwl/tests/wf-defaults/wf2.cwl
new file mode 100644
index 0000000..7ba96ee
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf2.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir2.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf3.cwl b/sdk/cwl/tests/wf-defaults/wf3.cwl
new file mode 100644
index 0000000..911650d
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf3.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir3.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf4.cwl b/sdk/cwl/tests/wf-defaults/wf4.cwl
new file mode 100644
index 0000000..d6e65af
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf4.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir4.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf5.cwl b/sdk/cwl/tests/wf-defaults/wf5.cwl
new file mode 100644
index 0000000..631af18
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf5.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ in: []
+ out: []
+ run: default-dir5.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf6.cwl b/sdk/cwl/tests/wf-defaults/wf6.cwl
new file mode 100644
index 0000000..bd26cc1
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf6.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ requirements:
+ arv:RunInSingleContainer: {}
+ in: []
+ out: []
+ run: default-dir6.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf7.cwl b/sdk/cwl/tests/wf-defaults/wf7.cwl
new file mode 100644
index 0000000..ac07b9d
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf7.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ SubworkflowFeatureRequirement: {}
+steps:
+ step1:
+ requirements:
+ arv:RunInSingleContainer: {}
+ in: []
+ out: []
+ run: default-dir7.cwl
\ No newline at end of file
commit 8eadbd97fba7c45896558d0aac508756ea557be7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu May 3 10:37:44 2018 -0400
12256: Fix anonymous directory default inputs with file literals
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 998890a..6fedb12 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -107,6 +107,7 @@ class ArvPathMapper(PathMapper):
elif obj["location"].startswith("_:") and "contents" in obj:
with c.open(path + "/" + obj["basename"], "w") as f:
f.write(obj["contents"].encode("utf-8"))
+ remap.append((obj["location"], path + "/" + obj["basename"]))
else:
raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list