[ARVADOS] updated: 559729f94c6676cc6bca707e36417da412f937f5
Git user
git at public.curoverse.com
Mon Sep 19 16:09:02 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/arvworkflow.py | 15 +++++++++++----
sdk/cwl/arvados_cwl/pathmapper.py | 2 +-
sdk/cwl/tests/test_job.py | 15 ++++++++++-----
sdk/cwl/tests/{ => wf}/scatter2.cwl | 34 +++++++++++++++-------------------
sdk/cwl/tests/wf/scatter2_subwf.cwl | 33 +++++++++++++++++++++++++++++++++
5 files changed, 70 insertions(+), 29 deletions(-)
rename sdk/cwl/tests/{ => wf}/scatter2.cwl (66%)
create mode 100644 sdk/cwl/tests/wf/scatter2_subwf.cwl
via 559729f94c6676cc6bca707e36417da412f937f5 (commit)
via 07e339237850f463d18ce6c06446441c6684fc58 (commit)
from 261d51d1c2551c9e7dc87a8fca23caccfc613df4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 559729f94c6676cc6bca707e36417da412f937f5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Sep 19 16:08:56 2016 -0400
10081: Fix quoting issue with workflow & input object documents. Improve test
for single-container subworkflow dispatch.
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 6883c10..3bf62ad 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -64,7 +64,7 @@ class ArvadosWorkflow(Workflow):
packed = pack(document_loader, workflowobj, uri, self.metadata)
def prune_directories(obj):
- if obj["location"].startswith("keep:"):
+ if obj["location"].startswith("keep:") and "listing" in obj:
del obj["listing"]
adjustDirObjs(joborder, prune_directories)
@@ -79,15 +79,15 @@ class ArvadosWorkflow(Workflow):
{
"class": "InitialWorkDirRequirement",
"listing": [{
- "entryname": "workflow.json",
- "entry": yaml.safe_dump(packed).replace('$(', '\$(').replace('${', '\${')
+ "entryname": "workflow.cwl",
+ "entry": yaml.safe_dump(packed).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
}, {
"entryname": "cwl.input.json",
- "entry": "$(JSON.stringify(inputs))"
+ "entry": yaml.safe_dump(joborder).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
}]
}],
"hints": workflowobj["hints"],
- "arguments": ["--no-container", "--move-outputs", "workflow.json", "cwl.input.json"]
+ "arguments": ["--no-container", "--move-outputs", "workflow.cwl", "cwl.input.json"]
}
kwargs["loader"] = self.doc_loader
kwargs["avsc_names"] = self.doc_schema
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 40a64ff..ffdade0 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -52,7 +52,7 @@ class ArvPathMapper(PathMapper):
elif srcobj["class"] == "Directory":
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
- for l in srcobj["listing"]:
+ for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
def addentry(self, obj, c, path, subdirs):
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 9a7199e..ea694e1 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -210,7 +210,7 @@ class TestWorkflow(unittest.TestCase):
tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
- mockcollection().portable_data_hash.return_value = "f101400a398097d4398cdb3eb5d1a7ca+118"
+ mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
@@ -225,7 +225,7 @@ class TestWorkflow(unittest.TestCase):
subwf = f.read()
mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{"sleeptime":5}')])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
runner.api.jobs().create.assert_called_with(
body={
@@ -238,10 +238,10 @@ class TestWorkflow(unittest.TestCase):
'HOME': '$(task.outdir)',
'TMPDIR': '$(task.tmpdir)'},
'task.vwd': {
- 'workflow.json': '$(task.keep)/f101400a398097d4398cdb3eb5d1a7ca+118/workflow.json',
- 'cwl.input.json': '$(task.keep)/f101400a398097d4398cdb3eb5d1a7ca+118/cwl.input.json'
+ 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
+ 'cwl.input.json': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.json'
},
- 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'workflow.json', u'cwl.input.json'],
+ 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'workflow.cwl', u'cwl.input.json'],
'task.stdout': 'cwl.output.json'}]},
'runtime_constraints': {
'min_scratch_mb_per_node': 2048,
diff --git a/sdk/cwl/tests/scatter2.cwl b/sdk/cwl/tests/wf/scatter2.cwl
similarity index 66%
rename from sdk/cwl/tests/scatter2.cwl
rename to sdk/cwl/tests/wf/scatter2.cwl
index 2952983..f73ec2b 100644
--- a/sdk/cwl/tests/scatter2.cwl
+++ b/sdk/cwl/tests/wf/scatter2.cwl
@@ -5,8 +5,11 @@ $namespaces:
inputs:
sleeptime:
type: int[]
- default: [44, 29, 14]
-outputs: []
+ default: [5]
+outputs:
+ out:
+ type: string[]
+ outputSource: scatterstep/out
requirements:
SubworkflowFeatureRequirement: {}
ScatterFeatureRequirement: {}
@@ -16,7 +19,7 @@ steps:
scatterstep:
in:
sleeptime: sleeptime
- out: []
+ out: [out]
scatter: sleeptime
hints:
- class: arv:RunInSingleContainer
@@ -25,11 +28,19 @@ steps:
id: mysub
inputs:
sleeptime: int
- outputs: []
+ outputs:
+ out:
+ type: string
+ outputSource: sleep1/out
steps:
sleep1:
in:
sleeptime: sleeptime
+ blurb:
+ valueFrom: |
+ ${
+ return String(inputs.sleeptime) + "b";
+ }
out: [out]
run:
class: CommandLineTool
@@ -43,18 +54,3 @@ steps:
outputBinding:
outputEval: "out"
baseCommand: sleep
- sleep2:
- in:
- sleeptime:
- source: sleeptime
- valueFrom: $(self+1)
- dep: sleep1/out
- out: []
- run:
- class: CommandLineTool
- inputs:
- sleeptime:
- type: int
- inputBinding: {position: 1}
- outputs: []
- baseCommand: sleep
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
new file mode 100644
index 0000000..0ae1cf0
--- /dev/null
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -0,0 +1,33 @@
+$graph:
+- class: Workflow
+ hints:
+ - {class: 'http://arvados.org/cwl#RunInSingleContainer'}
+ id: '#main'
+ inputs:
+ - {id: '#main/sleeptime', type: int}
+ outputs:
+ - {id: '#main/out', outputSource: '#main/sleep1/out', type: string}
+ requirements:
+ - {class: InlineJavascriptRequirement}
+ - {class: ScatterFeatureRequirement}
+ - {class: StepInputExpressionRequirement}
+ - {class: SubworkflowFeatureRequirement}
+ steps:
+ - id: '#main/sleep1'
+ in:
+ - {id: '#main/sleep1/blurb', valueFrom: "${\n return String(inputs.sleeptime)\
+ \ + \"b\";\n}\n"}
+ - {id: '#main/sleep1/sleeptime', source: '#main/sleeptime'}
+ out: ['#main/sleep1/out']
+ run:
+ baseCommand: sleep
+ class: CommandLineTool
+ inputs:
+ - id: '#main/sleep1/sleeptime'
+ inputBinding: {position: 1}
+ type: int
+ outputs:
+ - id: '#main/sleep1/out'
+ outputBinding: {outputEval: out}
+ type: string
+cwlVersion: v1.0
\ No newline at end of file
commit 07e339237850f463d18ce6c06446441c6684fc58
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Sep 19 14:58:55 2016 -0400
10081: Dump as yaml instead of json. Add test for extracted subworkflow.
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index eed4711..6883c10 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -7,6 +7,7 @@ from cwltool.pack import pack
from cwltool.load_tool import fetch_document
from cwltool.process import shortname
from cwltool.workflow import Workflow
+from cwltool.pathmapper import adjustDirObjs
import ruamel.yaml as yaml
@@ -61,6 +62,12 @@ class ArvadosWorkflow(Workflow):
workflowobj["requirements"] = self.requirements + workflowobj.get("requirements", [])
workflowobj["hints"] = self.hints + workflowobj.get("hints", [])
packed = pack(document_loader, workflowobj, uri, self.metadata)
+
+ def prune_directories(obj):
+ if obj["location"].startswith("keep:"):
+ del obj["listing"]
+ adjustDirObjs(joborder, prune_directories)
+
wf_runner = {
"class": "CommandLineTool",
"baseCommand": "cwltool",
@@ -73,7 +80,7 @@ class ArvadosWorkflow(Workflow):
"class": "InitialWorkDirRequirement",
"listing": [{
"entryname": "workflow.json",
- "entry": json.dumps(packed, sort_keys=True, indent=4).replace('$(', '\$(').replace('${', '\${')
+ "entry": yaml.safe_dump(packed).replace('$(', '\$(').replace('${', '\${')
}, {
"entryname": "cwl.input.json",
"entry": "$(JSON.stringify(inputs))"
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 9d97096..9a7199e 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -1,4 +1,3 @@
-
import logging
import mock
import unittest
@@ -222,6 +221,12 @@ class TestWorkflow(unittest.TestCase):
it.next().run()
it.next().run()
+ with open("tests/wf/scatter2_subwf.cwl") as f:
+ subwf = f.read()
+
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call('{"sleeptime":5}')])
+
runner.api.jobs().create.assert_called_with(
body={
'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list