[ARVADOS] updated: 1.1.4-191-g24ebcf6
Git user
git at public.curoverse.com
Fri May 4 10:30:19 EDT 2018
Summary of changes:
sdk/cwl/arvados_cwl/runner.py | 22 ++++++++++------------
sdk/cwl/tests/arvados-tests.sh | 3 +++
sdk/cwl/tests/arvados-tests.yml | 10 ++++++++++
sdk/cwl/tests/secondaryFiles/example1.cwl | 20 ++++++++++++++++++++
sdk/cwl/tests/secondaryFiles/example3.cwl | 12 ++++++++++++
.../cwl/tests/secondaryFiles/hello.txt | 0
.../cwl/tests/secondaryFiles/hello.txt.idx | 0
sdk/cwl/tests/secondaryFiles/inp3.yml | 3 +++
8 files changed, 58 insertions(+), 12 deletions(-)
create mode 100644 sdk/cwl/tests/secondaryFiles/example1.cwl
create mode 100644 sdk/cwl/tests/secondaryFiles/example3.cwl
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/secondaryFiles/hello.txt (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/secondaryFiles/hello.txt.idx (100%)
create mode 100644 sdk/cwl/tests/secondaryFiles/inp3.yml
via 24ebcf6675fa383b38ba6e594d553a2d2526eab7 (commit)
via cb03587052027b65d99eb89046c41d8c9c5227bc (commit)
from f63bd7af0faad9d0daa8d183d4b722c4bb42f383 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 24ebcf6675fa383b38ba6e594d553a2d2526eab7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 4 10:29:44 2018 -0400
13365: Add tests to confirm regressions are fixed.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
index d3c1e90..4869e3e 100755
--- a/sdk/cwl/tests/arvados-tests.sh
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -9,4 +9,7 @@ fi
if ! arv-get f225e6259bdd63bc7240599648dde9f1+97 > /dev/null ; then
arv-put --portable-data-hash hg19/*
fi
+if ! arv-get 4d8a70b1e63b2aad6984e40e338e2373+69 > /dev/null ; then
+ arv-put --portable-data-hash secondaryFiles/hello.txt*
+fi
exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 934fe47..1950867 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -172,3 +172,13 @@
out: null
tool: wf/runin-reqs-wf4.cwl
doc: "RunInSingleContainer discovers static resource request in subworkflow steps"
+
+- job: secondaryFiles/inp3.yml
+ output: {}
+ tool: secondaryFiles/example1.cwl
+ doc: Discover secondaryFiles at runtime if they are in keep
+
+- job: null
+ output: {}
+ tool: secondaryFiles/example3.cwl
+ doc: Discover secondaryFiles on default values
diff --git a/sdk/cwl/tests/secondaryFiles/example1.cwl b/sdk/cwl/tests/secondaryFiles/example1.cwl
new file mode 100644
index 0000000..540edcf
--- /dev/null
+++ b/sdk/cwl/tests/secondaryFiles/example1.cwl
@@ -0,0 +1,20 @@
+class: Workflow
+cwlVersion: v1.0
+inputs:
+ toplevel_input: File
+outputs: []
+steps:
+ step1:
+ in:
+ step_input: toplevel_input
+ out: []
+ run:
+ id: sub
+ class: CommandLineTool
+ inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ outputs: []
+ baseCommand: echo
diff --git a/sdk/cwl/tests/secondaryFiles/example3.cwl b/sdk/cwl/tests/secondaryFiles/example3.cwl
new file mode 100644
index 0000000..892973b
--- /dev/null
+++ b/sdk/cwl/tests/secondaryFiles/example3.cwl
@@ -0,0 +1,12 @@
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+ step_input:
+ type: File
+ secondaryFiles:
+ - .idx
+ default:
+ class: File
+ location: hello.txt
+outputs: []
+baseCommand: echo
diff --git a/sdk/cwl/tests/secondaryFiles/hello.txt b/sdk/cwl/tests/secondaryFiles/hello.txt
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/secondaryFiles/hello.txt.idx b/sdk/cwl/tests/secondaryFiles/hello.txt.idx
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/secondaryFiles/inp3.yml b/sdk/cwl/tests/secondaryFiles/inp3.yml
new file mode 100644
index 0000000..1107623
--- /dev/null
+++ b/sdk/cwl/tests/secondaryFiles/inp3.yml
@@ -0,0 +1,3 @@
+toplevel_input:
+ class: File
+ location: keep:4d8a70b1e63b2aad6984e40e338e2373+69/hello.txt
\ No newline at end of file
commit cb03587052027b65d99eb89046c41d8c9c5227bc
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 4 10:13:05 2018 -0400
13365: clean up discover_secondary_files
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 8db3e61..fe1ef22 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -64,22 +64,20 @@ def find_defaults(d, op):
for i in d.itervalues():
find_defaults(i, op)
+def setSecondary(t, fileobj, discovered):
+ if isinstance(fileobj, dict) and fileobj.get("class") == "File":
+ if "secondaryFiles" not in fileobj:
+ fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
+ if discovered is not None:
+ discovered[fileobj["location"]] = fileobj["secondaryFiles"]
+ elif isinstance(fileobj, list):
+ for e in fileobj:
+ setSecondary(t, e, discovered)
def discover_secondary_files(inputs, job_order, discovered=None):
for t in inputs:
- def setSecondary(fileobj):
- if isinstance(fileobj, dict) and fileobj.get("class") == "File":
- if "secondaryFiles" not in fileobj:
- fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
- if discovered is not None:
- discovered[fileobj["location"]] = fileobj["secondaryFiles"]
-
- if isinstance(fileobj, list):
- for e in fileobj:
- setSecondary(e)
-
if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
- setSecondary(job_order[shortname(t["id"])])
+ setSecondary(t, job_order[shortname(t["id"])], discovered)
def upload_dependencies(arvrunner, name, document_loader,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list