[ARVADOS] created: 1.2.0-423-g423936db5
Git user
git at public.curoverse.com
Mon Nov 19 11:10:48 EST 2018
at 423936db5b43ba7b60a9088d5b0be80e1dee0599 (commit)
commit 423936db5b43ba7b60a9088d5b0be80e1dee0599
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 22:01:45 2018 -0500
14198: Tests for hints in different places
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/hint-on-tool.cwl
similarity index 75%
copy from sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
copy to sdk/cwl/tests/federation/cases/hint-on-tool.cwl
index 67779f9aa..93e6d2c37 100644
--- a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/hint-on-tool.cwl
@@ -9,7 +9,7 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:runner-home-step-remote
+ dockerPull: arvados/fed-test:hint-on-tool
inputs:
inp:
type: File
@@ -24,8 +24,5 @@ steps:
in:
inp: inp
runOnCluster: runOnCluster
- hints:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum-tool-hint.cwl
diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/hint-on-wf.cwl
similarity index 72%
copy from sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
copy to sdk/cwl/tests/federation/cases/hint-on-wf.cwl
index a74d2f550..43236599e 100644
--- a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
+++ b/sdk/cwl/tests/federation/cases/hint-on-wf.cwl
@@ -9,11 +9,15 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:runner-remote-step-home
+ dockerPull: arvados/fed-test:hint-on-wf
+hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
inputBinding: {}
+ runOnCluster: string
outputs:
hash:
type: File
@@ -23,7 +27,4 @@ steps:
in:
inp: inp
out: [hash]
- hints:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum.cwl
diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 67779f9aa..182ca1ec4 100644
--- a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -11,9 +11,7 @@ requirements:
DockerRequirement:
dockerPull: arvados/fed-test:runner-home-step-remote
inputs:
- inp:
- type: File
- inputBinding: {}
+ inp: File
runOnCluster: string
outputs:
hash:
diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index a74d2f550..963c84f32 100644
--- a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -11,9 +11,8 @@ requirements:
DockerRequirement:
dockerPull: arvados/fed-test:runner-remote-step-home
inputs:
- inp:
- type: File
- inputBinding: {}
+ inp: File
+ runOnCluster: string
outputs:
hash:
type: File
@@ -22,6 +21,7 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
hints:
arv:ClusterTarget:
diff --git a/sdk/cwl/tests/federation/data/hint-on-tool.txt b/sdk/cwl/tests/federation/data/hint-on-tool.txt
new file mode 100644
index 000000000..c39612574
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/hint-on-tool.txt
@@ -0,0 +1,16 @@
+Call me hint-on-tool. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/hint-on-wf.txt b/sdk/cwl/tests/federation/data/hint-on-wf.txt
new file mode 100644
index 000000000..f4aa872e2
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/hint-on-wf.txt
@@ -0,0 +1,16 @@
+Call me hint-on-wf. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index d07cdfd18..a00e6d3d9 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -60,6 +60,12 @@ outputs:
threestep-remote-success:
type: Any
outputSource: threestep-remote/success
+ hint-on-wf-success:
+ type: Any
+ outputSource: hint-on-wf/success
+ hint-on-tool-success:
+ type: Any
+ outputSource: hint-on-tool/success
steps:
base-case:
@@ -495,9 +501,9 @@ steps:
scrub_image: {default: "arvados/fed-test:hint-on-wf"}
scrub_collections:
default:
- - 3bc373e38751fe13dcbd62778d583242+81 # input collection
- - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
- - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ - 862433f328041b2525c90b1dc3c462fd+62 # input collection
+ - 9a68b0b9720977faba8a28e75a4398b7+51 # md5sum output collection
+ - 6a601cddb36ee2f766783b1aa9ff8d66+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
@@ -532,8 +538,8 @@ steps:
scrub_image: {default: "arvados/fed-test:hint-on-tool"}
scrub_collections:
default:
- - 3bc373e38751fe13dcbd62778d583242+81 # input collection
- - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
- - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ - 6803004a4f8db9f8d1d54f6229851599+64 # input collection
+ - cacb0d56235564b5ff485c5b31215ab5+51 # md5sum output collection
+ - 2b50af43fdd84a9e906be2d54b92cddf+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
commit 920363d3419d69477c63e1bb23b3b7f070a341d7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 16:39:09 2018 -0500
14198: Tests prefer setting ClusterTarget hint on the workflow step.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl b/sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl
deleted file mode 100644
index a52b400cf..000000000
--- a/sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-cwlVersion: v1.0
-class: Workflow
-$namespaces:
- arv: "http://arvados.org/cwl#"
-requirements:
- InlineJavascriptRequirement: {}
- DockerRequirement:
- dockerPull: arvados/fed-test:remote-case
-inputs:
- inp:
- type: File
- inputBinding: {}
- runOnCluster: string
-outputs:
- hash:
- type: File
- outputSource: md5sum/hash
-steps:
- md5sum:
- in:
- inp: inp
- runOnCluster: runOnCluster
- out: [hash]
- run: md5sum-tool-hint.cwl
diff --git a/sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl b/sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl
deleted file mode 100644
index f380de791..000000000
--- a/sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-cwlVersion: v1.0
-class: Workflow
-$namespaces:
- arv: "http://arvados.org/cwl#"
-requirements:
- InlineJavascriptRequirement: {}
- DockerRequirement:
- dockerPull: arvados/fed-test:remote-case
-hints:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
-inputs:
- inp:
- type: File
- inputBinding: {}
- runOnCluster: string
-outputs:
- hash:
- type: File
- outputSource: md5sum/hash
-steps:
- md5sum:
- in:
- inp: inp
- runOnCluster: runOnCluster
- out: [hash]
- run: md5sum.cwl
diff --git a/sdk/cwl/tests/federation/cases/remote-case-step-hint.cwl b/sdk/cwl/tests/federation/cases/remote-case.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/cases/remote-case-step-hint.cwl
rename to sdk/cwl/tests/federation/cases/remote-case.cwl
diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 0621dd736..67779f9aa 100644
--- a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -24,5 +24,8 @@ steps:
in:
inp: inp
runOnCluster: runOnCluster
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index 6ce3cce41..a74d2f550 100644
--- a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -14,7 +14,6 @@ inputs:
inp:
type: File
inputBinding: {}
- runOnCluster: string
outputs:
hash:
type: File
@@ -23,6 +22,8 @@ steps:
md5sum:
in:
inp: inp
- runOnCluster: runOnCluster
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/scatter-gather.cwl b/sdk/cwl/tests/federation/cases/scatter-gather.cwl
index a0519b29d..07403ed92 100644
--- a/sdk/cwl/tests/federation/cases/scatter-gather.cwl
+++ b/sdk/cwl/tests/federation/cases/scatter-gather.cwl
@@ -26,6 +26,9 @@ steps:
scatter: [inp, runOnCluster]
scatterMethod: dotproduct
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
cat:
in:
diff --git a/sdk/cwl/tests/federation/cases/threestep-remote.cwl b/sdk/cwl/tests/federation/cases/threestep-remote.cwl
index f5a1b3e4a..8dffc18dd 100644
--- a/sdk/cwl/tests/federation/cases/threestep-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/threestep-remote.cwl
@@ -26,16 +26,25 @@ steps:
inp: inp
runOnCluster: clusterA
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
revB:
in:
inp: md5sum/hash
runOnCluster: clusterB
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev-input-to-output.cwl
revC:
in:
inp: revB/revhash
runOnCluster: clusterC
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev-input-to-output.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
index c6cc811cb..b924c54ec 100644
--- a/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
@@ -26,10 +26,16 @@ steps:
inp: inp
runOnCluster: md5sumCluster
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
rev:
in:
inp: md5sum/hash
runOnCluster: revCluster
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev.cwl
diff --git a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
index 6e8521b2b..c74c24778 100644
--- a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
@@ -26,10 +26,16 @@ steps:
inp: inp
runOnCluster: md5sumCluster
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
rev:
in:
inp: md5sum/hash
runOnCluster: revCluster
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev.cwl
diff --git a/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
index d1f5896f3..3722c99e3 100644
--- a/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
@@ -26,10 +26,16 @@ steps:
inp: inp
runOnCluster: md5sumCluster
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
rev:
in:
inp: md5sum/hash
runOnCluster: revCluster
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev-input-to-output.cwl
diff --git a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
index 7804b2024..e528914c5 100644
--- a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
@@ -26,10 +26,16 @@ steps:
inp: inp
runOnCluster: md5sumCluster
out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
rev:
in:
inp: md5sum/hash
runOnCluster: revCluster
out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
run: rev.cwl
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 40236ce37..d07cdfd18 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -101,7 +101,7 @@ steps:
runner-home-step-remote:
doc: |
Single step workflow with the runner on the home cluster and the
- step on the remote cluster.
+ step on the remote cluster. ClusterTarget hint is on the workflow step.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -463,3 +463,77 @@ steps:
- 8c86dbec7de7948871b5e168ede417e1+120 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ hint-on-wf:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster. ClusterTarget hint is at the workflow level.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/hint-on-wf.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/hint-on-wf.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:hint-on-wf"}
+ scrub_collections:
+ default:
+ - 3bc373e38751fe13dcbd62778d583242+81 # input collection
+ - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
+ - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ hint-on-tool:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster. ClusterTarget hint is at the tool level.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/hint-on-tool.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum-tool-hint.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/hint-on-tool.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:hint-on-tool"}
+ scrub_collections:
+ default:
+ - 3bc373e38751fe13dcbd62778d583242+81 # input collection
+ - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
+ - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit fb5ff6487ae564aa482d72b3ab7bed4c2401c104
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 16:18:19 2018 -0500
14198: Test cases for places that ClusterTarget can appear
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/md5sum-tool-hint.cwl b/sdk/cwl/tests/federation/cases/md5sum-tool-hint.cwl
new file mode 100644
index 000000000..726c33bbd
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/md5sum-tool-hint.cwl
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp: File
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputBinding:
+ glob: out.txt
+stdin: $(inputs.inp.path)
+stdout: out.txt
+arguments: ["md5sum", "-"]
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case-step-hint.cwl
similarity index 100%
copy from sdk/cwl/tests/federation/cases/remote-case.cwl
copy to sdk/cwl/tests/federation/cases/remote-case-step-hint.cwl
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl
similarity index 83%
copy from sdk/cwl/tests/federation/cases/remote-case.cwl
copy to sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl
index 66830620f..a52b400cf 100644
--- a/sdk/cwl/tests/federation/cases/remote-case.cwl
+++ b/sdk/cwl/tests/federation/cases/remote-case-tool-hint.cwl
@@ -25,7 +25,4 @@ steps:
inp: inp
runOnCluster: runOnCluster
out: [hash]
- hints:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
- run: md5sum.cwl
+ run: md5sum-tool-hint.cwl
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl
similarity index 87%
rename from sdk/cwl/tests/federation/cases/remote-case.cwl
rename to sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl
index 66830620f..f380de791 100644
--- a/sdk/cwl/tests/federation/cases/remote-case.cwl
+++ b/sdk/cwl/tests/federation/cases/remote-case-wf-hint.cwl
@@ -10,6 +10,9 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:remote-case
+hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,7 +28,4 @@ steps:
inp: inp
runOnCluster: runOnCluster
out: [hash]
- hints:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
run: md5sum.cwl
commit 8a57ab72f14f00ed08908fa1fa38b2f81643bcf7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 10:59:52 2018 -0500
14198: Set ClusterTarget hint on steps instead of tools
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl
index 57eff5975..4ab3b2096 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/base-case.cwl
@@ -25,4 +25,7 @@ steps:
inp: inp
runOnCluster: runOnCluster
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl
index 8a844e9dd..af119990a 100644
--- a/sdk/cwl/tests/federation/cases/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/md5sum.cwl
@@ -8,12 +8,9 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
- runOnCluster: string
outputs:
hash:
type: File
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case.cwl
index 02d1e953a..66830620f 100644
--- a/sdk/cwl/tests/federation/cases/remote-case.cwl
+++ b/sdk/cwl/tests/federation/cases/remote-case.cwl
@@ -25,4 +25,7 @@ steps:
inp: inp
runOnCluster: runOnCluster
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
diff --git a/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl b/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
index ac9f4e915..0c247a858 100644
--- a/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
+++ b/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
@@ -8,13 +8,10 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
ShellCommandRequirement: {}
inputs:
inp:
type: File
- runOnCluster: string
outputs:
original:
type: File
diff --git a/sdk/cwl/tests/federation/cases/rev.cwl b/sdk/cwl/tests/federation/cases/rev.cwl
index 13e7a87f2..8bbc565d9 100644
--- a/sdk/cwl/tests/federation/cases/rev.cwl
+++ b/sdk/cwl/tests/federation/cases/rev.cwl
@@ -8,12 +8,9 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
- runOnCluster: string
outputs:
revhash:
type: File
commit b9ccf340d62624263b582731d2fdd2e3782d56d5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 14 14:44:19 2018 -0500
14198: Rewrite README
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index e53793900..e5eb04c60 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -2,54 +2,43 @@
#
# SPDX-License-Identifier: Apache-2.0
-Things to test.
+Arvados federated workflow testing
-Single step --submit --no-wait workflow, matrix of:
+Requires cwltool 1.0.20181109150732 or later
-Runner on home/remote cluster
-Docker image on home/remote cluster
-Step on home/remote cluster
+Create main-test.json:
-Two step workflow, matrix of:
+{
+ "acr": "/path/to/arvados-cwl-runner",
+ "arvado_api_host_insecure": false,
+ "arvados_api_hosts": [
+ "c97qk.arvadosapi.com",
+ "4xphq.arvadosapi.com",
+ "9tee4.arvadosapi.com"
+ ],
+ "arvados_api_token": "...",
+ "arvados_cluster_ids": [
+ "c97qk",
+ "4xphq",
+ "9tee4"
+ ]
+}
-Step1 on home cluster -> Step2 on remote cluster
-Step1 on remote cluster -> Step2 on home cluster
+Or create an arvbox test cluster:
-Three step workflow:
+$ cwltool --enable-ext arvbox-make-federation.cwl --arvbox_base ~/.arvbox/ --in_acr /path/to/arvados-cwl-runner > main-test.json
-(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster
-Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
-Workflow which has a remote collection in InitialWorkDir, which is captured in output.
+Run tests:
-Workflow with file input that has a secondary file in a separate
-collection, which is remote.
+$ cwltool main.cwl main-test.json
-Need to pull Docker image
-Can't run in container because it doesn't have access to Docker from inside -> how can we workaround?
-Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this)
-Try setting up a virtualenv -> annoying because you need C dependencies to build
-Virtualenv sets up shop in /tmp instead of the real directory
---relocatable ???
+List test cases:
+$ cwltool --print-targets main.cwl
-arvados/jobs doesn't have the docker client
-arvbox has docker client
-- Still need venv or something to run cwltool to manage all this.
-- Unless we run cwltool in docker as well.
+Run a specific test case:
-Can use cwl-docker.sh to run cwltool in docker
-
-- arvbox needs Docker. This will *probably* work launched from container if base dir is under PWD
-- want to run arvados-cwl-runner. this needs Docker to pull images.
-
-For a-c-r solutions are one of:
-
-1) Don't run in Docker, use virtualenv instead.
-2) Access to Docker from inside Docker to pull images.
-2a) Docker-in-Docker
-2b) Bind mount Docker socket
-2c) Communicate to Docker daemon over TCP
-3) Arvados feature to pull images
+$ cwltool -t twostep-remote-copy-to-home main.cwl main-test.json
commit 2b646db057540526145948dcbe333a1547ba1351
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 13 23:10:07 2018 -0500
14198: Connect test cases to outputs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index fed85b426..40236ce37 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -51,6 +51,15 @@ outputs:
twostep-both-remote-success:
type: Any
outputSource: twostep-both-remote/success
+ twostep-remote-copy-to-home-success:
+ type: Any
+ outputSource: twostep-remote-copy-to-home/success
+ scatter-gather-success:
+ type: Any
+ outputSource: scatter-gather/success
+ threestep-remote-success:
+ type: Any
+ outputSource: threestep-remote/success
steps:
base-case:
commit f4766f228db82e7b4a37a03fff472bcd87a17a74
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 13 22:56:56 2018 -0500
14198: Add threestep-remote
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/threestep-remote.cwl b/sdk/cwl/tests/federation/cases/threestep-remote.cwl
new file mode 100644
index 000000000..f5a1b3e4a
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/threestep-remote.cwl
@@ -0,0 +1,41 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:threestep-remote
+ ScatterFeatureRequirement: {}
+inputs:
+ inp: File
+ clusterA: string
+ clusterB: string
+ clusterC: string
+outputs:
+ revhash:
+ type: File
+ outputSource: revC/revhash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: clusterA
+ out: [hash]
+ run: md5sum.cwl
+ revB:
+ in:
+ inp: md5sum/hash
+ runOnCluster: clusterB
+ out: [revhash]
+ run: rev-input-to-output.cwl
+ revC:
+ in:
+ inp: revB/revhash
+ runOnCluster: clusterC
+ out: [revhash]
+ run: rev-input-to-output.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/data/threestep-remote.txt b/sdk/cwl/tests/federation/data/threestep-remote.txt
new file mode 100644
index 000000000..39dd99bee
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/threestep-remote.txt
@@ -0,0 +1,16 @@
+Call me threestep-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 9058fde0d..fed85b426 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -413,3 +413,44 @@ steps:
- 89de265942800ae36549109969940363+117 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ threestep-remote:
+ doc: ""
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/threestep-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev-input-to-output.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/threestep-remote.txt
+ valueFrom: |-
+ ${
+ self["clusterA"] = inputs.arvados_cluster_ids[0];
+ self["clusterB"] = inputs.arvados_cluster_ids[1];
+ self["clusterC"] = inputs.arvados_cluster_ids[2];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:threestep-remote"}
+ scrub_collections:
+ default:
+ - 9fbf33e62876357fe134f619865cc5a5+68 # input collection
+ - 210c5f2a716f6689b04316acd4928c10+51 # md5sum output collection
+ - 3abea7506269d5ebf61fb17c78bbd2af+105 # revB output
+ - 9e1b3acb28949759ad07e4c9740bbaa5+113 # revC output
+ - 8c86dbec7de7948871b5e168ede417e1+120 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit 85fd63ed6f8c0b1abdf937ba49872989e3261a1c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 13 22:06:36 2018 -0500
14198: Add scatter-gather
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/cat.cwl b/sdk/cwl/tests/federation/cases/cat.cwl
new file mode 100644
index 000000000..17132fe61
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/cat.cwl
@@ -0,0 +1,14 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp:
+ type: File[]
+ inputBinding: {}
+outputs:
+ joined: stdout
+stdout: joined.txt
+baseCommand: cat
diff --git a/sdk/cwl/tests/federation/cases/scatter-gather.cwl b/sdk/cwl/tests/federation/cases/scatter-gather.cwl
new file mode 100644
index 000000000..a0519b29d
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/scatter-gather.cwl
@@ -0,0 +1,34 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:scatter-gather
+ ScatterFeatureRequirement: {}
+inputs:
+ shards: File[]
+ clusters: string[]
+outputs:
+ joined:
+ type: File
+ outputSource: cat/joined
+steps:
+ md5sum:
+ in:
+ inp: shards
+ runOnCluster: clusters
+ scatter: [inp, runOnCluster]
+ scatterMethod: dotproduct
+ out: [hash]
+ run: md5sum.cwl
+ cat:
+ in:
+ inp: md5sum/hash
+ out: [joined]
+ run: cat.cwl
diff --git a/sdk/cwl/tests/federation/data/scatter-gather-s1.txt b/sdk/cwl/tests/federation/data/scatter-gather-s1.txt
new file mode 100644
index 000000000..cc732e3e7
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/scatter-gather-s1.txt
@@ -0,0 +1,16 @@
+Call me scatter-gather-s1. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/scatter-gather-s2.txt b/sdk/cwl/tests/federation/data/scatter-gather-s2.txt
new file mode 100644
index 000000000..3b57ee1ea
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/scatter-gather-s2.txt
@@ -0,0 +1,16 @@
+Call me scatter-gather-s2. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/scatter-gather-s3.txt b/sdk/cwl/tests/federation/data/scatter-gather-s3.txt
new file mode 100644
index 000000000..06f77d285
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/scatter-gather-s3.txt
@@ -0,0 +1,16 @@
+Call me scatter-gather-s3. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 7e1d878af..9058fde0d 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -369,3 +369,47 @@ steps:
- 2d3a4a840077390a0d7788f169eaba89+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ scatter-gather:
+ doc: ""
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/scatter-gather.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/cat.cwl
+ obj:
+ default:
+ shards:
+ - class: File
+ location: data/scatter-gather-s1.txt
+ - class: File
+ location: data/scatter-gather-s2.txt
+ - class: File
+ location: data/scatter-gather-s3.txt
+ valueFrom: |-
+ ${
+ self["clusters"] = inputs.arvados_cluster_ids;
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:scatter-gather"}
+ scrub_collections:
+ default:
+ - 99cc18329bce1b4a5fe6c4cf60477668+209 # input collection
+ - 2e570e844e03c7027baad148642d726f+51 # s1 md5sum output collection
+ - 61c88ee7811d0b849b5c06376eb065a6+51 # s2 md5sum output collection
+ - 85aaf18d638045fe609e025d3a319b2a+51 # s3 md5sum output collection
+ - ec44bcba77e65128f1a8f843d881ede4+56 # cat output collection
+ - 89de265942800ae36549109969940363+117 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit 579f06c0b9567b2ef85fe9422111699685cd6cb0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 13 17:28:59 2018 -0500
14198: Add test twostep-remote-copy-to-home
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl b/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
new file mode 100644
index 000000000..ac9f4e915
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/rev-input-to-output.cwl
@@ -0,0 +1,30 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ ShellCommandRequirement: {}
+inputs:
+ inp:
+ type: File
+ runOnCluster: string
+outputs:
+ original:
+ type: File
+ outputBinding:
+ glob: $(inputs.inp.basename)
+ revhash:
+ type: stdout
+stdout: rev-$(inputs.inp.basename)
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ ln -s $(inputs.inp.path) $(inputs.inp.basename) &&
+ rev $(inputs.inp.basename)
diff --git a/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
new file mode 100644
index 000000000..d1f5896f3
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-copy-to-home.cwl
@@ -0,0 +1,35 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-remote-copy-to-home
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ run: rev-input-to-output.cwl
diff --git a/sdk/cwl/tests/federation/data/twostep-remote-copy-to-home.txt b/sdk/cwl/tests/federation/data/twostep-remote-copy-to-home.txt
new file mode 100644
index 000000000..c0f72ef1f
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-remote-copy-to-home.txt
@@ -0,0 +1,16 @@
+Call me twostep-remote-copy-to-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 2a6b61a3e..7e1d878af 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -324,3 +324,48 @@ steps:
- ddfa58a81953dad08436d571615dd584+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ twostep-remote-copy-to-home:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the remote cluster, the second step is on the home
+ cluster, and propagates its input file directly from input to
+ output by symlinking the input file in the output directory.
+ Tests that crunch-run will copy blocks from remote to local
+ when preparing output collection.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-remote-copy-to-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev-input-to-output.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-remote-copy-to-home.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-remote-copy-to-home"}
+ scrub_collections:
+ default:
+ - 538887bc29a3098bf79abdb8536d17bd+79 # input collection
+ - 14da0e0d52d7ab2945427074b275e9ee+51 # md5sum output collection
+ - 2d3a4a840077390a0d7788f169eaba89+112 # rev output collection
+ - 2d3a4a840077390a0d7788f169eaba89+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit 1ffbef072158d3f920f6fa78707c599528f1883f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 13 16:37:34 2018 -0500
14198: Fix collection PDHs for remote-case
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 732661220..2a6b61a3e 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -194,9 +194,9 @@ steps:
scrub_image: {default: "arvados/fed-test:remote-case"}
scrub_collections:
default:
- - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
- - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
- - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ - fccd49fdef8e452295f718208abafd88+69 # input collection
+ - 58c0e8ea6b148134ef8577ee11307eec+51 # md5sum output collection
+ - 1fd679c5ab64c123b9764024dbf560f0+112 # final output json
out: [out, success]
run: framework/testcase.cwl
commit f545e75fd93985b411e785733394a4098b4cb570
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:48:14 2018 -0500
14198: Add some basic doc comments.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 4b95d8c03..732661220 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -54,6 +54,9 @@ outputs:
steps:
base-case:
+ doc: |
+ Base case (no federation), single step workflow with both the
+ runner and step on the same cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -87,6 +90,9 @@ steps:
run: framework/testcase.cwl
runner-home-step-remote:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -121,6 +127,9 @@ steps:
run: framework/testcase.cwl
runner-remote-step-home:
+ doc: |
+ Single step workflow with the runner on the remote cluster and the
+ step on the home cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -155,6 +164,9 @@ steps:
run: framework/testcase.cwl
remote-case:
+ doc: |
+ Single step workflow with both the runner and the step on the
+ remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -189,6 +201,10 @@ steps:
run: framework/testcase.cwl
twostep-home-to-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the home cluster, the second step is on the remote
+ cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -227,6 +243,10 @@ steps:
run: framework/testcase.cwl
twostep-remote-to-home:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the remote cluster, the second step is on the home
+ cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -265,6 +285,9 @@ steps:
run: framework/testcase.cwl
twostep-both-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, both steps are
+ on the remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
commit afebc919fe84beec227b4dd73d8f702fdb054d9f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:33:05 2018 -0500
14198: Add remote-case and twostep-both-remote
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case.cwl
new file mode 100644
index 000000000..02d1e953a
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/remote-case.cwl
@@ -0,0 +1,28 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:remote-case
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
new file mode 100644
index 000000000..c6cc811cb
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
@@ -0,0 +1,35 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-both-remote
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ run: rev.cwl
diff --git a/sdk/cwl/tests/federation/data/remote-case-input.txt b/sdk/cwl/tests/federation/data/remote-case-input.txt
new file mode 100644
index 000000000..21e87fb1d
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/remote-case-input.txt
@@ -0,0 +1,16 @@
+Call me remote-case. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/twostep-both-remote.txt b/sdk/cwl/tests/federation/data/twostep-both-remote.txt
new file mode 100644
index 000000000..6218bb50e
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-both-remote.txt
@@ -0,0 +1,16 @@
+Call me twostep-both-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 602cc2fcb..4b95d8c03 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -39,13 +39,16 @@ outputs:
runner-remote-step-home-success:
type: Any
outputSource: runner-remote-step-home/success
+ remote-case-success:
+ type: Any
+ outputSource: remote-case/success
twostep-home-to-remote-success:
type: Any
outputSource: twostep-home-to-remote/success
twostep-remote-to-home-success:
type: Any
outputSource: twostep-remote-to-home/success
- twostep-both-remote:
+ twostep-both-remote-success:
type: Any
outputSource: twostep-both-remote/success
@@ -151,6 +154,40 @@ steps:
out: [out, success]
run: framework/testcase.cwl
+ remote-case:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/remote-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/remote-case-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
+ scrub_image: {default: "arvados/fed-test:remote-case"}
+ scrub_collections:
+ default:
+ - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
+ - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
+ - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
twostep-home-to-remote:
in:
arvados_api_token: arvados_api_token
@@ -264,5 +301,3 @@ steps:
- ddfa58a81953dad08436d571615dd584+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
-
- # also: twostep-all-remote
commit 9a801efd4710643fb58bbe5d32769398bc57010d
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:02:50 2018 -0500
14198: Add more twostep tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
index ffce6c3c4..6e8521b2b 100644
--- a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
@@ -33,6 +33,3 @@ steps:
runOnCluster: revCluster
out: [revhash]
run: rev.cwl
- requirements:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
diff --git a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
index 4541ebba2..7804b2024 100644
--- a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
@@ -14,7 +14,8 @@ inputs:
inp:
type: File
inputBinding: {}
- runOnCluster: string
+ md5sumCluster: string
+ revCluster: string
outputs:
hash:
type: File
@@ -23,13 +24,12 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: md5sumCluster
out: [hash]
run: md5sum.cwl
- requirements:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
rev:
in:
inp: md5sum/hash
+ runOnCluster: revCluster
out: [revhash]
run: rev.cwl
diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
index 52c780a87..6fe90813e 100644
--- a/sdk/cwl/tests/federation/framework/prepare.py
+++ b/sdk/cwl/tests/federation/framework/prepare.py
@@ -36,7 +36,6 @@ for cluster_id in config["arvados_cluster_ids"]:
for cluster_id in config["arvados_cluster_ids"]:
matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
- print("matches from %s was %s" % (cluster_id, matches))
for m in matches["items"]:
api.collections().delete(uuid=m["uuid"]).execute()
print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 346f97914..602cc2fcb 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -42,6 +42,12 @@ outputs:
twostep-home-to-remote-success:
type: Any
outputSource: twostep-home-to-remote/success
+ twostep-remote-to-home-success:
+ type: Any
+ outputSource: twostep-remote-to-home/success
+ twostep-both-remote:
+ type: Any
+ outputSource: twostep-both-remote/success
steps:
base-case:
@@ -183,4 +189,80 @@ steps:
out: [out, success]
run: framework/testcase.cwl
+ twostep-remote-to-home:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-remote-to-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-remote-to-home.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-remote-to-home"}
+ scrub_collections:
+ default:
+ - cce89b9f7b6e163978144051ce5f071a+74 # input collection
+ - 0c358c3af63644c6343766feff1b7238+51 # md5sum output collection
+ - 33fb7d512bf21f04847eca58cea46e74+51 # rev output collection
+ - 912e04aa3db04aba008cf5cd46c277b2+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-both-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-both-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-both-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-both-remote"}
+ scrub_collections:
+ default:
+ - 3c5e39939cf197d304ac1eac20841238+71 # input collection
+ - 3edb99aa607731593969cdab663d65b4+51 # md5sum output collection
+ - a91625b7139e60fe61a88cae42fbee13+51 # rev output collection
+ - ddfa58a81953dad08436d571615dd584+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
# also: twostep-all-remote
commit a95db1a9c3552f690ae43b9b37acf5332f675ae0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 10:47:23 2018 -0500
14198: Working on two step test cases
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl
index 2e0ff64c6..57eff5975 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/base-case.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl
index af119990a..8a844e9dd 100644
--- a/sdk/cwl/tests/federation/cases/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/md5sum.cwl
@@ -8,9 +8,12 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
+ runOnCluster: string
outputs:
hash:
type: File
diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/rev.cwl
similarity index 73%
copy from sdk/cwl/tests/federation/cases/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/rev.cwl
index af119990a..13e7a87f2 100644
--- a/sdk/cwl/tests/federation/cases/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/rev.cwl
@@ -8,14 +8,16 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
+ runOnCluster: string
outputs:
- hash:
+ revhash:
type: File
outputBinding:
glob: out.txt
-stdin: $(inputs.inp.path)
stdout: out.txt
-arguments: ["md5sum", "-"]
+arguments: [rev, $(inputs.inp)]
diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 70d017639..0621dd736 100644
--- a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:runner-home-step-remote
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index 33681f18e..6ce3cce41 100644
--- a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:runner-remote-step-home
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
similarity index 54%
copy from sdk/cwl/tests/federation/cases/base-case.cwl
copy to sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
index 2e0ff64c6..ffce6c3c4 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
@@ -9,14 +9,13 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
+ dockerPull: arvados/fed-test:twostep-home-to-remote
inputs:
inp:
type: File
inputBinding: {}
- runOnCluster: string
+ md5sumCluster: string
+ revCluster: string
outputs:
hash:
type: File
@@ -25,5 +24,15 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: md5sumCluster
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ run: rev.cwl
+ requirements:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
similarity index 65%
copy from sdk/cwl/tests/federation/cases/base-case.cwl
copy to sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
index 2e0ff64c6..4541ebba2 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
@@ -9,9 +9,7 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
+ dockerPull: arvados/fed-test:twostep-remote-to-home
inputs:
inp:
type: File
@@ -26,4 +24,12 @@ steps:
in:
inp: inp
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum.cwl
+ requirements:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ rev:
+ in:
+ inp: md5sum/hash
+ out: [revhash]
+ run: rev.cwl
diff --git a/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt b/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt
new file mode 100644
index 000000000..6430ad509
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt
@@ -0,0 +1,16 @@
+Call me twostep-home-to-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt b/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt
new file mode 100644
index 000000000..231802581
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt
@@ -0,0 +1,16 @@
+Call me twostep-remote-to-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
index 94a6a75ed..52c780a87 100644
--- a/sdk/cwl/tests/federation/framework/prepare.py
+++ b/sdk/cwl/tests/federation/framework/prepare.py
@@ -33,10 +33,10 @@ for cluster_id in config["arvados_cluster_ids"]:
for lk in search_links["items"]:
api.links().delete(uuid=lk["uuid"]).execute()
-
for cluster_id in config["arvados_cluster_ids"]:
matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ print("matches from %s was %s" % (cluster_id, matches))
for m in matches["items"]:
api.collections().delete(uuid=m["uuid"]).execute()
print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 1042350f9..346f97914 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -39,6 +39,9 @@ outputs:
runner-remote-step-home-success:
type: Any
outputSource: runner-remote-step-home/success
+ twostep-home-to-remote-success:
+ type: Any
+ outputSource: twostep-home-to-remote/success
steps:
base-case:
@@ -141,3 +144,43 @@ steps:
- ecb639201f454b6493757f5117f540df+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ twostep-home-to-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-home-to-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-home-to-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[0];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-home-to-remote"}
+ scrub_collections:
+ default:
+ - 268a54947fb75115cfe05bb54cc62c30+74 # input collection
+ - 400f03b8c5d2dc3dcb513a21b626ef88+51 # md5sum output collection
+ - 3738166916ca5f6f6ad12bf7e06b4a21+51 # rev output collection
+ - bc37c17a37aa25229e5de1339b27fbcc+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ # also: twostep-all-remote
commit 833e34673af439297e44a791858c9b6b4a5c436a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 8 22:19:15 2018 -0500
14198: Refactor arvbox federation setup
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
similarity index 94%
rename from sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
rename to sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 91e622022..9a08195a7 100644
--- a/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -44,7 +44,7 @@ steps:
containers: containers
arvbox_base: arvbox_base
out: [arvbox_data]
- run: arvbox-mkdir.cwl
+ run: arvbox/mkdir.cwl
start:
in:
container_name: containers
@@ -52,7 +52,7 @@ steps:
out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
- run: arvbox-start.cwl
+ run: arvbox/start.cwl
fed-config:
in:
container_name: containers
@@ -63,10 +63,10 @@ steps:
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
- run: arvbox-fed-config.cwl
+ run: arvbox/fed-config.cwl
setup-user:
in:
container_host: {source: start/container_host, valueFrom: "$(self[0])"}
superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
out: [test_user_uuid, test_user_token]
- run: arvbox-setup-user.cwl
+ run: arvbox/setup-user.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
deleted file mode 100644
index a6fd00823..000000000
--- a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-cwlVersion: v1.0
-class: Workflow
-$namespaces:
- arv: "http://arvados.org/cwl#"
- cwltool: "http://commonwl.org/cwltool#"
-requirements:
- cwltool:LoadListingRequirement:
- loadListing: no_listing
- SubworkflowFeatureRequirement: {}
-inputs:
- arvbox_base: Directory
- acr: string?
-outputs: []
-steps:
- run-arvbox:
- in:
- containers:
- default: [fedbox1, fedbox2, fedbox3]
- arvbox_base: arvbox_base
- out: [cluster_ids, container_hosts, test_user_uuid, test_user_token]
- run: arvbox-fed.cwl
- run-main:
- in:
- arvados_api_host_home: {source: run-arvbox/container_hosts, valueFrom: "$(self[0])"}
- arvados_home_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[0])"}
- arvados_api_token: run-arvbox/test_user_token
- arvado_api_host_insecure: {default: true}
- arvados_api_host_clusterB: {source: run-arvbox/container_hosts, valueFrom: "$(self[1])"}
- arvados_clusterB_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[1])"}
- arvados_api_host_clusterC: {source: run-arvbox/container_hosts, valueFrom: "$(self[2])"}
- arvados_clusterC_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[2])"}
- acr: acr
- out: [base-case-out, runner-home-step-remote-out]
- run: main.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
rename to sdk/cwl/tests/federation/arvbox/fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox/mkdir.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
rename to sdk/cwl/tests/federation/arvbox/mkdir.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox/setup-user.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
rename to sdk/cwl/tests/federation/arvbox/setup-user.cwl
diff --git a/sdk/cwl/tests/federation/framework/setup_user.py b/sdk/cwl/tests/federation/arvbox/setup_user.py
similarity index 100%
rename from sdk/cwl/tests/federation/framework/setup_user.py
rename to sdk/cwl/tests/federation/arvbox/setup_user.py
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox/start.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
rename to sdk/cwl/tests/federation/arvbox/start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox/stop.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
rename to sdk/cwl/tests/federation/arvbox/stop.cwl
commit 7bca46888446919d6c3f8fbd49721d8c6b709c09
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 8 16:13:18 2018 -0500
14198: Refactored. Tests use isolated inputs. Return success boolean
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/.licenseignore b/.licenseignore
index 5d3eff344..113bf4fa4 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -46,7 +46,7 @@ docker/jobs/apt.arvados.org.list
*/script/rails
sdk/cwl/tests/input/blorp.txt
sdk/cwl/tests/tool/blub.txt
-sdk/cwl/tests/federation/whale.txt
+sdk/cwl/tests/federation/data/*
sdk/go/manifest/testdata/*_manifest
sdk/java/.classpath
sdk/java/pom.xml
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-fed-config.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-fed.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
new file mode 100644
index 000000000..a6fd00823
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
@@ -0,0 +1,38 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvbox_base: Directory
+ acr: string?
+outputs: []
+steps:
+ run-arvbox:
+ in:
+ containers:
+ default: [fedbox1, fedbox2, fedbox3]
+ arvbox_base: arvbox_base
+ out: [cluster_ids, container_hosts, test_user_uuid, test_user_token]
+ run: arvbox-fed.cwl
+ run-main:
+ in:
+ arvados_api_host_home: {source: run-arvbox/container_hosts, valueFrom: "$(self[0])"}
+ arvados_home_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[0])"}
+ arvados_api_token: run-arvbox/test_user_token
+ arvado_api_host_insecure: {default: true}
+ arvados_api_host_clusterB: {source: run-arvbox/container_hosts, valueFrom: "$(self[1])"}
+ arvados_clusterB_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[1])"}
+ arvados_api_host_clusterC: {source: run-arvbox/container_hosts, valueFrom: "$(self[2])"}
+ arvados_clusterC_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[2])"}
+ acr: acr
+ out: [base-case-out, runner-home-step-remote-out]
+ run: main.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-mkdir.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-setup-user.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-start.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-stop.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl
similarity index 63%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/base-case.cwl
index 18f3fb0f9..2e0ff64c6 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/base-case.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:base-case
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl
similarity index 55%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/md5sum.cwl
index 18f3fb0f9..af119990a 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/md5sum.cwl
@@ -8,21 +8,14 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
- DockerRequirement:
- dockerPull: debian:9
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
- inputBinding: {}
- runOnCluster: string
outputs:
hash:
- type: string
+ type: File
outputBinding:
glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
+stdin: $(inputs.inp.path)
stdout: out.txt
-baseCommand: md5sum
+arguments: ["md5sum", "-"]
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
similarity index 63%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 18f3fb0f9..70d017639 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:runner-home-step-remote
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
similarity index 63%
rename from sdk/cwl/tests/federation/md5sum.cwl
rename to sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index 18f3fb0f9..33681f18e 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:runner-remote-step-home
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/base-case-input.txt
similarity index 93%
copy from sdk/cwl/tests/federation/whale.txt
copy to sdk/cwl/tests/federation/data/base-case-input.txt
index 425d1ed02..761b840cd 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/base-case-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me base-case. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
similarity index 92%
copy from sdk/cwl/tests/federation/whale.txt
copy to sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
index 425d1ed02..91ab77d12 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me runner-home-step-remote. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
similarity index 92%
rename from sdk/cwl/tests/federation/whale.txt
rename to sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
index 425d1ed02..e5673b848 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me runner-remote-step-home. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/framework/check-exist.cwl
similarity index 65%
copy from sdk/cwl/tests/federation/prepare.cwl
copy to sdk/cwl/tests/federation/framework/check-exist.cwl
index 6c82605bb..ebb0fb220 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/framework/check-exist.cwl
@@ -7,15 +7,11 @@ class: CommandLineTool
requirements:
InitialWorkDirRequirement:
listing:
- - entryname: input.json
- entry: $(JSON.stringify(inputs.obj))
- entryname: config.json
entry: |-
${
return JSON.stringify({
- arvados_cluster_ids: inputs.arvados_cluster_ids,
- scrub_images: inputs.scrub_images,
- scrub_collections: inputs.scrub_collections
+ check_collections: inputs.check_collections
});
}
EnvVarRequirement:
@@ -28,21 +24,19 @@ inputs:
arvados_api_token: string
arvado_api_host_insecure: boolean
arvados_api_host: string
- arvados_cluster_ids: string[]
- wf: File
- obj: Any
- scrub_images: string[]
- scrub_collections: string[]
+ check_collections: string[]
preparescript:
type: File
default:
class: File
- location: prepare.py
+ location: check_exist.py
inputBinding:
position: 1
outputs:
- done:
+ success:
type: boolean
outputBinding:
- outputEval: $(true)
+ glob: success
+ loadContents: true
+ outputEval: $(self[0].contents=="true")
baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/framework/check_exist.py b/sdk/cwl/tests/federation/framework/check_exist.py
new file mode 100644
index 000000000..b3338939e
--- /dev/null
+++ b/sdk/cwl/tests/federation/framework/check_exist.py
@@ -0,0 +1,25 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import json
+
+api = arvados.api()
+
+with open("config.json") as f:
+ config = json.load(f)
+
+success = True
+for c in config["check_collections"]:
+ try:
+ api.collections().get(uuid=c).execute()
+ except Exception as e:
+ print("Checking for %s got exception %s" % (c, e))
+ success = False
+
+with open("success", "w") as f:
+ if success:
+ f.write("true")
+ else:
+ f.write("false")
diff --git a/sdk/cwl/tests/federation/framework/dockerbuild.cwl b/sdk/cwl/tests/federation/framework/dockerbuild.cwl
new file mode 100644
index 000000000..d00b3e2a5
--- /dev/null
+++ b/sdk/cwl/tests/federation/framework/dockerbuild.cwl
@@ -0,0 +1,21 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ testcase: string
+outputs:
+ imagename:
+ type: string
+ outputBinding:
+ outputEval: $(inputs.testcase)
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: Dockerfile
+ entry: |-
+ FROM debian at sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b
+ LABEL org.arvados.testcase="$(inputs.testcase)"
+arguments: [docker, build, -t, $(inputs.testcase), "."]
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/framework/prepare.cwl
similarity index 92%
rename from sdk/cwl/tests/federation/prepare.cwl
rename to sdk/cwl/tests/federation/framework/prepare.cwl
index 6c82605bb..03f792c5e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/framework/prepare.cwl
@@ -14,7 +14,7 @@ requirements:
${
return JSON.stringify({
arvados_cluster_ids: inputs.arvados_cluster_ids,
- scrub_images: inputs.scrub_images,
+ scrub_images: [inputs.scrub_image],
scrub_collections: inputs.scrub_collections
});
}
@@ -31,7 +31,7 @@ inputs:
arvados_cluster_ids: string[]
wf: File
obj: Any
- scrub_images: string[]
+ scrub_image: string
scrub_collections: string[]
preparescript:
type: File
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
similarity index 100%
rename from sdk/cwl/tests/federation/prepare.py
rename to sdk/cwl/tests/federation/framework/prepare.py
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/framework/run-acr.cwl
similarity index 91%
rename from sdk/cwl/tests/federation/run-acr.cwl
rename to sdk/cwl/tests/federation/framework/run-acr.cwl
index 081186506..b720eaa12 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/framework/run-acr.cwl
@@ -42,7 +42,8 @@ outputs:
outputBinding:
glob: output.json
loadContents: true
- outputEval: $(JSON.parse(self[0].contents))
+ #outputEval: $(JSON.parse(self[0].contents))
+ outputEval: $(self[0].contents)
stdout: output.json
arguments:
- valueFrom: --disable-reuse
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/framework/setup_user.py
similarity index 100%
rename from sdk/cwl/tests/federation/setup_user.py
rename to sdk/cwl/tests/federation/framework/setup_user.py
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/framework/testcase.cwl
similarity index 71%
rename from sdk/cwl/tests/federation/testcase.cwl
rename to sdk/cwl/tests/federation/framework/testcase.cwl
index b5ef43091..89aa3f98b 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/framework/testcase.cwl
@@ -25,14 +25,22 @@ inputs:
acr: string?
wf: File
obj: Any
- scrub_images: string[]
+ scrub_image: string
scrub_collections: string[]
runner_cluster: string?
outputs:
out:
type: Any
outputSource: run-acr/out
+ success:
+ type: boolean
+ outputSource: check-result/success
steps:
+ dockerbuild:
+ in:
+ testcase: scrub_image
+ out: [imagename]
+ run: dockerbuild.cwl
prepare:
in:
arvados_api_token: arvados_api_token
@@ -41,13 +49,14 @@ steps:
arvados_cluster_ids: arvados_cluster_ids
wf: wf
obj: obj
- scrub_images: scrub_images
+ scrub_image: scrub_image
scrub_collections: scrub_collections
out: [done]
run: prepare.cwl
run-acr:
in:
prepare: prepare/done
+ image-ready: dockerbuild/imagename
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
@@ -57,3 +66,12 @@ steps:
obj: obj
out: [out]
run: run-acr.cwl
+ check-result:
+ in:
+ acr-done: run-acr/out
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ check_collections: scrub_collections
+ out: [success]
+ run: check-exist.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 4029d53d5..1042350f9 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -23,14 +23,22 @@ inputs:
arvados_api_hosts: string[]
arvados_cluster_ids: string[]
acr: string?
-
+ testcases:
+ type: string[]
+ default:
+ - base-case
+ - runner-home-step-remote
+ - runner-remote-step-home
outputs:
- base-case-out:
+ base-case-success:
+ type: Any
+ outputSource: base-case/success
+ runner-home-step-remote-success:
type: Any
- outputSource: base-case/out
- runner-home-step-remote-out:
+ outputSource: runner-home-step-remote/success
+ runner-remote-step-home-success:
type: Any
- outputSource: runner-home-step-remote/out
+ outputSource: runner-remote-step-home/success
steps:
base-case:
@@ -40,21 +48,31 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/base-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/base-case-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:base-case"}
+ scrub_collections:
+ default:
+ - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
+ - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
+ - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
runner-home-step-remote:
in:
@@ -63,22 +81,32 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/runner-home-step-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/runner-home-step-remote-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:runner-home-step-remote"}
+ scrub_collections:
+ default:
+ - 3bc373e38751fe13dcbd62778d583242+81 # input collection
+ - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
+ - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
runner-remote-step-home:
in:
@@ -87,19 +115,29 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/runner-remote-step-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/runner-remote-step-home-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:runner-remote-step-home"}
+ scrub_collections:
+ default:
+ - 25fe10d8e8530329a738de69d9bc8ab5+81 # input collection
+ - 7f052d1a04b851b6f73fba77c7802e1d+51 # md5sum output collection
+ - ecb639201f454b6493757f5117f540df+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit 2e3a0b9f54535b9fa36a8584b911a27e638aa844
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 16:40:27 2018 -0500
14198: Add runner-remote-step-home
Add license headers
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/.licenseignore b/.licenseignore
index 51a1e7cbd..5d3eff344 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -46,6 +46,7 @@ docker/jobs/apt.arvados.org.list
*/script/rails
sdk/cwl/tests/input/blorp.txt
sdk/cwl/tests/tool/blub.txt
+sdk/cwl/tests/federation/whale.txt
sdk/go/manifest/testdata/*_manifest
sdk/java/.classpath
sdk/java/pom.xml
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 721e6b775..e53793900 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
Things to test.
Single step --submit --no-wait workflow, matrix of:
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 173b9df72..77567ee89 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 8d7075a1d..91e622022 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
index b047beb9b..727d491a3 100644
--- a/sdk/cwl/tests/federation/arvbox-mkdir.cwl
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
index 684bc8c8d..0fddc1b87 100644
--- a/sdk/cwl/tests/federation/arvbox-setup-user.cwl
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index ae591d58f..f69775a53 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
index c4a5412de..2ea4c0f74 100644
--- a/sdk/cwl/tests/federation/arvbox-stop.cwl
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index d481f5c19..4029d53d5 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -1,4 +1,8 @@
#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
@@ -70,6 +74,31 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
+ out: [out]
+ run: testcase.cwl
+
+ runner-remote-step-home:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
scrub_images: {default: ["debian:9"]}
scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
index 55e1fbf8a..18f3fb0f9 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index 2463a466e..6c82605bb 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
requirements:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index a0ac95588..94a6a75ed 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import json
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index 887d113b9..081186506 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
@@ -11,7 +15,7 @@ inputs:
arvado_api_host_insecure:
type: boolean
default: false
- runner_remote_host:
+ runner_cluster:
type: string?
inputBinding:
prefix: --submit-runner-cluster
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2fc621521..a456976be 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import arvados.errors
import time
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index 66c510071..b5ef43091 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -1,4 +1,8 @@
#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
@@ -23,6 +27,7 @@ inputs:
obj: Any
scrub_images: string[]
scrub_collections: string[]
+ runner_cluster: string?
outputs:
out:
type: Any
@@ -46,6 +51,7 @@ steps:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ runner_cluster: runner_cluster
acr: acr
wf: wf
obj: obj
commit 81ee7a02f195c61cbd2c471607e16da5c65e5cf8
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 16:23:01 2018 -0500
14198: Use --always-submit-runner
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index f1d101260..d481f5c19 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -48,7 +48,7 @@ steps:
return self;
}
scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
run: testcase.cwl
@@ -71,6 +71,6 @@ steps:
return self;
}
scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index ed9599194..887d113b9 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -43,5 +43,7 @@ stdout: output.json
arguments:
- valueFrom: --disable-reuse
position: 2
+ - valueFrom: --always-submit-runner
+ position: 2
- valueFrom: input.json
position: 4
\ No newline at end of file
commit 0ac9e1beeaf8a090aea74af4346a7f98550526c4
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 15:13:28 2018 -0500
14198: Scrubbing works
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index bdf16482e..a0ac95588 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -6,16 +6,33 @@ api = arvados.api()
with open("config.json") as f:
config = json.load(f)
+scrub_collections = set(config["scrub_collections"])
+
for cluster_id in config["arvados_cluster_ids"]:
+ images = []
for scrub_image in config["scrub_images"]:
sp = scrub_image.split(":")
image_name = sp[0]
image_tag = sp[1] if len(sp) > 1 else "latest"
+ images.append('{}:{}'.format(image_name, image_tag))
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', 'in', images]],
+ cluster_id=cluster_id).execute()
+
+ head_uuids = [lk["head_uuid"] for lk in search_links["items"]]
+ cols = api.collections().list(filters=[["uuid", "in", head_uuids]],
+ cluster_id=cluster_id).execute()
+ for c in cols["items"]:
+ scrub_collections.add(c["portable_data_hash"])
+ for lk in search_links["items"]:
+ api.links().delete(uuid=lk["uuid"]).execute()
- search_links = api.links().list(
- filters=[['link_class', '=', 'docker_image_repo+tag'],
- ['name', '=',
- '{}:{}'.format(image_name, image_tag)]],
- cluster_id=cluster_id).execute()
- for s in search_links["items"]:
- print s
+
+for cluster_id in config["arvados_cluster_ids"]:
+ matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
+ select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ for m in matches["items"]:
+ api.collections().delete(uuid=m["uuid"]).execute()
+ print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
commit 148293daf42c2b13c5bb29ce7614ca5fbd330eb8
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:37:08 2018 -0500
14198: More work on scrub
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index e7101dc9e..f1d101260 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -47,6 +47,8 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
@@ -68,5 +70,7 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index e12aae464..2463a466e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -5,8 +5,15 @@ requirements:
listing:
- entryname: input.json
entry: $(JSON.stringify(inputs.obj))
- - entryname: clusters.json
- entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ - entryname: config.json
+ entry: |-
+ ${
+ return JSON.stringify({
+ arvados_cluster_ids: inputs.arvados_cluster_ids,
+ scrub_images: inputs.scrub_images,
+ scrub_collections: inputs.scrub_collections
+ });
+ }
EnvVarRequirement:
envDef:
ARVADOS_API_HOST: $(inputs.arvados_api_host)
@@ -20,6 +27,8 @@ inputs:
arvados_cluster_ids: string[]
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
preparescript:
type: File
default:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index 48055224f..bdf16482e 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,5 +1,21 @@
import arvados
+import json
api = arvados.api()
-print(api.users().current().execute())
+with open("config.json") as f:
+ config = json.load(f)
+
+for cluster_id in config["arvados_cluster_ids"]:
+ for scrub_image in config["scrub_images"]:
+ sp = scrub_image.split(":")
+ image_name = sp[0]
+ image_tag = sp[1] if len(sp) > 1 else "latest"
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', '=',
+ '{}:{}'.format(image_name, image_tag)]],
+ cluster_id=cluster_id).execute()
+ for s in search_links["items"]:
+ print s
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index a22d57374..66c510071 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -21,6 +21,8 @@ inputs:
acr: string?
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
outputs:
out:
type: Any
@@ -34,6 +36,8 @@ steps:
arvados_cluster_ids: arvados_cluster_ids
wf: wf
obj: obj
+ scrub_images: scrub_images
+ scrub_collections: scrub_collections
out: [done]
run: prepare.cwl
run-acr:
commit 43006cc3595ba200bca2a297ee9c633a352bb6b8
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:01:30 2018 -0500
14198: Working on prepare step.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
new file mode 100644
index 000000000..e12aae464
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -0,0 +1,35 @@
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ - entryname: clusters.json
+ entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure: boolean
+ arvados_api_host: string
+ arvados_cluster_ids: string[]
+ wf: File
+ obj: Any
+ preparescript:
+ type: File
+ default:
+ class: File
+ location: prepare.py
+ inputBinding:
+ position: 1
+outputs:
+ done:
+ type: boolean
+ outputBinding:
+ outputEval: $(true)
+baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
new file mode 100644
index 000000000..48055224f
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -0,0 +1,5 @@
+import arvados
+
+api = arvados.api()
+
+print(api.users().current().execute())
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index a178a2c88..ed9599194 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -6,9 +6,9 @@ inputs:
default: arvados-cwl-runner
inputBinding:
position: 1
- arv_host: string
- arv_token: string
- arv_insecure:
+ arvados_api_host: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
type: boolean
default: false
runner_remote_host:
@@ -28,9 +28,9 @@ requirements:
entry: $(JSON.stringify(inputs.obj))
EnvVarRequirement:
envDef:
- ARVADOS_API_HOST: $(inputs.arv_host)
- ARVADOS_API_TOKEN: $(inputs.arv_token)
- ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
InlineJavascriptRequirement: {}
outputs:
out:
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2b0e56c60..2fc621521 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -18,16 +18,19 @@ else:
u = api.users().create(body={
'first_name': 'Test',
'last_name': 'User',
- 'email': 'test at example.com'
+ 'email': 'test at example.com',
+ 'is_admin': False
}).execute()
api.users().activate(uuid=u["uuid"]).execute()
tok = api.api_client_authorizations().create(body={
- "owner_uuid": u["uuid"]
+ "api_client_authorization": {
+ "owner_uuid": u["uuid"]
+ }
}).execute()
with open("cwl.output.json", "w") as f:
json.dump({
"test_user_uuid": u["uuid"],
- "test_user_token": tok["api_token"]
+ "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"])
}, f)
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index f4b8f7320..a22d57374 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -26,11 +26,22 @@ outputs:
type: Any
outputSource: run-acr/out
steps:
+ prepare:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ arvados_cluster_ids: arvados_cluster_ids
+ wf: wf
+ obj: obj
+ out: [done]
+ run: prepare.cwl
run-acr:
in:
- arv_token: arvados_api_token
- arv_insecure: arvado_api_host_insecure
- arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ prepare: prepare/done
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
acr: acr
wf: wf
obj: obj
commit b3f5fab7195137b463da7ff1433fdf1894fdfe07
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 13:17:11 2018 -0500
14198: Initial test cases.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 4da2bac0b..721e6b775 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -19,4 +19,33 @@ Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
Workflow which has a remote collection in InitialWorkDir, which is captured in output.
Workflow with file input that has a secondary file in a separate
-collection, which is remote.
\ No newline at end of file
+collection, which is remote.
+
+
+Need to pull Docker image
+Can't run in container because it doesn't have access to Docker from inside -> how can we workaround?
+Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this)
+Try setting up a virtualenv -> annoying because you need C dependencies to build
+Virtualenv sets up shop in /tmp instead of the real directory
+--relocatable ???
+
+
+arvados/jobs doesn't have the docker client
+arvbox has docker client
+
+- Still need venv or something to run cwltool to manage all this.
+- Unless we run cwltool in docker as well.
+
+Can use cwl-docker.sh to run cwltool in docker
+
+- arvbox needs Docker. This will *probably* work launched from container if base dir is under PWD
+- want to run arvados-cwl-runner. this needs Docker to pull images.
+
+For a-c-r solutions are one of:
+
+1) Don't run in Docker, use virtualenv instead.
+2) Access to Docker from inside Docker to pull images.
+2a) Docker-in-Docker
+2b) Bind mount Docker socket
+2c) Communicate to Docker daemon over TCP
+3) Arvados feature to pull images
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 2ff6f79b1..8d7075a1d 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -8,22 +8,32 @@ requirements:
StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
+ InlineJavascriptRequirement: {}
inputs:
- containers: string[]
+ containers:
+ type: string[]
+ default: [fedbox1, fedbox2, fedbox3]
arvbox_base: Directory
+ in_acr: string?
+ insecure:
+ type: boolean
+ default: true
outputs:
- cluster_ids:
- type: string[]
- outputSource: start/cluster_id
- container_hosts:
- type: string[]
- outputSource: start/container_host
- test_user_uuid:
- type: string
- outputSource: setup-user/test_user_uuid
- test_user_token:
+ arvados_api_token:
type: string
outputSource: setup-user/test_user_token
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/container_host
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ acr:
+ type: string?
+ outputSource: in_acr
+ arvado_api_host_insecure:
+ type: boolean
+ outputSource: insecure
steps:
mkdir:
in:
@@ -55,4 +65,4 @@ steps:
container_host: {source: start/container_host, valueFrom: "$(self[0])"}
superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
out: [test_user_uuid, test_user_token]
- run: arvbox-setup-user.cwl
\ No newline at end of file
+ run: arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 2392b1149..e7101dc9e 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -7,16 +7,66 @@ $namespaces:
hints:
cwltool:Secrets:
secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
inputs:
- arvados_api_host_home: string
- arvados_home_id: string
arvados_api_token: string
arvado_api_host_insecure:
- type: bool
+ type: boolean
default: false
- arvados_api_host_clusterB: string
- arvados_clusterB_id: string
- arvados_api_host_clusterC: string
- arvados_clusterC_id: string
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
-outputs: []
+outputs:
+ base-case-out:
+ type: Any
+ outputSource: base-case/out
+ runner-home-step-remote-out:
+ type: Any
+ outputSource: runner-home-step-remote/out
+
+steps:
+ base-case:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
+
+ runner-home-step-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
new file mode 100644
index 000000000..55e1fbf8a
--- /dev/null
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -0,0 +1,24 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:9
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: string
+ outputBinding:
+ glob: out.txt
+ loadContents: true
+ outputEval: $(self[0].contents.substr(0, 32))
+stdout: out.txt
+baseCommand: md5sum
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
new file mode 100644
index 000000000..a178a2c88
--- /dev/null
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -0,0 +1,47 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ acr:
+ type: string?
+ default: arvados-cwl-runner
+ inputBinding:
+ position: 1
+ arv_host: string
+ arv_token: string
+ arv_insecure:
+ type: boolean
+ default: false
+ runner_remote_host:
+ type: string?
+ inputBinding:
+ prefix: --submit-runner-cluster
+ position: 2
+ wf:
+ type: File
+ inputBinding:
+ position: 3
+ obj: Any
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arv_host)
+ ARVADOS_API_TOKEN: $(inputs.arv_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ InlineJavascriptRequirement: {}
+outputs:
+ out:
+ type: Any
+ outputBinding:
+ glob: output.json
+ loadContents: true
+ outputEval: $(JSON.parse(self[0].contents))
+stdout: output.json
+arguments:
+ - valueFrom: --disable-reuse
+ position: 2
+ - valueFrom: input.json
+ position: 4
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
new file mode 100644
index 000000000..f4b8f7320
--- /dev/null
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -0,0 +1,38 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
+ wf: File
+ obj: Any
+outputs:
+ out:
+ type: Any
+ outputSource: run-acr/out
+steps:
+ run-acr:
+ in:
+ arv_token: arvados_api_token
+ arv_insecure: arvado_api_host_insecure
+ arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ acr: acr
+ wf: wf
+ obj: obj
+ out: [out]
+ run: run-acr.cwl
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/whale.txt
new file mode 100644
index 000000000..425d1ed02
--- /dev/null
+++ b/sdk/cwl/tests/federation/whale.txt
@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
commit a28c6869e1b0df197924c523d8b6a85f9f841e89
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 14:57:14 2018 -0500
14198: Initialize test user
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index a595d67cf..173b9df72 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -9,7 +9,11 @@ inputs:
cluster_ids: string[]
cluster_hosts: string[]
arvbox_data: Directory
-outputs: []
+outputs:
+ arvbox_data_out:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
@@ -23,7 +27,7 @@ requirements:
var remoteClusters = {};
for (var i = 0; i < inputs.cluster_ids.length; i++) {
remoteClusters[inputs.cluster_ids[i]] = {
- "Host": inputs.cluster_hosts[i]+":8000",
+ "Host": inputs.cluster_hosts[i],
"Proxy": true,
"Insecure": true
};
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 4b52223e0..2ff6f79b1 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -5,6 +5,7 @@ $namespaces:
cwltool: "http://commonwl.org/cwltool#"
requirements:
ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
inputs:
@@ -14,9 +15,15 @@ outputs:
cluster_ids:
type: string[]
outputSource: start/cluster_id
- container_ips:
+ container_hosts:
type: string[]
- outputSource: start/container_ip
+ outputSource: start/container_host
+ test_user_uuid:
+ type: string
+ outputSource: setup-user/test_user_uuid
+ test_user_token:
+ type: string
+ outputSource: setup-user/test_user_token
steps:
mkdir:
in:
@@ -28,7 +35,7 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip, arvbox_data]
+ out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
@@ -37,9 +44,15 @@ steps:
container_name: containers
this_cluster_id: start/cluster_id
cluster_ids: start/cluster_id
- cluster_hosts: start/container_ip
- arvbox_data: start/arvbox_data
+ cluster_hosts: start/container_host
+ arvbox_data: start/arvbox_data_out
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
run: arvbox-fed-config.cwl
+ setup-user:
+ in:
+ container_host: {source: start/container_host, valueFrom: "$(self[0])"}
+ superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
+ out: [test_user_uuid, test_user_token]
+ run: arvbox-setup-user.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
new file mode 100644
index 000000000..684bc8c8d
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.container_host)
+ ARVADOS_API_TOKEN: $(inputs.superuser_token)
+ ARVADOS_API_HOST_INSECURE: "true"
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ DockerRequirement:
+ dockerPull: arvados/jobs
+inputs:
+ container_host: string
+ superuser_token: string
+ make_user_script:
+ type: File
+ default:
+ class: File
+ location: setup_user.py
+outputs:
+ test_user_uuid: string
+ test_user_token: string
+arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 3485fba68..ae591d58f 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -21,7 +21,7 @@ outputs:
}
}
}
- container_ip:
+ container_host:
type: string
outputBinding:
glob: status.txt
@@ -31,11 +31,17 @@ outputs:
var sp = self[0].contents.split("\n");
for (var i = 0; i < sp.length; i++) {
if (sp[i].startsWith("Container IP: ")) {
- return sp[i].substr(14);
+ return sp[i].substr(14)+":8000";
}
}
}
- arvbox_data:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+ arvbox_data_out:
type: Directory
outputBinding:
outputEval: $(inputs.arvbox_data)
@@ -56,4 +62,7 @@ requirements:
arguments:
- shellQuote: false
valueFrom: |
- arvbox start dev && arvbox status > status.txt
+ set -e
+ arvbox start dev
+ arvbox status > status.txt
+ arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
new file mode 100644
index 000000000..2b0e56c60
--- /dev/null
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -0,0 +1,33 @@
+import arvados
+import arvados.errors
+import time
+import json
+
+while True:
+ try:
+ api = arvados.api()
+ break
+ except arvados.errors.ApiError:
+ time.sleep(2)
+
+existing = api.users().list(filters=[["email", "=", "test at example.com"],
+ ["is_active", "=", True]], limit=1).execute()
+if existing["items"]:
+ u = existing["items"][0]
+else:
+ u = api.users().create(body={
+ 'first_name': 'Test',
+ 'last_name': 'User',
+ 'email': 'test at example.com'
+ }).execute()
+ api.users().activate(uuid=u["uuid"]).execute()
+
+tok = api.api_client_authorizations().create(body={
+ "owner_uuid": u["uuid"]
+}).execute()
+
+with open("cwl.output.json", "w") as f:
+ json.dump({
+ "test_user_uuid": u["uuid"],
+ "test_user_token": tok["api_token"]
+ }, f)
commit 60ccccec73cf39b3379db322dc8eb7ba7b22ddf6
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 13:59:20 2018 -0500
14198: Start and configure a federation of N arvboxes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 157fe4f72..a595d67cf 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,36 +1,52 @@
cwlVersion: v1.0
class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
inputs:
container_name: string
- this_cluster: string
+ this_cluster_id: string
cluster_ids: string[]
cluster_hosts: string[]
- arvbox_base: Directory
+ arvbox_data: Directory
outputs: []
requirements:
EnvVarRequirement:
envDef:
ARVBOX_CONTAINER: $(inputs.container_name)
- ARVBOX_BASE: $(inputs.arvbox_base.path)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
InitialWorkDirRequirement:
listing:
- cluster_config.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
- }
- application.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
- }
+ - entryname: cluster_config.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = {
+ "Host": inputs.cluster_hosts[i]+":8000",
+ "Proxy": true,
+ "Insecure": true
+ };
+ }
+ var r = {"Clusters": {}};
+ r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+ return JSON.stringify(r);
+ }
+ - entryname: application.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
ShellCommandRequirement: {}
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
arguments:
- shellQuote: false
valueFrom: |
@@ -39,4 +55,4 @@ arguments:
arvbox sv restart api
arvbox sv restart controller
arvbox sv restart keepstore0
- arvbox sv restart keepstore1
\ No newline at end of file
+ arvbox sv restart keepstore1
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 91bd22287..4b52223e0 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -28,7 +28,18 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip]
+ out: [cluster_id, container_ip, arvbox_data]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
+ fed-config:
+ in:
+ container_name: containers
+ this_cluster_id: start/cluster_id
+ cluster_ids: start/cluster_id
+ cluster_hosts: start/container_ip
+ arvbox_data: start/arvbox_data
+ out: []
+ scatter: [container_name, this_cluster_id, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 2e12597ce..3485fba68 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -35,6 +35,10 @@ outputs:
}
}
}
+ arvbox_data:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
commit 5a0721144f1d702bc868257a98602521104c876e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 17:46:41 2018 -0500
14198: Federation integration testing WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
new file mode 100644
index 000000000..4da2bac0b
--- /dev/null
+++ b/sdk/cwl/tests/federation/README
@@ -0,0 +1,22 @@
+Things to test.
+
+Single step --submit --no-wait workflow, matrix of:
+
+Runner on home/remote cluster
+Docker image on home/remote cluster
+Step on home/remote cluster
+
+Two step workflow, matrix of:
+
+Step1 on home cluster -> Step2 on remote cluster
+Step1 on remote cluster -> Step2 on home cluster
+
+Three step workflow:
+
+(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster
+Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
+
+Workflow which has a remote collection in InitialWorkDir, which is captured in output.
+
+Workflow with file input that has a secondary file in a separate
+collection, which is remote.
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
new file mode 100644
index 000000000..157fe4f72
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -0,0 +1,42 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ container_name: string
+ this_cluster: string
+ cluster_ids: string[]
+ cluster_hosts: string[]
+ arvbox_base: Directory
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_BASE: $(inputs.arvbox_base.path)
+ InitialWorkDirRequirement:
+ listing:
+ cluster_config.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
+ }
+ application.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ ShellCommandRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
+ arvbox sv restart api
+ arvbox sv restart controller
+ arvbox sv restart keepstore0
+ arvbox sv restart keepstore1
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
new file mode 100644
index 000000000..91bd22287
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -0,0 +1,34 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ ScatterFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ containers: string[]
+ arvbox_base: Directory
+outputs:
+ cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ container_ips:
+ type: string[]
+ outputSource: start/container_ip
+steps:
+ mkdir:
+ in:
+ containers: containers
+ arvbox_base: arvbox_base
+ out: [arvbox_data]
+ run: arvbox-mkdir.cwl
+ start:
+ in:
+ container_name: containers
+ arvbox_data: mkdir/arvbox_data
+ out: [cluster_id, container_ip]
+ scatter: [container_name, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
new file mode 100644
index 000000000..b047beb9b
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -0,0 +1,43 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ containers:
+ type:
+ type: array
+ items: string
+ inputBinding:
+ position: 3
+ valueFrom: |
+ ${
+ return "base/"+self;
+ }
+ arvbox_base: Directory
+outputs:
+ arvbox_data:
+ type: Directory[]
+ outputBinding:
+ glob: |
+ ${
+ var r = [];
+ for (var i = 0; i < inputs.containers.length; i++) {
+ r.push("base/"+inputs.containers[i]);
+ }
+ return r;
+ }
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_base)
+ entryname: base
+ writable: true
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+arguments:
+ - mkdir
+ - "-p"
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
new file mode 100644
index 000000000..2e12597ce
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -0,0 +1,55 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+ arvbox_data: Directory
+outputs:
+ cluster_id:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Cluster id: ")) {
+ return sp[i].substr(12);
+ }
+ }
+ }
+ container_ip:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Container IP: ")) {
+ return sp[i].substr(14);
+ }
+ }
+ }
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
+ ShellCommandRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_data)
+ entryname: $(inputs.container_name)
+ writable: true
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ InlineJavascriptRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ arvbox start dev && arvbox status > status.txt
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
new file mode 100644
index 000000000..c4a5412de
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+arguments: [arvbox, stop]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
new file mode 100755
index 000000000..2392b1149
--- /dev/null
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -0,0 +1,22 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+inputs:
+ arvados_api_host_home: string
+ arvados_home_id: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: bool
+ default: false
+ arvados_api_host_clusterB: string
+ arvados_clusterB_id: string
+ arvados_api_host_clusterC: string
+ arvados_clusterC_id: string
+
+outputs: []
commit 50aa173d7fd43d8d509da962697c42ca03e5fa5e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 16:15:11 2018 -0500
14198: Evaluate ClusterTarget in the job step that declares it
Don't use get_requirement to inherit ClusterTarget.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 1a0c81b28..ee9dd25a1 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -10,16 +10,26 @@ from functools import partial
from schema_salad.sourceline import SourceLine
from cwltool.errors import WorkflowException
-def check_cluster_target(self, builder, runtimeContext):
- cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
- with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug):
- runtimeContext.cluster_target_id = id(cluster_target_req)
- runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster
- runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid
- if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in self.arvrunner.api._rootDesc["remoteHosts"]:
- raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster,
- ", ".join(self.arvrunner.api._rootDesc["remoteHosts"].keys())))
+def set_cluster_target(tool, arvrunner, builder, runtimeContext):
+ cluster_target_req = None
+ for field in ("hints", "requirements"):
+ if field not in tool:
+ continue
+ for item in tool[field]:
+ if item["class"] == "http://arvados.org/cwl#ClusterTarget":
+ cluster_target_req = item
+
+ if cluster_target_req is None:
+ return runtimeContext
+
+ with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug):
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster
+ runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid
+ if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"]:
+ raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster,
+ ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
+ return runtimeContext
class ArvadosCommandTool(CommandLineTool):
"""Wrap cwltool CommandLineTool to override selected methods."""
@@ -56,8 +66,7 @@ class ArvadosCommandTool(CommandLineTool):
joborder = builder.job
runtimeContext = runtimeContext.copy()
-
- check_cluster_target(self, builder, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index b26bee824..218006633 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -22,7 +22,7 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
from .pathmapper import ArvPathMapper, trim_listing
-from .arvtool import ArvadosCommandTool, check_cluster_target
+from .arvtool import ArvadosCommandTool, set_cluster_target
from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
@@ -134,9 +134,10 @@ class ArvadosWorkflowStep(WorkflowStep):
def job(self, joborder, output_callback, runtimeContext):
builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext)
- check_cluster_target(self, builder, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
+
class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
@@ -148,11 +149,12 @@ class ArvadosWorkflow(Workflow):
self.wf_reffiles = []
self.loadingContext = loadingContext
super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
+ self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
def job(self, joborder, output_callback, runtimeContext):
builder = self._init_job(joborder, runtimeContext)
- check_cluster_target(self, builder, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if not req:
commit 8c4a21620cdaa36eab8a3d1a8f423bbbe4b27bdf
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 14:21:07 2018 -0500
14198: Set arvrunner before calling super constructor
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 5f4f8867a..b26bee824 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -141,13 +141,13 @@ class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
def __init__(self, arvrunner, toolpath_object, loadingContext):
- super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
self.arvrunner = arvrunner
self.wf_pdh = None
self.dynamic_resource_req = []
self.static_resource_req = []
self.wf_reffiles = []
self.loadingContext = loadingContext
+ super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
def job(self, joborder, output_callback, runtimeContext):
commit 7c6caeb6fd43bd8e86a9eee9f6c588f79a649eea
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 14:12:51 2018 -0500
14198: Set arvrunner on ArvadosWorkflowStep
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 130b42b5c..1a0c81b28 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -6,7 +6,6 @@ from cwltool.command_line_tool import CommandLineTool
from .arvjob import ArvadosJob
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
-from .context import ClusterTarget
from functools import partial
from schema_salad.sourceline import SourceLine
from cwltool.errors import WorkflowException
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f06d569bf..5f4f8867a 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -123,12 +123,14 @@ class ArvadosWorkflowStep(WorkflowStep):
toolpath_object, # type: Dict[Text, Any]
pos, # type: int
loadingContext, # type: LoadingContext
+ arvrunner,
*argc,
**argv
): # type: (...) -> None
super(ArvadosWorkflowStep, self).__init__(toolpath_object, pos, loadingContext, *argc, **argv)
self.tool["class"] = "WorkflowStep"
+ self.arvrunner = arvrunner
def job(self, joborder, output_callback, runtimeContext):
builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext)
@@ -308,4 +310,4 @@ class ArvadosWorkflow(Workflow):
**argv
):
# (...) -> WorkflowStep
- return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, *argc, **argv)
+ return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, self.arvrunner, *argc, **argv)
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 8af3f20c8..7831e1cfd 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -9,8 +9,6 @@ class ArvLoadingContext(LoadingContext):
def __init__(self, kwargs=None):
super(ArvLoadingContext, self).__init__(kwargs)
-ClusterTarget = namedtuple("ClusterTarget", ("instance", "cluster_id", "owner_uuid"))
-
class ArvRuntimeContext(RuntimeContext):
def __init__(self, kwargs=None):
self.work_api = None
commit 8f40521cc7f9b4a619bfd88bf30e81d2d0a39f03
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 13:55:49 2018 -0500
14198: Set fake 'class' so _init_job is happy
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 7ea5b5873..f06d569bf 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -119,6 +119,17 @@ def get_overall_res_req(res_reqs):
return cmap(overall_res_req)
class ArvadosWorkflowStep(WorkflowStep):
+ def __init__(self,
+ toolpath_object, # type: Dict[Text, Any]
+ pos, # type: int
+ loadingContext, # type: LoadingContext
+ *argc,
+ **argv
+ ): # type: (...) -> None
+
+ super(ArvadosWorkflowStep, self).__init__(toolpath_object, pos, loadingContext, *argc, **argv)
+ self.tool["class"] = "WorkflowStep"
+
def job(self, joborder, output_callback, runtimeContext):
builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext)
check_cluster_target(self, builder, runtimeContext)
@@ -293,7 +304,8 @@ class ArvadosWorkflow(Workflow):
toolpath_object, # type: Dict[Text, Any]
pos, # type: int
loadingContext, # type: LoadingContext
- parentworkflowProv=None # type: Optional[CreateProvProfile]
+ *argc,
+ **argv
):
# (...) -> WorkflowStep
- return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv)
+ return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, *argc, **argv)
commit d5e2db11140f924eb7198cf8714c9f7eab6364c7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 11:44:55 2018 -0500
14198: Fix jobstep builder init
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index fa543e375..7ea5b5873 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -120,7 +120,7 @@ def get_overall_res_req(res_reqs):
class ArvadosWorkflowStep(WorkflowStep):
def job(self, joborder, output_callback, runtimeContext):
- builder = self._init_job(joborder, runtimeContext)
+ builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext)
check_cluster_target(self, builder, runtimeContext)
return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
commit 584e9a49d1b8161297f43ef248a82f5988ff62ab
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 11:28:27 2018 -0500
14198: Fix ruamel.yaml versioning
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 36aed3217..5d373282b 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
'cwltool==1.0.20181116032456',
'schema-salad==2.7.20181116024232',
'typing >= 3.6.4',
- 'ruamel.yaml >=0.13.11, <= 0.15.77',
+ 'ruamel.yaml >=0.15.54, <=0.15.77',
'arvados-python-client>=1.1.4.20180607143841',
'setuptools',
'ciso8601 >=1.0.6, <2.0.0',
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index e5126c12b..9b38f0714 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-python-client',
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.15.54',
+ 'ruamel.yaml >=0.15.54, <=0.15.77',
'setuptools',
'ws4py >=0.4.2',
'subprocess32 >=3.5.1',
commit 46459c623c283f2fc810d6a01c77cc29105450f2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 11:26:51 2018 -0500
14198: Construct ArvadosWorkflowStep
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8a7fb7f60..fa543e375 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -296,4 +296,4 @@ class ArvadosWorkflow(Workflow):
parentworkflowProv=None # type: Optional[CreateProvProfile]
):
# (...) -> WorkflowStep
- return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv)
+ return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv)
commit 9bd76cc66671046bde8bb5d6592c502088e6305f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 16 10:28:24 2018 -0500
14198: Update version deps for schema-salad, cwltool, ruamel.yaml, requests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/build.list b/build/build.list
index 4c3d740b0..3b2e17fa8 100644
--- a/build/build.list
+++ b/build/build.list
@@ -33,7 +33,7 @@ debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|pytho
debian8,debian9,centos7|six|1.10.0|2|python3|all
debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
-ubuntu1404|requests|2.4.3|2|python|all
+ubuntu1404|requests|2.6.1|2|python|all
centos7|contextlib2|0.5.4|2|python|all
centos7|isodate|0.5.4|2|python|all
centos7|python-daemon|2.1.2|1|python|all
@@ -44,7 +44,7 @@ centos7|networkx|1.11|0|python|all
centos7|psutil|5.0.1|0|python|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+all|ruamel.yaml|0.15.77|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
all|junit-xml|1.8|3|python|all
all|rdflib-jsonld|0.4.0|2|python|all
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index e80c06f00..8a7fb7f60 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -12,7 +12,7 @@ from schema_salad.sourceline import SourceLine, cmap
from cwltool.pack import pack
from cwltool.load_tool import fetch_document
from cwltool.process import shortname
-from cwltool.workflow import Workflow, WorkflowException
+from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.builder import Builder
from cwltool.context import LoadingContext
@@ -118,6 +118,12 @@ def get_overall_res_req(res_reqs):
overall_res_req["class"] = "ResourceRequirement"
return cmap(overall_res_req)
+class ArvadosWorkflowStep(WorkflowStep):
+ def job(self, joborder, output_callback, runtimeContext):
+ builder = self._init_job(joborder, runtimeContext)
+ check_cluster_target(self, builder, runtimeContext)
+ return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
+
class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
@@ -282,3 +288,12 @@ class ArvadosWorkflow(Workflow):
"id": "#"
})
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+
+ def make_workflow_step(self,
+ toolpath_object, # type: Dict[Text, Any]
+ pos, # type: int
+ loadingContext, # type: LoadingContext
+ parentworkflowProv=None # type: Optional[CreateProvProfile]
+ ):
+ # (...) -> WorkflowStep
+ return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv)
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 316a65252..b22b2ffd6 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -265,6 +265,12 @@ class CollectionFetcher(DefaultFetcher):
return super(CollectionFetcher, self).urljoin(base_url, url)
+ schemes = [u"file", u"http", u"https", u"mailto", u"keep"]
+
+ def supported_schemes(self): # type: () -> List[Text]
+ return self.schemes
+
+
workflow_uuid_pattern = re.compile(r'[a-z0-9]{5}-7fd4e-[a-z0-9]{15}')
pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}')
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 91a03ed86..36aed3217 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,12 +33,10 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20181114114354',
- 'schema-salad==2.7.20181017120439',
+ 'cwltool==1.0.20181116032456',
+ 'schema-salad==2.7.20181116024232',
'typing >= 3.6.4',
- # Need to limit ruamel.yaml version to 0.15.26 because of bug
- # https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping
- 'ruamel.yaml >=0.13.11, <= 0.15.26',
+ 'ruamel.yaml >=0.13.11, <= 0.15.77',
'arvados-python-client>=1.1.4.20180607143841',
'setuptools',
'ciso8601 >=1.0.6, <2.0.0',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index aa1f18052..d33956ccc 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,7 +20,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN pip install -U setuptools six
+RUN pip install -U setuptools six requests
ARG sdk
ARG runner
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 8f576196b..e5126c12b 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-python-client',
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.13.11, <= 0.15.26',
+ 'ruamel.yaml >=0.15.54',
'setuptools',
'ws4py >=0.4.2',
'subprocess32 >=3.5.1',
commit 014cee6569ba8c170c352d92aa35ddc61ed53b78
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 14 13:22:16 2018 -0500
14198: Update cwltool and schema-salad
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 72ef8fd4f..e80c06f00 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -132,7 +132,8 @@ class ArvadosWorkflow(Workflow):
def job(self, joborder, output_callback, runtimeContext):
- check_cluster_target(self, self._init_job(joborder, runtimeContext), runtimeContext)
+ builder = self._init_job(joborder, runtimeContext)
+ check_cluster_target(self, builder, runtimeContext)
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if not req:
@@ -161,11 +162,6 @@ class ArvadosWorkflow(Workflow):
packed = pack(document_loader, workflowobj, uri, self.metadata)
- builder = Builder(joborder,
- requirements=workflowobj["requirements"],
- hints=workflowobj["hints"],
- resources={})
-
def visit(item):
for t in ("hints", "requirements"):
if t not in item:
@@ -205,11 +201,6 @@ class ArvadosWorkflow(Workflow):
if self.dynamic_resource_req:
- builder = Builder(joborder,
- requirements=self.requirements,
- hints=self.hints,
- resources={})
-
# Evaluate dynamic resource requirements using current builder
rs = copy.copy(self.static_resource_req)
for dyn_rs in self.dynamic_resource_req:
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 2b7b31b9f..91a03ed86 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180806194258',
- 'schema-salad==2.7.20180719125426',
+ 'cwltool==1.0.20181114114354',
+ 'schema-salad==2.7.20181017120439',
'typing >= 3.6.4',
# Need to limit ruamel.yaml version to 0.15.26 because of bug
# https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping
commit 0765f4e130c8dd898b7ceb0c8b90668c50333255
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 16:24:02 2018 -0500
14198: Add --always-submit-runner
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index a29bafb22..605d33005 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -145,6 +145,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
default=None)
+ parser.add_argument("--always-submit-runner", action="store_true",
+ help="Always submit a runner to manage the workflow, even when running only a single CommandLineTool",
+ default=False)
+
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--submit-request-uuid", type=str,
default=None,
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 48f92b77f..8af3f20c8 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -35,6 +35,7 @@ class ArvRuntimeContext(RuntimeContext):
self.http_timeout = 300
self.submit_runner_cluster = None
self.cluster_target_id = 0
+ self.always_submit_runner = False
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index c6dc71629..994594023 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -586,7 +586,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if runtimeContext.submit:
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
- if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait:
+ if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
runtimeContext.runnerjob = tool.tool["id"]
runnerjob = tool.job(job_order,
self.output_callback,
commit 827c9b91ed1d6eb5e4a17c3d4234be722f7b0600
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 17:44:23 2018 -0500
14198: Add --submit-runner-cluster test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0fd824fbe..92ea553ea 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1449,6 +1449,25 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_container_cluster_id(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
commit 55f11de917a834f13ea22dc461ebcf65d341c259
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 10:16:48 2018 -0500
14198: Unit tests pass again
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 141c74de4..a29bafb22 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -36,10 +36,12 @@ from .executor import ArvCwlExecutor
# These arn't used directly in this file but
# other code expects to import them from here
from .arvcontainer import ArvadosContainer
+from .arvjob import ArvadosJob
from .arvtool import ArvadosCommandTool
-from .fsaccess import CollectionFsAccess, CollectionCache
+from .fsaccess import CollectionFsAccess, CollectionCache, CollectionFetcher
from .util import get_current_container
from .executor import RuntimeStatusLoggingHandler, DEFAULT_PRIORITY
+from .arvworkflow import ArvadosWorkflow
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 189282ec2..9a03372d3 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -30,6 +30,7 @@ from .pathmapper import VwdPathMapper, trim_listing
from .perf import Perf
from . import done
from ._version import __version__
+from .util import get_intermediate_collection_info
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index b689e94ef..72ef8fd4f 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -22,7 +22,7 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
from .pathmapper import ArvPathMapper, trim_listing
-from .arvtool import ArvadosCommandTool
+from .arvtool import ArvadosCommandTool, check_cluster_target
from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
commit 67f696530369f3d7533c0db7c24889ab04efc2af
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 2 17:05:20 2018 -0400
14198: test_submit passes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 53a4a17ee..0fd824fbe 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1216,43 +1216,45 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ def tearDown(self):
+ arvados_cwl.arvdocker.arv_docker_clear_cache()
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
- try:
- arvrunner = mock.MagicMock()
- arvrunner.project_uuid = ""
- api.return_value = mock.MagicMock()
- arvrunner.api = api.return_value
- arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
- "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "link_class": "docker_image_repo+tag",
- "name": "arvados/jobs:"+arvados_cwl.__version__,
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
- )
- find_one_image_hash.return_value = "123456"
-
- arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "owner_uuid": "",
- "manifest_text": "",
- "properties": ""
- }], "items_available": 1, "offset": 0},)
- arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "portable_data_hash": "9999999999999999999999999999999b+99"}
- self.assertEqual("9999999999999999999999999999999b+99",
- arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
- finally:
- arvados_cwl.arvdocker.arv_docker_clear_cache()
+ arvados_cwl.arvdocker.arv_docker_clear_cache()
+
+ arvrunner = mock.MagicMock()
+ arvrunner.project_uuid = ""
+ api.return_value = mock.MagicMock()
+ arvrunner.api = api.return_value
+ arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
+ "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "link_class": "docker_image_repo+tag",
+ "name": "arvados/jobs:"+arvados_cwl.__version__,
+ "owner_uuid": "",
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
+ {"items": [{"created_at": "",
+ "head_uuid": "",
+ "link_class": "docker_image_hash",
+ "name": "123456",
+ "owner_uuid": "",
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
+ )
+ find_one_image_hash.return_value = "123456"
+
+ arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "owner_uuid": "",
+ "manifest_text": "",
+ "properties": ""
+ }], "items_available": 1, "offset": 0},)
+ arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
+ arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "portable_data_hash": "9999999999999999999999999999999b+99"}
+ self.assertEqual("9999999999999999999999999999999b+99",
+ arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+
@stubs
def test_submit_secrets(self, stubs):
commit 81018b81b5a9d50b76c7c89a885ff95d440e39b2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 2 16:41:01 2018 -0400
14198: More test updates
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index ab4e7e510..53a4a17ee 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -50,7 +50,8 @@ def stubs(func):
stubs.docker_images = {
"arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
"debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
- "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")]
+ "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
+ "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
}
def kd(a, b, image_name=None, image_tag=None):
return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
@@ -117,6 +118,11 @@ def stubs(func):
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
"portable_data_hash": "999999999999999999999999999999d5+99",
"manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
+ "portable_data_hash": "999999999999999999999999999999d6+99",
+ "manifest_text": ""
}
}
stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
@@ -1739,12 +1745,12 @@ class TestTemplateInputs(unittest.TestCase):
"components": {
"inputs_test.cwl": {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': '999999999999999999999999999999d3+99',
'min_ram_mb_per_node': 1024
},
'script_parameters': {
'cwl:tool':
- '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
+ 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
'optionalFloatInput': None,
'fileInput': {
'type': 'File',
@@ -1805,7 +1811,7 @@ class TestTemplateInputs(unittest.TestCase):
params = expect_template[
"components"]["inputs_test.cwl"]["script_parameters"]
params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
- params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
+ params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
params["floatInput"]["value"] = 1.234
params["boolInput"]["value"] = True
diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index c84252c7b..cb2e5ff56 100644
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -25,7 +25,8 @@
"requirements": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8"
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
}
]
},
commit 919d5333ccfa43be3f14394cd7611d27881a5279
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 2 16:05:29 2018 -0400
14198: Bringing unit tests up to date WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 63fc3ea47..141c74de4 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -13,27 +13,33 @@ import sys
import re
import pkg_resources # part of setuptools
+from schema_salad.sourceline import SourceLine
+import schema_salad.validate as validate
import cwltool.main
import cwltool.workflow
import cwltool.process
-from schema_salad.sourceline import SourceLine
-import schema_salad.validate as validate
import cwltool.argparser
+from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
import arvados
import arvados.config
from arvados.keep import KeepClient
from arvados.errors import ApiError
import arvados.commands._util as arv_cmd
+from arvados.api import OrderedJsonModel
from .perf import Perf
from ._version import __version__
from .executor import ArvCwlExecutor
-from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
-
-from arvados.api import OrderedJsonModel
+# These arn't used directly in this file but
+# other code expects to import them from here
+from .arvcontainer import ArvadosContainer
+from .arvtool import ArvadosCommandTool
+from .fsaccess import CollectionFsAccess, CollectionCache
+from .util import get_current_container
+from .executor import RuntimeStatusLoggingHandler, DEFAULT_PRIORITY
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
@@ -43,8 +49,6 @@ arvados.log_handler.setFormatter(logging.Formatter(
'%(asctime)s %(name)s %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
-DEFAULT_PRIORITY = 500
-
def versionstring():
"""Print version string of key packages for provenance and debugging."""
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 100329b69..278ff08e9 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -12,7 +12,7 @@ import ciso8601
import uuid
import math
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
@@ -171,8 +171,8 @@ class ArvadosContainer(JobBase):
keepemptydirs(vwd)
if not runtimeContext.current_container:
- runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ runtimeContext.current_container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 1287fbb6e..189282ec2 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -18,7 +18,7 @@ from cwltool.job import JobBase
from schema_salad.sourceline import SourceLine
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import ruamel.yaml as yaml
import arvados.collection
@@ -77,9 +77,7 @@ class ArvadosJob(JobBase):
if vwd:
with Perf(metrics, "generatefiles.save_new %s" % self.name):
- if not runtimeContext.current_container:
- runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ info = get_intermediate_collection_info(self.name, None, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 9f0c91f11..7512d5bef 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -104,7 +104,7 @@ def run():
arvargs.output_tags = output_tags
arvargs.thread_count = 1
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache(
+ runner = arvados_cwl.ArvCwlExecutor(api_client=arvados.safeapi.ThreadSafeApiCache(
api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
arvargs=arvargs)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 8c2023e18..c6dc71629 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -23,6 +23,7 @@ import arvados.config
from arvados.keep import KeepClient
from arvados.errors import ApiError
+import arvados_cwl.util
from .arvcontainer import RunnerContainer
from .arvjob import RunnerJob, RunnerTemplate
from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
@@ -33,7 +34,6 @@ from .perf import Perf
from .pathmapper import NoFollowPathMapper
from .task_queue import TaskQueue
from .context import ArvLoadingContext, ArvRuntimeContext
-from .util import get_current_container
from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
@@ -43,6 +43,8 @@ from cwltool.command_line_tool import compute_checksums
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
+DEFAULT_PRIORITY = 500
+
class RuntimeStatusLoggingHandler(logging.Handler):
"""
Intercepts logging calls and report them as runtime statuses on runner
@@ -167,7 +169,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
# Add a custom logging handler to the root logger for runtime status reporting
# if running inside a container
- if get_current_container(self.api, self.num_retries, logger):
+ if arvados_cwl.util.get_current_container(self.api, self.num_retries, logger):
root_logger = logging.getLogger('')
handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
root_logger.addHandler(handler)
@@ -222,7 +224,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
activity statuses, for example in the RuntimeStatusLoggingHandler.
"""
with self.workflow_eval_lock:
- current = get_current_container(self.api, self.num_retries, logger)
+ current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current is None:
return
runtime_status = current.get('runtime_status', {})
@@ -465,7 +467,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
def set_crunch_output(self):
if self.work_api == "containers":
- current = get_current_container(self.api, self.num_retries, logger)
+ current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current is None:
return
try:
@@ -626,7 +628,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
runnerjob.run(submitargs)
return (runnerjob.uuid, "success")
- current_container = get_current_container(self.api, self.num_retries, logger)
+ current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current_container:
logger.info("Running inside container %s", current_container.get("uuid"))
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index d083b78f5..26c85d300 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -8,7 +8,7 @@ import uuid
import os
import urllib
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import arvados.commands.run
import arvados.collection
@@ -155,8 +155,8 @@ class ArvPathMapper(PathMapper):
for l in srcobj.get("listing", []):
self.addentry(l, c, ".", remap)
- container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+ container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
@@ -174,8 +174,8 @@ class ArvPathMapper(PathMapper):
num_retries=self.arvrunner.num_retries )
self.addentry(srcobj, c, ".", remap)
- container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+ container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 31a424d30..a846f2b00 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -26,7 +26,7 @@ from cwltool.pack import pack
import arvados.collection
import ruamel.yaml as yaml
-import arvdocker
+import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing
from ._version import __version__
from . import done
@@ -215,9 +215,9 @@ def upload_docker(arvrunner, tool):
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
else:
- arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
@@ -245,8 +245,7 @@ def packed_workflow(arvrunner, tool, merged_map):
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
if v.get("class") == "DockerRequirement":
- img = v.get("dockerImageId") or v.get("dockerPull")
- v["http://arvados.org/cwl#dockerCollectionPDH"] = arvdocker.cached_lookups[img]
+ v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
@@ -327,7 +326,7 @@ def arvados_jobs_image(arvrunner, img):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- return arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 46184325f..c875c0785 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -4,6 +4,7 @@
import arvados_cwl
import arvados_cwl.context
+import arvados_cwl.util
from arvados_cwl.arvdocker import arv_docker_clear_cache
import copy
import arvados.config
@@ -132,7 +133,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
@@ -219,7 +220,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 7200,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
@@ -351,7 +352,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
@@ -439,7 +440,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
@@ -465,7 +466,10 @@ class TestContainer(unittest.TestCase):
col().open.return_value = []
+ loadingContext, runtimeContext = self.helper(runner)
+
arvjob = arvados_cwl.ArvadosContainer(runner,
+ runtimeContext,
mock.MagicMock(),
{},
None,
@@ -496,7 +500,7 @@ class TestContainer(unittest.TestCase):
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
- @mock.patch("arvados_cwl.get_current_container")
+ @mock.patch("arvados_cwl.util.get_current_container")
@mock.patch("arvados.collection.CollectionReader")
@mock.patch("arvados.collection.Collection")
def test_child_failure(self, col, reader, gcc_mock):
@@ -507,11 +511,11 @@ class TestContainer(unittest.TestCase):
# Set up runner with mocked runtime_status_update()
self.assertFalse(gcc_mock.called)
runtime_status_update = mock.MagicMock()
- arvados_cwl.ArvCwlRunner.runtime_status_update = runtime_status_update
- runner = arvados_cwl.ArvCwlRunner(api)
+ arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update
+ runner = arvados_cwl.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
- # Make sure ArvCwlRunner thinks it's running inside a container so it
+ # Make sure ArvCwlExecutor thinks it's running inside a container so it
# adds the logging handler that will call runtime_status_update() mock
gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
self.assertTrue(gcc_mock.called)
@@ -536,7 +540,10 @@ class TestContainer(unittest.TestCase):
col().open.return_value = []
+ loadingContext, runtimeContext = self.helper(runner)
+
arvjob = arvados_cwl.ArvadosContainer(runner,
+ runtimeContext,
mock.MagicMock(),
{},
None,
@@ -648,7 +655,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999994+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
@@ -741,7 +748,7 @@ class TestContainer(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['md5sum', 'example.conf'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 20efe1513..2aaac0ae5 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -13,6 +13,7 @@ import StringIO
import arvados
import arvados_cwl
+import arvados_cwl.executor
import cwltool.process
from arvados.errors import ApiError
from schema_salad.ref_resolver import Loader
@@ -373,7 +374,7 @@ class TestWorkflow(unittest.TestCase):
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'jobs')
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
@@ -455,7 +456,7 @@ class TestWorkflow(unittest.TestCase):
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'jobs')
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
@@ -517,5 +518,5 @@ class TestWorkflow(unittest.TestCase):
api = mock.MagicMock()
api._rootDesc = copy.deepcopy(get_rootDesc())
del api._rootDesc.get('resources')['jobs']['methods']['create']
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index 590c82d20..baeb4145e 100644
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -12,6 +12,7 @@ import unittest
import arvados
import arvados_cwl
+import arvados_cwl.executor
from .mock_discovery import get_rootDesc
class TestMakeOutput(unittest.TestCase):
@@ -23,7 +24,7 @@ class TestMakeOutput(unittest.TestCase):
@mock.patch("arvados.collection.CollectionReader")
def test_make_output_collection(self, reader, col):
keep_client = mock.MagicMock()
- runner = arvados_cwl.ArvCwlRunner(self.api, keep_client=keep_client)
+ runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
final = mock.MagicMock()
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index eaa571142..fb3c257d9 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -14,6 +14,7 @@ import arvados
import arvados.keep
import arvados.collection
import arvados_cwl
+import arvados_cwl.executor
from cwltool.pathmapper import MapperEnt
from .mock_discovery import get_rootDesc
@@ -34,7 +35,7 @@ class TestPathmap(unittest.TestCase):
def test_keepref(self):
"""Test direct keep references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
p = ArvPathMapper(arvrunner, [{
"class": "File",
@@ -49,7 +50,7 @@ class TestPathmap(unittest.TestCase):
def test_upload(self, statfile, upl):
"""Test pathmapper uploading files."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
st = arvados.commands.run.UploadFile("", "tests/hw.py")
@@ -70,7 +71,7 @@ class TestPathmap(unittest.TestCase):
@mock.patch("arvados.commands.run.statfile")
def test_statfile(self, statfile, upl):
"""Test pathmapper handling ArvFile references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
# An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
# keep mount, so we can construct a direct reference directly without upload.
@@ -92,7 +93,7 @@ class TestPathmap(unittest.TestCase):
@mock.patch("os.stat")
def test_missing_file(self, stat):
"""Test pathmapper handling missing references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
stat.side_effect = OSError(2, "No such file or directory")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index f718a86b3..ab4e7e510 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -15,6 +15,7 @@ import unittest
import arvados
import arvados.collection
import arvados_cwl
+import arvados_cwl.executor
import arvados_cwl.runner
import arvados.keep
@@ -46,7 +47,15 @@ def stubs(func):
keep_client2.put.side_effect = putstub
stubs.keep_client = keep_client2
- stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ stubs.docker_images = {
+ "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
+ "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
+ "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")]
+ }
+ def kd(a, b, image_name=None, image_tag=None):
+ return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
+ stubs.keepdocker.side_effect = kd
+
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
@@ -69,7 +78,7 @@ def stubs(func):
def collection_createstub(created_collections, body, ensure_unique_name=None):
mt = body["manifest_text"]
- uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
+ uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
created_collections[uuid] = {
"uuid": uuid,
@@ -93,6 +102,21 @@ def stubs(func):
"uuid": "",
"portable_data_hash": "99999999999999999999999999999994+99",
"manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
+ "portable_data_hash": "999999999999999999999999999999d3+99",
+ "manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
+ "portable_data_hash": "999999999999999999999999999999d4+99",
+ "manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
+ "portable_data_hash": "999999999999999999999999999999d5+99",
+ "manifest_text": ""
}
}
stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
@@ -117,7 +141,7 @@ def stubs(func):
}
stubs.expect_job_spec = {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': '999999999999999999999999999999d3+99',
'min_ram_mb_per_node': 1024
},
'script_parameters': {
@@ -141,7 +165,7 @@ def stubs(func):
}],
'class': 'Directory'
},
- 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
+ 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
},
'repository': 'arvados',
'script_version': 'master',
@@ -155,7 +179,7 @@ def stubs(func):
'owner_uuid': None,
"components": {
"cwl-runner": {
- 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
+ 'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
'script_parameters': {
'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
'x': {"value": {
@@ -173,7 +197,7 @@ def stubs(func):
'size': 0
}
]}},
- 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
+ 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
'arv:debug': True,
'arv:enable_reuse': True,
'arv:on_error': 'continue'
@@ -247,7 +271,7 @@ def stubs(func):
'--enable-reuse', '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': '999999999999999999999999999999d3+99',
'output_path': '/var/spool/cwl',
'cwd': '/var/spool/cwl',
'runtime_constraints': {
@@ -277,10 +301,17 @@ def stubs(func):
class TestSubmit(unittest.TestCase):
- @mock.patch("arvados_cwl.runner.arv_docker_get_image")
+ @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
@mock.patch("time.sleep")
@stubs
def test_submit(self, stubs, tm, arvdock):
+ def get_image(api_client, dockerRequirement, pull_image, project_uuid):
+ if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
+ return '999999999999999999999999999999d3+99'
+ elif dockerRequirement["dockerPull"] == "debian:8":
+ return '999999999999999999999999999999d4+99'
+ arvdock.side_effect = get_image
+
capture_stdout = cStringIO.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug",
@@ -303,13 +334,14 @@ class TestSubmit(unittest.TestCase):
}), ensure_unique_name=False),
mock.call(body=JsonDiffMatcher({
'manifest_text':
- '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
+ ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
'replication_desired': None,
'name': 'submit_wf.cwl',
}), ensure_unique_name=True) ])
arvdock.assert_has_calls([
mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
+ mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
])
@@ -646,7 +678,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("arvados_cwl.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
- @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
@stubs
def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
def set_final_output(job_order, output_callback, runtimeContext):
@@ -667,7 +699,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("arvados_cwl.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
- @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
@stubs
def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
def set_final_output(job_order, output_callback, runtimeContext):
@@ -835,7 +867,7 @@ class TestSubmit(unittest.TestCase):
}, 'state': 'Committed',
'output_path': '/var/spool/cwl',
'name': 'expect_arvworkflow.cwl#main',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
'--eval-timeout=20', '--thread-count=4',
@@ -934,7 +966,11 @@ class TestSubmit(unittest.TestCase):
'id': '#submit_tool.cwl/x'}
],
'requirements': [
- {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
+ {
+ 'dockerPull': 'debian:8',
+ 'class': 'DockerRequirement',
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
],
'id': '#submit_tool.cwl',
'outputs': [],
@@ -953,7 +989,7 @@ class TestSubmit(unittest.TestCase):
}, 'state': 'Committed',
'output_path': '/var/spool/cwl',
'name': 'a test workflow',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
'--eval-timeout=20', '--thread-count=4',
@@ -1090,7 +1126,7 @@ class TestSubmit(unittest.TestCase):
except:
logging.exception("")
- stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
+ stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
@@ -1110,7 +1146,7 @@ class TestSubmit(unittest.TestCase):
except:
logging.exception("")
- stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
+ stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
@@ -1179,33 +1215,38 @@ class TestSubmit(unittest.TestCase):
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
- arvrunner = mock.MagicMock()
- arvrunner.project_uuid = ""
- api.return_value = mock.MagicMock()
- arvrunner.api = api.return_value
- arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
- "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "link_class": "docker_image_repo+tag",
- "name": "arvados/jobs:"+arvados_cwl.__version__,
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
- )
- find_one_image_hash.return_value = "123456"
-
- arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "owner_uuid": "",
- "manifest_text": "",
- "properties": ""
- }], "items_available": 1, "offset": 0},)
- arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
- arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ try:
+ arvrunner = mock.MagicMock()
+ arvrunner.project_uuid = ""
+ api.return_value = mock.MagicMock()
+ arvrunner.api = api.return_value
+ arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
+ "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "link_class": "docker_image_repo+tag",
+ "name": "arvados/jobs:"+arvados_cwl.__version__,
+ "owner_uuid": "",
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
+ {"items": [{"created_at": "",
+ "head_uuid": "",
+ "link_class": "docker_image_hash",
+ "name": "123456",
+ "owner_uuid": "",
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
+ )
+ find_one_image_hash.return_value = "123456"
+
+ arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "owner_uuid": "",
+ "manifest_text": "",
+ "properties": ""
+ }], "items_available": 1, "offset": 0},)
+ arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
+ arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "portable_data_hash": "9999999999999999999999999999999b+99"}
+ self.assertEqual("9999999999999999999999999999999b+99",
+ arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ finally:
+ arvados_cwl.arvdocker.arv_docker_clear_cache()
@stubs
def test_submit_secrets(self, stubs):
@@ -1235,7 +1276,7 @@ class TestSubmit(unittest.TestCase):
"/var/lib/cwl/workflow.json#main",
"/var/lib/cwl/cwl.input.json"
],
- "container_image": "arvados/jobs:"+arvados_cwl.__version__,
+ "container_image": "999999999999999999999999999999d3+99",
"cwd": "/var/spool/cwl",
"mounts": {
"/var/lib/cwl/cwl.input.json": {
@@ -1297,7 +1338,8 @@ class TestSubmit(unittest.TestCase):
"hints": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8"
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
},
{
"class": "http://commonwl.org/cwltool#Secrets",
@@ -1395,7 +1437,7 @@ class TestSubmit(unittest.TestCase):
logging.exception("")
stubs.api.container_requests().update.assert_called_with(
- uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
+ uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec), cluster_id="zzzzz")
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
diff --git a/sdk/cwl/tests/wf/submit_wf_packed.cwl b/sdk/cwl/tests/wf/submit_wf_packed.cwl
index 65704b4e5..83ba584b2 100644
--- a/sdk/cwl/tests/wf/submit_wf_packed.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_packed.cwl
@@ -8,6 +8,7 @@ $graph:
requirements:
- class: DockerRequirement
dockerPull: debian:8
+ 'http://arvados.org/cwl#dockerCollectionPDH': 999999999999999999999999999999d4+99
inputs:
- id: '#submit_tool.cwl/x'
type: File
commit a44b2e337b7f56844768d6d64cd21bd079d4c464
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 1 14:42:12 2018 -0400
14198: Add paging on container list, check for valid cluster id
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 94eaf9560..902b1ffba 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -247,9 +247,9 @@ $graph:
jsonldPredicate:
_id: "@type"
_type: "@vocab"
- clusterID:
+ cluster_id:
type: string?
doc: The cluster to run the container
- ownerUUID:
+ project_uuid:
type: string?
doc: The project that will own the container requests and intermediate collections
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 83307d331..130b42b5c 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -8,6 +8,19 @@ from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
from .context import ClusterTarget
from functools import partial
+from schema_salad.sourceline import SourceLine
+from cwltool.errors import WorkflowException
+
+def check_cluster_target(self, builder, runtimeContext):
+ cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
+ if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
+ with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug):
+ runtimeContext.cluster_target_id = id(cluster_target_req)
+ runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster
+ runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid
+ if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in self.arvrunner.api._rootDesc["remoteHosts"]:
+ raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster,
+ ", ".join(self.arvrunner.api._rootDesc["remoteHosts"].keys())))
class ArvadosCommandTool(CommandLineTool):
"""Wrap cwltool CommandLineTool to override selected methods."""
@@ -45,11 +58,7 @@ class ArvadosCommandTool(CommandLineTool):
runtimeContext = runtimeContext.copy()
- cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
- runtimeContext.cluster_target_id = id(cluster_target_req)
- runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("clusterID")) or runtimeContext.submit_runner_cluster
- runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("ownerUUID")) or runtimeContext.project_uuid
+ check_cluster_target(self, builder, runtimeContext)
if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 2f114f4ff..b689e94ef 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -132,11 +132,7 @@ class ArvadosWorkflow(Workflow):
def job(self, joborder, output_callback, runtimeContext):
- cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
- runtimeContext.cluster_target_id = id(cluster_target_req)
- runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("clusterID")) or runtimeContext.submit_runner_cluster
- runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("ownerUUID")) or runtimeContext.project_uuid
+ check_cluster_target(self, self._init_job(joborder, runtimeContext), runtimeContext)
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if not req:
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index bf81853be..8c2023e18 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -326,21 +326,26 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
elif self.work_api == "jobs":
table = self.poll_api.jobs()
- try:
- proc_states = table.list(filters=[["uuid", "in", keys]]).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.warn("Error checking states on API server: %s", e)
- remain_wait = self.poll_interval
- continue
+ pageSize = self.poll_api._rootDesc.get('maxItemsPerResponse', 1000)
- for p in proc_states["items"]:
- self.on_message({
- "object_uuid": p["uuid"],
- "event_type": "update",
- "properties": {
- "new_attributes": p
- }
- })
+ while keys:
+ page = keys[:pageSize]
+ keys = keys[pageSize:]
+ try:
+ proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
+ except Exception as e:
+ logger.warn("Error checking states on API server: %s", e)
+ remain_wait = self.poll_interval
+ continue
+
+ for p in proc_states["items"]:
+ self.on_message({
+ "object_uuid": p["uuid"],
+ "event_type": "update",
+ "properties": {
+ "new_attributes": p
+ }
+ })
finish_poll = time.time()
remain_wait = self.poll_interval - (finish_poll - begin_poll)
except:
@@ -631,17 +636,17 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
self.task_queue = TaskQueue(self.workflow_eval_lock, self.thread_count)
- if runnerjob:
- jobiter = iter((runnerjob,))
- else:
- if runtimeContext.cwl_runner_job is not None:
- self.uuid = runtimeContext.cwl_runner_job.get('uuid')
- jobiter = tool.job(job_order,
- self.output_callback,
- runtimeContext)
-
try:
self.workflow_eval_lock.acquire()
+ if runnerjob:
+ jobiter = iter((runnerjob,))
+ else:
+ if runtimeContext.cwl_runner_job is not None:
+ self.uuid = runtimeContext.cwl_runner_job.get('uuid')
+ jobiter = tool.job(job_order,
+ self.output_callback,
+ runtimeContext)
+
# Holds the lock while this code runs and releases it when
# it is safe to do so in self.workflow_eval_lock.wait(),
# at which point on_message can update job state and
@@ -681,7 +686,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
logger.error("Interrupted, workflow will be cancelled")
else:
- logger.error("Execution failed: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+ logger.error("Execution failed:\n%s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
if self.pipeline:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 2d0bc114f..771ef2b1f 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -50,6 +50,7 @@ class Arvados::V1::SchemaController < ApplicationController
defaultTrashLifetime: Rails.application.config.default_trash_lifetime,
blobSignatureTtl: Rails.application.config.blob_signature_ttl,
maxRequestSize: Rails.application.config.max_request_size,
+ maxItemsPerResponse: Rails.application.config.max_items_per_response,
dockerImageFormats: Rails.application.config.docker_image_formats,
crunchLogBytesPerEvent: Rails.application.config.crunch_log_bytes_per_event,
crunchLogSecondsBetweenEvents: Rails.application.config.crunch_log_seconds_between_events,
commit 8a466e1db2578cbfbc41bf7c9a9cbad9c60c551e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Oct 31 13:25:24 2018 -0400
14198: Use ArvRuntimeContext.submit_runner_cluster for target cluster
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 823b41ce9..100329b69 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -36,7 +36,7 @@ metrics = logging.getLogger('arvados.cwl-runner.metrics')
class ArvadosContainer(JobBase):
"""Submit and manage a Crunch container request for executing a CWL CommandLineTool."""
- def __init__(self, runner, cluster_target,
+ def __init__(self, runner, job_runtime,
builder, # type: Builder
joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
make_path_mapper, # type: Callable[..., PathMapper]
@@ -46,7 +46,7 @@ class ArvadosContainer(JobBase):
):
super(ArvadosContainer, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name)
self.arvrunner = runner
- self.cluster_target = cluster_target
+ self.job_runtime = job_runtime
self.running = False
self.uuid = None
@@ -61,6 +61,8 @@ class ArvadosContainer(JobBase):
# ArvadosContainer object by CommandLineTool.job() before
# run() is called.
+ runtimeContext = self.job_runtime
+
container_request = {
"command": self.command_line,
"name": self.name,
@@ -252,11 +254,8 @@ class ArvadosContainer(JobBase):
scheduling_parameters["max_run_time"] = self.timelimit
extra_submit_params = {}
- if self.cluster_target is not None:
- if self.cluster_target.cluster_id:
- extra_submit_params["cluster_id"] = self.cluster_target.cluster_id
- if self.cluster_target.owner_uuid:
- container_request["owner_uuid"] = self.cluster_target.owner_uuid
+ if runtimeContext.submit_runner_cluster:
+ extra_submit_params["cluster_id"] = runtimeContext.submit_runner_cluster
container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index e0997db5b..83307d331 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -18,7 +18,7 @@ class ArvadosCommandTool(CommandLineTool):
def make_job_runner(self, runtimeContext):
if runtimeContext.work_api == "containers":
- return partial(ArvadosContainer, self.arvrunner, runtimeContext.cluster_target)
+ return partial(ArvadosContainer, self.arvrunner, runtimeContext)
elif runtimeContext.work_api == "jobs":
return partial(ArvadosJob, self.arvrunner)
else:
@@ -46,10 +46,10 @@ class ArvadosCommandTool(CommandLineTool):
runtimeContext = runtimeContext.copy()
cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if runtimeContext.cluster_target is None or runtimeContext.cluster_target.instance != id(cluster_target_req):
- runtimeContext.cluster_target = ClusterTarget(id(cluster_target_req),
- builder.do_eval(cluster_target_req.get("clusterID")),
- builder.do_eval(cluster_target_req.get("ownerUUID")))
+ if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
+ runtimeContext.cluster_target_id = id(cluster_target_req)
+ runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("clusterID")) or runtimeContext.submit_runner_cluster
+ runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("ownerUUID")) or runtimeContext.project_uuid
if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f86641bfd..2f114f4ff 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -133,10 +133,10 @@ class ArvadosWorkflow(Workflow):
def job(self, joborder, output_callback, runtimeContext):
cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if runtimeContext.cluster_target is None or runtimeContext.cluster_target.instance != id(cluster_target_req):
- runtimeContext.cluster_target = ClusterTarget(id(cluster_target_req),
- builder.do_eval(cluster_target_req.get("clusterID")),
- builder.do_eval(cluster_target_req.get("ownerUUID")))
+ if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req):
+ runtimeContext.cluster_target_id = id(cluster_target_req)
+ runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("clusterID")) or runtimeContext.submit_runner_cluster
+ runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("ownerUUID")) or runtimeContext.project_uuid
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if not req:
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 23e7b91a0..48f92b77f 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -34,7 +34,7 @@ class ArvRuntimeContext(RuntimeContext):
self.current_container = None
self.http_timeout = 300
self.submit_runner_cluster = None
- self.cluster_target = None
+ self.cluster_target_id = 0
super(ArvRuntimeContext, self).__init__(kwargs)
commit b2f2333e942c343608008edb6c825d80d90ae50e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 17:21:05 2018 -0400
14198: Fix typo current -> current_container, add copyright header
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 7256e1d0d..bf81853be 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import argparse
import logging
import os
@@ -618,7 +622,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
return (runnerjob.uuid, "success")
current_container = get_current_container(self.api, self.num_retries, logger)
- if current:
+ if current_container:
logger.info("Running inside container %s", current_container.get("uuid"))
self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout)
commit 5c644b2f7633cc22a50aa5a156f5bb1325e0c98b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 17:11:59 2018 -0400
14198: Log current container
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 6eaa4b92c..7256e1d0d 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -617,6 +617,10 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
runnerjob.run(submitargs)
return (runnerjob.uuid, "success")
+ current_container = get_current_container(self.api, self.num_retries, logger)
+ if current:
+ logger.info("Running inside container %s", current_container.get("uuid"))
+
self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout)
self.polling_thread = threading.Thread(target=self.poll_states)
self.polling_thread.start()
commit e7be2f19642771f96d25152202f5b2e957a4cb4e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 16:14:50 2018 -0400
14198: Support expressions in TargetCluster[clusterID, ownerUUID]
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index d9466a83a..823b41ce9 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -36,7 +36,7 @@ metrics = logging.getLogger('arvados.cwl-runner.metrics')
class ArvadosContainer(JobBase):
"""Submit and manage a Crunch container request for executing a CWL CommandLineTool."""
- def __init__(self, runner,
+ def __init__(self, runner, cluster_target,
builder, # type: Builder
joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
make_path_mapper, # type: Callable[..., PathMapper]
@@ -46,6 +46,7 @@ class ArvadosContainer(JobBase):
):
super(ArvadosContainer, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name)
self.arvrunner = runner
+ self.cluster_target = cluster_target
self.running = False
self.uuid = None
@@ -251,13 +252,11 @@ class ArvadosContainer(JobBase):
scheduling_parameters["max_run_time"] = self.timelimit
extra_submit_params = {}
- cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- if cluster_target_req:
- cluster_id = cluster_target_req.get("clusterID")
- if cluster_id:
- extra_submit_params["cluster_id"] = cluster_id
- if cluster_target_req.get("ownerUUID"):
- container_request["owner_uuid"] = cluster_target_req.get("ownerUUID")
+ if self.cluster_target is not None:
+ if self.cluster_target.cluster_id:
+ extra_submit_params["cluster_id"] = self.cluster_target.cluster_id
+ if self.cluster_target.owner_uuid:
+ container_request["owner_uuid"] = self.cluster_target.owner_uuid
container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 119acc303..e0997db5b 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -6,6 +6,7 @@ from cwltool.command_line_tool import CommandLineTool
from .arvjob import ArvadosJob
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
+from .context import ClusterTarget
from functools import partial
class ArvadosCommandTool(CommandLineTool):
@@ -17,7 +18,7 @@ class ArvadosCommandTool(CommandLineTool):
def make_job_runner(self, runtimeContext):
if runtimeContext.work_api == "containers":
- return partial(ArvadosContainer, self.arvrunner)
+ return partial(ArvadosContainer, self.arvrunner, runtimeContext.cluster_target)
elif runtimeContext.work_api == "jobs":
return partial(ArvadosJob, self.arvrunner)
else:
@@ -44,6 +45,12 @@ class ArvadosCommandTool(CommandLineTool):
runtimeContext = runtimeContext.copy()
+ cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
+ if runtimeContext.cluster_target is None or runtimeContext.cluster_target.instance != id(cluster_target_req):
+ runtimeContext.cluster_target = ClusterTarget(id(cluster_target_req),
+ builder.do_eval(cluster_target_req.get("clusterID")),
+ builder.do_eval(cluster_target_req.get("ownerUUID")))
+
if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
if dockerReq and dockerReq.get("dockerOutputDirectory"):
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ae9062510..f86641bfd 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -131,158 +131,167 @@ class ArvadosWorkflow(Workflow):
self.loadingContext = loadingContext
def job(self, joborder, output_callback, runtimeContext):
+
+ cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
+ if runtimeContext.cluster_target is None or runtimeContext.cluster_target.instance != id(cluster_target_req):
+ runtimeContext.cluster_target = ClusterTarget(id(cluster_target_req),
+ builder.do_eval(cluster_target_req.get("clusterID")),
+ builder.do_eval(cluster_target_req.get("ownerUUID")))
+
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
- if req:
- with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if "id" not in self.tool:
- raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
- document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
+ if not req:
+ return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
- discover_secondary_files(self.tool["inputs"], joborder)
+ # RunInSingleContainer is true
- with Perf(metrics, "subworkflow upload_deps"):
- upload_dependencies(self.arvrunner,
- os.path.basename(joborder.get("id", "#")),
- document_loader,
- joborder,
- joborder.get("id", "#"),
- False)
+ with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if "id" not in self.tool:
+ raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
+ document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
+
+ discover_secondary_files(self.tool["inputs"], joborder)
+
+ with Perf(metrics, "subworkflow upload_deps"):
+ upload_dependencies(self.arvrunner,
+ os.path.basename(joborder.get("id", "#")),
+ document_loader,
+ joborder,
+ joborder.get("id", "#"),
+ False)
+
+ if self.wf_pdh is None:
+ workflowobj["requirements"] = dedup_reqs(self.requirements)
+ workflowobj["hints"] = dedup_reqs(self.hints)
+
+ packed = pack(document_loader, workflowobj, uri, self.metadata)
- if self.wf_pdh is None:
- workflowobj["requirements"] = dedup_reqs(self.requirements)
- workflowobj["hints"] = dedup_reqs(self.hints)
-
- packed = pack(document_loader, workflowobj, uri, self.metadata)
-
- builder = Builder(joborder,
- requirements=workflowobj["requirements"],
- hints=workflowobj["hints"],
- resources={})
-
- def visit(item):
- for t in ("hints", "requirements"):
- if t not in item:
- continue
- for req in item[t]:
- if req["class"] == "ResourceRequirement":
- dyn = False
- for k in max_res_pars + sum_res_pars:
- if k in req:
- if isinstance(req[k], basestring):
- if item["id"] == "#main":
- # only the top-level requirements/hints may contain expressions
- self.dynamic_resource_req.append(req)
- dyn = True
- break
- else:
- with SourceLine(req, k, WorkflowException):
- raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
- if not dyn:
- self.static_resource_req.append(req)
-
- visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
-
- if self.static_resource_req:
- self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
-
- upload_dependencies(self.arvrunner,
- runtimeContext.name,
- document_loader,
- packed,
- uri,
- False)
-
- # Discover files/directories referenced by the
- # workflow (mainly "default" values)
- visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
-
-
- if self.dynamic_resource_req:
builder = Builder(joborder,
- requirements=self.requirements,
- hints=self.hints,
+ requirements=workflowobj["requirements"],
+ hints=workflowobj["hints"],
resources={})
- # Evaluate dynamic resource requirements using current builder
- rs = copy.copy(self.static_resource_req)
- for dyn_rs in self.dynamic_resource_req:
- eval_req = {"class": "ResourceRequirement"}
- for a in max_res_pars + sum_res_pars:
- if a in dyn_rs:
- eval_req[a] = builder.do_eval(dyn_rs[a])
- rs.append(eval_req)
- job_res_reqs = [get_overall_res_req(rs)]
- else:
- job_res_reqs = self.static_resource_req
-
- with Perf(metrics, "subworkflow adjust"):
- joborder_resolved = copy.deepcopy(joborder)
- joborder_keepmount = copy.deepcopy(joborder)
-
- reffiles = []
- visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
-
- mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, runtimeContext.basedir,
- "/keep/%s",
- "/keep/%s/%s")
-
- # For containers API, we need to make sure any extra
- # referenced files (ie referenced by the workflow but
- # not in the inputs) are included in the mounts.
- if self.wf_reffiles:
- runtimeContext = runtimeContext.copy()
- runtimeContext.extra_reffiles = copy.deepcopy(self.wf_reffiles)
-
- def keepmount(obj):
- remove_redundant_fields(obj)
- with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if "location" not in obj:
- raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
- with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if obj["location"].startswith("keep:"):
- obj["location"] = mapper.mapper(obj["location"]).target
- if "listing" in obj:
- del obj["listing"]
- elif obj["location"].startswith("_:"):
- del obj["location"]
- else:
- raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
-
- visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
-
- def resolved(obj):
- if obj["location"].startswith("keep:"):
- obj["location"] = mapper.mapper(obj["location"]).resolved
-
- visit_class(joborder_resolved, ("File", "Directory"), resolved)
-
- if self.wf_pdh is None:
- adjustFileObjs(packed, keepmount)
- adjustDirObjs(packed, keepmount)
- self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
-
- wf_runner = cmap({
- "class": "CommandLineTool",
- "baseCommand": "cwltool",
- "inputs": self.tool["inputs"],
- "outputs": self.tool["outputs"],
- "stdout": "cwl.output.json",
- "requirements": self.requirements+job_res_reqs+[
- {"class": "InlineJavascriptRequirement"},
- {
- "class": "InitialWorkDirRequirement",
- "listing": [{
- "entryname": "workflow.cwl",
- "entry": '$({"class": "File", "location": "keep:%s/workflow.cwl"})' % self.wf_pdh
- }, {
- "entryname": "cwl.input.yml",
- "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
- }]
- }],
- "hints": self.hints,
- "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
- "id": "#"
- })
- return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+ def visit(item):
+ for t in ("hints", "requirements"):
+ if t not in item:
+ continue
+ for req in item[t]:
+ if req["class"] == "ResourceRequirement":
+ dyn = False
+ for k in max_res_pars + sum_res_pars:
+ if k in req:
+ if isinstance(req[k], basestring):
+ if item["id"] == "#main":
+ # only the top-level requirements/hints may contain expressions
+ self.dynamic_resource_req.append(req)
+ dyn = True
+ break
+ else:
+ with SourceLine(req, k, WorkflowException):
+ raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
+ if not dyn:
+ self.static_resource_req.append(req)
+
+ visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
+
+ if self.static_resource_req:
+ self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
+
+ upload_dependencies(self.arvrunner,
+ runtimeContext.name,
+ document_loader,
+ packed,
+ uri,
+ False)
+
+ # Discover files/directories referenced by the
+ # workflow (mainly "default" values)
+ visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
+
+
+ if self.dynamic_resource_req:
+ builder = Builder(joborder,
+ requirements=self.requirements,
+ hints=self.hints,
+ resources={})
+
+ # Evaluate dynamic resource requirements using current builder
+ rs = copy.copy(self.static_resource_req)
+ for dyn_rs in self.dynamic_resource_req:
+ eval_req = {"class": "ResourceRequirement"}
+ for a in max_res_pars + sum_res_pars:
+ if a in dyn_rs:
+ eval_req[a] = builder.do_eval(dyn_rs[a])
+ rs.append(eval_req)
+ job_res_reqs = [get_overall_res_req(rs)]
else:
- return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
+ job_res_reqs = self.static_resource_req
+
+ with Perf(metrics, "subworkflow adjust"):
+ joborder_resolved = copy.deepcopy(joborder)
+ joborder_keepmount = copy.deepcopy(joborder)
+
+ reffiles = []
+ visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
+
+ mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, runtimeContext.basedir,
+ "/keep/%s",
+ "/keep/%s/%s")
+
+ # For containers API, we need to make sure any extra
+ # referenced files (ie referenced by the workflow but
+ # not in the inputs) are included in the mounts.
+ if self.wf_reffiles:
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.extra_reffiles = copy.deepcopy(self.wf_reffiles)
+
+ def keepmount(obj):
+ remove_redundant_fields(obj)
+ with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if "location" not in obj:
+ raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
+ with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if obj["location"].startswith("keep:"):
+ obj["location"] = mapper.mapper(obj["location"]).target
+ if "listing" in obj:
+ del obj["listing"]
+ elif obj["location"].startswith("_:"):
+ del obj["location"]
+ else:
+ raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
+
+ visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
+
+ def resolved(obj):
+ if obj["location"].startswith("keep:"):
+ obj["location"] = mapper.mapper(obj["location"]).resolved
+
+ visit_class(joborder_resolved, ("File", "Directory"), resolved)
+
+ if self.wf_pdh is None:
+ adjustFileObjs(packed, keepmount)
+ adjustDirObjs(packed, keepmount)
+ self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
+
+ wf_runner = cmap({
+ "class": "CommandLineTool",
+ "baseCommand": "cwltool",
+ "inputs": self.tool["inputs"],
+ "outputs": self.tool["outputs"],
+ "stdout": "cwl.output.json",
+ "requirements": self.requirements+job_res_reqs+[
+ {"class": "InlineJavascriptRequirement"},
+ {
+ "class": "InitialWorkDirRequirement",
+ "listing": [{
+ "entryname": "workflow.cwl",
+ "entry": '$({"class": "File", "location": "keep:%s/workflow.cwl"})' % self.wf_pdh
+ }, {
+ "entryname": "cwl.input.yml",
+ "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
+ }]
+ }],
+ "hints": self.hints,
+ "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
+ "id": "#"
+ })
+ return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 4e1334c1c..23e7b91a0 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -3,11 +3,14 @@
# SPDX-License-Identifier: Apache-2.0
from cwltool.context import LoadingContext, RuntimeContext
+from collections import namedtuple
class ArvLoadingContext(LoadingContext):
def __init__(self, kwargs=None):
super(ArvLoadingContext, self).__init__(kwargs)
+ClusterTarget = namedtuple("ClusterTarget", ("instance", "cluster_id", "owner_uuid"))
+
class ArvRuntimeContext(RuntimeContext):
def __init__(self, kwargs=None):
self.work_api = None
@@ -31,6 +34,7 @@ class ArvRuntimeContext(RuntimeContext):
self.current_container = None
self.http_timeout = 300
self.submit_runner_cluster = None
+ self.cluster_target = None
super(ArvRuntimeContext, self).__init__(kwargs)
commit 967c767e604bfe41a9ddac49f394a0b946eb09d7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 15:19:05 2018 -0400
14198: Resolve to Docker images to PDH and set "http://arvados.org/cwl#dockerCollectionPDH"
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 7e22a7d39..d9466a83a 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -214,8 +214,7 @@ class ArvadosContainer(JobBase):
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
runtimeContext.pull_image,
- self.arvrunner.project_uuid,
- runtimeContext.submit_runner_cluster)
+ self.arvrunner.project_uuid)
api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement")
if api_req:
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 6bca07c88..84006b47d 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -21,6 +21,9 @@ cached_lookups_lock = threading.Lock()
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
+ if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
+ return dockerRequirement["http://arvados.org/cwl#dockerCollectionPDH"]
+
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement = copy.deepcopy(dockerRequirement)
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index bbfb8ffc6..6eaa4b92c 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -37,6 +37,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
from cwltool.command_line_tool import compute_checksums
logger = logging.getLogger('arvados.cwl-runner')
+metrics = logging.getLogger('arvados.cwl-runner.metrics')
class RuntimeStatusLoggingHandler(logging.Handler):
"""
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3b40552ac..31a424d30 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -26,7 +26,7 @@ from cwltool.pack import pack
import arvados.collection
import ruamel.yaml as yaml
-from .arvdocker import arv_docker_get_image
+import arvdocker
from .pathmapper import ArvPathMapper, trim_listing
from ._version import __version__
from . import done
@@ -215,9 +215,9 @@ def upload_docker(arvrunner, tool):
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
+ arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
else:
- arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
+ arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
@@ -244,6 +244,9 @@ def packed_workflow(arvrunner, tool, merged_map):
v["location"] = merged_map[cur_id].resolved[v["location"]]
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
+ if v.get("class") == "DockerRequirement":
+ img = v.get("dockerImageId") or v.get("dockerPull")
+ v["http://arvados.org/cwl#dockerCollectionPDH"] = arvdocker.cached_lookups[img]
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
@@ -324,7 +327,7 @@ def arvados_jobs_image(arvrunner, img):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- return arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
commit 5f9337c53ad135e977bda86bdbcfa2a59acbb5f1
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 15:02:23 2018 -0400
14198: Refactor and add support for --submit-runner-cluster
Rename ArvCwlRunner to ArvCwlExecutor and move into its own file.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 0866f69d6..63fc3ea47 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -10,19 +10,9 @@ import argparse
import logging
import os
import sys
-import threading
-import hashlib
-import copy
-import json
import re
-from functools import partial
import pkg_resources # part of setuptools
-import Queue
-import time
-import signal
-import thread
-from cwltool.errors import WorkflowException
import cwltool.main
import cwltool.workflow
import cwltool.process
@@ -36,23 +26,12 @@ from arvados.keep import KeepClient
from arvados.errors import ApiError
import arvados.commands._util as arv_cmd
-from .arvcontainer import ArvadosContainer, RunnerContainer
-from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
-from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
-from .arvtool import ArvadosCommandTool
-from .arvworkflow import ArvadosWorkflow, upload_workflow
-from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache
from .perf import Perf
-from .pathmapper import NoFollowPathMapper
-from .task_queue import TaskQueue
-from .context import ArvLoadingContext, ArvRuntimeContext
-from .util import get_current_container
from ._version import __version__
+from .executor import ArvCwlExecutor
-from cwltool.pack import pack
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
-from cwltool.command_line_tool import compute_checksums
from arvados.api import OrderedJsonModel
@@ -66,677 +45,6 @@ arvados.log_handler.setFormatter(logging.Formatter(
DEFAULT_PRIORITY = 500
-class RuntimeStatusLoggingHandler(logging.Handler):
- """
- Intercepts logging calls and report them as runtime statuses on runner
- containers.
- """
- def __init__(self, runtime_status_update_func):
- super(RuntimeStatusLoggingHandler, self).__init__()
- self.runtime_status_update = runtime_status_update_func
-
- def emit(self, record):
- kind = None
- if record.levelno >= logging.ERROR:
- kind = 'error'
- elif record.levelno >= logging.WARNING:
- kind = 'warning'
- if kind is not None:
- log_msg = record.getMessage()
- if '\n' in log_msg:
- # If the logged message is multi-line, use its first line as status
- # and the rest as detail.
- status, detail = log_msg.split('\n', 1)
- self.runtime_status_update(
- kind,
- "%s: %s" % (record.name, status),
- detail
- )
- else:
- self.runtime_status_update(
- kind,
- "%s: %s" % (record.name, record.getMessage())
- )
-
-class ArvCwlRunner(object):
- """Execute a CWL tool or workflow, submit work (using either jobs or
- containers API), wait for them to complete, and report output.
-
- """
-
- def __init__(self, api_client,
- arvargs=None,
- keep_client=None,
- num_retries=4,
- thread_count=4):
-
- if arvargs is None:
- arvargs = argparse.Namespace()
- arvargs.work_api = None
- arvargs.output_name = None
- arvargs.output_tags = None
- arvargs.thread_count = 1
-
- self.api = api_client
- self.processes = {}
- self.workflow_eval_lock = threading.Condition(threading.RLock())
- self.final_output = None
- self.final_status = None
- self.num_retries = num_retries
- self.uuid = None
- self.stop_polling = threading.Event()
- self.poll_api = None
- self.pipeline = None
- self.final_output_collection = None
- self.output_name = arvargs.output_name
- self.output_tags = arvargs.output_tags
- self.project_uuid = None
- self.intermediate_output_ttl = 0
- self.intermediate_output_collections = []
- self.trash_intermediate = False
- self.thread_count = arvargs.thread_count
- self.poll_interval = 12
- self.loadingContext = None
-
- if keep_client is not None:
- self.keep_client = keep_client
- else:
- self.keep_client = arvados.keep.KeepClient(api_client=self.api, num_retries=self.num_retries)
-
- self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries)
-
- self.fetcher_constructor = partial(CollectionFetcher,
- api_client=self.api,
- fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
- num_retries=self.num_retries)
-
- self.work_api = None
- expected_api = ["jobs", "containers"]
- for api in expected_api:
- try:
- methods = self.api._rootDesc.get('resources')[api]['methods']
- if ('httpMethod' in methods['create'] and
- (arvargs.work_api == api or arvargs.work_api is None)):
- self.work_api = api
- break
- except KeyError:
- pass
-
- if not self.work_api:
- if arvargs.work_api is None:
- raise Exception("No supported APIs")
- else:
- raise Exception("Unsupported API '%s', expected one of %s" % (arvargs.work_api, expected_api))
-
- if self.work_api == "jobs":
- logger.warn("""
-*******************************
-Using the deprecated 'jobs' API.
-
-To get rid of this warning:
-
-Users: read about migrating at
-http://doc.arvados.org/user/cwl/cwl-style.html#migrate
-and use the option --api=containers
-
-Admins: configure the cluster to disable the 'jobs' API as described at:
-http://doc.arvados.org/install/install-api-server.html#disable_api_methods
-*******************************""")
-
- self.loadingContext = ArvLoadingContext(vars(arvargs))
- self.loadingContext.fetcher_constructor = self.fetcher_constructor
- self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
- self.loadingContext.construct_tool_object = self.arv_make_tool
-
- # Add a custom logging handler to the root logger for runtime status reporting
- # if running inside a container
- if get_current_container(self.api, self.num_retries, logger):
- root_logger = logging.getLogger('')
- handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
- root_logger.addHandler(handler)
-
- def arv_make_tool(self, toolpath_object, loadingContext):
- if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
- return ArvadosCommandTool(self, toolpath_object, loadingContext)
- elif "class" in toolpath_object and toolpath_object["class"] == "Workflow":
- return ArvadosWorkflow(self, toolpath_object, loadingContext)
- else:
- return cwltool.workflow.default_make_tool(toolpath_object, loadingContext)
-
- def output_callback(self, out, processStatus):
- with self.workflow_eval_lock:
- if processStatus == "success":
- logger.info("Overall process status is %s", processStatus)
- state = "Complete"
- else:
- logger.error("Overall process status is %s", processStatus)
- state = "Failed"
- if self.pipeline:
- self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
- body={"state": state}).execute(num_retries=self.num_retries)
- self.final_status = processStatus
- self.final_output = out
- self.workflow_eval_lock.notifyAll()
-
-
- def start_run(self, runnable, runtimeContext):
- self.task_queue.add(partial(runnable.run, runtimeContext))
-
- def process_submitted(self, container):
- with self.workflow_eval_lock:
- self.processes[container.uuid] = container
-
- def process_done(self, uuid, record):
- with self.workflow_eval_lock:
- j = self.processes[uuid]
- logger.info("%s %s is %s", self.label(j), uuid, record["state"])
- self.task_queue.add(partial(j.done, record))
- del self.processes[uuid]
-
- def runtime_status_update(self, kind, message, detail=None):
- """
- Updates the runtime_status field on the runner container.
- Called when there's a need to report errors, warnings or just
- activity statuses, for example in the RuntimeStatusLoggingHandler.
- """
- with self.workflow_eval_lock:
- current = get_current_container(self.api, self.num_retries, logger)
- if current is None:
- return
- runtime_status = current.get('runtime_status', {})
- # In case of status being an error, only report the first one.
- if kind == 'error':
- if not runtime_status.get('error'):
- runtime_status.update({
- 'error': message
- })
- if detail is not None:
- runtime_status.update({
- 'errorDetail': detail
- })
- # Further errors are only mentioned as a count.
- else:
- # Get anything before an optional 'and N more' string.
- try:
- error_msg = re.match(
- r'^(.*?)(?=\s*\(and \d+ more\)|$)', runtime_status.get('error')).groups()[0]
- more_failures = re.match(
- r'.*\(and (\d+) more\)', runtime_status.get('error'))
- except TypeError:
- # Ignore tests stubbing errors
- return
- if more_failures:
- failure_qty = int(more_failures.groups()[0])
- runtime_status.update({
- 'error': "%s (and %d more)" % (error_msg, failure_qty+1)
- })
- else:
- runtime_status.update({
- 'error': "%s (and 1 more)" % error_msg
- })
- elif kind in ['warning', 'activity']:
- # Record the last warning/activity status without regard of
- # previous occurences.
- runtime_status.update({
- kind: message
- })
- if detail is not None:
- runtime_status.update({
- kind+"Detail": detail
- })
- else:
- # Ignore any other status kind
- return
- try:
- self.api.containers().update(uuid=current['uuid'],
- body={
- 'runtime_status': runtime_status,
- }).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.info("Couldn't update runtime_status: %s", e)
-
- def wrapped_callback(self, cb, obj, st):
- with self.workflow_eval_lock:
- cb(obj, st)
- self.workflow_eval_lock.notifyAll()
-
- def get_wrapped_callback(self, cb):
- return partial(self.wrapped_callback, cb)
-
- def on_message(self, event):
- if event.get("object_uuid") in self.processes and event["event_type"] == "update":
- uuid = event["object_uuid"]
- if event["properties"]["new_attributes"]["state"] == "Running":
- with self.workflow_eval_lock:
- j = self.processes[uuid]
- if j.running is False:
- j.running = True
- j.update_pipeline_component(event["properties"]["new_attributes"])
- logger.info("%s %s is Running", self.label(j), uuid)
- elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled", "Final"):
- self.process_done(uuid, event["properties"]["new_attributes"])
-
- def label(self, obj):
- return "[%s %s]" % (self.work_api[0:-1], obj.name)
-
- def poll_states(self):
- """Poll status of jobs or containers listed in the processes dict.
-
- Runs in a separate thread.
- """
-
- try:
- remain_wait = self.poll_interval
- while True:
- if remain_wait > 0:
- self.stop_polling.wait(remain_wait)
- if self.stop_polling.is_set():
- break
- with self.workflow_eval_lock:
- keys = list(self.processes.keys())
- if not keys:
- remain_wait = self.poll_interval
- continue
-
- begin_poll = time.time()
- if self.work_api == "containers":
- table = self.poll_api.container_requests()
- elif self.work_api == "jobs":
- table = self.poll_api.jobs()
-
- try:
- proc_states = table.list(filters=[["uuid", "in", keys]]).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.warn("Error checking states on API server: %s", e)
- remain_wait = self.poll_interval
- continue
-
- for p in proc_states["items"]:
- self.on_message({
- "object_uuid": p["uuid"],
- "event_type": "update",
- "properties": {
- "new_attributes": p
- }
- })
- finish_poll = time.time()
- remain_wait = self.poll_interval - (finish_poll - begin_poll)
- except:
- logger.exception("Fatal error in state polling thread.")
- with self.workflow_eval_lock:
- self.processes.clear()
- self.workflow_eval_lock.notifyAll()
- finally:
- self.stop_polling.set()
-
- def add_intermediate_output(self, uuid):
- if uuid:
- self.intermediate_output_collections.append(uuid)
-
- def trash_intermediate_output(self):
- logger.info("Cleaning up intermediate output collections")
- for i in self.intermediate_output_collections:
- try:
- self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
- except:
- logger.warn("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
- if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
- break
-
- def check_features(self, obj):
- if isinstance(obj, dict):
- if obj.get("writable") and self.work_api != "containers":
- raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
- if obj.get("class") == "DockerRequirement":
- if obj.get("dockerOutputDirectory"):
- if self.work_api != "containers":
- raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
- "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
- if not obj.get("dockerOutputDirectory").startswith('/'):
- raise SourceLine(obj, "dockerOutputDirectory", validate.ValidationException).makeError(
- "Option 'dockerOutputDirectory' must be an absolute path.")
- if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
- raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
- for v in obj.itervalues():
- self.check_features(v)
- elif isinstance(obj, list):
- for i,v in enumerate(obj):
- with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
- self.check_features(v)
-
- def make_output_collection(self, name, storage_classes, tagsString, outputObj):
- outputObj = copy.deepcopy(outputObj)
-
- files = []
- def capture(fileobj):
- files.append(fileobj)
-
- adjustDirObjs(outputObj, capture)
- adjustFileObjs(outputObj, capture)
-
- generatemapper = NoFollowPathMapper(files, "", "", separateDirs=False)
-
- final = arvados.collection.Collection(api_client=self.api,
- keep_client=self.keep_client,
- num_retries=self.num_retries)
-
- for k,v in generatemapper.items():
- if k.startswith("_:"):
- if v.type == "Directory":
- continue
- if v.type == "CreateFile":
- with final.open(v.target, "wb") as f:
- f.write(v.resolved.encode("utf-8"))
- continue
-
- if not k.startswith("keep:"):
- raise Exception("Output source is not in keep or a literal")
- sp = k.split("/")
- srccollection = sp[0][5:]
- try:
- reader = self.collection_cache.get(srccollection)
- srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
- final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
- except arvados.errors.ArgumentError as e:
- logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
- raise
- except IOError as e:
- logger.warn("While preparing output collection: %s", e)
-
- def rewrite(fileobj):
- fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
- for k in ("listing", "contents", "nameext", "nameroot", "dirname"):
- if k in fileobj:
- del fileobj[k]
-
- adjustDirObjs(outputObj, rewrite)
- adjustFileObjs(outputObj, rewrite)
-
- with final.open("cwl.output.json", "w") as f:
- json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
-
- final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
-
- logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
- final.api_response()["name"],
- final.manifest_locator())
-
- final_uuid = final.manifest_locator()
- tags = tagsString.split(',')
- for tag in tags:
- self.api.links().create(body={
- "head_uuid": final_uuid, "link_class": "tag", "name": tag
- }).execute(num_retries=self.num_retries)
-
- def finalcollection(fileobj):
- fileobj["location"] = "keep:%s/%s" % (final.portable_data_hash(), fileobj["location"])
-
- adjustDirObjs(outputObj, finalcollection)
- adjustFileObjs(outputObj, finalcollection)
-
- return (outputObj, final)
-
- def set_crunch_output(self):
- if self.work_api == "containers":
- current = get_current_container(self.api, self.num_retries, logger)
- if current is None:
- return
- try:
- self.api.containers().update(uuid=current['uuid'],
- body={
- 'output': self.final_output_collection.portable_data_hash(),
- }).execute(num_retries=self.num_retries)
- self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
- body={
- 'is_trashed': True
- }).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.info("Setting container output: %s", e)
- elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
- self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
- body={
- 'output': self.final_output_collection.portable_data_hash(),
- 'success': self.final_status == "success",
- 'progress':1.0
- }).execute(num_retries=self.num_retries)
-
- def arv_executor(self, tool, job_order, runtimeContext, logger=None):
- self.debug = runtimeContext.debug
-
- tool.visit(self.check_features)
-
- self.project_uuid = runtimeContext.project_uuid
- self.pipeline = None
- self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
- self.secret_store = runtimeContext.secret_store
-
- self.trash_intermediate = runtimeContext.trash_intermediate
- if self.trash_intermediate and self.work_api != "containers":
- raise Exception("--trash-intermediate is only supported with --api=containers.")
-
- self.intermediate_output_ttl = runtimeContext.intermediate_output_ttl
- if self.intermediate_output_ttl and self.work_api != "containers":
- raise Exception("--intermediate-output-ttl is only supported with --api=containers.")
- if self.intermediate_output_ttl < 0:
- raise Exception("Invalid value %d for --intermediate-output-ttl, cannot be less than zero" % self.intermediate_output_ttl)
-
- if runtimeContext.submit_request_uuid and self.work_api != "containers":
- raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
-
- if not runtimeContext.name:
- runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
-
- # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
- # Also uploads docker images.
- merged_map = upload_workflow_deps(self, tool)
-
- # Reload tool object which may have been updated by
- # upload_workflow_deps
- # Don't validate this time because it will just print redundant errors.
- loadingContext = self.loadingContext.copy()
- loadingContext.loader = tool.doc_loader
- loadingContext.avsc_names = tool.doc_schema
- loadingContext.metadata = tool.metadata
- loadingContext.do_validate = False
-
- tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
- loadingContext)
-
- # Upload local file references in the job order.
- job_order = upload_job_order(self, "%s input" % runtimeContext.name,
- tool, job_order)
-
- existing_uuid = runtimeContext.update_workflow
- if existing_uuid or runtimeContext.create_workflow:
- # Create a pipeline template or workflow record and exit.
- if self.work_api == "jobs":
- tmpl = RunnerTemplate(self, tool, job_order,
- runtimeContext.enable_reuse,
- uuid=existing_uuid,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- merged_map=merged_map)
- tmpl.save()
- # cwltool.main will write our return value to stdout.
- return (tmpl.uuid, "success")
- elif self.work_api == "containers":
- return (upload_workflow(self, tool, job_order,
- self.project_uuid,
- uuid=existing_uuid,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- merged_map=merged_map),
- "success")
-
- self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
- self.eval_timeout = runtimeContext.eval_timeout
-
- runtimeContext = runtimeContext.copy()
- runtimeContext.use_container = True
- runtimeContext.tmpdir_prefix = "tmp"
- runtimeContext.work_api = self.work_api
-
- if self.work_api == "containers":
- if self.ignore_docker_for_reuse:
- raise Exception("--ignore-docker-for-reuse not supported with containers API.")
- runtimeContext.outdir = "/var/spool/cwl"
- runtimeContext.docker_outdir = "/var/spool/cwl"
- runtimeContext.tmpdir = "/tmp"
- runtimeContext.docker_tmpdir = "/tmp"
- elif self.work_api == "jobs":
- if runtimeContext.priority != DEFAULT_PRIORITY:
- raise Exception("--priority not implemented for jobs API.")
- runtimeContext.outdir = "$(task.outdir)"
- runtimeContext.docker_outdir = "$(task.outdir)"
- runtimeContext.tmpdir = "$(task.tmpdir)"
-
- if runtimeContext.priority < 1 or runtimeContext.priority > 1000:
- raise Exception("--priority must be in the range 1..1000.")
-
- runnerjob = None
- if runtimeContext.submit:
- # Submit a runner job to run the workflow for us.
- if self.work_api == "containers":
- if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait:
- runtimeContext.runnerjob = tool.tool["id"]
- runnerjob = tool.job(job_order,
- self.output_callback,
- runtimeContext).next()
- else:
- runnerjob = RunnerContainer(self, tool, job_order, runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- on_error=runtimeContext.on_error,
- submit_runner_image=runtimeContext.submit_runner_image,
- intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
- merged_map=merged_map,
- priority=runtimeContext.priority,
- secret_store=self.secret_store)
- elif self.work_api == "jobs":
- runnerjob = RunnerJob(self, tool, job_order, runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- on_error=runtimeContext.on_error,
- submit_runner_image=runtimeContext.submit_runner_image,
- merged_map=merged_map)
- elif runtimeContext.cwl_runner_job is None and self.work_api == "jobs":
- # Create pipeline for local run
- self.pipeline = self.api.pipeline_instances().create(
- body={
- "owner_uuid": self.project_uuid,
- "name": runtimeContext.name if runtimeContext.name else shortname(tool.tool["id"]),
- "components": {},
- "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
- logger.info("Pipeline instance %s", self.pipeline["uuid"])
-
- if runnerjob and not runtimeContext.wait:
- submitargs = runtimeContext.copy()
- submitargs.submit = False
- runnerjob.run(submitargs)
- return (runnerjob.uuid, "success")
-
- self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout)
- self.polling_thread = threading.Thread(target=self.poll_states)
- self.polling_thread.start()
-
- self.task_queue = TaskQueue(self.workflow_eval_lock, self.thread_count)
-
- if runnerjob:
- jobiter = iter((runnerjob,))
- else:
- if runtimeContext.cwl_runner_job is not None:
- self.uuid = runtimeContext.cwl_runner_job.get('uuid')
- jobiter = tool.job(job_order,
- self.output_callback,
- runtimeContext)
-
- try:
- self.workflow_eval_lock.acquire()
- # Holds the lock while this code runs and releases it when
- # it is safe to do so in self.workflow_eval_lock.wait(),
- # at which point on_message can update job state and
- # process output callbacks.
-
- loopperf = Perf(metrics, "jobiter")
- loopperf.__enter__()
- for runnable in jobiter:
- loopperf.__exit__()
-
- if self.stop_polling.is_set():
- break
-
- if self.task_queue.error is not None:
- raise self.task_queue.error
-
- if runnable:
- with Perf(metrics, "run"):
- self.start_run(runnable, runtimeContext)
- else:
- if (self.task_queue.in_flight + len(self.processes)) > 0:
- self.workflow_eval_lock.wait(3)
- else:
- logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
- break
- loopperf.__enter__()
- loopperf.__exit__()
-
- while (self.task_queue.in_flight + len(self.processes)) > 0:
- if self.task_queue.error is not None:
- raise self.task_queue.error
- self.workflow_eval_lock.wait(3)
-
- except UnsupportedRequirement:
- raise
- except:
- if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
- logger.error("Interrupted, workflow will be cancelled")
- else:
- logger.error("Execution failed: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
- if self.pipeline:
- self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
- body={"state": "Failed"}).execute(num_retries=self.num_retries)
- if runnerjob and runnerjob.uuid and self.work_api == "containers":
- self.api.container_requests().update(uuid=runnerjob.uuid,
- body={"priority": "0"}).execute(num_retries=self.num_retries)
- finally:
- self.workflow_eval_lock.release()
- self.task_queue.drain()
- self.stop_polling.set()
- self.polling_thread.join()
- self.task_queue.join()
-
- if self.final_status == "UnsupportedRequirement":
- raise UnsupportedRequirement("Check log for details.")
-
- if self.final_output is None:
- raise WorkflowException("Workflow did not return a result.")
-
- if runtimeContext.submit and isinstance(runnerjob, Runner):
- logger.info("Final output collection %s", runnerjob.final_output)
- else:
- if self.output_name is None:
- self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
- if self.output_tags is None:
- self.output_tags = ""
-
- storage_classes = runtimeContext.storage_classes.strip().split(",")
- self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
- self.set_crunch_output()
-
- if runtimeContext.compute_checksum:
- adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
- adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
-
- if self.trash_intermediate and self.final_status == "success":
- self.trash_intermediate_output()
-
- return (self.final_output, self.final_status)
-
-
def versionstring():
"""Print version string of key packages for provenance and debugging."""
@@ -831,9 +139,13 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
default=None)
- parser.add_argument("--submit-request-uuid", type=str,
+ exgroup = parser.add_mutually_exclusive_group()
+ exgroup.add_argument("--submit-request-uuid", type=str,
default=None,
- help="Update and commit supplied container request instead of creating a new one (containers API only).")
+ help="Update and commit to supplied container request instead of creating a new one (containers API only).")
+ exgroup.add_argument("--submit-runner-cluster", type=str,
+ help="Submit toplevel runner to a remote cluster (containers API only)",
+ default=None)
parser.add_argument("--name", type=str,
help="Name to use for workflow execution instance.",
@@ -942,6 +254,10 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
add_arv_hints()
+ for key, val in cwltool.argparser.get_default_args().items():
+ if not hasattr(arvargs, key):
+ setattr(arvargs, key, val)
+
try:
if api_client is None:
api_client = arvados.safeapi.ThreadSafeApiCache(
@@ -952,7 +268,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- runner = ArvCwlRunner(api_client, arvargs, keep_client=keep_client, num_retries=4)
+ executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
except Exception as e:
logger.error(e)
return 1
@@ -977,22 +293,13 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
else:
arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
- for key, val in cwltool.argparser.get_default_args().items():
- if not hasattr(arvargs, key):
- setattr(arvargs, key, val)
-
- runtimeContext = ArvRuntimeContext(vars(arvargs))
- runtimeContext.make_fs_access = partial(CollectionFsAccess,
- collection_cache=runner.collection_cache)
- runtimeContext.http_timeout = arvargs.http_timeout
-
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
- executor=runner.arv_executor,
+ executor=executor.arv_executor,
versionfunc=versionstring,
job_order_object=job_order_object,
logger_handler=arvados.log_handler,
custom_schema_callback=add_arv_hints,
- loadingContext=runner.loadingContext,
- runtimeContext=runtimeContext)
+ loadingContext=executor.loadingContext,
+ runtimeContext=executor.runtimeContext)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b46711af4..7e22a7d39 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -212,9 +212,10 @@ class ArvadosContainer(JobBase):
docker_req = {"dockerImageId": "arvados/jobs"}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
- docker_req,
- runtimeContext.pull_image,
- self.arvrunner.project_uuid)
+ docker_req,
+ runtimeContext.pull_image,
+ self.arvrunner.project_uuid,
+ runtimeContext.submit_runner_cluster)
api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement")
if api_req:
@@ -490,14 +491,20 @@ class RunnerContainer(Runner):
if self.arvrunner.project_uuid:
job_spec["owner_uuid"] = self.arvrunner.project_uuid
+ extra_submit_params = {}
+ if runtimeContext.submit_runner_cluster:
+ extra_submit_params["cluster_id"] = runtimeContext.submit_runner_cluster
+
if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
uuid=runtimeContext.submit_request_uuid,
- body=job_spec
+ body=job_spec,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
else:
response = self.arvrunner.api.container_requests().create(
- body=job_spec
+ body=job_spec,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 48a3edec5..4e1334c1c 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -30,5 +30,9 @@ class ArvRuntimeContext(RuntimeContext):
self.storage_classes = "default"
self.current_container = None
self.http_timeout = 300
+ self.submit_runner_cluster = None
super(ArvRuntimeContext, self).__init__(kwargs)
+
+ if self.submit_request_uuid:
+ self.submit_runner_cluster = self.submit_request_uuid[0:5]
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/executor.py
similarity index 69%
copy from sdk/cwl/arvados_cwl/__init__.py
copy to sdk/cwl/arvados_cwl/executor.py
index 0866f69d6..bbfb8ffc6 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -1,43 +1,26 @@
-#!/usr/bin/env python
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-# Implement cwl-runner interface for submitting and running work on Arvados, using
-# either the Crunch jobs API or Crunch containers API.
-
import argparse
import logging
import os
import sys
import threading
-import hashlib
import copy
import json
import re
from functools import partial
-import pkg_resources # part of setuptools
-import Queue
import time
-import signal
-import thread
from cwltool.errors import WorkflowException
-import cwltool.main
import cwltool.workflow
-import cwltool.process
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
-import cwltool.argparser
import arvados
import arvados.config
from arvados.keep import KeepClient
from arvados.errors import ApiError
-import arvados.commands._util as arv_cmd
-from .arvcontainer import ArvadosContainer, RunnerContainer
-from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
+from .arvcontainer import RunnerContainer
+from .arvjob import RunnerJob, RunnerTemplate
from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
from .arvtool import ArvadosCommandTool
from .arvworkflow import ArvadosWorkflow, upload_workflow
@@ -49,22 +32,11 @@ from .context import ArvLoadingContext, ArvRuntimeContext
from .util import get_current_container
from ._version import __version__
-from cwltool.pack import pack
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
from cwltool.command_line_tool import compute_checksums
-from arvados.api import OrderedJsonModel
-
logger = logging.getLogger('arvados.cwl-runner')
-metrics = logging.getLogger('arvados.cwl-runner.metrics')
-logger.setLevel(logging.INFO)
-
-arvados.log_handler.setFormatter(logging.Formatter(
- '%(asctime)s %(name)s %(levelname)s: %(message)s',
- '%Y-%m-%d %H:%M:%S'))
-
-DEFAULT_PRIORITY = 500
class RuntimeStatusLoggingHandler(logging.Handler):
"""
@@ -98,7 +70,7 @@ class RuntimeStatusLoggingHandler(logging.Handler):
"%s: %s" % (record.name, record.getMessage())
)
-class ArvCwlRunner(object):
+class ArvCwlExecutor(object):
"""Execute a CWL tool or workflow, submit work (using either jobs or
containers API), wait for them to complete, and report output.
@@ -195,6 +167,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
root_logger.addHandler(handler)
+ self.runtimeContext = ArvRuntimeContext(vars(arvargs))
+ self.runtimeContext.make_fs_access = partial(CollectionFsAccess,
+ collection_cache=self.collection_cache)
+
+
def arv_make_tool(self, toolpath_object, loadingContext):
if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
return ArvadosCommandTool(self, toolpath_object, loadingContext)
@@ -735,264 +712,3 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
self.trash_intermediate_output()
return (self.final_output, self.final_status)
-
-
-def versionstring():
- """Print version string of key packages for provenance and debugging."""
-
- arvcwlpkg = pkg_resources.require("arvados-cwl-runner")
- arvpkg = pkg_resources.require("arvados-python-client")
- cwlpkg = pkg_resources.require("cwltool")
-
- return "%s %s, %s %s, %s %s" % (sys.argv[0], arvcwlpkg[0].version,
- "arvados-python-client", arvpkg[0].version,
- "cwltool", cwlpkg[0].version)
-
-
-def arg_parser(): # type: () -> argparse.ArgumentParser
- parser = argparse.ArgumentParser(description='Arvados executor for Common Workflow Language')
-
- parser.add_argument("--basedir", type=str,
- help="Base directory used to resolve relative references in the input, default to directory of input object file or current directory (if inputs piped/provided on command line).")
- parser.add_argument("--outdir", type=str, default=os.path.abspath('.'),
- help="Output directory, default current directory")
-
- parser.add_argument("--eval-timeout",
- help="Time to wait for a Javascript expression to evaluate before giving an error, default 20s.",
- type=float,
- default=20)
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--print-dot", action="store_true",
- help="Print workflow visualization in graphviz format and exit")
- exgroup.add_argument("--version", action="version", help="Print version and exit", version=versionstring())
- exgroup.add_argument("--validate", action="store_true", help="Validate CWL document only.")
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--verbose", action="store_true", help="Default logging")
- exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
- exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
-
- parser.add_argument("--metrics", action="store_true", help="Print timing metrics")
-
- parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--enable-reuse", action="store_true",
- default=True, dest="enable_reuse",
- help="Enable job or container reuse (default)")
- exgroup.add_argument("--disable-reuse", action="store_false",
- default=True, dest="enable_reuse",
- help="Disable job or container reuse")
-
- parser.add_argument("--project-uuid", type=str, metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
- parser.add_argument("--output-name", type=str, help="Name to use for collection that stores the final output.", default=None)
- parser.add_argument("--output-tags", type=str, help="Tags for the final output collection separated by commas, e.g., '--output-tags tag0,tag1,tag2'.", default=None)
- parser.add_argument("--ignore-docker-for-reuse", action="store_true",
- help="Ignore Docker image version when deciding whether to reuse past jobs.",
- default=False)
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--submit", action="store_true", help="Submit workflow to run on Arvados.",
- default=True, dest="submit")
- exgroup.add_argument("--local", action="store_false", help="Run workflow on local host (submits jobs to Arvados).",
- default=True, dest="submit")
- exgroup.add_argument("--create-template", action="store_true", help="(Deprecated) synonym for --create-workflow.",
- dest="create_workflow")
- exgroup.add_argument("--create-workflow", action="store_true", help="Create an Arvados workflow (if using the 'containers' API) or pipeline template (if using the 'jobs' API). See --api.")
- exgroup.add_argument("--update-workflow", type=str, metavar="UUID", help="Update an existing Arvados workflow or pipeline template with the given UUID.")
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--wait", action="store_true", help="After submitting workflow runner job, wait for completion.",
- default=True, dest="wait")
- exgroup.add_argument("--no-wait", action="store_false", help="Submit workflow runner job and exit.",
- default=True, dest="wait")
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--log-timestamps", action="store_true", help="Prefix logging lines with timestamp",
- default=True, dest="log_timestamps")
- exgroup.add_argument("--no-log-timestamps", action="store_false", help="No timestamp on logging lines",
- default=True, dest="log_timestamps")
-
- parser.add_argument("--api", type=str,
- default=None, dest="work_api",
- choices=("jobs", "containers"),
- help="Select work submission API. Default is 'jobs' if that API is available, otherwise 'containers'.")
-
- parser.add_argument("--compute-checksum", action="store_true", default=False,
- help="Compute checksum of contents while collecting outputs",
- dest="compute_checksum")
-
- parser.add_argument("--submit-runner-ram", type=int,
- help="RAM (in MiB) required for the workflow runner job (default 1024)",
- default=None)
-
- parser.add_argument("--submit-runner-image", type=str,
- help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
- default=None)
-
- parser.add_argument("--submit-request-uuid", type=str,
- default=None,
- help="Update and commit supplied container request instead of creating a new one (containers API only).")
-
- parser.add_argument("--name", type=str,
- help="Name to use for workflow execution instance.",
- default=None)
-
- parser.add_argument("--on-error", type=str,
- help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. "
- "Default is 'continue'.", default="continue", choices=("stop", "continue"))
-
- parser.add_argument("--enable-dev", action="store_true",
- help="Enable loading and running development versions "
- "of CWL spec.", default=False)
- parser.add_argument('--storage-classes', default="default", type=str,
- help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
-
- parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
- help="If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).",
- default=0)
-
- parser.add_argument("--priority", type=int,
- help="Workflow priority (range 1..1000, higher has precedence over lower, containers api only)",
- default=DEFAULT_PRIORITY)
-
- parser.add_argument("--disable-validate", dest="do_validate",
- action="store_false", default=True,
- help=argparse.SUPPRESS)
-
- parser.add_argument("--disable-js-validation",
- action="store_true", default=False,
- help=argparse.SUPPRESS)
-
- parser.add_argument("--thread-count", type=int,
- default=4, help="Number of threads to use for job submit and output collection.")
-
- parser.add_argument("--http-timeout", type=int,
- default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
-
- exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--trash-intermediate", action="store_true",
- default=False, dest="trash_intermediate",
- help="Immediately trash intermediate outputs on workflow success.")
- exgroup.add_argument("--no-trash-intermediate", action="store_false",
- default=False, dest="trash_intermediate",
- help="Do not trash intermediate outputs (default).")
-
- parser.add_argument("workflow", type=str, default=None, help="The workflow to execute")
- parser.add_argument("job_order", nargs=argparse.REMAINDER, help="The input object to the workflow.")
-
- return parser
-
-def add_arv_hints():
- cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
- cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
- res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
- use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
- res.close()
- cwltool.process.supportedProcessRequirements.extend([
- "http://arvados.org/cwl#RunInSingleContainer",
- "http://arvados.org/cwl#OutputDirType",
- "http://arvados.org/cwl#RuntimeConstraints",
- "http://arvados.org/cwl#PartitionRequirement",
- "http://arvados.org/cwl#APIRequirement",
- "http://commonwl.org/cwltool#LoadListingRequirement",
- "http://arvados.org/cwl#IntermediateOutput",
- "http://arvados.org/cwl#ReuseRequirement",
- "http://arvados.org/cwl#ClusterTarget"
- ])
-
-def exit_signal_handler(sigcode, frame):
- logger.error("Caught signal {}, exiting.".format(sigcode))
- sys.exit(-sigcode)
-
-def main(args, stdout, stderr, api_client=None, keep_client=None,
- install_sig_handlers=True):
- parser = arg_parser()
-
- job_order_object = None
- arvargs = parser.parse_args(args)
-
- if len(arvargs.storage_classes.strip().split(',')) > 1:
- logger.error("Multiple storage classes are not supported currently.")
- return 1
-
- arvargs.use_container = True
- arvargs.relax_path_checks = True
- arvargs.print_supported_versions = False
-
- if install_sig_handlers:
- arv_cmd.install_signal_handlers()
-
- if arvargs.update_workflow:
- if arvargs.update_workflow.find('-7fd4e-') == 5:
- want_api = 'containers'
- elif arvargs.update_workflow.find('-p5p6p-') == 5:
- want_api = 'jobs'
- else:
- want_api = None
- if want_api and arvargs.work_api and want_api != arvargs.work_api:
- logger.error('--update-workflow arg {!r} uses {!r} API, but --api={!r} specified'.format(
- arvargs.update_workflow, want_api, arvargs.work_api))
- return 1
- arvargs.work_api = want_api
-
- if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order:
- job_order_object = ({}, "")
-
- add_arv_hints()
-
- try:
- if api_client is None:
- api_client = arvados.safeapi.ThreadSafeApiCache(
- api_params={"model": OrderedJsonModel(), "timeout": arvargs.http_timeout},
- keep_params={"num_retries": 4})
- keep_client = api_client.keep
- # Make an API object now so errors are reported early.
- api_client.users().current().execute()
- if keep_client is None:
- keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- runner = ArvCwlRunner(api_client, arvargs, keep_client=keep_client, num_retries=4)
- except Exception as e:
- logger.error(e)
- return 1
-
- if arvargs.debug:
- logger.setLevel(logging.DEBUG)
- logging.getLogger('arvados').setLevel(logging.DEBUG)
-
- if arvargs.quiet:
- logger.setLevel(logging.WARN)
- logging.getLogger('arvados').setLevel(logging.WARN)
- logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-
- if arvargs.metrics:
- metrics.setLevel(logging.DEBUG)
- logging.getLogger("cwltool.metrics").setLevel(logging.DEBUG)
-
- if arvargs.log_timestamps:
- arvados.log_handler.setFormatter(logging.Formatter(
- '%(asctime)s %(name)s %(levelname)s: %(message)s',
- '%Y-%m-%d %H:%M:%S'))
- else:
- arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
-
- for key, val in cwltool.argparser.get_default_args().items():
- if not hasattr(arvargs, key):
- setattr(arvargs, key, val)
-
- runtimeContext = ArvRuntimeContext(vars(arvargs))
- runtimeContext.make_fs_access = partial(CollectionFsAccess,
- collection_cache=runner.collection_cache)
- runtimeContext.http_timeout = arvargs.http_timeout
-
- return cwltool.main.main(args=arvargs,
- stdout=stdout,
- stderr=stderr,
- executor=runner.arv_executor,
- versionfunc=versionstring,
- job_order_object=job_order_object,
- logger_handler=arvados.log_handler,
- custom_schema_callback=add_arv_hints,
- loadingContext=runner.loadingContext,
- runtimeContext=runtimeContext)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 41166c512..3b40552ac 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -324,10 +324,10 @@ def arvados_jobs_image(arvrunner, img):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
- return img
+
def upload_workflow_collection(arvrunner, name, packed):
collection = arvados.collection.Collection(api_client=arvrunner.api,
commit edd1366a8d09f950b5c3e67e1c41fd78d3b5310d
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 10:36:47 2018 -0400
14198: Use PDH for container_image instead of docker repo+tag
Needed to support federated container requests.
This reverts 6ea807b2caf6c934f170b2e4d89c23c4a08ca69c
Based on the commit comment, that change was made to accomodate the
Docker v1 to v2 image format migration (to enable the API server to
select the image with the correct format). However, the API server
subsequently gained the ability to detect if it needed to substitute a
PDH with a migrated image PDH in commit
a72205728f94f5261b657766e01f5767dc15d4b5 so now we want restore the
original behavior of locally resolving the image PDH and using that in
the container request.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 7508febb0..6bca07c88 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -31,7 +31,7 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
global cached_lookups_lock
with cached_lookups_lock:
if dockerRequirement["dockerImageId"] in cached_lookups:
- return dockerRequirement["dockerImageId"]
+ return cached_lookups[dockerRequirement["dockerImageId"]]
with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
sp = dockerRequirement["dockerImageId"].split(":")
@@ -70,10 +70,12 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if not images:
raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
+ pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
+
with cached_lookups_lock:
- cached_lookups[dockerRequirement["dockerImageId"]] = True
+ cached_lookups[dockerRequirement["dockerImageId"]] = pdh
- return dockerRequirement["dockerImageId"]
+ return pdh
def arv_docker_clear_cache():
global cached_lookups
commit 622ecfe2ce9cc3965bdbba0a4dc167d7d484c48e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Oct 24 14:45:21 2018 -0400
14198: Initial support ClusterTarget hint
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 2e1ea50a3..0866f69d6 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -898,7 +898,8 @@ def add_arv_hints():
"http://arvados.org/cwl#APIRequirement",
"http://commonwl.org/cwltool#LoadListingRequirement",
"http://arvados.org/cwl#IntermediateOutput",
- "http://arvados.org/cwl#ReuseRequirement"
+ "http://arvados.org/cwl#ReuseRequirement",
+ "http://arvados.org/cwl#ClusterTarget"
])
def exit_signal_handler(sigcode, frame):
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 4f762192a..94eaf9560 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -232,4 +232,24 @@ $graph:
coresMin:
type: int?
doc: Minimum cores allocated to cwl-runner
- jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
\ No newline at end of file
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
+
+- name: ClusterTarget
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify where a workflow step should run
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:ClusterTarget'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ clusterID:
+ type: string?
+ doc: The cluster to run the container
+ ownerUUID:
+ type: string?
+ doc: The project that will own the container requests and intermediate collections
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b4d01019f..b46711af4 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -250,6 +250,15 @@ class ArvadosContainer(JobBase):
if self.timelimit is not None:
scheduling_parameters["max_run_time"] = self.timelimit
+ extra_submit_params = {}
+ cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
+ if cluster_target_req:
+ cluster_id = cluster_target_req.get("clusterID")
+ if cluster_id:
+ extra_submit_params["cluster_id"] = cluster_id
+ if cluster_target_req.get("ownerUUID"):
+ container_request["owner_uuid"] = cluster_target_req.get("ownerUUID")
+
container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
@@ -277,11 +286,13 @@ class ArvadosContainer(JobBase):
if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
uuid=runtimeContext.submit_request_uuid,
- body=container_request
+ body=container_request,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
else:
response = self.arvrunner.api.container_requests().create(
- body=container_request
+ body=container_request,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list