[ARVADOS] created: 1.2.0-383-g78831de03
Git user
git at public.curoverse.com
Tue Nov 13 14:13:06 EST 2018
at 78831de038f39da27828bdb44ca1519b246a7acb (commit)
commit 78831de038f39da27828bdb44ca1519b246a7acb
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:48:14 2018 -0500
14198: Add some basic doc comments.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 4b95d8c03..732661220 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -54,6 +54,9 @@ outputs:
steps:
base-case:
+ doc: |
+ Base case (no federation), single step workflow with both the
+ runner and step on the same cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -87,6 +90,9 @@ steps:
run: framework/testcase.cwl
runner-home-step-remote:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -121,6 +127,9 @@ steps:
run: framework/testcase.cwl
runner-remote-step-home:
+ doc: |
+ Single step workflow with the runner on the remote cluster and the
+ step on the home cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -155,6 +164,9 @@ steps:
run: framework/testcase.cwl
remote-case:
+ doc: |
+ Single step workflow with both the runner and the step on the
+ remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -189,6 +201,10 @@ steps:
run: framework/testcase.cwl
twostep-home-to-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the home cluster, the second step is on the remote
+ cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -227,6 +243,10 @@ steps:
run: framework/testcase.cwl
twostep-remote-to-home:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the remote cluster, the second step is on the home
+ cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
@@ -265,6 +285,9 @@ steps:
run: framework/testcase.cwl
twostep-both-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, both steps are
+ on the remote cluster.
in:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
commit 6a3559f11c28521083a7943be738188aa2e85666
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:33:05 2018 -0500
14198: Add remote-case and twostep-both-remote
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/remote-case.cwl b/sdk/cwl/tests/federation/cases/remote-case.cwl
new file mode 100644
index 000000000..02d1e953a
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/remote-case.cwl
@@ -0,0 +1,28 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:remote-case
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
new file mode 100644
index 000000000..c6cc811cb
--- /dev/null
+++ b/sdk/cwl/tests/federation/cases/twostep-both-remote.cwl
@@ -0,0 +1,35 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-both-remote
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ run: rev.cwl
diff --git a/sdk/cwl/tests/federation/data/remote-case-input.txt b/sdk/cwl/tests/federation/data/remote-case-input.txt
new file mode 100644
index 000000000..21e87fb1d
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/remote-case-input.txt
@@ -0,0 +1,16 @@
+Call me remote-case. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/twostep-both-remote.txt b/sdk/cwl/tests/federation/data/twostep-both-remote.txt
new file mode 100644
index 000000000..6218bb50e
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-both-remote.txt
@@ -0,0 +1,16 @@
+Call me twostep-both-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 602cc2fcb..4b95d8c03 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -39,13 +39,16 @@ outputs:
runner-remote-step-home-success:
type: Any
outputSource: runner-remote-step-home/success
+ remote-case-success:
+ type: Any
+ outputSource: remote-case/success
twostep-home-to-remote-success:
type: Any
outputSource: twostep-home-to-remote/success
twostep-remote-to-home-success:
type: Any
outputSource: twostep-remote-to-home/success
- twostep-both-remote:
+ twostep-both-remote-success:
type: Any
outputSource: twostep-both-remote/success
@@ -151,6 +154,40 @@ steps:
out: [out, success]
run: framework/testcase.cwl
+ remote-case:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/remote-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/remote-case-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
+ scrub_image: {default: "arvados/fed-test:remote-case"}
+ scrub_collections:
+ default:
+ - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
+ - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
+ - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
twostep-home-to-remote:
in:
arvados_api_token: arvados_api_token
@@ -264,5 +301,3 @@ steps:
- ddfa58a81953dad08436d571615dd584+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
-
- # also: twostep-all-remote
commit 546a7a97f23c2e589aca379e3a6839a4e8461696
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 15:02:50 2018 -0500
14198: Add more twostep tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
index ffce6c3c4..6e8521b2b 100644
--- a/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
@@ -33,6 +33,3 @@ steps:
runOnCluster: revCluster
out: [revhash]
run: rev.cwl
- requirements:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
diff --git a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
index 4541ebba2..7804b2024 100644
--- a/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
@@ -14,7 +14,8 @@ inputs:
inp:
type: File
inputBinding: {}
- runOnCluster: string
+ md5sumCluster: string
+ revCluster: string
outputs:
hash:
type: File
@@ -23,13 +24,12 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: md5sumCluster
out: [hash]
run: md5sum.cwl
- requirements:
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
rev:
in:
inp: md5sum/hash
+ runOnCluster: revCluster
out: [revhash]
run: rev.cwl
diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
index 52c780a87..6fe90813e 100644
--- a/sdk/cwl/tests/federation/framework/prepare.py
+++ b/sdk/cwl/tests/federation/framework/prepare.py
@@ -36,7 +36,6 @@ for cluster_id in config["arvados_cluster_ids"]:
for cluster_id in config["arvados_cluster_ids"]:
matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
- print("matches from %s was %s" % (cluster_id, matches))
for m in matches["items"]:
api.collections().delete(uuid=m["uuid"]).execute()
print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 346f97914..602cc2fcb 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -42,6 +42,12 @@ outputs:
twostep-home-to-remote-success:
type: Any
outputSource: twostep-home-to-remote/success
+ twostep-remote-to-home-success:
+ type: Any
+ outputSource: twostep-remote-to-home/success
+ twostep-both-remote:
+ type: Any
+ outputSource: twostep-both-remote/success
steps:
base-case:
@@ -183,4 +189,80 @@ steps:
out: [out, success]
run: framework/testcase.cwl
+ twostep-remote-to-home:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-remote-to-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-remote-to-home.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-remote-to-home"}
+ scrub_collections:
+ default:
+ - cce89b9f7b6e163978144051ce5f071a+74 # input collection
+ - 0c358c3af63644c6343766feff1b7238+51 # md5sum output collection
+ - 33fb7d512bf21f04847eca58cea46e74+51 # rev output collection
+ - 912e04aa3db04aba008cf5cd46c277b2+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-both-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-both-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-both-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-both-remote"}
+ scrub_collections:
+ default:
+ - 3c5e39939cf197d304ac1eac20841238+71 # input collection
+ - 3edb99aa607731593969cdab663d65b4+51 # md5sum output collection
+ - a91625b7139e60fe61a88cae42fbee13+51 # rev output collection
+ - ddfa58a81953dad08436d571615dd584+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
# also: twostep-all-remote
commit e013102e944467c460d8b1d7a32260f3c98fc3ba
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Nov 9 10:47:23 2018 -0500
14198: Working on two step test cases
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl
index 2e0ff64c6..57eff5975 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/base-case.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl
index af119990a..8a844e9dd 100644
--- a/sdk/cwl/tests/federation/cases/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/md5sum.cwl
@@ -8,9 +8,12 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
+ runOnCluster: string
outputs:
hash:
type: File
diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/rev.cwl
similarity index 73%
copy from sdk/cwl/tests/federation/cases/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/rev.cwl
index af119990a..13e7a87f2 100644
--- a/sdk/cwl/tests/federation/cases/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/rev.cwl
@@ -8,14 +8,16 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
+ runOnCluster: string
outputs:
- hash:
+ revhash:
type: File
outputBinding:
glob: out.txt
-stdin: $(inputs.inp.path)
stdout: out.txt
-arguments: ["md5sum", "-"]
+arguments: [rev, $(inputs.inp)]
diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 70d017639..0621dd736 100644
--- a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:runner-home-step-remote
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index 33681f18e..6ce3cce41 100644
--- a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -10,8 +10,6 @@ requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
dockerPull: arvados/fed-test:runner-remote-step-home
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
@@ -25,5 +23,6 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: runOnCluster
out: [hash]
run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
similarity index 54%
copy from sdk/cwl/tests/federation/cases/base-case.cwl
copy to sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
index 2e0ff64c6..ffce6c3c4 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-home-to-remote.cwl
@@ -9,14 +9,13 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
+ dockerPull: arvados/fed-test:twostep-home-to-remote
inputs:
inp:
type: File
inputBinding: {}
- runOnCluster: string
+ md5sumCluster: string
+ revCluster: string
outputs:
hash:
type: File
@@ -25,5 +24,15 @@ steps:
md5sum:
in:
inp: inp
+ runOnCluster: md5sumCluster
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ run: rev.cwl
+ requirements:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
similarity index 65%
copy from sdk/cwl/tests/federation/cases/base-case.cwl
copy to sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
index 2e0ff64c6..4541ebba2 100644
--- a/sdk/cwl/tests/federation/cases/base-case.cwl
+++ b/sdk/cwl/tests/federation/cases/twostep-remote-to-home.cwl
@@ -9,9 +9,7 @@ $namespaces:
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: arvados/fed-test:base-case
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
+ dockerPull: arvados/fed-test:twostep-remote-to-home
inputs:
inp:
type: File
@@ -26,4 +24,12 @@ steps:
in:
inp: inp
out: [hash]
- run: md5sum.cwl
\ No newline at end of file
+ run: md5sum.cwl
+ requirements:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ rev:
+ in:
+ inp: md5sum/hash
+ out: [revhash]
+ run: rev.cwl
diff --git a/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt b/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt
new file mode 100644
index 000000000..6430ad509
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-home-to-remote.txt
@@ -0,0 +1,16 @@
+Call me twostep-home-to-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt b/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt
new file mode 100644
index 000000000..231802581
--- /dev/null
+++ b/sdk/cwl/tests/federation/data/twostep-remote-to-home.txt
@@ -0,0 +1,16 @@
+Call me twostep-remote-to-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
index 94a6a75ed..52c780a87 100644
--- a/sdk/cwl/tests/federation/framework/prepare.py
+++ b/sdk/cwl/tests/federation/framework/prepare.py
@@ -33,10 +33,10 @@ for cluster_id in config["arvados_cluster_ids"]:
for lk in search_links["items"]:
api.links().delete(uuid=lk["uuid"]).execute()
-
for cluster_id in config["arvados_cluster_ids"]:
matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ print("matches from %s was %s" % (cluster_id, matches))
for m in matches["items"]:
api.collections().delete(uuid=m["uuid"]).execute()
print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 1042350f9..346f97914 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -39,6 +39,9 @@ outputs:
runner-remote-step-home-success:
type: Any
outputSource: runner-remote-step-home/success
+ twostep-home-to-remote-success:
+ type: Any
+ outputSource: twostep-home-to-remote/success
steps:
base-case:
@@ -141,3 +144,43 @@ steps:
- ecb639201f454b6493757f5117f540df+112 # runner output json
out: [out, success]
run: framework/testcase.cwl
+
+ twostep-home-to-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-home-to-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-home-to-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[0];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-home-to-remote"}
+ scrub_collections:
+ default:
+ - 268a54947fb75115cfe05bb54cc62c30+74 # input collection
+ - 400f03b8c5d2dc3dcb513a21b626ef88+51 # md5sum output collection
+ - 3738166916ca5f6f6ad12bf7e06b4a21+51 # rev output collection
+ - bc37c17a37aa25229e5de1339b27fbcc+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ # also: twostep-all-remote
commit cdcf4a2d5413ce5718389df3977f2be62902d071
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 8 22:19:15 2018 -0500
14198: Refactor arvbox federation setup
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
similarity index 94%
rename from sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
rename to sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 91e622022..9a08195a7 100644
--- a/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -44,7 +44,7 @@ steps:
containers: containers
arvbox_base: arvbox_base
out: [arvbox_data]
- run: arvbox-mkdir.cwl
+ run: arvbox/mkdir.cwl
start:
in:
container_name: containers
@@ -52,7 +52,7 @@ steps:
out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
- run: arvbox-start.cwl
+ run: arvbox/start.cwl
fed-config:
in:
container_name: containers
@@ -63,10 +63,10 @@ steps:
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
- run: arvbox-fed-config.cwl
+ run: arvbox/fed-config.cwl
setup-user:
in:
container_host: {source: start/container_host, valueFrom: "$(self[0])"}
superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
out: [test_user_uuid, test_user_token]
- run: arvbox-setup-user.cwl
+ run: arvbox/setup-user.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
deleted file mode 100644
index a6fd00823..000000000
--- a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-cwlVersion: v1.0
-class: Workflow
-$namespaces:
- arv: "http://arvados.org/cwl#"
- cwltool: "http://commonwl.org/cwltool#"
-requirements:
- cwltool:LoadListingRequirement:
- loadListing: no_listing
- SubworkflowFeatureRequirement: {}
-inputs:
- arvbox_base: Directory
- acr: string?
-outputs: []
-steps:
- run-arvbox:
- in:
- containers:
- default: [fedbox1, fedbox2, fedbox3]
- arvbox_base: arvbox_base
- out: [cluster_ids, container_hosts, test_user_uuid, test_user_token]
- run: arvbox-fed.cwl
- run-main:
- in:
- arvados_api_host_home: {source: run-arvbox/container_hosts, valueFrom: "$(self[0])"}
- arvados_home_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[0])"}
- arvados_api_token: run-arvbox/test_user_token
- arvado_api_host_insecure: {default: true}
- arvados_api_host_clusterB: {source: run-arvbox/container_hosts, valueFrom: "$(self[1])"}
- arvados_clusterB_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[1])"}
- arvados_api_host_clusterC: {source: run-arvbox/container_hosts, valueFrom: "$(self[2])"}
- arvados_clusterC_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[2])"}
- acr: acr
- out: [base-case-out, runner-home-step-remote-out]
- run: main.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
rename to sdk/cwl/tests/federation/arvbox/fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox/mkdir.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
rename to sdk/cwl/tests/federation/arvbox/mkdir.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox/setup-user.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
rename to sdk/cwl/tests/federation/arvbox/setup-user.cwl
diff --git a/sdk/cwl/tests/federation/framework/setup_user.py b/sdk/cwl/tests/federation/arvbox/setup_user.py
similarity index 100%
rename from sdk/cwl/tests/federation/framework/setup_user.py
rename to sdk/cwl/tests/federation/arvbox/setup_user.py
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox/start.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
rename to sdk/cwl/tests/federation/arvbox/start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox/stop.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
rename to sdk/cwl/tests/federation/arvbox/stop.cwl
commit c356ddf7c6652230874ac1058fa8b070297514e8
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 8 16:13:18 2018 -0500
14198: Refactored. Tests use isolated inputs. Return success boolean
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/.licenseignore b/.licenseignore
index 5d3eff344..113bf4fa4 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -46,7 +46,7 @@ docker/jobs/apt.arvados.org.list
*/script/rails
sdk/cwl/tests/input/blorp.txt
sdk/cwl/tests/tool/blub.txt
-sdk/cwl/tests/federation/whale.txt
+sdk/cwl/tests/federation/data/*
sdk/go/manifest/testdata/*_manifest
sdk/java/.classpath
sdk/java/pom.xml
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-fed-config.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-fed.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl
diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
new file mode 100644
index 000000000..a6fd00823
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl
@@ -0,0 +1,38 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvbox_base: Directory
+ acr: string?
+outputs: []
+steps:
+ run-arvbox:
+ in:
+ containers:
+ default: [fedbox1, fedbox2, fedbox3]
+ arvbox_base: arvbox_base
+ out: [cluster_ids, container_hosts, test_user_uuid, test_user_token]
+ run: arvbox-fed.cwl
+ run-main:
+ in:
+ arvados_api_host_home: {source: run-arvbox/container_hosts, valueFrom: "$(self[0])"}
+ arvados_home_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[0])"}
+ arvados_api_token: run-arvbox/test_user_token
+ arvado_api_host_insecure: {default: true}
+ arvados_api_host_clusterB: {source: run-arvbox/container_hosts, valueFrom: "$(self[1])"}
+ arvados_clusterB_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[1])"}
+ arvados_api_host_clusterC: {source: run-arvbox/container_hosts, valueFrom: "$(self[2])"}
+ arvados_clusterC_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[2])"}
+ acr: acr
+ out: [base-case-out, runner-home-step-remote-out]
+ run: main.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-mkdir.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-setup-user.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-start.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
similarity index 100%
rename from sdk/cwl/tests/federation/arvbox-stop.cwl
rename to sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl
similarity index 63%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/base-case.cwl
index 18f3fb0f9..2e0ff64c6 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/base-case.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:base-case
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl
similarity index 55%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/md5sum.cwl
index 18f3fb0f9..af119990a 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/md5sum.cwl
@@ -8,21 +8,14 @@ $namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
- DockerRequirement:
- dockerPull: debian:9
- arv:ClusterTarget:
- cluster_id: $(inputs.runOnCluster)
inputs:
inp:
type: File
- inputBinding: {}
- runOnCluster: string
outputs:
hash:
- type: string
+ type: File
outputBinding:
glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
+stdin: $(inputs.inp.path)
stdout: out.txt
-baseCommand: md5sum
+arguments: ["md5sum", "-"]
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
similarity index 63%
copy from sdk/cwl/tests/federation/md5sum.cwl
copy to sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
index 18f3fb0f9..70d017639 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:runner-home-step-remote
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
similarity index 63%
rename from sdk/cwl/tests/federation/md5sum.cwl
rename to sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
index 18f3fb0f9..33681f18e 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
cwlVersion: v1.0
-class: CommandLineTool
+class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:9
+ dockerPull: arvados/fed-test:runner-remote-step-home
arv:ClusterTarget:
cluster_id: $(inputs.runOnCluster)
inputs:
@@ -19,10 +19,11 @@ inputs:
runOnCluster: string
outputs:
hash:
- type: string
- outputBinding:
- glob: out.txt
- loadContents: true
- outputEval: $(self[0].contents.substr(0, 32))
-stdout: out.txt
-baseCommand: md5sum
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/base-case-input.txt
similarity index 93%
copy from sdk/cwl/tests/federation/whale.txt
copy to sdk/cwl/tests/federation/data/base-case-input.txt
index 425d1ed02..761b840cd 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/base-case-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me base-case. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
similarity index 92%
copy from sdk/cwl/tests/federation/whale.txt
copy to sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
index 425d1ed02..91ab77d12 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me runner-home-step-remote. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
similarity index 92%
rename from sdk/cwl/tests/federation/whale.txt
rename to sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
index 425d1ed02..e5673b848 100644
--- a/sdk/cwl/tests/federation/whale.txt
+++ b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt
@@ -1,4 +1,4 @@
-Call me Ishmael. Some years ago--never mind how long precisely--having
+Call me runner-remote-step-home. Some years ago--never mind how long precisely--having
little or no money in my purse, and nothing particular to interest me on
shore, I thought I would sail about a little and see the watery part of
the world. It is a way I have of driving off the spleen and regulating
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/framework/check-exist.cwl
similarity index 65%
copy from sdk/cwl/tests/federation/prepare.cwl
copy to sdk/cwl/tests/federation/framework/check-exist.cwl
index 6c82605bb..ebb0fb220 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/framework/check-exist.cwl
@@ -7,15 +7,11 @@ class: CommandLineTool
requirements:
InitialWorkDirRequirement:
listing:
- - entryname: input.json
- entry: $(JSON.stringify(inputs.obj))
- entryname: config.json
entry: |-
${
return JSON.stringify({
- arvados_cluster_ids: inputs.arvados_cluster_ids,
- scrub_images: inputs.scrub_images,
- scrub_collections: inputs.scrub_collections
+ check_collections: inputs.check_collections
});
}
EnvVarRequirement:
@@ -28,21 +24,19 @@ inputs:
arvados_api_token: string
arvado_api_host_insecure: boolean
arvados_api_host: string
- arvados_cluster_ids: string[]
- wf: File
- obj: Any
- scrub_images: string[]
- scrub_collections: string[]
+ check_collections: string[]
preparescript:
type: File
default:
class: File
- location: prepare.py
+ location: check_exist.py
inputBinding:
position: 1
outputs:
- done:
+ success:
type: boolean
outputBinding:
- outputEval: $(true)
+ glob: success
+ loadContents: true
+ outputEval: $(self[0].contents=="true")
baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/framework/check_exist.py b/sdk/cwl/tests/federation/framework/check_exist.py
new file mode 100644
index 000000000..b3338939e
--- /dev/null
+++ b/sdk/cwl/tests/federation/framework/check_exist.py
@@ -0,0 +1,25 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import json
+
+api = arvados.api()
+
+with open("config.json") as f:
+ config = json.load(f)
+
+success = True
+for c in config["check_collections"]:
+ try:
+ api.collections().get(uuid=c).execute()
+ except Exception as e:
+ print("Checking for %s got exception %s" % (c, e))
+ success = False
+
+with open("success", "w") as f:
+ if success:
+ f.write("true")
+ else:
+ f.write("false")
diff --git a/sdk/cwl/tests/federation/framework/dockerbuild.cwl b/sdk/cwl/tests/federation/framework/dockerbuild.cwl
new file mode 100644
index 000000000..d00b3e2a5
--- /dev/null
+++ b/sdk/cwl/tests/federation/framework/dockerbuild.cwl
@@ -0,0 +1,21 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ testcase: string
+outputs:
+ imagename:
+ type: string
+ outputBinding:
+ outputEval: $(inputs.testcase)
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: Dockerfile
+ entry: |-
+ FROM debian at sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b
+ LABEL org.arvados.testcase="$(inputs.testcase)"
+arguments: [docker, build, -t, $(inputs.testcase), "."]
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/framework/prepare.cwl
similarity index 92%
rename from sdk/cwl/tests/federation/prepare.cwl
rename to sdk/cwl/tests/federation/framework/prepare.cwl
index 6c82605bb..03f792c5e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/framework/prepare.cwl
@@ -14,7 +14,7 @@ requirements:
${
return JSON.stringify({
arvados_cluster_ids: inputs.arvados_cluster_ids,
- scrub_images: inputs.scrub_images,
+ scrub_images: [inputs.scrub_image],
scrub_collections: inputs.scrub_collections
});
}
@@ -31,7 +31,7 @@ inputs:
arvados_cluster_ids: string[]
wf: File
obj: Any
- scrub_images: string[]
+ scrub_image: string
scrub_collections: string[]
preparescript:
type: File
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
similarity index 100%
rename from sdk/cwl/tests/federation/prepare.py
rename to sdk/cwl/tests/federation/framework/prepare.py
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/framework/run-acr.cwl
similarity index 91%
rename from sdk/cwl/tests/federation/run-acr.cwl
rename to sdk/cwl/tests/federation/framework/run-acr.cwl
index 081186506..b720eaa12 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/framework/run-acr.cwl
@@ -42,7 +42,8 @@ outputs:
outputBinding:
glob: output.json
loadContents: true
- outputEval: $(JSON.parse(self[0].contents))
+ #outputEval: $(JSON.parse(self[0].contents))
+ outputEval: $(self[0].contents)
stdout: output.json
arguments:
- valueFrom: --disable-reuse
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/framework/setup_user.py
similarity index 100%
rename from sdk/cwl/tests/federation/setup_user.py
rename to sdk/cwl/tests/federation/framework/setup_user.py
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/framework/testcase.cwl
similarity index 71%
rename from sdk/cwl/tests/federation/testcase.cwl
rename to sdk/cwl/tests/federation/framework/testcase.cwl
index b5ef43091..89aa3f98b 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/framework/testcase.cwl
@@ -25,14 +25,22 @@ inputs:
acr: string?
wf: File
obj: Any
- scrub_images: string[]
+ scrub_image: string
scrub_collections: string[]
runner_cluster: string?
outputs:
out:
type: Any
outputSource: run-acr/out
+ success:
+ type: boolean
+ outputSource: check-result/success
steps:
+ dockerbuild:
+ in:
+ testcase: scrub_image
+ out: [imagename]
+ run: dockerbuild.cwl
prepare:
in:
arvados_api_token: arvados_api_token
@@ -41,13 +49,14 @@ steps:
arvados_cluster_ids: arvados_cluster_ids
wf: wf
obj: obj
- scrub_images: scrub_images
+ scrub_image: scrub_image
scrub_collections: scrub_collections
out: [done]
run: prepare.cwl
run-acr:
in:
prepare: prepare/done
+ image-ready: dockerbuild/imagename
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
@@ -57,3 +66,12 @@ steps:
obj: obj
out: [out]
run: run-acr.cwl
+ check-result:
+ in:
+ acr-done: run-acr/out
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ check_collections: scrub_collections
+ out: [success]
+ run: check-exist.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 4029d53d5..1042350f9 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -23,14 +23,22 @@ inputs:
arvados_api_hosts: string[]
arvados_cluster_ids: string[]
acr: string?
-
+ testcases:
+ type: string[]
+ default:
+ - base-case
+ - runner-home-step-remote
+ - runner-remote-step-home
outputs:
- base-case-out:
+ base-case-success:
+ type: Any
+ outputSource: base-case/success
+ runner-home-step-remote-success:
type: Any
- outputSource: base-case/out
- runner-home-step-remote-out:
+ outputSource: runner-home-step-remote/success
+ runner-remote-step-home-success:
type: Any
- outputSource: runner-home-step-remote/out
+ outputSource: runner-remote-step-home/success
steps:
base-case:
@@ -40,21 +48,31 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/base-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/base-case-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:base-case"}
+ scrub_collections:
+ default:
+ - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
+ - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
+ - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
runner-home-step-remote:
in:
@@ -63,22 +81,32 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/runner-home-step-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/runner-home-step-remote-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:runner-home-step-remote"}
+ scrub_collections:
+ default:
+ - 3bc373e38751fe13dcbd62778d583242+81 # input collection
+ - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
+ - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
runner-remote-step-home:
in:
@@ -87,19 +115,29 @@ steps:
arvados_api_hosts: arvados_api_hosts
arvados_cluster_ids: arvados_cluster_ids
acr: acr
- wf: {default: {class: File, location: md5sum.cwl}}
+ wf:
+ default:
+ class: File
+ location: cases/runner-remote-step-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
obj:
default:
inp:
class: File
- location: whale.txt
+ location: data/runner-remote-step-home-input.txt
valueFrom: |-
${
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
- scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
- out: [out]
- run: testcase.cwl
+ scrub_image: {default: "arvados/fed-test:runner-remote-step-home"}
+ scrub_collections:
+ default:
+ - 25fe10d8e8530329a738de69d9bc8ab5+81 # input collection
+ - 7f052d1a04b851b6f73fba77c7802e1d+51 # md5sum output collection
+ - ecb639201f454b6493757f5117f540df+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
commit e2c5d210ac79275417e5dba8048629eaecc66c0b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 16:40:27 2018 -0500
14198: Add runner-remote-step-home
Add license headers
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/.licenseignore b/.licenseignore
index 51a1e7cbd..5d3eff344 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -46,6 +46,7 @@ docker/jobs/apt.arvados.org.list
*/script/rails
sdk/cwl/tests/input/blorp.txt
sdk/cwl/tests/tool/blub.txt
+sdk/cwl/tests/federation/whale.txt
sdk/go/manifest/testdata/*_manifest
sdk/java/.classpath
sdk/java/pom.xml
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 721e6b775..e53793900 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
Things to test.
Single step --submit --no-wait workflow, matrix of:
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 173b9df72..77567ee89 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 8d7075a1d..91e622022 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
index b047beb9b..727d491a3 100644
--- a/sdk/cwl/tests/federation/arvbox-mkdir.cwl
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
index 684bc8c8d..0fddc1b87 100644
--- a/sdk/cwl/tests/federation/arvbox-setup-user.cwl
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index ae591d58f..f69775a53 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
index c4a5412de..2ea4c0f74 100644
--- a/sdk/cwl/tests/federation/arvbox-stop.cwl
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index d481f5c19..4029d53d5 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -1,4 +1,8 @@
#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
@@ -70,6 +74,31 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
+ out: [out]
+ run: testcase.cwl
+
+ runner-remote-step-home:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
scrub_images: {default: ["debian:9"]}
scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
index 55e1fbf8a..18f3fb0f9 100644
--- a/sdk/cwl/tests/federation/md5sum.cwl
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index 2463a466e..6c82605bb 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
requirements:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index a0ac95588..94a6a75ed 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import json
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index 887d113b9..081186506 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
@@ -11,7 +15,7 @@ inputs:
arvado_api_host_insecure:
type: boolean
default: false
- runner_remote_host:
+ runner_cluster:
type: string?
inputBinding:
prefix: --submit-runner-cluster
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2fc621521..a456976be 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import arvados.errors
import time
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index 66c510071..b5ef43091 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -1,4 +1,8 @@
#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
@@ -23,6 +27,7 @@ inputs:
obj: Any
scrub_images: string[]
scrub_collections: string[]
+ runner_cluster: string?
outputs:
out:
type: Any
@@ -46,6 +51,7 @@ steps:
arvados_api_token: arvados_api_token
arvado_api_host_insecure: arvado_api_host_insecure
arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ runner_cluster: runner_cluster
acr: acr
wf: wf
obj: obj
commit 9279477e3d80997daf225e033faa544dd231a356
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 16:23:01 2018 -0500
14198: Use --always-submit-runner
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index f1d101260..d481f5c19 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -48,7 +48,7 @@ steps:
return self;
}
scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
run: testcase.cwl
@@ -71,6 +71,6 @@ steps:
return self;
}
scrub_images: {default: ["debian:9"]}
- scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
out: [out]
run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index ed9599194..887d113b9 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -43,5 +43,7 @@ stdout: output.json
arguments:
- valueFrom: --disable-reuse
position: 2
+ - valueFrom: --always-submit-runner
+ position: 2
- valueFrom: input.json
position: 4
\ No newline at end of file
commit 1d72731a1f22380cd034da8bb0c79d5b48a7c3c4
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 15:13:28 2018 -0500
14198: Scrubbing works
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index bdf16482e..a0ac95588 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -6,16 +6,33 @@ api = arvados.api()
with open("config.json") as f:
config = json.load(f)
+scrub_collections = set(config["scrub_collections"])
+
for cluster_id in config["arvados_cluster_ids"]:
+ images = []
for scrub_image in config["scrub_images"]:
sp = scrub_image.split(":")
image_name = sp[0]
image_tag = sp[1] if len(sp) > 1 else "latest"
+ images.append('{}:{}'.format(image_name, image_tag))
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', 'in', images]],
+ cluster_id=cluster_id).execute()
+
+ head_uuids = [lk["head_uuid"] for lk in search_links["items"]]
+ cols = api.collections().list(filters=[["uuid", "in", head_uuids]],
+ cluster_id=cluster_id).execute()
+ for c in cols["items"]:
+ scrub_collections.add(c["portable_data_hash"])
+ for lk in search_links["items"]:
+ api.links().delete(uuid=lk["uuid"]).execute()
- search_links = api.links().list(
- filters=[['link_class', '=', 'docker_image_repo+tag'],
- ['name', '=',
- '{}:{}'.format(image_name, image_tag)]],
- cluster_id=cluster_id).execute()
- for s in search_links["items"]:
- print s
+
+for cluster_id in config["arvados_cluster_ids"]:
+ matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
+ select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ for m in matches["items"]:
+ api.collections().delete(uuid=m["uuid"]).execute()
+ print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
commit 1a21d0f19e125a85693ebf08f345205e6e12dd14
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:37:08 2018 -0500
14198: More work on scrub
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index e7101dc9e..f1d101260 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -47,6 +47,8 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
@@ -68,5 +70,7 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index e12aae464..2463a466e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -5,8 +5,15 @@ requirements:
listing:
- entryname: input.json
entry: $(JSON.stringify(inputs.obj))
- - entryname: clusters.json
- entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ - entryname: config.json
+ entry: |-
+ ${
+ return JSON.stringify({
+ arvados_cluster_ids: inputs.arvados_cluster_ids,
+ scrub_images: inputs.scrub_images,
+ scrub_collections: inputs.scrub_collections
+ });
+ }
EnvVarRequirement:
envDef:
ARVADOS_API_HOST: $(inputs.arvados_api_host)
@@ -20,6 +27,8 @@ inputs:
arvados_cluster_ids: string[]
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
preparescript:
type: File
default:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index 48055224f..bdf16482e 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,5 +1,21 @@
import arvados
+import json
api = arvados.api()
-print(api.users().current().execute())
+with open("config.json") as f:
+ config = json.load(f)
+
+for cluster_id in config["arvados_cluster_ids"]:
+ for scrub_image in config["scrub_images"]:
+ sp = scrub_image.split(":")
+ image_name = sp[0]
+ image_tag = sp[1] if len(sp) > 1 else "latest"
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', '=',
+ '{}:{}'.format(image_name, image_tag)]],
+ cluster_id=cluster_id).execute()
+ for s in search_links["items"]:
+ print s
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index a22d57374..66c510071 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -21,6 +21,8 @@ inputs:
acr: string?
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
outputs:
out:
type: Any
@@ -34,6 +36,8 @@ steps:
arvados_cluster_ids: arvados_cluster_ids
wf: wf
obj: obj
+ scrub_images: scrub_images
+ scrub_collections: scrub_collections
out: [done]
run: prepare.cwl
run-acr:
commit 1959956cecaad080746426e08d7349a9199e24da
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:01:30 2018 -0500
14198: Working on prepare step.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
new file mode 100644
index 000000000..e12aae464
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -0,0 +1,35 @@
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ - entryname: clusters.json
+ entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure: boolean
+ arvados_api_host: string
+ arvados_cluster_ids: string[]
+ wf: File
+ obj: Any
+ preparescript:
+ type: File
+ default:
+ class: File
+ location: prepare.py
+ inputBinding:
+ position: 1
+outputs:
+ done:
+ type: boolean
+ outputBinding:
+ outputEval: $(true)
+baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
new file mode 100644
index 000000000..48055224f
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -0,0 +1,5 @@
+import arvados
+
+api = arvados.api()
+
+print(api.users().current().execute())
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index a178a2c88..ed9599194 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -6,9 +6,9 @@ inputs:
default: arvados-cwl-runner
inputBinding:
position: 1
- arv_host: string
- arv_token: string
- arv_insecure:
+ arvados_api_host: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
type: boolean
default: false
runner_remote_host:
@@ -28,9 +28,9 @@ requirements:
entry: $(JSON.stringify(inputs.obj))
EnvVarRequirement:
envDef:
- ARVADOS_API_HOST: $(inputs.arv_host)
- ARVADOS_API_TOKEN: $(inputs.arv_token)
- ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
InlineJavascriptRequirement: {}
outputs:
out:
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2b0e56c60..2fc621521 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -18,16 +18,19 @@ else:
u = api.users().create(body={
'first_name': 'Test',
'last_name': 'User',
- 'email': 'test at example.com'
+ 'email': 'test at example.com',
+ 'is_admin': False
}).execute()
api.users().activate(uuid=u["uuid"]).execute()
tok = api.api_client_authorizations().create(body={
- "owner_uuid": u["uuid"]
+ "api_client_authorization": {
+ "owner_uuid": u["uuid"]
+ }
}).execute()
with open("cwl.output.json", "w") as f:
json.dump({
"test_user_uuid": u["uuid"],
- "test_user_token": tok["api_token"]
+ "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"])
}, f)
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index f4b8f7320..a22d57374 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -26,11 +26,22 @@ outputs:
type: Any
outputSource: run-acr/out
steps:
+ prepare:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ arvados_cluster_ids: arvados_cluster_ids
+ wf: wf
+ obj: obj
+ out: [done]
+ run: prepare.cwl
run-acr:
in:
- arv_token: arvados_api_token
- arv_insecure: arvado_api_host_insecure
- arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ prepare: prepare/done
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
acr: acr
wf: wf
obj: obj
commit 2444242969b0b421d5f6b1cd7214cadb69ce6e9e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 13:17:11 2018 -0500
14198: Initial test cases.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 4da2bac0b..721e6b775 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -19,4 +19,33 @@ Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
Workflow which has a remote collection in InitialWorkDir, which is captured in output.
Workflow with file input that has a secondary file in a separate
-collection, which is remote.
\ No newline at end of file
+collection, which is remote.
+
+
+Need to pull Docker image
+Can't run in container because it doesn't have access to Docker from inside -> how can we workaround?
+Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this)
+Try setting up a virtualenv -> annoying because you need C dependencies to build
+Virtualenv sets up shop in /tmp instead of the real directory
+--relocatable ???
+
+
+arvados/jobs doesn't have the docker client
+arvbox has docker client
+
+- Still need venv or something to run cwltool to manage all this.
+- Unless we run cwltool in docker as well.
+
+Can use cwl-docker.sh to run cwltool in docker
+
+- arvbox needs Docker. This will *probably* work launched from container if base dir is under PWD
+- want to run arvados-cwl-runner. this needs Docker to pull images.
+
+For a-c-r solutions are one of:
+
+1) Don't run in Docker, use virtualenv instead.
+2) Access to Docker from inside Docker to pull images.
+2a) Docker-in-Docker
+2b) Bind mount Docker socket
+2c) Communicate to Docker daemon over TCP
+3) Arvados feature to pull images
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 2ff6f79b1..8d7075a1d 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -8,22 +8,32 @@ requirements:
StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
+ InlineJavascriptRequirement: {}
inputs:
- containers: string[]
+ containers:
+ type: string[]
+ default: [fedbox1, fedbox2, fedbox3]
arvbox_base: Directory
+ in_acr: string?
+ insecure:
+ type: boolean
+ default: true
outputs:
- cluster_ids:
- type: string[]
- outputSource: start/cluster_id
- container_hosts:
- type: string[]
- outputSource: start/container_host
- test_user_uuid:
- type: string
- outputSource: setup-user/test_user_uuid
- test_user_token:
+ arvados_api_token:
type: string
outputSource: setup-user/test_user_token
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/container_host
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ acr:
+ type: string?
+ outputSource: in_acr
+ arvado_api_host_insecure:
+ type: boolean
+ outputSource: insecure
steps:
mkdir:
in:
@@ -55,4 +65,4 @@ steps:
container_host: {source: start/container_host, valueFrom: "$(self[0])"}
superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
out: [test_user_uuid, test_user_token]
- run: arvbox-setup-user.cwl
\ No newline at end of file
+ run: arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 2392b1149..e7101dc9e 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -7,16 +7,66 @@ $namespaces:
hints:
cwltool:Secrets:
secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
inputs:
- arvados_api_host_home: string
- arvados_home_id: string
arvados_api_token: string
arvado_api_host_insecure:
- type: bool
+ type: boolean
default: false
- arvados_api_host_clusterB: string
- arvados_clusterB_id: string
- arvados_api_host_clusterC: string
- arvados_clusterC_id: string
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
-outputs: []
+outputs:
+ base-case-out:
+ type: Any
+ outputSource: base-case/out
+ runner-home-step-remote-out:
+ type: Any
+ outputSource: runner-home-step-remote/out
+
+steps:
+ base-case:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
+
+ runner-home-step-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
new file mode 100644
index 000000000..55e1fbf8a
--- /dev/null
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -0,0 +1,24 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:9
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: string
+ outputBinding:
+ glob: out.txt
+ loadContents: true
+ outputEval: $(self[0].contents.substr(0, 32))
+stdout: out.txt
+baseCommand: md5sum
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
new file mode 100644
index 000000000..a178a2c88
--- /dev/null
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -0,0 +1,47 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ acr:
+ type: string?
+ default: arvados-cwl-runner
+ inputBinding:
+ position: 1
+ arv_host: string
+ arv_token: string
+ arv_insecure:
+ type: boolean
+ default: false
+ runner_remote_host:
+ type: string?
+ inputBinding:
+ prefix: --submit-runner-cluster
+ position: 2
+ wf:
+ type: File
+ inputBinding:
+ position: 3
+ obj: Any
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arv_host)
+ ARVADOS_API_TOKEN: $(inputs.arv_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ InlineJavascriptRequirement: {}
+outputs:
+ out:
+ type: Any
+ outputBinding:
+ glob: output.json
+ loadContents: true
+ outputEval: $(JSON.parse(self[0].contents))
+stdout: output.json
+arguments:
+ - valueFrom: --disable-reuse
+ position: 2
+ - valueFrom: input.json
+ position: 4
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
new file mode 100644
index 000000000..f4b8f7320
--- /dev/null
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -0,0 +1,38 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
+ wf: File
+ obj: Any
+outputs:
+ out:
+ type: Any
+ outputSource: run-acr/out
+steps:
+ run-acr:
+ in:
+ arv_token: arvados_api_token
+ arv_insecure: arvado_api_host_insecure
+ arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ acr: acr
+ wf: wf
+ obj: obj
+ out: [out]
+ run: run-acr.cwl
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/whale.txt
new file mode 100644
index 000000000..425d1ed02
--- /dev/null
+++ b/sdk/cwl/tests/federation/whale.txt
@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
commit ce919ec5805a0216b7c8d36da03def0ca66c0718
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 14:57:14 2018 -0500
14198: Initialize test user
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index a595d67cf..173b9df72 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -9,7 +9,11 @@ inputs:
cluster_ids: string[]
cluster_hosts: string[]
arvbox_data: Directory
-outputs: []
+outputs:
+ arvbox_data_out:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
@@ -23,7 +27,7 @@ requirements:
var remoteClusters = {};
for (var i = 0; i < inputs.cluster_ids.length; i++) {
remoteClusters[inputs.cluster_ids[i]] = {
- "Host": inputs.cluster_hosts[i]+":8000",
+ "Host": inputs.cluster_hosts[i],
"Proxy": true,
"Insecure": true
};
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 4b52223e0..2ff6f79b1 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -5,6 +5,7 @@ $namespaces:
cwltool: "http://commonwl.org/cwltool#"
requirements:
ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
inputs:
@@ -14,9 +15,15 @@ outputs:
cluster_ids:
type: string[]
outputSource: start/cluster_id
- container_ips:
+ container_hosts:
type: string[]
- outputSource: start/container_ip
+ outputSource: start/container_host
+ test_user_uuid:
+ type: string
+ outputSource: setup-user/test_user_uuid
+ test_user_token:
+ type: string
+ outputSource: setup-user/test_user_token
steps:
mkdir:
in:
@@ -28,7 +35,7 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip, arvbox_data]
+ out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
@@ -37,9 +44,15 @@ steps:
container_name: containers
this_cluster_id: start/cluster_id
cluster_ids: start/cluster_id
- cluster_hosts: start/container_ip
- arvbox_data: start/arvbox_data
+ cluster_hosts: start/container_host
+ arvbox_data: start/arvbox_data_out
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
run: arvbox-fed-config.cwl
+ setup-user:
+ in:
+ container_host: {source: start/container_host, valueFrom: "$(self[0])"}
+ superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
+ out: [test_user_uuid, test_user_token]
+ run: arvbox-setup-user.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
new file mode 100644
index 000000000..684bc8c8d
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.container_host)
+ ARVADOS_API_TOKEN: $(inputs.superuser_token)
+ ARVADOS_API_HOST_INSECURE: "true"
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ DockerRequirement:
+ dockerPull: arvados/jobs
+inputs:
+ container_host: string
+ superuser_token: string
+ make_user_script:
+ type: File
+ default:
+ class: File
+ location: setup_user.py
+outputs:
+ test_user_uuid: string
+ test_user_token: string
+arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 3485fba68..ae591d58f 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -21,7 +21,7 @@ outputs:
}
}
}
- container_ip:
+ container_host:
type: string
outputBinding:
glob: status.txt
@@ -31,11 +31,17 @@ outputs:
var sp = self[0].contents.split("\n");
for (var i = 0; i < sp.length; i++) {
if (sp[i].startsWith("Container IP: ")) {
- return sp[i].substr(14);
+ return sp[i].substr(14)+":8000";
}
}
}
- arvbox_data:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+ arvbox_data_out:
type: Directory
outputBinding:
outputEval: $(inputs.arvbox_data)
@@ -56,4 +62,7 @@ requirements:
arguments:
- shellQuote: false
valueFrom: |
- arvbox start dev && arvbox status > status.txt
+ set -e
+ arvbox start dev
+ arvbox status > status.txt
+ arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
new file mode 100644
index 000000000..2b0e56c60
--- /dev/null
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -0,0 +1,33 @@
+import arvados
+import arvados.errors
+import time
+import json
+
+while True:
+ try:
+ api = arvados.api()
+ break
+ except arvados.errors.ApiError:
+ time.sleep(2)
+
+existing = api.users().list(filters=[["email", "=", "test at example.com"],
+ ["is_active", "=", True]], limit=1).execute()
+if existing["items"]:
+ u = existing["items"][0]
+else:
+ u = api.users().create(body={
+ 'first_name': 'Test',
+ 'last_name': 'User',
+ 'email': 'test at example.com'
+ }).execute()
+ api.users().activate(uuid=u["uuid"]).execute()
+
+tok = api.api_client_authorizations().create(body={
+ "owner_uuid": u["uuid"]
+}).execute()
+
+with open("cwl.output.json", "w") as f:
+ json.dump({
+ "test_user_uuid": u["uuid"],
+ "test_user_token": tok["api_token"]
+ }, f)
commit 7e39196ac5be04479ed1a75275068357608598ef
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 13:59:20 2018 -0500
14198: Start and configure a federation of N arvboxes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 157fe4f72..a595d67cf 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,36 +1,52 @@
cwlVersion: v1.0
class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
inputs:
container_name: string
- this_cluster: string
+ this_cluster_id: string
cluster_ids: string[]
cluster_hosts: string[]
- arvbox_base: Directory
+ arvbox_data: Directory
outputs: []
requirements:
EnvVarRequirement:
envDef:
ARVBOX_CONTAINER: $(inputs.container_name)
- ARVBOX_BASE: $(inputs.arvbox_base.path)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
InitialWorkDirRequirement:
listing:
- cluster_config.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
- }
- application.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
- }
+ - entryname: cluster_config.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = {
+ "Host": inputs.cluster_hosts[i]+":8000",
+ "Proxy": true,
+ "Insecure": true
+ };
+ }
+ var r = {"Clusters": {}};
+ r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+ return JSON.stringify(r);
+ }
+ - entryname: application.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
ShellCommandRequirement: {}
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
arguments:
- shellQuote: false
valueFrom: |
@@ -39,4 +55,4 @@ arguments:
arvbox sv restart api
arvbox sv restart controller
arvbox sv restart keepstore0
- arvbox sv restart keepstore1
\ No newline at end of file
+ arvbox sv restart keepstore1
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 91bd22287..4b52223e0 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -28,7 +28,18 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip]
+ out: [cluster_id, container_ip, arvbox_data]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
+ fed-config:
+ in:
+ container_name: containers
+ this_cluster_id: start/cluster_id
+ cluster_ids: start/cluster_id
+ cluster_hosts: start/container_ip
+ arvbox_data: start/arvbox_data
+ out: []
+ scatter: [container_name, this_cluster_id, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 2e12597ce..3485fba68 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -35,6 +35,10 @@ outputs:
}
}
}
+ arvbox_data:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
commit cd25215ae9d727511bd7b680883975433c514fc9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 17:46:41 2018 -0500
14198: Federation integration testing WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
new file mode 100644
index 000000000..4da2bac0b
--- /dev/null
+++ b/sdk/cwl/tests/federation/README
@@ -0,0 +1,22 @@
+Things to test.
+
+Single step --submit --no-wait workflow, matrix of:
+
+Runner on home/remote cluster
+Docker image on home/remote cluster
+Step on home/remote cluster
+
+Two step workflow, matrix of:
+
+Step1 on home cluster -> Step2 on remote cluster
+Step1 on remote cluster -> Step2 on home cluster
+
+Three step workflow:
+
+(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster
+Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
+
+Workflow which has a remote collection in InitialWorkDir, which is captured in output.
+
+Workflow with file input that has a secondary file in a separate
+collection, which is remote.
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
new file mode 100644
index 000000000..157fe4f72
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -0,0 +1,42 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ container_name: string
+ this_cluster: string
+ cluster_ids: string[]
+ cluster_hosts: string[]
+ arvbox_base: Directory
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_BASE: $(inputs.arvbox_base.path)
+ InitialWorkDirRequirement:
+ listing:
+ cluster_config.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
+ }
+ application.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ ShellCommandRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
+ arvbox sv restart api
+ arvbox sv restart controller
+ arvbox sv restart keepstore0
+ arvbox sv restart keepstore1
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
new file mode 100644
index 000000000..91bd22287
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -0,0 +1,34 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ ScatterFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ containers: string[]
+ arvbox_base: Directory
+outputs:
+ cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ container_ips:
+ type: string[]
+ outputSource: start/container_ip
+steps:
+ mkdir:
+ in:
+ containers: containers
+ arvbox_base: arvbox_base
+ out: [arvbox_data]
+ run: arvbox-mkdir.cwl
+ start:
+ in:
+ container_name: containers
+ arvbox_data: mkdir/arvbox_data
+ out: [cluster_id, container_ip]
+ scatter: [container_name, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
new file mode 100644
index 000000000..b047beb9b
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -0,0 +1,43 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ containers:
+ type:
+ type: array
+ items: string
+ inputBinding:
+ position: 3
+ valueFrom: |
+ ${
+ return "base/"+self;
+ }
+ arvbox_base: Directory
+outputs:
+ arvbox_data:
+ type: Directory[]
+ outputBinding:
+ glob: |
+ ${
+ var r = [];
+ for (var i = 0; i < inputs.containers.length; i++) {
+ r.push("base/"+inputs.containers[i]);
+ }
+ return r;
+ }
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_base)
+ entryname: base
+ writable: true
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+arguments:
+ - mkdir
+ - "-p"
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
new file mode 100644
index 000000000..2e12597ce
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -0,0 +1,55 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+ arvbox_data: Directory
+outputs:
+ cluster_id:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Cluster id: ")) {
+ return sp[i].substr(12);
+ }
+ }
+ }
+ container_ip:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Container IP: ")) {
+ return sp[i].substr(14);
+ }
+ }
+ }
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
+ ShellCommandRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_data)
+ entryname: $(inputs.container_name)
+ writable: true
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ InlineJavascriptRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ arvbox start dev && arvbox status > status.txt
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
new file mode 100644
index 000000000..c4a5412de
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+arguments: [arvbox, stop]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
new file mode 100755
index 000000000..2392b1149
--- /dev/null
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -0,0 +1,22 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+inputs:
+ arvados_api_host_home: string
+ arvados_home_id: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: bool
+ default: false
+ arvados_api_host_clusterB: string
+ arvados_clusterB_id: string
+ arvados_api_host_clusterC: string
+ arvados_clusterC_id: string
+
+outputs: []
commit 8fb2fa4a432401a97dd48c72505b3944bf92a449
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Oct 25 15:03:23 2018 -0400
14198: Add genericFederatedRequestHandler support for links
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index 350c2a034..09b3bcd42 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -27,7 +27,6 @@ var containersRe = regexp.MustCompile(fmt.Sprintf(pathPattern, "containers", "dz
var containerRequestsRe = regexp.MustCompile(fmt.Sprintf(pathPattern, "container_requests", "xvhdp"))
var collectionsRe = regexp.MustCompile(fmt.Sprintf(pathPattern, "collections", "4zz18"))
var collectionsByPDHRe = regexp.MustCompile(`^/arvados/v1/collections/([0-9a-fA-F]{32}\+[0-9]+)+$`)
-
var linksRe = regexp.MustCompile(fmt.Sprintf(pathPattern, "links", "o0j2j"))
func (h *Handler) remoteClusterRequest(remoteID string, req *http.Request) (*http.Response, error) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list