[ARVADOS] created: 1.2.0-324-gb47410ab6
Git user
git at public.curoverse.com
Wed Nov 7 15:16:11 EST 2018
at b47410ab657a42db9e9ef1d7e9d8eae4aaf35685 (commit)
commit b47410ab657a42db9e9ef1d7e9d8eae4aaf35685
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 15:13:28 2018 -0500
14198: Scrubbing works
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index bdf16482e..a0ac95588 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -6,16 +6,33 @@ api = arvados.api()
with open("config.json") as f:
config = json.load(f)
+scrub_collections = set(config["scrub_collections"])
+
for cluster_id in config["arvados_cluster_ids"]:
+ images = []
for scrub_image in config["scrub_images"]:
sp = scrub_image.split(":")
image_name = sp[0]
image_tag = sp[1] if len(sp) > 1 else "latest"
+ images.append('{}:{}'.format(image_name, image_tag))
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', 'in', images]],
+ cluster_id=cluster_id).execute()
+
+ head_uuids = [lk["head_uuid"] for lk in search_links["items"]]
+ cols = api.collections().list(filters=[["uuid", "in", head_uuids]],
+ cluster_id=cluster_id).execute()
+ for c in cols["items"]:
+ scrub_collections.add(c["portable_data_hash"])
+ for lk in search_links["items"]:
+ api.links().delete(uuid=lk["uuid"]).execute()
- search_links = api.links().list(
- filters=[['link_class', '=', 'docker_image_repo+tag'],
- ['name', '=',
- '{}:{}'.format(image_name, image_tag)]],
- cluster_id=cluster_id).execute()
- for s in search_links["items"]:
- print s
+
+for cluster_id in config["arvados_cluster_ids"]:
+ matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
+ select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ for m in matches["items"]:
+ api.collections().delete(uuid=m["uuid"]).execute()
+ print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
commit 81063f060d85c2ac7db43f30641926b5688d434b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:37:08 2018 -0500
14198: More work on scrub
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index e7101dc9e..f1d101260 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -47,6 +47,8 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[0];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
@@ -68,5 +70,7 @@ steps:
self["runOnCluster"] = inputs.arvados_cluster_ids[1];
return self;
}
+ scrub_images: {default: ["debian:9"]}
+ scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
out: [out]
run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index e12aae464..2463a466e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -5,8 +5,15 @@ requirements:
listing:
- entryname: input.json
entry: $(JSON.stringify(inputs.obj))
- - entryname: clusters.json
- entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ - entryname: config.json
+ entry: |-
+ ${
+ return JSON.stringify({
+ arvados_cluster_ids: inputs.arvados_cluster_ids,
+ scrub_images: inputs.scrub_images,
+ scrub_collections: inputs.scrub_collections
+ });
+ }
EnvVarRequirement:
envDef:
ARVADOS_API_HOST: $(inputs.arvados_api_host)
@@ -20,6 +27,8 @@ inputs:
arvados_cluster_ids: string[]
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
preparescript:
type: File
default:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index 48055224f..bdf16482e 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,5 +1,21 @@
import arvados
+import json
api = arvados.api()
-print(api.users().current().execute())
+with open("config.json") as f:
+ config = json.load(f)
+
+for cluster_id in config["arvados_cluster_ids"]:
+ for scrub_image in config["scrub_images"]:
+ sp = scrub_image.split(":")
+ image_name = sp[0]
+ image_tag = sp[1] if len(sp) > 1 else "latest"
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', '=',
+ '{}:{}'.format(image_name, image_tag)]],
+ cluster_id=cluster_id).execute()
+ for s in search_links["items"]:
+ print s
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index a22d57374..66c510071 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -21,6 +21,8 @@ inputs:
acr: string?
wf: File
obj: Any
+ scrub_images: string[]
+ scrub_collections: string[]
outputs:
out:
type: Any
@@ -34,6 +36,8 @@ steps:
arvados_cluster_ids: arvados_cluster_ids
wf: wf
obj: obj
+ scrub_images: scrub_images
+ scrub_collections: scrub_collections
out: [done]
run: prepare.cwl
run-acr:
commit 99324aa7ef31ef45e2a7979c7d81549ef0587d2b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 14:01:30 2018 -0500
14198: Working on prepare step.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
new file mode 100644
index 000000000..e12aae464
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -0,0 +1,35 @@
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ - entryname: clusters.json
+ entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure: boolean
+ arvados_api_host: string
+ arvados_cluster_ids: string[]
+ wf: File
+ obj: Any
+ preparescript:
+ type: File
+ default:
+ class: File
+ location: prepare.py
+ inputBinding:
+ position: 1
+outputs:
+ done:
+ type: boolean
+ outputBinding:
+ outputEval: $(true)
+baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
new file mode 100644
index 000000000..48055224f
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -0,0 +1,5 @@
+import arvados
+
+api = arvados.api()
+
+print(api.users().current().execute())
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index a178a2c88..ed9599194 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -6,9 +6,9 @@ inputs:
default: arvados-cwl-runner
inputBinding:
position: 1
- arv_host: string
- arv_token: string
- arv_insecure:
+ arvados_api_host: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
type: boolean
default: false
runner_remote_host:
@@ -28,9 +28,9 @@ requirements:
entry: $(JSON.stringify(inputs.obj))
EnvVarRequirement:
envDef:
- ARVADOS_API_HOST: $(inputs.arv_host)
- ARVADOS_API_TOKEN: $(inputs.arv_token)
- ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
InlineJavascriptRequirement: {}
outputs:
out:
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2b0e56c60..2fc621521 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -18,16 +18,19 @@ else:
u = api.users().create(body={
'first_name': 'Test',
'last_name': 'User',
- 'email': 'test at example.com'
+ 'email': 'test at example.com',
+ 'is_admin': False
}).execute()
api.users().activate(uuid=u["uuid"]).execute()
tok = api.api_client_authorizations().create(body={
- "owner_uuid": u["uuid"]
+ "api_client_authorization": {
+ "owner_uuid": u["uuid"]
+ }
}).execute()
with open("cwl.output.json", "w") as f:
json.dump({
"test_user_uuid": u["uuid"],
- "test_user_token": tok["api_token"]
+ "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"])
}, f)
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index f4b8f7320..a22d57374 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -26,11 +26,22 @@ outputs:
type: Any
outputSource: run-acr/out
steps:
+ prepare:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ arvados_cluster_ids: arvados_cluster_ids
+ wf: wf
+ obj: obj
+ out: [done]
+ run: prepare.cwl
run-acr:
in:
- arv_token: arvados_api_token
- arv_insecure: arvado_api_host_insecure
- arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ prepare: prepare/done
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
acr: acr
wf: wf
obj: obj
commit 4e637ed5a0b4b12543c1c543bd18af984576acb8
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Nov 7 13:17:11 2018 -0500
14198: Initial test cases.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 4da2bac0b..721e6b775 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -19,4 +19,33 @@ Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
Workflow which has a remote collection in InitialWorkDir, which is captured in output.
Workflow with file input that has a secondary file in a separate
-collection, which is remote.
\ No newline at end of file
+collection, which is remote.
+
+
+Need to pull Docker image
+Can't run in container because it doesn't have access to Docker from inside -> how can we workaround?
+Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this)
+Try setting up a virtualenv -> annoying because you need C dependencies to build
+Virtualenv sets up shop in /tmp instead of the real directory
+--relocatable ???
+
+
+arvados/jobs doesn't have the docker client
+arvbox has docker client
+
+- Still need venv or something to run cwltool to manage all this.
+- Unless we run cwltool in docker as well.
+
+Can use cwl-docker.sh to run cwltool in docker
+
+- arvbox needs Docker. This will *probably* work launched from container if base dir is under PWD
+- want to run arvados-cwl-runner. this needs Docker to pull images.
+
+For a-c-r solutions are one of:
+
+1) Don't run in Docker, use virtualenv instead.
+2) Access to Docker from inside Docker to pull images.
+2a) Docker-in-Docker
+2b) Bind mount Docker socket
+2c) Communicate to Docker daemon over TCP
+3) Arvados feature to pull images
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 2ff6f79b1..8d7075a1d 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -8,22 +8,32 @@ requirements:
StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
+ InlineJavascriptRequirement: {}
inputs:
- containers: string[]
+ containers:
+ type: string[]
+ default: [fedbox1, fedbox2, fedbox3]
arvbox_base: Directory
+ in_acr: string?
+ insecure:
+ type: boolean
+ default: true
outputs:
- cluster_ids:
- type: string[]
- outputSource: start/cluster_id
- container_hosts:
- type: string[]
- outputSource: start/container_host
- test_user_uuid:
- type: string
- outputSource: setup-user/test_user_uuid
- test_user_token:
+ arvados_api_token:
type: string
outputSource: setup-user/test_user_token
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/container_host
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ acr:
+ type: string?
+ outputSource: in_acr
+ arvado_api_host_insecure:
+ type: boolean
+ outputSource: insecure
steps:
mkdir:
in:
@@ -55,4 +65,4 @@ steps:
container_host: {source: start/container_host, valueFrom: "$(self[0])"}
superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
out: [test_user_uuid, test_user_token]
- run: arvbox-setup-user.cwl
\ No newline at end of file
+ run: arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 2392b1149..e7101dc9e 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -7,16 +7,66 @@ $namespaces:
hints:
cwltool:Secrets:
secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
inputs:
- arvados_api_host_home: string
- arvados_home_id: string
arvados_api_token: string
arvado_api_host_insecure:
- type: bool
+ type: boolean
default: false
- arvados_api_host_clusterB: string
- arvados_clusterB_id: string
- arvados_api_host_clusterC: string
- arvados_clusterC_id: string
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
-outputs: []
+outputs:
+ base-case-out:
+ type: Any
+ outputSource: base-case/out
+ runner-home-step-remote-out:
+ type: Any
+ outputSource: runner-home-step-remote/out
+
+steps:
+ base-case:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
+
+ runner-home-step-remote:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf: {default: {class: File, location: md5sum.cwl}}
+ obj:
+ default:
+ inp:
+ class: File
+ location: whale.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ out: [out]
+ run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
new file mode 100644
index 000000000..55e1fbf8a
--- /dev/null
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -0,0 +1,24 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:9
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: string
+ outputBinding:
+ glob: out.txt
+ loadContents: true
+ outputEval: $(self[0].contents.substr(0, 32))
+stdout: out.txt
+baseCommand: md5sum
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
new file mode 100644
index 000000000..a178a2c88
--- /dev/null
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -0,0 +1,47 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ acr:
+ type: string?
+ default: arvados-cwl-runner
+ inputBinding:
+ position: 1
+ arv_host: string
+ arv_token: string
+ arv_insecure:
+ type: boolean
+ default: false
+ runner_remote_host:
+ type: string?
+ inputBinding:
+ prefix: --submit-runner-cluster
+ position: 2
+ wf:
+ type: File
+ inputBinding:
+ position: 3
+ obj: Any
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arv_host)
+ ARVADOS_API_TOKEN: $(inputs.arv_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+ InlineJavascriptRequirement: {}
+outputs:
+ out:
+ type: Any
+ outputBinding:
+ glob: output.json
+ loadContents: true
+ outputEval: $(JSON.parse(self[0].contents))
+stdout: output.json
+arguments:
+ - valueFrom: --disable-reuse
+ position: 2
+ - valueFrom: input.json
+ position: 4
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
new file mode 100644
index 000000000..f4b8f7320
--- /dev/null
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -0,0 +1,38 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
+ wf: File
+ obj: Any
+outputs:
+ out:
+ type: Any
+ outputSource: run-acr/out
+steps:
+ run-acr:
+ in:
+ arv_token: arvados_api_token
+ arv_insecure: arvado_api_host_insecure
+ arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ acr: acr
+ wf: wf
+ obj: obj
+ out: [out]
+ run: run-acr.cwl
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/whale.txt
new file mode 100644
index 000000000..425d1ed02
--- /dev/null
+++ b/sdk/cwl/tests/federation/whale.txt
@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
commit 8e7f4087fe20e87a8b2924255a7fd70db0e935e9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 14:57:14 2018 -0500
14198: Initialize test user
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index a595d67cf..173b9df72 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -9,7 +9,11 @@ inputs:
cluster_ids: string[]
cluster_hosts: string[]
arvbox_data: Directory
-outputs: []
+outputs:
+ arvbox_data_out:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
@@ -23,7 +27,7 @@ requirements:
var remoteClusters = {};
for (var i = 0; i < inputs.cluster_ids.length; i++) {
remoteClusters[inputs.cluster_ids[i]] = {
- "Host": inputs.cluster_hosts[i]+":8000",
+ "Host": inputs.cluster_hosts[i],
"Proxy": true,
"Insecure": true
};
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 4b52223e0..2ff6f79b1 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -5,6 +5,7 @@ $namespaces:
cwltool: "http://commonwl.org/cwltool#"
requirements:
ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
inputs:
@@ -14,9 +15,15 @@ outputs:
cluster_ids:
type: string[]
outputSource: start/cluster_id
- container_ips:
+ container_hosts:
type: string[]
- outputSource: start/container_ip
+ outputSource: start/container_host
+ test_user_uuid:
+ type: string
+ outputSource: setup-user/test_user_uuid
+ test_user_token:
+ type: string
+ outputSource: setup-user/test_user_token
steps:
mkdir:
in:
@@ -28,7 +35,7 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip, arvbox_data]
+ out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
@@ -37,9 +44,15 @@ steps:
container_name: containers
this_cluster_id: start/cluster_id
cluster_ids: start/cluster_id
- cluster_hosts: start/container_ip
- arvbox_data: start/arvbox_data
+ cluster_hosts: start/container_host
+ arvbox_data: start/arvbox_data_out
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
run: arvbox-fed-config.cwl
+ setup-user:
+ in:
+ container_host: {source: start/container_host, valueFrom: "$(self[0])"}
+ superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
+ out: [test_user_uuid, test_user_token]
+ run: arvbox-setup-user.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
new file mode 100644
index 000000000..684bc8c8d
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.container_host)
+ ARVADOS_API_TOKEN: $(inputs.superuser_token)
+ ARVADOS_API_HOST_INSECURE: "true"
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ DockerRequirement:
+ dockerPull: arvados/jobs
+inputs:
+ container_host: string
+ superuser_token: string
+ make_user_script:
+ type: File
+ default:
+ class: File
+ location: setup_user.py
+outputs:
+ test_user_uuid: string
+ test_user_token: string
+arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 3485fba68..ae591d58f 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -21,7 +21,7 @@ outputs:
}
}
}
- container_ip:
+ container_host:
type: string
outputBinding:
glob: status.txt
@@ -31,11 +31,17 @@ outputs:
var sp = self[0].contents.split("\n");
for (var i = 0; i < sp.length; i++) {
if (sp[i].startsWith("Container IP: ")) {
- return sp[i].substr(14);
+ return sp[i].substr(14)+":8000";
}
}
}
- arvbox_data:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+ arvbox_data_out:
type: Directory
outputBinding:
outputEval: $(inputs.arvbox_data)
@@ -56,4 +62,7 @@ requirements:
arguments:
- shellQuote: false
valueFrom: |
- arvbox start dev && arvbox status > status.txt
+ set -e
+ arvbox start dev
+ arvbox status > status.txt
+ arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
new file mode 100644
index 000000000..2b0e56c60
--- /dev/null
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -0,0 +1,33 @@
+import arvados
+import arvados.errors
+import time
+import json
+
+while True:
+ try:
+ api = arvados.api()
+ break
+ except arvados.errors.ApiError:
+ time.sleep(2)
+
+existing = api.users().list(filters=[["email", "=", "test at example.com"],
+ ["is_active", "=", True]], limit=1).execute()
+if existing["items"]:
+ u = existing["items"][0]
+else:
+ u = api.users().create(body={
+ 'first_name': 'Test',
+ 'last_name': 'User',
+ 'email': 'test at example.com'
+ }).execute()
+ api.users().activate(uuid=u["uuid"]).execute()
+
+tok = api.api_client_authorizations().create(body={
+ "owner_uuid": u["uuid"]
+}).execute()
+
+with open("cwl.output.json", "w") as f:
+ json.dump({
+ "test_user_uuid": u["uuid"],
+ "test_user_token": tok["api_token"]
+ }, f)
commit 39fde057d9edb020eb91b4c98b8f14478568bc28
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Nov 6 13:59:20 2018 -0500
14198: Start and configure a federation of N arvboxes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 157fe4f72..a595d67cf 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,36 +1,52 @@
cwlVersion: v1.0
class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
inputs:
container_name: string
- this_cluster: string
+ this_cluster_id: string
cluster_ids: string[]
cluster_hosts: string[]
- arvbox_base: Directory
+ arvbox_data: Directory
outputs: []
requirements:
EnvVarRequirement:
envDef:
ARVBOX_CONTAINER: $(inputs.container_name)
- ARVBOX_BASE: $(inputs.arvbox_base.path)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
InitialWorkDirRequirement:
listing:
- cluster_config.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
- }
- application.yml.override: |
- ${
- var remoteClusters = {};
- for (var i = 0; i < cluster_ids.length; i++) {
- remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
- }
- return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
- }
+ - entryname: cluster_config.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = {
+ "Host": inputs.cluster_hosts[i]+":8000",
+ "Proxy": true,
+ "Insecure": true
+ };
+ }
+ var r = {"Clusters": {}};
+ r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+ return JSON.stringify(r);
+ }
+ - entryname: application.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
ShellCommandRequirement: {}
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
arguments:
- shellQuote: false
valueFrom: |
@@ -39,4 +55,4 @@ arguments:
arvbox sv restart api
arvbox sv restart controller
arvbox sv restart keepstore0
- arvbox sv restart keepstore1
\ No newline at end of file
+ arvbox sv restart keepstore1
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 91bd22287..4b52223e0 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -28,7 +28,18 @@ steps:
in:
container_name: containers
arvbox_data: mkdir/arvbox_data
- out: [cluster_id, container_ip]
+ out: [cluster_id, container_ip, arvbox_data]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
run: arvbox-start.cwl
+ fed-config:
+ in:
+ container_name: containers
+ this_cluster_id: start/cluster_id
+ cluster_ids: start/cluster_id
+ cluster_hosts: start/container_ip
+ arvbox_data: start/arvbox_data
+ out: []
+ scatter: [container_name, this_cluster_id, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 2e12597ce..3485fba68 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -35,6 +35,10 @@ outputs:
}
}
}
+ arvbox_data:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
requirements:
EnvVarRequirement:
envDef:
commit 935c60c3f67bd88ac975bf02b7409038157e2324
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 17:46:41 2018 -0500
14198: Federation integration testing WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
new file mode 100644
index 000000000..4da2bac0b
--- /dev/null
+++ b/sdk/cwl/tests/federation/README
@@ -0,0 +1,22 @@
+Things to test.
+
+Single step --submit --no-wait workflow, matrix of:
+
+Runner on home/remote cluster
+Docker image on home/remote cluster
+Step on home/remote cluster
+
+Two step workflow, matrix of:
+
+Step1 on home cluster -> Step2 on remote cluster
+Step1 on remote cluster -> Step2 on home cluster
+
+Three step workflow:
+
+(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster
+Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
+
+Workflow which has a remote collection in InitialWorkDir, which is captured in output.
+
+Workflow with file input that has a secondary file in a separate
+collection, which is remote.
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
new file mode 100644
index 000000000..157fe4f72
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -0,0 +1,42 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ container_name: string
+ this_cluster: string
+ cluster_ids: string[]
+ cluster_hosts: string[]
+ arvbox_base: Directory
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_BASE: $(inputs.arvbox_base.path)
+ InitialWorkDirRequirement:
+ listing:
+ cluster_config.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
+ }
+ application.yml.override: |
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ ShellCommandRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
+ arvbox sv restart api
+ arvbox sv restart controller
+ arvbox sv restart keepstore0
+ arvbox sv restart keepstore1
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
new file mode 100644
index 000000000..91bd22287
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -0,0 +1,34 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ ScatterFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ containers: string[]
+ arvbox_base: Directory
+outputs:
+ cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ container_ips:
+ type: string[]
+ outputSource: start/container_ip
+steps:
+ mkdir:
+ in:
+ containers: containers
+ arvbox_base: arvbox_base
+ out: [arvbox_data]
+ run: arvbox-mkdir.cwl
+ start:
+ in:
+ container_name: containers
+ arvbox_data: mkdir/arvbox_data
+ out: [cluster_id, container_ip]
+ scatter: [container_name, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
new file mode 100644
index 000000000..b047beb9b
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -0,0 +1,43 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ containers:
+ type:
+ type: array
+ items: string
+ inputBinding:
+ position: 3
+ valueFrom: |
+ ${
+ return "base/"+self;
+ }
+ arvbox_base: Directory
+outputs:
+ arvbox_data:
+ type: Directory[]
+ outputBinding:
+ glob: |
+ ${
+ var r = [];
+ for (var i = 0; i < inputs.containers.length; i++) {
+ r.push("base/"+inputs.containers[i]);
+ }
+ return r;
+ }
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_base)
+ entryname: base
+ writable: true
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+arguments:
+ - mkdir
+ - "-p"
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
new file mode 100644
index 000000000..2e12597ce
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -0,0 +1,55 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+ arvbox_data: Directory
+outputs:
+ cluster_id:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Cluster id: ")) {
+ return sp[i].substr(12);
+ }
+ }
+ }
+ container_ip:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Container IP: ")) {
+ return sp[i].substr(14);
+ }
+ }
+ }
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
+ ShellCommandRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_data)
+ entryname: $(inputs.container_name)
+ writable: true
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ InlineJavascriptRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ arvbox start dev && arvbox status > status.txt
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
new file mode 100644
index 000000000..c4a5412de
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+arguments: [arvbox, stop]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
new file mode 100755
index 000000000..2392b1149
--- /dev/null
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -0,0 +1,22 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+inputs:
+ arvados_api_host_home: string
+ arvados_home_id: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: bool
+ default: false
+ arvados_api_host_clusterB: string
+ arvados_clusterB_id: string
+ arvados_api_host_clusterC: string
+ arvados_clusterC_id: string
+
+outputs: []
commit 99aa7102ee0516753bfe998e8aa2ecea7999e64e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Nov 5 17:44:23 2018 -0500
14198: Add --submit-runner-cluster test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0fd824fbe..92ea553ea 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1449,6 +1449,25 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_container_cluster_id(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list