[ARVADOS] updated: 1.3.0-1898-g76509a793
Git user
git at public.curoverse.com
Thu Nov 21 20:07:39 UTC 2019
Summary of changes:
.../tests/federation/arvbox-make-federation.cwl | 8 ++
sdk/cwl/tests/federation/arvbox/fed-config.cwl | 6 ++
sdk/cwl/tests/federation/arvbox/start.cwl | 23 +++++-
sdk/python/tests/fed-migrate/CWLFile | 28 +++++++
.../tests/fed-migrate/arvbox-make-federation.cwl | 12 +++
sdk/python/tests/fed-migrate/cwlex.cwl | 41 ++++++++++
sdk/python/tests/fed-migrate/fed-migrate.cwl | 95 ++++------------------
sdk/python/tests/fed-migrate/fed-migrate.cwlex | 33 ++------
sdk/python/tests/fed-migrate/set_login.py | 10 ---
9 files changed, 143 insertions(+), 113 deletions(-)
create mode 100644 sdk/python/tests/fed-migrate/CWLFile
create mode 100644 sdk/python/tests/fed-migrate/cwlex.cwl
delete mode 100644 sdk/python/tests/fed-migrate/set_login.py
via 76509a7932f48f945904163fd414b60da1d601a0 (commit)
from 0c7319e50b1e5c35d8645b26758f19bce0f32cb0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 76509a7932f48f945904163fd414b60da1d601a0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Nov 21 14:47:46 2019 -0500
Refactor arvbox federation test scripts, no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-make-federation.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 81b542057..5872dbef5 100644
--- a/sdk/cwl/tests/federation/arvbox-make-federation.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -27,6 +27,12 @@ inputs:
default:
class: File
location: ../../../../tools/arvbox/bin/arvbox
+ branch:
+ type: string
+ default: master
+ logincluster:
+ type: boolean
+ default: false
outputs:
arvados_api_token:
type: string
@@ -64,6 +70,7 @@ steps:
container_name: containers
arvbox_data: mkdir/arvbox_data
arvbox_bin: arvbox
+ branch: branch
out: [cluster_id, container_host, arvbox_data_out, superuser_token]
scatter: [container_name, arvbox_data]
scatterMethod: dotproduct
@@ -76,6 +83,7 @@ steps:
cluster_hosts: start/container_host
arvbox_data: start/arvbox_data_out
arvbox_bin: arvbox
+ logincluster: logincluster
out: []
scatter: [container_name, this_cluster_id, arvbox_data]
scatterMethod: dotproduct
diff --git a/sdk/cwl/tests/federation/arvbox/fed-config.cwl b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
index 76523a56b..37936df63 100644
--- a/sdk/cwl/tests/federation/arvbox/fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
@@ -14,6 +14,9 @@ inputs:
cluster_hosts: string[]
arvbox_data: Directory
arvbox_bin: File
+ logincluster:
+ type: boolean
+ default: false
outputs:
arvbox_data_out:
type: Directory
@@ -39,6 +42,9 @@ requirements:
}
var r = {"Clusters": {}};
r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+ if (r["Clusters"][inputs.this_cluster_id]) {
+ r["Clusters"][inputs.this_cluster_id]["Login"] = {"LoginCluster": inputs.cluster_ids[0]};
+ }
return JSON.stringify(r);
}
- entryname: application.yml.override
diff --git a/sdk/cwl/tests/federation/arvbox/start.cwl b/sdk/cwl/tests/federation/arvbox/start.cwl
index a0b3e1864..d26a6b28e 100644
--- a/sdk/cwl/tests/federation/arvbox/start.cwl
+++ b/sdk/cwl/tests/federation/arvbox/start.cwl
@@ -11,6 +11,9 @@ inputs:
container_name: string
arvbox_data: Directory
arvbox_bin: File
+ branch:
+ type: string
+ default: master
outputs:
cluster_id:
type: string
@@ -68,6 +71,24 @@ arguments:
- shellQuote: false
valueFrom: |
set -ex
- $(inputs.arvbox_bin.path) start dev
+ mkdir -p $ARVBOX_DATA
+ if ! test -d $ARVBOX_DATA/arvados ; then
+ cd $ARVBOX_DATA
+ git clone https://github.com/curoverse/arvados.git
+ fi
+ cd $ARVBOX_DATA/arvados
+ gitver=`git rev-parse HEAD`
+ git fetch
+ git checkout -f $(inputs.branch)
+ git pull
+ pulled=`git rev-parse HEAD`
+ git --no-pager log -n1 $pulled
+
+ cd $(runtime.outdir)
+ if test "$gitver" = "$pulled" ; then
+ $(inputs.arvbox_bin.path) start dev
+ else
+ $(inputs.arvbox_bin.path) restart dev
+ fi
$(inputs.arvbox_bin.path) status > status.txt
$(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
diff --git a/sdk/python/tests/fed-migrate/CWLFile b/sdk/python/tests/fed-migrate/CWLFile
new file mode 100644
index 000000000..18b2ed877
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/CWLFile
@@ -0,0 +1,28 @@
+cwlVersion: v1.0
+class: Workflow
+requirements:
+ ScatterFeatureRequirement: {}
+inputs:
+ exfiles:
+ type: string[]
+ default:
+ - fed-migrate.cwlex
+ - run-test.cwlex
+ dir:
+ type: Directory
+ default:
+ class: Directory
+ location: .
+outputs:
+ out:
+ type: File[]
+ outputSource: step1/converted
+
+steps:
+ step1:
+ in:
+ inpdir: dir
+ inpfile: exfiles
+ out: [converted]
+ scatter: inpfile
+ run: cwlex.cwl
diff --git a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
index 5057d4cb1..0aa6f177a 100644
--- a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
+++ b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
@@ -3,8 +3,12 @@ class: Workflow
$namespaces:
arv: "http://arvados.org/cwl#"
cwltool: "http://commonwl.org/cwltool#"
+
inputs:
arvbox_base: Directory
+ branch:
+ type: string
+ default: master
outputs:
arvados_api_hosts:
type: string[]
@@ -21,13 +25,21 @@ outputs:
arvbox_bin:
type: File
outputSource: start/arvbox_bin
+ refspec:
+ type: string
+ outputSource: branch
requirements:
SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
cwltool:LoadListingRequirement:
loadListing: no_listing
steps:
start:
in:
arvbox_base: arvbox_base
+ branch: branch
+ logincluster:
+ default: true
out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers, arvbox_bin]
run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
diff --git a/sdk/python/tests/fed-migrate/cwlex.cwl b/sdk/python/tests/fed-migrate/cwlex.cwl
new file mode 100644
index 000000000..1e72fedb5
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/cwlex.cwl
@@ -0,0 +1,41 @@
+#!/usr/bin/env cwl-runner
+arguments:
+ - cwlex
+ - '$(inputs.inp ? inputs.inp.path : inputs.inpdir.path+''/''+inputs.inpfile)'
+class: CommandLineTool
+cwlVersion: v1.0
+id: '#main'
+inputs:
+ - id: inp
+ type:
+ - 'null'
+ - File
+ - id: inpdir
+ type:
+ - 'null'
+ - Directory
+ - id: inpfile
+ type:
+ - 'null'
+ - string
+ - id: outname
+ type:
+ - 'null'
+ - string
+outputs:
+ - id: converted
+ outputBinding:
+ glob: $(outname(inputs))
+ type: File
+requirements:
+ - class: DockerRequirement
+ dockerPull: commonworkflowlanguage/cwlex
+ - class: InlineJavascriptRequirement
+ expressionLib:
+ - |
+
+ function outname(inputs) {
+ return inputs.outname ? inputs.outname : (inputs.inp ? inputs.inp.nameroot+'.cwl' : inputs.inpfile.replace(/(.*).cwlex/, '$1.cwl'));
+ }
+stdout: $(outname(inputs))
+
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl
index 1c8fcca59..19c2b58ef 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwl
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl
@@ -337,7 +337,7 @@ $graph:
type: string
- id: arvbox_bin
type: File
- - default: 15531-logincluster-migrate
+ - default: master
id: refspec
type: string
outputs:
@@ -407,73 +407,15 @@ $graph:
type: string
outputs:
- id: supertok
- outputSource: superuser_tok_3/superuser_token
+ outputSource: superuser_tok_2/superuser_token
type: string
requirements:
- - class: EnvVarRequirement
- envDef:
- ARVBOX_CONTAINER: $(inputs.container)
+ InlineJavascriptRequirement: {}
steps:
- id: main_2_embed_1
in:
- cluster_id:
- source: cluster_id
- container:
- source: container
- logincluster:
- source: logincluster
- set_login:
- default:
- class: File
- location: set_login.py
- out:
- - c
- run:
- arguments:
- - sh
- - _script
- class: CommandLineTool
- id: main_2_embed_1_embed
- inputs:
- - id: container
- type: string
- - id: cluster_id
- type: string
- - id: logincluster
- type: string
- - id: set_login
- type: File
- outputs:
- - id: c
- outputBinding:
- outputEval: $(inputs.container)
- type: string
- requirements:
- InitialWorkDirRequirement:
- listing:
- - entry: >
- set -x
-
- docker cp
- $(inputs.container):/var/lib/arvados/cluster_config.yml.override
- .
-
- chmod +w cluster_config.yml.override
-
- python $(inputs.set_login.path)
- cluster_config.yml.override $(inputs.cluster_id)
- $(inputs.logincluster)
-
- docker cp cluster_config.yml.override
- $(inputs.container):/var/lib/arvados
- entryname: _script
- InlineJavascriptRequirement: {}
- - id: main_2_embed_2
- in:
arvbox_bin:
source: arvbox_bin
- c:
- source: main_2_embed_1/c
container:
source: container
host:
@@ -487,7 +429,7 @@ $graph:
- sh
- _script
class: CommandLineTool
- id: main_2_embed_2_embed
+ id: main_2_embed_1_embed
inputs:
- id: container
type: string
@@ -495,21 +437,21 @@ $graph:
type: string
- id: arvbox_bin
type: File
- - id: c
- type: string
- id: refspec
type: string
outputs:
- id: d
outputBinding:
- outputEval: $(inputs.c)
+ outputEval: $(inputs.container)
type: string
requirements:
InitialWorkDirRequirement:
listing:
- - entry: >
+ - entry: >+
set -xe
+ export ARVBOX_CONTAINER="$(inputs.container)"
+
$(inputs.arvbox_bin.path) pipe <<EOF
cd /usr/src/arvados
@@ -532,27 +474,26 @@ $graph:
https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
>/dev/null ; do sleep 3 ; done
- export ARVADOS_API_HOST=$(inputs.host)
-
- export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path)
- cat /var/lib/arvados/superuser_token)
-
- export ARVADOS_API_HOST_INSECURE=1
ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
cat /var/lib/arvados/vm-uuid)
- while ! python -c "import arvados ;
- arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
- 2>/dev/null ; do sleep 3; done
+ ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat
+ /var/lib/arvados/superuser_token)
+
+ while ! curl --fail --insecure --silent -H
+ "Authorization: Bearer $ARVADOS_API_TOKEN"
+ https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID
+ >/dev/null ; do sleep 3 ; done
+
entryname: _script
InlineJavascriptRequirement: {}
- - id: superuser_tok_3
+ - id: superuser_tok_2
in:
container:
source: container
d:
- source: main_2_embed_2/d
+ source: main_2_embed_1/d
out:
- superuser_token
run: '#superuser_tok'
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
index 22bc95a83..e0beaa91d 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwlex
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
@@ -8,7 +8,7 @@ def workflow main(
arvbox_containers string[],
fed_migrate="arv-federation-migrate",
arvbox_bin File,
- refspec="15531-logincluster-migrate"
+ refspec="master"
) {
logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
@@ -18,27 +18,10 @@ def workflow main(
arvados_api_hosts as host
do run workflow(logincluster, arvbox_bin, refspec)
{
- requirements {
- EnvVarRequirement {
- envDef: {
- ARVBOX_CONTAINER: "$(inputs.container)"
- }
- }
- }
-
- run tool(container, cluster_id, logincluster, set_login = File("set_login.py")) {
-sh <<<
-set -x
-docker cp $(inputs.container):/var/lib/arvados/cluster_config.yml.override .
-chmod +w cluster_config.yml.override
-python $(inputs.set_login.path) cluster_config.yml.override $(inputs.cluster_id) $(inputs.logincluster)
-docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
->>>
- return container as c
- }
- run tool(container, host, arvbox_bin, c, refspec) {
+ run tool(container, host, arvbox_bin, refspec) {
sh <<<
set -xe
+export ARVBOX_CONTAINER="$(inputs.container)"
$(inputs.arvbox_bin.path) pipe <<EOF
cd /usr/src/arvados
git fetch
@@ -50,13 +33,13 @@ EOF
$(inputs.arvbox_bin.path) hotreset
while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
-export ARVADOS_API_HOST=$(inputs.host)
-export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
-export ARVADOS_API_HOST_INSECURE=1
+
ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid)
-while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
+ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
+while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID >/dev/null ; do sleep 3 ; done
+
>>>
- return c as d
+ return container as d
}
supertok = superuser_tok(container, d)
return supertok
diff --git a/sdk/python/tests/fed-migrate/set_login.py b/sdk/python/tests/fed-migrate/set_login.py
deleted file mode 100644
index 2900af182..000000000
--- a/sdk/python/tests/fed-migrate/set_login.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import json
-import sys
-
-f = open(sys.argv[1], "r+")
-j = json.load(f)
-j["Clusters"][sys.argv[2]]["Login"] = {"LoginCluster": sys.argv[3]}
-for r in j["Clusters"][sys.argv[2]]["RemoteClusters"]:
- j["Clusters"][sys.argv[2]]["RemoteClusters"][r]["Insecure"] = True
-f.seek(0)
-json.dump(j, f)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list