[ARVADOS] updated: 1.3.0-1898-g76509a793

Git user git at public.curoverse.com
Thu Nov 21 20:07:39 UTC 2019


Summary of changes:
 .../tests/federation/arvbox-make-federation.cwl    |  8 ++
 sdk/cwl/tests/federation/arvbox/fed-config.cwl     |  6 ++
 sdk/cwl/tests/federation/arvbox/start.cwl          | 23 +++++-
 sdk/python/tests/fed-migrate/CWLFile               | 28 +++++++
 .../tests/fed-migrate/arvbox-make-federation.cwl   | 12 +++
 sdk/python/tests/fed-migrate/cwlex.cwl             | 41 ++++++++++
 sdk/python/tests/fed-migrate/fed-migrate.cwl       | 95 ++++------------------
 sdk/python/tests/fed-migrate/fed-migrate.cwlex     | 33 ++------
 sdk/python/tests/fed-migrate/set_login.py          | 10 ---
 9 files changed, 143 insertions(+), 113 deletions(-)
 create mode 100644 sdk/python/tests/fed-migrate/CWLFile
 create mode 100644 sdk/python/tests/fed-migrate/cwlex.cwl
 delete mode 100644 sdk/python/tests/fed-migrate/set_login.py

       via  76509a7932f48f945904163fd414b60da1d601a0 (commit)
      from  0c7319e50b1e5c35d8645b26758f19bce0f32cb0 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 76509a7932f48f945904163fd414b60da1d601a0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Nov 21 14:47:46 2019 -0500

    Refactor arvbox federation test scripts, no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/arvbox-make-federation.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 81b542057..5872dbef5 100644
--- a/sdk/cwl/tests/federation/arvbox-make-federation.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -27,6 +27,12 @@ inputs:
     default:
       class: File
       location: ../../../../tools/arvbox/bin/arvbox
+  branch:
+    type: string
+    default: master
+  logincluster:
+    type: boolean
+    default: false
 outputs:
   arvados_api_token:
     type: string
@@ -64,6 +70,7 @@ steps:
       container_name: containers
       arvbox_data: mkdir/arvbox_data
       arvbox_bin: arvbox
+      branch: branch
     out: [cluster_id, container_host, arvbox_data_out, superuser_token]
     scatter: [container_name, arvbox_data]
     scatterMethod: dotproduct
@@ -76,6 +83,7 @@ steps:
       cluster_hosts: start/container_host
       arvbox_data: start/arvbox_data_out
       arvbox_bin: arvbox
+      logincluster: logincluster
     out: []
     scatter: [container_name, this_cluster_id, arvbox_data]
     scatterMethod: dotproduct
diff --git a/sdk/cwl/tests/federation/arvbox/fed-config.cwl b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
index 76523a56b..37936df63 100644
--- a/sdk/cwl/tests/federation/arvbox/fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
@@ -14,6 +14,9 @@ inputs:
   cluster_hosts: string[]
   arvbox_data: Directory
   arvbox_bin: File
+  logincluster:
+    type: boolean
+    default: false
 outputs:
   arvbox_data_out:
     type: Directory
@@ -39,6 +42,9 @@ requirements:
           }
           var r = {"Clusters": {}};
           r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+          if (r["Clusters"][inputs.this_cluster_id]) {
+            r["Clusters"][inputs.this_cluster_id]["Login"] = {"LoginCluster": inputs.cluster_ids[0]};
+          }
           return JSON.stringify(r);
           }
       - entryname: application.yml.override
diff --git a/sdk/cwl/tests/federation/arvbox/start.cwl b/sdk/cwl/tests/federation/arvbox/start.cwl
index a0b3e1864..d26a6b28e 100644
--- a/sdk/cwl/tests/federation/arvbox/start.cwl
+++ b/sdk/cwl/tests/federation/arvbox/start.cwl
@@ -11,6 +11,9 @@ inputs:
   container_name: string
   arvbox_data: Directory
   arvbox_bin: File
+  branch:
+    type: string
+    default: master
 outputs:
   cluster_id:
     type: string
@@ -68,6 +71,24 @@ arguments:
   - shellQuote: false
     valueFrom: |
       set -ex
-      $(inputs.arvbox_bin.path) start dev
+      mkdir -p $ARVBOX_DATA
+      if ! test -d $ARVBOX_DATA/arvados ; then
+        cd $ARVBOX_DATA
+        git clone https://github.com/curoverse/arvados.git
+      fi
+      cd $ARVBOX_DATA/arvados
+      gitver=`git rev-parse HEAD`
+      git fetch
+      git checkout -f $(inputs.branch)
+      git pull
+      pulled=`git rev-parse HEAD`
+      git --no-pager log -n1 $pulled
+
+      cd $(runtime.outdir)
+      if test "$gitver" = "$pulled" ; then
+        $(inputs.arvbox_bin.path) start dev
+      else
+        $(inputs.arvbox_bin.path) restart dev
+      fi
       $(inputs.arvbox_bin.path) status > status.txt
       $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
diff --git a/sdk/python/tests/fed-migrate/CWLFile b/sdk/python/tests/fed-migrate/CWLFile
new file mode 100644
index 000000000..18b2ed877
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/CWLFile
@@ -0,0 +1,28 @@
+cwlVersion: v1.0
+class: Workflow
+requirements:
+  ScatterFeatureRequirement: {}
+inputs:
+  exfiles:
+    type: string[]
+    default:
+      - fed-migrate.cwlex
+      - run-test.cwlex
+  dir:
+    type: Directory
+    default:
+      class: Directory
+      location: .
+outputs:
+  out:
+    type: File[]
+    outputSource: step1/converted
+
+steps:
+  step1:
+    in:
+      inpdir: dir
+      inpfile: exfiles
+    out: [converted]
+    scatter: inpfile
+    run: cwlex.cwl
diff --git a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
index 5057d4cb1..0aa6f177a 100644
--- a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
+++ b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
@@ -3,8 +3,12 @@ class: Workflow
 $namespaces:
   arv: "http://arvados.org/cwl#"
   cwltool: "http://commonwl.org/cwltool#"
+
 inputs:
   arvbox_base: Directory
+  branch:
+    type: string
+    default: master
 outputs:
   arvados_api_hosts:
     type: string[]
@@ -21,13 +25,21 @@ outputs:
   arvbox_bin:
     type: File
     outputSource: start/arvbox_bin
+  refspec:
+    type: string
+    outputSource: branch
 requirements:
   SubworkflowFeatureRequirement: {}
+  ScatterFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
   cwltool:LoadListingRequirement:
     loadListing: no_listing
 steps:
   start:
     in:
       arvbox_base: arvbox_base
+      branch: branch
+      logincluster:
+        default: true
     out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers, arvbox_bin]
     run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
diff --git a/sdk/python/tests/fed-migrate/cwlex.cwl b/sdk/python/tests/fed-migrate/cwlex.cwl
new file mode 100644
index 000000000..1e72fedb5
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/cwlex.cwl
@@ -0,0 +1,41 @@
+#!/usr/bin/env cwl-runner
+arguments:
+  - cwlex
+  - '$(inputs.inp ? inputs.inp.path : inputs.inpdir.path+''/''+inputs.inpfile)'
+class: CommandLineTool
+cwlVersion: v1.0
+id: '#main'
+inputs:
+  - id: inp
+    type:
+      - 'null'
+      - File
+  - id: inpdir
+    type:
+      - 'null'
+      - Directory
+  - id: inpfile
+    type:
+      - 'null'
+      - string
+  - id: outname
+    type:
+      - 'null'
+      - string
+outputs:
+  - id: converted
+    outputBinding:
+      glob: $(outname(inputs))
+    type: File
+requirements:
+  - class: DockerRequirement
+    dockerPull: commonworkflowlanguage/cwlex
+  - class: InlineJavascriptRequirement
+    expressionLib:
+      - |
+
+        function outname(inputs) {
+          return inputs.outname ? inputs.outname : (inputs.inp ? inputs.inp.nameroot+'.cwl' : inputs.inpfile.replace(/(.*).cwlex/, '$1.cwl'));
+        }
+stdout: $(outname(inputs))
+
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl
index 1c8fcca59..19c2b58ef 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwl
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl
@@ -337,7 +337,7 @@ $graph:
         type: string
       - id: arvbox_bin
         type: File
-      - default: 15531-logincluster-migrate
+      - default: master
         id: refspec
         type: string
     outputs:
@@ -407,73 +407,15 @@ $graph:
               type: string
           outputs:
             - id: supertok
-              outputSource: superuser_tok_3/superuser_token
+              outputSource: superuser_tok_2/superuser_token
               type: string
           requirements:
-            - class: EnvVarRequirement
-              envDef:
-                ARVBOX_CONTAINER: $(inputs.container)
+            InlineJavascriptRequirement: {}
           steps:
             - id: main_2_embed_1
               in:
-                cluster_id:
-                  source: cluster_id
-                container:
-                  source: container
-                logincluster:
-                  source: logincluster
-                set_login:
-                  default:
-                    class: File
-                    location: set_login.py
-              out:
-                - c
-              run:
-                arguments:
-                  - sh
-                  - _script
-                class: CommandLineTool
-                id: main_2_embed_1_embed
-                inputs:
-                  - id: container
-                    type: string
-                  - id: cluster_id
-                    type: string
-                  - id: logincluster
-                    type: string
-                  - id: set_login
-                    type: File
-                outputs:
-                  - id: c
-                    outputBinding:
-                      outputEval: $(inputs.container)
-                    type: string
-                requirements:
-                  InitialWorkDirRequirement:
-                    listing:
-                      - entry: >
-                          set -x
-
-                          docker cp
-                          $(inputs.container):/var/lib/arvados/cluster_config.yml.override
-                          .
-
-                          chmod +w cluster_config.yml.override
-
-                          python $(inputs.set_login.path)
-                          cluster_config.yml.override $(inputs.cluster_id)
-                          $(inputs.logincluster)
-
-                          docker cp cluster_config.yml.override
-                          $(inputs.container):/var/lib/arvados
-                        entryname: _script
-                  InlineJavascriptRequirement: {}
-            - id: main_2_embed_2
-              in:
                 arvbox_bin:
                   source: arvbox_bin
-                c:
-                  source: main_2_embed_1/c
                 container:
                   source: container
                 host:
@@ -487,7 +429,7 @@ $graph:
                   - sh
                   - _script
                 class: CommandLineTool
-                id: main_2_embed_2_embed
+                id: main_2_embed_1_embed
                 inputs:
                   - id: container
                     type: string
@@ -495,21 +437,21 @@ $graph:
                     type: string
                   - id: arvbox_bin
                     type: File
-                  - id: c
-                    type: string
                   - id: refspec
                     type: string
                 outputs:
                   - id: d
                     outputBinding:
-                      outputEval: $(inputs.c)
+                      outputEval: $(inputs.container)
                     type: string
                 requirements:
                   InitialWorkDirRequirement:
                     listing:
-                      - entry: >
+                      - entry: >+
                           set -xe
 
+                          export ARVBOX_CONTAINER="$(inputs.container)"
+
                           $(inputs.arvbox_bin.path) pipe <<EOF
 
                           cd /usr/src/arvados
@@ -532,27 +474,26 @@ $graph:
                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
                           >/dev/null ; do sleep 3 ; done
 
-                          export ARVADOS_API_HOST=$(inputs.host)
-
-                          export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path)
-                          cat /var/lib/arvados/superuser_token)
-
-                          export ARVADOS_API_HOST_INSECURE=1
 
                           ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
                           cat /var/lib/arvados/vm-uuid)
 
-                          while ! python -c "import arvados ;
-                          arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
-                          2>/dev/null ; do sleep 3; done
+                          ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat
+                          /var/lib/arvados/superuser_token)
+
+                          while ! curl --fail --insecure --silent -H
+                          "Authorization: Bearer $ARVADOS_API_TOKEN"
+                          https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID
+                          >/dev/null ; do sleep 3 ; done
+
                         entryname: _script
                   InlineJavascriptRequirement: {}
-            - id: superuser_tok_3
+            - id: superuser_tok_2
               in:
                 container:
                   source: container
                 d:
-                  source: main_2_embed_2/d
+                  source: main_2_embed_1/d
               out:
                 - superuser_token
               run: '#superuser_tok'
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
index 22bc95a83..e0beaa91d 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwlex
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
@@ -8,7 +8,7 @@ def workflow main(
   arvbox_containers string[],
   fed_migrate="arv-federation-migrate",
   arvbox_bin File,
-  refspec="15531-logincluster-migrate"
+  refspec="master"
 ) {
 
   logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
@@ -18,27 +18,10 @@ def workflow main(
 	  arvados_api_hosts as host
     do run workflow(logincluster, arvbox_bin, refspec)
   {
-    requirements {
-      EnvVarRequirement {
-        envDef: {
-          ARVBOX_CONTAINER: "$(inputs.container)"
-        }
-      }
-    }
-
-    run tool(container, cluster_id, logincluster, set_login = File("set_login.py")) {
-sh <<<
-set -x
-docker cp $(inputs.container):/var/lib/arvados/cluster_config.yml.override .
-chmod +w cluster_config.yml.override
-python $(inputs.set_login.path) cluster_config.yml.override $(inputs.cluster_id) $(inputs.logincluster)
-docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
->>>
-      return container as c
-    }
-    run tool(container, host, arvbox_bin, c, refspec) {
+    run tool(container, host, arvbox_bin, refspec) {
 sh <<<
 set -xe
+export ARVBOX_CONTAINER="$(inputs.container)"
 $(inputs.arvbox_bin.path) pipe <<EOF
 cd /usr/src/arvados
 git fetch
@@ -50,13 +33,13 @@ EOF
 $(inputs.arvbox_bin.path) hotreset
 
 while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
-export ARVADOS_API_HOST=$(inputs.host)
-export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
-export ARVADOS_API_HOST_INSECURE=1
+
 ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid)
-while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
+ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
+while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID >/dev/null ; do sleep 3 ; done
+
 >>>
-      return c as d
+      return container as d
     }
     supertok = superuser_tok(container, d)
     return supertok
diff --git a/sdk/python/tests/fed-migrate/set_login.py b/sdk/python/tests/fed-migrate/set_login.py
deleted file mode 100644
index 2900af182..000000000
--- a/sdk/python/tests/fed-migrate/set_login.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import json
-import sys
-
-f = open(sys.argv[1], "r+")
-j = json.load(f)
-j["Clusters"][sys.argv[2]]["Login"] = {"LoginCluster": sys.argv[3]}
-for r in j["Clusters"][sys.argv[2]]["RemoteClusters"]:
-    j["Clusters"][sys.argv[2]]["RemoteClusters"][r]["Insecure"] = True
-f.seek(0)
-json.dump(j, f)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list