[ARVADOS] created: 1.2.0-326-ga4e764bbe

Git user git at public.curoverse.com
Wed Nov 7 16:25:10 EST 2018


        at  a4e764bbe11e6d49ddaa8a52c233b43d162ce00a (commit)


commit a4e764bbe11e6d49ddaa8a52c233b43d162ce00a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Nov 7 16:23:01 2018 -0500

    14198: Use --always-submit-runner
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index f1d101260..d481f5c19 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -48,7 +48,7 @@ steps:
           return self;
           }
       scrub_images: {default: ["debian:9"]}
-      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
     out: [out]
     run: testcase.cwl
 
@@ -71,6 +71,6 @@ steps:
           return self;
           }
       scrub_images: {default: ["debian:9"]}
-      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
+      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57", "67beab1cda8fe7d7e623323dc4287b5b+51"]}
     out: [out]
     run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index ed9599194..887d113b9 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -43,5 +43,7 @@ stdout: output.json
 arguments:
   - valueFrom: --disable-reuse
     position: 2
+  - valueFrom: --always-submit-runner
+    position: 2
   - valueFrom: input.json
     position: 4
\ No newline at end of file

commit 0be62a84535672721c311ea43964d1d25146e1ff
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Nov 7 15:13:28 2018 -0500

    14198: Scrubbing works
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index bdf16482e..a0ac95588 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -6,16 +6,33 @@ api = arvados.api()
 with open("config.json") as f:
     config = json.load(f)
 
+scrub_collections = set(config["scrub_collections"])
+
 for cluster_id in config["arvados_cluster_ids"]:
+    images = []
     for scrub_image in config["scrub_images"]:
         sp = scrub_image.split(":")
         image_name = sp[0]
         image_tag = sp[1] if len(sp) > 1 else "latest"
+        images.append('{}:{}'.format(image_name, image_tag))
+
+    search_links = api.links().list(
+        filters=[['link_class', '=', 'docker_image_repo+tag'],
+                 ['name', 'in', images]],
+        cluster_id=cluster_id).execute()
+
+    head_uuids = [lk["head_uuid"] for lk in search_links["items"]]
+    cols = api.collections().list(filters=[["uuid", "in", head_uuids]],
+                                  cluster_id=cluster_id).execute()
+    for c in cols["items"]:
+        scrub_collections.add(c["portable_data_hash"])
+    for lk in search_links["items"]:
+        api.links().delete(uuid=lk["uuid"]).execute()
 
-        search_links = api.links().list(
-            filters=[['link_class', '=', 'docker_image_repo+tag'],
-                     ['name', '=',
-                      '{}:{}'.format(image_name, image_tag)]],
-            cluster_id=cluster_id).execute()
-        for s in search_links["items"]:
-            print s
+
+for cluster_id in config["arvados_cluster_ids"]:
+    matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
+                                     select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+    for m in matches["items"]:
+        api.collections().delete(uuid=m["uuid"]).execute()
+        print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))

commit 56b57f8bba52f5b6001a2ecdcc44ffbd7f6dda4f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Nov 7 14:37:08 2018 -0500

    14198: More work on scrub
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index e7101dc9e..f1d101260 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -47,6 +47,8 @@ steps:
           self["runOnCluster"] = inputs.arvados_cluster_ids[0];
           return self;
           }
+      scrub_images: {default: ["debian:9"]}
+      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
     out: [out]
     run: testcase.cwl
 
@@ -68,5 +70,7 @@ steps:
           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
           return self;
           }
+      scrub_images: {default: ["debian:9"]}
+      scrub_collections: {default: ["cba47aefe5eb3a014a26ec00316b30c1+57"]}
     out: [out]
     run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
index e12aae464..2463a466e 100644
--- a/sdk/cwl/tests/federation/prepare.cwl
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -5,8 +5,15 @@ requirements:
     listing:
       - entryname: input.json
         entry: $(JSON.stringify(inputs.obj))
-      - entryname: clusters.json
-        entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+      - entryname: config.json
+        entry: |-
+          ${
+          return JSON.stringify({
+            arvados_cluster_ids: inputs.arvados_cluster_ids,
+            scrub_images: inputs.scrub_images,
+            scrub_collections: inputs.scrub_collections
+          });
+          }
   EnvVarRequirement:
     envDef:
       ARVADOS_API_HOST: $(inputs.arvados_api_host)
@@ -20,6 +27,8 @@ inputs:
   arvados_cluster_ids: string[]
   wf: File
   obj: Any
+  scrub_images: string[]
+  scrub_collections: string[]
   preparescript:
     type: File
     default:
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
index 48055224f..bdf16482e 100644
--- a/sdk/cwl/tests/federation/prepare.py
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -1,5 +1,21 @@
 import arvados
+import json
 
 api = arvados.api()
 
-print(api.users().current().execute())
+with open("config.json") as f:
+    config = json.load(f)
+
+for cluster_id in config["arvados_cluster_ids"]:
+    for scrub_image in config["scrub_images"]:
+        sp = scrub_image.split(":")
+        image_name = sp[0]
+        image_tag = sp[1] if len(sp) > 1 else "latest"
+
+        search_links = api.links().list(
+            filters=[['link_class', '=', 'docker_image_repo+tag'],
+                     ['name', '=',
+                      '{}:{}'.format(image_name, image_tag)]],
+            cluster_id=cluster_id).execute()
+        for s in search_links["items"]:
+            print s
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index a22d57374..66c510071 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -21,6 +21,8 @@ inputs:
   acr: string?
   wf: File
   obj: Any
+  scrub_images: string[]
+  scrub_collections: string[]
 outputs:
   out:
     type: Any
@@ -34,6 +36,8 @@ steps:
       arvados_cluster_ids: arvados_cluster_ids
       wf: wf
       obj: obj
+      scrub_images: scrub_images
+      scrub_collections: scrub_collections
     out: [done]
     run: prepare.cwl
   run-acr:

commit cfe7f72716cb4dd0cc805fc8e77df799971db1fb
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Nov 7 14:01:30 2018 -0500

    14198: Working on prepare step.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/prepare.cwl b/sdk/cwl/tests/federation/prepare.cwl
new file mode 100644
index 000000000..e12aae464
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.cwl
@@ -0,0 +1,35 @@
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: input.json
+        entry: $(JSON.stringify(inputs.obj))
+      - entryname: clusters.json
+        entry: $(JSON.stringify(inputs.arvados_cluster_ids))
+  EnvVarRequirement:
+    envDef:
+      ARVADOS_API_HOST: $(inputs.arvados_api_host)
+      ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+      ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+  InlineJavascriptRequirement: {}
+inputs:
+  arvados_api_token: string
+  arvado_api_host_insecure: boolean
+  arvados_api_host: string
+  arvados_cluster_ids: string[]
+  wf: File
+  obj: Any
+  preparescript:
+    type: File
+    default:
+      class: File
+      location: prepare.py
+    inputBinding:
+      position: 1
+outputs:
+  done:
+    type: boolean
+    outputBinding:
+      outputEval: $(true)
+baseCommand: python2
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/prepare.py b/sdk/cwl/tests/federation/prepare.py
new file mode 100644
index 000000000..48055224f
--- /dev/null
+++ b/sdk/cwl/tests/federation/prepare.py
@@ -0,0 +1,5 @@
+import arvados
+
+api = arvados.api()
+
+print(api.users().current().execute())
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
index a178a2c88..ed9599194 100644
--- a/sdk/cwl/tests/federation/run-acr.cwl
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -6,9 +6,9 @@ inputs:
     default: arvados-cwl-runner
     inputBinding:
       position: 1
-  arv_host: string
-  arv_token: string
-  arv_insecure:
+  arvados_api_host: string
+  arvados_api_token: string
+  arvado_api_host_insecure:
     type: boolean
     default: false
   runner_remote_host:
@@ -28,9 +28,9 @@ requirements:
         entry: $(JSON.stringify(inputs.obj))
   EnvVarRequirement:
     envDef:
-      ARVADOS_API_HOST: $(inputs.arv_host)
-      ARVADOS_API_TOKEN: $(inputs.arv_token)
-      ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+      ARVADOS_API_HOST: $(inputs.arvados_api_host)
+      ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+      ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
   InlineJavascriptRequirement: {}
 outputs:
   out:
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
index 2b0e56c60..2fc621521 100644
--- a/sdk/cwl/tests/federation/setup_user.py
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -18,16 +18,19 @@ else:
     u = api.users().create(body={
         'first_name': 'Test',
         'last_name': 'User',
-        'email': 'test at example.com'
+        'email': 'test at example.com',
+        'is_admin': False
     }).execute()
     api.users().activate(uuid=u["uuid"]).execute()
 
 tok = api.api_client_authorizations().create(body={
-    "owner_uuid": u["uuid"]
+    "api_client_authorization": {
+        "owner_uuid": u["uuid"]
+    }
 }).execute()
 
 with open("cwl.output.json", "w") as f:
     json.dump({
         "test_user_uuid": u["uuid"],
-        "test_user_token": tok["api_token"]
+        "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"])
     }, f)
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
index f4b8f7320..a22d57374 100644
--- a/sdk/cwl/tests/federation/testcase.cwl
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -26,11 +26,22 @@ outputs:
     type: Any
     outputSource: run-acr/out
 steps:
+  prepare:
+    in:
+      arvados_api_token: arvados_api_token
+      arvado_api_host_insecure: arvado_api_host_insecure
+      arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+      arvados_cluster_ids: arvados_cluster_ids
+      wf: wf
+      obj: obj
+    out: [done]
+    run: prepare.cwl
   run-acr:
     in:
-      arv_token: arvados_api_token
-      arv_insecure: arvado_api_host_insecure
-      arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+      prepare: prepare/done
+      arvados_api_token: arvados_api_token
+      arvado_api_host_insecure: arvado_api_host_insecure
+      arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
       acr: acr
       wf: wf
       obj: obj

commit ad69639c625db66a86297459006005314368023a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed Nov 7 13:17:11 2018 -0500

    14198: Initial test cases.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
index 4da2bac0b..721e6b775 100644
--- a/sdk/cwl/tests/federation/README
+++ b/sdk/cwl/tests/federation/README
@@ -19,4 +19,33 @@ Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
 Workflow which has a remote collection in InitialWorkDir, which is captured in output.
 
 Workflow with file input that has a secondary file in a separate
-collection, which is remote.
\ No newline at end of file
+collection, which is remote.
+
+
+Need to pull Docker image
+Can't run in container because it doesn't have access to Docker from inside -> how can we workaround?
+Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this)
+Try setting up a virtualenv -> annoying because you need C dependencies to build
+Virtualenv sets up shop in /tmp instead of the real directory
+--relocatable ???
+
+
+arvados/jobs doesn't have the docker client
+arvbox has docker client
+
+- Still need venv or something to run cwltool to manage all this.
+- Unless we run cwltool in docker as well.
+
+Can use cwl-docker.sh to run cwltool in docker
+
+- arvbox needs Docker.  This will *probably* work launched from container if base dir is under PWD
+- want to run arvados-cwl-runner.  this needs Docker to pull images.
+
+For a-c-r solutions are one of:
+
+1) Don't run in Docker, use virtualenv instead.
+2) Access to Docker from inside Docker to pull images.
+2a) Docker-in-Docker
+2b) Bind mount Docker socket
+2c) Communicate to Docker daemon over TCP
+3) Arvados feature to pull images
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 2ff6f79b1..8d7075a1d 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -8,22 +8,32 @@ requirements:
   StepInputExpressionRequirement: {}
   cwltool:LoadListingRequirement:
     loadListing: no_listing
+  InlineJavascriptRequirement: {}
 inputs:
-  containers: string[]
+  containers:
+    type: string[]
+    default: [fedbox1, fedbox2, fedbox3]
   arvbox_base: Directory
+  in_acr: string?
+  insecure:
+    type: boolean
+    default: true
 outputs:
-  cluster_ids:
-    type: string[]
-    outputSource: start/cluster_id
-  container_hosts:
-    type: string[]
-    outputSource: start/container_host
-  test_user_uuid:
-    type: string
-    outputSource: setup-user/test_user_uuid
-  test_user_token:
+  arvados_api_token:
     type: string
     outputSource: setup-user/test_user_token
+  arvados_api_hosts:
+    type: string[]
+    outputSource: start/container_host
+  arvados_cluster_ids:
+    type: string[]
+    outputSource: start/cluster_id
+  acr:
+    type: string?
+    outputSource: in_acr
+  arvado_api_host_insecure:
+    type: boolean
+    outputSource: insecure
 steps:
   mkdir:
     in:
@@ -55,4 +65,4 @@ steps:
       container_host: {source: start/container_host, valueFrom: "$(self[0])"}
       superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
     out: [test_user_uuid, test_user_token]
-    run: arvbox-setup-user.cwl
\ No newline at end of file
+    run: arvbox-setup-user.cwl
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
index 2392b1149..e7101dc9e 100755
--- a/sdk/cwl/tests/federation/main.cwl
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -7,16 +7,66 @@ $namespaces:
 hints:
   cwltool:Secrets:
     secrets: [arvados_api_token]
+requirements:
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+  SubworkflowFeatureRequirement: {}
 inputs:
-  arvados_api_host_home: string
-  arvados_home_id: string
   arvados_api_token: string
   arvado_api_host_insecure:
-    type: bool
+    type: boolean
     default: false
-  arvados_api_host_clusterB: string
-  arvados_clusterB_id: string
-  arvados_api_host_clusterC: string
-  arvados_clusterC_id: string
+  arvados_api_hosts: string[]
+  arvados_cluster_ids: string[]
+  acr: string?
 
-outputs: []
+outputs:
+  base-case-out:
+    type: Any
+    outputSource: base-case/out
+  runner-home-step-remote-out:
+    type: Any
+    outputSource: runner-home-step-remote/out
+
+steps:
+  base-case:
+    in:
+      arvados_api_token: arvados_api_token
+      arvado_api_host_insecure: arvado_api_host_insecure
+      arvados_api_hosts: arvados_api_hosts
+      arvados_cluster_ids: arvados_cluster_ids
+      acr: acr
+      wf: {default: {class: File, location: md5sum.cwl}}
+      obj:
+        default:
+          inp:
+            class: File
+            location: whale.txt
+        valueFrom: |-
+          ${
+          self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+          return self;
+          }
+    out: [out]
+    run: testcase.cwl
+
+  runner-home-step-remote:
+    in:
+      arvados_api_token: arvados_api_token
+      arvado_api_host_insecure: arvado_api_host_insecure
+      arvados_api_hosts: arvados_api_hosts
+      arvados_cluster_ids: arvados_cluster_ids
+      acr: acr
+      wf: {default: {class: File, location: md5sum.cwl}}
+      obj:
+        default:
+          inp:
+            class: File
+            location: whale.txt
+        valueFrom: |-
+          ${
+          self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+          return self;
+          }
+    out: [out]
+    run: testcase.cwl
diff --git a/sdk/cwl/tests/federation/md5sum.cwl b/sdk/cwl/tests/federation/md5sum.cwl
new file mode 100644
index 000000000..55e1fbf8a
--- /dev/null
+++ b/sdk/cwl/tests/federation/md5sum.cwl
@@ -0,0 +1,24 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  InlineJavascriptRequirement: {}
+  DockerRequirement:
+    dockerPull: debian:9
+  arv:ClusterTarget:
+    cluster_id: $(inputs.runOnCluster)
+inputs:
+  inp:
+    type: File
+    inputBinding: {}
+  runOnCluster: string
+outputs:
+  hash:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self[0].contents.substr(0, 32))
+stdout: out.txt
+baseCommand: md5sum
diff --git a/sdk/cwl/tests/federation/run-acr.cwl b/sdk/cwl/tests/federation/run-acr.cwl
new file mode 100644
index 000000000..a178a2c88
--- /dev/null
+++ b/sdk/cwl/tests/federation/run-acr.cwl
@@ -0,0 +1,47 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  acr:
+    type: string?
+    default: arvados-cwl-runner
+    inputBinding:
+      position: 1
+  arv_host: string
+  arv_token: string
+  arv_insecure:
+    type: boolean
+    default: false
+  runner_remote_host:
+    type: string?
+    inputBinding:
+      prefix: --submit-runner-cluster
+      position: 2
+  wf:
+    type: File
+    inputBinding:
+      position: 3
+  obj: Any
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: input.json
+        entry: $(JSON.stringify(inputs.obj))
+  EnvVarRequirement:
+    envDef:
+      ARVADOS_API_HOST: $(inputs.arv_host)
+      ARVADOS_API_TOKEN: $(inputs.arv_token)
+      ARVADOS_API_HOST_INSECURE: $(""+inputs.arv_insecure)
+  InlineJavascriptRequirement: {}
+outputs:
+  out:
+    type: Any
+    outputBinding:
+      glob: output.json
+      loadContents: true
+      outputEval: $(JSON.parse(self[0].contents))
+stdout: output.json
+arguments:
+  - valueFrom: --disable-reuse
+    position: 2
+  - valueFrom: input.json
+    position: 4
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/testcase.cwl b/sdk/cwl/tests/federation/testcase.cwl
new file mode 100644
index 000000000..f4b8f7320
--- /dev/null
+++ b/sdk/cwl/tests/federation/testcase.cwl
@@ -0,0 +1,38 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+hints:
+  cwltool:Secrets:
+    secrets: [arvados_api_token]
+requirements:
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+  SubworkflowFeatureRequirement: {}
+inputs:
+  arvados_api_token: string
+  arvado_api_host_insecure:
+    type: boolean
+    default: false
+  arvados_api_hosts: string[]
+  arvados_cluster_ids: string[]
+  acr: string?
+  wf: File
+  obj: Any
+outputs:
+  out:
+    type: Any
+    outputSource: run-acr/out
+steps:
+  run-acr:
+    in:
+      arv_token: arvados_api_token
+      arv_insecure: arvado_api_host_insecure
+      arv_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+      acr: acr
+      wf: wf
+      obj: obj
+    out: [out]
+    run: run-acr.cwl
diff --git a/sdk/cwl/tests/federation/whale.txt b/sdk/cwl/tests/federation/whale.txt
new file mode 100644
index 000000000..425d1ed02
--- /dev/null
+++ b/sdk/cwl/tests/federation/whale.txt
@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.

commit 657d9973e6aead5b73581be774990291b44eebfb
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Nov 6 14:57:14 2018 -0500

    14198: Initialize test user
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index a595d67cf..173b9df72 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -9,7 +9,11 @@ inputs:
   cluster_ids: string[]
   cluster_hosts: string[]
   arvbox_data: Directory
-outputs: []
+outputs:
+  arvbox_data_out:
+    type: Directory
+    outputBinding:
+      outputEval: $(inputs.arvbox_data)
 requirements:
   EnvVarRequirement:
     envDef:
@@ -23,7 +27,7 @@ requirements:
           var remoteClusters = {};
           for (var i = 0; i < inputs.cluster_ids.length; i++) {
             remoteClusters[inputs.cluster_ids[i]] = {
-              "Host": inputs.cluster_hosts[i]+":8000",
+              "Host": inputs.cluster_hosts[i],
               "Proxy": true,
               "Insecure": true
             };
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 4b52223e0..2ff6f79b1 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -5,6 +5,7 @@ $namespaces:
   cwltool: "http://commonwl.org/cwltool#"
 requirements:
   ScatterFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
   cwltool:LoadListingRequirement:
     loadListing: no_listing
 inputs:
@@ -14,9 +15,15 @@ outputs:
   cluster_ids:
     type: string[]
     outputSource: start/cluster_id
-  container_ips:
+  container_hosts:
     type: string[]
-    outputSource: start/container_ip
+    outputSource: start/container_host
+  test_user_uuid:
+    type: string
+    outputSource: setup-user/test_user_uuid
+  test_user_token:
+    type: string
+    outputSource: setup-user/test_user_token
 steps:
   mkdir:
     in:
@@ -28,7 +35,7 @@ steps:
     in:
       container_name: containers
       arvbox_data: mkdir/arvbox_data
-    out: [cluster_id, container_ip, arvbox_data]
+    out: [cluster_id, container_host, arvbox_data_out, superuser_token]
     scatter: [container_name, arvbox_data]
     scatterMethod: dotproduct
     run: arvbox-start.cwl
@@ -37,9 +44,15 @@ steps:
       container_name: containers
       this_cluster_id: start/cluster_id
       cluster_ids: start/cluster_id
-      cluster_hosts: start/container_ip
-      arvbox_data: start/arvbox_data
+      cluster_hosts: start/container_host
+      arvbox_data: start/arvbox_data_out
     out: []
     scatter: [container_name, this_cluster_id, arvbox_data]
     scatterMethod: dotproduct
     run: arvbox-fed-config.cwl
+  setup-user:
+    in:
+      container_host: {source: start/container_host, valueFrom: "$(self[0])"}
+      superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
+    out: [test_user_uuid, test_user_token]
+    run: arvbox-setup-user.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
new file mode 100644
index 000000000..684bc8c8d
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-setup-user.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  EnvVarRequirement:
+    envDef:
+      ARVADOS_API_HOST: $(inputs.container_host)
+      ARVADOS_API_TOKEN: $(inputs.superuser_token)
+      ARVADOS_API_HOST_INSECURE: "true"
+  cwltool:LoadListingRequirement:
+    loadListing: no_listing
+  InlineJavascriptRequirement: {}
+  cwltool:InplaceUpdateRequirement:
+    inplaceUpdate: true
+  DockerRequirement:
+    dockerPull: arvados/jobs
+inputs:
+  container_host: string
+  superuser_token: string
+  make_user_script:
+    type: File
+    default:
+      class: File
+      location: setup_user.py
+outputs:
+  test_user_uuid: string
+  test_user_token: string
+arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 3485fba68..ae591d58f 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -21,7 +21,7 @@ outputs:
           }
         }
         }
-  container_ip:
+  container_host:
     type: string
     outputBinding:
       glob: status.txt
@@ -31,11 +31,17 @@ outputs:
         var sp = self[0].contents.split("\n");
         for (var i = 0; i < sp.length; i++) {
           if (sp[i].startsWith("Container IP: ")) {
-            return sp[i].substr(14);
+            return sp[i].substr(14)+":8000";
           }
         }
         }
-  arvbox_data:
+  superuser_token:
+    type: string
+    outputBinding:
+      glob: superuser_token.txt
+      loadContents: true
+      outputEval: $(self[0].contents.trim())
+  arvbox_data_out:
     type: Directory
     outputBinding:
       outputEval: $(inputs.arvbox_data)
@@ -56,4 +62,7 @@ requirements:
 arguments:
   - shellQuote: false
     valueFrom: |
-      arvbox start dev && arvbox status > status.txt
+      set -e
+      arvbox start dev
+      arvbox status > status.txt
+      arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/setup_user.py b/sdk/cwl/tests/federation/setup_user.py
new file mode 100644
index 000000000..2b0e56c60
--- /dev/null
+++ b/sdk/cwl/tests/federation/setup_user.py
@@ -0,0 +1,33 @@
+import arvados
+import arvados.errors
+import time
+import json
+
+while True:
+    try:
+        api = arvados.api()
+        break
+    except arvados.errors.ApiError:
+        time.sleep(2)
+
+existing = api.users().list(filters=[["email", "=", "test at example.com"],
+                                     ["is_active", "=", True]], limit=1).execute()
+if existing["items"]:
+    u = existing["items"][0]
+else:
+    u = api.users().create(body={
+        'first_name': 'Test',
+        'last_name': 'User',
+        'email': 'test at example.com'
+    }).execute()
+    api.users().activate(uuid=u["uuid"]).execute()
+
+tok = api.api_client_authorizations().create(body={
+    "owner_uuid": u["uuid"]
+}).execute()
+
+with open("cwl.output.json", "w") as f:
+    json.dump({
+        "test_user_uuid": u["uuid"],
+        "test_user_token": tok["api_token"]
+    }, f)

commit fb4f550441035a7aed35a9b2192b21e8f76a6981
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Nov 6 13:59:20 2018 -0500

    14198: Start and configure a federation of N arvboxes
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
index 157fe4f72..a595d67cf 100644
--- a/sdk/cwl/tests/federation/arvbox-fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -1,36 +1,52 @@
 cwlVersion: v1.0
 class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
 inputs:
   container_name: string
-  this_cluster: string
+  this_cluster_id: string
   cluster_ids: string[]
   cluster_hosts: string[]
-  arvbox_base: Directory
+  arvbox_data: Directory
 outputs: []
 requirements:
   EnvVarRequirement:
     envDef:
       ARVBOX_CONTAINER: $(inputs.container_name)
-      ARVBOX_BASE: $(inputs.arvbox_base.path)
+      ARVBOX_DATA: $(inputs.arvbox_data.path)
   InitialWorkDirRequirement:
     listing:
-      cluster_config.yml.override: |
-        ${
-        var remoteClusters = {};
-        for (var i = 0; i < cluster_ids.length; i++) {
-          remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
-        }
-        return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
-        }
-      application.yml.override: |
-        ${
-        var remoteClusters = {};
-        for (var i = 0; i < cluster_ids.length; i++) {
-          remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
-        }
-        return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
-        }
+      - entryname: cluster_config.yml.override
+        entry: >-
+          ${
+          var remoteClusters = {};
+          for (var i = 0; i < inputs.cluster_ids.length; i++) {
+            remoteClusters[inputs.cluster_ids[i]] = {
+              "Host": inputs.cluster_hosts[i]+":8000",
+              "Proxy": true,
+              "Insecure": true
+            };
+          }
+          var r = {"Clusters": {}};
+          r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+          return JSON.stringify(r);
+          }
+      - entryname: application.yml.override
+        entry: >-
+          ${
+          var remoteClusters = {};
+          for (var i = 0; i < inputs.cluster_ids.length; i++) {
+            remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+          }
+          return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+          }
+  cwltool:LoadListingRequirement:
+    loadListing: no_listing
   ShellCommandRequirement: {}
+  InlineJavascriptRequirement: {}
+  cwltool:InplaceUpdateRequirement:
+    inplaceUpdate: true
 arguments:
   - shellQuote: false
     valueFrom: |
@@ -39,4 +55,4 @@ arguments:
       arvbox sv restart api
       arvbox sv restart controller
       arvbox sv restart keepstore0
-      arvbox sv restart keepstore1
\ No newline at end of file
+      arvbox sv restart keepstore1
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
index 91bd22287..4b52223e0 100644
--- a/sdk/cwl/tests/federation/arvbox-fed.cwl
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -28,7 +28,18 @@ steps:
     in:
       container_name: containers
       arvbox_data: mkdir/arvbox_data
-    out: [cluster_id, container_ip]
+    out: [cluster_id, container_ip, arvbox_data]
     scatter: [container_name, arvbox_data]
     scatterMethod: dotproduct
     run: arvbox-start.cwl
+  fed-config:
+    in:
+      container_name: containers
+      this_cluster_id: start/cluster_id
+      cluster_ids: start/cluster_id
+      cluster_hosts: start/container_ip
+      arvbox_data: start/arvbox_data
+    out: []
+    scatter: [container_name, this_cluster_id, arvbox_data]
+    scatterMethod: dotproduct
+    run: arvbox-fed-config.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
index 2e12597ce..3485fba68 100644
--- a/sdk/cwl/tests/federation/arvbox-start.cwl
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -35,6 +35,10 @@ outputs:
           }
         }
         }
+  arvbox_data:
+    type: Directory
+    outputBinding:
+      outputEval: $(inputs.arvbox_data)
 requirements:
   EnvVarRequirement:
     envDef:

commit 07f3c5e1d1b8f329eca198b72bbe300f56b64c05
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Nov 5 17:46:41 2018 -0500

    14198: Federation integration testing WIP
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README
new file mode 100644
index 000000000..4da2bac0b
--- /dev/null
+++ b/sdk/cwl/tests/federation/README
@@ -0,0 +1,22 @@
+Things to test.
+
+Single step --submit --no-wait workflow, matrix of:
+
+Runner on home/remote cluster
+Docker image on home/remote cluster
+Step on home/remote cluster
+
+Two step workflow, matrix of:
+
+Step1 on home cluster -> Step2 on remote cluster
+Step1 on remote cluster -> Step2 on home cluster
+
+Three step workflow:
+
+(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster
+Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster
+
+Workflow which has a remote collection in InitialWorkDir, which is captured in output.
+
+Workflow with file input that has a secondary file in a separate
+collection, which is remote.
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
new file mode 100644
index 000000000..157fe4f72
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed-config.cwl
@@ -0,0 +1,42 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  container_name: string
+  this_cluster: string
+  cluster_ids: string[]
+  cluster_hosts: string[]
+  arvbox_base: Directory
+outputs: []
+requirements:
+  EnvVarRequirement:
+    envDef:
+      ARVBOX_CONTAINER: $(inputs.container_name)
+      ARVBOX_BASE: $(inputs.arvbox_base.path)
+  InitialWorkDirRequirement:
+    listing:
+      cluster_config.yml.override: |
+        ${
+        var remoteClusters = {};
+        for (var i = 0; i < cluster_ids.length; i++) {
+          remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+        }
+        return JSON.stringify({"Cluster": {inputs.this_cluster: {"RemoteClusters": remoteClusters}}});
+        }
+      application.yml.override: |
+        ${
+        var remoteClusters = {};
+        for (var i = 0; i < cluster_ids.length; i++) {
+          remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+        }
+        return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+        }
+  ShellCommandRequirement: {}
+arguments:
+  - shellQuote: false
+    valueFrom: |
+      docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+      docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
+      arvbox sv restart api
+      arvbox sv restart controller
+      arvbox sv restart keepstore0
+      arvbox sv restart keepstore1
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox-fed.cwl
new file mode 100644
index 000000000..91bd22287
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-fed.cwl
@@ -0,0 +1,34 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  ScatterFeatureRequirement: {}
+  cwltool:LoadListingRequirement:
+    loadListing: no_listing
+inputs:
+  containers: string[]
+  arvbox_base: Directory
+outputs:
+  cluster_ids:
+    type: string[]
+    outputSource: start/cluster_id
+  container_ips:
+    type: string[]
+    outputSource: start/container_ip
+steps:
+  mkdir:
+    in:
+      containers: containers
+      arvbox_base: arvbox_base
+    out: [arvbox_data]
+    run: arvbox-mkdir.cwl
+  start:
+    in:
+      container_name: containers
+      arvbox_data: mkdir/arvbox_data
+    out: [cluster_id, container_ip]
+    scatter: [container_name, arvbox_data]
+    scatterMethod: dotproduct
+    run: arvbox-start.cwl
diff --git a/sdk/cwl/tests/federation/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
new file mode 100644
index 000000000..b047beb9b
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-mkdir.cwl
@@ -0,0 +1,43 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+inputs:
+  containers:
+    type:
+      type: array
+      items: string
+      inputBinding:
+        position: 3
+        valueFrom: |
+          ${
+          return "base/"+self;
+          }
+  arvbox_base: Directory
+outputs:
+  arvbox_data:
+    type: Directory[]
+    outputBinding:
+      glob: |
+        ${
+        var r = [];
+        for (var i = 0; i < inputs.containers.length; i++) {
+          r.push("base/"+inputs.containers[i]);
+        }
+        return r;
+        }
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entry: $(inputs.arvbox_base)
+        entryname: base
+        writable: true
+  cwltool:LoadListingRequirement:
+    loadListing: no_listing
+  InlineJavascriptRequirement: {}
+  cwltool:InplaceUpdateRequirement:
+    inplaceUpdate: true
+arguments:
+  - mkdir
+  - "-p"
diff --git a/sdk/cwl/tests/federation/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox-start.cwl
new file mode 100644
index 000000000..2e12597ce
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-start.cwl
@@ -0,0 +1,55 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+inputs:
+  container_name: string
+  arvbox_data: Directory
+outputs:
+  cluster_id:
+    type: string
+    outputBinding:
+      glob: status.txt
+      loadContents: true
+      outputEval: |
+        ${
+        var sp = self[0].contents.split("\n");
+        for (var i = 0; i < sp.length; i++) {
+          if (sp[i].startsWith("Cluster id: ")) {
+            return sp[i].substr(12);
+          }
+        }
+        }
+  container_ip:
+    type: string
+    outputBinding:
+      glob: status.txt
+      loadContents: true
+      outputEval: |
+        ${
+        var sp = self[0].contents.split("\n");
+        for (var i = 0; i < sp.length; i++) {
+          if (sp[i].startsWith("Container IP: ")) {
+            return sp[i].substr(14);
+          }
+        }
+        }
+requirements:
+  EnvVarRequirement:
+    envDef:
+      ARVBOX_CONTAINER: $(inputs.container_name)
+      ARVBOX_DATA: $(inputs.arvbox_data.path)
+  ShellCommandRequirement: {}
+  InitialWorkDirRequirement:
+    listing:
+      - entry: $(inputs.arvbox_data)
+        entryname: $(inputs.container_name)
+        writable: true
+  cwltool:InplaceUpdateRequirement:
+    inplaceUpdate: true
+  InlineJavascriptRequirement: {}
+arguments:
+  - shellQuote: false
+    valueFrom: |
+      arvbox start dev && arvbox status > status.txt
diff --git a/sdk/cwl/tests/federation/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox-stop.cwl
new file mode 100644
index 000000000..c4a5412de
--- /dev/null
+++ b/sdk/cwl/tests/federation/arvbox-stop.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+inputs:
+  container_name: string
+outputs: []
+requirements:
+  EnvVarRequirement:
+    envDef:
+      ARVBOX_CONTAINER: $(inputs.container_name)
+arguments: [arvbox, stop]
\ No newline at end of file
diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl
new file mode 100755
index 000000000..2392b1149
--- /dev/null
+++ b/sdk/cwl/tests/federation/main.cwl
@@ -0,0 +1,22 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+hints:
+  cwltool:Secrets:
+    secrets: [arvados_api_token]
+inputs:
+  arvados_api_host_home: string
+  arvados_home_id: string
+  arvados_api_token: string
+  arvado_api_host_insecure:
+    type: bool
+    default: false
+  arvados_api_host_clusterB: string
+  arvados_clusterB_id: string
+  arvados_api_host_clusterC: string
+  arvados_clusterC_id: string
+
+outputs: []

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list