[ARVADOS] updated: 1.3.0-1731-g8e2b4c9cd

Git user git at public.curoverse.com
Tue Oct 8 19:56:59 UTC 2019


Summary of changes:
 apps/workbench/Gemfile                             |   3 -
 apps/workbench/Gemfile.lock                        |  22 +-
 .../app/views/projects/_show_dashboard.html.erb    |   2 +-
 .../test/controllers/projects_controller_test.rb   |  13 +-
 .../test/integration/application_layout_test.rb    |  17 +-
 apps/workbench/test/integration/logins_test.rb     |   2 +-
 .../test/integration/user_profile_test.rb          |   4 +-
 apps/workbench/test/integration/work_units_test.rb |   2 +-
 build/run-library.sh                               |  32 +-
 build/run-tests.sh                                 |  25 +-
 doc/_config.yml                                    |   3 +-
 doc/_includes/_assign_volume_uuid.liquid           |   7 +
 .../collection-versioning.html.textile.liquid      |   2 +-
 doc/admin/config-migration.html.textile.liquid     |  11 +-
 doc/admin/federation.html.textile.liquid           |   8 +-
 doc/admin/keep-balance.html.textile.liquid         |  43 ++
 doc/admin/upgrading.html.textile.liquid            |  29 +-
 doc/api/methods/containers.html.textile.liquid     |   4 +-
 doc/api/permission-model.html.textile.liquid       |   2 +-
 ...onfigure-azure-blob-storage.html.textile.liquid | 152 ++--
 .../configure-fs-storage.html.textile.liquid       | 117 +--
 ...configure-s3-object-storage.html.textile.liquid | 186 +++--
 .../install-keep-balance.html.textile.liquid       |  83 +--
 doc/install/install-keepstore.html.textile.liquid  | 202 ++----
 lib/config/cmd_test.go                             |  10 +-
 lib/config/config.default.yml                      | 189 ++++-
 lib/config/deprecated.go                           | 138 +++-
 lib/config/deprecated_keepstore.go                 | 688 ++++++++++++++++++
 lib/config/deprecated_keepstore_test.go            | 791 +++++++++++++++++++++
 lib/config/deprecated_test.go                      |  59 +-
 lib/config/export.go                               |  22 +
 lib/config/generated_config.go                     | 189 ++++-
 lib/config/load.go                                 |  19 +-
 lib/config/load_test.go                            |  45 +-
 lib/controller/cmd.go                              |   3 +-
 lib/controller/handler_test.go                     |   3 +-
 lib/dispatchcloud/cmd.go                           |   4 +-
 lib/dispatchcloud/dispatcher.go                    |  11 +-
 lib/dispatchcloud/dispatcher_test.go               |   2 +
 lib/service/cmd.go                                 |  91 ++-
 lib/service/cmd_test.go                            |  72 +-
 lib/service/tls.go                                 |  81 +++
 sdk/cli/arvados-cli.gemspec                        |   2 +
 sdk/cli/test/test_arv-collection-create.rb         |   4 +-
 sdk/cli/test/test_arv-get.rb                       |   2 +-
 sdk/cli/test/test_arv-keep-get.rb                  |  16 +-
 sdk/cli/test/test_arv-keep-put.rb                  |  14 +-
 sdk/cli/test/test_arv-tag.rb                       |   2 +-
 sdk/cli/test/test_arv-ws.rb                        |   4 +-
 sdk/cwl/arvados_cwl/arvworkflow.py                 |   6 +
 sdk/cwl/tests/arvados-tests.yml                    |   6 +
 .../tests/federation/arvbox-make-federation.cwl    |  10 +
 sdk/cwl/tests/federation/arvbox/fed-config.cwl     |   9 +-
 sdk/cwl/tests/federation/arvbox/start.cwl          |   9 +-
 sdk/cwl/tests/wf/runin-reqs-wf.cwl                 |   4 +-
 sdk/cwl/tests/wf/runin-reqs-wf2.cwl                |   4 +-
 sdk/cwl/tests/wf/runin-reqs-wf3.cwl                |   4 +-
 sdk/cwl/tests/wf/runin-reqs-wf4.cwl                |   4 +-
 .../wf/{runin-reqs-wf4.cwl => runin-reqs-wf5.cwl}  |   2 +-
 sdk/go/arvados/config.go                           |  78 +-
 sdk/go/arvados/keep_service.go                     |  25 +-
 sdk/go/arvadostest/run_servers.go                  |  10 +-
 sdk/go/ctxlog/log.go                               |   5 +-
 sdk/go/httpserver/httpserver.go                    |   7 +-
 sdk/go/httpserver/request_limiter.go               |  38 +-
 sdk/go/httpserver/request_limiter_test.go          |   4 +-
 sdk/python/arvados/commands/federation_migrate.py  |  11 +-
 .../tests/fed-migrate/arvbox-make-federation.cwl   |   5 +-
 sdk/python/tests/fed-migrate/fed-migrate.cwl       |  46 +-
 sdk/python/tests/fed-migrate/fed-migrate.cwlex     |  23 +-
 sdk/python/tests/fed-migrate/run-test.cwlex        |   3 +-
 sdk/python/tests/run_test_server.py                | 132 ++--
 sdk/python/tests/test_arv_put.py                   |  21 +-
 sdk/python/tests/test_keep_client.py               |  67 +-
 sdk/ruby/arvados.gemspec                           |   2 +
 .../arvados/v1/keep_services_controller.rb         |   7 +-
 services/api/app/models/container.rb               |   4 +
 services/api/app/models/keep_service.rb            |  58 ++
 .../arvados/v1/keep_services_controller_test.rb    |  29 +
 services/api/test/unit/container_test.rb           |   9 +
 services/arv-git-httpd/auth_handler_test.go        |   3 +-
 services/arv-git-httpd/git_handler_test.go         |   3 +-
 services/arv-git-httpd/gitolite_test.go            |   3 +-
 services/arv-git-httpd/integration_test.go         |   3 +-
 services/crunch-run/git_mount_test.go              |   3 +-
 services/fuse/tests/integration_test.py            |   2 +-
 services/fuse/tests/test_exec.py                   |   2 +-
 services/health/main.go                            |   3 +-
 services/keep-balance/balance.go                   |  36 +-
 services/keep-balance/balance_run_test.go          | 188 +++--
 services/keep-balance/balance_test.go              |  10 +-
 services/keep-balance/collection_test.go           |   2 +-
 services/keep-balance/integration_test.go          |  28 +-
 services/keep-balance/keep-balance.service         |   1 -
 services/keep-balance/main.go                      | 130 ++--
 services/keep-balance/main_test.go                 |  46 --
 services/keep-balance/metrics.go                   |   4 +-
 services/keep-balance/server.go                    | 140 +---
 services/keep-balance/usage.go                     | 106 ---
 services/keep-web/handler_test.go                  |   3 +-
 services/keep-web/server_test.go                   |   3 +-
 services/keepproxy/keepproxy_test.go               |   3 +-
 services/keepstore/azure_blob_volume.go            | 307 +++-----
 services/keepstore/azure_blob_volume_test.go       | 255 +++----
 services/keepstore/bufferpool.go                   |  15 +-
 services/keepstore/bufferpool_test.go              |  14 +-
 services/keepstore/command.go                      | 219 ++++++
 services/keepstore/command_test.go                 |  29 +
 services/keepstore/config.go                       | 226 ------
 services/keepstore/config_test.go                  |  14 -
 services/keepstore/deprecated.go                   |  47 --
 services/keepstore/handler_test.go                 | 592 +++++++--------
 services/keepstore/handlers.go                     | 176 +++--
 .../keepstore/handlers_with_generic_volume_test.go | 127 ----
 services/keepstore/keepstore.go                    | 199 +-----
 services/keepstore/keepstore.service               |   1 -
 services/keepstore/keepstore_test.go               | 456 ------------
 services/keepstore/metrics.go                      |  22 -
 services/keepstore/mounts_test.go                  |  64 +-
 services/keepstore/perms.go                        |  12 +-
 services/keepstore/perms_test.go                   |  46 +-
 services/keepstore/proxy_remote.go                 |  19 +-
 services/keepstore/proxy_remote_test.go            |  29 +-
 services/keepstore/pull_worker.go                  |  51 +-
 services/keepstore/pull_worker_integration_test.go |  74 +-
 services/keepstore/pull_worker_test.go             |  84 ++-
 services/keepstore/s3_volume.go                    | 490 +++++++------
 services/keepstore/s3_volume_test.go               | 177 +++--
 services/keepstore/server.go                       |  78 --
 services/keepstore/server_test.go                  |  47 --
 services/keepstore/status_test.go                  |   4 +-
 services/keepstore/trash_worker.go                 |  23 +-
 services/keepstore/trash_worker_test.go            | 125 ++--
 services/keepstore/unix_volume.go                  | 205 ++----
 services/keepstore/unix_volume_test.go             | 273 ++++---
 services/keepstore/usage.go                        | 162 -----
 services/keepstore/volume.go                       | 124 ++--
 services/keepstore/volume_generic_test.go          | 329 ++++-----
 services/keepstore/volume_test.go                  |  79 +-
 services/login-sync/Gemfile.lock                   |  11 +-
 services/login-sync/arvados-login-sync.gemspec     |   2 +
 services/ws/server_test.go                         |   7 +-
 tools/arvbox/bin/arvbox                            |   2 +
 tools/arvbox/lib/arvbox/docker/cluster-config.sh   |  25 +-
 tools/arvbox/lib/arvbox/docker/keep-setup.sh       |   9 +-
 .../lib/arvbox/docker/service/composer/run-service |   2 +-
 tools/arvbox/lib/arvbox/docker/service/nginx/run   |   7 +
 .../arvbox/docker/service/workbench2/run-service   |   2 +-
 148 files changed, 5371 insertions(+), 4686 deletions(-)
 create mode 100644 doc/_includes/_assign_volume_uuid.liquid
 create mode 100644 doc/admin/keep-balance.html.textile.liquid
 create mode 100644 lib/config/deprecated_keepstore.go
 create mode 100644 lib/config/deprecated_keepstore_test.go
 create mode 100644 lib/service/tls.go
 copy sdk/cwl/tests/wf/{runin-reqs-wf4.cwl => runin-reqs-wf5.cwl} (95%)
 delete mode 100644 services/keep-balance/main_test.go
 delete mode 100644 services/keep-balance/usage.go
 create mode 100644 services/keepstore/command.go
 create mode 100644 services/keepstore/command_test.go
 delete mode 100644 services/keepstore/config.go
 delete mode 100644 services/keepstore/config_test.go
 delete mode 100644 services/keepstore/deprecated.go
 delete mode 100644 services/keepstore/handlers_with_generic_volume_test.go
 delete mode 100644 services/keepstore/keepstore_test.go
 delete mode 100644 services/keepstore/server.go
 delete mode 100644 services/keepstore/server_test.go
 delete mode 100644 services/keepstore/usage.go

       via  8e2b4c9cd3ef1028056703e87131e633e6455c12 (commit)
       via  400210f8c9d8b111a3efdaa76c8be579ea5666cb (commit)
       via  6d67bbfdfc00a7a280f2d08e64b6198c5e4cba90 (commit)
       via  0c92112a718841ef7bfaa77ee8f207897ca04298 (commit)
       via  709141e7ce9c98560067398f34a36ca8669cf820 (commit)
       via  861599984cc86e100bd1c6dc36500a6f2334052d (commit)
       via  1c6464fc1b6e3cde61b64fc07308a3934966fe74 (commit)
       via  b2e1db4f3c44c6f41c37bfebf032526cf09bf9a6 (commit)
       via  80c1c0c533c7d7eedbef85805e157f0ab75c864b (commit)
       via  84c753c29346450bae7efd8f8bcd11aa7ea71109 (commit)
       via  5ffded80f04bc1b38574b0a4eee54c6ceb9b9112 (commit)
       via  b4be060bb0ee42281fc3a044c60d2b55c74475f8 (commit)
       via  fd38b59aac9b4453cf04fb7d6e1b8ad51549d6c3 (commit)
       via  30db189f713285d03facc83ac8018942a89afa81 (commit)
       via  4d7567b7b3577b561dd064d397c10a9331c3ee16 (commit)
       via  f1bcbdae5b6c24790b6b03823d04ecf4a675fb6e (commit)
       via  cf0c4c381a2b34130f072096038a430e3c6bbe55 (commit)
       via  ae60ced94e3ba9e80c994880886b11eefffd39c2 (commit)
       via  3911cd836c4e937262d48f9b0af703a9d7d68cdd (commit)
       via  f672f727fe79bf6642a2daab641a1ef5c84648df (commit)
       via  e86ee860d99036ff4ac61f6635b738f3a408e446 (commit)
       via  6125e8bc03e27d073535c2313743fba3c3e6b27d (commit)
       via  fbbba2116c046a3c20a71ea0268501a1a5b802e5 (commit)
       via  6f62ef31c63c6f04f9cb3fc772b93bead5a58345 (commit)
       via  7174c5c2d0ad4e450401045070c4726c1004de7e (commit)
       via  b174ae5f406a0254e017a9428ca3d3deda177b97 (commit)
       via  dc491966feeeeb4a136b1e4cfee64eab5ffe5f14 (commit)
       via  d22da8c4ae59fa50bfecaf3c5857fe8c0cc5fae7 (commit)
       via  f902c301c8c40707bb25f8dae01537f8a50be286 (commit)
       via  ca7f17d5aa220b2daea126fb58081d5d219216ea (commit)
       via  5612cb8542511ea96108604499b8b7e37e3804c2 (commit)
       via  fdd56a5f193c4d8c561059c74d3aa4e850a483d1 (commit)
       via  e8e461a99945a74968de8dd438e54649ce006216 (commit)
       via  25a7acb26323d159a60e43b037a19e255b6abdbd (commit)
       via  1d54582d7d80362602a017ebf9dbf79d2712b6f0 (commit)
       via  55e5a470d6430d2026b94892112be6d985bcef09 (commit)
       via  18b8c3b584a9337977df997624176b5a66b256e9 (commit)
       via  295aa2a6640f3694bedbefb3f728a00be98adae4 (commit)
       via  7d2f01a15a9e0ecf7bce74e1d271b6fb2993bbc3 (commit)
       via  4a919918a4ce37b5290793f02fa959db1c073590 (commit)
       via  ae9fc56d6bc8f73be3de765f03c9607cc5d21d98 (commit)
       via  a57fe1c6957883d973f87c6d99f169a7d5bea0ea (commit)
       via  cf3dc59e23bb9da48d67b1ff6e35932c17c64ea3 (commit)
       via  623a04c28cf042c8d0fbe3b3982d85d54e3a30ed (commit)
       via  3185e134320ff732e2bc3ef791133862aeb494ab (commit)
       via  a994f208a6ab576261721735e4283e7a01759d00 (commit)
       via  fa4a1e7741cf780656bf1dfa5a622e40a315c67a (commit)
       via  52b7b2934d5d74ee67ca13f8d1cc95f1379faddc (commit)
       via  8db3a110976bbb9288214eeb3064d31cb12c1c82 (commit)
       via  c8bfd534fc6e33b0b37f9fed1ee6232159edb631 (commit)
       via  fcb08f0ca7794b05c77a15ab8bd46b978cb07778 (commit)
       via  61bf0c73e5f8f1895912550a6031d8583a911e8d (commit)
       via  afaa86c118f4c08eef9f357610cf83bee62e5ac4 (commit)
       via  22cff6bb354d6980c33a0f471d5860f968e853a6 (commit)
       via  110580d079cdb0b0a773ecf1671c1f97f1736cc6 (commit)
       via  8a879fd6ffb38927150be85c63d926cd6a4c0d42 (commit)
       via  7c0257925a75185937c4a84dfd077458a24c53f9 (commit)
       via  915d1913a691d980c3362d14ad50d6f6dc3ff9af (commit)
       via  ae03057c03ca6b8543c93a4b3938ba06dad41571 (commit)
       via  0020a0bc96ee13203fbdc2af28ffa077799213d0 (commit)
       via  da79c28c17eb5de3196ab49718b86c3f73db2380 (commit)
       via  3b1947092b856e2c3bdb733828b1c951ac158b06 (commit)
       via  0bb53c1cbcbcb8b3be50e6ecf3fdf0bb7cbd96b5 (commit)
       via  6527c64378ebd1970be69f6502db181c8272da5a (commit)
       via  f5862345c56afa3ea3565db9e6e0edb365b9fa03 (commit)
       via  3cec8c10796bb6de9c02c96bbe7dd644c7366e42 (commit)
       via  608a1591840dfbeac3d5580c505b50bfface5929 (commit)
       via  18de39ed805bd1abd36ff4fcf83b5b130d321e74 (commit)
       via  28f250ee0a43e873760d43b39119e5710de75fe8 (commit)
       via  1b6b38c5d2e181f357cab883b63917cb5eaa778c (commit)
       via  3056c56352955911bc8e844bc1e655cec5abe242 (commit)
       via  faeaccc982ec3fc89263342e26e9bed4b0ee50ed (commit)
       via  1267366d66cadcc09972721bf7381f96f8fe73d6 (commit)
       via  a5b7b00aff9debab6607afb09b387de627bc2603 (commit)
       via  ce0df2b962b83137d97b75f594aa28fd694fb015 (commit)
       via  55dc07e524cf221c3e572fb86de01255be6c759c (commit)
       via  9dcec2ce4a077f14204fdfd6c4b1ec208ea281ab (commit)
       via  32d0b751985b6e8adef29a71c3e4542e87f7c54f (commit)
       via  0e0f69c6164339f4d0babb1d9d5a68fae24c01d5 (commit)
       via  453fe7852de9b01a10ace4ded7b3b7825326e599 (commit)
       via  fc86370e31e1bb3a5d7b234419e29015c165bc67 (commit)
       via  baa2bf80cc078868191494ccb1631c426f4e0496 (commit)
       via  cf5d136d81bd22ce5340243643a4734f3cf20856 (commit)
       via  07724cb2413ea22e583b9d579b514d81a97911be (commit)
       via  e201aabcc4d6d457a662ad0b4e51f3524b5bc865 (commit)
       via  ef2d67b9251839b44146f1df16f8cdd2d8faa38f (commit)
       via  3dec49afe102f61d6db045aa8dadec442d055b71 (commit)
       via  76ee36c49088274ab4e8465c0f4b20878d66c706 (commit)
       via  6653f96c23ff461bc4cadf5184a95e1c9142f7e6 (commit)
       via  11597243529c762114fc34907f772c331d6f6ef5 (commit)
       via  e5437560d2f30350370a1c96397716ac56a7398d (commit)
       via  d99713e58b0b9eb284fc3ea7f4008fa80bc5bcc5 (commit)
       via  a0ce99d4067878a7a32a9a2db44eb4bc0929d879 (commit)
       via  08d2f48c3cf3eac9d71261172677c54f03669fe5 (commit)
       via  04dd8d57e5ef90dd68876ffe5c72003645c6e3a2 (commit)
       via  c9a3f26fb35a6903178c3dbbc4dc3decb0ba2cf0 (commit)
       via  b144996ab73b15233f027713faf3114f34985e72 (commit)
       via  90d84d8578b760b493ac76b22c42bc284868bc0c (commit)
       via  20a69e126c68236a3c73daab101e351346f8bfaf (commit)
       via  f04693da1811e670d4cbb981debeecf14d79137c (commit)
      from  a798952d64584cdcec2a13dc0df434692d2b521e (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8e2b4c9cd3ef1028056703e87131e633e6455c12
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Oct 8 15:22:04 2019 -0400

    15531: Moved federation section in admin guide
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/doc/_config.yml b/doc/_config.yml
index 0ae26584e..344456d1f 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -172,8 +172,6 @@ navbar:
       - admin/collection-versioning.html.textile.liquid
       - admin/collection-managed-properties.html.textile.liquid
       - admin/keep-balance.html.textile.liquid
-    - Other:
-      - admin/federation.html.textile.liquid
       - admin/controlling-container-reuse.html.textile.liquid
       - admin/logs-table-management.html.textile.liquid
     - Other:

commit 400210f8c9d8b111a3efdaa76c8be579ea5666cb
Merge: 6d67bbfdf 0c92112a7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Oct 8 15:21:32 2019 -0400

    Merge branch 'master' into 15531-logincluster-migrate
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --cc doc/_config.yml
index 314b5b5cd,c4fad997f..0ae26584e
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@@ -150,8 -150,6 +150,7 @@@ navbar
        - admin/index.html.textile.liquid
      - Configuration:
        - admin/config.html.textile.liquid
 +      - admin/federation.html.textile.liquid
-       - admin/collection-managed-properties.html.textile.liquid
      - Upgrading and migrations:
        - admin/upgrading.html.textile.liquid
        - admin/config-migration.html.textile.liquid
@@@ -171,12 -171,13 +170,16 @@@
        - admin/cloudtest.html.textile.liquid
      - Data Management:
        - admin/collection-versioning.html.textile.liquid
+       - admin/collection-managed-properties.html.textile.liquid
+       - admin/keep-balance.html.textile.liquid
+     - Other:
+       - admin/federation.html.textile.liquid
        - admin/controlling-container-reuse.html.textile.liquid
        - admin/logs-table-management.html.textile.liquid
 +    - Other:
        - admin/troubleshooting.html.textile.liquid
 +      - install/migrate-docker19.html.textile.liquid
 +      - admin/upgrade-crunch2.html.textile.liquid
    installguide:
      - Overview:
        - install/index.html.textile.liquid

commit 6d67bbfdfc00a7a280f2d08e64b6198c5e4cba90
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Tue Oct 8 15:16:47 2019 -0400

    15531: Fixing up test framework
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/doc/admin/federation.html.textile.liquid b/doc/admin/federation.html.textile.liquid
index 85258955b..b1f1506e4 100644
--- a/doc/admin/federation.html.textile.liquid
+++ b/doc/admin/federation.html.textile.liquid
@@ -44,7 +44,7 @@ A federation of clusters can be configured to use a separate user database per c
 
 If clusters belong to separate organizations, each cluster will have its own user database for the members of that organization.  Through federation, a user from one organization can be granted access to the cluster of another organization.  The admin of the seond cluster controls access on a individual basis by choosing to activate or deactivate accounts from other organizations (with the default policy the value of  @ActivateUsers@).
 
-On the other hand, if all clusters belong to the same organization, and users in that organization should have access to all the clusters, user management can be simplified by setting the @LoginCluster@ which manages the user database used by all other clusters in the federation.  The @LoginCluster@ configuration should be set for all clusters in the federation to the id of one cluster in the federation which will be the 'master cluster'.  This directs all logins to the LoginCluster, and the LoginCluster will issue API tokens which are valid on any cluster in the federation.  Users are activated or deactivated across the entire federation based on their status on the master cluster.
+On the other hand, if all clusters belong to the same organization, and users in that organization should have access to all the clusters, user management can be simplified by setting the @LoginCluster@ which manages the user database used by all other clusters in the federation.  To do this, choose one cluster in the federation which will be the 'login cluster'.  Set the the @Login.LoginCluster@ configuration value on all clusters in the federation to the cluster id of the login cluster.  After setting @LoginCluster@, restart arvados-api-server and arvados-controller.
 
 <pre>
 Clusters:
@@ -53,7 +53,11 @@ Clusters:
       LoginCluster: clsr1
 </pre>
 
-To migrate users of an existing federation with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
+The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which are valid on any cluster in the federation.  Users are activated or deactivated across the entire federation based on their status on the master cluster.
+
+Note: tokens issued by the master cluster need to be periodically re-validated when used on other clusters in the federation.  The period between revalidation attempts is configured with @Login.RemoteTokenRefresh at .  The default is 5 minutes.  A longer period reduces overhead from validating tokens, but means it will take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
+
+To migrate users of existing clusters with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
 
 h2. Testing
 
diff --git a/sdk/cwl/tests/federation/arvbox-make-federation.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 341ce1228..81b542057 100644
--- a/sdk/cwl/tests/federation/arvbox-make-federation.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -22,6 +22,11 @@ inputs:
   insecure:
     type: boolean
     default: true
+  arvbox:
+    type: File
+    default:
+      class: File
+      location: ../../../../tools/arvbox/bin/arvbox
 outputs:
   arvados_api_token:
     type: string
@@ -44,6 +49,9 @@ outputs:
   arvbox_containers:
     type: string[]
     outputSource: containers
+  arvbox_bin:
+    type: File
+    outputSource: arvbox
 steps:
   mkdir:
     in:
@@ -55,6 +63,7 @@ steps:
     in:
       container_name: containers
       arvbox_data: mkdir/arvbox_data
+      arvbox_bin: arvbox
     out: [cluster_id, container_host, arvbox_data_out, superuser_token]
     scatter: [container_name, arvbox_data]
     scatterMethod: dotproduct
@@ -66,6 +75,7 @@ steps:
       cluster_ids: start/cluster_id
       cluster_hosts: start/container_host
       arvbox_data: start/arvbox_data_out
+      arvbox_bin: arvbox
     out: []
     scatter: [container_name, this_cluster_id, arvbox_data]
     scatterMethod: dotproduct
diff --git a/sdk/cwl/tests/federation/arvbox/fed-config.cwl b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
index 77567ee89..76523a56b 100644
--- a/sdk/cwl/tests/federation/arvbox/fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvbox/fed-config.cwl
@@ -13,6 +13,7 @@ inputs:
   cluster_ids: string[]
   cluster_hosts: string[]
   arvbox_data: Directory
+  arvbox_bin: File
 outputs:
   arvbox_data_out:
     type: Directory
@@ -60,7 +61,7 @@ arguments:
     valueFrom: |
       docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
       docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
-      arvbox sv restart api
-      arvbox sv restart controller
-      arvbox sv restart keepstore0
-      arvbox sv restart keepstore1
+      $(inputs.arvbox_bin.path) sv restart api
+      $(inputs.arvbox_bin.path) sv restart controller
+      $(inputs.arvbox_bin.path) sv restart keepstore0
+      $(inputs.arvbox_bin.path) sv restart keepstore1
diff --git a/sdk/cwl/tests/federation/arvbox/start.cwl b/sdk/cwl/tests/federation/arvbox/start.cwl
index f69775a53..a0b3e1864 100644
--- a/sdk/cwl/tests/federation/arvbox/start.cwl
+++ b/sdk/cwl/tests/federation/arvbox/start.cwl
@@ -10,6 +10,7 @@ $namespaces:
 inputs:
   container_name: string
   arvbox_data: Directory
+  arvbox_bin: File
 outputs:
   cluster_id:
     type: string
@@ -66,7 +67,7 @@ requirements:
 arguments:
   - shellQuote: false
     valueFrom: |
-      set -e
-      arvbox start dev
-      arvbox status > status.txt
-      arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
+      set -ex
+      $(inputs.arvbox_bin.path) start dev
+      $(inputs.arvbox_bin.path) status > status.txt
+      $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index a7d9414d9..e533a642c 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -3,6 +3,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+#
+# Migration tool for merging user accounts belonging to the same user
+# but on separate clusters to use a single user account managed by a
+# specific cluster.
+#
+# If you're working on this, see
+# arvados/sdk/python/tests/fed-migrate/README for information about
+# the testing infrastructure.
+
 import arvados
 import arvados.util
 import arvados.errors
@@ -67,7 +76,7 @@ def connect_clusters(args):
             continue
 
         if not cur["is_admin"]:
-            errors.append("Not admin of %s" % host)
+            errors.append("User %s is not admin on %s" % (cur["uuid"], arv._rootDesc["uuidPrefix"]))
             continue
 
         for r in clusters:
diff --git a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
index c3fcbdcb3..5057d4cb1 100644
--- a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
+++ b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
@@ -18,6 +18,9 @@ outputs:
   arvbox_containers:
     type: string[]
     outputSource: start/arvbox_containers
+  arvbox_bin:
+    type: File
+    outputSource: start/arvbox_bin
 requirements:
   SubworkflowFeatureRequirement: {}
   cwltool:LoadListingRequirement:
@@ -26,5 +29,5 @@ steps:
   start:
     in:
       arvbox_base: arvbox_base
-    out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers]
+    out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers, arvbox_bin]
     run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl
index 313946dd3..cb686847e 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwl
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl
@@ -131,6 +131,7 @@ $graph:
             - class: EnvVarRequirement
               envDef:
                 ARVADOS_API_HOST: $(inputs.host)
+                ARVADOS_API_HOST_INSECURE: '1'
                 ARVADOS_API_TOKEN: $(inputs.token)
           steps:
             - id: main_2_embed_1
@@ -334,6 +335,11 @@ $graph:
       - default: arv-federation-migrate
         id: fed_migrate
         type: string
+      - id: arvbox_bin
+        type: File
+      - default: 15531-logincluster-migrate
+        id: refspec
+        type: string
     outputs:
       - id: supertok
         outputSource: main_2/supertok
@@ -369,6 +375,8 @@ $graph:
               type: string
       - id: main_2
         in:
+          arvbox_bin:
+            source: arvbox_bin
           cluster_id:
             source: arvados_cluster_ids
           container:
@@ -377,6 +385,8 @@ $graph:
             source: arvados_api_hosts
           logincluster:
             source: main_1/logincluster
+          refspec:
+            source: refspec
         out:
           - supertok
         run:
@@ -391,6 +401,10 @@ $graph:
               type: string
             - id: logincluster
               type: string
+            - id: arvbox_bin
+              type: File
+            - id: refspec
+              type: string
           outputs:
             - id: supertok
               outputSource: superuser_tok_3/superuser_token
@@ -456,12 +470,16 @@ $graph:
                   InlineJavascriptRequirement: {}
             - id: main_2_embed_2
               in:
+                arvbox_bin:
+                  source: arvbox_bin
                 c:
                   source: main_2_embed_1/c
                 container:
                   source: container
                 host:
                   source: host
+                refspec:
+                  source: refspec
               out:
                 - d
               run:
@@ -475,8 +493,12 @@ $graph:
                     type: string
                   - id: host
                     type: string
+                  - id: arvbox_bin
+                    type: File
                   - id: c
                     type: string
+                  - id: refspec
+                    type: string
                 outputs:
                   - id: d
                     outputBinding:
@@ -486,9 +508,21 @@ $graph:
                   InitialWorkDirRequirement:
                     listing:
                       - entry: >
-                          set -x
+                          set -xe
+
+                          $(inputs.arvbox_bin.path) pipe <<EOF
+
+                          cd /usr/src/arvados
+
+                          git fetch
+
+                          git checkout -f $(inputs.refspec)
+
+                          EOF
+
+
+                          $(inputs.arvbox_bin.path) hotreset
 
-                          arvbox hotreset
 
                           while ! curl --fail --insecure --silent
                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
@@ -496,13 +530,13 @@ $graph:
 
                           export ARVADOS_API_HOST=$(inputs.host)
 
-                          export ARVADOS_API_TOKEN=\$(arvbox cat
-                          /var/lib/arvados/superuser_token)
+                          export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path)
+                          cat /var/lib/arvados/superuser_token)
 
                           export ARVADOS_API_HOST_INSECURE=1
 
-                          ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
-                          /var/lib/arvados/vm-uuid)
+                          ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
+                          cat /var/lib/arvados/vm-uuid)
 
                           while ! python -c "import arvados ;
                           arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
index c39093807..2d1325016 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwlex
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
@@ -6,7 +6,9 @@ def workflow main(
   arvados_cluster_ids string[],
   superuser_tokens string[],
   arvbox_containers string[],
-  fed_migrate="arv-federation-migrate"
+  fed_migrate="arv-federation-migrate",
+  arvbox_bin File,
+  refspec="15531-logincluster-migrate"
 ) {
 
   logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
@@ -14,7 +16,7 @@ def workflow main(
   scatter arvbox_containers as container,
           arvados_cluster_ids as cluster_id,
 	  arvados_api_hosts as host
-    do run workflow(logincluster)
+    do run workflow(logincluster, arvbox_bin, refspec)
   {
     requirements {
       EnvVarRequirement {
@@ -34,15 +36,22 @@ docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
 >>>
       return container as c
     }
-    run tool(container, host, c) {
+    run tool(container, host, arvbox_bin, c, refspec) {
 sh <<<
-set -x
-arvbox hotreset
+set -xe
+$(inputs.arvbox_bin.path) pipe <<EOF
+cd /usr/src/arvados
+git fetch
+git checkout -f $(inputs.refspec)
+EOF
+
+$(inputs.arvbox_bin.path) hotreset
+
 while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
 export ARVADOS_API_HOST=$(inputs.host)
-export ARVADOS_API_TOKEN=\$(arvbox cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
 export ARVADOS_API_HOST_INSECURE=1
-ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat /var/lib/arvados/vm-uuid)
+ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid)
 while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
 >>>
       return c as d
diff --git a/sdk/python/tests/fed-migrate/run-test.cwlex b/sdk/python/tests/fed-migrate/run-test.cwlex
index ef37c5152..55ac6a740 100644
--- a/sdk/python/tests/fed-migrate/run-test.cwlex
+++ b/sdk/python/tests/fed-migrate/run-test.cwlex
@@ -22,7 +22,8 @@ def workflow main(
       EnvVarRequirement {
         envDef: {
           ARVADOS_API_HOST: "$(inputs.host)",
-          ARVADOS_API_TOKEN: "$(inputs.token)"
+          ARVADOS_API_TOKEN: "$(inputs.token)",
+	  ARVADOS_API_HOST_INSECURE: "1"
 	}
       }
     }
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index bdbd5fa23..246229dec 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -19,11 +19,13 @@ if ! which docker >/dev/null 2>/dev/null ; then
 fi
 
 if test -z "$ARVBOX_DOCKER" ; then
+    set +e
     if which greadlink >/dev/null 2>/dev/null ; then
         ARVBOX_DOCKER=$(greadlink -f $(dirname $0)/../lib/arvbox/docker)
     else
         ARVBOX_DOCKER=$(readlink -f $(dirname $0)/../lib/arvbox/docker)
     fi
+    set -e
 fi
 
 if test -z "$ARVBOX_CONTAINER" ; then

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list