[arvados] updated: 2.7.0-5387-g3fa9678a0a
git repository hosting
git at public.arvados.org
Mon Nov 20 19:37:12 UTC 2023
Summary of changes:
build/rails-package-scripts/arvados-api-server.sh | 2 +-
build/rails-package-scripts/postinst.sh | 7 +-
build/run-tests.sh | 18 +-
cmd/arvados-client/cmd_test.go | 5 +-
cmd/arvados-package/cmd.go | 2 +-
cmd/arvados-server/arvados-controller.service | 1 +
cmd/arvados-server/arvados-dispatch-cloud.service | 1 +
cmd/arvados-server/arvados-dispatch-lsf.service | 1 +
cmd/arvados-server/arvados-git-httpd.service | 1 +
cmd/arvados-server/arvados-health.service | 1 +
cmd/arvados-server/arvados-ws.service | 1 +
cmd/arvados-server/crunch-dispatch-slurm.service | 1 +
cmd/arvados-server/keep-balance.service | 3 +-
cmd/arvados-server/keep-web.service | 1 +
cmd/arvados-server/keepproxy.service | 1 +
cmd/arvados-server/keepstore.service | 1 +
doc/Gemfile.lock | 23 +-
doc/_includes/_install_ruby_and_bundler.liquid | 10 +-
doc/admin/keep-balance.html.textile.liquid | 6 +-
doc/admin/keep-faster-gc-s3.html.textile.liquid | 7 +-
doc/admin/upgrading.html.textile.liquid | 6 +
.../install-keep-balance.html.textile.liquid | 2 +-
lib/boot/cmd.go | 2 +-
lib/boot/supervisor.go | 5 +-
lib/boot/workbench2.go | 27 ++-
lib/cli/get.go | 4 +-
lib/cloud/ec2/ec2.go | 33 ++-
lib/cloud/ec2/ec2_test.go | 51 ++++-
lib/cmd/cmd.go | 26 ++-
lib/cmd/parseflags.go | 4 +-
lib/config/cmd_test.go | 2 +-
lib/config/config.default.yml | 20 ++
lib/config/export.go | 3 +
lib/controller/federation/conn.go | 64 ++++--
lib/controller/federation/login_test.go | 39 ----
lib/controller/federation/logout_test.go | 246 ++++++++++++++++++++
lib/crunchrun/executor_test.go | 4 +
lib/dispatchcloud/container/queue.go | 23 +-
lib/dispatchcloud/container/queue_test.go | 11 +-
lib/dispatchcloud/dispatcher.go | 2 +-
lib/dispatchcloud/dispatcher_test.go | 15 +-
lib/dispatchcloud/node_size.go | 65 ++++--
lib/dispatchcloud/node_size_test.go | 68 ++++--
lib/dispatchcloud/scheduler/run_queue.go | 111 +++++----
lib/dispatchcloud/scheduler/run_queue_test.go | 6 +-
lib/dispatchcloud/test/queue.go | 10 +-
lib/dispatchcloud/test/stub_driver.go | 16 +-
lib/install/deps.go | 249 +++++++++++++--------
sdk/cwl/arvados_cwl/runner.py | 5 +-
sdk/cwl/setup.py | 1 -
sdk/go/arvados/config.go | 3 +
sdk/python/arvados/commands/keepdocker.py | 5 +-
sdk/python/tests/run_test_server.py | 22 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 8 +-
services/fuse/arvados_fuse/unmount.py | 10 +
services/keep-balance/balance.go | 47 ++--
services/keep-balance/balance_run_test.go | 45 ++--
services/keep-balance/balance_test.go | 12 +-
services/keep-balance/change_set.go | 25 ++-
services/keep-balance/integration_test.go | 3 -
services/keep-balance/main.go | 15 +-
services/keep-balance/server.go | 6 +-
services/workbench2/Makefile | 44 +++-
.../workbench2/cypress/integration/login.spec.js | 19 +-
.../cypress/integration/page-not-found.spec.js | 5 +-
.../workbench2/cypress/integration/project.spec.js | 2 +-
services/workbench2/cypress/support/commands.js | 2 +-
services/workbench2/docker/Dockerfile | 3 +-
.../collection-panel-files.tsx | 2 +
services/workbench2/src/index.tsx | 17 +-
.../collection-service/collection-service.ts | 11 +-
.../process-logs-panel-actions.ts | 48 ++--
.../store/process-panel/process-panel-actions.ts | 8 +-
.../project-panel-middleware-service.ts | 7 +-
.../workbench2/src/store/trash/trash-actions.ts | 174 +++++++-------
.../src/store/workbench/workbench-actions.ts | 6 +
.../views/collection-panel/collection-panel.tsx | 7 +-
.../src/views/not-found-panel/not-found-panel.tsx | 27 +++
.../src/views/process-panel/process-panel-root.tsx | 18 +-
.../src/views/project-panel/project-panel.tsx | 25 ++-
.../workflow-panel/registered-workflow-panel.tsx | 9 +-
services/workbench2/tools/arvados_config.yml | 6 +-
services/workbench2/tools/run-integration-tests.sh | 23 +-
83 files changed, 1282 insertions(+), 595 deletions(-)
create mode 100644 lib/controller/federation/logout_test.go
via 3fa9678a0a6caef757209f8666de66fb0896c953 (commit)
via c9f0e701ab84ed9b322cb242f32287d765335fe7 (commit)
via b927ca5fe5083a0bd170427c919774ab32cf50f7 (commit)
via 506bd0a728728c055635ca01d7e6d8d2d6f263df (commit)
via 9dc967877cb3599ce3584aca57e8c3672f94bd0a (commit)
via b6b7a5d8948a3749dabe1a983368065dd21e4599 (commit)
via 0e131fba4895c95b4829c3c66da62f5bd1111832 (commit)
via c1bb2409cf5d6d062b5e549f43124ba4ce6e6008 (commit)
via e98eb84cc8f860bfad29b2b79e72603fccad673c (commit)
via 1f7aeab7038ed8e2aa0f824185b392c8c13e6623 (commit)
via 9fa5faed898bf23fcea8f4e7946e540473e42e08 (commit)
via dc6fb79ab1c3ba1aaa0e33261dea132ae40cb981 (commit)
via 5f778e6945d8c3c216922f9aa1f45253a1c62376 (commit)
via 6f6a06e13d12389ba045bc6694cb83b97f714e78 (commit)
via 46fe1e60a1cd96a39163911edd821b3e316ca606 (commit)
via 74c506146023934d0f052e2bd4971d143b41404d (commit)
via 434508e84040487628d0e3efaa9ef86a85719ea7 (commit)
via 7bc4217495b405c98ea4cad8178b8136383486cf (commit)
via 3bea410bc6f2bde352cb351c9065ee1c46f1b631 (commit)
via b2ef730e0f524bc8e08fbc241c80c170d9f27763 (commit)
via f5fd7e2dc13b912e1032a54119f3fb60973f77e2 (commit)
via 5f3520a80c66558c2a6b1a48ebc0eedb4d5e8f96 (commit)
via b9b2513a621b6704157a1078e3b2ec75b3c92d1c (commit)
via d3cb0cad7f052987651a9f0b2237329cf3997c58 (commit)
via 91d9e964f9769347044ebaac556cd56360d96d2a (commit)
via 50bccff93109fe6ef9c370d2a858049d8b2e22cd (commit)
via c68e4ba51336a871dff26ae9f8dc7eb7e316083d (commit)
via 3236e0cbbe0fcb9e684d63a0ffac4d1f4a0d96b8 (commit)
via ae4cf50d72adebc6df338ef54ec5f2755ad73b8f (commit)
via 4f06e4dc7e9eb5ad04703dbbaa47370e6baeb9a1 (commit)
via 3433f306caf560017377b32adf7a23842ba9ab31 (commit)
via 6ff39245b93dd55e60c7433af64f2244b0091c6e (commit)
via d087ef17a59688c8e38c3c72192f295d8bca28c2 (commit)
via 10b08e0c05d176775c4f0cf13f6e77025bb1e636 (commit)
via f4fdee016f05ca986c55c7d769ce2f1a6bec5541 (commit)
via a5ce6f3013a9924c40c71b04940759e127c98858 (commit)
via d6e41e6884090f5ff42e9fa5bb53852c7bcc47c4 (commit)
via 28b83594452b195cc1e8b412bdd3a3bb2a91e2d9 (commit)
via f3b8d03f7063b162355bccfd71aeb2b8b67bbdbb (commit)
via 03caa5fccf60f32c4fe1d6aac297c64ade6a69b9 (commit)
via e14e00a5fb79be5cba5c8b9b0ae5863011f60daf (commit)
via 2454b8583c13b754c233b0e44cdbc95873cd14cc (commit)
via dc524f398ed78c9820ef90b87d2ec4422d4f9132 (commit)
via e8d36edca6d1601a50f3712bc7d1c7258a23fcc7 (commit)
via 5e2a9f3aeb1e72b9776f3a9974f8f4bffde83f8a (commit)
via d719195a223a00b1f7706f4f734e0cd59618e54e (commit)
via a53bf7d89e01e98dcdc62b5357a0404d5489c9a1 (commit)
via b34c4b9234777d68b675aebf77680b8dd8708a6d (commit)
via ac312e0acae4fd114114081c9f4791d05e640831 (commit)
via 4ad6191d53207a8b2d4c0c8a30b18119daaa5fbc (commit)
via b77707a7b06d57145a7829458d476baf8573317e (commit)
via 814cc4a5534ec1fc47dd4425d6577f1aa0aaa01a (commit)
via 7a391142ec0c692e56f6aa63114d15ac995e4608 (commit)
via a369357c88259a337536d6586ee25bda83e3a8b4 (commit)
via 4ad36ef0580a438f10c0f76a99c7502f5ed4473b (commit)
via bf31106bd4a74dd41cae846e1d51e02e033e9a90 (commit)
via d4a4a96d0690c3527acb71a624c58e696249f879 (commit)
via 04f7b48fcae392b3f2d3ff1adfd822a18fb9b951 (commit)
via 9031b369f5b31d6ea44573c9539b5bd6aa153cd5 (commit)
via 486623f4ded7e95e085a590e2b5e6cb988b1ad7c (commit)
via 5bc07be644dfaa923df770f0411735302cb68ddf (commit)
via 9a9a55d66bbc28b460c8ffdf1973fd567e1ea6c3 (commit)
via 889be1993b1c270b361f3502f33ead1dae95c9a2 (commit)
via 0e9f41a15027a66c69ef351846b213b2e2075642 (commit)
via d6446b03e2f5d5079a870bdd7b963456dc12b485 (commit)
via a73295e64f58fe027b995e0cca3d103d4e2289ff (commit)
via 88aedea4fdf827524c620830ec11681e5cd5f527 (commit)
via 66cee5a8021e73271650e0997ca7f757e419d169 (commit)
via da204dccd3df12b5c885068768f95c84e4703047 (commit)
via 1028c0630dac2a2bff363da1390bbf942e7fe7ae (commit)
via e3ac3dafe5432c81a294403e1996c306a93d48f8 (commit)
via 1875af9bcf4a1afe435176e952e63341a9ae9c03 (commit)
via 8e913832ad289fd28700f4081046b7de48688130 (commit)
via 5b30fab0552f623a35ff5b89b90dab82dfcee258 (commit)
via 26e2f985f4c2b511c59977e4c63158649f296624 (commit)
via e11c6c520d8d01397f58a5cda1fe7e5e5e06acab (commit)
via 75b017ae9d566d523e448aaeb863c4d89b3548fe (commit)
from 357858970aef82a2a87b5105d133a52196987381 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3fa9678a0a6caef757209f8666de66fb0896c953
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 20 14:37:02 2023 -0500
21126: Update keep-balance admin doc page.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/keep-balance.html.textile.liquid b/doc/admin/keep-balance.html.textile.liquid
index 2785930de8..4d18307cf7 100644
--- a/doc/admin/keep-balance.html.textile.liquid
+++ b/doc/admin/keep-balance.html.textile.liquid
@@ -30,14 +30,12 @@ The @Collections.BalancePeriod@ value in @/etc/arvados/config.yml@ determines th
Keep-balance can also be run with the @-once@ flag to do a single scan/balance operation and then exit. The exit code will be zero if the operation was successful.
-h3. Committing
-
-Keep-balance computes and reports changes but does not implement them by sending pull and trash lists to the Keep services unless the @-commit-pull@ and @-commit-trash@ flags are used.
-
h3. Additional configuration
For configuring resource usage tuning and lost block reporting, please see the @Collections.BlobMissingReport@, @Collections.BalanceCollectionBatch@, @Collections.BalanceCollectionBuffers@ option in the "default config.yml file":{{site.baseurl}}/admin/config.html.
+The @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries determine the maximum number of pull and trash operations keep-balance will attempt to apply on each keepstore server. If both values are zero, keep-balance will operate in "dry run" mode, where all changes are computed but none are committed.
+
h3. Limitations
Keep-balance does not attempt to discover whether committed pull and trash requests ever get carried out -- only that they are accepted by the Keep services. If some services are full, new copies of under-replicated blocks might never get made, only repeatedly requested.
diff --git a/doc/install/install-keep-balance.html.textile.liquid b/doc/install/install-keep-balance.html.textile.liquid
index bb4ae7b3d8..05d27b7cb4 100644
--- a/doc/install/install-keep-balance.html.textile.liquid
+++ b/doc/install/install-keep-balance.html.textile.liquid
@@ -24,7 +24,7 @@ Keep-balance can be installed anywhere with network access to Keep services, arv
{% include 'notebox_begin' %}
-If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, edit your keep-balance startup script to use the flags @-commit-pulls=false -commit-trash=false -commit-confirmed-fields=false at .
+If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, set the @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries to zero.
{% include 'notebox_end' %}
commit c9f0e701ab84ed9b322cb242f32287d765335fe7
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 20 14:25:09 2023 -0500
21126: Update faster-gc doc page with new keep-balance configs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/keep-faster-gc-s3.html.textile.liquid b/doc/admin/keep-faster-gc-s3.html.textile.liquid
index 872a772d2d..7f7cdd921f 100644
--- a/doc/admin/keep-faster-gc-s3.html.textile.liquid
+++ b/doc/admin/keep-faster-gc-s3.html.textile.liquid
@@ -19,12 +19,11 @@ The following steps can be used to temporarily disable writes on an S3 bucket to
# Shut down the @keep-balance@ service.
# Update the target volume configuration with @ReadOnly: true@, @AllowTrashWhileReadOnly: true@, and @UnsafeDelete: true at .
# In the @Collections@ configuration section, set @BlobTrashLifetime: 0 at . Note this will effectively disable garbage collection on other S3-backed volumes, and garbage collection will delete blocks outright (bypassing the recoverable trash phase) on any non-S3 volumes.
+# In the @Collections@ configuration section, set @BalancePullLimit: 0 at .
# Restart all @keepstore@ services with the updated configuration.
-# Run @keep-balance@ with command line options @-commit-pulls=false -commit-trash=true at . For example, use @sudo systemctl edit keep-balance@ to override the default command line: <notextile><pre><code>[Service]
-ExecStart=
-ExecStart=/usr/bin/keep-balance -commit-pulls=false -commit-trash=true</code></pre></notextile> ...and then start the service using @sudo systemctl start keep-balance at . Note that if you do not use @-commit-pulls=false@ here, keep-balance will also start copying data from the target volume to other volumes, which can slow down garbage collection progress.
+# Start the @keep-balance@ service.
# Garbage collection should now proceed faster on the target volume. Monitor progress by watching @keep-balance@ logs and metrics. When garbage collection is complete, keep-balance logs will show an empty changeset: <notextile><pre><code>zzzzz-bi6l4-0123456789abcdef (keep0.zzzzz.arvadosapi.com:25107, disk): ChangeSet{Pulls:0, Trashes:0}</code></pre></notextile>
# Revert @BlobTrashLifetime@ to its previous value (or the default value, @336h@) and set @UnsafeDelete: false@ on the target volume.
# Optionally, revert to @ReadOnly: false@ and @AllowTrashWhileReadOnly: false@ on the target volume and/or set @ReadOnly: true@ and @AllowTrashWhileReadOnly: true@ on the new volume, depending on where you intend to store new data in future. If the target volume has @PrefixLength: 0@ and the new volume has @PrefixLength: 3@, you can skip this step: new data will be stored on the new volume, some existing data will be moved automatically to other volumes, and some will be left on the target volume as long as it's needed.
-# Revert the @-commit-pulls=false@ change to the @keep-balance@ startup script, and restart @keep-balance at .
+# Remove the @BalancePullLimit@ configuration entry (or restore its previous value), and restart @keep-balance at .
# Restart all @keepstore@ services with the updated configuration.
commit b927ca5fe5083a0bd170427c919774ab32cf50f7
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 20 14:24:59 2023 -0500
21126: Update doc bundle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/Gemfile.lock b/doc/Gemfile.lock
index 420e13146f..c2ef90d878 100644
--- a/doc/Gemfile.lock
+++ b/doc/Gemfile.lock
@@ -1,20 +1,29 @@
GEM
remote: https://rubygems.org/
specs:
- RedCloth (4.3.2)
+ RedCloth (4.3.3)
coderay (1.1.3)
- colorize (1.0.5)
+ colorize (1.1.0)
commonjs (0.2.7)
- kramdown (1.17.0)
+ kramdown (2.4.0)
+ rexml
+ kramdown-parser-gfm (1.1.0)
+ kramdown (~> 2.0)
+ kramdown-syntax-coderay (1.0.1)
+ coderay (~> 1.1)
+ kramdown (~> 2.0)
less (2.6.0)
commonjs (~> 0.2.7)
liquid (4.0.4)
makerakeworkwell (1.0.4)
rake (>= 0.9.2, < 15)
- rake (13.0.6)
- zenweb (3.10.6)
+ rake (13.1.0)
+ rexml (3.2.6)
+ zenweb (3.11.0)
coderay (~> 1.0)
- kramdown (~> 1.4)
+ kramdown (~> 2.0)
+ kramdown-parser-gfm (~> 1.0)
+ kramdown-syntax-coderay (~> 1.0)
less (~> 2.0)
makerakeworkwell (~> 1.0)
rake (>= 0.9, < 15)
@@ -29,4 +38,4 @@ DEPENDENCIES
zenweb
BUNDLED WITH
- 2.1.4
+ 2.2.19
commit 506bd0a728728c055635ca01d7e6d8d2d6f263df
Merge: 357858970a 9dc967877c
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 20 14:14:38 2023 -0500
21126: Merge branch 'main' into 21126-trash-when-ro
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list