[arvados] updated: 2.5.0-189-g87380299e
git repository hosting
git at public.arvados.org
Tue Feb 21 16:58:20 UTC 2023
Summary of changes:
.../workbench/config/initializers/reload_config.rb | 4 +-
build/build-dev-docker-jobs-image.sh | 22 +-
doc/images/ArvadosLogoRegSmall.png | Bin 0 -> 214783 bytes
go.mod | 41 +--
go.sum | 158 ++++++---
lib/config/config.default.yml | 8 +-
lib/config/export.go | 2 +-
lib/controller/localdb/login_oidc.go | 25 +-
lib/controller/localdb/login_oidc_test.go | 70 +++-
sdk/cwl/arvados_cwl/__init__.py | 9 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 25 +-
sdk/cwl/arvados_cwl/arvdocker.py | 32 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 392 +++++++++++++++++++--
sdk/cwl/arvados_cwl/context.py | 1 +
sdk/cwl/arvados_cwl/executor.py | 98 ++++--
sdk/cwl/arvados_cwl/fsaccess.py | 10 +-
sdk/cwl/arvados_cwl/runner.py | 192 +++++-----
sdk/cwl/setup.py | 4 +-
sdk/cwl/test_with_arvbox.sh | 11 +
sdk/cwl/tests/arvados-tests.yml | 2 +-
.../dmel_r6.16/WholeGenome/genome.dict | 0
.../dmel_r6.16/WholeGenome/genome.fa.fai | 0
.../Homo_sapiens/GRCh38.p2/WholeGenome/genome.dict | 0
.../GRCh38.p2/WholeGenome/genome.fa.fai | 0
.../collection_per_tool/collection_per_tool.cwl | 4 +-
.../collection_per_tool_wrapper.cwl | 2 +-
sdk/cwl/tests/test_container.py | 19 +-
sdk/cwl/tests/test_copy_deps.py | 72 ++--
sdk/cwl/tests/test_submit.py | 257 +++++---------
sdk/cwl/tests/wf/expect_upload_wrapper.cwl | 5 +-
sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl | 5 +-
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf2.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf5.cwl | 3 +-
sdk/cwl/tests/wf/secret_wf.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_no_reuse.cwl | 4 +-
sdk/cwl/tests/wf/submit_wf_process_properties.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_runner_resources.cwl | 2 +-
...ct_upload_wrapper.cwl => submit_wf_wrapper.cwl} | 30 +-
sdk/dev-jobs.dockerfile | 3 +
sdk/go/arvados/client.go | 2 +-
sdk/go/arvados/config.go | 2 +-
sdk/go/arvados/fs_collection.go | 59 +++-
sdk/go/arvados/fs_collection_test.go | 34 +-
sdk/go/arvados/limiter_test.go | 139 ++++----
sdk/go/arvadostest/oidc_provider.go | 11 +-
sdk/python/arvados/util.py | 17 +
services/api/config/initializers/reload_config.rb | 4 +-
.../api/db/migrate/20221230155924_bigint_id.rb | 5 +
.../pillars/nginx_collections_configuration.sls | 5 +
.../aws/pillars/nginx_controller_configuration.sls | 5 +
.../aws/pillars/nginx_download_configuration.sls | 5 +
.../aws/pillars/nginx_keepproxy_configuration.sls | 5 +
.../aws/pillars/nginx_webshell_configuration.sls | 5 +
.../aws/pillars/nginx_websocket_configuration.sls | 5 +
.../aws/pillars/nginx_workbench2_configuration.sls | 5 +
.../aws/pillars/nginx_workbench_configuration.sls | 5 +
.../multi_host/aws/pillars/ssl_key_encrypted.sls | 11 +
.../multi_host/aws/states/custom_certs.sls | 10 +-
.../multi_host/aws/states/ssl_key_encrypted.sls | 71 ++++
tools/salt-install/installer.sh | 8 +-
.../local.params.example.multiple_hosts | 6 +
...l.params.example.single_host_multiple_hostnames | 5 +
...ocal.params.example.single_host_single_hostname | 5 +
tools/salt-install/provision.sh | 31 +-
.../salt-install/terraform/aws/services/locals.tf | 3 +
tools/salt-install/terraform/aws/services/main.tf | 44 ++-
.../salt-install/terraform/aws/services/outputs.tf | 4 +
.../terraform/aws/services/terraform.tfvars | 4 +
.../terraform/aws/services/variables.tf | 6 +
73 files changed, 1384 insertions(+), 667 deletions(-)
create mode 100644 doc/images/ArvadosLogoRegSmall.png
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.dict (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa.fai (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.dict (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa.fai (100%)
copy sdk/cwl/tests/wf/{expect_upload_wrapper.cwl => submit_wf_wrapper.cwl} (54%)
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/ssl_key_encrypted.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/ssl_key_encrypted.sls
via 87380299e967cf8921ffbfd98c52d0f2a13613e1 (commit)
via b9a5ff791910fc8c49b02521c48a5bf25ccc4259 (commit)
via aa208f2ea4f7d0bc05628399cde30fd9c3747a8c (commit)
via b4792d835e27f7b5deaaff51adf648f395c3a999 (commit)
via c088a40d5a626c42d6779bca10c8a6f84a86f7bc (commit)
via fcfc961430e59fc19a0cee846c32bbe822463aa0 (commit)
via e03ebe77c257e44645c64c8cd71e5e7c115991ef (commit)
via 94dcdefbf8fe264daa28c5b15f68b304c683e390 (commit)
via bc3c1a3334cf5b39d73ac7f59311dccc93e8a00f (commit)
via 0d17360aaf96a28edddcb0c4394b1965a765088a (commit)
via a68ec8b7bed718acbbc55900e41847b1d319874c (commit)
via e10646b6cd9e178c284f14658d3a55149582db3b (commit)
via e9928bf0228070a287c086eb91cf1b278bded2db (commit)
via cffecc32488ab5c7ef44d1d855fb8c0e55d76b1f (commit)
via 2de2c96925cc3439305f16dced7f89bd9124853d (commit)
via 284f37a08fcdff15012b9f731000c57c1d7c56f1 (commit)
via df5588faa32a61d40968cf5c0ef50bdfb36985e3 (commit)
via d4093dea56055b8630b4ba4ca04ad0c891770db8 (commit)
via 7602078b5c8f56f6dddfd89a36cf65cb2e1c88d0 (commit)
via acb7d560e17143a8a82af26a774a6a334f868494 (commit)
via e1577a7fbba344f1c7d784d085d5f8b099bc28bf (commit)
via 7da7fa921c00de34babb3de13400f1d413042fe8 (commit)
via e51ba73431f40cfcb6a8f2c80bb5373f23609fcd (commit)
via aece35d20c892aa7b3df29b15ac0a8a167b0b3ab (commit)
via 1d8b1200de6bc1fc6df70f611446e1e6466244fd (commit)
via 511fc7559a6ad00468c9a452bdd1de63ad2c1f77 (commit)
via f3a1fcb306856fc904c7e8051ccb69ea85e5640f (commit)
via 1c50d29ea10fa90a379ee1d3fe7d8e05681d8622 (commit)
via cfba0af9859716e3a771dc03d205379ac6b8834f (commit)
via 1bd3fa5209c808445ee8fa3bcfde8e88b0fd32d7 (commit)
via ca210064364c6f45db1e2d6a936f21940a3cf03f (commit)
via 952458464e04ff17e2c8fc0ce89812912e09737c (commit)
via 4b1a3fdcb5065f7a34eeeda662e31f01d8224392 (commit)
via 4b2b6ca09aec9dfb9f3871bd202484b1ae4063b9 (commit)
via c4c57180c9ee9f79a1d272710aa7b8747d4d0c38 (commit)
via 083b86a4e748900bcc285cac8bfd2ecdd36679f6 (commit)
via 4e765b926f58f169bfca38268df9bdda84b55503 (commit)
via 0879d8adc1156f2484d6cc0e7b369e40f5eee356 (commit)
via 62ede2cf371f51cbe8bac07c36ddf904e428262b (commit)
via 39b216f1ecad8a9e607447eecf1d2ba90d6eaabf (commit)
via 995df5b03c9304bc956b02e50225ad2e1f9dfd8d (commit)
via 9f42cb85807ebad098aaf6e0ab3218f763b712e2 (commit)
via a54c0f72656d883ae8f27d5074e35f60e61dce09 (commit)
via dfc13b2f30d752963bd9715189aafbf74fdf5360 (commit)
via 2930c49ba1a06f947d06424a37b0dc93505fad18 (commit)
via ec7e22ab6ad88b0e5930e98760334dc0ef14a5be (commit)
via 37eb070f55b5ae0c622fb4bf0a946c9dd49b2752 (commit)
via 4f3739a4967959db6783408d8aad2137b9ebdab5 (commit)
via 6e45aef0eaea08b9fb3ba85783af8552757f8e6a (commit)
via a5bb9338a46680c3976544618d0a3fd5f0b65a96 (commit)
via fb27923f7ce22eddde0852d24d6c94323de41330 (commit)
via fcdf73b20710064a80807e3d8b579a11b629d193 (commit)
via cc095ef1f8c86b05a8e58f15c4086a2caa861ec5 (commit)
via 60da027b51ed0d29ae2aad263b9ebdf8cfb6f9b3 (commit)
via 5ed84016e9e9a210ecf26a01ad50d2d6a51694a5 (commit)
via 01f34b3dbf9d37a551f90c3025d42202b222713f (commit)
via d74058dabbd7bf7360ace73fd6b2874c6ae52809 (commit)
via d2464c017d342cda08842631f5aa6266f191fa41 (commit)
via 21c017bebaa9e8811170c113da77b9054def7117 (commit)
via 347ece2d7c00703487b87998eab06c5edf8796db (commit)
via 91dc5cddefb9d5f09c261b5097165517ea5fb33a (commit)
via 165ec76d4d36b03449fcd756cb03261f688c1a75 (commit)
via 07c70cc59ef9f17ee3baf98d69be76d9a6585235 (commit)
via 50bd5e3f71a1eb18c7721fc44a934413463baa7c (commit)
via 55e73facf08cc8651eb71529a2ab4e0657d3c870 (commit)
via b420ec626f8cb5cd7a8b4252dfc2be76ba3ba844 (commit)
via 62e7af59cbad5577423b844213be7b2f59709602 (commit)
via 9e1828a4263d7efc8fe7149c8c63ce0477551e8c (commit)
via 28b5ebc08c632a7d947883910b565ffdc6f85b68 (commit)
via 472742c1b4c2abd62325471094f25bbbb4770ede (commit)
via fec959159de7bb7bed944bd15868e11be0ecdcfa (commit)
via 58287d8af844ed344736db29590b2b06c58c97c8 (commit)
via 0920e90f2454f47f1e4cf97ebbceb69e8c443e59 (commit)
via 0eee268be6c2d47f310f49bf74bd356aaf5ad163 (commit)
via 889e9fa0fe0faf311770d6bf7639432853544d79 (commit)
from bfb9e29c250bcfb34a6b1813ca46953503ca05e6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 87380299e967cf8921ffbfd98c52d0f2a13613e1
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 11:57:27 2023 -0500
19973: Split unit test cases into multiple funcs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/limiter_test.go b/sdk/go/arvados/limiter_test.go
index d1f5e837a..d32ab9699 100644
--- a/sdk/go/arvados/limiter_test.go
+++ b/sdk/go/arvados/limiter_test.go
@@ -18,90 +18,93 @@ var _ = Suite(&limiterSuite{})
type limiterSuite struct{}
-func (*limiterSuite) TestLimiter(c *C) {
+func (*limiterSuite) TestUnlimitedBeforeFirstReport(c *C) {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
defer cancel()
rl := requestLimiter{}
- // unlimited concurrency before first call to Report()
- {
- var wg sync.WaitGroup
- wg.Add(1000)
- for i := 0; i < 1000; i++ {
- go func() {
- rl.Acquire(ctx)
- wg.Done()
- }()
- }
- wg.Wait()
- c.Check(rl.current, Equals, int64(1000))
- wg.Add(1000)
- for i := 0; i < 1000; i++ {
- go func() {
- rl.Release()
- wg.Done()
- }()
- }
- wg.Wait()
- c.Check(rl.current, Equals, int64(0))
+ var wg sync.WaitGroup
+ wg.Add(1000)
+ for i := 0; i < 1000; i++ {
+ go func() {
+ rl.Acquire(ctx)
+ wg.Done()
+ }()
}
-
- // context cancels while waiting for Acquire
- {
- rl.limit = 1
- rl.Acquire(ctx)
- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Millisecond))
- defer cancel()
- rl.Acquire(ctx)
- c.Check(rl.current, Equals, int64(2))
- c.Check(ctx.Err(), NotNil)
- rl.Release()
- rl.Release()
- c.Check(rl.current, Equals, int64(0))
+ wg.Wait()
+ c.Check(rl.current, Equals, int64(1000))
+ wg.Add(1000)
+ for i := 0; i < 1000; i++ {
+ go func() {
+ rl.Release()
+ wg.Done()
+ }()
}
+ wg.Wait()
+ c.Check(rl.current, Equals, int64(0))
+}
+
+func (*limiterSuite) TestCancelWhileWaitingForAcquire(c *C) {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+ defer cancel()
+ rl := requestLimiter{}
+
+ rl.limit = 1
+ rl.Acquire(ctx)
+ ctxShort, cancel := context.WithDeadline(ctx, time.Now().Add(time.Millisecond))
+ defer cancel()
+ rl.Acquire(ctxShort)
+ c.Check(rl.current, Equals, int64(2))
+ c.Check(ctxShort.Err(), NotNil)
+ rl.Release()
+ rl.Release()
+ c.Check(rl.current, Equals, int64(0))
+}
+
+func (*limiterSuite) TestReducedLimitAndQuietPeriod(c *C) {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+ defer cancel()
+ rl := requestLimiter{}
// Use a short quiet period to make tests faster
defer func(orig time.Duration) { requestLimiterQuietPeriod = orig }(requestLimiterQuietPeriod)
requestLimiterQuietPeriod = time.Second / 10
- // Immediately after a 503, limit is decreased, and Acquire()
- // waits for a quiet period
- {
- rl.limit = 0
- for i := 0; i < 5; i++ {
- rl.Acquire(ctx)
- }
- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(requestLimiterQuietPeriod/10))
- defer cancel()
- rl.Report(&http.Response{StatusCode: http.StatusServiceUnavailable}, nil)
- c.Check(rl.limit, Equals, int64(3))
-
- for i := 0; i < 5; i++ {
- rl.Release()
- }
-
- // Even with all slots released, we can't Acquire in
- // the quiet period.
- acquire := time.Now()
+ for i := 0; i < 5; i++ {
rl.Acquire(ctx)
- c.Check(ctx.Err(), Equals, context.DeadlineExceeded)
- c.Check(time.Since(acquire) < requestLimiterQuietPeriod/2, Equals, true)
- c.Check(rl.quietUntil.Sub(time.Now()) > requestLimiterQuietPeriod/2, Equals, true)
- rl.Release()
}
-
- // Acquire waits for the quiet period to expire.
- {
- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(requestLimiterQuietPeriod*2))
- defer cancel()
- acquire := time.Now()
- rl.Acquire(ctx)
- c.Check(time.Since(acquire) > requestLimiterQuietPeriod/10, Equals, true)
- c.Check(time.Since(acquire) < requestLimiterQuietPeriod, Equals, true)
- c.Check(ctx.Err(), IsNil)
+ rl.Report(&http.Response{StatusCode: http.StatusServiceUnavailable}, nil)
+ c.Check(rl.limit, Equals, int64(3))
+ for i := 0; i < 5; i++ {
rl.Release()
}
+ // Even with all slots released, we can't Acquire in the quiet
+ // period.
+
+ // (a) If our context expires before the end of the quiet
+ // period, we get back DeadlineExceeded -- without waiting for
+ // the end of the quiet period.
+ acquire := time.Now()
+ ctxShort, cancel := context.WithDeadline(ctx, time.Now().Add(requestLimiterQuietPeriod/10))
+ defer cancel()
+ rl.Acquire(ctxShort)
+ c.Check(ctxShort.Err(), Equals, context.DeadlineExceeded)
+ c.Check(time.Since(acquire) < requestLimiterQuietPeriod/2, Equals, true)
+ c.Check(rl.quietUntil.Sub(time.Now()) > requestLimiterQuietPeriod/2, Equals, true)
+ rl.Release()
+
+ // (b) If our context does not expire first, Acquire waits for
+ // the end of the quiet period.
+ ctxLong, cancel := context.WithDeadline(ctx, time.Now().Add(requestLimiterQuietPeriod*2))
+ defer cancel()
+ acquire = time.Now()
+ rl.Acquire(ctxLong)
+ c.Check(time.Since(acquire) > requestLimiterQuietPeriod/10, Equals, true)
+ c.Check(time.Since(acquire) < requestLimiterQuietPeriod, Equals, true)
+ c.Check(ctxLong.Err(), IsNil)
+ rl.Release()
+
// OK to call Report() with nil Response and non-nil error.
rl.Report(nil, errors.New("network error"))
}
commit b9a5ff791910fc8c49b02521c48a5bf25ccc4259
Merge: aa208f2ea b4792d835
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 10:24:52 2023 -0500
19973: Merge branch 'main'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit aa208f2ea4f7d0bc05628399cde30fd9c3747a8c
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 10:22:24 2023 -0500
19973: un-embed requestLimiter.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 4eed6086a..05176214a 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -82,7 +82,7 @@ type Client struct {
// differs from an outgoing connection limit (a feature
// provided by http.Transport) when concurrent calls are
// multiplexed on a single http2 connection.
- requestLimiter
+ requestLimiter requestLimiter
last503 atomic.Value
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list