[ARVADOS] updated: 1.3.0-453-g8660244c5
Git user
git at public.curoverse.com
Fri Mar 8 15:58:33 EST 2019
Summary of changes:
.licenseignore | 3 +-
apps/workbench/.gitignore | 3 +
apps/workbench/app/views/layouts/body.html.erb | 15 ++
apps/workbench/config/application.default.yml | 8 +
.../config/initializers/validate_wb2_url_config.rb | 5 +-
apps/workbench/lib/config_validators.rb | 28 ++++
apps/workbench/package-build.version | 1 -
.../test/integration/application_layout_test.rb | 24 +++
apps/workbench/test/the.patch | 3 -
build/package-build-dockerfiles/centos7/Dockerfile | 6 +-
build/package-build-dockerfiles/debian8/Dockerfile | 5 +-
build/package-build-dockerfiles/debian9/Dockerfile | 5 +-
.../ubuntu1404/Dockerfile | 5 +-
.../ubuntu1604/Dockerfile | 5 +-
.../ubuntu1804/Dockerfile | 3 -
build/run-build-docker-jobs-image.sh | 48 ++++--
build/run-build-packages-python-and-ruby.sh | 3 -
build/run-build-packages.sh | 15 --
build/run-library.sh | 50 ++++--
build/run-tests.sh | 1 -
cmd/arvados-server/arvados-dispatch-cloud.service | 2 +-
doc/_includes/_install_compute_docker.liquid | 2 +-
doc/admin/upgrading.html.textile.liquid | 57 +++++++
...nstall-manual-prerequisites.html.textile.liquid | 4 +-
doc/user/topics/arv-docker.html.textile.liquid | 2 +-
docker/jobs/1078ECD7.key | 30 ++++
docker/jobs/Dockerfile | 26 +--
docker/jobs/apt.arvados.org-dev.list | 2 +
docker/jobs/apt.arvados.org-stable.list | 2 +
docker/jobs/apt.arvados.org-testing.list | 2 +
docker/jobs/apt.arvados.org.list | 2 -
lib/cloud/azure/azure.go | 183 +++++++++------------
lib/cloud/azure/azure_test.go | 40 +++--
lib/cloud/interfaces.go | 21 ++-
lib/controller/cmd.go | 4 +-
lib/controller/federation_test.go | 7 +-
lib/controller/handler_test.go | 12 +-
lib/controller/server_test.go | 18 +-
lib/dispatchcloud/cmd.go | 6 +-
lib/dispatchcloud/container/queue.go | 21 ++-
lib/dispatchcloud/dispatcher.go | 22 ++-
lib/dispatchcloud/dispatcher_test.go | 12 +-
lib/dispatchcloud/node_size.go | 2 +-
lib/dispatchcloud/scheduler/fix_stale_locks.go | 7 +-
lib/dispatchcloud/scheduler/run_queue.go | 85 ++++++----
lib/dispatchcloud/scheduler/run_queue_test.go | 11 +-
lib/dispatchcloud/scheduler/scheduler.go | 20 ++-
lib/dispatchcloud/scheduler/sync.go | 84 ++++++----
lib/dispatchcloud/ssh_executor/executor.go | 6 +-
lib/dispatchcloud/ssh_executor/executor_test.go | 2 +
lib/dispatchcloud/test/logger.go | 19 ---
lib/dispatchcloud/test/ssh_service.go | 8 +-
lib/dispatchcloud/test/stub_driver.go | 31 +++-
lib/dispatchcloud/worker/pool.go | 162 ++++++++++--------
lib/dispatchcloud/worker/pool_test.go | 9 +-
lib/dispatchcloud/worker/verify.go | 56 +++++++
lib/dispatchcloud/worker/worker.go | 51 ++++--
lib/dispatchcloud/worker/worker_test.go | 5 +-
lib/service/cmd.go | 16 +-
sdk/cwl/arvados_cwl/__init__.py | 6 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 16 +-
sdk/cwl/arvados_cwl/arvdocker.py | 1 +
sdk/cwl/arvados_cwl/arvjob.py | 10 +-
sdk/cwl/arvados_cwl/executor.py | 56 ++++---
sdk/cwl/arvados_cwl/fsaccess.py | 4 +-
sdk/cwl/arvados_cwl/runner.py | 12 +-
sdk/cwl/setup.py | 7 +-
sdk/cwl/tests/test_container.py | 55 +++++--
sdk/go/arvados/config.go | 15 +-
sdk/go/ctxlog/log.go | 68 ++++++--
.../src/main/java/org/arvados/sdk/Arvados.java | 1 -
sdk/python/arvados/_normalize_stream.py | 6 +-
sdk/python/arvados/collection.py | 21 ++-
sdk/python/arvados/commands/keepdocker.py | 16 +-
sdk/python/arvados/commands/put.py | 18 +-
sdk/python/setup.py | 7 +-
sdk/python/tests/test_arv_put.py | 12 ++
sdk/ruby/lib/arvados/collection.rb | 10 +-
sdk/ruby/test/test_collection.rb | 10 ++
services/api/.gitignore | 3 +
services/api/app/models/container.rb | 3 +
services/api/test/unit/container_test.rb | 8 +
services/crunch-run/background.go | 77 ++++-----
services/crunch-run/crunchrun.go | 26 +++
services/fuse/arvados_fuse/fresh.py | 2 +-
services/fuse/setup.py | 5 +-
services/keepstore/azure_blob_volume.go | 9 +-
services/keepstore/azure_blob_volume_test.go | 14 ++
services/keepstore/config.go | 2 +-
services/keepstore/handler_test.go | 18 ++
services/keepstore/handlers.go | 5 +
services/keepstore/keepstore.go | 1 +
services/keepstore/keepstore_test.go | 2 +
services/keepstore/metrics.go | 21 ++-
services/keepstore/mounts_test.go | 18 +-
services/keepstore/s3_volume.go | 6 +-
services/keepstore/s3_volume_test.go | 15 +-
services/keepstore/stats_ticker.go | 1 +
services/keepstore/unix_volume.go | 6 +-
services/keepstore/unix_volume_test.go | 9 +-
services/keepstore/volume.go | 3 +-
services/keepstore/volume_generic_test.go | 80 +++++++++
services/keepstore/volume_test.go | 17 +-
services/ws/session_v0.go | 2 +-
.../arvbox/docker/service/workbench/run-service | 1 +
105 files changed, 1336 insertions(+), 656 deletions(-)
copy build/package-testing/test-package-python27-python-arvados-cwl-runner.sh => apps/workbench/config/initializers/validate_wb2_url_config.rb (59%)
mode change 100755 => 100644
create mode 100644 apps/workbench/lib/config_validators.rb
delete mode 100644 apps/workbench/package-build.version
delete mode 100644 apps/workbench/test/the.patch
create mode 100644 docker/jobs/1078ECD7.key
create mode 100644 docker/jobs/apt.arvados.org-dev.list
create mode 100644 docker/jobs/apt.arvados.org-stable.list
create mode 100644 docker/jobs/apt.arvados.org-testing.list
delete mode 100644 docker/jobs/apt.arvados.org.list
delete mode 100644 lib/dispatchcloud/test/logger.go
create mode 100644 lib/dispatchcloud/worker/verify.go
via 8660244c570cecdb5b4c25a3809cfe01fafebada (commit)
via 3470ee572e9841bbcb45ca5f1de71a238d60d1bd (commit)
via 7f3f0b4f3be061ebba1ce1b64ee5574873ef181d (commit)
via 53df63726d26692f68e6582f486859a801e444bd (commit)
via 646ea4328be842f4baa194205618c01910ec49db (commit)
via 35a23c29e610809d19635aba3cc1956128cb75d2 (commit)
via 124db7dd6288165a0fe772d141b5f1058f87655f (commit)
via d4da0c6adf235aa103337430fd73cf3bfba64fd9 (commit)
via 12ca5b7a44a0600374dd0d6f11dbda66a96ca1ff (commit)
via 07da7af1bac6dbf346ad6b97924c41564ce42997 (commit)
via b20862479a05afec430f10067b8b417418b11eee (commit)
via d911d9335fe92615ee479349952eff150f25e354 (commit)
via d91d9cd878ead1be98122a0ddaf550c684e0ac38 (commit)
via 81693ee04f18e68558c9206705ef589cca2460b0 (commit)
via f1d38ded6991f7061e2e4d9f61db3501c9c90b2b (commit)
via 0818f8a837bd9d60132f0c7bc8cd06f15e0002d2 (commit)
via 703b21198473ca619b1c83726bc654563384d96f (commit)
via d99fc30eb4a6e3f798dfb80ea010e313490683ae (commit)
via 58bc1b8355b0a09c09fe56d1fca82dfdb830efee (commit)
via fd86f7f461bdca63ba500570094a9a452d28eb57 (commit)
via 338ab239adbc259d5cd070158b4e571925b9f81b (commit)
via 3af38eca348413c6f11f6526b2ee2ca7cb53e348 (commit)
via a310d114bdc06b20cd007e6aff14b409e1c11e32 (commit)
via 494b80a7e5f324bbd09a6b879149a1c57bdc637b (commit)
via b16f84a8c5562cd5c3939e8c445dc86df2be2d4b (commit)
via dd969e81dfbe0ff1b60c85841aaf01e51b45c574 (commit)
via 8c4fb97b1d34b5f8fc50d239698a08c35a63dac3 (commit)
via e231fef37079916d0dc6babf93d669d474598ced (commit)
via f1943e92b2c83d2884eeaa2829f1b4a07d23b9ee (commit)
via d9e5d620d8e68b349bb79b9749a74110603a89d4 (commit)
via ffef38539dd36e224abb75f0480cff1deb319124 (commit)
via 1c35232248baebeaa69e7e99d9827e7b9983fe6c (commit)
via 37c46ca294ce95242ac424c8375a130d783f6d1e (commit)
via 926dcfeab145e7c714c5531fa044bac6f0f0f992 (commit)
via 023858a629bedfccdb5d17602643aaaa0a223e1a (commit)
via 39a98eef822a5f68a0666c33b97ff1a373bce10c (commit)
via 23cced39fdeeec170460fca4799a3fa54f8b8aae (commit)
via 11103ee6c45acceb1ebdd02c4a190bbabbf8621c (commit)
via d0584563200b0fe69e508b2fc5b0ddb223ebccc0 (commit)
via 8fd360faf91921c557f44f079b127a4fa5830486 (commit)
via 08123387faf00b0af64ea01f7196d8c5ca3fae1e (commit)
via 61a02d598416ef85b5f580be0eecfc7b2d551195 (commit)
via 48ee36541552d5394234d3057a6c1d5957f40d2e (commit)
via 0dbe95af08f5e63086ee14a287350a36d3405812 (commit)
via ebf4385eee0408a357641254aac688ecc713be6f (commit)
via 50ecc8767d360670b0f8906da4daa87e56779e54 (commit)
via d1fcb330f81e0f3d3a2a817340cbd499a47b047e (commit)
via 689cd158566b0370153e521aceff2758e77f2e8a (commit)
via 5462e33891e660cfe16b40332db739f7aa9fc405 (commit)
via 7e5f0e9ca6756099f761cc3f392476f362cd1645 (commit)
via 7aae48ab36090eb33859e65e39132eb8c74362c5 (commit)
via a1aefe79abf1d84ee9bec3b5896a7ba30d42e014 (commit)
via 408e81ed610fa238229150e4a47f457139fcf867 (commit)
via 6ab33cb91de748f4f8ec8e12377f0b0650804db1 (commit)
via 0fc8778c47397f7e5de8481e0a3e7716ccc53d21 (commit)
via e7ce0cd2e144e9139cdbdaaab364803039627922 (commit)
via 601764a10575232ada34a7c0dc5ec61195094e7f (commit)
via a986c70c083ed53f9b2d0d112853d1a1155eedea (commit)
via ca2d946973b6ae25dd594ddecec54e02b83bc44e (commit)
via 61ddf5ce517ce0f876e7279fa2a5aa375b9fbdd8 (commit)
via 9f57bcbab5b244365ca4d0ece8490d7540c72b86 (commit)
via 9559f6df3d943cff3cd946c406535971fdf35fb8 (commit)
via ffd42e0f61c902eb1a1a14df61ab53838be01431 (commit)
via c9f9b4cf4659ef801b059a006ce46e3849926f71 (commit)
via 4a449778017df5e8910a69565c27e70d64996d71 (commit)
via d97388bdbfeb6a43cb86996012a1db0ba4a8871f (commit)
via f2e8b80d3a3e98417e4e56f7069176a9b88eacbc (commit)
via 6d852fc2b60e140decfafc92a971a5d1f027e854 (commit)
via 5d8ebb8d00de8c3afa59045fa929536ff8973881 (commit)
via 36e1f63fde4005f38a8d9825abc8244057d9bb58 (commit)
via 28209cbb68aaa3bcf2832ca739d6c1ed89f4d3fd (commit)
via 970af93afee8838a6e6be52e897c639ba13c7d3f (commit)
via 79693e508b0c2a61bc006994c6245697e32c2586 (commit)
via e1e0f678978cd2e6d81cba1f61de06effa961afa (commit)
via 87bf45c8c4db3c7dd58edabab429286dcabfcd7d (commit)
via 91b39ff3fbba69e54a330a7949c9ed4bbb3117ff (commit)
via 80c48b78fe5029fe3389275b9e51fe0be65cb11d (commit)
via abd21f165dbfd84eaea9ca1d7f20ce195c52d679 (commit)
via d6fbaeba4da4ca1dcd70fe48a8875a84c17214da (commit)
via 30ca2a11cbe11e054ea60ed01c3e94d422dddac6 (commit)
via 3d662ef38da0395af4e477acd6c3a20de9280084 (commit)
via f6d551a68e30f533676bcf7ed17b199b21990c7b (commit)
via 3d7b9154170eb4217119ed2279a6e8c365d9288c (commit)
via d3cef2f89fbcbf9bac380a7c834f13117d187058 (commit)
via de9a5e2703cca69b0ba6e8e3e6102ee267b7447e (commit)
via 0de109fe60bbe505fa0192b24fe29e96963a0bdf (commit)
via e96f8774c0245399d69a30c93c67f197a91622fc (commit)
via 832235d35bbc21967206e59798605bd45ebaa859 (commit)
via 601eeec8916916330c4fb9a7d615a3918fcc36ff (commit)
via 97a1babd776addd419fb5c050157786bdbd6232f (commit)
via c4c77dc1efad8ebebb049f62f1756f5cef971660 (commit)
via efe3cb087cc269daa5e41f669ecbb4c0d14bcfc6 (commit)
via d71e2f174f9a58b315037750ac301225ec68e11f (commit)
via 13fa76866cc2266812df44c410cd5cdfe16d5e73 (commit)
via ed317e6b28c7fc69d1af2899e782b92b1dad0a79 (commit)
via d2bdc5af995f829240593cdfa580f670a2d30e27 (commit)
via bcabab96dde14aaa7c4d410e86d11a70abf71595 (commit)
via e6cb52f3e0cd84e0a7a4185d1fa2b38b9a5f8c5a (commit)
via df2884530baa3218ce58253a0272379568504da7 (commit)
from 92d1b6e05e042a0781070e7287b1ceb3e094e852 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8660244c570cecdb5b4c25a3809cfe01fafebada
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Fri Mar 8 17:56:17 2019 -0300
13937: Adds tests for operation & I/O counters.
Added {"operation": "any"} to opsCounters so that it's easy to check on tests,
because the kind of operation depends on the driver and the tests were
implemented as part of the volume generic tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 85d0a1eea..8a5ffde26 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -29,6 +29,7 @@ import (
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/ghodss/yaml"
+ "github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
@@ -745,6 +746,19 @@ func (v *TestableAzureBlobVolume) Teardown() {
v.azStub.Close()
}
+func (v *TestableAzureBlobVolume) GetMetricsVecs() (opsCounters, errCounters, ioBytes *prometheus.CounterVec) {
+ opsCounters = v.container.stats.opsCounters
+ errCounters = v.container.stats.errCounters
+ ioBytes = v.container.stats.ioBytes
+ return
+}
+
+func (v *TestableAzureBlobVolume) Start(vm *volumeMetricsVecs) error {
+ // Override original Start() to be able to assign CounterVecs with a dummy DeviceID
+ v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(prometheus.Labels{"device_id": "azure_blob_volume_test"})
+ return nil
+}
+
func makeEtag() string {
return fmt.Sprintf("0x%x", rand.Int63())
}
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 6fd6b653e..3d7a630e0 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -493,3 +493,10 @@ func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) {
func (v *TestableS3Volume) Teardown() {
v.server.Quit()
}
+
+func (v *TestableS3Volume) GetMetricsVecs() (opsCounters, errCounters, ioBytes *prometheus.CounterVec) {
+ opsCounters = v.bucket.stats.opsCounters
+ errCounters = v.bucket.stats.errCounters
+ ioBytes = v.bucket.stats.ioBytes
+ return
+}
diff --git a/services/keepstore/stats_ticker.go b/services/keepstore/stats_ticker.go
index 342b9e320..29daeb6dd 100644
--- a/services/keepstore/stats_ticker.go
+++ b/services/keepstore/stats_ticker.go
@@ -74,6 +74,7 @@ func (s *statsTicker) TickOps(operations ...string) {
return
}
for _, opType := range operations {
+ s.opsCounters.With(prometheus.Labels{"operation": "any"}).Inc()
s.opsCounters.With(prometheus.Labels{"operation": opType}).Inc()
}
}
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index 8b2bc2d54..6582642ff 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -74,6 +74,13 @@ func (v *TestableUnixVolume) Teardown() {
}
}
+func (v *TestableUnixVolume) GetMetricsVecs() (opsCounters, errCounters, ioBytes *prometheus.CounterVec) {
+ opsCounters = v.os.stats.opsCounters
+ errCounters = v.os.stats.errCounters
+ ioBytes = v.os.stats.ioBytes
+ return
+}
+
// serialize = false; readonly = false
func TestUnixVolumeWithGenericTests(t *testing.T) {
DoGenericVolumeTests(t, func(t TB) TestableVolume {
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index 23a17fd09..964e68bb9 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -18,6 +18,8 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
)
type TB interface {
@@ -75,6 +77,8 @@ func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
testStatus(t, factory)
+ testMetrics(t, factory)
+
testString(t, factory)
testUpdateReadOnly(t, factory)
@@ -533,6 +537,82 @@ func testStatus(t TB, factory TestableVolumeFactory) {
}
}
+func getValueFrom(cv *prometheus.CounterVec, lbls prometheus.Labels) float64 {
+ c, _ := cv.GetMetricWith(lbls)
+ pb := &dto.Metric{}
+ c.Write(pb)
+ return pb.GetCounter().GetValue()
+}
+
+func testMetrics(t TB, factory TestableVolumeFactory) {
+ var err error
+
+ v := factory(t)
+ defer v.Teardown()
+ reg := prometheus.NewRegistry()
+ vm := newVolumeMetricsVecs(reg)
+
+ err = v.Start(vm)
+ if err != nil {
+ t.Error("Failed Start(): ", err)
+ }
+ opsC, _, ioC := v.GetMetricsVecs()
+
+ if ioC == nil {
+ t.Error("ioBytes CounterVec is nil")
+ return
+ }
+
+ if getValueFrom(ioC, prometheus.Labels{"direction": "out"})+
+ getValueFrom(ioC, prometheus.Labels{"direction": "in"}) > 0 {
+ t.Error("ioBytes counter should be zero")
+ }
+
+ if opsC == nil {
+ t.Error("opsCounter CounterVec is nil")
+ return
+ }
+
+ var c, anyOpCounter float64
+ anyOpCounter = getValueFrom(opsC, prometheus.Labels{"operation": "any"})
+ // Test Put if volume is writable
+ if v.Writable() {
+ err = v.Put(context.Background(), TestHash, TestBlock)
+ if err != nil {
+ t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
+ }
+ // Check that the operations counter increased
+ c = getValueFrom(opsC, prometheus.Labels{"operation": "any"})
+ if c <= anyOpCounter {
+ t.Error("Operation(s) not counted on Put")
+ }
+ anyOpCounter = c
+ // Check that bytes counter is > 0
+ if getValueFrom(ioC, prometheus.Labels{"direction": "out"}) == 0 {
+ t.Error("ioBytes{direction=out} counter shouldn't be zero")
+ }
+ } else {
+ v.PutRaw(TestHash, TestBlock)
+ }
+
+ buf := make([]byte, BlockSize)
+ _, err = v.Get(context.Background(), TestHash, buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Check that the operations counter increased
+ c = getValueFrom(opsC, prometheus.Labels{"operation": "any"})
+ if c <= anyOpCounter {
+ t.Error("Operation(s) not counted on Get")
+ }
+ anyOpCounter = c
+ // Check that the bytes "in" counter is > 0
+ if getValueFrom(ioC, prometheus.Labels{"direction": "in"}) == 0 {
+ t.Error("ioBytes{direction=in} counter shouldn't be zero")
+ }
+}
+
// Invoke String for the volume; expect non-empty result
// Test should pass for both writable and read-only volumes
func testString(t TB, factory TestableVolumeFactory) {
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 9a745b629..5feae3c29 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -15,6 +15,8 @@ import (
"strings"
"sync"
"time"
+
+ "github.com/prometheus/client_golang/prometheus"
)
// A TestableVolume allows test suites to manipulate the state of an
@@ -22,6 +24,10 @@ import (
// impractical to achieve with a sequence of normal Volume operations.
type TestableVolume interface {
Volume
+
+ // Get prometheus metrics
+ GetMetricsVecs() (opsCounters, errCounters, ioBytes *prometheus.CounterVec)
+
// [Over]write content for a locator with the given data,
// bypassing all constraints like readonly and serialize.
PutRaw(locator string, data []byte)
commit 3470ee572e9841bbcb45ca5f1de71a238d60d1bd
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Thu Mar 7 11:42:43 2019 -0300
13937: Simplifies volume counter stats assignments. Fixes test.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 432347396..6b5b233c2 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -186,9 +186,7 @@ func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.container.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
- v.container.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
- v.container.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
+ v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(lbls)
return nil
}
diff --git a/services/keepstore/metrics.go b/services/keepstore/metrics.go
index d119bd803..235c41891 100644
--- a/services/keepstore/metrics.go
+++ b/services/keepstore/metrics.go
@@ -128,3 +128,10 @@ func newVolumeMetricsVecs(reg *prometheus.Registry) *volumeMetricsVecs {
return m
}
+
+func (vm *volumeMetricsVecs) getCounterVecsFor(lbls prometheus.Labels) (opsCV, errCV, ioCV *prometheus.CounterVec) {
+ opsCV = vm.opsCounters.MustCurryWith(lbls)
+ errCV = vm.errCounters.MustCurryWith(lbls)
+ ioCV = vm.ioBytes.MustCurryWith(lbls)
+ return
+}
diff --git a/services/keepstore/mounts_test.go b/services/keepstore/mounts_test.go
index ac30c369a..7c932ee02 100644
--- a/services/keepstore/mounts_test.go
+++ b/services/keepstore/mounts_test.go
@@ -149,15 +149,15 @@ func (s *MountsSuite) TestMetrics(c *check.C) {
c.Check(found["time_to_status_seconds"], check.Equals, true)
metricsNames := []string{
- "arvados_keepstore_bufferpool_buffers_in_use",
- "arvados_keepstore_bufferpool_buffers_max",
- "arvados_keepstore_bufferpool_bytes_allocated",
- "arvados_keepstore_pull_queue_in_progress",
- "arvados_keepstore_pull_queue_queued",
- "arvados_keepstore_requests_current",
- "arvados_keepstore_requests_max",
- "arvados_keepstore_trash_queue_in_progress",
- "arvados_keepstore_trash_queue_queued",
+ "arvados_keepstore_bufferpool_inuse_buffers",
+ "arvados_keepstore_bufferpool_max_buffers",
+ "arvados_keepstore_bufferpool_allocated_bytes",
+ "arvados_keepstore_pull_queue_inprogress_entries",
+ "arvados_keepstore_pull_queue_pending_entries",
+ "arvados_keepstore_concurrent_requests",
+ "arvados_keepstore_max_concurrent_requests",
+ "arvados_keepstore_trash_queue_inprogress_entries",
+ "arvados_keepstore_trash_queue_pending_entries",
"request_duration_seconds",
"time_to_status_seconds",
}
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 33279e5ef..4c39dcd5c 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -251,9 +251,7 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.bucket.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
- v.bucket.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
- v.bucket.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
+ v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls)
return nil
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 32628f729..96f458720 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -232,9 +232,7 @@ func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.os.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
- v.os.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
- v.os.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
+ v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = vm.getCounterVecsFor(lbls)
_, err := v.os.Stat(v.Root)
commit 7f3f0b4f3be061ebba1ce1b64ee5574873ef181d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Mar 6 19:25:10 2019 -0300
13937: Fixes metrics naming
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/keepstore/metrics.go b/services/keepstore/metrics.go
index 4a154bd7b..d119bd803 100644
--- a/services/keepstore/metrics.go
+++ b/services/keepstore/metrics.go
@@ -20,7 +20,7 @@ func (m *nodeMetrics) setupBufferPoolMetrics(b *bufferPool) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: "bufferpool_bytes_allocated",
+ Name: "bufferpool_allocated_bytes",
Help: "Number of bytes allocated to buffers",
},
func() float64 { return float64(b.Alloc()) },
@@ -29,7 +29,7 @@ func (m *nodeMetrics) setupBufferPoolMetrics(b *bufferPool) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: "bufferpool_buffers_max",
+ Name: "bufferpool_max_buffers",
Help: "Maximum number of buffers allowed",
},
func() float64 { return float64(b.Cap()) },
@@ -38,7 +38,7 @@ func (m *nodeMetrics) setupBufferPoolMetrics(b *bufferPool) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: "bufferpool_buffers_in_use",
+ Name: "bufferpool_inuse_buffers",
Help: "Number of buffers in use",
},
func() float64 { return float64(b.Len()) },
@@ -50,7 +50,7 @@ func (m *nodeMetrics) setupWorkQueueMetrics(q *WorkQueue, qName string) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: fmt.Sprintf("%s_queue_in_progress", qName),
+ Name: fmt.Sprintf("%s_queue_inprogress_entries", qName),
Help: fmt.Sprintf("Number of %s requests in progress", qName),
},
func() float64 { return float64(getWorkQueueStatus(q).InProgress) },
@@ -59,7 +59,7 @@ func (m *nodeMetrics) setupWorkQueueMetrics(q *WorkQueue, qName string) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: fmt.Sprintf("%s_queue_queued", qName),
+ Name: fmt.Sprintf("%s_queue_pending_entries", qName),
Help: fmt.Sprintf("Number of queued %s requests", qName),
},
func() float64 { return float64(getWorkQueueStatus(q).Queued) },
@@ -71,7 +71,7 @@ func (m *nodeMetrics) setupRequestMetrics(rc httpserver.RequestCounter) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: "requests_current",
+ Name: "concurrent_requests",
Help: "Number of requests in progress",
},
func() float64 { return float64(rc.Current()) },
@@ -80,7 +80,7 @@ func (m *nodeMetrics) setupRequestMetrics(rc httpserver.RequestCounter) {
prometheus.GaugeOpts{
Namespace: "arvados",
Subsystem: "keepstore",
- Name: "requests_max",
+ Name: "max_concurrent_requests",
Help: "Maximum number of concurrent requests",
},
func() float64 { return float64(rc.Max()) },
commit 53df63726d26692f68e6582f486859a801e444bd
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Mar 6 19:12:20 2019 -0300
13937: Simplifies arg passing on Start() funcs
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 66956b89e..432347396 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -148,7 +148,7 @@ func (v *AzureBlobVolume) Type() string {
}
// Start implements Volume.
-func (v *AzureBlobVolume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error {
+func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
if v.ContainerName == "" {
return errors.New("no container name given")
}
@@ -186,9 +186,9 @@ func (v *AzureBlobVolume) Start(opsCounters, errCounters, ioBytes *prometheus.Co
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.container.stats.opsCounters = opsCounters.MustCurryWith(lbls)
- v.container.stats.errCounters = errCounters.MustCurryWith(lbls)
- v.container.stats.ioBytes = ioBytes.MustCurryWith(lbls)
+ v.container.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
+ v.container.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
+ v.container.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
return nil
}
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
index 0902b99eb..7f056d833 100644
--- a/services/keepstore/config.go
+++ b/services/keepstore/config.go
@@ -146,7 +146,7 @@ func (cfg *Config) Start(reg *prometheus.Registry) error {
}
vm := newVolumeMetricsVecs(reg)
for _, v := range cfg.Volumes {
- if err := v.Start(vm.opsCounters, vm.errCounters, vm.ioBytes); err != nil {
+ if err := v.Start(vm); err != nil {
return fmt.Errorf("volume %s: %s", v, err)
}
log.Printf("Using volume %v (writable=%v)", v, v.Writable())
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 0bf0d9a10..33279e5ef 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -199,7 +199,7 @@ func (*S3Volume) Type() string {
// Start populates private fields and verifies the configuration is
// valid.
-func (v *S3Volume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error {
+func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
region, ok := aws.Regions[v.Region]
if v.Endpoint == "" {
if !ok {
@@ -251,9 +251,9 @@ func (v *S3Volume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVe
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.bucket.stats.opsCounters = opsCounters.MustCurryWith(lbls)
- v.bucket.stats.errCounters = errCounters.MustCurryWith(lbls)
- v.bucket.stats.ioBytes = ioBytes.MustCurryWith(lbls)
+ v.bucket.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
+ v.bucket.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
+ v.bucket.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
return nil
}
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index baa9dda9e..6fd6b653e 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -172,7 +172,7 @@ func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Con
vol.Endpoint = srv.URL
v = &TestableS3Volume{S3Volume: &vol}
metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics.opsCounters, metrics.errCounters, metrics.ioBytes)
+ v.Start(metrics)
ctx, cancel := context.WithCancel(context.Background())
@@ -433,7 +433,7 @@ func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration,
serverClock: clock,
}
metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics.opsCounters, metrics.errCounters, metrics.ioBytes)
+ v.Start(metrics)
err = v.bucket.PutBucket(s3.ACL("private"))
c.Assert(err, check.IsNil)
return v
@@ -451,7 +451,7 @@ Volumes:
c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
}
-func (v *TestableS3Volume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error {
+func (v *TestableS3Volume) Start(vm *volumeMetricsVecs) error {
tmp, err := ioutil.TempFile("", "keepstore")
v.c.Assert(err, check.IsNil)
defer os.Remove(tmp.Name())
@@ -462,7 +462,7 @@ func (v *TestableS3Volume) Start(opsCounters, errCounters, ioBytes *prometheus.C
v.S3Volume.AccessKeyFile = tmp.Name()
v.S3Volume.SecretKeyFile = tmp.Name()
- v.c.Assert(v.S3Volume.Start(opsCounters, errCounters, ioBytes), check.IsNil)
+ v.c.Assert(v.S3Volume.Start(vm), check.IsNil)
return nil
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 8bdbf9362..32628f729 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -220,7 +220,7 @@ func (v *UnixVolume) Type() string {
}
// Start implements Volume
-func (v *UnixVolume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error {
+func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
if v.Serialize {
v.locker = &sync.Mutex{}
}
@@ -232,9 +232,9 @@ func (v *UnixVolume) Start(opsCounters, errCounters, ioBytes *prometheus.Counter
}
// Set up prometheus metrics
lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.os.stats.opsCounters = opsCounters.MustCurryWith(lbls)
- v.os.stats.errCounters = errCounters.MustCurryWith(lbls)
- v.os.stats.ioBytes = ioBytes.MustCurryWith(lbls)
+ v.os.stats.opsCounters = vm.opsCounters.MustCurryWith(lbls)
+ v.os.stats.errCounters = vm.errCounters.MustCurryWith(lbls)
+ v.os.stats.ioBytes = vm.ioBytes.MustCurryWith(lbls)
_, err := v.os.Stat(v.Root)
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index fe20f33d6..8b2bc2d54 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -117,7 +117,7 @@ func TestReplicationDefault1(t *testing.T) {
ReadOnly: true,
}
metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- if err := v.Start(metrics.opsCounters, metrics.errCounters, metrics.ioBytes); err != nil {
+ if err := v.Start(metrics); err != nil {
t.Error(err)
}
if got := v.Replication(); got != 1 {
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 39e2d5206..52b9b1b24 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -14,7 +14,6 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
- "github.com/prometheus/client_golang/prometheus"
)
type BlockWriter interface {
@@ -40,7 +39,7 @@ type Volume interface {
// Do whatever private setup tasks and configuration checks
// are needed. Return non-nil if the volume is unusable (e.g.,
// invalid config).
- Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error
+ Start(vm *volumeMetricsVecs) error
// Get a block: copy the block data into buf, and return the
// number of bytes copied.
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index df6a09e3a..9a745b629 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -15,8 +15,6 @@ import (
"strings"
"sync"
"time"
-
- "github.com/prometheus/client_golang/prometheus"
)
// A TestableVolume allows test suites to manipulate the state of an
@@ -214,7 +212,7 @@ func (v *MockVolume) Type() string {
return "Mock"
}
-func (v *MockVolume) Start(opsCounters, errCounters, ioBytes *prometheus.CounterVec) error {
+func (v *MockVolume) Start(vm *volumeMetricsVecs) error {
return nil
}
commit 646ea4328be842f4baa194205618c01910ec49db
Merge: 92d1b6e05 35a23c29e
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Mar 6 18:09:28 2019 -0300
Merge branch 'master' into 13937-keepstore-prometheus
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list