[ARVADOS] updated: 1.1.4-723-g413db07b4
Git user
git at public.curoverse.com
Mon Jul 30 15:54:14 EDT 2018
Summary of changes:
.../app/assets/javascripts/models/session_db.js | 11 +-
doc/_config.yml | 16 +-
doc/admin/health-checks.html.textile.liquid | 70 +++++
doc/admin/management-token.html.textile.liquid | 56 ++++
doc/admin/metrics.html.textile.liquid | 163 +++++++++++
doc/api/execution.html.textile.liquid | 28 ++
doc/css/images.css | 8 +
doc/install/cheat_sheet.html.textile.liquid | 2 +-
doc/user/composer/c1.png | Bin 0 -> 17420 bytes
doc/user/composer/c10.png | Bin 0 -> 33020 bytes
doc/user/composer/c11.png | Bin 0 -> 26144 bytes
doc/user/composer/c12.png | Bin 0 -> 175462 bytes
doc/user/composer/c13.png | Bin 0 -> 11645 bytes
doc/user/composer/c14.png | Bin 0 -> 13116 bytes
doc/user/composer/c15.png | Bin 0 -> 50066 bytes
doc/user/composer/c16.png | Bin 0 -> 32472 bytes
doc/user/composer/c17.png | Bin 0 -> 34259 bytes
doc/user/composer/c18.png | Bin 0 -> 59780 bytes
doc/user/composer/c19.png | Bin 0 -> 53580 bytes
doc/user/composer/c2.png | Bin 0 -> 41693 bytes
doc/user/composer/c20.png | Bin 0 -> 23625 bytes
doc/user/composer/c21.png | Bin 0 -> 11379 bytes
doc/user/composer/c22.png | Bin 0 -> 13713 bytes
doc/user/composer/c23.png | Bin 0 -> 12253 bytes
doc/user/composer/c24.png | Bin 0 -> 19931 bytes
doc/user/composer/c2b.png | Bin 0 -> 14285 bytes
doc/user/composer/c2c.png | Bin 0 -> 9123 bytes
doc/user/composer/c3.png | Bin 0 -> 38471 bytes
doc/user/composer/c4.png | Bin 0 -> 8811 bytes
doc/user/composer/c5.png | Bin 0 -> 24020 bytes
doc/user/composer/c6.png | Bin 0 -> 41978 bytes
doc/user/composer/c7.png | Bin 0 -> 26904 bytes
doc/user/composer/c8.png | Bin 0 -> 44358 bytes
doc/user/composer/c9.png | Bin 0 -> 18920 bytes
doc/user/composer/composer.html.textile.liquid | 119 ++++++++
doc/user/cwl/cwl-style.html.textile.liquid | 2 +
doc/user/topics/arv-docker.html.textile.liquid | 10 +-
...tutorial-workflow-workbench.html.textile.liquid | 4 +
.../writing-cwl-workflow.html.textile.liquid | 6 +-
lib/controller/federation.go | 117 ++++++++
lib/controller/federation_test.go | 301 +++++++++++++++++++++
lib/controller/handler.go | 146 +++++-----
lib/controller/handler_test.go | 14 +-
lib/controller/proxy.go | 83 ++++++
lib/controller/server_test.go | 68 +++++
lib/crunchstat/crunchstat.go | 64 ++++-
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cli/bin/crunch-job | 10 +
sdk/go/arvados/api_client_authorization.go | 4 +
sdk/go/arvados/config.go | 26 +-
sdk/go/arvados/postgresql.go | 20 ++
sdk/go/arvadostest/fixtures.go | 2 +
sdk/go/auth/auth.go | 24 +-
sdk/go/auth/salt.go | 48 ++++
sdk/python/tests/run_test_server.py | 17 +-
sdk/ruby/lib/arvados/google_api_client.rb | 5 +
services/api/Gemfile | 3 +
services/api/Gemfile.lock | 13 +-
services/api/app/controllers/static_controller.rb | 2 +-
services/api/db/structure.sql | 3 +-
services/api/lib/crunch_dispatch.rb | 5 +
services/api/test/integration/cross_origin_test.rb | 4 +-
services/crunch-run/crunchrun.go | 1 +
services/crunchstat/crunchstat.go | 4 +-
services/login-sync/arvados-login-sync.gemspec | 2 +-
services/nodemanager/tests/fake_azure.cfg.template | 6 +-
services/ws/config.go | 4 +-
services/ws/event_source.go | 16 --
services/ws/event_source_test.go | 30 +-
services/ws/server.go | 2 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 8 +-
tools/arvbox/lib/arvbox/docker/common.sh | 4 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 2 +-
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 23 +-
.../lib/arvbox/docker/service/api/run-service | 4 +-
.../docker/service/arv-git-httpd/run-service | 2 +-
.../lib/arvbox/docker/service/composer/run-service | 2 +-
.../service/{api => controller}/log/main/.gitstub | 0
.../docker/service/{api => controller}/log/run | 0
.../lib/arvbox/docker/service/controller/run | 50 ++++
.../service/crunch-dispatch-local/run-service | 2 +-
tools/arvbox/lib/arvbox/docker/service/doc/run | 10 +-
.../lib/arvbox/docker/service/doc/run-service | 31 +--
.../lib/arvbox/docker/service/gitolite/run-service | 4 +-
.../lib/arvbox/docker/service/keep-web/run-service | 2 +-
.../arvbox/docker/service/keepproxy/run-service | 2 +-
.../service/{api => nginx}/log/main/.gitstub | 0
.../arvbox/docker/service/{api => nginx}/log/run | 0
.../lib/arvbox/docker/service/{api => nginx}/run | 0
.../lib/arvbox/docker/service/nginx/run-service | 54 ++++
tools/arvbox/lib/arvbox/docker/service/ready/run | 10 +-
.../lib/arvbox/docker/service/ready/run-service | 4 +-
.../lib/arvbox/docker/service/sso/run-service | 2 +-
tools/arvbox/lib/arvbox/docker/service/vm/run | 2 +-
.../lib/arvbox/docker/service/vm/run-service | 2 +-
.../arvbox/docker/service/websockets/run-service | 2 +-
.../arvbox/docker/service/workbench/run-service | 6 +-
...application_yml_override.py => yml_override.py} | 9 +-
99 files changed, 1555 insertions(+), 225 deletions(-)
create mode 100644 doc/admin/health-checks.html.textile.liquid
create mode 100644 doc/admin/management-token.html.textile.liquid
create mode 100644 doc/admin/metrics.html.textile.liquid
create mode 100644 doc/user/composer/c1.png
create mode 100644 doc/user/composer/c10.png
create mode 100644 doc/user/composer/c11.png
create mode 100644 doc/user/composer/c12.png
create mode 100644 doc/user/composer/c13.png
create mode 100644 doc/user/composer/c14.png
create mode 100644 doc/user/composer/c15.png
create mode 100644 doc/user/composer/c16.png
create mode 100644 doc/user/composer/c17.png
create mode 100644 doc/user/composer/c18.png
create mode 100644 doc/user/composer/c19.png
create mode 100644 doc/user/composer/c2.png
create mode 100644 doc/user/composer/c20.png
create mode 100644 doc/user/composer/c21.png
create mode 100644 doc/user/composer/c22.png
create mode 100644 doc/user/composer/c23.png
create mode 100644 doc/user/composer/c24.png
create mode 100644 doc/user/composer/c2b.png
create mode 100644 doc/user/composer/c2c.png
create mode 100644 doc/user/composer/c3.png
create mode 100644 doc/user/composer/c4.png
create mode 100644 doc/user/composer/c5.png
create mode 100644 doc/user/composer/c6.png
create mode 100644 doc/user/composer/c7.png
create mode 100644 doc/user/composer/c8.png
create mode 100644 doc/user/composer/c9.png
create mode 100644 doc/user/composer/composer.html.textile.liquid
create mode 100644 lib/controller/federation.go
create mode 100644 lib/controller/federation_test.go
create mode 100644 lib/controller/proxy.go
create mode 100644 lib/controller/server_test.go
create mode 100644 sdk/go/arvados/postgresql.go
create mode 100644 sdk/go/auth/salt.go
copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/controller/run
mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/doc/run
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/run (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/nginx/run-service
mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/ready/run
rename tools/arvbox/lib/arvbox/docker/{application_yml_override.py => yml_override.py} (79%)
discards 6d7e9f721e14a43976961753ef40cc35f2b2288f (commit)
via 413db07b4c81ea08663f90f31ee03227349d2be4 (commit)
via 996776292b5bfaeadb89457943ea1fba0700d030 (commit)
via cd5cd1855c02319ca31410943c4d944133294b17 (commit)
via 676d3522be04f774ba3a7db38d88b284252e0408 (commit)
via 43175f034f8015c6ca89a04fa8d9a322cf156265 (commit)
via c0b02133b2988b05772de5f2716d152aec4864ca (commit)
via 904e0b03fd558491d9dee39d7679a6c77eb6abd5 (commit)
via d6c7e2952413c33a4af9b1599e4000b9df622661 (commit)
via 35ac725af39a6326323cac66b28c532e49e27510 (commit)
via 8fdac0b6f954b6265798390b95e61f8192b85630 (commit)
via f5d7521ca506d63f631c603938cac5f40663bcca (commit)
via 22f96cd164a9a86e577b1ca4de24a281e7fbfa17 (commit)
via 641ef213571f793bb290a182dee3c4325bc85096 (commit)
via cdc146d447ae1ba42275d4ce7f34bd0b04158306 (commit)
via 92fba2405fc9cd6bbb6b1c4f5a7774f15b242696 (commit)
via c775680aefe1b6813b50f0a97dc073caeb4befbc (commit)
via 85f6919fae720c41474e60f5c21710f70e446070 (commit)
via c41b3893a26d542142de86a641e43077df3bda16 (commit)
via 3ae140fa072b2f2fbc8576c20ffd81fe463e78a5 (commit)
via b211e857d304f7fbe8787d2b65a307da841d047b (commit)
via 57153b5a518646399e9b62ad693ce213dae21972 (commit)
via 0853b16ad82a6b4411f04910d0035a2c653cfc29 (commit)
via b9a322e1aba0030f2dc13c2cf71fb581876c68cc (commit)
via d63984fdc1cf7c48cab9a57640415e85c8bfc48f (commit)
via 0a3d7a02236cbec448203a1b2218b5e0630d1c00 (commit)
via a77e606772cd4909f2401f55cd7c3c08f8325fb8 (commit)
via cec8e2705d260c9df1042858941419a3b9160c0e (commit)
via 9cc572d6a44262e21251372e28b549cfc09e681a (commit)
via 115e837fd16eecd80868267b0a6f99a531d4cc93 (commit)
via febdebbb58592be73dcf7d4bd4b2c7ff96657741 (commit)
via bef56b9a22efac9ce73006623080e84a0b57f243 (commit)
via b3b3d214f54d7909f12e60a911b02676818400b1 (commit)
via 711711827bb0c3564836707bb7d4453c60c6a98c (commit)
via 3271e860e871b19a2f3be68376c57c2ce88993ac (commit)
via 6c66177816fe128595454e9890d2c82a984cbc90 (commit)
via 66fc74001e777ac7ceff2b02cfc459b1368f42f3 (commit)
via b3c1156e1d4ed9f15a36bc51f2cd125a65aab3d3 (commit)
via a93ef946eb1e73ee190ea4ff19c4f9278235530c (commit)
via 5eda2f0ee6fcab67d3a74517a85946435931c67b (commit)
via 8d3ecc67ad5d208e5327b81dffda0c10ef0a8378 (commit)
via 3934348cf5166ed53248bed7f2aa3b98d067cbc2 (commit)
via acf545571425b2a2d8a26cb703f75926b6b2a987 (commit)
via b29c13b372bd39075eabda49ea57b0cb6d5a8452 (commit)
via 81d5bd428962c1819c0c60de5bc5d93710798424 (commit)
via d22f92638664d9744a38e60dce9f0d95a2d393f0 (commit)
via 6b8a3d3d912573ab432c319121eaffd0bbe2216b (commit)
via 424298694945c9f66676bd401753be44f6fc5fa8 (commit)
via 64a357242cc7ba9fb877a9b3d622160318098c5c (commit)
via 57074c15587686e066d7e44adec7b6c7154d6ac4 (commit)
via 95de13bdab14eb803a4c9e2243df50e1eb0df69a (commit)
via e1633d0a8365f63db56777524b1281d90bd9ee46 (commit)
via 890195dc78d25d9a8f1e513197e4adc71be4f146 (commit)
via 967821ec0ac00aef677f943e709ea079ebc30f78 (commit)
via 59cf374273bfae0baec8370526a747bbb8e5edb7 (commit)
via a6ab70e907a47e8e28aae2dd3eb357aa72c8b673 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (6d7e9f721e14a43976961753ef40cc35f2b2288f)
\
N -- N -- N (413db07b4c81ea08663f90f31ee03227349d2be4)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 413db07b4c81ea08663f90f31ee03227349d2be4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Jul 30 12:03:07 2018 -0400
13198: Add cache stats to metrics.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 636c0306c..e669e326c 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -732,7 +732,7 @@ do_test_once() {
do_install() {
skipit=false
- if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+ if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
retry do_install_once ${@}
else
skipit=true
diff --git a/sdk/go/httpserver/metrics.go b/sdk/go/httpserver/metrics.go
index 77525a80f..b52068e95 100644
--- a/sdk/go/httpserver/metrics.go
+++ b/sdk/go/httpserver/metrics.go
@@ -99,10 +99,17 @@ func (m *metrics) ServeAPI(next http.Handler) http.Handler {
// For the metrics to be accurate, the caller must ensure every
// request passed to the Handler also passes through
// LogRequests(logger, ...), and vice versa.
-func Instrument(logger *logrus.Logger, next http.Handler) Handler {
+//
+// If registry is nil, a new registry is created.
+//
+// If logger is nil, logrus.StandardLogger() is used.
+func Instrument(registry *prometheus.Registry, logger *logrus.Logger, next http.Handler) Handler {
if logger == nil {
logger = logrus.StandardLogger()
}
+ if registry == nil {
+ registry = prometheus.NewRegistry()
+ }
reqDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "request_duration_seconds",
Help: "Summary of request duration.",
@@ -111,7 +118,6 @@ func Instrument(logger *logrus.Logger, next http.Handler) Handler {
Name: "time_to_status_seconds",
Help: "Summary of request TTFB.",
}, []string{"code", "method"})
- registry := prometheus.NewRegistry()
registry.MustRegister(timeToStatus)
registry.MustRegister(reqDuration)
m := &metrics{
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 59e8de3bc..b2bab7821 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -12,8 +12,11 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
+ "github.com/prometheus/client_golang/prometheus"
)
+const metricsUpdateInterval = time.Second / 10
+
type cache struct {
TTL arvados.Duration
UUIDTTL arvados.Duration
@@ -22,13 +25,16 @@ type cache struct {
MaxPermissionEntries int
MaxUUIDEntries int
+ registry *prometheus.Registry
stats cacheStats
+ metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
setupOnce sync.Once
}
+// cacheStats is EOL - add new metrics to cacheMetrics instead
type cacheStats struct {
Requests uint64 `json:"Cache.Requests"`
CollectionBytes uint64 `json:"Cache.CollectionBytes"`
@@ -39,6 +45,68 @@ type cacheStats struct {
APICalls uint64 `json:"Cache.APICalls"`
}
+type cacheMetrics struct {
+ requests prometheus.Counter
+ collectionBytes prometheus.Gauge
+ collectionEntries prometheus.Gauge
+ collectionHits prometheus.Counter
+ pdhHits prometheus.Counter
+ permissionHits prometheus.Counter
+ apiCalls prometheus.Counter
+}
+
+func (m *cacheMetrics) setup(reg *prometheus.Registry) {
+ m.requests = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "requests",
+ Help: "Number of targetID-to-manifest lookups handled.",
+ })
+ reg.MustRegister(m.requests)
+ m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "hits",
+ Help: "Number of pdh-to-manifest cache hits.",
+ })
+ reg.MustRegister(m.collectionHits)
+ m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "pdh_hits",
+ Help: "Number of uuid-to-pdh cache hits.",
+ })
+ reg.MustRegister(m.pdhHits)
+ m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "permission_hits",
+ Help: "Number of targetID-to-permission cache hits.",
+ })
+ reg.MustRegister(m.permissionHits)
+ m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "api_calls",
+ Help: "Number of outgoing API calls made by cache.",
+ })
+ reg.MustRegister(m.apiCalls)
+ m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifest_bytes",
+ Help: "Total size of all manifests in cache.",
+ })
+ reg.MustRegister(m.collectionBytes)
+ m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifests",
+ Help: "Number of manifests in cache.",
+ })
+ reg.MustRegister(m.collectionEntries)
+}
+
type cachedPDH struct {
expire time.Time
pdh string
@@ -67,6 +135,22 @@ func (c *cache) setup() {
if err != nil {
panic(err)
}
+
+ reg := c.registry
+ if reg == nil {
+ reg = prometheus.NewRegistry()
+ }
+ c.metrics.setup(reg)
+ go func() {
+ for range time.Tick(metricsUpdateInterval) {
+ c.updateGauges()
+ }
+ }()
+}
+
+func (c *cache) updateGauges() {
+ c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
+ c.metrics.collectionEntries.Set(float64(c.collections.Len()))
}
var selectPDH = map[string]interface{}{
@@ -113,6 +197,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
c.setupOnce.Do(c.setup)
atomic.AddUint64(&c.stats.Requests, 1)
+ c.metrics.requests.Inc()
permOK := false
permKey := arv.ApiToken + "\000" + targetID
@@ -124,6 +209,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
} else {
permOK = true
atomic.AddUint64(&c.stats.PermissionHits, 1)
+ c.metrics.permissionHits.Inc()
}
}
@@ -137,6 +223,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
} else {
pdh = ent.pdh
atomic.AddUint64(&c.stats.PDHHits, 1)
+ c.metrics.pdhHits.Inc()
}
}
@@ -153,6 +240,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
// _and_ the current token has permission, we can
// use our cached manifest.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
var current arvados.Collection
err := arv.Get("collections", targetID, selectPDH, ¤t)
if err != nil {
@@ -181,6 +269,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
// Collection manifest is not cached.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
err := arv.Get("collections", targetID, nil, &collection)
if err != nil {
return nil, err
@@ -261,16 +350,16 @@ func (c *cache) collectionBytes() uint64 {
}
func (c *cache) lookupCollection(key string) *arvados.Collection {
- if ent, cached := c.collections.Get(key); !cached {
+ e, cached := c.collections.Get(key)
+ if !cached {
+ return nil
+ }
+ ent := e.(*cachedCollection)
+ if ent.expire.Before(time.Now()) {
+ c.collections.Remove(key)
return nil
- } else {
- ent := ent.(*cachedCollection)
- if ent.expire.Before(time.Now()) {
- c.collections.Remove(key)
- return nil
- } else {
- atomic.AddUint64(&c.stats.CollectionHits, 1)
- return ent.collection
- }
}
+ atomic.AddUint64(&c.stats.CollectionHits, 1)
+ c.metrics.collectionHits.Inc()
+ return ent.collection
}
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 58ec348c8..68ff8a7b0 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -8,6 +8,7 @@ import (
"net/http"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/prometheus/client_golang/prometheus"
)
type server struct {
@@ -17,7 +18,9 @@ type server struct {
func (srv *server) Start() error {
h := &handler{Config: srv.Config}
- mh := httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+ reg := prometheus.NewRegistry()
+ h.Config.Cache.registry = reg
+ mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
srv.Handler = mh
srv.Addr = srv.Config.Listen
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 6688cc2ee..7e738cb9f 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -306,14 +306,19 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
- req.Host = arvadostest.FooCollection + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp, err = http.DefaultClient.Do(req)
- c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- buf, _ := ioutil.ReadAll(resp.Body)
- c.Check(buf, check.DeepEquals, []byte("foo"))
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ s.testServer.Config.Cache.updateGauges()
req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
resp, err = http.DefaultClient.Do(req)
@@ -327,6 +332,12 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
Value float64
}
}
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
var ents []struct {
Name string
Help string
@@ -336,24 +347,38 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
Name string
Value string
}
+ Counter counter
+ Gauge gauge
Summary summary
}
}
json.NewDecoder(resp.Body).Decode(&ents)
- flat := map[string]summary{}
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
for _, e := range ents {
for _, m := range e.Metric {
labels := map[string]string{}
for _, lbl := range m.Label {
labels[lbl.Name] = lbl.Value
}
- flat[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
}
}
- c.Check(flat["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
- c.Check(flat["request_duration_seconds/get/200"].SampleCount, check.Equals, "2")
- c.Check(flat["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
- c.Check(flat["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+ c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+ // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index d19be61e9..c31ab9c2e 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -88,7 +88,7 @@ func MakeRESTRouter() http.Handler {
rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
- stack := httpserver.Instrument(nil,
+ stack := httpserver.Instrument(nil, nil,
httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
return stack.ServeAPI(stack)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list