[ARVADOS] updated: 1.1.4-723-g413db07b4

Git user git at public.curoverse.com
Mon Jul 30 15:54:14 EDT 2018


Summary of changes:
 .../app/assets/javascripts/models/session_db.js    |  11 +-
 doc/_config.yml                                    |  16 +-
 doc/admin/health-checks.html.textile.liquid        |  70 +++++
 doc/admin/management-token.html.textile.liquid     |  56 ++++
 doc/admin/metrics.html.textile.liquid              | 163 +++++++++++
 doc/api/execution.html.textile.liquid              |  28 ++
 doc/css/images.css                                 |   8 +
 doc/install/cheat_sheet.html.textile.liquid        |   2 +-
 doc/user/composer/c1.png                           | Bin 0 -> 17420 bytes
 doc/user/composer/c10.png                          | Bin 0 -> 33020 bytes
 doc/user/composer/c11.png                          | Bin 0 -> 26144 bytes
 doc/user/composer/c12.png                          | Bin 0 -> 175462 bytes
 doc/user/composer/c13.png                          | Bin 0 -> 11645 bytes
 doc/user/composer/c14.png                          | Bin 0 -> 13116 bytes
 doc/user/composer/c15.png                          | Bin 0 -> 50066 bytes
 doc/user/composer/c16.png                          | Bin 0 -> 32472 bytes
 doc/user/composer/c17.png                          | Bin 0 -> 34259 bytes
 doc/user/composer/c18.png                          | Bin 0 -> 59780 bytes
 doc/user/composer/c19.png                          | Bin 0 -> 53580 bytes
 doc/user/composer/c2.png                           | Bin 0 -> 41693 bytes
 doc/user/composer/c20.png                          | Bin 0 -> 23625 bytes
 doc/user/composer/c21.png                          | Bin 0 -> 11379 bytes
 doc/user/composer/c22.png                          | Bin 0 -> 13713 bytes
 doc/user/composer/c23.png                          | Bin 0 -> 12253 bytes
 doc/user/composer/c24.png                          | Bin 0 -> 19931 bytes
 doc/user/composer/c2b.png                          | Bin 0 -> 14285 bytes
 doc/user/composer/c2c.png                          | Bin 0 -> 9123 bytes
 doc/user/composer/c3.png                           | Bin 0 -> 38471 bytes
 doc/user/composer/c4.png                           | Bin 0 -> 8811 bytes
 doc/user/composer/c5.png                           | Bin 0 -> 24020 bytes
 doc/user/composer/c6.png                           | Bin 0 -> 41978 bytes
 doc/user/composer/c7.png                           | Bin 0 -> 26904 bytes
 doc/user/composer/c8.png                           | Bin 0 -> 44358 bytes
 doc/user/composer/c9.png                           | Bin 0 -> 18920 bytes
 doc/user/composer/composer.html.textile.liquid     | 119 ++++++++
 doc/user/cwl/cwl-style.html.textile.liquid         |   2 +
 doc/user/topics/arv-docker.html.textile.liquid     |  10 +-
 ...tutorial-workflow-workbench.html.textile.liquid |   4 +
 .../writing-cwl-workflow.html.textile.liquid       |   6 +-
 lib/controller/federation.go                       | 117 ++++++++
 lib/controller/federation_test.go                  | 301 +++++++++++++++++++++
 lib/controller/handler.go                          | 146 +++++-----
 lib/controller/handler_test.go                     |  14 +-
 lib/controller/proxy.go                            |  83 ++++++
 lib/controller/server_test.go                      |  68 +++++
 lib/crunchstat/crunchstat.go                       |  64 ++++-
 sdk/cli/arvados-cli.gemspec                        |   2 +-
 sdk/cli/bin/crunch-job                             |  10 +
 sdk/go/arvados/api_client_authorization.go         |   4 +
 sdk/go/arvados/config.go                           |  26 +-
 sdk/go/arvados/postgresql.go                       |  20 ++
 sdk/go/arvadostest/fixtures.go                     |   2 +
 sdk/go/auth/auth.go                                |  24 +-
 sdk/go/auth/salt.go                                |  48 ++++
 sdk/python/tests/run_test_server.py                |  17 +-
 sdk/ruby/lib/arvados/google_api_client.rb          |   5 +
 services/api/Gemfile                               |   3 +
 services/api/Gemfile.lock                          |  13 +-
 services/api/app/controllers/static_controller.rb  |   2 +-
 services/api/db/structure.sql                      |   3 +-
 services/api/lib/crunch_dispatch.rb                |   5 +
 services/api/test/integration/cross_origin_test.rb |   4 +-
 services/crunch-run/crunchrun.go                   |   1 +
 services/crunchstat/crunchstat.go                  |   4 +-
 services/login-sync/arvados-login-sync.gemspec     |   2 +-
 services/nodemanager/tests/fake_azure.cfg.template |   6 +-
 services/ws/config.go                              |   4 +-
 services/ws/event_source.go                        |  16 --
 services/ws/event_source_test.go                   |  30 +-
 services/ws/server.go                              |   2 +-
 tools/arvbox/lib/arvbox/docker/Dockerfile.base     |   2 +-
 tools/arvbox/lib/arvbox/docker/api-setup.sh        |   8 +-
 tools/arvbox/lib/arvbox/docker/common.sh           |   4 +-
 tools/arvbox/lib/arvbox/docker/crunch-setup.sh     |   2 +-
 tools/arvbox/lib/arvbox/docker/keep-setup.sh       |  23 +-
 .../lib/arvbox/docker/service/api/run-service      |   4 +-
 .../docker/service/arv-git-httpd/run-service       |   2 +-
 .../lib/arvbox/docker/service/composer/run-service |   2 +-
 .../service/{api => controller}/log/main/.gitstub  |   0
 .../docker/service/{api => controller}/log/run     |   0
 .../lib/arvbox/docker/service/controller/run       |  50 ++++
 .../service/crunch-dispatch-local/run-service      |   2 +-
 tools/arvbox/lib/arvbox/docker/service/doc/run     |  10 +-
 .../lib/arvbox/docker/service/doc/run-service      |  31 +--
 .../lib/arvbox/docker/service/gitolite/run-service |   4 +-
 .../lib/arvbox/docker/service/keep-web/run-service |   2 +-
 .../arvbox/docker/service/keepproxy/run-service    |   2 +-
 .../service/{api => nginx}/log/main/.gitstub       |   0
 .../arvbox/docker/service/{api => nginx}/log/run   |   0
 .../lib/arvbox/docker/service/{api => nginx}/run   |   0
 .../lib/arvbox/docker/service/nginx/run-service    |  54 ++++
 tools/arvbox/lib/arvbox/docker/service/ready/run   |  10 +-
 .../lib/arvbox/docker/service/ready/run-service    |   4 +-
 .../lib/arvbox/docker/service/sso/run-service      |   2 +-
 tools/arvbox/lib/arvbox/docker/service/vm/run      |   2 +-
 .../lib/arvbox/docker/service/vm/run-service       |   2 +-
 .../arvbox/docker/service/websockets/run-service   |   2 +-
 .../arvbox/docker/service/workbench/run-service    |   6 +-
 ...application_yml_override.py => yml_override.py} |   9 +-
 99 files changed, 1555 insertions(+), 225 deletions(-)
 create mode 100644 doc/admin/health-checks.html.textile.liquid
 create mode 100644 doc/admin/management-token.html.textile.liquid
 create mode 100644 doc/admin/metrics.html.textile.liquid
 create mode 100644 doc/user/composer/c1.png
 create mode 100644 doc/user/composer/c10.png
 create mode 100644 doc/user/composer/c11.png
 create mode 100644 doc/user/composer/c12.png
 create mode 100644 doc/user/composer/c13.png
 create mode 100644 doc/user/composer/c14.png
 create mode 100644 doc/user/composer/c15.png
 create mode 100644 doc/user/composer/c16.png
 create mode 100644 doc/user/composer/c17.png
 create mode 100644 doc/user/composer/c18.png
 create mode 100644 doc/user/composer/c19.png
 create mode 100644 doc/user/composer/c2.png
 create mode 100644 doc/user/composer/c20.png
 create mode 100644 doc/user/composer/c21.png
 create mode 100644 doc/user/composer/c22.png
 create mode 100644 doc/user/composer/c23.png
 create mode 100644 doc/user/composer/c24.png
 create mode 100644 doc/user/composer/c2b.png
 create mode 100644 doc/user/composer/c2c.png
 create mode 100644 doc/user/composer/c3.png
 create mode 100644 doc/user/composer/c4.png
 create mode 100644 doc/user/composer/c5.png
 create mode 100644 doc/user/composer/c6.png
 create mode 100644 doc/user/composer/c7.png
 create mode 100644 doc/user/composer/c8.png
 create mode 100644 doc/user/composer/c9.png
 create mode 100644 doc/user/composer/composer.html.textile.liquid
 create mode 100644 lib/controller/federation.go
 create mode 100644 lib/controller/federation_test.go
 create mode 100644 lib/controller/proxy.go
 create mode 100644 lib/controller/server_test.go
 create mode 100644 sdk/go/arvados/postgresql.go
 create mode 100644 sdk/go/auth/salt.go
 copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/main/.gitstub (100%)
 copy tools/arvbox/lib/arvbox/docker/service/{api => controller}/log/run (100%)
 create mode 100755 tools/arvbox/lib/arvbox/docker/service/controller/run
 mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/doc/run
 copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/main/.gitstub (100%)
 copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/log/run (100%)
 copy tools/arvbox/lib/arvbox/docker/service/{api => nginx}/run (100%)
 create mode 100755 tools/arvbox/lib/arvbox/docker/service/nginx/run-service
 mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/ready/run
 rename tools/arvbox/lib/arvbox/docker/{application_yml_override.py => yml_override.py} (79%)

  discards  6d7e9f721e14a43976961753ef40cc35f2b2288f (commit)
       via  413db07b4c81ea08663f90f31ee03227349d2be4 (commit)
       via  996776292b5bfaeadb89457943ea1fba0700d030 (commit)
       via  cd5cd1855c02319ca31410943c4d944133294b17 (commit)
       via  676d3522be04f774ba3a7db38d88b284252e0408 (commit)
       via  43175f034f8015c6ca89a04fa8d9a322cf156265 (commit)
       via  c0b02133b2988b05772de5f2716d152aec4864ca (commit)
       via  904e0b03fd558491d9dee39d7679a6c77eb6abd5 (commit)
       via  d6c7e2952413c33a4af9b1599e4000b9df622661 (commit)
       via  35ac725af39a6326323cac66b28c532e49e27510 (commit)
       via  8fdac0b6f954b6265798390b95e61f8192b85630 (commit)
       via  f5d7521ca506d63f631c603938cac5f40663bcca (commit)
       via  22f96cd164a9a86e577b1ca4de24a281e7fbfa17 (commit)
       via  641ef213571f793bb290a182dee3c4325bc85096 (commit)
       via  cdc146d447ae1ba42275d4ce7f34bd0b04158306 (commit)
       via  92fba2405fc9cd6bbb6b1c4f5a7774f15b242696 (commit)
       via  c775680aefe1b6813b50f0a97dc073caeb4befbc (commit)
       via  85f6919fae720c41474e60f5c21710f70e446070 (commit)
       via  c41b3893a26d542142de86a641e43077df3bda16 (commit)
       via  3ae140fa072b2f2fbc8576c20ffd81fe463e78a5 (commit)
       via  b211e857d304f7fbe8787d2b65a307da841d047b (commit)
       via  57153b5a518646399e9b62ad693ce213dae21972 (commit)
       via  0853b16ad82a6b4411f04910d0035a2c653cfc29 (commit)
       via  b9a322e1aba0030f2dc13c2cf71fb581876c68cc (commit)
       via  d63984fdc1cf7c48cab9a57640415e85c8bfc48f (commit)
       via  0a3d7a02236cbec448203a1b2218b5e0630d1c00 (commit)
       via  a77e606772cd4909f2401f55cd7c3c08f8325fb8 (commit)
       via  cec8e2705d260c9df1042858941419a3b9160c0e (commit)
       via  9cc572d6a44262e21251372e28b549cfc09e681a (commit)
       via  115e837fd16eecd80868267b0a6f99a531d4cc93 (commit)
       via  febdebbb58592be73dcf7d4bd4b2c7ff96657741 (commit)
       via  bef56b9a22efac9ce73006623080e84a0b57f243 (commit)
       via  b3b3d214f54d7909f12e60a911b02676818400b1 (commit)
       via  711711827bb0c3564836707bb7d4453c60c6a98c (commit)
       via  3271e860e871b19a2f3be68376c57c2ce88993ac (commit)
       via  6c66177816fe128595454e9890d2c82a984cbc90 (commit)
       via  66fc74001e777ac7ceff2b02cfc459b1368f42f3 (commit)
       via  b3c1156e1d4ed9f15a36bc51f2cd125a65aab3d3 (commit)
       via  a93ef946eb1e73ee190ea4ff19c4f9278235530c (commit)
       via  5eda2f0ee6fcab67d3a74517a85946435931c67b (commit)
       via  8d3ecc67ad5d208e5327b81dffda0c10ef0a8378 (commit)
       via  3934348cf5166ed53248bed7f2aa3b98d067cbc2 (commit)
       via  acf545571425b2a2d8a26cb703f75926b6b2a987 (commit)
       via  b29c13b372bd39075eabda49ea57b0cb6d5a8452 (commit)
       via  81d5bd428962c1819c0c60de5bc5d93710798424 (commit)
       via  d22f92638664d9744a38e60dce9f0d95a2d393f0 (commit)
       via  6b8a3d3d912573ab432c319121eaffd0bbe2216b (commit)
       via  424298694945c9f66676bd401753be44f6fc5fa8 (commit)
       via  64a357242cc7ba9fb877a9b3d622160318098c5c (commit)
       via  57074c15587686e066d7e44adec7b6c7154d6ac4 (commit)
       via  95de13bdab14eb803a4c9e2243df50e1eb0df69a (commit)
       via  e1633d0a8365f63db56777524b1281d90bd9ee46 (commit)
       via  890195dc78d25d9a8f1e513197e4adc71be4f146 (commit)
       via  967821ec0ac00aef677f943e709ea079ebc30f78 (commit)
       via  59cf374273bfae0baec8370526a747bbb8e5edb7 (commit)
       via  a6ab70e907a47e8e28aae2dd3eb357aa72c8b673 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (6d7e9f721e14a43976961753ef40cc35f2b2288f)
            \
             N -- N -- N (413db07b4c81ea08663f90f31ee03227349d2be4)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 413db07b4c81ea08663f90f31ee03227349d2be4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Jul 30 12:03:07 2018 -0400

    13198: Add cache stats to metrics.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 636c0306c..e669e326c 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -732,7 +732,7 @@ do_test_once() {
 do_install() {
   skipit=false
 
-  if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+  if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
       retry do_install_once ${@}
   else
       skipit=true
diff --git a/sdk/go/httpserver/metrics.go b/sdk/go/httpserver/metrics.go
index 77525a80f..b52068e95 100644
--- a/sdk/go/httpserver/metrics.go
+++ b/sdk/go/httpserver/metrics.go
@@ -99,10 +99,17 @@ func (m *metrics) ServeAPI(next http.Handler) http.Handler {
 // For the metrics to be accurate, the caller must ensure every
 // request passed to the Handler also passes through
 // LogRequests(logger, ...), and vice versa.
-func Instrument(logger *logrus.Logger, next http.Handler) Handler {
+//
+// If registry is nil, a new registry is created.
+//
+// If logger is nil, logrus.StandardLogger() is used.
+func Instrument(registry *prometheus.Registry, logger *logrus.Logger, next http.Handler) Handler {
 	if logger == nil {
 		logger = logrus.StandardLogger()
 	}
+	if registry == nil {
+		registry = prometheus.NewRegistry()
+	}
 	reqDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
 		Name: "request_duration_seconds",
 		Help: "Summary of request duration.",
@@ -111,7 +118,6 @@ func Instrument(logger *logrus.Logger, next http.Handler) Handler {
 		Name: "time_to_status_seconds",
 		Help: "Summary of request TTFB.",
 	}, []string{"code", "method"})
-	registry := prometheus.NewRegistry()
 	registry.MustRegister(timeToStatus)
 	registry.MustRegister(reqDuration)
 	m := &metrics{
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 59e8de3bc..b2bab7821 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -12,8 +12,11 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
+const metricsUpdateInterval = time.Second / 10
+
 type cache struct {
 	TTL                  arvados.Duration
 	UUIDTTL              arvados.Duration
@@ -22,13 +25,16 @@ type cache struct {
 	MaxPermissionEntries int
 	MaxUUIDEntries       int
 
+	registry    *prometheus.Registry
 	stats       cacheStats
+	metrics     cacheMetrics
 	pdhs        *lru.TwoQueueCache
 	collections *lru.TwoQueueCache
 	permissions *lru.TwoQueueCache
 	setupOnce   sync.Once
 }
 
+// cacheStats is EOL - add new metrics to cacheMetrics instead
 type cacheStats struct {
 	Requests          uint64 `json:"Cache.Requests"`
 	CollectionBytes   uint64 `json:"Cache.CollectionBytes"`
@@ -39,6 +45,68 @@ type cacheStats struct {
 	APICalls          uint64 `json:"Cache.APICalls"`
 }
 
+type cacheMetrics struct {
+	requests          prometheus.Counter
+	collectionBytes   prometheus.Gauge
+	collectionEntries prometheus.Gauge
+	collectionHits    prometheus.Counter
+	pdhHits           prometheus.Counter
+	permissionHits    prometheus.Counter
+	apiCalls          prometheus.Counter
+}
+
+func (m *cacheMetrics) setup(reg *prometheus.Registry) {
+	m.requests = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "requests",
+		Help:      "Number of targetID-to-manifest lookups handled.",
+	})
+	reg.MustRegister(m.requests)
+	m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "hits",
+		Help:      "Number of pdh-to-manifest cache hits.",
+	})
+	reg.MustRegister(m.collectionHits)
+	m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "pdh_hits",
+		Help:      "Number of uuid-to-pdh cache hits.",
+	})
+	reg.MustRegister(m.pdhHits)
+	m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "permission_hits",
+		Help:      "Number of targetID-to-permission cache hits.",
+	})
+	reg.MustRegister(m.permissionHits)
+	m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "api_calls",
+		Help:      "Number of outgoing API calls made by cache.",
+	})
+	reg.MustRegister(m.apiCalls)
+	m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "cached_manifest_bytes",
+		Help:      "Total size of all manifests in cache.",
+	})
+	reg.MustRegister(m.collectionBytes)
+	m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "arvados",
+		Subsystem: "keepweb_collectioncache",
+		Name:      "cached_manifests",
+		Help:      "Number of manifests in cache.",
+	})
+	reg.MustRegister(m.collectionEntries)
+}
+
 type cachedPDH struct {
 	expire time.Time
 	pdh    string
@@ -67,6 +135,22 @@ func (c *cache) setup() {
 	if err != nil {
 		panic(err)
 	}
+
+	reg := c.registry
+	if reg == nil {
+		reg = prometheus.NewRegistry()
+	}
+	c.metrics.setup(reg)
+	go func() {
+		for range time.Tick(metricsUpdateInterval) {
+			c.updateGauges()
+		}
+	}()
+}
+
+func (c *cache) updateGauges() {
+	c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
+	c.metrics.collectionEntries.Set(float64(c.collections.Len()))
 }
 
 var selectPDH = map[string]interface{}{
@@ -113,6 +197,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 	c.setupOnce.Do(c.setup)
 
 	atomic.AddUint64(&c.stats.Requests, 1)
+	c.metrics.requests.Inc()
 
 	permOK := false
 	permKey := arv.ApiToken + "\000" + targetID
@@ -124,6 +209,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		} else {
 			permOK = true
 			atomic.AddUint64(&c.stats.PermissionHits, 1)
+			c.metrics.permissionHits.Inc()
 		}
 	}
 
@@ -137,6 +223,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		} else {
 			pdh = ent.pdh
 			atomic.AddUint64(&c.stats.PDHHits, 1)
+			c.metrics.pdhHits.Inc()
 		}
 	}
 
@@ -153,6 +240,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		// _and_ the current token has permission, we can
 		// use our cached manifest.
 		atomic.AddUint64(&c.stats.APICalls, 1)
+		c.metrics.apiCalls.Inc()
 		var current arvados.Collection
 		err := arv.Get("collections", targetID, selectPDH, &current)
 		if err != nil {
@@ -181,6 +269,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 
 	// Collection manifest is not cached.
 	atomic.AddUint64(&c.stats.APICalls, 1)
+	c.metrics.apiCalls.Inc()
 	err := arv.Get("collections", targetID, nil, &collection)
 	if err != nil {
 		return nil, err
@@ -261,16 +350,16 @@ func (c *cache) collectionBytes() uint64 {
 }
 
 func (c *cache) lookupCollection(key string) *arvados.Collection {
-	if ent, cached := c.collections.Get(key); !cached {
+	e, cached := c.collections.Get(key)
+	if !cached {
+		return nil
+	}
+	ent := e.(*cachedCollection)
+	if ent.expire.Before(time.Now()) {
+		c.collections.Remove(key)
 		return nil
-	} else {
-		ent := ent.(*cachedCollection)
-		if ent.expire.Before(time.Now()) {
-			c.collections.Remove(key)
-			return nil
-		} else {
-			atomic.AddUint64(&c.stats.CollectionHits, 1)
-			return ent.collection
-		}
 	}
+	atomic.AddUint64(&c.stats.CollectionHits, 1)
+	c.metrics.collectionHits.Inc()
+	return ent.collection
 }
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 58ec348c8..68ff8a7b0 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -8,6 +8,7 @@ import (
 	"net/http"
 
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type server struct {
@@ -17,7 +18,9 @@ type server struct {
 
 func (srv *server) Start() error {
 	h := &handler{Config: srv.Config}
-	mh := httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+	reg := prometheus.NewRegistry()
+	h.Config.Cache.registry = reg
+	mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
 	h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
 	srv.Handler = mh
 	srv.Addr = srv.Config.Listen
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 6688cc2ee..7e738cb9f 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -306,14 +306,19 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 	resp, err := http.DefaultClient.Do(req)
 	c.Assert(err, check.IsNil)
 	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
-	req, _ = http.NewRequest("GET", origin+"/foo", nil)
-	req.Host = arvadostest.FooCollection + ".example.com"
-	req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-	resp, err = http.DefaultClient.Do(req)
-	c.Assert(err, check.IsNil)
-	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
-	buf, _ := ioutil.ReadAll(resp.Body)
-	c.Check(buf, check.DeepEquals, []byte("foo"))
+	for i := 0; i < 2; i++ {
+		req, _ = http.NewRequest("GET", origin+"/foo", nil)
+		req.Host = arvadostest.FooCollection + ".example.com"
+		req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+		resp, err = http.DefaultClient.Do(req)
+		c.Assert(err, check.IsNil)
+		c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+		buf, _ := ioutil.ReadAll(resp.Body)
+		c.Check(buf, check.DeepEquals, []byte("foo"))
+		resp.Body.Close()
+	}
+
+	s.testServer.Config.Cache.updateGauges()
 
 	req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 	resp, err = http.DefaultClient.Do(req)
@@ -327,6 +332,12 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 			Value    float64
 		}
 	}
+	type counter struct {
+		Value int64
+	}
+	type gauge struct {
+		Value float64
+	}
 	var ents []struct {
 		Name   string
 		Help   string
@@ -336,24 +347,38 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 				Name  string
 				Value string
 			}
+			Counter counter
+			Gauge   gauge
 			Summary summary
 		}
 	}
 	json.NewDecoder(resp.Body).Decode(&ents)
-	flat := map[string]summary{}
+	summaries := map[string]summary{}
+	gauges := map[string]gauge{}
+	counters := map[string]counter{}
 	for _, e := range ents {
 		for _, m := range e.Metric {
 			labels := map[string]string{}
 			for _, lbl := range m.Label {
 				labels[lbl.Name] = lbl.Value
 			}
-			flat[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+			summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+			counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+			gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 		}
 	}
-	c.Check(flat["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
-	c.Check(flat["request_duration_seconds/get/200"].SampleCount, check.Equals, "2")
-	c.Check(flat["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
-	c.Check(flat["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+	c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+	c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+	c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+	c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+	c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+	c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+	c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+	c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+	c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+	c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+	// FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+	c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
 
 	// If the Host header indicates a collection, /metrics.json
 	// refers to a file in the collection -- the metrics handler
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index d19be61e9..c31ab9c2e 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -88,7 +88,7 @@ func MakeRESTRouter() http.Handler {
 
 	rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
 
-	stack := httpserver.Instrument(nil,
+	stack := httpserver.Instrument(nil, nil,
 		httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
 	return stack.ServeAPI(stack)
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list