[ARVADOS] updated: 1.1.4-669-g6d7e9f721
Git user
git at public.curoverse.com
Mon Jul 30 12:04:06 EDT 2018
Summary of changes:
README.md | 4 +-
apps/workbench/Gemfile.lock | 4 +-
build/run-library.sh | 3 +-
build/run-tests.sh | 2 +-
doc/_includes/_skip_sso_server_install.liquid | 12 -
...opy_pipeline_from_curoverse.html.textile.liquid | 20 +-
doc/install/index.html.textile.liquid | 4 +-
.../install-nodemanager.html.textile.liquid | 27 +
.../firstpipeline.html.textile.liquid | 2 +-
doc/user/cwl/cwl-runner.html.textile.liquid | 10 +-
doc/user/topics/arv-copy.html.textile.liquid | 4 +-
.../running-external-program.html.textile.liquid | 4 +-
sdk/R/R/Arvados.R | 2796 ++++++++++----------
sdk/R/R/ArvadosFile.R | 20 +-
sdk/R/R/Collection.R | 22 +-
sdk/R/R/CollectionTree.R | 10 +-
sdk/R/R/HttpParser.R | 6 +-
sdk/R/R/HttpRequest.R | 15 +-
sdk/R/R/RESTService.R | 15 +-
sdk/R/R/Subcollection.R | 14 +-
sdk/R/R/autoGenAPI.R | 16 +-
sdk/R/tests/testthat/fakes/FakeHttpParser.R | 6 +-
sdk/R/tests/testthat/fakes/FakeHttpRequest.R | 29 +-
sdk/R/tests/testthat/fakes/FakeRESTService.R | 4 +-
sdk/R/tests/testthat/test-ArvadosFile.R | 46 +-
sdk/R/tests/testthat/test-Collection.R | 6 +-
sdk/R/tests/testthat/test-CollectionTree.R | 20 +-
sdk/R/tests/testthat/test-HttpParser.R | 10 +-
sdk/R/tests/testthat/test-HttpRequest.R | 85 +-
sdk/R/tests/testthat/test-RESTService.R | 99 +-
sdk/R/tests/testthat/test-Subcollection.R | 62 +-
sdk/R/tests/testthat/test-util.R | 20 +-
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/go/dispatch/dispatch.go | 12 +-
sdk/go/httpserver/metrics.go | 10 +-
services/api/Gemfile | 2 -
services/api/Gemfile.lock | 17 +-
services/api/config/initializers/oj_mimic_json.rb | 11 +
services/api/config/initializers/time_format.rb | 2 +
services/api/lib/safe_json.rb | 6 +
services/api/test/test_helper.rb | 2 +-
.../crunch-dispatch-slurm_test.go | 14 +-
services/crunch-dispatch-slurm/squeue.go | 24 +-
services/crunch-dispatch-slurm/squeue_test.go | 44 +
services/keep-web/cache.go | 109 +-
services/keep-web/server.go | 5 +-
services/keep-web/server_test.go | 53 +-
services/keepproxy/keepproxy_test.go | 2 +-
services/keepstore/handlers.go | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 24 +-
.../arvnodeman/computenode/driver/__init__.py | 8 +-
.../arvnodeman/computenode/driver/azure.py | 4 +-
.../arvnodeman/computenode/driver/dummy.py | 2 +-
.../arvnodeman/computenode/driver/ec2.py | 4 +-
.../arvnodeman/computenode/driver/gce.py | 5 +-
services/nodemanager/arvnodeman/config.py | 3 +-
services/nodemanager/arvnodeman/daemon.py | 69 +-
services/nodemanager/arvnodeman/jobqueue.py | 5 +-
services/nodemanager/arvnodeman/launcher.py | 3 +-
services/nodemanager/doc/azure.example.cfg | 10 +
services/nodemanager/doc/ec2.example.cfg | 9 +
services/nodemanager/doc/gce.example.cfg | 9 +
services/nodemanager/tests/integration_test.py | 5 +-
.../nodemanager/tests/test_computenode_dispatch.py | 14 +-
services/nodemanager/tests/test_daemon.py | 16 +
65 files changed, 2152 insertions(+), 1752 deletions(-)
delete mode 100644 doc/_includes/_skip_sso_server_install.liquid
create mode 100644 services/api/config/initializers/oj_mimic_json.rb
via 6d7e9f721e14a43976961753ef40cc35f2b2288f (commit)
via 8df5b745955fafef08056a742680979b81faeb1e (commit)
via d6e1bfee59569d79f0f3e24620280e70aa161403 (commit)
via 0985649e03f3112be396fb67ee7507ff47a37571 (commit)
via d91eb0fae8d2ef4c2348769694573bf725ef16d4 (commit)
via 2f4a5bef7433097558b89844c84d4cc40a447c53 (commit)
via 8ab6b482342b95ad35775867bcdb8fd691b78fb7 (commit)
via 58a026e09bda4c1e2374347615c325007c64fac4 (commit)
via e2cc6c1935565ee965f38501a4974e9d0e6b29e5 (commit)
via d2bb622dce5e765e55942e4da6348f949883d9fc (commit)
via c09663a2b1342e755daafcf8e4a4c2b121e4a253 (commit)
via 649d52a70fd75e255437deb1798fa9f09697b78c (commit)
via fba33040ea730ccc30035557226fc1a1de32ba6e (commit)
via 7f0f3ab4e13dd808b7b761039c857d55c7843175 (commit)
via 9a80d15b7cab21efe16ec2b543dfb566bea9def4 (commit)
via 297c4aaf43858eff5022a1e72eb8e09660bde4b0 (commit)
via 080c940d7a8134a6e277a53b7e45eb27e2b2c87f (commit)
via eef816fa8450fd3ce5388f4701ed888691ba897a (commit)
via 55137e6828bf11f76c3f9ec61e4a76954f5d6fa1 (commit)
via cb64653912a604809c3429f7dbda781741acd18a (commit)
via 3c23a0a57bde2978402e5d46ca2b003becee6d58 (commit)
via 42012c4746d6dd81eb3c72fddb2fdd36cff381e7 (commit)
via 6fe8e52020d421797306e5c6536afbcee761510a (commit)
via a4166f402b34e018940ae1df726351e8c52ac1c1 (commit)
via 355173ba2e8c42b29011493d1d8c7cc4d69295c6 (commit)
via ab3afbb684bc1b32577c2696e13882123bfff7d2 (commit)
via 321e025cde18b8069dfc8977754af39f2efe505a (commit)
via 265fe64e5b7a931736f156c3cb446fbbbc27f018 (commit)
via 83d08d7ccbc622ec97948929c83fb91f96743ca2 (commit)
via 051b9e763b0a14f9039f536859bd2c1580faa31c (commit)
via c9653ef070009e03061e59f2e13ea06cf7c16b01 (commit)
via 0279d2f8a27170c151833c470b88cf361aa3b91e (commit)
via ef7718f230f076d369f8c66064e7b542d24e57ed (commit)
via 9ba887f887aa3736ab50728ccd2c0ae20e45d2a4 (commit)
via ec9ecf93b25f849e8480ece08b330c22b13bfe5f (commit)
via e0033099ba12ae2c42d41d3d049b9d759663d35d (commit)
via d4df74752927fbd464567f13723ba0dc5d9e5ff0 (commit)
via cd5141d405db649f0b99ca55b46ff68f16dc13b9 (commit)
via 336095d3ba51bf434c11a6d9d67e510bad165a0a (commit)
via 8d75c13b5601cb3450ad253cdddfef41499ba493 (commit)
via 5c2e355aca8dd456a646e743652a1be87113113f (commit)
via 0c27a910dc594c3d9eee5a69866cac220c0b18ba (commit)
via 1980d00a9afeaea0bc6f266892e43de14ccd297e (commit)
via 7c893856e22073b10dc147b887b0fd64b11a1f81 (commit)
via 09afabfb3de546d55c3a33ae0b68291edf414fcb (commit)
via 29776dabd84b8070b647ee38289214b7a56b2c1c (commit)
via 146f7179c88c0d25acc91ff2b5faff8e7f98d1be (commit)
via bc248010427eded4610d7f337d3ee01d4d06538a (commit)
via ce85cd0b2605a6cbfb96bf18c1f2913d33b60f44 (commit)
via 7c1e0140fb0d059299af08a102d30d65dfb60990 (commit)
via f4fd17ccc26166a195b7344b84d95e9ff2315a36 (commit)
via cf24a6137c3b70060fc5a9197f751595b1033131 (commit)
via 0a2cc29e409f85381a8ee1ae1f0097937eb11804 (commit)
via fe8e22166ce16ec8ee78c53e49d0ab23f767d9a6 (commit)
via 116f934f0359b27ae3bd3047860fc5e7f27bdcf5 (commit)
from 0011b5236fc9a562bc13f943f9a431c496b2b7cd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6d7e9f721e14a43976961753ef40cc35f2b2288f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Jul 30 12:03:07 2018 -0400
13198: Add cache stats to metrics.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 636c0306c..e669e326c 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -732,7 +732,7 @@ do_test_once() {
do_install() {
skipit=false
- if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+ if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
retry do_install_once ${@}
else
skipit=true
diff --git a/sdk/go/httpserver/metrics.go b/sdk/go/httpserver/metrics.go
index 77525a80f..b52068e95 100644
--- a/sdk/go/httpserver/metrics.go
+++ b/sdk/go/httpserver/metrics.go
@@ -99,10 +99,17 @@ func (m *metrics) ServeAPI(next http.Handler) http.Handler {
// For the metrics to be accurate, the caller must ensure every
// request passed to the Handler also passes through
// LogRequests(logger, ...), and vice versa.
-func Instrument(logger *logrus.Logger, next http.Handler) Handler {
+//
+// If registry is nil, a new registry is created.
+//
+// If logger is nil, logrus.StandardLogger() is used.
+func Instrument(registry *prometheus.Registry, logger *logrus.Logger, next http.Handler) Handler {
if logger == nil {
logger = logrus.StandardLogger()
}
+ if registry == nil {
+ registry = prometheus.NewRegistry()
+ }
reqDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "request_duration_seconds",
Help: "Summary of request duration.",
@@ -111,7 +118,6 @@ func Instrument(logger *logrus.Logger, next http.Handler) Handler {
Name: "time_to_status_seconds",
Help: "Summary of request TTFB.",
}, []string{"code", "method"})
- registry := prometheus.NewRegistry()
registry.MustRegister(timeToStatus)
registry.MustRegister(reqDuration)
m := &metrics{
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 59e8de3bc..b2bab7821 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -12,8 +12,11 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
+ "github.com/prometheus/client_golang/prometheus"
)
+const metricsUpdateInterval = time.Second / 10
+
type cache struct {
TTL arvados.Duration
UUIDTTL arvados.Duration
@@ -22,13 +25,16 @@ type cache struct {
MaxPermissionEntries int
MaxUUIDEntries int
+ registry *prometheus.Registry
stats cacheStats
+ metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
setupOnce sync.Once
}
+// cacheStats is EOL - add new metrics to cacheMetrics instead
type cacheStats struct {
Requests uint64 `json:"Cache.Requests"`
CollectionBytes uint64 `json:"Cache.CollectionBytes"`
@@ -39,6 +45,68 @@ type cacheStats struct {
APICalls uint64 `json:"Cache.APICalls"`
}
+type cacheMetrics struct {
+ requests prometheus.Counter
+ collectionBytes prometheus.Gauge
+ collectionEntries prometheus.Gauge
+ collectionHits prometheus.Counter
+ pdhHits prometheus.Counter
+ permissionHits prometheus.Counter
+ apiCalls prometheus.Counter
+}
+
+func (m *cacheMetrics) setup(reg *prometheus.Registry) {
+ m.requests = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "requests",
+ Help: "Number of targetID-to-manifest lookups handled.",
+ })
+ reg.MustRegister(m.requests)
+ m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "hits",
+ Help: "Number of pdh-to-manifest cache hits.",
+ })
+ reg.MustRegister(m.collectionHits)
+ m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "pdh_hits",
+ Help: "Number of uuid-to-pdh cache hits.",
+ })
+ reg.MustRegister(m.pdhHits)
+ m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "permission_hits",
+ Help: "Number of targetID-to-permission cache hits.",
+ })
+ reg.MustRegister(m.permissionHits)
+ m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "api_calls",
+ Help: "Number of outgoing API calls made by cache.",
+ })
+ reg.MustRegister(m.apiCalls)
+ m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifest_bytes",
+ Help: "Total size of all manifests in cache.",
+ })
+ reg.MustRegister(m.collectionBytes)
+ m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifests",
+ Help: "Number of manifests in cache.",
+ })
+ reg.MustRegister(m.collectionEntries)
+}
+
type cachedPDH struct {
expire time.Time
pdh string
@@ -67,6 +135,22 @@ func (c *cache) setup() {
if err != nil {
panic(err)
}
+
+ reg := c.registry
+ if reg == nil {
+ reg = prometheus.NewRegistry()
+ }
+ c.metrics.setup(reg)
+ go func() {
+ for range time.Tick(metricsUpdateInterval) {
+ c.updateGauges()
+ }
+ }()
+}
+
+func (c *cache) updateGauges() {
+ c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
+ c.metrics.collectionEntries.Set(float64(c.collections.Len()))
}
var selectPDH = map[string]interface{}{
@@ -113,6 +197,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
c.setupOnce.Do(c.setup)
atomic.AddUint64(&c.stats.Requests, 1)
+ c.metrics.requests.Inc()
permOK := false
permKey := arv.ApiToken + "\000" + targetID
@@ -124,6 +209,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
} else {
permOK = true
atomic.AddUint64(&c.stats.PermissionHits, 1)
+ c.metrics.permissionHits.Inc()
}
}
@@ -137,6 +223,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
} else {
pdh = ent.pdh
atomic.AddUint64(&c.stats.PDHHits, 1)
+ c.metrics.pdhHits.Inc()
}
}
@@ -153,6 +240,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
// _and_ the current token has permission, we can
// use our cached manifest.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
var current arvados.Collection
err := arv.Get("collections", targetID, selectPDH, ¤t)
if err != nil {
@@ -181,6 +269,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
// Collection manifest is not cached.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
err := arv.Get("collections", targetID, nil, &collection)
if err != nil {
return nil, err
@@ -261,16 +350,16 @@ func (c *cache) collectionBytes() uint64 {
}
func (c *cache) lookupCollection(key string) *arvados.Collection {
- if ent, cached := c.collections.Get(key); !cached {
+ e, cached := c.collections.Get(key)
+ if !cached {
+ return nil
+ }
+ ent := e.(*cachedCollection)
+ if ent.expire.Before(time.Now()) {
+ c.collections.Remove(key)
return nil
- } else {
- ent := ent.(*cachedCollection)
- if ent.expire.Before(time.Now()) {
- c.collections.Remove(key)
- return nil
- } else {
- atomic.AddUint64(&c.stats.CollectionHits, 1)
- return ent.collection
- }
}
+ atomic.AddUint64(&c.stats.CollectionHits, 1)
+ c.metrics.collectionHits.Inc()
+ return ent.collection
}
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 58ec348c8..68ff8a7b0 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -8,6 +8,7 @@ import (
"net/http"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/prometheus/client_golang/prometheus"
)
type server struct {
@@ -17,7 +18,9 @@ type server struct {
func (srv *server) Start() error {
h := &handler{Config: srv.Config}
- mh := httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+ reg := prometheus.NewRegistry()
+ h.Config.Cache.registry = reg
+ mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
srv.Handler = mh
srv.Addr = srv.Config.Listen
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 6688cc2ee..7e738cb9f 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -306,14 +306,19 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
- req.Host = arvadostest.FooCollection + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp, err = http.DefaultClient.Do(req)
- c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- buf, _ := ioutil.ReadAll(resp.Body)
- c.Check(buf, check.DeepEquals, []byte("foo"))
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ s.testServer.Config.Cache.updateGauges()
req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
resp, err = http.DefaultClient.Do(req)
@@ -327,6 +332,12 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
Value float64
}
}
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
var ents []struct {
Name string
Help string
@@ -336,24 +347,38 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
Name string
Value string
}
+ Counter counter
+ Gauge gauge
Summary summary
}
}
json.NewDecoder(resp.Body).Decode(&ents)
- flat := map[string]summary{}
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
for _, e := range ents {
for _, m := range e.Metric {
labels := map[string]string{}
for _, lbl := range m.Label {
labels[lbl.Name] = lbl.Value
}
- flat[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
}
}
- c.Check(flat["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
- c.Check(flat["request_duration_seconds/get/200"].SampleCount, check.Equals, "2")
- c.Check(flat["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
- c.Check(flat["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+ c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+ // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index d19be61e9..c31ab9c2e 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -88,7 +88,7 @@ func MakeRESTRouter() http.Handler {
rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
- stack := httpserver.Instrument(nil,
+ stack := httpserver.Instrument(nil, nil,
httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
return stack.ServeAPI(stack)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list