[ARVADOS] updated: 1.1.4-759-g6db4c9452
Git user
git at public.curoverse.com
Tue Aug 7 18:09:37 EDT 2018
Summary of changes:
doc/_config.yml | 1 +
doc/admin/upgrading.html.textile.liquid | 46 +-
doc/css/code.css | 9 +
doc/install/install-api-server.html.textile.liquid | 68 +-
doc/install/install-controller.html.textile.liquid | 180 +
doc/user/cwl/cwl-style.html.textile.liquid | 4 +-
sdk/R/DESCRIPTION | 2 +-
sdk/R/R/Arvados.R | 3552 ++++++++++----------
sdk/R/R/ArvadosFile.R | 93 +-
sdk/R/R/Collection.R | 119 +-
sdk/R/R/CollectionTree.R | 84 +-
sdk/R/R/HttpRequest.R | 2 +-
sdk/R/R/RESTService.R | 22 +-
sdk/R/R/Subcollection.R | 88 +-
sdk/R/R/zzz.R | 4 +
sdk/R/README.Rmd | 53 +-
sdk/R/man/Arvados.Rd | 1 +
sdk/R/man/ArvadosFile.Rd | 6 +-
sdk/R/man/Collection.Rd | 8 +-
sdk/R/man/Subcollection.Rd | 6 +-
sdk/R/man/users.merge.Rd | 22 +
sdk/R/tests/testthat/fakes/FakeHttpRequest.R | 4 +
sdk/R/tests/testthat/fakes/FakeRESTService.R | 12 +-
sdk/R/tests/testthat/test-ArvadosFile.R | 90 +-
sdk/R/tests/testthat/test-Collection.R | 81 +-
sdk/R/tests/testthat/test-RESTService.R | 34 +
sdk/R/tests/testthat/test-Subcollection.R | 86 +-
sdk/cwl/arvados_cwl/__init__.py | 15 +
sdk/cwl/arvados_cwl/fsaccess.py | 11 +
sdk/cwl/setup.py | 4 +-
sdk/cwl/tests/13931-size-job.yml | 3 +
sdk/cwl/tests/13931-size.cwl | 10 +
sdk/cwl/tests/arvados-tests.sh | 4 +
sdk/cwl/tests/arvados-tests.yml | 11 +
sdk/cwl/tests/test_container.py | 3 +-
sdk/cwl/tests/test_job.py | 4 +-
sdk/cwl/tests/test_submit.py | 10 +-
sdk/cwl/tests/wf/expect_packed.cwl | 3 +-
sdk/dev-jobs.dockerfile | 2 +-
sdk/go/dispatch/dispatch.go | 6 +
services/api/config/application.rb | 3 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 6 -
.../crunch-dispatch-slurm_test.go | 2 +-
services/crunch-dispatch-slurm/squeue.go | 67 +-
services/keep-web/cache.go | 33 -
services/keep-web/cache_test.go | 87 +-
services/keep-web/handler.go | 9 +-
services/keep-web/status_test.go | 1 -
services/nodemanager/arvnodeman/config.py | 3 +
services/nodemanager/arvnodeman/launcher.py | 10 +-
50 files changed, 2819 insertions(+), 2165 deletions(-)
create mode 100644 doc/install/install-controller.html.textile.liquid
create mode 100644 sdk/R/man/users.merge.Rd
create mode 100644 sdk/cwl/tests/13931-size-job.yml
create mode 100644 sdk/cwl/tests/13931-size.cwl
via 6db4c94527903f403e77ba4fce2d1d5fe4e29b03 (commit)
via 9f9cf2e1da7233d6b341602faeaa69454b7e201f (commit)
via 61611952a2ac9a8f40fddd1a259c3eae5cd8d6b0 (commit)
via 94d277295c4a8d331622e4077ace1eb4570d54e5 (commit)
via d94ed391d56f17f23e98a8e116849a44cac27e3b (commit)
via fb95e36d2f0525fc7e024854b1bb3705027ca779 (commit)
via d22b0f215c0c6c94277d7114756f497cd7a3f07b (commit)
via 9a752eae723dfe23d5b98db8ba7ca600ab9f9573 (commit)
via d49a9ee4cd176e35f1929c1bb60caa751b53903d (commit)
via 0fe48142de81e20967eaac3972e35d115aaaa497 (commit)
via df29ab509d3a9f6275fade0b7b8c9b5cd8d6f74d (commit)
via 718e3b0c18916a2a48d16cb5cb67d972ea5be6b8 (commit)
via 4c393c35f171c5b9e76972223a6c2739c02e0072 (commit)
via a464c2a216412032b58688194b4ff730fbe3a28f (commit)
via da14703fb4e1a249f47685b29310c4c69441ff08 (commit)
via 562130229d7e85cbf813179a315772594e298127 (commit)
via 7c7514411d3ed3b4d4695f55eb1a02f452a894a4 (commit)
via 336638f432e46f61cc87b958580e098e81cea921 (commit)
via e43159541b9da21730d3da27568530f47f7194f4 (commit)
via 73167d511b4ed1a0441bafa0f7bac26263b20231 (commit)
via 353cfc20742aaa069eecba2b291cb56a78e4d497 (commit)
via 3847e0a3ecba006108130fa890e55c020a6dc3e9 (commit)
via f4faf6f01860cb4683536a2c5e793a93b4193a42 (commit)
via c4e13fdbb3f7de9d587f9c07b4711c08c7eae09e (commit)
via ccfad8f850f6d3edc43e7757fbeed112864efd18 (commit)
via 0be45d32255d83c410aad4e34560c478917ee6c4 (commit)
via b727cb3c8c32389f03e6ae71ad128c518a2c2a8c (commit)
via e8ebb2101fa89294c91b404953cc2fbabb796e40 (commit)
via de76be07dcbb36d38f6714480570cddaeabe2194 (commit)
via c8a4dee5e52feed137ca3cb4c4a4e224efbb694f (commit)
via d80674f09b714ee2ced537f18d73d325672db3c2 (commit)
via db33fa2b2d1f09a186bc07a2956cb0edae380d2c (commit)
via c9bfa91e3c3ec90778795969fdd3787578a76ed4 (commit)
via 342c15f625fda8746e7d74510a298286b3a52196 (commit)
via c7cbb121dec3c356bf7d72087d601c7593410b8c (commit)
via 74cffe96768727e8b727cdb54358729c91bec130 (commit)
from 413db07b4c81ea08663f90f31ee03227349d2be4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6db4c94527903f403e77ba4fce2d1d5fe4e29b03
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Aug 7 18:09:25 2018 -0400
13198: Remove old counters from status.json.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index b2bab7821..8336b78f9 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -6,7 +6,6 @@ package main
import (
"sync"
- "sync/atomic"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -26,7 +25,6 @@ type cache struct {
MaxUUIDEntries int
registry *prometheus.Registry
- stats cacheStats
metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
@@ -34,17 +32,6 @@ type cache struct {
setupOnce sync.Once
}
-// cacheStats is EOL - add new metrics to cacheMetrics instead
-type cacheStats struct {
- Requests uint64 `json:"Cache.Requests"`
- CollectionBytes uint64 `json:"Cache.CollectionBytes"`
- CollectionEntries int `json:"Cache.CollectionEntries"`
- CollectionHits uint64 `json:"Cache.CollectionHits"`
- PDHHits uint64 `json:"Cache.UUIDHits"`
- PermissionHits uint64 `json:"Cache.PermissionHits"`
- APICalls uint64 `json:"Cache.APICalls"`
-}
-
type cacheMetrics struct {
requests prometheus.Counter
collectionBytes prometheus.Gauge
@@ -157,19 +144,6 @@ var selectPDH = map[string]interface{}{
"select": []string{"portable_data_hash"},
}
-func (c *cache) Stats() cacheStats {
- c.setupOnce.Do(c.setup)
- return cacheStats{
- Requests: atomic.LoadUint64(&c.stats.Requests),
- CollectionBytes: c.collectionBytes(),
- CollectionEntries: c.collections.Len(),
- CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits),
- PDHHits: atomic.LoadUint64(&c.stats.PDHHits),
- PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits),
- APICalls: atomic.LoadUint64(&c.stats.APICalls),
- }
-}
-
// Update saves a modified version (fs) to an existing collection
// (coll) and, if successful, updates the relevant cache entries so
// subsequent calls to Get() reflect the modifications.
@@ -195,8 +169,6 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad
func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
c.setupOnce.Do(c.setup)
-
- atomic.AddUint64(&c.stats.Requests, 1)
c.metrics.requests.Inc()
permOK := false
@@ -208,7 +180,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
c.permissions.Remove(permKey)
} else {
permOK = true
- atomic.AddUint64(&c.stats.PermissionHits, 1)
c.metrics.permissionHits.Inc()
}
}
@@ -222,7 +193,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
c.pdhs.Remove(targetID)
} else {
pdh = ent.pdh
- atomic.AddUint64(&c.stats.PDHHits, 1)
c.metrics.pdhHits.Inc()
}
}
@@ -239,7 +209,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
// likely, the cached PDH is still correct; if so,
// _and_ the current token has permission, we can
// use our cached manifest.
- atomic.AddUint64(&c.stats.APICalls, 1)
c.metrics.apiCalls.Inc()
var current arvados.Collection
err := arv.Get("collections", targetID, selectPDH, ¤t)
@@ -268,7 +237,6 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
}
// Collection manifest is not cached.
- atomic.AddUint64(&c.stats.APICalls, 1)
c.metrics.apiCalls.Inc()
err := arv.Get("collections", targetID, nil, &collection)
if err != nil {
@@ -359,7 +327,6 @@ func (c *cache) lookupCollection(key string) *arvados.Collection {
c.collections.Remove(key)
return nil
}
- atomic.AddUint64(&c.stats.CollectionHits, 1)
c.metrics.collectionHits.Inc()
return ent.collection
}
diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go
index cddeaf489..d147573ee 100644
--- a/services/keep-web/cache_test.go
+++ b/services/keep-web/cache_test.go
@@ -5,17 +5,36 @@
package main
import (
+ "bytes"
+
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/common/expfmt"
"gopkg.in/check.v1"
)
+func (s *UnitSuite) checkCacheMetrics(c *check.C, reg *prometheus.Registry, regs ...string) {
+ mfs, err := reg.Gather()
+ c.Check(err, check.IsNil)
+ buf := &bytes.Buffer{}
+ enc := expfmt.NewEncoder(buf, expfmt.FmtText)
+ for _, mf := range mfs {
+ c.Check(enc.Encode(mf), check.IsNil)
+ }
+ mm := buf.String()
+ for _, reg := range regs {
+ c.Check(mm, check.Matches, `(?ms).*collectioncache_`+reg+`\n.*`)
+ }
+}
+
func (s *UnitSuite) TestCache(c *check.C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
// Hit the same collection 5 times using the same token. Only
// the first req should cause an API call; the next 4 should
@@ -29,11 +48,12 @@ func (s *UnitSuite) TestCache(c *check.C) {
c.Check(coll.PortableDataHash, check.Equals, arvadostest.FooPdh)
c.Check(coll.ManifestText[:2], check.Equals, ". ")
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(5))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 5",
+ "hits 4",
+ "permission_hits 4",
+ "pdh_hits 4",
+ "api_calls 1")
// Hit the same collection 2 more times, this time requesting
// it by PDH and using a different token. The first req should
@@ -49,11 +69,12 @@ func (s *UnitSuite) TestCache(c *check.C) {
c.Check(coll2.ManifestText[:2], check.Equals, ". ")
c.Check(coll2.ManifestText, check.Not(check.Equals), coll.ManifestText)
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+1))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 6",
+ "hits 4",
+ "permission_hits 4",
+ "pdh_hits 4",
+ "api_calls 2")
coll2, err = cache.Get(arv, arvadostest.FooPdh, false)
c.Check(err, check.Equals, nil)
@@ -61,11 +82,12 @@ func (s *UnitSuite) TestCache(c *check.C) {
c.Check(coll2.PortableDataHash, check.Equals, arvadostest.FooPdh)
c.Check(coll2.ManifestText[:2], check.Equals, ". ")
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+2))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+1))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 7",
+ "hits 5",
+ "permission_hits 5",
+ "pdh_hits 4",
+ "api_calls 2")
// Alternating between two collections N times should produce
// only 2 more API calls.
@@ -80,11 +102,12 @@ func (s *UnitSuite) TestCache(c *check.C) {
_, err := cache.Get(arv, target, false)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(5+2+20))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+1+18))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1+18))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0+18))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1+2))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 27",
+ "hits 23",
+ "permission_hits 23",
+ "pdh_hits 22",
+ "api_calls 4")
}
func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) {
@@ -92,17 +115,19 @@ func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) {
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooPdh, forceReload)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(4))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(0))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 4",
+ "hits 3",
+ "permission_hits 1",
+ "pdh_hits 0",
+ "api_calls 3")
}
func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) {
@@ -110,15 +135,17 @@ func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) {
c.Assert(err, check.Equals, nil)
cache := DefaultConfig().Cache
+ cache.registry = prometheus.NewRegistry()
for _, forceReload := range []bool{false, true, false, true} {
_, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
c.Check(err, check.Equals, nil)
}
- c.Check(cache.Stats().Requests, check.Equals, uint64(4))
- c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
- c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
- c.Check(cache.Stats().PDHHits, check.Equals, uint64(3))
- c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+ s.checkCacheMetrics(c, cache.registry,
+ "requests 4",
+ "hits 3",
+ "permission_hits 1",
+ "pdh_hits 3",
+ "api_calls 3")
}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index d0ba431aa..bb77e5859 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -91,14 +91,7 @@ func (h *handler) setup() {
}
func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
- status := struct {
- cacheStats
- Version string
- }{
- cacheStats: h.Config.Cache.Stats(),
- Version: version,
- }
- json.NewEncoder(w).Encode(status)
+ json.NewEncoder(w).Encode(struct{ Version string }{version})
}
// updateOnSuccess wraps httpserver.ResponseWriter. If the handler
diff --git a/services/keep-web/status_test.go b/services/keep-web/status_test.go
index 0a2b9eb98..62db198dd 100644
--- a/services/keep-web/status_test.go
+++ b/services/keep-web/status_test.go
@@ -30,7 +30,6 @@ func (s *UnitSuite) TestStatus(c *check.C) {
var status map[string]interface{}
err := json.NewDecoder(resp.Body).Decode(&status)
c.Check(err, check.IsNil)
- c.Check(status["Cache.Requests"], check.Equals, float64(0))
c.Check(status["Version"], check.Not(check.Equals), "")
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list