[ARVADOS] updated: 9d894536b8a7044fdfa168f81a38c3408e6cd7b4
Git user
git at public.curoverse.com
Wed Jun 7 10:49:15 EDT 2017
Summary of changes:
services/keep-web/cache.go | 76 ++++++++++++++++++++++++++---------------
services/keep-web/cache_test.go | 36 +++++++++++++++++++
services/keep-web/handler.go | 13 +++++++
services/keep-web/main.go | 10 +++---
services/keep-web/usage.go | 8 ++---
5 files changed, 107 insertions(+), 36 deletions(-)
via 9d894536b8a7044fdfa168f81a38c3408e6cd7b4 (commit)
via d22400822b3489c621e5dbd902749b1a547ca579 (commit)
via 6246cacc17e8b90519143e717b7241e527678be9 (commit)
via eb21778470992713458dc42e4a115bd0a619c5de (commit)
via a15a137c593d24649e2960471d7273acad695186 (commit)
from 46580c9d21578ec9f0638c9cb464703c7ede00b3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9d894536b8a7044fdfa168f81a38c3408e6cd7b4
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 7 10:47:37 2017 -0400
11809: Skip lookups in forceReload case. Add forceReload tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 7970165..ab7c653 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -89,7 +89,8 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
permOK := false
permKey := arv.ApiToken + "\000" + targetID
- if ent, cached := c.permissions.Get(permKey); cached {
+ if forceReload {
+ } else if ent, cached := c.permissions.Get(permKey); cached {
ent := ent.(*cachedPermission)
if ent.expire.Before(time.Now()) {
c.permissions.Remove(permKey)
@@ -102,6 +103,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
var pdh string
if arvadosclient.PDHMatch(targetID) {
pdh = targetID
+ } else if forceReload {
} else if ent, cached := c.pdhs.Get(targetID); cached {
ent := ent.(*cachedPDH)
if ent.expire.Before(time.Now()) {
@@ -112,13 +114,14 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
}
}
- collection := c.lookupCollection(pdh)
-
- if collection != nil && permOK && !forceReload {
- return collection, nil
+ var collection map[string]interface{}
+ if pdh != "" {
+ collection = c.lookupCollection(pdh)
}
- if collection != nil {
+ if collection != nil && permOK {
+ return collection, nil
+ } else if collection != nil {
// Ask API for current PDH for this targetID. Most
// likely, the cached PDH is still correct; if so,
// _and_ the current token has permission, we can
diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go
index 9538543..f8aa2b1 100644
--- a/services/keep-web/cache_test.go
+++ b/services/keep-web/cache_test.go
@@ -66,3 +66,39 @@ func (s *UnitSuite) TestCache(c *check.C) {
c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0+18))
c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1+2))
}
+
+func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+
+ cache := DefaultConfig().Cache
+
+ for _, forceReload := range []bool{false, true, false, true} {
+ _, err := cache.Get(arv, arvadostest.FooPdh, forceReload)
+ c.Check(err, check.Equals, nil)
+ }
+
+ c.Check(cache.Stats().Requests, check.Equals, uint64(4))
+ c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
+ c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
+ c.Check(cache.Stats().PDHHits, check.Equals, uint64(0))
+ c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+}
+
+func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+
+ cache := DefaultConfig().Cache
+
+ for _, forceReload := range []bool{false, true, false, true} {
+ _, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
+ c.Check(err, check.Equals, nil)
+ }
+
+ c.Check(cache.Stats().Requests, check.Equals, uint64(4))
+ c.Check(cache.Stats().CollectionHits, check.Equals, uint64(1))
+ c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
+ c.Check(cache.Stats().PDHHits, check.Equals, uint64(1))
+ c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+}
commit d22400822b3489c621e5dbd902749b1a547ca579
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 7 10:32:09 2017 -0400
11809: Increase default cache sizes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 0cb62d1..f17522c 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -38,10 +38,10 @@ func DefaultConfig() *Config {
Listen: ":80",
Cache: cache{
TTL: arvados.Duration(5 * time.Minute),
- MaxCollectionEntries: 100,
+ MaxCollectionEntries: 1000,
MaxCollectionBytes: 100000000,
- MaxPermissionEntries: 100,
- MaxUUIDEntries: 100,
+ MaxPermissionEntries: 1000,
+ MaxUUIDEntries: 1000,
},
}
}
commit 6246cacc17e8b90519143e717b7241e527678be9
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 7 10:27:59 2017 -0400
11809: Prune collection cache asynchronously.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index b9e3e3e..7970165 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -176,7 +176,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
collection: collection,
})
if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
- c.pruneCollections()
+ go c.pruneCollections()
}
return collection, nil
}
commit eb21778470992713458dc42e4a115bd0a619c5de
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 7 10:27:06 2017 -0400
11809: Rename FooEntries -> MaxFooEntries in cache config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 26f6627..b9e3e3e 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -12,11 +12,11 @@ import (
)
type cache struct {
- TTL arvados.Duration
- CollectionEntries int
- CollectionBytes int64
- PermissionEntries int
- UUIDEntries int
+ TTL arvados.Duration
+ MaxCollectionEntries int
+ MaxCollectionBytes int64
+ MaxPermissionEntries int
+ MaxUUIDEntries int
stats cacheStats
pdhs *lru.TwoQueueCache
@@ -51,15 +51,15 @@ type cachedPermission struct {
func (c *cache) setup() {
var err error
- c.pdhs, err = lru.New2Q(c.UUIDEntries)
+ c.pdhs, err = lru.New2Q(c.MaxUUIDEntries)
if err != nil {
panic(err)
}
- c.collections, err = lru.New2Q(c.CollectionEntries)
+ c.collections, err = lru.New2Q(c.MaxCollectionEntries)
if err != nil {
panic(err)
}
- c.permissions, err = lru.New2Q(c.PermissionEntries)
+ c.permissions, err = lru.New2Q(c.MaxPermissionEntries)
if err != nil {
panic(err)
}
@@ -175,7 +175,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
expire: exp,
collection: collection,
})
- if int64(len(collection["manifest_text"].(string))) > c.CollectionBytes/int64(c.CollectionEntries) {
+ if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
c.pruneCollections()
}
return collection, nil
@@ -212,7 +212,7 @@ func (c *cache) pruneCollections() {
}
}
for i, k := range keys {
- if size <= c.CollectionBytes {
+ if size <= c.MaxCollectionBytes {
break
}
if expired[i] {
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index bcdcd8b..0cb62d1 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -37,11 +37,11 @@ func DefaultConfig() *Config {
return &Config{
Listen: ":80",
Cache: cache{
- TTL: arvados.Duration(5 * time.Minute),
- CollectionEntries: 100,
- CollectionBytes: 100000000,
- PermissionEntries: 100,
- UUIDEntries: 100,
+ TTL: arvados.Duration(5 * time.Minute),
+ MaxCollectionEntries: 100,
+ MaxCollectionBytes: 100000000,
+ MaxPermissionEntries: 100,
+ MaxUUIDEntries: 100,
},
}
}
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
index b854f53..603dd4d 100644
--- a/services/keep-web/usage.go
+++ b/services/keep-web/usage.go
@@ -71,19 +71,19 @@ Cache.TTL:
Maximum time to cache collection data and permission checks.
-Cache.CollectionEntries:
+Cache.MaxCollectionEntries:
Maximum number of collection cache entries.
-Cache.CollectionBytes:
+Cache.MaxCollectionBytes:
Approximate memory limit for collection cache.
-Cache.PermissionEntries:
+Cache.MaxPermissionEntries:
Maximum number of permission cache entries.
-Cache.UUIDEntries:
+Cache.MaxUUIDEntries:
Maximum number of UUID cache entries.
commit a15a137c593d24649e2960471d7273acad695186
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jun 7 10:23:41 2017 -0400
11809: Add /status.json handler.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index e8bf399..26f6627 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -26,11 +26,13 @@ type cache struct {
}
type cacheStats struct {
- Requests uint64
- CollectionHits uint64
- PDHHits uint64
- PermissionHits uint64
- APICalls uint64
+ Requests uint64 `json:"Cache.Requests"`
+ CollectionBytes uint64 `json:"Cache.CollectionBytes"`
+ CollectionEntries int `json:"Cache.CollectionEntries"`
+ CollectionHits uint64 `json:"Cache.CollectionHits"`
+ PDHHits uint64 `json:"Cache.UUIDHits"`
+ PermissionHits uint64 `json:"Cache.PermissionHits"`
+ APICalls uint64 `json:"Cache.APICalls"`
}
type cachedPDH struct {
@@ -68,12 +70,15 @@ var selectPDH = map[string]interface{}{
}
func (c *cache) Stats() cacheStats {
+ c.setupOnce.Do(c.setup)
return cacheStats{
- Requests: atomic.LoadUint64(&c.stats.Requests),
- CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits),
- PDHHits: atomic.LoadUint64(&c.stats.PDHHits),
- PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits),
- APICalls: atomic.LoadUint64(&c.stats.APICalls),
+ Requests: atomic.LoadUint64(&c.stats.Requests),
+ CollectionBytes: c.collectionBytes(),
+ CollectionEntries: c.collections.Len(),
+ CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits),
+ PDHHits: atomic.LoadUint64(&c.stats.PDHHits),
+ PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits),
+ APICalls: atomic.LoadUint64(&c.stats.APICalls),
}
}
@@ -219,6 +224,20 @@ func (c *cache) pruneCollections() {
}
}
+// collectionBytes returns the approximate memory size of the
+// collection cache.
+func (c *cache) collectionBytes() uint64 {
+ var size uint64
+ for _, k := range c.collections.Keys() {
+ v, ok := c.collections.Peek(k)
+ if !ok {
+ continue
+ }
+ size += uint64(len(v.(*cachedCollection).collection["manifest_text"].(string)))
+ }
+ return size
+}
+
func (c *cache) lookupCollection(pdh string) map[string]interface{} {
if pdh == "" {
return nil
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 45cd27f..42c37b8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,6 +1,7 @@
package main
import (
+ "encoding/json"
"fmt"
"html"
"io"
@@ -67,6 +68,15 @@ func (h *handler) setup() {
keepclient.RefreshServiceDiscoveryOnSIGHUP()
}
+func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
+ status := struct {
+ cacheStats
+ }{
+ cacheStats: h.Config.Cache.Stats(),
+ }
+ json.NewEncoder(w).Encode(status)
+}
+
// ServeHTTP implements http.Handler.
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h.setupOnce.Do(h.setup)
@@ -151,6 +161,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// http://ID.collections.example/PATH...
credentialsOK = true
targetPath = pathParts
+ } else if r.URL.Path == "/status.json" {
+ h.serveStatus(w, r)
+ return
} else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID/PATH...
targetID = parseCollectionIDFromURL(pathParts[0][2:])
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list