[ARVADOS] updated: 9d894536b8a7044fdfa168f81a38c3408e6cd7b4

Git user git at public.curoverse.com
Wed Jun 7 10:49:15 EDT 2017


Summary of changes:
 services/keep-web/cache.go      | 76 ++++++++++++++++++++++++++---------------
 services/keep-web/cache_test.go | 36 +++++++++++++++++++
 services/keep-web/handler.go    | 13 +++++++
 services/keep-web/main.go       | 10 +++---
 services/keep-web/usage.go      |  8 ++---
 5 files changed, 107 insertions(+), 36 deletions(-)

       via  9d894536b8a7044fdfa168f81a38c3408e6cd7b4 (commit)
       via  d22400822b3489c621e5dbd902749b1a547ca579 (commit)
       via  6246cacc17e8b90519143e717b7241e527678be9 (commit)
       via  eb21778470992713458dc42e4a115bd0a619c5de (commit)
       via  a15a137c593d24649e2960471d7273acad695186 (commit)
      from  46580c9d21578ec9f0638c9cb464703c7ede00b3 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 9d894536b8a7044fdfa168f81a38c3408e6cd7b4
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 7 10:47:37 2017 -0400

    11809: Skip lookups in forceReload case. Add forceReload tests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 7970165..ab7c653 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -89,7 +89,8 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 
 	permOK := false
 	permKey := arv.ApiToken + "\000" + targetID
-	if ent, cached := c.permissions.Get(permKey); cached {
+	if forceReload {
+	} else if ent, cached := c.permissions.Get(permKey); cached {
 		ent := ent.(*cachedPermission)
 		if ent.expire.Before(time.Now()) {
 			c.permissions.Remove(permKey)
@@ -102,6 +103,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 	var pdh string
 	if arvadosclient.PDHMatch(targetID) {
 		pdh = targetID
+	} else if forceReload {
 	} else if ent, cached := c.pdhs.Get(targetID); cached {
 		ent := ent.(*cachedPDH)
 		if ent.expire.Before(time.Now()) {
@@ -112,13 +114,14 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		}
 	}
 
-	collection := c.lookupCollection(pdh)
-
-	if collection != nil && permOK && !forceReload {
-		return collection, nil
+	var collection map[string]interface{}
+	if pdh != "" {
+		collection = c.lookupCollection(pdh)
 	}
 
-	if collection != nil {
+	if collection != nil && permOK {
+		return collection, nil
+	} else if collection != nil {
 		// Ask API for current PDH for this targetID. Most
 		// likely, the cached PDH is still correct; if so,
 		// _and_ the current token has permission, we can
diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go
index 9538543..f8aa2b1 100644
--- a/services/keep-web/cache_test.go
+++ b/services/keep-web/cache_test.go
@@ -66,3 +66,39 @@ func (s *UnitSuite) TestCache(c *check.C) {
 	c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0+18))
 	c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1+2))
 }
+
+func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, check.Equals, nil)
+
+	cache := DefaultConfig().Cache
+
+	for _, forceReload := range []bool{false, true, false, true} {
+		_, err := cache.Get(arv, arvadostest.FooPdh, forceReload)
+		c.Check(err, check.Equals, nil)
+	}
+
+	c.Check(cache.Stats().Requests, check.Equals, uint64(4))
+	c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3))
+	c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
+	c.Check(cache.Stats().PDHHits, check.Equals, uint64(0))
+	c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+}
+
+func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, check.Equals, nil)
+
+	cache := DefaultConfig().Cache
+
+	for _, forceReload := range []bool{false, true, false, true} {
+		_, err := cache.Get(arv, arvadostest.FooCollection, forceReload)
+		c.Check(err, check.Equals, nil)
+	}
+
+	c.Check(cache.Stats().Requests, check.Equals, uint64(4))
+	c.Check(cache.Stats().CollectionHits, check.Equals, uint64(1))
+	c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1))
+	c.Check(cache.Stats().PDHHits, check.Equals, uint64(1))
+	c.Check(cache.Stats().APICalls, check.Equals, uint64(3))
+}

commit d22400822b3489c621e5dbd902749b1a547ca579
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 7 10:32:09 2017 -0400

    11809: Increase default cache sizes.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 0cb62d1..f17522c 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -38,10 +38,10 @@ func DefaultConfig() *Config {
 		Listen: ":80",
 		Cache: cache{
 			TTL:                  arvados.Duration(5 * time.Minute),
-			MaxCollectionEntries: 100,
+			MaxCollectionEntries: 1000,
 			MaxCollectionBytes:   100000000,
-			MaxPermissionEntries: 100,
-			MaxUUIDEntries:       100,
+			MaxPermissionEntries: 1000,
+			MaxUUIDEntries:       1000,
 		},
 	}
 }

commit 6246cacc17e8b90519143e717b7241e527678be9
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 7 10:27:59 2017 -0400

    11809: Prune collection cache asynchronously.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index b9e3e3e..7970165 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -176,7 +176,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		collection: collection,
 	})
 	if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
-		c.pruneCollections()
+		go c.pruneCollections()
 	}
 	return collection, nil
 }

commit eb21778470992713458dc42e4a115bd0a619c5de
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 7 10:27:06 2017 -0400

    11809: Rename FooEntries -> MaxFooEntries in cache config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 26f6627..b9e3e3e 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -12,11 +12,11 @@ import (
 )
 
 type cache struct {
-	TTL               arvados.Duration
-	CollectionEntries int
-	CollectionBytes   int64
-	PermissionEntries int
-	UUIDEntries       int
+	TTL                  arvados.Duration
+	MaxCollectionEntries int
+	MaxCollectionBytes   int64
+	MaxPermissionEntries int
+	MaxUUIDEntries       int
 
 	stats       cacheStats
 	pdhs        *lru.TwoQueueCache
@@ -51,15 +51,15 @@ type cachedPermission struct {
 
 func (c *cache) setup() {
 	var err error
-	c.pdhs, err = lru.New2Q(c.UUIDEntries)
+	c.pdhs, err = lru.New2Q(c.MaxUUIDEntries)
 	if err != nil {
 		panic(err)
 	}
-	c.collections, err = lru.New2Q(c.CollectionEntries)
+	c.collections, err = lru.New2Q(c.MaxCollectionEntries)
 	if err != nil {
 		panic(err)
 	}
-	c.permissions, err = lru.New2Q(c.PermissionEntries)
+	c.permissions, err = lru.New2Q(c.MaxPermissionEntries)
 	if err != nil {
 		panic(err)
 	}
@@ -175,7 +175,7 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo
 		expire:     exp,
 		collection: collection,
 	})
-	if int64(len(collection["manifest_text"].(string))) > c.CollectionBytes/int64(c.CollectionEntries) {
+	if int64(len(collection["manifest_text"].(string))) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) {
 		c.pruneCollections()
 	}
 	return collection, nil
@@ -212,7 +212,7 @@ func (c *cache) pruneCollections() {
 		}
 	}
 	for i, k := range keys {
-		if size <= c.CollectionBytes {
+		if size <= c.MaxCollectionBytes {
 			break
 		}
 		if expired[i] {
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index bcdcd8b..0cb62d1 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -37,11 +37,11 @@ func DefaultConfig() *Config {
 	return &Config{
 		Listen: ":80",
 		Cache: cache{
-			TTL:               arvados.Duration(5 * time.Minute),
-			CollectionEntries: 100,
-			CollectionBytes:   100000000,
-			PermissionEntries: 100,
-			UUIDEntries:       100,
+			TTL:                  arvados.Duration(5 * time.Minute),
+			MaxCollectionEntries: 100,
+			MaxCollectionBytes:   100000000,
+			MaxPermissionEntries: 100,
+			MaxUUIDEntries:       100,
 		},
 	}
 }
diff --git a/services/keep-web/usage.go b/services/keep-web/usage.go
index b854f53..603dd4d 100644
--- a/services/keep-web/usage.go
+++ b/services/keep-web/usage.go
@@ -71,19 +71,19 @@ Cache.TTL:
 
     Maximum time to cache collection data and permission checks.
 
-Cache.CollectionEntries:
+Cache.MaxCollectionEntries:
 
     Maximum number of collection cache entries.
 
-Cache.CollectionBytes:
+Cache.MaxCollectionBytes:
 
     Approximate memory limit for collection cache.
 
-Cache.PermissionEntries:
+Cache.MaxPermissionEntries:
 
     Maximum number of permission cache entries.
 
-Cache.UUIDEntries:
+Cache.MaxUUIDEntries:
 
     Maximum number of UUID cache entries.
 

commit a15a137c593d24649e2960471d7273acad695186
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Jun 7 10:23:41 2017 -0400

    11809: Add /status.json handler.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index e8bf399..26f6627 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -26,11 +26,13 @@ type cache struct {
 }
 
 type cacheStats struct {
-	Requests       uint64
-	CollectionHits uint64
-	PDHHits        uint64
-	PermissionHits uint64
-	APICalls       uint64
+	Requests          uint64 `json:"Cache.Requests"`
+	CollectionBytes   uint64 `json:"Cache.CollectionBytes"`
+	CollectionEntries int    `json:"Cache.CollectionEntries"`
+	CollectionHits    uint64 `json:"Cache.CollectionHits"`
+	PDHHits           uint64 `json:"Cache.UUIDHits"`
+	PermissionHits    uint64 `json:"Cache.PermissionHits"`
+	APICalls          uint64 `json:"Cache.APICalls"`
 }
 
 type cachedPDH struct {
@@ -68,12 +70,15 @@ var selectPDH = map[string]interface{}{
 }
 
 func (c *cache) Stats() cacheStats {
+	c.setupOnce.Do(c.setup)
 	return cacheStats{
-		Requests:       atomic.LoadUint64(&c.stats.Requests),
-		CollectionHits: atomic.LoadUint64(&c.stats.CollectionHits),
-		PDHHits:        atomic.LoadUint64(&c.stats.PDHHits),
-		PermissionHits: atomic.LoadUint64(&c.stats.PermissionHits),
-		APICalls:       atomic.LoadUint64(&c.stats.APICalls),
+		Requests:          atomic.LoadUint64(&c.stats.Requests),
+		CollectionBytes:   c.collectionBytes(),
+		CollectionEntries: c.collections.Len(),
+		CollectionHits:    atomic.LoadUint64(&c.stats.CollectionHits),
+		PDHHits:           atomic.LoadUint64(&c.stats.PDHHits),
+		PermissionHits:    atomic.LoadUint64(&c.stats.PermissionHits),
+		APICalls:          atomic.LoadUint64(&c.stats.APICalls),
 	}
 }
 
@@ -219,6 +224,20 @@ func (c *cache) pruneCollections() {
 	}
 }
 
+// collectionBytes returns the approximate memory size of the
+// collection cache.
+func (c *cache) collectionBytes() uint64 {
+	var size uint64
+	for _, k := range c.collections.Keys() {
+		v, ok := c.collections.Peek(k)
+		if !ok {
+			continue
+		}
+		size += uint64(len(v.(*cachedCollection).collection["manifest_text"].(string)))
+	}
+	return size
+}
+
 func (c *cache) lookupCollection(pdh string) map[string]interface{} {
 	if pdh == "" {
 		return nil
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 45cd27f..42c37b8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"html"
 	"io"
@@ -67,6 +68,15 @@ func (h *handler) setup() {
 	keepclient.RefreshServiceDiscoveryOnSIGHUP()
 }
 
+func (h *handler) serveStatus(w http.ResponseWriter, r *http.Request) {
+	status := struct {
+		cacheStats
+	}{
+		cacheStats: h.Config.Cache.Stats(),
+	}
+	json.NewEncoder(w).Encode(status)
+}
+
 // ServeHTTP implements http.Handler.
 func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	h.setupOnce.Do(h.setup)
@@ -151,6 +161,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 		// http://ID.collections.example/PATH...
 		credentialsOK = true
 		targetPath = pathParts
+	} else if r.URL.Path == "/status.json" {
+		h.serveStatus(w, r)
+		return
 	} else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
 		// /c=ID/PATH...
 		targetID = parseCollectionIDFromURL(pathParts[0][2:])

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list