[arvados] updated: 2.4.2-26-gc5279947a
git repository hosting
git at public.arvados.org
Mon Sep 19 18:18:59 UTC 2022
Summary of changes:
sdk/go/arvados/fs_base.go | 12 ++++++++++-
sdk/go/arvados/fs_collection.go | 44 +++++++++++++++++++++--------------------
sdk/go/arvados/fs_project.go | 3 +++
services/keep-web/cache.go | 29 ++++++++++-----------------
services/keep-web/handler.go | 6 ++++++
5 files changed, 53 insertions(+), 41 deletions(-)
via c5279947a9a418c60db2ff9364e63aed024181eb (commit)
from 12a14f217939ecb4d0ee6485a036f67fb299c36b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c5279947a9a418c60db2ff9364e63aed024181eb
Author: Tom Clegg <tom at curii.com>
Date: Mon Sep 19 14:08:20 2022 -0400
19428: cherry-pick '19428-webdav-performance' to 2.4-staging
refs #19428
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index ce9253ab3..4cf3566b7 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -409,10 +409,20 @@ func (n *treenode) Sync() error {
}
func (n *treenode) MemorySize() (size int64) {
+ // To avoid making other callers wait while we count the
+ // entire filesystem size, we lock the node only long enough
+ // to copy the list of children. We accept that the resulting
+ // size will sometimes be misleading (e.g., we will
+ // double-count an item that moves from A to B after we check
+ // A's size but before we check B's size).
n.RLock()
- defer n.RUnlock()
debugPanicIfNotLocked(n, false)
+ todo := make([]inode, 0, len(n.inodes))
for _, inode := range n.inodes {
+ todo = append(todo, inode)
+ }
+ n.RUnlock()
+ for _, inode := range todo {
size += inode.MemorySize()
}
return 64 + size
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index ccfbdc4da..9c0ff2e4f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -442,8 +442,6 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
}
func (fs *collectionFileSystem) MemorySize() int64 {
- fs.fileSystem.root.Lock()
- defer fs.fileSystem.root.Unlock()
return fs.fileSystem.root.(*dirnode).MemorySize()
}
@@ -575,6 +573,19 @@ func (fn *filenode) FS() FileSystem {
return fn.fs
}
+func (fn *filenode) MemorySize() (size int64) {
+ fn.RLock()
+ defer fn.RUnlock()
+ size = 64
+ for _, seg := range fn.segments {
+ size += 64
+ if seg, ok := seg.(*memSegment); ok {
+ size += int64(seg.Len())
+ }
+ }
+ return
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -1149,27 +1160,18 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
return cg.Wait()
}
-// caller must have write lock.
func (dn *dirnode) MemorySize() (size int64) {
- for _, name := range dn.sortedNames() {
- node := dn.inodes[name]
- node.Lock()
- defer node.Unlock()
- switch node := node.(type) {
- case *dirnode:
- size += node.MemorySize()
- case *filenode:
- size += 64
- for _, seg := range node.segments {
- switch seg := seg.(type) {
- case *memSegment:
- size += int64(seg.Len())
- }
- size += 64
- }
- }
+ dn.RLock()
+ todo := make([]inode, 0, len(dn.inodes))
+ for _, node := range dn.inodes {
+ todo = append(todo, node)
+ }
+ dn.RUnlock()
+ size = 64
+ for _, node := range todo {
+ size += node.MemorySize()
}
- return 64 + size
+ return
}
// caller must have write lock.
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 380fb9c6d..9386d7cbc 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -78,6 +78,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
return nil, err
}
+ pagesize := 100000
var inodes []inode
// When #17424 is resolved, remove the outer loop here and use
@@ -98,6 +99,8 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
Count: "none",
Filters: filters,
Order: "uuid",
+ Select: []string{"uuid", "name", "modified_at", "properties"},
+ Limit: &pagesize,
}
for {
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index d2c79326a..2d200814f 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -275,9 +275,10 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, *cachedSessi
// until approximate remaining size <= maxsize/2
func (c *cache) pruneSessions() {
now := time.Now()
- var size int64
keys := c.sessions.Keys()
- for _, token := range keys {
+ sizes := make([]int64, len(keys))
+ var size int64
+ for i, token := range keys {
ent, ok := c.sessions.Peek(token)
if !ok {
continue
@@ -288,27 +289,17 @@ func (c *cache) pruneSessions() {
continue
}
if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
- size += fs.MemorySize()
+ sizes[i] = fs.MemorySize()
+ size += sizes[i]
}
}
// Remove tokens until reaching size limit, starting with the
// least frequently used entries (which Keys() returns last).
- for i := len(keys) - 1; i >= 0; i-- {
- token := keys[i]
- if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
- break
- }
- ent, ok := c.sessions.Peek(token)
- if !ok {
- continue
- }
- s := ent.(*cachedSession)
- fs, _ := s.fs.Load().(arvados.CustomFileSystem)
- if fs == nil {
- continue
+ for i := len(keys) - 1; i >= 0 && size > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2; i-- {
+ if sizes[i] > 0 {
+ c.sessions.Remove(keys[i])
+ size -= sizes[i]
}
- c.sessions.Remove(token)
- size -= fs.MemorySize()
}
}
@@ -501,7 +492,7 @@ func (c *cache) GetTokenUser(token string) (*arvados.User, error) {
c.metrics.apiCalls.Inc()
var current arvados.User
- err = sess.client.RequestAndDecode(¤t, "GET", "/arvados/v1/users/current", nil, nil)
+ err = sess.client.RequestAndDecode(¤t, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return nil, err
}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index d7796d44d..4c6145463 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -486,7 +486,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Check configured permission
_, sess, err := h.Config.Cache.GetSession(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
+ }
tokenUser, err = h.Config.Cache.GetTokenUser(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
+ }
if webdavMethod[r.Method] {
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list