[arvados] updated: 2.1.0-2622-g87a93969b

git repository hosting git at public.arvados.org
Mon Aug 29 21:43:57 UTC 2022


Summary of changes:
 sdk/go/arvados/fs_base.go       | 12 +++++++++++-
 sdk/go/arvados/fs_collection.go |  8 ++++----
 sdk/go/arvados/fs_project.go    |  3 +++
 services/keep-web/cache.go      | 11 +++++++++--
 services/keep-web/handler.go    |  6 ++++++
 5 files changed, 33 insertions(+), 7 deletions(-)

       via  87a93969ba0b4eaa1d8c63af5c039e7fed908a31 (commit)
       via  0ea242c2b4bb65b6dcf732668a5c557338195552 (commit)
       via  9f5abe254fb5b0d9899d1ef4dba8355e637c2371 (commit)
       via  bc1b2cfd10aa9c53b8b2ebf6d089eb1769956e3e (commit)
       via  499c10afc48ab2ed7cd873f8a03f39668d2ce39c (commit)
      from  b896ceb55db0593631718cb13ee95b2414afe8f9 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 87a93969ba0b4eaa1d8c63af5c039e7fed908a31
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 17:01:08 2022 -0400

    19363: Avoid locking entire filesystem for MemorySize.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index ce9253ab3..4cf3566b7 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -409,10 +409,20 @@ func (n *treenode) Sync() error {
 }
 
 func (n *treenode) MemorySize() (size int64) {
+	// To avoid making other callers wait while we count the
+	// entire filesystem size, we lock the node only long enough
+	// to copy the list of children. We accept that the resulting
+	// size will sometimes be misleading (e.g., we will
+	// double-count an item that moves from A to B after we check
+	// A's size but before we check B's size).
 	n.RLock()
-	defer n.RUnlock()
 	debugPanicIfNotLocked(n, false)
+	todo := make([]inode, 0, len(n.inodes))
 	for _, inode := range n.inodes {
+		todo = append(todo, inode)
+	}
+	n.RUnlock()
+	for _, inode := range todo {
 		size += inode.MemorySize()
 	}
 	return 64 + size
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index ccfbdc4da..81bc3242f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -442,8 +442,8 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
 }
 
 func (fs *collectionFileSystem) MemorySize() int64 {
-	fs.fileSystem.root.Lock()
-	defer fs.fileSystem.root.Unlock()
+	fs.fileSystem.root.RLock()
+	defer fs.fileSystem.root.RUnlock()
 	return fs.fileSystem.root.(*dirnode).MemorySize()
 }
 
@@ -1153,8 +1153,7 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
 func (dn *dirnode) MemorySize() (size int64) {
 	for _, name := range dn.sortedNames() {
 		node := dn.inodes[name]
-		node.Lock()
-		defer node.Unlock()
+		node.RLock()
 		switch node := node.(type) {
 		case *dirnode:
 			size += node.MemorySize()
@@ -1168,6 +1167,7 @@ func (dn *dirnode) MemorySize() (size int64) {
 				size += 64
 			}
 		}
+		node.RUnlock()
 	}
 	return 64 + size
 }

commit 0ea242c2b4bb65b6dcf732668a5c557338195552
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 17:00:42 2022 -0400

    19363: Fix unchecked errors.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 602b111b7..cd886bda1 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -458,7 +458,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 
 	// Check configured permission
 	_, sess, err := h.Cache.GetSession(arv.ApiToken)
+	if err != nil {
+		http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
+	}
 	tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
+	if err != nil {
+		http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
+	}
 
 	if webdavMethod[r.Method] {
 		if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {

commit 9f5abe254fb5b0d9899d1ef4dba8355e637c2371
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:59:58 2022 -0400

    19363: Fix leading "/" in path, skip the 301 redirect.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 61d48b03b..75eecc450 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -516,7 +516,7 @@ func (c *cache) GetTokenUser(token string) (*arvados.User, error) {
 	c.metrics.apiCalls.Inc()
 	var current arvados.User
 
-	err = sess.client.RequestAndDecode(&current, "GET", "/arvados/v1/users/current", nil, nil)
+	err = sess.client.RequestAndDecode(&current, "GET", "arvados/v1/users/current", nil, nil)
 	if err != nil {
 		return nil, err
 	}

commit bc1b2cfd10aa9c53b8b2ebf6d089eb1769956e3e
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:59:20 2022 -0400

    19363: Skip brand new sessions when pruning.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 16e828c8e..61d48b03b 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -285,7 +285,7 @@ func (c *cache) pruneSessions() {
 	now := time.Now()
 	var size int64
 	keys := c.sessions.Keys()
-	for _, token := range keys {
+	for idx, token := range keys {
 		ent, ok := c.sessions.Peek(token)
 		if !ok {
 			continue
@@ -293,6 +293,7 @@ func (c *cache) pruneSessions() {
 		s := ent.(*cachedSession)
 		if s.expire.Before(now) {
 			c.sessions.Remove(token)
+			keys[idx] = ""
 			continue
 		}
 		if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
@@ -303,6 +304,12 @@ func (c *cache) pruneSessions() {
 	// least frequently used entries (which Keys() returns last).
 	for i := len(keys) - 1; i >= 0; i-- {
 		token := keys[i]
+		if token == "" {
+			// removed this session in the loop above;
+			// don't prune it, even if it's already been
+			// reinserted.
+			continue
+		}
 		if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
 			break
 		}

commit 499c10afc48ab2ed7cd873f8a03f39668d2ce39c
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:54:49 2022 -0400

    19363: Use a larger page size when retrieving project contents.
    
    refs #19363
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index 380fb9c6d..9386d7cbc 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -78,6 +78,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
 		return nil, err
 	}
 
+	pagesize := 100000
 	var inodes []inode
 
 	// When #17424 is resolved, remove the outer loop here and use
@@ -98,6 +99,8 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
 			Count:   "none",
 			Filters: filters,
 			Order:   "uuid",
+			Select:  []string{"uuid", "name", "modified_at", "properties"},
+			Limit:   &pagesize,
 		}
 
 		for {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list