[arvados] created: 2.1.0-2869-gacc627482

git repository hosting git at public.arvados.org
Mon Aug 29 21:59:23 UTC 2022


        at  acc627482f0c9e8bdaa190dffd653bfb93a0e62f (commit)


commit acc627482f0c9e8bdaa190dffd653bfb93a0e62f
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 17:01:08 2022 -0400

    19363: Avoid locking entire filesystem for MemorySize.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 2ad4d1f85..6da639eda 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -420,10 +420,20 @@ func (n *treenode) Sync() error {
 }
 
 func (n *treenode) MemorySize() (size int64) {
+	// To avoid making other callers wait while we count the
+	// entire filesystem size, we lock the node only long enough
+	// to copy the list of children. We accept that the resulting
+	// size will sometimes be misleading (e.g., we will
+	// double-count an item that moves from A to B after we check
+	// A's size but before we check B's size).
 	n.RLock()
-	defer n.RUnlock()
 	debugPanicIfNotLocked(n, false)
+	todo := make([]inode, 0, len(n.inodes))
 	for _, inode := range n.inodes {
+		todo = append(todo, inode)
+	}
+	n.RUnlock()
+	for _, inode := range todo {
 		size += inode.MemorySize()
 	}
 	return 64 + size
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 26012e240..759a0497e 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -443,8 +443,8 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
 }
 
 func (fs *collectionFileSystem) MemorySize() int64 {
-	fs.fileSystem.root.Lock()
-	defer fs.fileSystem.root.Unlock()
+	fs.fileSystem.root.RLock()
+	defer fs.fileSystem.root.RUnlock()
 	return fs.fileSystem.root.(*dirnode).MemorySize()
 }
 
@@ -1154,8 +1154,7 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
 func (dn *dirnode) MemorySize() (size int64) {
 	for _, name := range dn.sortedNames() {
 		node := dn.inodes[name]
-		node.Lock()
-		defer node.Unlock()
+		node.RLock()
 		switch node := node.(type) {
 		case *dirnode:
 			size += node.MemorySize()
@@ -1169,6 +1168,7 @@ func (dn *dirnode) MemorySize() (size int64) {
 				size += 64
 			}
 		}
+		node.RUnlock()
 	}
 	return 64 + size
 }

commit 1b29840c9f97b55cf357d9825e0b58348eff72e4
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 17:00:42 2022 -0400

    19363: Fix unchecked errors.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3a1d9acde..b04add1c4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -486,7 +486,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 
 	// Check configured permission
 	_, sess, err := h.Cache.GetSession(arv.ApiToken)
+	if err != nil {
+		http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
+	}
 	tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
+	if err != nil {
+		http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
+	}
 
 	if webdavMethod[r.Method] {
 		if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {

commit bdb0245310b6b4a083fd010ea7022ef63400c1d3
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:59:58 2022 -0400

    19363: Fix leading "/" in path, skip the 301 redirect.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index e10419f5d..b08abad85 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -507,7 +507,7 @@ func (c *cache) GetTokenUser(token string) (*arvados.User, error) {
 	c.metrics.apiCalls.Inc()
 	var current arvados.User
 
-	err = sess.client.RequestAndDecode(&current, "GET", "/arvados/v1/users/current", nil, nil)
+	err = sess.client.RequestAndDecode(&current, "GET", "arvados/v1/users/current", nil, nil)
 	if err != nil {
 		return nil, err
 	}

commit 6f51168f5fce636d5ded92f452932dd056253daf
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:59:20 2022 -0400

    19363: Skip brand new sessions when pruning.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index d5fdc4997..e10419f5d 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -276,7 +276,7 @@ func (c *cache) pruneSessions() {
 	now := time.Now()
 	var size int64
 	keys := c.sessions.Keys()
-	for _, token := range keys {
+	for idx, token := range keys {
 		ent, ok := c.sessions.Peek(token)
 		if !ok {
 			continue
@@ -284,6 +284,7 @@ func (c *cache) pruneSessions() {
 		s := ent.(*cachedSession)
 		if s.expire.Before(now) {
 			c.sessions.Remove(token)
+			keys[idx] = ""
 			continue
 		}
 		if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
@@ -294,6 +295,12 @@ func (c *cache) pruneSessions() {
 	// least frequently used entries (which Keys() returns last).
 	for i := len(keys) - 1; i >= 0; i-- {
 		token := keys[i]
+		if token == "" {
+			// removed this session in the loop above;
+			// don't prune it, even if it's already been
+			// reinserted.
+			continue
+		}
 		if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
 			break
 		}

commit c2f110e029452ff80b150d9af2c0c1c1ac8f8aa9
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 29 16:54:49 2022 -0400

    19363: Use a larger page size when retrieving project contents.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index bea1f76e2..faab6e4f0 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -79,6 +79,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
 		return nil, err
 	}
 
+	pagesize := 100000
 	var inodes []inode
 
 	// When #17424 is resolved, remove the outer loop here and use
@@ -100,6 +101,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
 			Filters: filters,
 			Order:   "uuid",
 			Select:  []string{"uuid", "name", "modified_at", "properties"},
+			Limit:   &pagesize,
 		}
 
 		for {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list