[arvados] created: 2.1.0-2869-gacc627482
git repository hosting
git at public.arvados.org
Mon Aug 29 21:59:23 UTC 2022
at acc627482f0c9e8bdaa190dffd653bfb93a0e62f (commit)
commit acc627482f0c9e8bdaa190dffd653bfb93a0e62f
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 17:01:08 2022 -0400
19363: Avoid locking entire filesystem for MemorySize.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 2ad4d1f85..6da639eda 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -420,10 +420,20 @@ func (n *treenode) Sync() error {
}
func (n *treenode) MemorySize() (size int64) {
+ // To avoid making other callers wait while we count the
+ // entire filesystem size, we lock the node only long enough
+ // to copy the list of children. We accept that the resulting
+ // size will sometimes be misleading (e.g., we will
+ // double-count an item that moves from A to B after we check
+ // A's size but before we check B's size).
n.RLock()
- defer n.RUnlock()
debugPanicIfNotLocked(n, false)
+ todo := make([]inode, 0, len(n.inodes))
for _, inode := range n.inodes {
+ todo = append(todo, inode)
+ }
+ n.RUnlock()
+ for _, inode := range todo {
size += inode.MemorySize()
}
return 64 + size
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 26012e240..759a0497e 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -443,8 +443,8 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
}
func (fs *collectionFileSystem) MemorySize() int64 {
- fs.fileSystem.root.Lock()
- defer fs.fileSystem.root.Unlock()
+ fs.fileSystem.root.RLock()
+ defer fs.fileSystem.root.RUnlock()
return fs.fileSystem.root.(*dirnode).MemorySize()
}
@@ -1154,8 +1154,7 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
func (dn *dirnode) MemorySize() (size int64) {
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
- node.Lock()
- defer node.Unlock()
+ node.RLock()
switch node := node.(type) {
case *dirnode:
size += node.MemorySize()
@@ -1169,6 +1168,7 @@ func (dn *dirnode) MemorySize() (size int64) {
size += 64
}
}
+ node.RUnlock()
}
return 64 + size
}
commit 1b29840c9f97b55cf357d9825e0b58348eff72e4
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 17:00:42 2022 -0400
19363: Fix unchecked errors.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3a1d9acde..b04add1c4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -486,7 +486,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Check configured permission
_, sess, err := h.Cache.GetSession(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
+ }
tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
+ }
if webdavMethod[r.Method] {
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
commit bdb0245310b6b4a083fd010ea7022ef63400c1d3
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:59:58 2022 -0400
19363: Fix leading "/" in path, skip the 301 redirect.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index e10419f5d..b08abad85 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -507,7 +507,7 @@ func (c *cache) GetTokenUser(token string) (*arvados.User, error) {
c.metrics.apiCalls.Inc()
var current arvados.User
- err = sess.client.RequestAndDecode(¤t, "GET", "/arvados/v1/users/current", nil, nil)
+ err = sess.client.RequestAndDecode(¤t, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return nil, err
}
commit 6f51168f5fce636d5ded92f452932dd056253daf
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:59:20 2022 -0400
19363: Skip brand new sessions when pruning.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index d5fdc4997..e10419f5d 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -276,7 +276,7 @@ func (c *cache) pruneSessions() {
now := time.Now()
var size int64
keys := c.sessions.Keys()
- for _, token := range keys {
+ for idx, token := range keys {
ent, ok := c.sessions.Peek(token)
if !ok {
continue
@@ -284,6 +284,7 @@ func (c *cache) pruneSessions() {
s := ent.(*cachedSession)
if s.expire.Before(now) {
c.sessions.Remove(token)
+ keys[idx] = ""
continue
}
if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
@@ -294,6 +295,12 @@ func (c *cache) pruneSessions() {
// least frequently used entries (which Keys() returns last).
for i := len(keys) - 1; i >= 0; i-- {
token := keys[i]
+ if token == "" {
+ // removed this session in the loop above;
+ // don't prune it, even if it's already been
+ // reinserted.
+ continue
+ }
if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
break
}
commit c2f110e029452ff80b150d9af2c0c1c1ac8f8aa9
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:54:49 2022 -0400
19363: Use a larger page size when retrieving project contents.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index bea1f76e2..faab6e4f0 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -79,6 +79,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
return nil, err
}
+ pagesize := 100000
var inodes []inode
// When #17424 is resolved, remove the outer loop here and use
@@ -100,6 +101,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
Filters: filters,
Order: "uuid",
Select: []string{"uuid", "name", "modified_at", "properties"},
+ Limit: &pagesize,
}
for {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list