[arvados] created: 2.1.0-2870-g2c0638b04
git repository hosting
git at public.arvados.org
Tue Aug 30 14:37:51 UTC 2022
at 2c0638b0444652591fa18d6cae2d1977ee5e5731 (commit)
commit 2c0638b0444652591fa18d6cae2d1977ee5e5731
Author: Tom Clegg <tom at curii.com>
Date: Tue Aug 30 10:35:58 2022 -0400
19428: Avoid locking entire collection in MemorySize.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 759a0497e..f09c60a57 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -443,8 +443,6 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
}
func (fs *collectionFileSystem) MemorySize() int64 {
- fs.fileSystem.root.RLock()
- defer fs.fileSystem.root.RUnlock()
return fs.fileSystem.root.(*dirnode).MemorySize()
}
@@ -576,6 +574,19 @@ func (fn *filenode) FS() FileSystem {
return fn.fs
}
+func (fn *filenode) MemorySize() (size int64) {
+ fn.RLock()
+ defer fn.RUnlock()
+ size = 64
+ for _, seg := range fn.segments {
+ size += 64
+ if seg, ok := seg.(*memSegment); ok {
+ size += int64(seg.Len())
+ }
+ }
+ return
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@ -1150,27 +1161,18 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
return cg.Wait()
}
-// caller must have write lock.
func (dn *dirnode) MemorySize() (size int64) {
- for _, name := range dn.sortedNames() {
- node := dn.inodes[name]
- node.RLock()
- switch node := node.(type) {
- case *dirnode:
- size += node.MemorySize()
- case *filenode:
- size += 64
- for _, seg := range node.segments {
- switch seg := seg.(type) {
- case *memSegment:
- size += int64(seg.Len())
- }
- size += 64
- }
- }
- node.RUnlock()
+ dn.RLock()
+ todo := make([]inode, 0, len(dn.inodes))
+ for _, node := range dn.inodes {
+ todo = append(todo, node)
}
- return 64 + size
+ dn.RUnlock()
+ size = 64
+ for _, node := range todo {
+ size += node.MemorySize()
+ }
+ return
}
// caller must have write lock.
commit 41eaf4e3b1601b8794f9e1b1abdd610ff3b1855b
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 17:01:08 2022 -0400
19428: Avoid locking entire filesystem for MemorySize.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 2ad4d1f85..6da639eda 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -420,10 +420,20 @@ func (n *treenode) Sync() error {
}
func (n *treenode) MemorySize() (size int64) {
+ // To avoid making other callers wait while we count the
+ // entire filesystem size, we lock the node only long enough
+ // to copy the list of children. We accept that the resulting
+ // size will sometimes be misleading (e.g., we will
+ // double-count an item that moves from A to B after we check
+ // A's size but before we check B's size).
n.RLock()
- defer n.RUnlock()
debugPanicIfNotLocked(n, false)
+ todo := make([]inode, 0, len(n.inodes))
for _, inode := range n.inodes {
+ todo = append(todo, inode)
+ }
+ n.RUnlock()
+ for _, inode := range todo {
size += inode.MemorySize()
}
return 64 + size
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 26012e240..759a0497e 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -443,8 +443,8 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
}
func (fs *collectionFileSystem) MemorySize() int64 {
- fs.fileSystem.root.Lock()
- defer fs.fileSystem.root.Unlock()
+ fs.fileSystem.root.RLock()
+ defer fs.fileSystem.root.RUnlock()
return fs.fileSystem.root.(*dirnode).MemorySize()
}
@@ -1154,8 +1154,7 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
func (dn *dirnode) MemorySize() (size int64) {
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
- node.Lock()
- defer node.Unlock()
+ node.RLock()
switch node := node.(type) {
case *dirnode:
size += node.MemorySize()
@@ -1169,6 +1168,7 @@ func (dn *dirnode) MemorySize() (size int64) {
size += 64
}
}
+ node.RUnlock()
}
return 64 + size
}
commit 082aafae830637728213ec59017c9aae9e3ecc4b
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 17:00:42 2022 -0400
19428: Fix unchecked errors.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3a1d9acde..b04add1c4 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -486,7 +486,13 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Check configured permission
_, sess, err := h.Cache.GetSession(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "session cache: "+err.Error(), http.StatusInternalServerError)
+ }
tokenUser, err = h.Cache.GetTokenUser(arv.ApiToken)
+ if err != nil {
+ http.Error(w, "user lookup: "+err.Error(), http.StatusInternalServerError)
+ }
if webdavMethod[r.Method] {
if !h.userPermittedToUploadOrDownload(r.Method, tokenUser) {
commit 21a93d6e30f152c52e3952e56c9e498bdb695a8c
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:59:58 2022 -0400
19428: Fix leading "/" in path, skip the 301 redirect.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index e10419f5d..b08abad85 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -507,7 +507,7 @@ func (c *cache) GetTokenUser(token string) (*arvados.User, error) {
c.metrics.apiCalls.Inc()
var current arvados.User
- err = sess.client.RequestAndDecode(¤t, "GET", "/arvados/v1/users/current", nil, nil)
+ err = sess.client.RequestAndDecode(¤t, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return nil, err
}
commit 40eb699fdd45523cd29cac5e13d268d3c5f1e3fb
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:59:20 2022 -0400
19428: Skip brand new sessions when pruning.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index d5fdc4997..e10419f5d 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -276,7 +276,7 @@ func (c *cache) pruneSessions() {
now := time.Now()
var size int64
keys := c.sessions.Keys()
- for _, token := range keys {
+ for idx, token := range keys {
ent, ok := c.sessions.Peek(token)
if !ok {
continue
@@ -284,6 +284,7 @@ func (c *cache) pruneSessions() {
s := ent.(*cachedSession)
if s.expire.Before(now) {
c.sessions.Remove(token)
+ keys[idx] = ""
continue
}
if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
@@ -294,6 +295,12 @@ func (c *cache) pruneSessions() {
// least frequently used entries (which Keys() returns last).
for i := len(keys) - 1; i >= 0; i-- {
token := keys[i]
+ if token == "" {
+ // removed this session in the loop above;
+ // don't prune it, even if it's already been
+ // reinserted.
+ continue
+ }
if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
break
}
commit 190ac69810f0bda6b6125ad859c6498539904ccf
Author: Tom Clegg <tom at curii.com>
Date: Mon Aug 29 16:54:49 2022 -0400
19428: Use a larger page size when retrieving project contents.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
index bea1f76e2..faab6e4f0 100644
--- a/sdk/go/arvados/fs_project.go
+++ b/sdk/go/arvados/fs_project.go
@@ -79,6 +79,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
return nil, err
}
+ pagesize := 100000
var inodes []inode
// When #17424 is resolved, remove the outer loop here and use
@@ -100,6 +101,7 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode,
Filters: filters,
Order: "uuid",
Select: []string{"uuid", "name", "modified_at", "properties"},
+ Limit: &pagesize,
}
for {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list