[arvados] updated: 2.1.0-2920-g35db49571
git repository hosting
git at public.arvados.org
Tue Sep 20 20:21:27 UTC 2022
Summary of changes:
doc/admin/upgrading.html.textile.liquid | 6 +
doc/api/methods/collections.html.textile.liquid | 2 +-
lib/config/config.default.yml | 16 +-
lib/config/deprecated.go | 9 -
lib/config/deprecated_test.go | 3 -
sdk/go/arvados/config.go | 11 +-
sdk/go/arvados/fs_base.go | 10 +-
sdk/go/arvados/fs_collection.go | 102 ++++--
sdk/go/arvados/fs_collection_test.go | 19 +-
sdk/go/arvados/fs_site.go | 43 ++-
sdk/go/arvadostest/fixtures.go | 8 +-
services/keep-web/cache.go | 352 +++-----------------
services/keep-web/cache_test.go | 242 +++++++-------
services/keep-web/handler.go | 408 ++++++++++--------------
services/keep-web/handler_test.go | 55 +++-
services/keep-web/main.go | 4 -
services/keep-web/s3.go | 38 +--
services/keep-web/server_test.go | 21 +-
services/keep-web/webdav.go | 17 +-
services/keepproxy/keepproxy_test.go | 2 +-
20 files changed, 548 insertions(+), 820 deletions(-)
via 35db495717a628e0a6ef52a453b8d8ced793c41b (commit)
via a88aa282a8da52589053244f4ef41283d51f629a (commit)
via f1ec6199fbe9b6abdba3a9eba95eda7b46eb5265 (commit)
via b8fe3b514556be00f877d63d957368727b608a61 (commit)
via 6115e62fba5208f53ef99cf7f3096e53e9290490 (commit)
via 25162c18b4290f7763cd545f2d94d96650ebc9d9 (commit)
via bd387cabd3fb58e8456f38ba54bb4af0be6500bb (commit)
via 94cf0f9216f09ae37e04686945ff2731bdc74db8 (commit)
via 6b0d4ac8df4b5b4255eb56b1d76865f06089ca2a (commit)
from 28d7ed94b33411909941c72289bc227a38e80cda (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 35db495717a628e0a6ef52a453b8d8ced793c41b
Merge: 28d7ed94b a88aa282a
Author: Tom Clegg <tom at curii.com>
Date: Tue Sep 20 16:09:01 2022 -0400
Merge branch '19362-all-webdav-via-sitefs'
refs #19362
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --cc sdk/go/arvados/fs_collection.go
index f09c60a57,256f513cf..a26c876b9
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@@ -574,19 -602,6 +600,16 @@@ func (fn *filenode) FS() FileSystem
return fn.fs
}
+func (fn *filenode) MemorySize() (size int64) {
+ fn.RLock()
+ defer fn.RUnlock()
+ size = 64
+ for _, seg := range fn.segments {
- size += 64
- if seg, ok := seg.(*memSegment); ok {
- size += int64(seg.Len())
- }
++ size += seg.memorySize()
+ }
+ return
+}
+
// Read reads file data from a single segment, starting at startPtr,
// into p. startPtr is assumed not to be up-to-date. Caller must have
// RLock or Lock.
@@@ -1629,6 -1654,6 +1652,7 @@@ type segment interface
// Return a new segment with a subsection of the data from this
// one. length<0 means length=Len()-off.
Slice(off int, length int) segment
++ memorySize() int64
}
type memSegment struct {
@@@ -1707,6 -1732,6 +1731,10 @@@ func (me *memSegment) ReadAt(p []byte,
return
}
++func (me *memSegment) memorySize() int64 {
++ return 64 + int64(len(me.buf))
++}
++
type storedSegment struct {
kc fsBackend
locator string
@@@ -1744,6 -1769,6 +1772,10 @@@ func (se storedSegment) ReadAt(p []byte
return se.kc.ReadAt(se.locator, p, int(off)+se.offset)
}
++func (se storedSegment) memorySize() int64 {
++ return 64 + int64(len(se.locator))
++}
++
func canonicalName(name string) string {
name = path.Clean("/" + name)
if name == "/" || name == "./" {
diff --cc sdk/go/arvados/fs_collection_test.go
index c2cac3c6c,da3166509..73689e4ea
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@@ -1209,11 -1209,11 +1209,12 @@@ func (s *CollectionFSSuite) TestFlushFu
}
nDirs := int64(8)
++ nFiles := int64(67)
megabyte := make([]byte, 1<<20)
for i := int64(0); i < nDirs; i++ {
dir := fmt.Sprintf("dir%d", i)
fs.Mkdir(dir, 0755)
-- for j := 0; j < 67; j++ {
++ for j := int64(0); j < nFiles; j++ {
f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
c.Assert(err, check.IsNil)
defer f.Close()
@@@ -1221,8 -1221,8 +1222,8 @@@
c.Assert(err, check.IsNil)
}
}
- inodebytes := int64((nDirs*(67*2+1) + 1) * 64)
- c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67<<20)+inodebytes)
- inodebytes := int64((nDirs*(67+1) + 1) * 64)
- c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67*(1<<20+8))+inodebytes)
++ inodebytes := int64((nDirs*(nFiles+1) + 1) * 64)
++ c.Check(fs.MemorySize(), check.Equals, nDirs*nFiles*(1<<20+64)+inodebytes)
c.Check(flushed, check.Equals, int64(0))
waitForFlush := func(expectUnflushed, expectFlushed int64) {
@@@ -1233,27 -1233,29 +1234,29 @@@
}
// Nothing flushed yet
- waitForFlush((nDirs*67)<<20+inodebytes, 0)
- waitForFlush(nDirs*67*(1<<20+8)+inodebytes, 0)
++ waitForFlush(nDirs*nFiles*(1<<20+64)+inodebytes, 0)
// Flushing a non-empty dir "/" is non-recursive and there are
// no top-level files, so this has no effect
fs.Flush("/", false)
- waitForFlush((nDirs*67)<<20+inodebytes, 0)
- waitForFlush(nDirs*67*(1<<20+8)+inodebytes, 0)
++ waitForFlush(nDirs*nFiles*(1<<20+64)+inodebytes, 0)
// Flush the full block in dir0
fs.Flush("dir0", false)
- waitForFlush((nDirs*67-64)<<20+inodebytes, 64<<20)
- bigloclen := int64(32 + 9 + 51 + 40) // md5 + "+" + "67xxxxxx" + "+Axxxxxx..." + 40 (see (*dirnode)MemorySize)
- waitForFlush((nDirs*67-64)*(1<<20+8)+inodebytes+bigloclen*64, 64<<20)
++ bigloclen := int64(32 + 9 + 51 + 64) // md5 + "+" + "67xxxxxx" + "+Axxxxxx..." + 64 (see (storedSegment)memorySize)
++ waitForFlush((nDirs*nFiles-64)*(1<<20+64)+inodebytes+bigloclen*64, 64<<20)
err = fs.Flush("dir-does-not-exist", false)
c.Check(err, check.NotNil)
// Flush full blocks in all dirs
fs.Flush("", false)
- waitForFlush(nDirs*3<<20+inodebytes, nDirs*64<<20)
- waitForFlush(nDirs*3*(1<<20+8)+inodebytes+bigloclen*64*nDirs, nDirs*64<<20)
++ waitForFlush(nDirs*3*(1<<20+64)+inodebytes+bigloclen*64*nDirs, nDirs*64<<20)
// Flush non-full blocks, too
fs.Flush("", true)
- waitForFlush(inodebytes, nDirs*67<<20)
- smallloclen := int64(32 + 8 + 51 + 40) // md5 + "+" + "3xxxxxx" + "+Axxxxxx..." + 40 (see (*dirnode)MemorySize)
++ smallloclen := int64(32 + 8 + 51 + 64) // md5 + "+" + "3xxxxxx" + "+Axxxxxx..." + 64 (see (storedSegment)memorySize)
+ waitForFlush(inodebytes+bigloclen*64*nDirs+smallloclen*3*nDirs, nDirs*67<<20)
}
// Even when writing lots of files/dirs from different goroutines, as
diff --cc services/keep-web/cache.go
index 7ec8639ab,3ba3e60ab..db06d6350
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@@ -294,11 -203,22 +205,11 @@@ func (c *cache) pruneSessions()
}
// Remove tokens until reaching size limit, starting with the
// least frequently used entries (which Keys() returns last).
- for i := len(keys) - 1; i >= 0 && size > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2; i-- {
- for i := len(keys) - 1; i >= 0; i-- {
- token := keys[i]
- if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes {
- break
- }
- ent, ok := c.sessions.Peek(token)
- if !ok {
- continue
- }
- s := ent.(*cachedSession)
- fs, _ := s.fs.Load().(arvados.CustomFileSystem)
- if fs == nil {
- continue
++ for i := len(keys) - 1; i >= 0 && size > c.cluster.Collections.WebDAVCache.MaxCollectionBytes; i-- {
+ if sizes[i] > 0 {
+ c.sessions.Remove(keys[i])
+ size -= sizes[i]
}
- c.sessions.Remove(token)
- size -= fs.MemorySize()
}
}
diff --cc services/keep-web/server_test.go
index 61c540808,1305416c4..30f755e1f
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@@ -475,15 -476,7 +476,7 @@@ func (s *IntegrationSuite) TestMetrics(
c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
- c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
- c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
- c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
- c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
- c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
- // FooCollection's cached manifest size is 45 ("1f4b0....+45")
- // plus one 51-byte blob signature; session fs counts 3 inodes
- // * 64 bytes.
- c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51+64*3))
- c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(445))
++ c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(384))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list