[arvados] updated: 2.7.0-5659-gec8d4899d7
git repository hosting
git at public.arvados.org
Thu Dec 28 20:41:03 UTC 2023
Summary of changes:
doc/admin/upgrading.html.textile.liquid | 6 ++++++
doc/install/install-keep-web.html.textile.liquid | 9 +++++++++
sdk/go/arvados/keep_cache.go | 24 ++++++++++++++----------
3 files changed, 29 insertions(+), 10 deletions(-)
via ec8d4899d74ea76c30cb394dfcda6a54cd9e6652 (commit)
via 279efb4dd345bcb1beee2c77ac14d66e57103b9f (commit)
from d1f5285cdf294074bb1835b502f0dc2f638f7399 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ec8d4899d74ea76c30cb394dfcda6a54cd9e6652
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 28 15:40:14 2023 -0500
20318: Add install/upgrade notes about /var/cache.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 81b32c1444..7b8c20fdf9 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -32,6 +32,12 @@ h2(#main). development main
"previous: Upgrading to 2.7.1":#v2_7_1
+h3. WebDAV service uses @/var/cache@ for file content
+
+ at keep-web@ now stores copies of recently accessed data blocks in @/var/cache/arvados/keep@ instead of in memory. That directory will be created automatically. The default cache size is 10% of the filesystem size. Use the new @Collections.WebDAVCache.DiskCacheSize@ config to specify a different percentage or an absolute size.
+
+If the previously supported @MaxBlockEntries@ config is present, remove it to avoid warning messages at startup.
+
h2(#2_7_1). v2.7.1 (2023-12-12)
"previous: Upgrading to 2.7.0":#v2_7_0
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index b3c6386129..0b051e715d 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -163,6 +163,15 @@ Normally, Keep-web accepts requests for multiple collections using the same host
In such cases -- for example, a site which is not reachable from the internet, where some data is world-readable from Arvados's perspective but is intended to be available only to users within the local network -- the downstream proxy should configured to return 401 for all paths beginning with "/c="
{% include 'notebox_end' %}
+h3. Configure filesystem cache size
+
+Keep-web stores copies of recently accessed data blocks in @/var/cache/arvados/keep at . The cache size defaults to 10% of the size of the filesystem where that directory is located (typically @/var@) and can be customized with the @DiskCacheSize@ config entry.
+
+<notextile>
+<pre><code> Collections:
+ WebDAVCache:
+ DiskCacheSize: 20 GiB</code></pre></notextile>
+
{% assign arvados_component = 'keep-web' %}
{% include 'install_packages' %}
commit 279efb4dd345bcb1beee2c77ac14d66e57103b9f
Author: Tom Clegg <tom at curii.com>
Date: Thu Dec 28 14:13:24 2023 -0500
20318: Sync cache state after 1% churn instead of 5 minute timer.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/keep_cache.go b/sdk/go/arvados/keep_cache.go
index 1d12f4cdc6..b6b2a9da66 100644
--- a/sdk/go/arvados/keep_cache.go
+++ b/sdk/go/arvados/keep_cache.go
@@ -61,7 +61,6 @@ type sharedCache struct {
maxSize ByteSizeOrPercent
tidying int32 // see tidy()
- tidyHoldUntil time.Time
defaultMaxSize int64
// The "heldopen" fields are used to open cache files for
@@ -79,8 +78,10 @@ type sharedCache struct {
writingCond *sync.Cond
writingLock sync.Mutex
- sizeMeasured int64 // actual size on disk after last tidy(); zero if not measured yet
- sizeEstimated int64 // last measured size, plus files we have written since
+ sizeMeasured int64 // actual size on disk after last tidy(); zero if not measured yet
+ sizeEstimated int64 // last measured size, plus files we have written since
+ lastFileCount int64 // number of files on disk at last count
+ writesSinceTidy int64 // number of files written since last tidy()
}
type writeprogress struct {
@@ -97,9 +98,8 @@ type openFileEnt struct {
}
const (
- cacheFileSuffix = ".keepcacheblock"
- tmpFileSuffix = ".tmp"
- tidyHoldDuration = 5 * time.Minute // time to re-check cache size even if estimated size is below max
+ cacheFileSuffix = ".keepcacheblock"
+ tmpFileSuffix = ".tmp"
)
func (cache *DiskCache) setup() {
@@ -557,6 +557,7 @@ func (cache *DiskCache) BlockRead(ctx context.Context, opts BlockReadOptions) (i
// Start a tidy() goroutine, unless one is already running / recently
// finished.
func (cache *DiskCache) gotidy() {
+ writes := atomic.AddInt64(&cache.writesSinceTidy, 1)
// Skip if another tidy goroutine is running in this process.
n := atomic.AddInt32(&cache.tidying, 1)
if n != 1 {
@@ -564,17 +565,18 @@ func (cache *DiskCache) gotidy() {
return
}
// Skip if sizeEstimated is based on an actual measurement and
- // is below maxSize, and we haven't reached the "recheck
- // anyway" time threshold.
+ // is below maxSize, and we haven't done very many writes
+ // since last tidy (defined as 1% of number of cache files at
+ // last count).
if cache.sizeMeasured > 0 &&
atomic.LoadInt64(&cache.sizeEstimated) < atomic.LoadInt64(&cache.defaultMaxSize) &&
- time.Now().Before(cache.tidyHoldUntil) {
+ writes < cache.lastFileCount/100 {
atomic.AddInt32(&cache.tidying, -1)
return
}
go func() {
cache.tidy()
- cache.tidyHoldUntil = time.Now().Add(tidyHoldDuration)
+ atomic.StoreInt64(&cache.writesSinceTidy, 0)
atomic.AddInt32(&cache.tidying, -1)
}()
}
@@ -677,6 +679,7 @@ func (cache *DiskCache) tidy() {
if totalsize <= maxsize || len(ents) == 1 {
atomic.StoreInt64(&cache.sizeMeasured, totalsize)
atomic.StoreInt64(&cache.sizeEstimated, totalsize)
+ cache.lastFileCount = int64(len(ents))
return
}
@@ -710,4 +713,5 @@ func (cache *DiskCache) tidy() {
}
atomic.StoreInt64(&cache.sizeMeasured, totalsize)
atomic.StoreInt64(&cache.sizeEstimated, totalsize)
+ cache.lastFileCount = int64(len(ents) - deleted)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list