[arvados] updated: 2.7.0-6008-g62168c2db5

[git repository hosting]

Summary of changes:
 services/keepstore/s3_volume.go |  4 +--
 services/keepstore/volume.go    | 64 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 63 insertions(+), 5 deletions(-)

       via  62168c2db5c36de2362cd1d5785b598b187bbef3 (commit)
       via  e6d11bb6b7d0008ba31e0ea07d577e7787236c74 (commit)
      from  022107bd52092c658208e74161581c6bedda4a5f (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 62168c2db5c36de2362cd1d5785b598b187bbef3
Author: Tom Clegg <tom at curii.com>
Date:   Fri Feb 16 10:22:28 2024 -0500

    2960: Tweak s3 concurrent download parameters.
    
    Trials show performance difference is negligible between 11x 6MiB and
    13x 5MiB. Given that, I figure lower concurrency wins because it
    involves less processing.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index d4b90540ea..dc857c3264 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -42,8 +42,8 @@ const (
 	s3DefaultConnectTimeout     = arvados.Duration(time.Minute)
 	maxClockSkew                = 600 * time.Second
 	nearlyRFC1123               = "Mon, 2 Jan 2006 15:04:05 GMT"
-	s3downloaderPartSize        = 5 * 1024 * 1024
-	s3downloaderReadConcurrency = 13
+	s3downloaderPartSize        = 6 * 1024 * 1024
+	s3downloaderReadConcurrency = 11
 	s3uploaderPartSize          = 5 * 1024 * 1024
 	s3uploaderWriteConcurrency  = 5
 )

commit e6d11bb6b7d0008ba31e0ea07d577e7787236c74
Author: Tom Clegg <tom at curii.com>
Date:   Thu Feb 15 19:05:33 2024 -0500

    2960: Add back volume implementation requirements.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index f1b6781da6..cd61804913 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -15,15 +15,73 @@ import (
 
 // volume is the interface to a back-end storage device.
 type volume interface {
+	// Return a unique identifier for the backend device. If
+	// possible, this should be chosen such that keepstore
+	// processes running on different hosts, and accessing the
+	// same backend device, will return the same string.
+	//
+	// This helps keep-balance avoid redundantly downloading
+	// multiple index listings for the same backend device.
+	DeviceID() string
+
+	// Copy a block from the backend device to writeTo.
+	//
+	// As with all volume methods, the hash argument is a
+	// 32-character hexadecimal string.
+	//
+	// Data can be written to writeTo in any order, and concurrent
+	// calls to writeTo.WriteAt() are allowed.  However, BlockRead
+	// must not do multiple writes that intersect with any given
+	// byte offset.
+	//
+	// BlockRead is not expected to verify data integrity.
+	//
+	// If the indicated block does not exist, or has been trashed,
+	// BlockRead must return os.ErrNotExist.
 	BlockRead(ctx context.Context, hash string, writeTo io.WriterAt) error
+
+	// Store a block on the backend device, and set its timestamp
+	// to the current time.
+	//
+	// The implementation must ensure that regardless of any
+	// errors encountered while writing, a partially written block
+	// is not left behind: a subsequent BlockRead call must return
+	// either a) the data previously stored under the given hash,
+	// if any, or b) os.ErrNotExist.
 	BlockWrite(ctx context.Context, hash string, data []byte) error
-	DeviceID() string
+
+	// Update the indicated block's stored timestamp to the
+	// current time.
 	BlockTouch(hash string) error
+
+	// Return the indicated block's stored timestamp.
+	Mtime(hash string) (time.Time, error)
+
+	// Mark the indicated block as trash, such that -- unless it
+	// is untrashed before time.Now() + BlobTrashLifetime --
+	// BlockRead returns os.ErrNotExist and the block is not
+	// listed by Index.
 	BlockTrash(hash string) error
+
+	// Un-mark the indicated block as trash. If the block has not
+	// been trashed, return os.ErrNotExist.
 	BlockUntrash(hash string) error
-	Index(ctx context.Context, prefix string, writeTo io.Writer) error
-	Mtime(hash string) (time.Time, error)
+
+	// Permanently delete all blocks that have been marked as
+	// trash for BlobTrashLifetime or longer.
 	EmptyTrash()
+
+	// Write an index of all non-trashed blocks available on the
+	// backend device whose hash begins with the given prefix
+	// (prefix is a string of zero or more hexadecimal digits).
+	//
+	// Each block is written as "{hash}+{size} {timestamp}\n"
+	// where timestamp is a decimal-formatted number of
+	// nanoseconds since the UTC Unix epoch.
+	//
+	// Index should abort and return ctx.Err() if ctx is cancelled
+	// before indexing is complete.
+	Index(ctx context.Context, prefix string, writeTo io.Writer) error
 }
 
 type volumeDriver func(newVolumeParams) (volume, error)

-----------------------------------------------------------------------


hooks/post-receive
--