[arvados] updated: 2.7.0-6226-gde7373882c

[git repository hosting]

Summary of changes:
 lib/config/config.default.yml    |  2 +-
 lib/install/deps.go              |  4 ++++
 services/keep-web/s3_test.go     |  6 +++++-
 services/keep-web/writebuffer.go | 22 +++++++++++++++++++++-
 4 files changed, 31 insertions(+), 3 deletions(-)

       via  de7373882c91e6f794cf44284bd7e556dd008e9e (commit)
       via  162de933411450bcd968fe2e030819b8ebb426e3 (commit)
       via  7365b64314406f24e14a910b19b77f7fec3d7ef0 (commit)
      from  b31f4337370c6afd1f97ad4e9827bcf652933684 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit de7373882c91e6f794cf44284bd7e556dd008e9e
Author: Tom Clegg <tom at curii.com>
Date:   Thu Mar 28 10:40:12 2024 -0400

    21606: Change default output buffer size to 0.
    
    See https://dev.arvados.org/issues/21606#note-8.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index a081c2c993..aadd548fd9 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -804,7 +804,7 @@ Clusters:
       # Per-connection output buffer for WebDAV downloads. May improve
       # throughput for large files, particularly when storage volumes
       # have high latency.
-      WebDAVOutputBuffer: 1M
+      WebDAVOutputBuffer: 0
 
     Login:
       # One of the following mechanisms (Google, PAM, LDAP, or

commit 162de933411450bcd968fe2e030819b8ebb426e3
Author: Tom Clegg <tom at curii.com>
Date:   Thu Mar 28 10:39:34 2024 -0400

    21606: Improve code comments.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/keep-web/writebuffer.go b/services/keep-web/writebuffer.go
index f309b69484..90bdcb476b 100644
--- a/services/keep-web/writebuffer.go
+++ b/services/keep-web/writebuffer.go
@@ -11,10 +11,19 @@ import (
 	"sync/atomic"
 )
 
+// writeBuffer uses a ring buffer to implement an asynchronous write
+// buffer.
+//
+// rpos==wpos means the buffer is empty.
+//
+// rpos==(wpos+1)%size means the buffer is full.
+//
+// size<2 means the buffer is always empty and full, so in this case
+// writeBuffer writes through synchronously.
 type writeBuffer struct {
 	out       io.Writer
 	buf       []byte
-	writesize int
+	writesize int           // max bytes flush() should write in a single out.Write()
 	wpos      atomic.Int64  // index in buf where writer (Write()) will write to next
 	wsignal   chan struct{} // receives a value after wpos or closed changes
 	rpos      atomic.Int64  // index in buf where reader (flush()) will read from next
@@ -63,6 +72,7 @@ func (wb *writeBuffer) Write(p []byte) (int, error) {
 	wpos := int(wb.wpos.Load())
 	rpos := int(wb.rpos.Load())
 	for len(todo) > 0 {
+		// wait until the buffer is not full.
 		for rpos == (wpos+1)%len(wb.buf) {
 			select {
 			case <-wb.flushed:
@@ -74,6 +84,8 @@ func (wb *writeBuffer) Write(p []byte) (int, error) {
 				rpos = int(wb.rpos.Load())
 			}
 		}
+		// determine next contiguous portion of buffer that is
+		// available.
 		var avail []byte
 		if rpos == 0 {
 			avail = wb.buf[wpos : len(wb.buf)-1]
@@ -101,6 +113,7 @@ func (wb *writeBuffer) flush() {
 	wpos := 0
 	closed := false
 	for {
+		// wait until buffer is not empty.
 		for rpos == wpos {
 			if closed {
 				return
@@ -109,6 +122,8 @@ func (wb *writeBuffer) flush() {
 			closed = wb.closed.Load()
 			wpos = int(wb.wpos.Load())
 		}
+		// determine next contiguous portion of buffer that is
+		// ready to write through.
 		var ready []byte
 		if rpos < wpos {
 			ready = wb.buf[rpos:wpos]
@@ -131,6 +146,11 @@ func (wb *writeBuffer) flush() {
 	}
 }
 
+// responseWriter enables inserting an io.Writer-wrapper (like
+// *writeBuffer) into an http.ResponseWriter stack.
+//
+// It passes Write() calls to an io.Writer, and all other calls to an
+// http.ResponseWriter.
 type responseWriter struct {
 	io.Writer
 	http.ResponseWriter

commit 7365b64314406f24e14a910b19b77f7fec3d7ef0
Author: Tom Clegg <tom at curii.com>
Date:   Thu Mar 21 11:17:12 2024 -0400

    21606: Update test failing on debian 12. Install s3cmd if available.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/install/deps.go b/lib/install/deps.go
index f9795cf872..08312d738c 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -235,6 +235,10 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
 			} else {
 				pkgs = append(pkgs, "firefox")
 			}
+			if osv.Debian && osv.Major >= 11 {
+				// not available in Debian <11
+				pkgs = append(pkgs, "s3cmd")
+			}
 		}
 		if dev || test {
 			pkgs = append(pkgs,
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 0e821696b7..79b3712c6b 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -1295,7 +1295,11 @@ func (s *IntegrationSuite) TestS3cmd(c *check.C) {
 	cmd = exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.URL[7:], "--host-bucket="+s.testServer.URL[7:], "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "get", "s3://"+arvadostest.FooCollection+"/foo,;$[|]bar", tmpfile)
 	buf, err = cmd.CombinedOutput()
 	c.Check(err, check.NotNil)
-	c.Check(string(buf), check.Matches, `(?ms).*NoSuchKey.*\n`)
+	// As of commit b7520e5c25e1bf25c1a8bf5aa2eadb299be8f606
+	// (between debian bullseye and bookworm versions), s3cmd
+	// started catching the NoSuchKey error code and replacing it
+	// with "Source object '%s' does not exist.".
+	c.Check(string(buf), check.Matches, `(?ms).*(NoSuchKey|Source object.*does not exist).*\n`)
 }
 
 func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {

-----------------------------------------------------------------------


hooks/post-receive
--