[ARVADOS] created: 1.1.0-17-ga3851ee

[Git user]

        at  a3851eec63fb52c1f8121395d0092f4aff25405f (commit)


commit a3851eec63fb52c1f8121395d0092f4aff25405f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Oct 11 16:28:43 2017 -0400

    12416: Merge Put() into PutReader().
    
    Put() was only used for empty blocks, and passing a nil reader to
    PutReader works just as well.
    
    This ensures the "put empty block" fix applies to all writes.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index bd929fb..e6a53d0 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -420,7 +420,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
 		if err != nil {
 			return
 		}
-		err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+		err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{})
 	}()
 	select {
 	case <-ctx.Done():
@@ -452,7 +452,7 @@ func (v *S3Volume) Touch(loc string) error {
 	} else if err != nil {
 		return err
 	}
-	err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{})
 	return v.translateError(err)
 }
 
@@ -466,7 +466,7 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) {
 	err = v.translateError(err)
 	if os.IsNotExist(err) {
 		// The data object X exists, but recent/X is missing.
-		err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+		err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{})
 		if err != nil {
 			log.Printf("error: creating %q: %s", "recent/"+loc, err)
 			return zeroTime, v.translateError(err)
@@ -648,7 +648,7 @@ func (v *S3Volume) Untrash(loc string) error {
 	if err != nil {
 		return err
 	}
-	err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{})
 	return v.translateError(err)
 }
 
@@ -927,21 +927,18 @@ func (b *s3bucket) Head(path string, headers map[string][]string) (*http.Respons
 }
 
 func (b *s3bucket) PutReader(path string, r io.Reader, length int64, contType string, perm s3.ACL, options s3.Options) error {
-	err := b.Bucket.PutReader(path, NewCountingReader(r, b.stats.TickOutBytes), length, contType, perm, options)
-	b.stats.Tick(&b.stats.Ops, &b.stats.PutOps)
-	b.stats.TickErr(err)
-	return err
-}
-
-func (b *s3bucket) Put(path string, data []byte, contType string, perm s3.ACL, options s3.Options) error {
-        var reader io.ReadCloser
-	// goamz will only send Content-Length: 0 when io.Reader is nil due to net.http.Request.ContentLength
-	// behavior. otherwise, Content-Length header is omitted which will cause some S3-like services (such
-	// as Ceph RadosGW) to fail to create empty objects
-        if data != nil {
-               reader = NewCountingReader(bytes.NewBuffer(data), b.stats.TickOutBytes)
+	if length == 0 {
+		// goamz will only send Content-Length: 0 when reader
+		// is nil due to net.http.Request.ContentLength
+		// behavior.  Otherwise, Content-Length header is
+		// omitted which will cause some S3 services
+		// (including AWS and Ceph RadosGW) to fail to create
+		// empty objects.
+		r = nil
+	} else {
+		r = NewCountingReader(r, b.stats.TickOutBytes)
 	}
-	err := b.Bucket.PutReader(path, reader, int64(len(data)), contType, perm, options)
+	err := b.Bucket.PutReader(path, r, length, contType, perm, options)
 	b.stats.Tick(&b.stats.Ops, &b.stats.PutOps)
 	b.stats.TickErr(err)
 	return err

commit 3bdff577515638d273c1df5ba9d702cdf3d5abbd
Merge: 0bb435a 94dd316
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Oct 12 09:14:26 2017 -0400

    12416: Merge branch 'fix/keepstore-s3-radosgw-empty-object' of https://github.com/wtsi-hgi/arvados into 12416-s3-empty-object
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>


commit 94dd3166ce6120a49049597493771d33fc089bfd
Author: Joshua C. Randall <jcrandall at alum.mit.edu>
Date:   Sun Oct 8 12:10:33 2017 +0000

    fix s3_volume so that it can create empty recent/* objects on Ceph RadosGW
    
    Arvados-DCO-1.1-Signed-off-by: Joshua C. Randall <jcrandall at alum.mit.edu>

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 3f08f6e..bd929fb 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -934,7 +934,14 @@ func (b *s3bucket) PutReader(path string, r io.Reader, length int64, contType st
 }
 
 func (b *s3bucket) Put(path string, data []byte, contType string, perm s3.ACL, options s3.Options) error {
-	err := b.Bucket.PutReader(path, NewCountingReader(bytes.NewBuffer(data), b.stats.TickOutBytes), int64(len(data)), contType, perm, options)
+        var reader io.ReadCloser
+	// goamz will only send Content-Length: 0 when io.Reader is nil due to net.http.Request.ContentLength
+	// behavior. otherwise, Content-Length header is omitted which will cause some S3-like services (such
+	// as Ceph RadosGW) to fail to create empty objects
+        if data != nil {
+               reader = NewCountingReader(bytes.NewBuffer(data), b.stats.TickOutBytes)
+	}
+	err := b.Bucket.PutReader(path, reader, int64(len(data)), contType, perm, options)
 	b.stats.Tick(&b.stats.Ops, &b.stats.PutOps)
 	b.stats.TickErr(err)
 	return err

-----------------------------------------------------------------------


hooks/post-receive
--