[ARVADOS] created: 1.3.0-824-g9a7d66b57

Git user git at public.curoverse.com
Thu Apr 25 19:15:04 UTC 2019


        at  9a7d66b571215d6b65f0475023b2b3bb972f39eb (commit)


commit 9a7d66b571215d6b65f0475023b2b3bb972f39eb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Apr 25 13:54:40 2019 -0400

    14399: Don't insert error text in the middle of an index response.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 51dd73a51..9a4d02df8 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -277,13 +277,19 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
 
 	for _, v := range vols {
 		if err := v.IndexTo(prefix, resp); err != nil {
-			// The only errors returned by IndexTo are
-			// write errors returned by resp.Write(),
-			// which probably means the client has
-			// disconnected and this error will never be
-			// reported to the client -- but it will
-			// appear in our own error log.
-			http.Error(resp, err.Error(), http.StatusInternalServerError)
+			// We can't send an error message to the
+			// client because we might have already sent
+			// headers and index content. All we can do is
+			// log the error in our own logs, and (in
+			// cases where headers haven't been sent yet)
+			// set a 500 status.
+			//
+			// If headers have already been sent, the
+			// client must notice the lack of trailing
+			// newline as an indication that the response
+			// is incomplete.
+			log.Printf("index error from volume %s: %s", v, err)
+			http.Error(resp, "", http.StatusInternalServerError)
 			return
 		}
 	}

commit 7a63d2f831bb253a63dc39f1acc4cfa0113883af
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Apr 25 13:54:03 2019 -0400

    14399: Retry Azure ListBlobs call after 503 errors.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 6b5b233c2..834c883b6 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -26,7 +26,11 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+const (
+	azureDefaultRequestTimeout       = arvados.Duration(10 * time.Minute)
+	azureDefaultListBlobsMaxAttempts = 12
+	azureDefaultListBlobsRetryDelay  = arvados.Duration(10 * time.Second)
+)
 
 var (
 	azureMaxGetBytes           int
@@ -108,6 +112,8 @@ type AzureBlobVolume struct {
 	ReadOnly              bool
 	RequestTimeout        arvados.Duration
 	StorageClasses        []string
+	ListBlobsRetryDelay   arvados.Duration
+	ListBlobsMaxAttempts  int
 
 	azClient  storage.Client
 	container *azureContainer
@@ -149,6 +155,12 @@ func (v *AzureBlobVolume) Type() string {
 
 // Start implements Volume.
 func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
+	if v.ListBlobsRetryDelay == 0 {
+		v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
+	}
+	if v.ListBlobsMaxAttempts == 0 {
+		v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
+	}
 	if v.ContainerName == "" {
 		return errors.New("no container name given")
 	}
@@ -486,8 +498,8 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
 		Prefix:  prefix,
 		Include: &storage.IncludeBlobDataset{Metadata: true},
 	}
-	for {
-		resp, err := v.container.ListBlobs(params)
+	for page := 1; ; page++ {
+		resp, err := v.listBlobs(page, params)
 		if err != nil {
 			return err
 		}
@@ -517,6 +529,23 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
 	}
 }
 
+// call v.container.ListBlobs, retrying if needed.
+func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters) (resp storage.BlobListResponse, err error) {
+	for i := 0; i < v.ListBlobsMaxAttempts; i++ {
+		resp, err = v.container.ListBlobs(params)
+		log.Printf("az err %T %q %#v", err, err, err)
+		err = v.translateError(err)
+		if err == VolumeBusyError {
+			log.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
+			time.Sleep(time.Duration(v.ListBlobsRetryDelay))
+			continue
+		} else {
+			break
+		}
+	}
+	return
+}
+
 // Trash a Keep block.
 func (v *AzureBlobVolume) Trash(loc string) error {
 	if v.ReadOnly {
@@ -674,8 +703,8 @@ func (v *AzureBlobVolume) EmptyTrash() {
 	}
 
 	params := storage.ListBlobsParameters{Include: &storage.IncludeBlobDataset{Metadata: true}}
-	for {
-		resp, err := v.container.ListBlobs(params)
+	for page := 1; ; page++ {
+		resp, err := v.listBlobs(page, params)
 		if err != nil {
 			log.Printf("EmptyTrash: ListBlobs: %v", err)
 			break
@@ -835,3 +864,7 @@ func (c *azureContainer) DeleteBlob(bname string, opts *storage.DeleteBlobOption
 	c.stats.TickErr(err)
 	return err
 }
+
+type retryableError interface {
+	RetryAfterDelay() time.Duration
+}
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index cfad7577c..8d02def14 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/ghodss/yaml"
 	"github.com/prometheus/client_golang/prometheus"
@@ -65,8 +66,9 @@ type azBlob struct {
 
 type azStubHandler struct {
 	sync.Mutex
-	blobs map[string]*azBlob
-	race  chan chan struct{}
+	blobs      map[string]*azBlob
+	race       chan chan struct{}
+	didlist503 bool
 }
 
 func newAzStubHandler() *azStubHandler {
@@ -281,6 +283,11 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		rw.WriteHeader(http.StatusAccepted)
 	case r.Method == "GET" && r.Form.Get("comp") == "list" && r.Form.Get("restype") == "container":
 		// "List Blobs" API
+		if !h.didlist503 {
+			h.didlist503 = true
+			rw.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
 		prefix := container + "|" + r.Form.Get("prefix")
 		marker := r.Form.Get("marker")
 
@@ -388,14 +395,17 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
 			t.Fatal(err)
 		}
 	}
+	azClient.Sender = &singleSender{}
 
 	bs := azClient.GetBlobService()
 	v := &AzureBlobVolume{
-		ContainerName:    container,
-		ReadOnly:         readonly,
-		AzureReplication: replication,
-		azClient:         azClient,
-		container:        &azureContainer{ctr: bs.GetContainerReference(container)},
+		ContainerName:        container,
+		ReadOnly:             readonly,
+		AzureReplication:     replication,
+		ListBlobsMaxAttempts: 2,
+		ListBlobsRetryDelay:  arvados.Duration(time.Millisecond),
+		azClient:             azClient,
+		container:            &azureContainer{ctr: bs.GetContainerReference(container)},
 	}
 
 	return &TestableAzureBlobVolume{

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list