[ARVADOS] created: 39536d8dd7f0a6ab89e106cd065830f1cbb067b1

Git user git at public.curoverse.com
Mon Nov 7 10:36:58 EST 2016


        at  39536d8dd7f0a6ab89e106cd065830f1cbb067b1 (commit)


commit 39536d8dd7f0a6ab89e106cd065830f1cbb067b1
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Nov 7 10:32:01 2016 -0500

    10468: Add RequestTimeout to Azure volume config.

diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index d2163f6..d2c4620 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"regexp"
 	"strconv"
@@ -15,9 +16,12 @@ import (
 	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/curoverse/azure-sdk-for-go/storage"
 )
 
+const azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+
 var (
 	azureMaxGetBytes           int
 	azureStorageAccountName    string
@@ -95,6 +99,7 @@ type AzureBlobVolume struct {
 	ContainerName         string
 	AzureReplication      int
 	ReadOnly              bool
+	RequestTimeout        arvados.Duration
 
 	azClient storage.Client
 	bsClient storage.BlobStorageClient
@@ -108,6 +113,7 @@ func (*AzureBlobVolume) Examples() []Volume {
 			StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
 			ContainerName:         "example-container-name",
 			AzureReplication:      3,
+			RequestTimeout:        azureDefaultRequestTimeout,
 		},
 	}
 }
@@ -133,6 +139,13 @@ func (v *AzureBlobVolume) Start() error {
 	if err != nil {
 		return fmt.Errorf("creating Azure storage client: %s", err)
 	}
+
+	if v.RequestTimeout == 0 {
+		v.RequestTimeout = azureDefaultRequestTimeout
+	}
+	v.azClient.HTTPClient = &http.Client{
+		Timeout: time.Duration(v.RequestTimeout),
+	}
 	v.bsClient = v.azClient.GetBlobService()
 
 	ok, err := v.bsClient.ContainerExists(v.ContainerName)

commit 36bbbee25ab89a499f4015fb39845cc2d911aa63
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Nov 7 10:15:55 2016 -0500

    10468: Use long S3 timeouts by default, instead of waiting forever.

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 6ddcaf1..a0cf450 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -209,6 +209,16 @@ func (v *S3Volume) Start() error {
 	if err != nil {
 		return err
 	}
+
+	// Zero timeouts mean "wait forever", which is a bad
+	// default. Default to long timeouts instead.
+	if v.ConnectTimeout == 0 {
+		v.ConnectTimeout = arvados.Duration(time.Minute)
+	}
+	if v.ReadTimeout == 0 {
+		v.ReadTimeout = arvados.Duration(10 * time.Minute)
+	}
+
 	client := s3.New(auth, region)
 	client.ConnectTimeout = time.Duration(v.ConnectTimeout)
 	client.ReadTimeout = time.Duration(v.ReadTimeout)

commit 1c1f12b1f2c32cdee5fab278f38a65ec246cbbf0
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Nov 7 10:09:48 2016 -0500

    10468: Add ConnectTimeout and ReadTimeout configs for S3 volumes.

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index caed35b..6ddcaf1 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -134,6 +134,8 @@ type S3Volume struct {
 	LocationConstraint bool
 	IndexPageSize      int
 	S3Replication      int
+	ConnectTimeout     arvados.Duration
+	ReadTimeout        arvados.Duration
 	RaceWindow         arvados.Duration
 	ReadOnly           bool
 	UnsafeDelete       bool
@@ -147,24 +149,28 @@ type S3Volume struct {
 func (*S3Volume) Examples() []Volume {
 	return []Volume{
 		&S3Volume{
-			AccessKeyFile: "/etc/aws_s3_access_key.txt",
-			SecretKeyFile: "/etc/aws_s3_secret_key.txt",
-			Endpoint:      "",
-			Region:        "us-east-1",
-			Bucket:        "example-bucket-name",
-			IndexPageSize: 1000,
-			S3Replication: 2,
-			RaceWindow:    arvados.Duration(24 * time.Hour),
+			AccessKeyFile:  "/etc/aws_s3_access_key.txt",
+			SecretKeyFile:  "/etc/aws_s3_secret_key.txt",
+			Endpoint:       "",
+			Region:         "us-east-1",
+			Bucket:         "example-bucket-name",
+			IndexPageSize:  1000,
+			S3Replication:  2,
+			RaceWindow:     arvados.Duration(24 * time.Hour),
+			ConnectTimeout: arvados.Duration(time.Minute),
+			ReadTimeout:    arvados.Duration(5 * time.Minute),
 		},
 		&S3Volume{
-			AccessKeyFile: "/etc/gce_s3_access_key.txt",
-			SecretKeyFile: "/etc/gce_s3_secret_key.txt",
-			Endpoint:      "https://storage.googleapis.com",
-			Region:        "",
-			Bucket:        "example-bucket-name",
-			IndexPageSize: 1000,
-			S3Replication: 2,
-			RaceWindow:    arvados.Duration(24 * time.Hour),
+			AccessKeyFile:  "/etc/gce_s3_access_key.txt",
+			SecretKeyFile:  "/etc/gce_s3_secret_key.txt",
+			Endpoint:       "https://storage.googleapis.com",
+			Region:         "",
+			Bucket:         "example-bucket-name",
+			IndexPageSize:  1000,
+			S3Replication:  2,
+			RaceWindow:     arvados.Duration(24 * time.Hour),
+			ConnectTimeout: arvados.Duration(time.Minute),
+			ReadTimeout:    arvados.Duration(5 * time.Minute),
 		},
 	}
 }
@@ -203,8 +209,11 @@ func (v *S3Volume) Start() error {
 	if err != nil {
 		return err
 	}
+	client := s3.New(auth, region)
+	client.ConnectTimeout = time.Duration(v.ConnectTimeout)
+	client.ReadTimeout = time.Duration(v.ReadTimeout)
 	v.bucket = &s3.Bucket{
-		S3:   s3.New(auth, region),
+		S3:   client,
 		Name: v.Bucket,
 	}
 	return nil

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list