[ARVADOS] created: 39536d8dd7f0a6ab89e106cd065830f1cbb067b1
Git user
git at public.curoverse.com
Mon Nov 7 10:36:58 EST 2016
at 39536d8dd7f0a6ab89e106cd065830f1cbb067b1 (commit)
commit 39536d8dd7f0a6ab89e106cd065830f1cbb067b1
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 7 10:32:01 2016 -0500
10468: Add RequestTimeout to Azure volume config.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index d2163f6..d2c4620 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -8,6 +8,7 @@ import (
"io"
"io/ioutil"
"log"
+ "net/http"
"os"
"regexp"
"strconv"
@@ -15,9 +16,12 @@ import (
"sync"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/curoverse/azure-sdk-for-go/storage"
)
+const azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+
var (
azureMaxGetBytes int
azureStorageAccountName string
@@ -95,6 +99,7 @@ type AzureBlobVolume struct {
ContainerName string
AzureReplication int
ReadOnly bool
+ RequestTimeout arvados.Duration
azClient storage.Client
bsClient storage.BlobStorageClient
@@ -108,6 +113,7 @@ func (*AzureBlobVolume) Examples() []Volume {
StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
ContainerName: "example-container-name",
AzureReplication: 3,
+ RequestTimeout: azureDefaultRequestTimeout,
},
}
}
@@ -133,6 +139,13 @@ func (v *AzureBlobVolume) Start() error {
if err != nil {
return fmt.Errorf("creating Azure storage client: %s", err)
}
+
+ if v.RequestTimeout == 0 {
+ v.RequestTimeout = azureDefaultRequestTimeout
+ }
+ v.azClient.HTTPClient = &http.Client{
+ Timeout: time.Duration(v.RequestTimeout),
+ }
v.bsClient = v.azClient.GetBlobService()
ok, err := v.bsClient.ContainerExists(v.ContainerName)
commit 36bbbee25ab89a499f4015fb39845cc2d911aa63
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 7 10:15:55 2016 -0500
10468: Use long S3 timeouts by default, instead of waiting forever.
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 6ddcaf1..a0cf450 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -209,6 +209,16 @@ func (v *S3Volume) Start() error {
if err != nil {
return err
}
+
+ // Zero timeouts mean "wait forever", which is a bad
+ // default. Default to long timeouts instead.
+ if v.ConnectTimeout == 0 {
+ v.ConnectTimeout = arvados.Duration(time.Minute)
+ }
+ if v.ReadTimeout == 0 {
+ v.ReadTimeout = arvados.Duration(10 * time.Minute)
+ }
+
client := s3.New(auth, region)
client.ConnectTimeout = time.Duration(v.ConnectTimeout)
client.ReadTimeout = time.Duration(v.ReadTimeout)
commit 1c1f12b1f2c32cdee5fab278f38a65ec246cbbf0
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 7 10:09:48 2016 -0500
10468: Add ConnectTimeout and ReadTimeout configs for S3 volumes.
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index caed35b..6ddcaf1 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -134,6 +134,8 @@ type S3Volume struct {
LocationConstraint bool
IndexPageSize int
S3Replication int
+ ConnectTimeout arvados.Duration
+ ReadTimeout arvados.Duration
RaceWindow arvados.Duration
ReadOnly bool
UnsafeDelete bool
@@ -147,24 +149,28 @@ type S3Volume struct {
func (*S3Volume) Examples() []Volume {
return []Volume{
&S3Volume{
- AccessKeyFile: "/etc/aws_s3_access_key.txt",
- SecretKeyFile: "/etc/aws_s3_secret_key.txt",
- Endpoint: "",
- Region: "us-east-1",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
+ AccessKeyFile: "/etc/aws_s3_access_key.txt",
+ SecretKeyFile: "/etc/aws_s3_secret_key.txt",
+ Endpoint: "",
+ Region: "us-east-1",
+ Bucket: "example-bucket-name",
+ IndexPageSize: 1000,
+ S3Replication: 2,
+ RaceWindow: arvados.Duration(24 * time.Hour),
+ ConnectTimeout: arvados.Duration(time.Minute),
+ ReadTimeout: arvados.Duration(5 * time.Minute),
},
&S3Volume{
- AccessKeyFile: "/etc/gce_s3_access_key.txt",
- SecretKeyFile: "/etc/gce_s3_secret_key.txt",
- Endpoint: "https://storage.googleapis.com",
- Region: "",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
+ AccessKeyFile: "/etc/gce_s3_access_key.txt",
+ SecretKeyFile: "/etc/gce_s3_secret_key.txt",
+ Endpoint: "https://storage.googleapis.com",
+ Region: "",
+ Bucket: "example-bucket-name",
+ IndexPageSize: 1000,
+ S3Replication: 2,
+ RaceWindow: arvados.Duration(24 * time.Hour),
+ ConnectTimeout: arvados.Duration(time.Minute),
+ ReadTimeout: arvados.Duration(5 * time.Minute),
},
}
}
@@ -203,8 +209,11 @@ func (v *S3Volume) Start() error {
if err != nil {
return err
}
+ client := s3.New(auth, region)
+ client.ConnectTimeout = time.Duration(v.ConnectTimeout)
+ client.ReadTimeout = time.Duration(v.ReadTimeout)
v.bucket = &s3.Bucket{
- S3: s3.New(auth, region),
+ S3: client,
Name: v.Bucket,
}
return nil
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list