[ARVADOS] created: 8a1a20bc8c2aebc0d11be7c9982745005ea71a35
Git user
git at public.curoverse.com
Wed Apr 26 17:11:11 EDT 2017
at 8a1a20bc8c2aebc0d11be7c9982745005ea71a35 (commit)
commit 8a1a20bc8c2aebc0d11be7c9982745005ea71a35
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 26 17:11:01 2017 -0400
11537: Add Via header to proxied keepstore requests.
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 2892031..f3e3960 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"log"
+ "net/http"
"os"
"os/signal"
"reflect"
@@ -22,7 +23,9 @@ func (this *KeepClient) DiscoverKeepServers() error {
if this.Arvados.KeepServiceURIs != nil {
this.foundNonDiskSvc = true
this.replicasPerService = 0
- this.setClientSettingsNonDisk()
+ if c, ok := this.Client.(*http.Client); ok {
+ this.setClientSettingsNonDisk(c)
+ }
roots := make(map[string]string)
for i, uri := range this.Arvados.KeepServiceURIs {
roots[fmt.Sprintf("00000-bi6l4-%015d", i)] = uri
@@ -134,10 +137,12 @@ func (this *KeepClient) loadKeepServers(list svcList) error {
gatewayRoots[service.Uuid] = url
}
- if this.foundNonDiskSvc {
- this.setClientSettingsNonDisk()
- } else {
- this.setClientSettingsDisk()
+ if client, ok := this.Client.(*http.Client); ok {
+ if this.foundNonDiskSvc {
+ this.setClientSettingsNonDisk(client)
+ } else {
+ this.setClientSettingsDisk(client)
+ }
}
this.SetServiceRoots(localRoots, writableLocalRoots, gatewayRoots)
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 4f84afc..95048a9 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -6,8 +6,6 @@ import (
"crypto/md5"
"errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
"net/http"
@@ -15,6 +13,9 @@ import (
"strconv"
"strings"
"sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/streamer"
)
// A Keep "block" is 64MB.
@@ -62,6 +63,10 @@ var ErrIncompleteIndex = errors.New("Got incomplete index")
const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
+type HTTPClient interface {
+ Do(*http.Request) (*http.Response, error)
+}
+
// Information about Arvados and Keep servers.
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
@@ -70,7 +75,7 @@ type KeepClient struct {
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
lock sync.RWMutex
- Client *http.Client
+ Client HTTPClient
Retries int
BlockCache *BlockCache
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 9adbb48..460dfba 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -4,7 +4,6 @@ import (
"crypto/md5"
"errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
"log"
@@ -15,6 +14,8 @@ import (
"regexp"
"strings"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/streamer"
)
// Function used to emit debug messages. The easiest way to enable
@@ -45,49 +46,45 @@ func Md5String(s string) string {
// Set timeouts applicable when connecting to non-disk services
// (assumed to be over the Internet).
-func (this *KeepClient) setClientSettingsNonDisk() {
- if this.Client.Timeout == 0 {
- // Maximum time to wait for a complete response
- this.Client.Timeout = 300 * time.Second
-
- // TCP and TLS connection settings
- this.Client.Transport = &http.Transport{
- Dial: (&net.Dialer{
- // The maximum time to wait to set up
- // the initial TCP connection.
- Timeout: 30 * time.Second,
-
- // The TCP keep alive heartbeat
- // interval.
- KeepAlive: 120 * time.Second,
- }).Dial,
-
- TLSHandshakeTimeout: 10 * time.Second,
- }
+func (*KeepClient) setClientSettingsNonDisk(client *http.Client) {
+ // Maximum time to wait for a complete response
+ client.Timeout = 300 * time.Second
+
+ // TCP and TLS connection settings
+ client.Transport = &http.Transport{
+ Dial: (&net.Dialer{
+ // The maximum time to wait to set up
+ // the initial TCP connection.
+ Timeout: 30 * time.Second,
+
+ // The TCP keep alive heartbeat
+ // interval.
+ KeepAlive: 120 * time.Second,
+ }).Dial,
+
+ TLSHandshakeTimeout: 10 * time.Second,
}
}
// Set timeouts applicable when connecting to keepstore services directly
// (assumed to be on the local network).
-func (this *KeepClient) setClientSettingsDisk() {
- if this.Client.Timeout == 0 {
- // Maximum time to wait for a complete response
- this.Client.Timeout = 20 * time.Second
-
- // TCP and TLS connection timeouts
- this.Client.Transport = &http.Transport{
- Dial: (&net.Dialer{
- // The maximum time to wait to set up
- // the initial TCP connection.
- Timeout: 2 * time.Second,
-
- // The TCP keep alive heartbeat
- // interval.
- KeepAlive: 180 * time.Second,
- }).Dial,
-
- TLSHandshakeTimeout: 4 * time.Second,
- }
+func (*KeepClient) setClientSettingsDisk(client *http.Client) {
+ // Maximum time to wait for a complete response
+ client.Timeout = 20 * time.Second
+
+ // TCP and TLS connection timeouts
+ client.Transport = &http.Transport{
+ Dial: (&net.Dialer{
+ // The maximum time to wait to set up
+ // the initial TCP connection.
+ Timeout: 2 * time.Second,
+
+ // The TCP keep alive heartbeat
+ // interval.
+ KeepAlive: 180 * time.Second,
+ }).Dial,
+
+ TLSHandshakeTimeout: 4 * time.Second,
}
}
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 76a8a15..22bab57 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -129,7 +129,7 @@ func main() {
if cfg.DefaultReplicas > 0 {
kc.Want_replicas = cfg.DefaultReplicas
}
- kc.Client.Timeout = time.Duration(cfg.Timeout)
+ kc.Client.(*http.Client).Timeout = time.Duration(cfg.Timeout)
go kc.RefreshServices(5*time.Minute, 3*time.Second)
listener, err = net.Listen("tcp", cfg.Listen)
@@ -329,6 +329,7 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
}()
kc := *this.KeepClient
+ kc.Client = &proxyClient{client: kc.Client, proto: req.Proto}
var pass bool
var tok string
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list