[ARVADOS] updated: 1.1.4-192-gfd3a655
Git user
git at public.curoverse.com
Thu May 3 14:06:42 EDT 2018
Summary of changes:
sdk/go/keepclient/keepclient.go | 15 +++++++++------
sdk/go/keepclient/keepclient_test.go | 6 +++---
sdk/go/keepclient/support.go | 28 +++++++++++++---------------
3 files changed, 25 insertions(+), 24 deletions(-)
via fd3a65529ae255a8c57552760707690437623b0a (commit)
from f691b3f7dfba6a337504211f1db44a980586afdd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fd3a65529ae255a8c57552760707690437623b0a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 3 14:03:40 2018 -0400
12167: Generate only one request ID per get/put.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 459197c..d88e767 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -202,6 +202,8 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
}
+ reqid := kc.getRequestID()
+
var expectLength int64
if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 {
expectLength = -1
@@ -234,7 +236,8 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
continue
}
- kc.setRequestHeaders(req)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+ req.Header.Add("X-Request-Id", reqid)
resp, err := kc.httpClient().Do(req)
if err != nil {
// Probably a network error, may be transient,
@@ -352,7 +355,8 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
return nil, err
}
- kc.setRequestHeaders(req)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+ req.Header.Set("X-Request-Id", kc.getRequestID())
resp, err := kc.httpClient().Do(req)
if err != nil {
return nil, err
@@ -543,12 +547,11 @@ func (kc *KeepClient) httpClient() HTTPClient {
var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
-func (kc *KeepClient) setRequestHeaders(req *http.Request) {
- req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+func (kc *KeepClient) getRequestID() string {
if kc.RequestID != "" {
- req.Header.Set("X-Request-Id", kc.RequestID)
+ return kc.RequestID
} else {
- req.Header.Set("X-Request-Id", reqIDGen.Next())
+ return reqIDGen.Next()
}
}
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 3922709..6815e1b 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -153,7 +153,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
@@ -174,7 +174,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
<-st.handled
@@ -227,7 +227,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
func(kc *KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index a40a574..bfe8d5b 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -11,7 +11,6 @@ import (
"io"
"io/ioutil"
"log"
- "math/rand"
"net/http"
"os"
"strings"
@@ -57,13 +56,13 @@ type uploadStatus struct {
}
func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
- upload_status chan<- uploadStatus, expectedLength int64, requestID int32) {
+ upload_status chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
- DebugPrintf("DEBUG: [%08x] Error creating request PUT %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%s] Error creating request PUT %v error: %v", reqid, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
@@ -77,13 +76,14 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
// to be empty, so don't set req.Body.
}
- this.setRequestHeaders(req)
+ req.Header.Add("X-Request-Id", reqid)
+ req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
var resp *http.Response
if resp, err = this.httpClient().Do(req); err != nil {
- DebugPrintf("DEBUG: [%08x] Upload failed %v error: %v", requestID, url, err.Error())
+ DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
@@ -99,16 +99,16 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
- DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, err2.Error(), response)
+ DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, err2.Error(), response)
upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
- DebugPrintf("DEBUG: [%08x] Upload %v success", requestID, url)
+ DebugPrintf("DEBUG: [%s] Upload %v success", reqid, url)
upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
if resp.StatusCode >= 300 && response == "" {
response = resp.Status
}
- DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
+ DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, resp.StatusCode, response)
upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
@@ -118,9 +118,7 @@ func (this *KeepClient) putReplicas(
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- // Generate an arbitrary ID to identify this specific
- // transaction in debug logs.
- requestID := rand.Int31()
+ reqid := this.getRequestID()
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
@@ -167,8 +165,8 @@ func (this *KeepClient) putReplicas(
for active*replicasPerThread < replicasTodo {
// Start some upload requests
if next_server < len(sv) {
- DebugPrintf("DEBUG: [%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
- go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, requestID)
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[next_server])
+ go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, reqid)
next_server += 1
active += 1
} else {
@@ -184,8 +182,8 @@ func (this *KeepClient) putReplicas(
}
}
}
- DebugPrintf("DEBUG: [%08x] Replicas remaining to write: %v active uploads: %v",
- requestID, replicasTodo, active)
+ DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v",
+ reqid, replicasTodo, active)
// Now wait for something to happen.
if active > 0 {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list