[ARVADOS] created: 1.1.4-142-g97ad088
Git user
git at public.curoverse.com
Tue May 1 09:29:53 EDT 2018
at 97ad08809663d2adcb00947a1dcc0f922f5937bc (commit)
commit 97ad08809663d2adcb00947a1dcc0f922f5937bc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 25 10:58:06 2018 -0400
12167: Propagate X-Request-Id in API calls.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 9247bc4..91da5a3 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -122,6 +122,9 @@ type ArvadosClient struct {
// Number of retries
Retries int
+
+ // X-Request-Id for outgoing requests
+ RequestID string
}
var CertFiles = []string{
@@ -266,6 +269,9 @@ func (c *ArvadosClient) CallRaw(method string, resourceType string, uuid string,
// Add api token header
req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
+ if c.RequestID != "" {
+ req.Header.Add("X-Request-Id", c.RequestID)
+ }
if c.External {
req.Header.Add("X-External-Client", "1")
}
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 0b17d97..c2bf6bc 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -257,6 +257,7 @@ func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, r
var err error
arv := *kc.Arvados
arv.ApiToken = tok
+ arv.RequestID = req.Header.Get("X-Request-Id")
if op == "read" {
err = arv.Call("HEAD", "keep_services", "", "accessible", nil, nil)
} else {
commit 49bf45439d31edbaad88a12c5958c03720824ec9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 25 00:34:19 2018 -0400
12167: Propagate X-Request-Id in keep requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 620bdbe..459197c 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -22,6 +22,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/asyncbuf"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
// A Keep "block" is 64MB.
@@ -99,6 +100,7 @@ type KeepClient struct {
HTTPClient HTTPClient
Retries int
BlockCache *BlockCache
+ RequestID string
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
@@ -232,7 +234,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
errs = append(errs, fmt.Sprintf("%s: %v", url, err))
continue
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+ kc.setRequestHeaders(req)
resp, err := kc.httpClient().Do(req)
if err != nil {
// Probably a network error, may be transient,
@@ -350,7 +352,7 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error
return nil, err
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken))
+ kc.setRequestHeaders(req)
resp, err := kc.httpClient().Do(req)
if err != nil {
return nil, err
@@ -539,6 +541,17 @@ func (kc *KeepClient) httpClient() HTTPClient {
return c
}
+var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
+
+func (kc *KeepClient) setRequestHeaders(req *http.Request) {
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+ if kc.RequestID != "" {
+ req.Header.Set("X-Request-Id", kc.RequestID)
+ } else {
+ req.Header.Set("X-Request-Id", reqIDGen.Next())
+ }
+}
+
type Locator struct {
Hash string
Size int // -1 if data size is not known
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 3791250..a40a574 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -77,7 +77,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
// to be empty, so don't set req.Body.
}
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken))
+ this.setRequestHeaders(req)
req.Header.Add("Content-Type", "application/octet-stream")
req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 63e2f37..02e7cb1 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -416,6 +416,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
+ kc.RequestID = r.Header.Get("X-Request-Id")
var basename string
if len(targetPath) > 0 {
@@ -528,6 +529,7 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
+ kc.RequestID = r.Header.Get("X-Request-Id")
client := (&arvados.Client{
APIHost: arv.ApiServer,
AuthToken: arv.ApiToken,
diff --git a/services/keepproxy/proxy_client.go b/services/keepproxy/proxy_client.go
index 3fa2671..06dfa1c 100644
--- a/services/keepproxy/proxy_client.go
+++ b/services/keepproxy/proxy_client.go
@@ -20,6 +20,6 @@ type proxyClient struct {
func (pc *proxyClient) Do(req *http.Request) (*http.Response, error) {
req.Header.Add("Via", pc.proto+" "+viaAlias)
- req.Header.Add("X-Request-Id", pc.requestID)
+ req.Header.Set("X-Request-Id", pc.requestID)
return pc.client.Do(req)
}
commit 4ac32fd2d53957520517e8713a5d92e762a97e5b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 25 00:20:47 2018 -0400
12167: Fix impossible fixture.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 807047e..7ff67f8 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -533,7 +533,7 @@ replication_desired_2_confirmed_2:
replication_confirmed: 2
updated_at: 2015-02-07 00:24:52.983381227 Z
uuid: zzzzz-4zz18-434zv1tnnf2rygp
- manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:3:foo 3:6:bar\n"
+ manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 37b51d194a7513e45b56f6524f2d51f2+3 0:3:foo 3:3:bar\n"
name: replication want=2 have=2
storage_classes_desired_default_unconfirmed:
commit 08b96e3ff47ef0f3b147442c14d6f5e404c93540
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 25 00:14:16 2018 -0400
12167: Log keep-web requests and responses as JSON.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/httpserver/logger.go b/sdk/go/httpserver/logger.go
index 569931a..1a4b7c5 100644
--- a/sdk/go/httpserver/logger.go
+++ b/sdk/go/httpserver/logger.go
@@ -32,7 +32,9 @@ func LogRequests(h http.Handler) http.Handler {
"remoteAddr": req.RemoteAddr,
"reqForwardedFor": req.Header.Get("X-Forwarded-For"),
"reqMethod": req.Method,
+ "reqHost": req.Host,
"reqPath": req.URL.Path[1:],
+ "reqQuery": req.URL.RawQuery,
"reqBytes": req.ContentLength,
})
logRequest(w, req, lgr)
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 4ffac26..63e2f37 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -10,7 +10,6 @@ import (
"html"
"html/template"
"io"
- "log"
"net/http"
"net/url"
"os"
@@ -26,6 +25,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ log "github.com/Sirupsen/logrus"
"golang.org/x/net/webdav"
)
@@ -191,13 +191,12 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
} else if w.WroteStatus() == 0 {
w.WriteHeader(statusCode)
} else if w.WroteStatus() != statusCode {
- httpserver.Log(r.RemoteAddr, "WARNING",
+ log.WithField("RequestID", r.Header.Get("X-Request-Id")).Warn(
fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
}
if statusText == "" {
statusText = http.StatusText(statusCode)
}
- httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
}()
if strings.HasPrefix(r.URL.Path, "/_health/") && r.Method == "GET" {
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 4894ceb..03bd8f1 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -632,7 +632,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
s.testServer.Handler.ServeHTTP(resp, req)
var cookies []*http.Cookie
@@ -643,7 +643,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
}
cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
for _, c := range cookies {
@@ -671,7 +671,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
@@ -687,7 +687,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
Host: u.Host,
URL: u,
RequestURI: u.RequestURI(),
- Header: trial.header,
+ Header: copyHeader(trial.header),
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
resp = httptest.NewRecorder()
@@ -723,3 +723,11 @@ func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
}
+
+func copyHeader(h http.Header) http.Header {
+ hc := http.Header{}
+ for k, v := range h {
+ hc[k] = append([]string(nil), v...)
+ }
+ return hc
+}
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 724af27..d09fce7 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -7,12 +7,12 @@ package main
import (
"flag"
"fmt"
- "log"
"os"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/config"
+ log "github.com/Sirupsen/logrus"
"github.com/coreos/go-systemd/daemon"
)
@@ -65,6 +65,10 @@ func init() {
if os.Getenv("ARVADOS_API_TOKEN") == "" {
os.Setenv("ARVADOS_API_TOKEN", "xxx")
}
+
+ log.SetFormatter(&log.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ })
}
func main() {
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index aed2989..2995bd3 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -14,7 +14,7 @@ type server struct {
}
func (srv *server) Start() error {
- srv.Handler = httpserver.AddRequestIDs(&handler{Config: srv.Config})
+ srv.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(&handler{Config: srv.Config}))
srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
commit fd86245d5c68c0c82224030e98b7f26974dc1b5c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Apr 23 23:17:41 2018 -0400
12167: Propagate X-Request-Id in API calls.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 24f3faa..ce2c5ae 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -6,6 +6,7 @@ package arvados
import (
"bytes"
+ "context"
"crypto/tls"
"encoding/json"
"fmt"
@@ -19,6 +20,8 @@ import (
"regexp"
"strings"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
)
// A Client is an HTTP client with an API endpoint and a set of
@@ -50,6 +53,8 @@ type Client struct {
KeepServiceURIs []string `json:",omitempty"`
dd *DiscoveryDocument
+
+ ctx context.Context
}
// The default http.Client used by a Client with Insecure==true and
@@ -92,11 +97,25 @@ func NewClientFromEnv() *Client {
}
}
-// Do adds authentication headers and then calls (*http.Client)Do().
+var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
+
+// Do adds Authorization and X-Request-Id headers and then calls
+// (*http.Client)Do().
func (c *Client) Do(req *http.Request) (*http.Response, error) {
if c.AuthToken != "" {
req.Header.Add("Authorization", "OAuth2 "+c.AuthToken)
}
+
+ reqid, ok := c.context().Value(contextKeyRequestID).(string)
+ if !ok {
+ reqid = reqIDGen.Next()
+ }
+ if req.Header.Get("X-Request-Id") == "" {
+ if req.Header == nil {
+ req.Header = http.Header{}
+ }
+ req.Header.Set("X-Request-Id", reqid)
+ }
return c.httpClient().Do(req)
}
@@ -225,6 +244,23 @@ func (c *Client) UpdateBody(rsc resource) io.Reader {
return bytes.NewBufferString(v.Encode())
}
+type contextKey string
+
+var contextKeyRequestID contextKey = "X-Request-Id"
+
+func (c *Client) WithRequestID(reqid string) *Client {
+ cc := *c
+ cc.ctx = context.WithValue(cc.context(), contextKeyRequestID, reqid)
+ return &cc
+}
+
+func (c *Client) context() context.Context {
+ if c.ctx == nil {
+ return context.Background()
+ }
+ return c.ctx
+}
+
func (c *Client) httpClient() *http.Client {
switch {
case c.Client != nil:
diff --git a/sdk/go/httpserver/id_generator.go b/sdk/go/httpserver/id_generator.go
index d2c3a41..6452136 100644
--- a/sdk/go/httpserver/id_generator.go
+++ b/sdk/go/httpserver/id_generator.go
@@ -45,6 +45,9 @@ func AddRequestIDs(h http.Handler) http.Handler {
gen := &IDGenerator{Prefix: "req-"}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Header.Get("X-Request-Id") == "" {
+ if req.Header == nil {
+ req.Header = http.Header{}
+ }
req.Header.Set("X-Request-Id", gen.Next())
}
h.ServeHTTP(w, req)
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 8b61b54..4ffac26 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -424,11 +424,11 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
applyContentDispositionHdr(w, r, basename, attachment)
- client := &arvados.Client{
+ client := (&arvados.Client{
APIHost: arv.ApiServer,
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
- }
+ }).WithRequestID(r.Header.Get("X-Request-Id"))
fs, err := collection.FileSystem(client, kc)
if err != nil {
@@ -529,11 +529,11 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- client := &arvados.Client{
+ client := (&arvados.Client{
APIHost: arv.ApiServer,
AuthToken: arv.ApiToken,
Insecure: arv.ApiInsecure,
- }
+ }).WithRequestID(r.Header.Get("X-Request-Id"))
fs := client.SiteFileSystem(kc)
f, err := fs.Open(r.URL.Path)
if os.IsNotExist(err) {
diff --git a/services/keep-web/server.go b/services/keep-web/server.go
index 0edcf31..aed2989 100644
--- a/services/keep-web/server.go
+++ b/services/keep-web/server.go
@@ -14,7 +14,7 @@ type server struct {
}
func (srv *server) Start() error {
- srv.Handler = &handler{Config: srv.Config}
+ srv.Handler = httpserver.AddRequestIDs(&handler{Config: srv.Config})
srv.Addr = srv.Config.Listen
return srv.Server.Start()
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list