[ARVADOS] updated: 9959bf0f5631daa84e8afa7de145154390259c67
Git user
git at public.curoverse.com
Tue Oct 3 15:18:24 EDT 2017
Summary of changes:
sdk/go/arvados/config.go | 28 +++++----
sdk/go/health/aggregator.go | 89 +++++++++++++++------------
sdk/go/health/aggregator_test.go | 127 +++++++++++++++++++++++++++++++++------
services/health/main.go | 32 ++++++++--
4 files changed, 198 insertions(+), 78 deletions(-)
via 9959bf0f5631daa84e8afa7de145154390259c67 (commit)
via f0e6212bbe6e34f6ac4e928a236e77cca852895d (commit)
via 825e37afbde9b1f0cb43451170fc300f64943ada (commit)
from 63218507eec156df91f80c86ee05d680d67b8336 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9959bf0f5631daa84e8afa7de145154390259c67
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Oct 3 15:15:20 2017 -0400
12260: Add -config arg. Drop non-resolvable hostname support.
Change check key to svctype+http://host:port/path.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index d7bae38..84e66b3 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -3,20 +3,21 @@ package arvados
import (
"fmt"
"os"
- "strings"
"git.curoverse.com/arvados.git/sdk/go/config"
)
+const DefaultConfigFile = "/etc/arvados/config.yml"
+
type Config struct {
Clusters map[string]Cluster
}
// GetConfig returns the current system config, loading it from
-// /etc if needed.
-func GetConfig() (*Config, error) {
+// configFile if needed.
+func GetConfig(configFile string) (*Config, error) {
var cfg Config
- err := config.LoadFile(&cfg, "/etc/arvados/config.yml")
+ err := config.LoadFile(&cfg, configFile)
return &cfg, err
}
@@ -63,14 +64,8 @@ func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) {
// error is returned if the appropriate configuration can't be
// determined (e.g., this does not appear to be a system node).
func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
- // Generally node is "a.b.ca", use the first of {"a.b.ca",
- // "a.b", "a"} that has an entry in SystemNodes.
- labels := strings.Split(node, ".")
- for j := len(labels); j > 0; j-- {
- hostpart := strings.Join(labels[:j], ".")
- if cfg, ok := cc.SystemNodes[hostpart]; ok {
- return &cfg, nil
- }
+ if cfg, ok := cc.SystemNodes[node]; ok {
+ return &cfg, nil
}
// If node is not listed, but "*" gives a default system node
// config, use the default config.
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 1ba1ee3..e881db8 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -49,15 +49,6 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Content-Type", "application/json")
- if agg.Config == nil {
- cfg, err := arvados.GetConfig()
- if err != nil {
- err = fmt.Errorf("arvados.GetConfig(): %s", err)
- sendErr(http.StatusInternalServerError, err)
- return
- }
- agg.Config = cfg
- }
cluster, err := agg.Config.GetCluster("")
if err != nil {
err = fmt.Errorf("arvados.GetCluster(): %s", err)
@@ -119,11 +110,20 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
wg.Add(1)
go func(node string) {
defer wg.Done()
- pingResp := agg.ping(node, addr, cluster)
+ var pingResp CheckResponse
+ url, err := agg.pingURL(node, addr)
+ if err != nil {
+ pingResp = CheckResponse{
+ Health: "ERROR",
+ Error: err.Error(),
+ }
+ } else {
+ pingResp = agg.ping(url, cluster)
+ }
mtx.Lock()
defer mtx.Unlock()
- resp.Checks[node+"/"+svc+"/_health/ping"] = pingResp
+ resp.Checks[svc+"+"+url] = pingResp
svHealth := resp.Services[svc]
if pingResp.OK() {
svHealth.N++
@@ -148,7 +148,12 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
return resp
}
-func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result CheckResponse) {
+func (agg *Aggregator) pingURL(node, addr string) (string, error) {
+ _, port, err := net.SplitHostPort(addr)
+ return "http://" + node + ":" + port + "/_health/ping", err
+}
+
+func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckResponse) {
t0 := time.Now()
var err error
@@ -159,11 +164,7 @@ func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result
}
}()
- _, port, err := net.SplitHostPort(addr)
- if err != nil {
- return
- }
- req, err := http.NewRequest("GET", "http://"+node+":"+port+"/_health/ping", nil)
+ req, err := http.NewRequest("GET", url, nil)
if err != nil {
return
}
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index b66671b..048886a 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -2,6 +2,8 @@ package health
import (
"encoding/json"
+ "fmt"
+ "net"
"net/http"
"net/http/httptest"
"strings"
@@ -102,12 +104,13 @@ func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
func (s *AggregatorSuite) TestHealthy(c *check.C) {
srv, listen := s.stubServer(&healthyHandler{})
defer srv.Close()
+ _, port, _ := net.SplitHostPort(listen)
s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
Keepstore: arvados.Keepstore{Listen: listen},
}
s.handler.ServeHTTP(s.resp, s.req)
resp := s.checkOK(c)
- ep := resp.Checks["localhost/keepstore/_health/ping"]
+ ep := resp.Checks[fmt.Sprintf("keepstore+http://localhost:%d/_health/ping", port)]
c.Check(ep.Health, check.Equals, "OK")
c.Check(ep.Status, check.Equals, 200)
}
@@ -115,8 +118,10 @@ func (s *AggregatorSuite) TestHealthy(c *check.C) {
func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
srvH, listenH := s.stubServer(&healthyHandler{})
defer srvH.Close()
+ _, portH, _ := net.SplitHostPort(listenH)
srvU, listenU := s.stubServer(&unhealthyHandler{})
defer srvU.Close()
+ _, portU, _ := net.SplitHostPort(listenU)
s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
Keepstore: arvados.Keepstore{Listen: listenH},
}
@@ -125,10 +130,10 @@ func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
}
s.handler.ServeHTTP(s.resp, s.req)
resp := s.checkUnhealthy(c)
- ep := resp.Checks["localhost/keepstore/_health/ping"]
+ ep := resp.Checks[fmt.Sprintf("keepstore+http://localhost:%d/_health/ping", portH)]
c.Check(ep.Health, check.Equals, "OK")
c.Check(ep.Status, check.Equals, 200)
- ep = resp.Checks["127.0.0.1/keepstore/_health/ping"]
+ ep = resp.Checks[fmt.Sprintf("keepstore+http://127.0.0.1:%d/_health/ping", portU)]
c.Check(ep.Health, check.Equals, "ERROR")
c.Check(ep.Status, check.Equals, 200)
}
diff --git a/services/health/main.go b/services/health/main.go
index 3e089fa..b6358de 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "flag"
"net/http"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -10,10 +11,13 @@ import (
)
func main() {
+ configFile := flag.String("config", arvados.DefaultConfigFile, "`path` to arvados configuration file")
+ flag.Parse()
+
log.SetFormatter(&log.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
})
- cfg, err := arvados.GetConfig()
+ cfg, err := arvados.GetConfig(*configFile)
if err != nil {
log.Fatal(err)
}
@@ -26,11 +30,18 @@ func main() {
log.Fatal(err)
}
+ log := log.WithField("Service", "Health")
srv := &httpserver.Server{
Addr: nodeCfg.Health.Listen,
Server: http.Server{
Handler: &health.Aggregator{
Config: cfg,
+ Log: func(req *http.Request, err error) {
+ log.WithField("RemoteAddr", req.RemoteAddr).
+ WithField("Path", req.URL.Path).
+ WithError(err).
+ Info("HTTP request")
+ },
},
},
}
commit f0e6212bbe6e34f6ac4e928a236e77cca852895d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Oct 3 10:55:42 2017 -0400
12260: Fix remote ping auth. Make server work.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 537de2a..d7bae38 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -24,7 +24,9 @@ func GetConfig() (*Config, error) {
// cluster, or the default/only configured cluster if clusterID is "".
func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
if clusterID == "" {
- if len(sc.Clusters) != 1 {
+ if len(sc.Clusters) == 0 {
+ return nil, fmt.Errorf("no clusters configured")
+ } else if len(sc.Clusters) > 1 {
return nil, fmt.Errorf("multiple clusters configured, cannot choose")
} else {
for id, cc := range sc.Clusters {
@@ -79,9 +81,14 @@ func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
}
type SystemNode struct {
+ Health Health
Keepstore Keepstore
}
+type Health struct {
+ Listen string
+}
+
type Keepstore struct {
Listen string
}
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 88a338e..1ba1ee3 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -41,7 +41,7 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
agg.setupOnce.Do(agg.setup)
sendErr := func(statusCode int, err error) {
resp.WriteHeader(statusCode)
- json.NewEncoder(resp).Encode(map[string]interface{}{"error": err})
+ json.NewEncoder(resp).Encode(map[string]string{"error": err.Error()})
if agg.Log != nil {
agg.Log(req, err)
}
@@ -119,7 +119,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
wg.Add(1)
go func(node string) {
defer wg.Done()
- pingResp := agg.ping(node, addr)
+ pingResp := agg.ping(node, addr, cluster)
mtx.Lock()
defer mtx.Unlock()
@@ -148,7 +148,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
return resp
}
-func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
+func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result CheckResponse) {
t0 := time.Now()
var err error
@@ -167,6 +167,7 @@ func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
if err != nil {
return
}
+ req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
ctx, cancel := context.WithCancel(req.Context())
go func() {
@@ -184,6 +185,7 @@ func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
result.Status = resp.StatusCode
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
+ err = fmt.Errorf("cannot decode response: %s", err)
return
}
if resp.StatusCode != http.StatusOK {
diff --git a/services/health/main.go b/services/health/main.go
index 7f4d648..3e089fa 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -1,6 +1,8 @@
package main
import (
+ "net/http"
+
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -11,18 +13,27 @@ func main() {
log.SetFormatter(&log.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
})
- sysConf, err := arvados.GetSystemConfig()
+ cfg, err := arvados.GetConfig()
+ if err != nil {
+ log.Fatal(err)
+ }
+ clusterCfg, err := cfg.GetCluster("")
+ if err != nil {
+ log.Fatal(err)
+ }
+ nodeCfg, err := clusterCfg.GetThisSystemNode()
if err != nil {
log.Fatal(err)
}
srv := &httpserver.Server{
- Addr: ":", // FIXME: should be dictated by Health on this SystemNode
- Handler: &health.Aggregator{
- SystemConfig: sysConf,
+ Addr: nodeCfg.Health.Listen,
+ Server: http.Server{
+ Handler: &health.Aggregator{
+ Config: cfg,
+ },
},
}
- srv.HandleFunc()
if err := srv.Start(); err != nil {
log.Fatal(err)
}
commit 825e37afbde9b1f0cb43451170fc300f64943ada
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Oct 2 22:36:35 2017 -0400
12260: Improve data structures.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 5c46c1a..88a338e 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -72,28 +72,39 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sendErr(http.StatusNotFound, errNotFound)
return
}
- json.NewEncoder(resp).Encode(agg.checkClusterHealth(cluster))
+ json.NewEncoder(resp).Encode(agg.ClusterHealth(cluster))
if agg.Log != nil {
agg.Log(req, nil)
}
}
-type serviceHealth struct {
+type ServiceHealth struct {
Health string `json:"health"`
N int `json:"n"`
}
-type clusterHealthResponse struct {
- Health string `json:"health"`
- Endpoints map[string]map[string]interface{} `json:"endpoints"`
- Services map[string]serviceHealth `json:"services"`
+type ClusterHealthResponse struct {
+ Health string `json:"health"`
+ Checks map[string]CheckResponse `json:"checks"`
+ Services map[string]ServiceHealth `json:"services"`
}
-func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealthResponse {
- resp := clusterHealthResponse{
- Health: "OK",
- Endpoints: make(map[string]map[string]interface{}),
- Services: make(map[string]serviceHealth),
+type CheckResponse struct {
+ Status int `json:"status"`
+ Health string `json:"health"`
+ Error string `json:"error,omitempty"`
+ ResponseTime json.Number `json:"responseTime"`
+}
+
+func (r *CheckResponse) OK() bool {
+ return r.Health == "OK" && r.Status == http.StatusOK
+}
+
+func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResponse {
+ resp := ClusterHealthResponse{
+ Health: "OK",
+ Checks: make(map[string]CheckResponse),
+ Services: make(map[string]ServiceHealth),
}
mtx := sync.Mutex{}
@@ -106,21 +117,21 @@ func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealt
continue
}
wg.Add(1)
- go func() {
+ go func(node string) {
defer wg.Done()
pingResp := agg.ping(node, addr)
mtx.Lock()
defer mtx.Unlock()
- resp.Endpoints[node+"/"+svc+"/_health/ping"] = pingResp
+ resp.Checks[node+"/"+svc+"/_health/ping"] = pingResp
svHealth := resp.Services[svc]
- if agg.isOK(pingResp) {
+ if pingResp.OK() {
svHealth.N++
} else {
resp.Health = "ERROR"
}
resp.Services[svc] = svHealth
- }()
+ }(node)
}
}
wg.Wait()
@@ -137,20 +148,14 @@ func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealt
return resp
}
-func (agg *Aggregator) isOK(result map[string]interface{}) bool {
- h, ok := result["health"].(string)
- return ok && h == "OK"
-}
-
-func (agg *Aggregator) ping(node, addr string) (result map[string]interface{}) {
+func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
t0 := time.Now()
- result = make(map[string]interface{})
var err error
defer func() {
- result["responseTime"] = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
+ result.ResponseTime = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
if err != nil {
- result["health"], result["error"] = "ERROR", err
+ result.Health, result.Error = "ERROR", err.Error()
}
}()
@@ -176,11 +181,12 @@ func (agg *Aggregator) ping(node, addr string) (result map[string]interface{}) {
if err != nil {
return
}
- err = json.NewDecoder(resp.Body).Decode(result)
+ result.Status = resp.StatusCode
+ err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return
}
- if resp.StatusCode != 200 {
+ if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
return
}
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index eb6adc8..b66671b 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -57,49 +58,130 @@ func (s *AggregatorSuite) TestEmptyConfig(c *check.C) {
s.checkOK(c)
}
+func (s *AggregatorSuite) stubServer(handler http.Handler) (*httptest.Server, string) {
+ srv := httptest.NewServer(handler)
+ var port string
+ if parts := strings.Split(srv.URL, ":"); len(parts) < 3 {
+ panic(srv.URL)
+ } else {
+ port = parts[len(parts)-1]
+ }
+ return srv, ":" + port
+}
+
type unhealthyHandler struct{}
func (*unhealthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- resp.Write([]byte(`{"health":"ERROR"}`))
+ if req.URL.Path == "/_health/ping" {
+ resp.Write([]byte(`{"health":"ERROR","error":"the bends"}`))
+ } else {
+ http.Error(resp, "not found", http.StatusNotFound)
+ }
}
func (s *AggregatorSuite) TestUnhealthy(c *check.C) {
- srv := httptest.NewServer(&unhealthyHandler{})
+ srv, listen := s.stubServer(&unhealthyHandler{})
defer srv.Close()
+ s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ Keepstore: arvados.Keepstore{Listen: listen},
+ }
+ s.handler.ServeHTTP(s.resp, s.req)
+ s.checkUnhealthy(c)
+}
- var port string
- if parts := strings.Split(srv.URL, ":"); len(parts) < 3 {
- panic(srv.URL)
+type healthyHandler struct{}
+
+func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if req.URL.Path == "/_health/ping" {
+ resp.Write([]byte(`{"health":"OK"}`))
} else {
- port = parts[len(parts)-1]
+ http.Error(resp, "not found", http.StatusNotFound)
+ }
+}
+
+func (s *AggregatorSuite) TestHealthy(c *check.C) {
+ srv, listen := s.stubServer(&healthyHandler{})
+ defer srv.Close()
+ s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ Keepstore: arvados.Keepstore{Listen: listen},
}
+ s.handler.ServeHTTP(s.resp, s.req)
+ resp := s.checkOK(c)
+ ep := resp.Checks["localhost/keepstore/_health/ping"]
+ c.Check(ep.Health, check.Equals, "OK")
+ c.Check(ep.Status, check.Equals, 200)
+}
+
+func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
+ srvH, listenH := s.stubServer(&healthyHandler{})
+ defer srvH.Close()
+ srvU, listenU := s.stubServer(&unhealthyHandler{})
+ defer srvU.Close()
s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
- Keepstore: arvados.Keepstore{Listen: ":" + port},
+ Keepstore: arvados.Keepstore{Listen: listenH},
+ }
+ s.handler.Config.Clusters["zzzzz"].SystemNodes["127.0.0.1"] = arvados.SystemNode{
+ Keepstore: arvados.Keepstore{Listen: listenU},
}
s.handler.ServeHTTP(s.resp, s.req)
- s.checkUnhealthy(c)
+ resp := s.checkUnhealthy(c)
+ ep := resp.Checks["localhost/keepstore/_health/ping"]
+ c.Check(ep.Health, check.Equals, "OK")
+ c.Check(ep.Status, check.Equals, 200)
+ ep = resp.Checks["127.0.0.1/keepstore/_health/ping"]
+ c.Check(ep.Health, check.Equals, "ERROR")
+ c.Check(ep.Status, check.Equals, 200)
}
func (s *AggregatorSuite) checkError(c *check.C) {
c.Check(s.resp.Code, check.Not(check.Equals), http.StatusOK)
- var body map[string]interface{}
- err := json.NewDecoder(s.resp.Body).Decode(&body)
+ var resp ClusterHealthResponse
+ err := json.NewDecoder(s.resp.Body).Decode(&resp)
c.Check(err, check.IsNil)
- c.Check(body["health"], check.Not(check.Equals), "OK")
+ c.Check(resp.Health, check.Not(check.Equals), "OK")
+}
+
+func (s *AggregatorSuite) checkUnhealthy(c *check.C) ClusterHealthResponse {
+ return s.checkResult(c, "ERROR")
+}
+
+func (s *AggregatorSuite) checkOK(c *check.C) ClusterHealthResponse {
+ return s.checkResult(c, "OK")
}
-func (s *AggregatorSuite) checkUnhealthy(c *check.C) {
+func (s *AggregatorSuite) checkResult(c *check.C, health string) ClusterHealthResponse {
c.Check(s.resp.Code, check.Equals, http.StatusOK)
- var body map[string]interface{}
- err := json.NewDecoder(s.resp.Body).Decode(&body)
+ var resp ClusterHealthResponse
+ err := json.NewDecoder(s.resp.Body).Decode(&resp)
c.Check(err, check.IsNil)
- c.Check(body["health"], check.Equals, "ERROR")
+ c.Check(resp.Health, check.Equals, health)
+ return resp
}
-func (s *AggregatorSuite) checkOK(c *check.C) {
- c.Check(s.resp.Code, check.Equals, http.StatusOK)
- var body map[string]interface{}
- err := json.NewDecoder(s.resp.Body).Decode(&body)
+type slowHandler struct{}
+
+func (*slowHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if req.URL.Path == "/_health/ping" {
+ time.Sleep(3 * time.Second)
+ resp.Write([]byte(`{"health":"OK"}`))
+ } else {
+ http.Error(resp, "not found", http.StatusNotFound)
+ }
+}
+
+func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
+ s.handler.timeout = arvados.Duration(100 * time.Millisecond)
+ srv, listen := s.stubServer(&slowHandler{})
+ defer srv.Close()
+ s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ Keepstore: arvados.Keepstore{Listen: listen},
+ }
+ s.handler.ServeHTTP(s.resp, s.req)
+ resp := s.checkUnhealthy(c)
+ ep := resp.Checks["localhost/keepstore/_health/ping"]
+ c.Check(ep.Health, check.Equals, "ERROR")
+ c.Check(ep.Status, check.Equals, 0)
+ rt, err := ep.ResponseTime.Float64()
c.Check(err, check.IsNil)
- c.Check(body["health"], check.Equals, "OK")
+ c.Check(rt > 0.005, check.Equals, true)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list