[ARVADOS] updated: 9959bf0f5631daa84e8afa7de145154390259c67

Git user git at public.curoverse.com
Tue Oct 3 15:18:24 EDT 2017


Summary of changes:
 sdk/go/arvados/config.go         |  28 +++++----
 sdk/go/health/aggregator.go      |  89 +++++++++++++++------------
 sdk/go/health/aggregator_test.go | 127 +++++++++++++++++++++++++++++++++------
 services/health/main.go          |  32 ++++++++--
 4 files changed, 198 insertions(+), 78 deletions(-)

       via  9959bf0f5631daa84e8afa7de145154390259c67 (commit)
       via  f0e6212bbe6e34f6ac4e928a236e77cca852895d (commit)
       via  825e37afbde9b1f0cb43451170fc300f64943ada (commit)
      from  63218507eec156df91f80c86ee05d680d67b8336 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 9959bf0f5631daa84e8afa7de145154390259c67
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Tue Oct 3 15:15:20 2017 -0400

    12260: Add -config arg. Drop non-resolvable hostname support.
    
    Change check key to svctype+http://host:port/path.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index d7bae38..84e66b3 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -3,20 +3,21 @@ package arvados
 import (
 	"fmt"
 	"os"
-	"strings"
 
 	"git.curoverse.com/arvados.git/sdk/go/config"
 )
 
+const DefaultConfigFile = "/etc/arvados/config.yml"
+
 type Config struct {
 	Clusters map[string]Cluster
 }
 
 // GetConfig returns the current system config, loading it from
-// /etc if needed.
-func GetConfig() (*Config, error) {
+// configFile if needed.
+func GetConfig(configFile string) (*Config, error) {
 	var cfg Config
-	err := config.LoadFile(&cfg, "/etc/arvados/config.yml")
+	err := config.LoadFile(&cfg, configFile)
 	return &cfg, err
 }
 
@@ -63,14 +64,8 @@ func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) {
 // error is returned if the appropriate configuration can't be
 // determined (e.g., this does not appear to be a system node).
 func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
-	// Generally node is "a.b.ca", use the first of {"a.b.ca",
-	// "a.b", "a"} that has an entry in SystemNodes.
-	labels := strings.Split(node, ".")
-	for j := len(labels); j > 0; j-- {
-		hostpart := strings.Join(labels[:j], ".")
-		if cfg, ok := cc.SystemNodes[hostpart]; ok {
-			return &cfg, nil
-		}
+	if cfg, ok := cc.SystemNodes[node]; ok {
+		return &cfg, nil
 	}
 	// If node is not listed, but "*" gives a default system node
 	// config, use the default config.
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 1ba1ee3..e881db8 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -49,15 +49,6 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 
 	resp.Header().Set("Content-Type", "application/json")
 
-	if agg.Config == nil {
-		cfg, err := arvados.GetConfig()
-		if err != nil {
-			err = fmt.Errorf("arvados.GetConfig(): %s", err)
-			sendErr(http.StatusInternalServerError, err)
-			return
-		}
-		agg.Config = cfg
-	}
 	cluster, err := agg.Config.GetCluster("")
 	if err != nil {
 		err = fmt.Errorf("arvados.GetCluster(): %s", err)
@@ -119,11 +110,20 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 			wg.Add(1)
 			go func(node string) {
 				defer wg.Done()
-				pingResp := agg.ping(node, addr, cluster)
+				var pingResp CheckResponse
+				url, err := agg.pingURL(node, addr)
+				if err != nil {
+					pingResp = CheckResponse{
+						Health: "ERROR",
+						Error:  err.Error(),
+					}
+				} else {
+					pingResp = agg.ping(url, cluster)
+				}
 
 				mtx.Lock()
 				defer mtx.Unlock()
-				resp.Checks[node+"/"+svc+"/_health/ping"] = pingResp
+				resp.Checks[svc+"+"+url] = pingResp
 				svHealth := resp.Services[svc]
 				if pingResp.OK() {
 					svHealth.N++
@@ -148,7 +148,12 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 	return resp
 }
 
-func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result CheckResponse) {
+func (agg *Aggregator) pingURL(node, addr string) (string, error) {
+	_, port, err := net.SplitHostPort(addr)
+	return "http://" + node + ":" + port + "/_health/ping", err
+}
+
+func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckResponse) {
 	t0 := time.Now()
 
 	var err error
@@ -159,11 +164,7 @@ func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result
 		}
 	}()
 
-	_, port, err := net.SplitHostPort(addr)
-	if err != nil {
-		return
-	}
-	req, err := http.NewRequest("GET", "http://"+node+":"+port+"/_health/ping", nil)
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return
 	}
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index b66671b..048886a 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -2,6 +2,8 @@ package health
 
 import (
 	"encoding/json"
+	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -102,12 +104,13 @@ func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 func (s *AggregatorSuite) TestHealthy(c *check.C) {
 	srv, listen := s.stubServer(&healthyHandler{})
 	defer srv.Close()
+	_, port, _ := net.SplitHostPort(listen)
 	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
 		Keepstore: arvados.Keepstore{Listen: listen},
 	}
 	s.handler.ServeHTTP(s.resp, s.req)
 	resp := s.checkOK(c)
-	ep := resp.Checks["localhost/keepstore/_health/ping"]
+	ep := resp.Checks[fmt.Sprintf("keepstore+http://localhost:%d/_health/ping", port)]
 	c.Check(ep.Health, check.Equals, "OK")
 	c.Check(ep.Status, check.Equals, 200)
 }
@@ -115,8 +118,10 @@ func (s *AggregatorSuite) TestHealthy(c *check.C) {
 func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
 	srvH, listenH := s.stubServer(&healthyHandler{})
 	defer srvH.Close()
+	_, portH, _ := net.SplitHostPort(listenH)
 	srvU, listenU := s.stubServer(&unhealthyHandler{})
 	defer srvU.Close()
+	_, portU, _ := net.SplitHostPort(listenU)
 	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
 		Keepstore: arvados.Keepstore{Listen: listenH},
 	}
@@ -125,10 +130,10 @@ func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
 	}
 	s.handler.ServeHTTP(s.resp, s.req)
 	resp := s.checkUnhealthy(c)
-	ep := resp.Checks["localhost/keepstore/_health/ping"]
+	ep := resp.Checks[fmt.Sprintf("keepstore+http://localhost:%d/_health/ping", portH)]
 	c.Check(ep.Health, check.Equals, "OK")
 	c.Check(ep.Status, check.Equals, 200)
-	ep = resp.Checks["127.0.0.1/keepstore/_health/ping"]
+	ep = resp.Checks[fmt.Sprintf("keepstore+http://127.0.0.1:%d/_health/ping", portU)]
 	c.Check(ep.Health, check.Equals, "ERROR")
 	c.Check(ep.Status, check.Equals, 200)
 }
diff --git a/services/health/main.go b/services/health/main.go
index 3e089fa..b6358de 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"net/http"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -10,10 +11,13 @@ import (
 )
 
 func main() {
+	configFile := flag.String("config", arvados.DefaultConfigFile, "`path` to arvados configuration file")
+	flag.Parse()
+
 	log.SetFormatter(&log.JSONFormatter{
 		TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
 	})
-	cfg, err := arvados.GetConfig()
+	cfg, err := arvados.GetConfig(*configFile)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -26,11 +30,18 @@ func main() {
 		log.Fatal(err)
 	}
 
+	log := log.WithField("Service", "Health")
 	srv := &httpserver.Server{
 		Addr: nodeCfg.Health.Listen,
 		Server: http.Server{
 			Handler: &health.Aggregator{
 				Config: cfg,
+				Log: func(req *http.Request, err error) {
+					log.WithField("RemoteAddr", req.RemoteAddr).
+						WithField("Path", req.URL.Path).
+						WithError(err).
+						Info("HTTP request")
+				},
 			},
 		},
 	}

commit f0e6212bbe6e34f6ac4e928a236e77cca852895d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Tue Oct 3 10:55:42 2017 -0400

    12260: Fix remote ping auth. Make server work.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 537de2a..d7bae38 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -24,7 +24,9 @@ func GetConfig() (*Config, error) {
 // cluster, or the default/only configured cluster if clusterID is "".
 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
 	if clusterID == "" {
-		if len(sc.Clusters) != 1 {
+		if len(sc.Clusters) == 0 {
+			return nil, fmt.Errorf("no clusters configured")
+		} else if len(sc.Clusters) > 1 {
 			return nil, fmt.Errorf("multiple clusters configured, cannot choose")
 		} else {
 			for id, cc := range sc.Clusters {
@@ -79,9 +81,14 @@ func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
 }
 
 type SystemNode struct {
+	Health    Health
 	Keepstore Keepstore
 }
 
+type Health struct {
+	Listen string
+}
+
 type Keepstore struct {
 	Listen string
 }
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 88a338e..1ba1ee3 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -41,7 +41,7 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	agg.setupOnce.Do(agg.setup)
 	sendErr := func(statusCode int, err error) {
 		resp.WriteHeader(statusCode)
-		json.NewEncoder(resp).Encode(map[string]interface{}{"error": err})
+		json.NewEncoder(resp).Encode(map[string]string{"error": err.Error()})
 		if agg.Log != nil {
 			agg.Log(req, err)
 		}
@@ -119,7 +119,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 			wg.Add(1)
 			go func(node string) {
 				defer wg.Done()
-				pingResp := agg.ping(node, addr)
+				pingResp := agg.ping(node, addr, cluster)
 
 				mtx.Lock()
 				defer mtx.Unlock()
@@ -148,7 +148,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 	return resp
 }
 
-func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
+func (agg *Aggregator) ping(node, addr string, cluster *arvados.Cluster) (result CheckResponse) {
 	t0 := time.Now()
 
 	var err error
@@ -167,6 +167,7 @@ func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
 	if err != nil {
 		return
 	}
+	req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
 
 	ctx, cancel := context.WithCancel(req.Context())
 	go func() {
@@ -184,6 +185,7 @@ func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
 	result.Status = resp.StatusCode
 	err = json.NewDecoder(resp.Body).Decode(&result)
 	if err != nil {
+		err = fmt.Errorf("cannot decode response: %s", err)
 		return
 	}
 	if resp.StatusCode != http.StatusOK {
diff --git a/services/health/main.go b/services/health/main.go
index 7f4d648..3e089fa 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -1,6 +1,8 @@
 package main
 
 import (
+	"net/http"
+
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/health"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -11,18 +13,27 @@ func main() {
 	log.SetFormatter(&log.JSONFormatter{
 		TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
 	})
-	sysConf, err := arvados.GetSystemConfig()
+	cfg, err := arvados.GetConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+	clusterCfg, err := cfg.GetCluster("")
+	if err != nil {
+		log.Fatal(err)
+	}
+	nodeCfg, err := clusterCfg.GetThisSystemNode()
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	srv := &httpserver.Server{
-		Addr: ":", // FIXME: should be dictated by Health on this SystemNode
-		Handler: &health.Aggregator{
-			SystemConfig: sysConf,
+		Addr: nodeCfg.Health.Listen,
+		Server: http.Server{
+			Handler: &health.Aggregator{
+				Config: cfg,
+			},
 		},
 	}
-	srv.HandleFunc()
 	if err := srv.Start(); err != nil {
 		log.Fatal(err)
 	}

commit 825e37afbde9b1f0cb43451170fc300f64943ada
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Oct 2 22:36:35 2017 -0400

    12260: Improve data structures.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 5c46c1a..88a338e 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -72,28 +72,39 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 		sendErr(http.StatusNotFound, errNotFound)
 		return
 	}
-	json.NewEncoder(resp).Encode(agg.checkClusterHealth(cluster))
+	json.NewEncoder(resp).Encode(agg.ClusterHealth(cluster))
 	if agg.Log != nil {
 		agg.Log(req, nil)
 	}
 }
 
-type serviceHealth struct {
+type ServiceHealth struct {
 	Health string `json:"health"`
 	N      int    `json:"n"`
 }
 
-type clusterHealthResponse struct {
-	Health    string                            `json:"health"`
-	Endpoints map[string]map[string]interface{} `json:"endpoints"`
-	Services  map[string]serviceHealth          `json:"services"`
+type ClusterHealthResponse struct {
+	Health   string                   `json:"health"`
+	Checks   map[string]CheckResponse `json:"checks"`
+	Services map[string]ServiceHealth `json:"services"`
 }
 
-func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealthResponse {
-	resp := clusterHealthResponse{
-		Health:    "OK",
-		Endpoints: make(map[string]map[string]interface{}),
-		Services:  make(map[string]serviceHealth),
+type CheckResponse struct {
+	Status       int         `json:"status"`
+	Health       string      `json:"health"`
+	Error        string      `json:"error,omitempty"`
+	ResponseTime json.Number `json:"responseTime"`
+}
+
+func (r *CheckResponse) OK() bool {
+	return r.Health == "OK" && r.Status == http.StatusOK
+}
+
+func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResponse {
+	resp := ClusterHealthResponse{
+		Health:   "OK",
+		Checks:   make(map[string]CheckResponse),
+		Services: make(map[string]ServiceHealth),
 	}
 
 	mtx := sync.Mutex{}
@@ -106,21 +117,21 @@ func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealt
 				continue
 			}
 			wg.Add(1)
-			go func() {
+			go func(node string) {
 				defer wg.Done()
 				pingResp := agg.ping(node, addr)
 
 				mtx.Lock()
 				defer mtx.Unlock()
-				resp.Endpoints[node+"/"+svc+"/_health/ping"] = pingResp
+				resp.Checks[node+"/"+svc+"/_health/ping"] = pingResp
 				svHealth := resp.Services[svc]
-				if agg.isOK(pingResp) {
+				if pingResp.OK() {
 					svHealth.N++
 				} else {
 					resp.Health = "ERROR"
 				}
 				resp.Services[svc] = svHealth
-			}()
+			}(node)
 		}
 	}
 	wg.Wait()
@@ -137,20 +148,14 @@ func (agg *Aggregator) checkClusterHealth(cluster *arvados.Cluster) clusterHealt
 	return resp
 }
 
-func (agg *Aggregator) isOK(result map[string]interface{}) bool {
-	h, ok := result["health"].(string)
-	return ok && h == "OK"
-}
-
-func (agg *Aggregator) ping(node, addr string) (result map[string]interface{}) {
+func (agg *Aggregator) ping(node, addr string) (result CheckResponse) {
 	t0 := time.Now()
-	result = make(map[string]interface{})
 
 	var err error
 	defer func() {
-		result["responseTime"] = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
+		result.ResponseTime = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
 		if err != nil {
-			result["health"], result["error"] = "ERROR", err
+			result.Health, result.Error = "ERROR", err.Error()
 		}
 	}()
 
@@ -176,11 +181,12 @@ func (agg *Aggregator) ping(node, addr string) (result map[string]interface{}) {
 	if err != nil {
 		return
 	}
-	err = json.NewDecoder(resp.Body).Decode(result)
+	result.Status = resp.StatusCode
+	err = json.NewDecoder(resp.Body).Decode(&result)
 	if err != nil {
 		return
 	}
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
 		return
 	}
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index eb6adc8..b66671b 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -57,49 +58,130 @@ func (s *AggregatorSuite) TestEmptyConfig(c *check.C) {
 	s.checkOK(c)
 }
 
+func (s *AggregatorSuite) stubServer(handler http.Handler) (*httptest.Server, string) {
+	srv := httptest.NewServer(handler)
+	var port string
+	if parts := strings.Split(srv.URL, ":"); len(parts) < 3 {
+		panic(srv.URL)
+	} else {
+		port = parts[len(parts)-1]
+	}
+	return srv, ":" + port
+}
+
 type unhealthyHandler struct{}
 
 func (*unhealthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	resp.Write([]byte(`{"health":"ERROR"}`))
+	if req.URL.Path == "/_health/ping" {
+		resp.Write([]byte(`{"health":"ERROR","error":"the bends"}`))
+	} else {
+		http.Error(resp, "not found", http.StatusNotFound)
+	}
 }
 
 func (s *AggregatorSuite) TestUnhealthy(c *check.C) {
-	srv := httptest.NewServer(&unhealthyHandler{})
+	srv, listen := s.stubServer(&unhealthyHandler{})
 	defer srv.Close()
+	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+		Keepstore: arvados.Keepstore{Listen: listen},
+	}
+	s.handler.ServeHTTP(s.resp, s.req)
+	s.checkUnhealthy(c)
+}
 
-	var port string
-	if parts := strings.Split(srv.URL, ":"); len(parts) < 3 {
-		panic(srv.URL)
+type healthyHandler struct{}
+
+func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	if req.URL.Path == "/_health/ping" {
+		resp.Write([]byte(`{"health":"OK"}`))
 	} else {
-		port = parts[len(parts)-1]
+		http.Error(resp, "not found", http.StatusNotFound)
+	}
+}
+
+func (s *AggregatorSuite) TestHealthy(c *check.C) {
+	srv, listen := s.stubServer(&healthyHandler{})
+	defer srv.Close()
+	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+		Keepstore: arvados.Keepstore{Listen: listen},
 	}
+	s.handler.ServeHTTP(s.resp, s.req)
+	resp := s.checkOK(c)
+	ep := resp.Checks["localhost/keepstore/_health/ping"]
+	c.Check(ep.Health, check.Equals, "OK")
+	c.Check(ep.Status, check.Equals, 200)
+}
+
+func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
+	srvH, listenH := s.stubServer(&healthyHandler{})
+	defer srvH.Close()
+	srvU, listenU := s.stubServer(&unhealthyHandler{})
+	defer srvU.Close()
 	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
-		Keepstore: arvados.Keepstore{Listen: ":" + port},
+		Keepstore: arvados.Keepstore{Listen: listenH},
+	}
+	s.handler.Config.Clusters["zzzzz"].SystemNodes["127.0.0.1"] = arvados.SystemNode{
+		Keepstore: arvados.Keepstore{Listen: listenU},
 	}
 	s.handler.ServeHTTP(s.resp, s.req)
-	s.checkUnhealthy(c)
+	resp := s.checkUnhealthy(c)
+	ep := resp.Checks["localhost/keepstore/_health/ping"]
+	c.Check(ep.Health, check.Equals, "OK")
+	c.Check(ep.Status, check.Equals, 200)
+	ep = resp.Checks["127.0.0.1/keepstore/_health/ping"]
+	c.Check(ep.Health, check.Equals, "ERROR")
+	c.Check(ep.Status, check.Equals, 200)
 }
 
 func (s *AggregatorSuite) checkError(c *check.C) {
 	c.Check(s.resp.Code, check.Not(check.Equals), http.StatusOK)
-	var body map[string]interface{}
-	err := json.NewDecoder(s.resp.Body).Decode(&body)
+	var resp ClusterHealthResponse
+	err := json.NewDecoder(s.resp.Body).Decode(&resp)
 	c.Check(err, check.IsNil)
-	c.Check(body["health"], check.Not(check.Equals), "OK")
+	c.Check(resp.Health, check.Not(check.Equals), "OK")
+}
+
+func (s *AggregatorSuite) checkUnhealthy(c *check.C) ClusterHealthResponse {
+	return s.checkResult(c, "ERROR")
+}
+
+func (s *AggregatorSuite) checkOK(c *check.C) ClusterHealthResponse {
+	return s.checkResult(c, "OK")
 }
 
-func (s *AggregatorSuite) checkUnhealthy(c *check.C) {
+func (s *AggregatorSuite) checkResult(c *check.C, health string) ClusterHealthResponse {
 	c.Check(s.resp.Code, check.Equals, http.StatusOK)
-	var body map[string]interface{}
-	err := json.NewDecoder(s.resp.Body).Decode(&body)
+	var resp ClusterHealthResponse
+	err := json.NewDecoder(s.resp.Body).Decode(&resp)
 	c.Check(err, check.IsNil)
-	c.Check(body["health"], check.Equals, "ERROR")
+	c.Check(resp.Health, check.Equals, health)
+	return resp
 }
 
-func (s *AggregatorSuite) checkOK(c *check.C) {
-	c.Check(s.resp.Code, check.Equals, http.StatusOK)
-	var body map[string]interface{}
-	err := json.NewDecoder(s.resp.Body).Decode(&body)
+type slowHandler struct{}
+
+func (*slowHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	if req.URL.Path == "/_health/ping" {
+		time.Sleep(3 * time.Second)
+		resp.Write([]byte(`{"health":"OK"}`))
+	} else {
+		http.Error(resp, "not found", http.StatusNotFound)
+	}
+}
+
+func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
+	s.handler.timeout = arvados.Duration(100 * time.Millisecond)
+	srv, listen := s.stubServer(&slowHandler{})
+	defer srv.Close()
+	s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+		Keepstore: arvados.Keepstore{Listen: listen},
+	}
+	s.handler.ServeHTTP(s.resp, s.req)
+	resp := s.checkUnhealthy(c)
+	ep := resp.Checks["localhost/keepstore/_health/ping"]
+	c.Check(ep.Health, check.Equals, "ERROR")
+	c.Check(ep.Status, check.Equals, 0)
+	rt, err := ep.ResponseTime.Float64()
 	c.Check(err, check.IsNil)
-	c.Check(body["health"], check.Equals, "OK")
+	c.Check(rt > 0.005, check.Equals, true)
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list