[ARVADOS] created: c5a8ad7751e13560a6cde34395ea76f380c8a80d

Git user git at public.curoverse.com
Mon Jun 26 15:13:02 EDT 2017


        at  c5a8ad7751e13560a6cde34395ea76f380c8a80d (commit)


commit c5a8ad7751e13560a6cde34395ea76f380c8a80d
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 26 15:12:12 2017 -0400

    11901: Report db stats (notably OpenConnections) in /debug.json
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 6a308b3..daf9a94 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -263,6 +263,7 @@ func (ps *pgEventSource) DebugStatus() interface{} {
 		"QueueDelay":   stats.Duration(ps.lastQDelay),
 		"Sinks":        len(ps.sinks),
 		"SinksBlocked": blocked,
+		"DBStats":      ps.db.Stats(),
 	}
 }
 

commit 47b2e4988e9bfb69ef00de1c900a20f714af5f2f
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 26 14:53:53 2017 -0400

    11901: Require management token for health checks.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7e21da4..cdab463 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -7,6 +7,7 @@ const (
 	AdminToken              = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h"
 	AnonymousToken          = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
 	DataManagerToken        = "320mkve8qkswstz7ff61glpk3mhgghmg67wmic7elw4z41pke1"
+	ManagementToken         = "jg3ajndnq63sywcd50gbs5dskdc9ckkysb0nsqmfz08nwf17nl"
 	ActiveUserUUID          = "zzzzz-tpzed-xurymjxw79nv3jz"
 	SpectatorUserUUID       = "zzzzz-tpzed-l1s2piq4t4mps8r"
 	UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
diff --git a/services/ws/config.go b/services/ws/config.go
index 79c2f23..cf82cf8 100644
--- a/services/ws/config.go
+++ b/services/ws/config.go
@@ -17,6 +17,8 @@ type wsConfig struct {
 	PingTimeout      arvados.Duration
 	ClientEventQueue int
 	ServerEventQueue int
+
+	ManagementToken string
 }
 
 func defaultConfig() wsConfig {
diff --git a/services/ws/router.go b/services/ws/router.go
index b2c94e7..7774497 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -53,10 +53,13 @@ func (rtr *router) setup() {
 	rtr.mux = http.NewServeMux()
 	rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0))
 	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
-	rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
-	rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
-	rtr.mux.HandleFunc("/_health/ping", jsonHandler(rtr.HealthFunc(func() error { return nil })))
-	rtr.mux.HandleFunc("/_health/db", jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
+	rtr.mux.Handle("/debug.json", rtr.jsonHandler(rtr.DebugStatus))
+	rtr.mux.Handle("/status.json", rtr.jsonHandler(rtr.Status))
+
+	health := http.NewServeMux()
+	rtr.mux.Handle("/_health/", rtr.mgmtAuth(health))
+	health.Handle("/_health/ping", rtr.jsonHandler(rtr.HealthFunc(func() error { return nil })))
+	health.Handle("/_health/db", rtr.jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
 }
 
 func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -142,16 +145,30 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	rtr.mux.ServeHTTP(resp, req)
 }
 
-func jsonHandler(fn func() interface{}) http.HandlerFunc {
-	return func(resp http.ResponseWriter, req *http.Request) {
-		logger := logger(req.Context())
-		resp.Header().Set("Content-Type", "application/json")
-		enc := json.NewEncoder(resp)
+func (rtr *router) mgmtAuth(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if rtr.Config.ManagementToken == "" {
+			http.Error(w, "disabled", http.StatusNotFound)
+		} else if ah := r.Header.Get("Authorization"); ah == "" {
+			http.Error(w, "authorization required", http.StatusUnauthorized)
+		} else if ah != "Bearer "+rtr.Config.ManagementToken {
+			http.Error(w, "authorization error", http.StatusForbidden)
+		} else {
+			h.ServeHTTP(w, r)
+		}
+	})
+}
+
+func (rtr *router) jsonHandler(fn func() interface{}) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		logger := logger(r.Context())
+		w.Header().Set("Content-Type", "application/json")
+		enc := json.NewEncoder(w)
 		err := enc.Encode(fn())
 		if err != nil {
 			msg := "encode failed"
 			logger.WithError(err).Error(msg)
-			http.Error(resp, msg, http.StatusInternalServerError)
+			http.Error(w, msg, http.StatusInternalServerError)
 		}
-	}
+	})
 }
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 57c734a..da7440d 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	check "gopkg.in/check.v1"
 )
 
@@ -28,6 +29,7 @@ func (*serverSuite) testConfig() *wsConfig {
 	cfg.Client = *(arvados.NewClientFromEnv())
 	cfg.Postgres = testDBConfig()
 	cfg.Listen = ":"
+	cfg.ManagementToken = arvadostest.ManagementToken
 	return &cfg
 }
 
@@ -64,9 +66,21 @@ func (s *serverSuite) TestBadDB(c *check.C) {
 func (s *serverSuite) TestHealth(c *check.C) {
 	go s.srv.Run()
 	s.srv.WaitReady()
-	resp, err := http.Get("http://" + s.srv.listener.Addr().String() + "/_health/ping")
-	c.Check(err, check.IsNil)
-	buf, err := ioutil.ReadAll(resp.Body)
-	c.Check(err, check.IsNil)
-	c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
+	for _, token := range []string{"", "foo", s.cfg.ManagementToken} {
+		req, err := http.NewRequest("GET", "http://"+s.srv.listener.Addr().String()+"/_health/ping", nil)
+		c.Assert(err, check.IsNil)
+		if token != "" {
+			req.Header.Add("Authorization", "Bearer "+token)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		c.Check(err, check.IsNil)
+		if token == s.cfg.ManagementToken {
+			c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+			buf, err := ioutil.ReadAll(resp.Body)
+			c.Check(err, check.IsNil)
+			c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
+		} else {
+			c.Check(resp.StatusCode, check.Not(check.Equals), http.StatusOK)
+		}
+	}
 }

commit 7800f12fccea5675d71159ddf7c868f4074f8f56
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 26 10:10:26 2017 -0400

    11901: Add /_health/ping and /_health/db health checks.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/ws/event.go b/services/ws/event.go
index 304f86b..fd280ae 100644
--- a/services/ws/event.go
+++ b/services/ws/event.go
@@ -17,6 +17,7 @@ type eventSink interface {
 type eventSource interface {
 	NewSink() eventSink
 	DB() *sql.DB
+	DBHealth() error
 }
 
 type event struct {
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 7c1b584..6a308b3 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -242,6 +242,12 @@ func (ps *pgEventSource) DB() *sql.DB {
 	return ps.db
 }
 
+func (ps *pgEventSource) DBHealth() error {
+	ctx, _ := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
+	var i int
+	return ps.db.QueryRowContext(ctx, "SELECT 1").Scan(&i)
+}
+
 func (ps *pgEventSource) DebugStatus() interface{} {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
diff --git a/services/ws/event_source_test.go b/services/ws/event_source_test.go
index b157cfa..94e3ba3 100644
--- a/services/ws/event_source_test.go
+++ b/services/ws/event_source_test.go
@@ -105,4 +105,6 @@ func (*eventSourceSuite) TestEventSource(c *check.C) {
 	case <-time.After(10 * time.Second):
 		c.Fatal("timed out")
 	}
+
+	c.Check(pges.DBHealth(), check.IsNil)
 }
diff --git a/services/ws/router.go b/services/ws/router.go
index 15b825f..b2c94e7 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -55,6 +55,8 @@ func (rtr *router) setup() {
 	rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
 	rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
 	rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
+	rtr.mux.HandleFunc("/_health/ping", jsonHandler(rtr.HealthFunc(func() error { return nil })))
+	rtr.mux.HandleFunc("/_health/db", jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
 }
 
 func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -102,6 +104,21 @@ func (rtr *router) DebugStatus() interface{} {
 	return s
 }
 
+var pingResponseOK = map[string]string{"health": "OK"}
+
+func (rtr *router) HealthFunc(f func() error) func() interface{} {
+	return func() interface{} {
+		err := f()
+		if err == nil {
+			return pingResponseOK
+		}
+		return map[string]string{
+			"health": "ERROR",
+			"error":  err.Error(),
+		}
+	}
+}
+
 func (rtr *router) Status() interface{} {
 	return map[string]interface{}{
 		"Clients": atomic.LoadInt64(&rtr.status.ReqsActive),
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index d74f7df..57c734a 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -1,6 +1,8 @@
 package main
 
 import (
+	"io/ioutil"
+	"net/http"
 	"sync"
 	"time"
 
@@ -11,9 +13,17 @@ import (
 var _ = check.Suite(&serverSuite{})
 
 type serverSuite struct {
+	cfg *wsConfig
+	srv *server
+	wg  sync.WaitGroup
 }
 
-func testConfig() *wsConfig {
+func (s *serverSuite) SetUpTest(c *check.C) {
+	s.cfg = s.testConfig()
+	s.srv = &server{wsConfig: s.cfg}
+}
+
+func (*serverSuite) testConfig() *wsConfig {
 	cfg := defaultConfig()
 	cfg.Client = *(arvados.NewClientFromEnv())
 	cfg.Postgres = testDBConfig()
@@ -24,20 +34,18 @@ func testConfig() *wsConfig {
 // TestBadDB ensures Run() returns an error (instead of panicking or
 // deadlocking) if it can't connect to the database server at startup.
 func (s *serverSuite) TestBadDB(c *check.C) {
-	cfg := testConfig()
-	cfg.Postgres["password"] = "1234"
-	srv := &server{wsConfig: cfg}
+	s.cfg.Postgres["password"] = "1234"
 
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		err := srv.Run()
+		err := s.srv.Run()
 		c.Check(err, check.NotNil)
 		wg.Done()
 	}()
 	wg.Add(1)
 	go func() {
-		srv.WaitReady()
+		s.srv.WaitReady()
 		wg.Done()
 	}()
 
@@ -53,9 +61,12 @@ func (s *serverSuite) TestBadDB(c *check.C) {
 	}
 }
 
-func newTestServer() *server {
-	srv := &server{wsConfig: testConfig()}
-	go srv.Run()
-	srv.WaitReady()
-	return srv
+func (s *serverSuite) TestHealth(c *check.C) {
+	go s.srv.Run()
+	s.srv.WaitReady()
+	resp, err := http.Get("http://" + s.srv.listener.Addr().String() + "/_health/ping")
+	c.Check(err, check.IsNil)
+	buf, err := ioutil.ReadAll(resp.Body)
+	c.Check(err, check.IsNil)
+	c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
 }
diff --git a/services/ws/session_v0_test.go b/services/ws/session_v0_test.go
index 85e3656..f6fe3f6 100644
--- a/services/ws/session_v0_test.go
+++ b/services/ws/session_v0_test.go
@@ -25,11 +25,13 @@ func init() {
 var _ = check.Suite(&v0Suite{})
 
 type v0Suite struct {
-	token    string
-	toDelete []string
+	serverSuite serverSuite
+	token       string
+	toDelete    []string
 }
 
 func (s *v0Suite) SetUpTest(c *check.C) {
+	s.serverSuite.SetUpTest(c)
 	s.token = arvadostest.ActiveToken
 }
 
@@ -227,7 +229,9 @@ func (s *v0Suite) expectLog(c *check.C, r *json.Decoder) *arvados.Log {
 }
 
 func (s *v0Suite) testClient() (*server, *websocket.Conn, *json.Decoder, *json.Encoder) {
-	srv := newTestServer()
+	go s.serverSuite.srv.Run()
+	s.serverSuite.srv.WaitReady()
+	srv := s.serverSuite.srv
 	conn, err := websocket.Dial("ws://"+srv.listener.Addr().String()+"/websocket?api_token="+s.token, "", "http://"+srv.listener.Addr().String())
 	if err != nil {
 		panic(err)

commit d25db7c02aa07e9d4812a029753c2b8606cf35b1
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 26 09:47:36 2017 -0400

    11901: Fix unclosed db.Rows object.
    
    rows is closed implicitly when rows.Next() returns false, but when the
    client hangs up, sendOldEvents returns without retrieving all
    results. In this case rows.Close() needs to be called explicitly.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>

diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index 44e2a1d..f8645eb 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -157,6 +157,7 @@ func (sub *v0subscribe) sendOldEvents(sess *v0session) {
 		sess.log.WithError(err).Error("db.Query failed")
 		return
 	}
+	defer rows.Close()
 	for rows.Next() {
 		var id uint64
 		err := rows.Scan(&id)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list