[ARVADOS] created: c5a8ad7751e13560a6cde34395ea76f380c8a80d
Git user
git at public.curoverse.com
Mon Jun 26 15:13:02 EDT 2017
at c5a8ad7751e13560a6cde34395ea76f380c8a80d (commit)
commit c5a8ad7751e13560a6cde34395ea76f380c8a80d
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 26 15:12:12 2017 -0400
11901: Report db stats (notably OpenConnections) in /debug.json
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 6a308b3..daf9a94 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -263,6 +263,7 @@ func (ps *pgEventSource) DebugStatus() interface{} {
"QueueDelay": stats.Duration(ps.lastQDelay),
"Sinks": len(ps.sinks),
"SinksBlocked": blocked,
+ "DBStats": ps.db.Stats(),
}
}
commit 47b2e4988e9bfb69ef00de1c900a20f714af5f2f
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 26 14:53:53 2017 -0400
11901: Require management token for health checks.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7e21da4..cdab463 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -7,6 +7,7 @@ const (
AdminToken = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h"
AnonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
DataManagerToken = "320mkve8qkswstz7ff61glpk3mhgghmg67wmic7elw4z41pke1"
+ ManagementToken = "jg3ajndnq63sywcd50gbs5dskdc9ckkysb0nsqmfz08nwf17nl"
ActiveUserUUID = "zzzzz-tpzed-xurymjxw79nv3jz"
SpectatorUserUUID = "zzzzz-tpzed-l1s2piq4t4mps8r"
UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
diff --git a/services/ws/config.go b/services/ws/config.go
index 79c2f23..cf82cf8 100644
--- a/services/ws/config.go
+++ b/services/ws/config.go
@@ -17,6 +17,8 @@ type wsConfig struct {
PingTimeout arvados.Duration
ClientEventQueue int
ServerEventQueue int
+
+ ManagementToken string
}
func defaultConfig() wsConfig {
diff --git a/services/ws/router.go b/services/ws/router.go
index b2c94e7..7774497 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -53,10 +53,13 @@ func (rtr *router) setup() {
rtr.mux = http.NewServeMux()
rtr.mux.Handle("/websocket", rtr.makeServer(newSessionV0))
rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
- rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
- rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
- rtr.mux.HandleFunc("/_health/ping", jsonHandler(rtr.HealthFunc(func() error { return nil })))
- rtr.mux.HandleFunc("/_health/db", jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
+ rtr.mux.Handle("/debug.json", rtr.jsonHandler(rtr.DebugStatus))
+ rtr.mux.Handle("/status.json", rtr.jsonHandler(rtr.Status))
+
+ health := http.NewServeMux()
+ rtr.mux.Handle("/_health/", rtr.mgmtAuth(health))
+ health.Handle("/_health/ping", rtr.jsonHandler(rtr.HealthFunc(func() error { return nil })))
+ health.Handle("/_health/db", rtr.jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
}
func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -142,16 +145,30 @@ func (rtr *router) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
rtr.mux.ServeHTTP(resp, req)
}
-func jsonHandler(fn func() interface{}) http.HandlerFunc {
- return func(resp http.ResponseWriter, req *http.Request) {
- logger := logger(req.Context())
- resp.Header().Set("Content-Type", "application/json")
- enc := json.NewEncoder(resp)
+func (rtr *router) mgmtAuth(h http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if rtr.Config.ManagementToken == "" {
+ http.Error(w, "disabled", http.StatusNotFound)
+ } else if ah := r.Header.Get("Authorization"); ah == "" {
+ http.Error(w, "authorization required", http.StatusUnauthorized)
+ } else if ah != "Bearer "+rtr.Config.ManagementToken {
+ http.Error(w, "authorization error", http.StatusForbidden)
+ } else {
+ h.ServeHTTP(w, r)
+ }
+ })
+}
+
+func (rtr *router) jsonHandler(fn func() interface{}) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ logger := logger(r.Context())
+ w.Header().Set("Content-Type", "application/json")
+ enc := json.NewEncoder(w)
err := enc.Encode(fn())
if err != nil {
msg := "encode failed"
logger.WithError(err).Error(msg)
- http.Error(resp, msg, http.StatusInternalServerError)
+ http.Error(w, msg, http.StatusInternalServerError)
}
- }
+ })
}
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 57c734a..da7440d 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -7,6 +7,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -28,6 +29,7 @@ func (*serverSuite) testConfig() *wsConfig {
cfg.Client = *(arvados.NewClientFromEnv())
cfg.Postgres = testDBConfig()
cfg.Listen = ":"
+ cfg.ManagementToken = arvadostest.ManagementToken
return &cfg
}
@@ -64,9 +66,21 @@ func (s *serverSuite) TestBadDB(c *check.C) {
func (s *serverSuite) TestHealth(c *check.C) {
go s.srv.Run()
s.srv.WaitReady()
- resp, err := http.Get("http://" + s.srv.listener.Addr().String() + "/_health/ping")
- c.Check(err, check.IsNil)
- buf, err := ioutil.ReadAll(resp.Body)
- c.Check(err, check.IsNil)
- c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
+ for _, token := range []string{"", "foo", s.cfg.ManagementToken} {
+ req, err := http.NewRequest("GET", "http://"+s.srv.listener.Addr().String()+"/_health/ping", nil)
+ c.Assert(err, check.IsNil)
+ if token != "" {
+ req.Header.Add("Authorization", "Bearer "+token)
+ }
+ resp, err := http.DefaultClient.Do(req)
+ c.Check(err, check.IsNil)
+ if token == s.cfg.ManagementToken {
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
+ } else {
+ c.Check(resp.StatusCode, check.Not(check.Equals), http.StatusOK)
+ }
+ }
}
commit 7800f12fccea5675d71159ddf7c868f4074f8f56
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 26 10:10:26 2017 -0400
11901: Add /_health/ping and /_health/db health checks.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/ws/event.go b/services/ws/event.go
index 304f86b..fd280ae 100644
--- a/services/ws/event.go
+++ b/services/ws/event.go
@@ -17,6 +17,7 @@ type eventSink interface {
type eventSource interface {
NewSink() eventSink
DB() *sql.DB
+ DBHealth() error
}
type event struct {
diff --git a/services/ws/event_source.go b/services/ws/event_source.go
index 7c1b584..6a308b3 100644
--- a/services/ws/event_source.go
+++ b/services/ws/event_source.go
@@ -242,6 +242,12 @@ func (ps *pgEventSource) DB() *sql.DB {
return ps.db
}
+func (ps *pgEventSource) DBHealth() error {
+ ctx, _ := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
+ var i int
+ return ps.db.QueryRowContext(ctx, "SELECT 1").Scan(&i)
+}
+
func (ps *pgEventSource) DebugStatus() interface{} {
ps.mtx.Lock()
defer ps.mtx.Unlock()
diff --git a/services/ws/event_source_test.go b/services/ws/event_source_test.go
index b157cfa..94e3ba3 100644
--- a/services/ws/event_source_test.go
+++ b/services/ws/event_source_test.go
@@ -105,4 +105,6 @@ func (*eventSourceSuite) TestEventSource(c *check.C) {
case <-time.After(10 * time.Second):
c.Fatal("timed out")
}
+
+ c.Check(pges.DBHealth(), check.IsNil)
}
diff --git a/services/ws/router.go b/services/ws/router.go
index 15b825f..b2c94e7 100644
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -55,6 +55,8 @@ func (rtr *router) setup() {
rtr.mux.Handle("/arvados/v1/events.ws", rtr.makeServer(newSessionV1))
rtr.mux.HandleFunc("/debug.json", jsonHandler(rtr.DebugStatus))
rtr.mux.HandleFunc("/status.json", jsonHandler(rtr.Status))
+ rtr.mux.HandleFunc("/_health/ping", jsonHandler(rtr.HealthFunc(func() error { return nil })))
+ rtr.mux.HandleFunc("/_health/db", jsonHandler(rtr.HealthFunc(rtr.eventSource.DBHealth)))
}
func (rtr *router) makeServer(newSession sessionFactory) *websocket.Server {
@@ -102,6 +104,21 @@ func (rtr *router) DebugStatus() interface{} {
return s
}
+var pingResponseOK = map[string]string{"health": "OK"}
+
+func (rtr *router) HealthFunc(f func() error) func() interface{} {
+ return func() interface{} {
+ err := f()
+ if err == nil {
+ return pingResponseOK
+ }
+ return map[string]string{
+ "health": "ERROR",
+ "error": err.Error(),
+ }
+ }
+}
+
func (rtr *router) Status() interface{} {
return map[string]interface{}{
"Clients": atomic.LoadInt64(&rtr.status.ReqsActive),
diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index d74f7df..57c734a 100644
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -1,6 +1,8 @@
package main
import (
+ "io/ioutil"
+ "net/http"
"sync"
"time"
@@ -11,9 +13,17 @@ import (
var _ = check.Suite(&serverSuite{})
type serverSuite struct {
+ cfg *wsConfig
+ srv *server
+ wg sync.WaitGroup
}
-func testConfig() *wsConfig {
+func (s *serverSuite) SetUpTest(c *check.C) {
+ s.cfg = s.testConfig()
+ s.srv = &server{wsConfig: s.cfg}
+}
+
+func (*serverSuite) testConfig() *wsConfig {
cfg := defaultConfig()
cfg.Client = *(arvados.NewClientFromEnv())
cfg.Postgres = testDBConfig()
@@ -24,20 +34,18 @@ func testConfig() *wsConfig {
// TestBadDB ensures Run() returns an error (instead of panicking or
// deadlocking) if it can't connect to the database server at startup.
func (s *serverSuite) TestBadDB(c *check.C) {
- cfg := testConfig()
- cfg.Postgres["password"] = "1234"
- srv := &server{wsConfig: cfg}
+ s.cfg.Postgres["password"] = "1234"
var wg sync.WaitGroup
wg.Add(1)
go func() {
- err := srv.Run()
+ err := s.srv.Run()
c.Check(err, check.NotNil)
wg.Done()
}()
wg.Add(1)
go func() {
- srv.WaitReady()
+ s.srv.WaitReady()
wg.Done()
}()
@@ -53,9 +61,12 @@ func (s *serverSuite) TestBadDB(c *check.C) {
}
}
-func newTestServer() *server {
- srv := &server{wsConfig: testConfig()}
- go srv.Run()
- srv.WaitReady()
- return srv
+func (s *serverSuite) TestHealth(c *check.C) {
+ go s.srv.Run()
+ s.srv.WaitReady()
+ resp, err := http.Get("http://" + s.srv.listener.Addr().String() + "/_health/ping")
+ c.Check(err, check.IsNil)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, `{"health":"OK"}`+"\n")
}
diff --git a/services/ws/session_v0_test.go b/services/ws/session_v0_test.go
index 85e3656..f6fe3f6 100644
--- a/services/ws/session_v0_test.go
+++ b/services/ws/session_v0_test.go
@@ -25,11 +25,13 @@ func init() {
var _ = check.Suite(&v0Suite{})
type v0Suite struct {
- token string
- toDelete []string
+ serverSuite serverSuite
+ token string
+ toDelete []string
}
func (s *v0Suite) SetUpTest(c *check.C) {
+ s.serverSuite.SetUpTest(c)
s.token = arvadostest.ActiveToken
}
@@ -227,7 +229,9 @@ func (s *v0Suite) expectLog(c *check.C, r *json.Decoder) *arvados.Log {
}
func (s *v0Suite) testClient() (*server, *websocket.Conn, *json.Decoder, *json.Encoder) {
- srv := newTestServer()
+ go s.serverSuite.srv.Run()
+ s.serverSuite.srv.WaitReady()
+ srv := s.serverSuite.srv
conn, err := websocket.Dial("ws://"+srv.listener.Addr().String()+"/websocket?api_token="+s.token, "", "http://"+srv.listener.Addr().String())
if err != nil {
panic(err)
commit d25db7c02aa07e9d4812a029753c2b8606cf35b1
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 26 09:47:36 2017 -0400
11901: Fix unclosed db.Rows object.
rows is closed implicitly when rows.Next() returns false, but when the
client hangs up, sendOldEvents returns without retrieving all
results. In this case rows.Close() needs to be called explicitly.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index 44e2a1d..f8645eb 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -157,6 +157,7 @@ func (sub *v0subscribe) sendOldEvents(sess *v0session) {
sess.log.WithError(err).Error("db.Query failed")
return
}
+ defer rows.Close()
for rows.Next() {
var id uint64
err := rows.Scan(&id)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list