[ARVADOS] updated: 1.1.4-384-gb0e5a42a4
Git user
git at public.curoverse.com
Fri Jun 15 15:22:40 EDT 2018
Summary of changes:
build/run-library.sh | 5 +----
build/run-tests.sh | 5 +++--
lib/controller/cmd.go | 6 ++----
lib/controller/handler.go | 18 ++++++++++------
lib/controller/handler_test.go | 4 ++--
lib/service/cmd.go | 17 +++++++++++----
sdk/go/arvados/config.go | 42 ++++++++++++++++---------------------
sdk/go/health/aggregator.go | 10 ++++-----
sdk/go/health/aggregator_test.go | 12 +++++------
sdk/python/tests/run_test_server.py | 2 +-
services/health/main.go | 2 +-
11 files changed, 64 insertions(+), 59 deletions(-)
via b0e5a42a42254af27902a0a17b983957d502db85 (commit)
via 1b5156270c5cb8d7a4a1b095d981f1a84a98554f (commit)
via b58c06e93fc1392aea0347ea099376b41ec4b7c3 (commit)
via a1f0e517f6b37ea987c0146a4ca93f50715f00f2 (commit)
from 21c5372c6b670820e842e01336eb6b191d6e10b7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b0e5a42a42254af27902a0a17b983957d502db85
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Jun 15 15:10:00 2018 -0400
13497: Rename SystemNodes to NodeProfiles in config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/cmd.go b/lib/controller/cmd.go
index 02fb3afba..94eb2580b 100644
--- a/lib/controller/cmd.go
+++ b/lib/controller/cmd.go
@@ -12,6 +12,6 @@ import (
var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
-func newHandler(cluster *arvados.Cluster, node *arvados.SystemNode) service.Handler {
- return &Handler{Cluster: cluster, Node: node}
+func newHandler(cluster *arvados.Cluster, np *arvados.NodeProfile) service.Handler {
+ return &Handler{Cluster: cluster, NodeProfile: np}
}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 643a932a5..59c2f2a61 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -20,8 +20,8 @@ import (
)
type Handler struct {
- Cluster *arvados.Cluster
- Node *arvados.SystemNode
+ Cluster *arvados.Cluster
+ NodeProfile *arvados.NodeProfile
setupOnce sync.Once
handlerStack http.Handler
@@ -63,7 +63,7 @@ var dropHeaders = map[string]bool{
}
func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
- urlOut, err := findRailsAPI(h.Cluster, h.Node)
+ urlOut, err := findRailsAPI(h.Cluster, h.NodeProfile)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -123,8 +123,8 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
// For now, findRailsAPI always uses the rails API running on this
// node.
-func findRailsAPI(cluster *arvados.Cluster, node *arvados.SystemNode) (*url.URL, error) {
- hostport := node.RailsAPI.Listen
+func findRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, error) {
+ hostport := np.RailsAPI.Listen
if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" {
// ":12345" => connect to indicated port on localhost
hostport = "localhost" + hostport
@@ -134,7 +134,7 @@ func findRailsAPI(cluster *arvados.Cluster, node *arvados.SystemNode) (*url.URL,
return nil, err
}
proto := "http"
- if node.RailsAPI.TLS {
+ if np.RailsAPI.TLS {
proto = "https"
}
return url.Parse(proto + "://" + hostport)
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 70a337a6c..981ad7ab9 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -35,14 +35,14 @@ type HandlerSuite struct {
func (s *HandlerSuite) SetUpTest(c *check.C) {
s.cluster = &arvados.Cluster{
ClusterID: "zzzzz",
- SystemNodes: map[string]arvados.SystemNode{
+ NodeProfiles: map[string]arvados.NodeProfile{
"*": {
Controller: arvados.SystemServiceInstance{Listen: ":"},
RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true},
},
},
}
- node := s.cluster.SystemNodes["*"]
+ node := s.cluster.NodeProfiles["*"]
s.handler = newHandler(s.cluster, &node)
}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index a144c01a6..08f8e02da 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -23,7 +23,7 @@ type Handler interface {
CheckHealth() error
}
-type NewHandlerFunc func(*arvados.Cluster, *arvados.SystemNode) Handler
+type NewHandlerFunc func(*arvados.Cluster, *arvados.NodeProfile) Handler
type command struct {
newHandler NewHandlerFunc
@@ -59,7 +59,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
configFile := flags.String("config", arvados.DefaultConfigFile, "Site configuration `file`")
- hostName := flags.String("host", "", "Host profile `name` to use in SystemNodes config (if blank, use hostname reported by OS)")
+ nodeProfile := flags.String("node-profile", "", "`Name` of NodeProfiles config entry to use (if blank, use hostname reported by OS)")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -75,7 +75,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
if err != nil {
return 1
}
- node, err := cluster.GetSystemNode(*hostName)
+ node, err := cluster.GetNodeProfile(*nodeProfile)
if err != nil {
return 1
}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index a74c6d8d6..8856c9295 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -51,7 +51,7 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
- SystemNodes map[string]SystemNode
+ NodeProfiles map[string]NodeProfile
InstanceTypes []InstanceType
HTTPRequestTimeout Duration
}
@@ -65,17 +65,11 @@ type InstanceType struct {
Price float64
}
-// GetThisSystemNode returns a SystemNode for the node we're running
-// on right now.
-func (cc *Cluster) GetThisSystemNode() (*SystemNode, error) {
- return cc.GetSystemNode("")
-}
-
-// GetSystemNode returns a SystemNode for the given hostname. An error
-// is returned if the appropriate configuration can't be determined
-// (e.g., this does not appear to be a system node). If node is empty,
-// use the OS-reported hostname.
-func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
+// GetNodeProfile returns a NodeProfile for the given hostname. An
+// error is returned if the appropriate configuration can't be
+// determined (e.g., this does not appear to be a system node). If
+// node is empty, use the OS-reported hostname.
+func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) {
if node == "" {
hostname, err := os.Hostname()
if err != nil {
@@ -83,18 +77,18 @@ func (cc *Cluster) GetSystemNode(node string) (*SystemNode, error) {
}
node = hostname
}
- if cfg, ok := cc.SystemNodes[node]; ok {
+ if cfg, ok := cc.NodeProfiles[node]; ok {
return &cfg, nil
}
// If node is not listed, but "*" gives a default system node
// config, use the default config.
- if cfg, ok := cc.SystemNodes["*"]; ok {
+ if cfg, ok := cc.NodeProfiles["*"]; ok {
return &cfg, nil
}
return nil, fmt.Errorf("config does not provision host %q as a system node", node)
}
-type SystemNode struct {
+type NodeProfile struct {
Controller SystemServiceInstance `json:"arvados-controller"`
Health SystemServiceInstance `json:"arvados-health"`
Keepproxy SystemServiceInstance `json:"keepproxy"`
@@ -121,16 +115,16 @@ const (
// ServicePorts returns the configured listening address (or "" if
// disabled) for each service on the node.
-func (sn *SystemNode) ServicePorts() map[ServiceName]string {
+func (np *NodeProfile) ServicePorts() map[ServiceName]string {
return map[ServiceName]string{
- ServiceNameRailsAPI: sn.RailsAPI.Listen,
- ServiceNameController: sn.Controller.Listen,
- ServiceNameNodemanager: sn.Nodemanager.Listen,
- ServiceNameWorkbench: sn.Workbench.Listen,
- ServiceNameWebsocket: sn.Websocket.Listen,
- ServiceNameKeepweb: sn.Keepweb.Listen,
- ServiceNameKeepproxy: sn.Keepproxy.Listen,
- ServiceNameKeepstore: sn.Keepstore.Listen,
+ ServiceNameRailsAPI: np.RailsAPI.Listen,
+ ServiceNameController: np.Controller.Listen,
+ ServiceNameNodemanager: np.Nodemanager.Listen,
+ ServiceNameWorkbench: np.Workbench.Listen,
+ ServiceNameWebsocket: np.Websocket.Listen,
+ ServiceNameKeepweb: np.Keepweb.Listen,
+ ServiceNameKeepproxy: np.Keepproxy.Listen,
+ ServiceNameKeepstore: np.Keepstore.Listen,
}
}
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 3f3a91800..a6cb8798a 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -113,8 +113,8 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
mtx := sync.Mutex{}
wg := sync.WaitGroup{}
- for node, nodeConfig := range cluster.SystemNodes {
- for svc, addr := range nodeConfig.ServicePorts() {
+ for profileName, profile := range cluster.NodeProfiles {
+ for svc, addr := range profile.ServicePorts() {
// Ensure svc is listed in resp.Services.
mtx.Lock()
if _, ok := resp.Services[svc]; !ok {
@@ -128,10 +128,10 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
}
wg.Add(1)
- go func(node string, svc arvados.ServiceName, addr string) {
+ go func(profileName string, svc arvados.ServiceName, addr string) {
defer wg.Done()
var result CheckResult
- url, err := agg.pingURL(node, addr)
+ url, err := agg.pingURL(profileName, addr)
if err != nil {
result = CheckResult{
Health: "ERROR",
@@ -152,7 +152,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
} else {
resp.Health = "ERROR"
}
- }(node, svc, addr)
+ }(profileName, svc, addr)
}
}
wg.Wait()
diff --git a/sdk/go/health/aggregator_test.go b/sdk/go/health/aggregator_test.go
index 4c652254b..a96ed136c 100644
--- a/sdk/go/health/aggregator_test.go
+++ b/sdk/go/health/aggregator_test.go
@@ -34,7 +34,7 @@ func (s *AggregatorSuite) SetUpTest(c *check.C) {
Clusters: map[string]arvados.Cluster{
"zzzzz": {
ManagementToken: arvadostest.ManagementToken,
- SystemNodes: map[string]arvados.SystemNode{},
+ NodeProfiles: map[string]arvados.NodeProfile{},
},
},
}}
@@ -86,7 +86,7 @@ func (*unhealthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
func (s *AggregatorSuite) TestUnhealthy(c *check.C) {
srv, listen := s.stubServer(&unhealthyHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listen},
}
s.handler.ServeHTTP(s.resp, s.req)
@@ -106,7 +106,7 @@ func (*healthyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
func (s *AggregatorSuite) TestHealthy(c *check.C) {
srv, listen := s.stubServer(&healthyHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Controller: arvados.SystemServiceInstance{Listen: listen},
Keepproxy: arvados.SystemServiceInstance{Listen: listen},
Keepstore: arvados.SystemServiceInstance{Listen: listen},
@@ -130,7 +130,7 @@ func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
defer srvH.Close()
srvU, listenU := s.stubServer(&unhealthyHandler{})
defer srvU.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Controller: arvados.SystemServiceInstance{Listen: listenH},
Keepproxy: arvados.SystemServiceInstance{Listen: listenH},
Keepstore: arvados.SystemServiceInstance{Listen: listenH},
@@ -140,7 +140,7 @@ func (s *AggregatorSuite) TestHealthyAndUnhealthy(c *check.C) {
Websocket: arvados.SystemServiceInstance{Listen: listenH},
Workbench: arvados.SystemServiceInstance{Listen: listenH},
}
- s.handler.Config.Clusters["zzzzz"].SystemNodes["127.0.0.1"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["127.0.0.1"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listenU},
}
s.handler.ServeHTTP(s.resp, s.req)
@@ -194,7 +194,7 @@ func (s *AggregatorSuite) TestPingTimeout(c *check.C) {
s.handler.timeout = arvados.Duration(100 * time.Millisecond)
srv, listen := s.stubServer(&slowHandler{})
defer srv.Close()
- s.handler.Config.Clusters["zzzzz"].SystemNodes["localhost"] = arvados.SystemNode{
+ s.handler.Config.Clusters["zzzzz"].NodeProfiles["localhost"] = arvados.NodeProfile{
Keepstore: arvados.SystemServiceInstance{Listen: listen},
}
s.handler.ServeHTTP(s.resp, s.req)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index dc4d721f1..f7ca6daf6 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -408,7 +408,7 @@ def run_controller():
f.write("""
Clusters:
zzzzz:
- SystemNodes:
+ NodeProfiles:
"*":
"arvados-controller":
Listen: ":{}"
diff --git a/services/health/main.go b/services/health/main.go
index 376d4830b..1d2ec47a6 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -41,7 +41,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
- nodeCfg, err := clusterCfg.GetThisSystemNode()
+ nodeCfg, err := clusterCfg.GetNodeProfile("")
if err != nil {
log.Fatal(err)
}
commit 1b5156270c5cb8d7a4a1b095d981f1a84a98554f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Jun 15 15:08:59 2018 -0400
13497: Abort startup if Rails API cannot be found.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/cmd.go b/lib/controller/cmd.go
index e006b6594..02fb3afba 100644
--- a/lib/controller/cmd.go
+++ b/lib/controller/cmd.go
@@ -5,8 +5,6 @@
package controller
import (
- "net/http"
-
"git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -14,6 +12,6 @@ import (
var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
-func newHandler(cluster *arvados.Cluster, node *arvados.SystemNode) http.Handler {
+func newHandler(cluster *arvados.Cluster, node *arvados.SystemNode) service.Handler {
return &Handler{Cluster: cluster, Node: node}
}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index ad765bafa..643a932a5 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -33,6 +33,12 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
h.handlerStack.ServeHTTP(w, req)
}
+func (h *Handler) CheckHealth() error {
+ h.setupOnce.Do(h.setup)
+ _, err := findRailsAPI(h.Cluster, h.NodeProfile)
+ return err
+}
+
func (h *Handler) setup() {
mux := http.NewServeMux()
mux.Handle("/_health/", &health.Handler{
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index ab7886066..a144c01a6 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -18,7 +18,12 @@ import (
"github.com/coreos/go-systemd/daemon"
)
-type NewHandlerFunc func(*arvados.Cluster, *arvados.SystemNode) http.Handler
+type Handler interface {
+ http.Handler
+ CheckHealth() error
+}
+
+type NewHandlerFunc func(*arvados.Cluster, *arvados.SystemNode) Handler
type command struct {
newHandler NewHandlerFunc
@@ -79,9 +84,13 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
err = fmt.Errorf("configuration does not enable the %s service on this host", c.svcName)
return 1
}
+ handler := c.newHandler(cluster, node)
+ if err = handler.CheckHealth(); err != nil {
+ return 1
+ }
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, c.newHandler(cluster, node))),
+ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
},
Addr: listen,
}
commit b58c06e93fc1392aea0347ea099376b41ec4b7c3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Jun 15 15:06:02 2018 -0400
13497: Support "run-tests.sh --only go".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index ad7960275..210f11f20 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -652,8 +652,9 @@ do_test() {
;;
esac
if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
- (-z "${only}" || "${only}" == "${suite}" || \
- "${only}" == "${1}") ]]; then
+ (-z "${only}" || "${only}" == "${suite}" || \
+ "${only}" == "${1}") ||
+ "${only}" == "${2}" ]]; then
retry do_test_once ${@}
else
title "Skipping ${1} tests"
commit a1f0e517f6b37ea987c0146a4ca93f50715f00f2
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Jun 15 15:05:17 2018 -0400
13497: Bump version numbers for Go packages when lib/ changes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index fb4df6a79..4b18d037b 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -129,10 +129,7 @@ package_go_binary() {
# Arvados SDK and the SDK has changed.
declare -a checkdirs=(vendor)
if grep -qr git.curoverse.com/arvados .; then
- checkdirs+=(sdk/go)
- if [[ "$prog" -eq "crunch-dispatch-slurm" ]]; then
- checkdirs+=(lib/dispatchcloud)
- fi
+ checkdirs+=(sdk/go lib)
fi
for dir in ${checkdirs[@]}; do
cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list