[ARVADOS] created: 1.3.0-962-gd6358ef9f

Git user git at public.curoverse.com
Fri May 24 19:17:58 UTC 2019


        at  d6358ef9fc0d8474827830a7ea0a451832e1fbec (commit)


commit d6358ef9fc0d8474827830a7ea0a451832e1fbec
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 15:15:25 2019 -0400

    15003: Remove NodeProfiles section from integration test config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index b2ee18296..fea0578ab 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -419,25 +419,26 @@ Clusters:
     PostgreSQL:
       ConnectionPool: 32
       Connection:
-        host: {}
-        dbname: {}
-        user: {}
-        password: {}
-    NodeProfiles:
-      "*":
-        "arvados-controller":
-          Listen: ":{}"
-        "arvados-api-server":
-          Listen: ":{}"
-          TLS: true
-          Insecure: true
+        host: {dbhost}
+        dbname: {dbname}
+        user: {dbuser}
+        password: {dbpass}
+    TLS:
+      Insecure: true
+    Services:
+      Controller:
+        InternalURLs:
+          "http://localhost:{controllerport}": {{}}
+      RailsAPI:
+        InternalURLs:
+          "https://localhost:{railsport}": {{}}
         """.format(
-            _dbconfig('host'),
-            _dbconfig('database'),
-            _dbconfig('username'),
-            _dbconfig('password'),
-            port,
-            rails_api_port,
+            dbhost=_dbconfig('host'),
+            dbname=_dbconfig('database'),
+            dbuser=_dbconfig('username'),
+            dbpass=_dbconfig('password'),
+            controllerport=port,
+            railsport=rails_api_port,
         ))
     logf = open(_logfilename('controller'), 'a')
     controller = subprocess.Popen(

commit db1cbf4f40a83c4af347be9737d7bca7ccd772ab
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 15:12:20 2019 -0400

    15003: Remove debug printf.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index fdcf3b4ac..56ccd8883 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
@@ -168,8 +167,6 @@ func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, err
 			listener.Close()
 			return url.Host, nil
 		}
-		log.Print(err)
-
 	}
 	return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
 }

commit 15c688d1c4b41232536d0a275dd5c0fdb8879a00
Merge: 8ee263231 cdc0e1acb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 15:11:45 2019 -0400

    15003: Merge branch 'master'
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --cc lib/service/cmd.go
index cfc40778f,955b21f9e..fdcf3b4ac
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@@ -10,8 -10,7 +10,9 @@@ import 
  	"flag"
  	"fmt"
  	"io"
+ 	"io/ioutil"
 +	"log"
 +	"net"
  	"net/http"
  	"net/url"
  	"os"

commit 8ee2632316f61a63f5af52e84e1538e26850005e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 14:48:24 2019 -0400

    15003: Update docs to new config struct.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/doc/install/install-dispatch-cloud.html.textile.liquid b/doc/install/install-dispatch-cloud.html.textile.liquid
index 2ecf3c377..bc3be8f1d 100644
--- a/doc/install/install-dispatch-cloud.html.textile.liquid
+++ b/doc/install/install-dispatch-cloud.html.textile.liquid
@@ -72,14 +72,14 @@ Add or update the following portions of your cluster configuration file, @/etc/a
       DispatchCloud:
         InternalURLs:
           "http://localhost:9006": {}
-    CloudVMs:
-      # BootProbeCommand is a shell command that succeeds when an instance is ready for service
-      BootProbeCommand: "sudo systemctl status docker"
+    Containers:
+      CloudVMs:
+        # BootProbeCommand is a shell command that succeeds when an instance is ready for service
+        BootProbeCommand: "sudo systemctl status docker"
 
-      <b># --- driver-specific configuration goes here --- see Amazon and Azure examples below ---</b>
+        <b># --- driver-specific configuration goes here --- see Amazon and Azure examples below ---</b>
 
-    Dispatch:
-      PrivateKey: |
+      DispatchPrivateKey: |
         -----BEGIN RSA PRIVATE KEY-----
         MIIEpQIBAAKCAQEAqXoCzcOBkFQ7w4dvXf9B++1ctgZRqEbgRYL3SstuMV4oawks
         ttUuxJycDdsPmeYcHsKo8vsEZpN6iYsX6ZZzhkO5nEayUTU8sBjmg1ZCTo4QqKXr
@@ -111,18 +111,19 @@ Minimal configuration example for Amazon EC2:
 <notextile>
 <pre><code>Clusters:
   <span class="userinput">uuid_prefix</span>:
-    CloudVMs:
-      ImageID: ami-01234567890abcdef
-      Driver: ec2
-      DriverParameters:
-        AccessKeyID: EALMF21BJC7MKNF9FVVR
-        SecretAccessKey: yKJAPmoCQOMtYWzEUQ1tKTyrocTcbH60CRvGP3pM
-        SecurityGroupIDs:
-        - sg-0123abcd
-        SubnetID: subnet-0123abcd
-        Region: us-east-1
-        EBSVolumeType: gp2
-        AdminUsername: debian
+    Containers:
+      CloudVMs:
+        ImageID: ami-01234567890abcdef
+        Driver: ec2
+        DriverParameters:
+          AccessKeyID: EALMF21BJC7MKNF9FVVR
+          SecretAccessKey: yKJAPmoCQOMtYWzEUQ1tKTyrocTcbH60CRvGP3pM
+          SecurityGroupIDs:
+          - sg-0123abcd
+          SubnetID: subnet-0123abcd
+          Region: us-east-1
+          EBSVolumeType: gp2
+          AdminUsername: debian
 </code></pre>
 </notextile>
 
@@ -131,23 +132,24 @@ Minimal configuration example for Azure:
 <notextile>
 <pre><code>Clusters:
   <span class="userinput">uuid_prefix</span>:
-    CloudVMs:
-      ImageID: "https://zzzzzzzz.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.55555555-5555-5555-5555-555555555555.vhd"
-      Driver: azure
-      DriverParameters:
-        SubscriptionID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-        ClientID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-        ClientSecret: 2WyXt0XFbEtutnf2hp528t6Wk9S5bOHWkRaaWwavKQo=
-        TenantID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-        CloudEnvironment: AzurePublicCloud
-        ResourceGroup: zzzzz
-        Location: centralus
-        Network: zzzzz
-        Subnet: zzzzz-subnet-private
-        StorageAccount: example
-        BlobContainer: vhds
-        DeleteDanglingResourcesAfter: 20s
-        AdminUsername: arvados
+    Containers:
+      CloudVMs:
+        ImageID: "https://zzzzzzzz.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.55555555-5555-5555-5555-555555555555.vhd"
+        Driver: azure
+        DriverParameters:
+          SubscriptionID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+          ClientID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+          ClientSecret: 2WyXt0XFbEtutnf2hp528t6Wk9S5bOHWkRaaWwavKQo=
+          TenantID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+          CloudEnvironment: AzurePublicCloud
+          ResourceGroup: zzzzz
+          Location: centralus
+          Network: zzzzz
+          Subnet: zzzzz-subnet-private
+          StorageAccount: example
+          BlobContainer: vhds
+          DeleteDanglingResourcesAfter: 20s
+          AdminUsername: arvados
 </code></pre>
 </notextile>
 

commit b3d57ff3ccf9c612a11fcf53a451a0f61a362da6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 13:55:37 2019 -0400

    15003: Update dispatchcloud to new config struct.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 71ff9c784..3bf0ee9bd 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -95,7 +95,7 @@ func (disp *dispatcher) Close() {
 // Make a worker.Executor for the given instance.
 func (disp *dispatcher) newExecutor(inst cloud.Instance) worker.Executor {
 	exr := ssh_executor.New(inst)
-	exr.SetTargetPort(disp.Cluster.CloudVMs.SSHPort)
+	exr.SetTargetPort(disp.Cluster.Containers.CloudVMs.SSHPort)
 	exr.SetSigners(disp.sshKey)
 	return exr
 }
@@ -126,8 +126,8 @@ func (disp *dispatcher) initialize() {
 	disp.stop = make(chan struct{}, 1)
 	disp.stopped = make(chan struct{})
 
-	if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Dispatch.PrivateKey)); err != nil {
-		disp.logger.Fatalf("error parsing configured Dispatch.PrivateKey: %s", err)
+	if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Containers.DispatchPrivateKey)); err != nil {
+		disp.logger.Fatalf("error parsing configured Containers.DispatchPrivateKey: %s", err)
 	} else {
 		disp.sshKey = key
 	}
@@ -167,11 +167,11 @@ func (disp *dispatcher) run() {
 	defer disp.instanceSet.Stop()
 	defer disp.pool.Stop()
 
-	staleLockTimeout := time.Duration(disp.Cluster.Dispatch.StaleLockTimeout)
+	staleLockTimeout := time.Duration(disp.Cluster.Containers.StaleLockTimeout)
 	if staleLockTimeout == 0 {
 		staleLockTimeout = defaultStaleLockTimeout
 	}
-	pollInterval := time.Duration(disp.Cluster.Dispatch.PollInterval)
+	pollInterval := time.Duration(disp.Cluster.Containers.CloudVMs.PollInterval)
 	if pollInterval <= 0 {
 		pollInterval = defaultPollInterval
 	}
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 00157b75c..6b8620ade 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -49,23 +49,23 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 	}
 
 	s.cluster = &arvados.Cluster{
-		CloudVMs: arvados.CloudVMs{
-			Driver:               "test",
-			SyncInterval:         arvados.Duration(10 * time.Millisecond),
-			TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
-			TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
-			TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
-			TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
-			MaxCloudOpsPerSecond: 500,
-		},
-		Dispatch: arvados.Dispatch{
-			PrivateKey:         string(dispatchprivraw),
-			PollInterval:       arvados.Duration(5 * time.Millisecond),
-			ProbeInterval:      arvados.Duration(5 * time.Millisecond),
+		Containers: arvados.ContainersConfig{
+			DispatchPrivateKey: string(dispatchprivraw),
 			StaleLockTimeout:   arvados.Duration(5 * time.Millisecond),
-			MaxProbesPerSecond: 1000,
-			TimeoutSignal:      arvados.Duration(3 * time.Millisecond),
-			TimeoutTERM:        arvados.Duration(20 * time.Millisecond),
+			CloudVMs: arvados.CloudVMsConfig{
+				Driver:               "test",
+				SyncInterval:         arvados.Duration(10 * time.Millisecond),
+				TimeoutIdle:          arvados.Duration(150 * time.Millisecond),
+				TimeoutBooting:       arvados.Duration(150 * time.Millisecond),
+				TimeoutProbe:         arvados.Duration(15 * time.Millisecond),
+				TimeoutShutdown:      arvados.Duration(5 * time.Millisecond),
+				MaxCloudOpsPerSecond: 500,
+				PollInterval:         arvados.Duration(5 * time.Millisecond),
+				ProbeInterval:        arvados.Duration(5 * time.Millisecond),
+				MaxProbesPerSecond:   1000,
+				TimeoutSignal:        arvados.Duration(3 * time.Millisecond),
+				TimeoutTERM:          arvados.Duration(20 * time.Millisecond),
+			},
 		},
 		InstanceTypes: arvados.InstanceTypeMap{
 			test.InstanceType(1).Name:  test.InstanceType(1),
@@ -76,16 +76,9 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
 			test.InstanceType(8).Name:  test.InstanceType(8),
 			test.InstanceType(16).Name: test.InstanceType(16),
 		},
-		NodeProfiles: map[string]arvados.NodeProfile{
-			"*": {
-				Controller:    arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_API_HOST")},
-				DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
-			},
-		},
-		Services: arvados.Services{
-			Controller: arvados.Service{ExternalURL: arvados.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}},
-		},
 	}
+	arvadostest.SetServiceURL(&s.cluster.Services.DispatchCloud, "http://localhost:/")
+	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "https://"+os.Getenv("ARVADOS_API_HOST")+"/")
 
 	arvClient, err := arvados.NewClientFromConfig(s.cluster)
 	c.Check(err, check.IsNil)
@@ -242,7 +235,7 @@ func (s *DispatcherSuite) TestAPIDisabled(c *check.C) {
 
 func (s *DispatcherSuite) TestInstancesAPI(c *check.C) {
 	s.cluster.ManagementToken = "abcdefgh"
-	s.cluster.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
+	s.cluster.Containers.CloudVMs.TimeoutBooting = arvados.Duration(time.Second)
 	drivers["test"] = s.stubDriver
 	s.disp.setupOnce.Do(s.disp.initialize)
 	s.disp.queue = &test.Queue{}
diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go
index eb1e48737..5ec0f73e7 100644
--- a/lib/dispatchcloud/driver.go
+++ b/lib/dispatchcloud/driver.go
@@ -22,12 +22,12 @@ var drivers = map[string]cloud.Driver{
 }
 
 func newInstanceSet(cluster *arvados.Cluster, setID cloud.InstanceSetID, logger logrus.FieldLogger) (cloud.InstanceSet, error) {
-	driver, ok := drivers[cluster.CloudVMs.Driver]
+	driver, ok := drivers[cluster.Containers.CloudVMs.Driver]
 	if !ok {
-		return nil, fmt.Errorf("unsupported cloud driver %q", cluster.CloudVMs.Driver)
+		return nil, fmt.Errorf("unsupported cloud driver %q", cluster.Containers.CloudVMs.Driver)
 	}
-	is, err := driver.InstanceSet(cluster.CloudVMs.DriverParameters, setID, logger)
-	if maxops := cluster.CloudVMs.MaxCloudOpsPerSecond; maxops > 0 {
+	is, err := driver.InstanceSet(cluster.Containers.CloudVMs.DriverParameters, setID, logger)
+	if maxops := cluster.Containers.CloudVMs.MaxCloudOpsPerSecond; maxops > 0 {
 		is = &rateLimitedInstanceSet{
 			InstanceSet: is,
 			ticker:      time.NewTicker(time.Second / time.Duration(maxops)),
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 014ab93bf..84b61fc00 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -97,18 +97,18 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe
 		arvClient:          arvClient,
 		instanceSet:        &throttledInstanceSet{InstanceSet: instanceSet},
 		newExecutor:        newExecutor,
-		bootProbeCommand:   cluster.CloudVMs.BootProbeCommand,
-		imageID:            cloud.ImageID(cluster.CloudVMs.ImageID),
+		bootProbeCommand:   cluster.Containers.CloudVMs.BootProbeCommand,
+		imageID:            cloud.ImageID(cluster.Containers.CloudVMs.ImageID),
 		instanceTypes:      cluster.InstanceTypes,
-		maxProbesPerSecond: cluster.Dispatch.MaxProbesPerSecond,
-		probeInterval:      duration(cluster.Dispatch.ProbeInterval, defaultProbeInterval),
-		syncInterval:       duration(cluster.CloudVMs.SyncInterval, defaultSyncInterval),
-		timeoutIdle:        duration(cluster.CloudVMs.TimeoutIdle, defaultTimeoutIdle),
-		timeoutBooting:     duration(cluster.CloudVMs.TimeoutBooting, defaultTimeoutBooting),
-		timeoutProbe:       duration(cluster.CloudVMs.TimeoutProbe, defaultTimeoutProbe),
-		timeoutShutdown:    duration(cluster.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown),
-		timeoutTERM:        duration(cluster.Dispatch.TimeoutTERM, defaultTimeoutTERM),
-		timeoutSignal:      duration(cluster.Dispatch.TimeoutSignal, defaultTimeoutSignal),
+		maxProbesPerSecond: cluster.Containers.CloudVMs.MaxProbesPerSecond,
+		probeInterval:      duration(cluster.Containers.CloudVMs.ProbeInterval, defaultProbeInterval),
+		syncInterval:       duration(cluster.Containers.CloudVMs.SyncInterval, defaultSyncInterval),
+		timeoutIdle:        duration(cluster.Containers.CloudVMs.TimeoutIdle, defaultTimeoutIdle),
+		timeoutBooting:     duration(cluster.Containers.CloudVMs.TimeoutBooting, defaultTimeoutBooting),
+		timeoutProbe:       duration(cluster.Containers.CloudVMs.TimeoutProbe, defaultTimeoutProbe),
+		timeoutShutdown:    duration(cluster.Containers.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown),
+		timeoutTERM:        duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM),
+		timeoutSignal:      duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal),
 		installPublicKey:   installPublicKey,
 		stop:               make(chan bool),
 	}
diff --git a/lib/dispatchcloud/worker/pool_test.go b/lib/dispatchcloud/worker/pool_test.go
index fc33a7ab2..693953668 100644
--- a/lib/dispatchcloud/worker/pool_test.go
+++ b/lib/dispatchcloud/worker/pool_test.go
@@ -76,13 +76,13 @@ func (suite *PoolSuite) TestResumeAfterRestart(c *check.C) {
 	}
 
 	cluster := &arvados.Cluster{
-		Dispatch: arvados.Dispatch{
-			MaxProbesPerSecond: 1000,
-			ProbeInterval:      arvados.Duration(time.Millisecond * 10),
-		},
-		CloudVMs: arvados.CloudVMs{
-			BootProbeCommand: "true",
-			SyncInterval:     arvados.Duration(time.Millisecond * 10),
+		Containers: arvados.ContainersConfig{
+			CloudVMs: arvados.CloudVMsConfig{
+				BootProbeCommand:   "true",
+				MaxProbesPerSecond: 1000,
+				ProbeInterval:      arvados.Duration(time.Millisecond * 10),
+				SyncInterval:       arvados.Duration(time.Millisecond * 10),
+			},
 		},
 		InstanceTypes: arvados.InstanceTypeMap{
 			type1.Name: type1,
diff --git a/sdk/go/arvadostest/stub.go b/sdk/go/arvadostest/stub.go
index 6b24a38fd..2b6e40c95 100644
--- a/sdk/go/arvadostest/stub.go
+++ b/sdk/go/arvadostest/stub.go
@@ -51,4 +51,5 @@ func SetServiceURL(service *arvados.Service, internalURL string) {
 		panic(err)
 	}
 	service.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL(*u): {}}
+	service.ExternalURL = arvados.URL(*u)
 }

commit a26a249eed0299c56e6583a4fae2b9f6dc16bf7f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 13:34:16 2019 -0400

    15003: Use lib/cmd and lib/service packages for health service.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index d8c0a4abf..acfdbb7f8 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -28,7 +28,7 @@ type Aggregator struct {
 	httpClient *http.Client
 	timeout    arvados.Duration
 
-	Config *arvados.Config
+	Cluster *arvados.Cluster
 
 	// If non-nil, Log is called after handling each request.
 	Log func(*http.Request, error)
@@ -42,6 +42,10 @@ func (agg *Aggregator) setup() {
 	}
 }
 
+func (agg *Aggregator) CheckHealth() error {
+	return nil
+}
+
 func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	agg.setupOnce.Do(agg.setup)
 	sendErr := func(statusCode int, err error) {
@@ -54,13 +58,7 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 
 	resp.Header().Set("Content-Type", "application/json")
 
-	cluster, err := agg.Config.GetCluster("")
-	if err != nil {
-		err = fmt.Errorf("arvados.GetCluster(): %s", err)
-		sendErr(http.StatusInternalServerError, err)
-		return
-	}
-	if !agg.checkAuth(req, cluster) {
+	if !agg.checkAuth(req) {
 		sendErr(http.StatusUnauthorized, errUnauthorized)
 		return
 	}
@@ -68,7 +66,7 @@ func (agg *Aggregator) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 		sendErr(http.StatusNotFound, errNotFound)
 		return
 	}
-	json.NewEncoder(resp).Encode(agg.ClusterHealth(cluster))
+	json.NewEncoder(resp).Encode(agg.ClusterHealth())
 	if agg.Log != nil {
 		agg.Log(req, nil)
 	}
@@ -104,7 +102,7 @@ type ServiceHealth struct {
 	N      int    `json:"n"`
 }
 
-func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResponse {
+func (agg *Aggregator) ClusterHealth() ClusterHealthResponse {
 	resp := ClusterHealthResponse{
 		Health:   "OK",
 		Checks:   make(map[string]CheckResult),
@@ -113,7 +111,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 
 	mtx := sync.Mutex{}
 	wg := sync.WaitGroup{}
-	for svcName, svc := range cluster.Services.Map() {
+	for svcName, svc := range agg.Cluster.Services.Map() {
 		// Ensure svc is listed in resp.Services.
 		mtx.Lock()
 		if _, ok := resp.Services[svcName]; !ok {
@@ -133,7 +131,7 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 						Error:  err.Error(),
 					}
 				} else {
-					result = agg.ping(pingURL, cluster)
+					result = agg.ping(pingURL)
 				}
 
 				mtx.Lock()
@@ -168,7 +166,7 @@ func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) {
 	return base.Parse("/_health/ping")
 }
 
-func (agg *Aggregator) ping(target *url.URL, cluster *arvados.Cluster) (result CheckResult) {
+func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
 	t0 := time.Now()
 
 	var err error
@@ -185,7 +183,7 @@ func (agg *Aggregator) ping(target *url.URL, cluster *arvados.Cluster) (result C
 	if err != nil {
 		return
 	}
-	req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+	req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
 
 	ctx, cancel := context.WithTimeout(req.Context(), time.Duration(agg.timeout))
 	defer cancel()
@@ -211,10 +209,10 @@ func (agg *Aggregator) ping(target *url.URL, cluster *arvados.Cluster) (result C
 	return
 }
 
-func (agg *Aggregator) checkAuth(req *http.Request, cluster *arvados.Cluster) bool {
+func (agg *Aggregator) checkAuth(req *http.Request) bool {
 	creds := auth.CredentialsFromRequest(req)
 	for _, token := range creds.Tokens {
-		if token != "" && token == cluster.ManagementToken {
+		if token != "" && token == agg.Cluster.ManagementToken {
 			return true
 		}
 	}
diff --git a/services/health/main.go b/services/health/main.go
index 21fcf4d67..2f66b2461 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -5,67 +5,24 @@
 package main
 
 import (
-	"flag"
-	"fmt"
-	"net/http"
+	"context"
+	"os"
 
+	"git.curoverse.com/arvados.git/lib/cmd"
+	"git.curoverse.com/arvados.git/lib/service"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/health"
-	"git.curoverse.com/arvados.git/sdk/go/httpserver"
-	log "github.com/sirupsen/logrus"
 )
 
-var version = "dev"
-
-func main() {
-	configFile := flag.String("config", arvados.DefaultConfigFile, "`path` to arvados configuration file")
-	getVersion := flag.Bool("version", false, "Print version information and exit.")
-	flag.Parse()
-
-	// Print version information if requested
-	if *getVersion {
-		fmt.Printf("arvados-health %s\n", version)
-		return
-	}
-
-	log.SetFormatter(&log.JSONFormatter{
-		TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
-	})
-	log.Printf("arvados-health %s started", version)
+var (
+	version             = "dev"
+	command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
+)
 
-	cfg, err := arvados.GetConfig(*configFile)
-	if err != nil {
-		log.Fatal(err)
-	}
-	clusterCfg, err := cfg.GetCluster("")
-	if err != nil {
-		log.Fatal(err)
-	}
-	nodeCfg, err := clusterCfg.GetNodeProfile("")
-	if err != nil {
-		log.Fatal(err)
-	}
+func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string) service.Handler {
+	return &health.Aggregator{Cluster: cluster}
+}
 
-	log := log.WithField("Service", "Health")
-	srv := &httpserver.Server{
-		Addr: nodeCfg.Health.Listen,
-		Server: http.Server{
-			Handler: &health.Aggregator{
-				Config: cfg,
-				Log: func(req *http.Request, err error) {
-					log.WithField("RemoteAddr", req.RemoteAddr).
-						WithField("Path", req.URL.Path).
-						WithError(err).
-						Info("HTTP request")
-				},
-			},
-		},
-	}
-	if err := srv.Start(); err != nil {
-		log.Fatal(err)
-	}
-	log.WithField("Listen", srv.Addr).Info("listening")
-	if err := srv.Wait(); err != nil {
-		log.Fatal(err)
-	}
+func main() {
+	os.Exit(command.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
 }

commit 48a3b3a3c28a6590fdf3d2b750192706cb751fae
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Fri May 24 13:22:12 2019 -0400

    15003: Remove NodeProfiles section from cluster config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/doc/install/install-controller.html.textile.liquid b/doc/install/install-controller.html.textile.liquid
index 3e94b290d..394aa0fdf 100644
--- a/doc/install/install-controller.html.textile.liquid
+++ b/doc/install/install-controller.html.textile.liquid
@@ -92,12 +92,13 @@ Create the cluster configuration file @/etc/arvados/config.yml@ using the follow
 <notextile>
 <pre><code>Clusters:
   <span class="userinput">uuid_prefix</span>:
-    NodeProfiles:
-      apiserver:
-        arvados-controller:
-          Listen: ":<span class="userinput">9004</span>" # must match the "upstream controller" section of your Nginx config
+    Services:
+      Controller:
+        InternalURLs:
+          "http://localhost:<span class="userinput">9004</span>": {} # must match the "upstream controller" section of your Nginx config
+      RailsAPI:
         arvados-api-server:
-          Listen: ":<span class="userinput">8000</span>" # must match the "upstream api" section of your Nginx config
+          "http://localhost:<span class="userinput">8000</span>": {} # must match the "upstream api" section of your Nginx config
     PostgreSQL:
       ConnectionPool: 128
       Connection:
diff --git a/doc/install/install-dispatch-cloud.html.textile.liquid b/doc/install/install-dispatch-cloud.html.textile.liquid
index 42c814b87..2ecf3c377 100644
--- a/doc/install/install-dispatch-cloud.html.textile.liquid
+++ b/doc/install/install-dispatch-cloud.html.textile.liquid
@@ -66,14 +66,12 @@ Add or update the following portions of your cluster configuration file, @/etc/a
   <span class="userinput">uuid_prefix</span>:
     ManagementToken: xyzzy
     SystemRootToken: <span class="userinput">zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>
-    NodeProfiles:
-      # The key "apiserver" corresponds to ARVADOS_NODE_PROFILE in environment file (see below).
-      apiserver:
-        arvados-dispatch-cloud:
-          Listen: ":9006"
     Services:
       Controller:
         ExternalURL: "https://<span class="userinput">uuid_prefix.arvadosapi.com</span>"
+      DispatchCloud:
+        InternalURLs:
+          "http://localhost:9006": {}
     CloudVMs:
       # BootProbeCommand is a shell command that succeeds when an instance is ready for service
       BootProbeCommand: "sudo systemctl status docker"
@@ -153,13 +151,6 @@ Minimal configuration example for Azure:
 </code></pre>
 </notextile>
 
-Create the host configuration file @/etc/arvados/environment at .
-
-<notextile>
-<pre><code>ARVADOS_NODE_PROFILE=apiserver
-</code></pre>
-</notextile>
-
 h2. Install the dispatcher
 
 First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index c8f943f3c..8ffa2a583 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -20,13 +20,33 @@ type deprRequestLimits struct {
 
 type deprCluster struct {
 	RequestLimits deprRequestLimits
-	NodeProfiles  map[string]arvados.NodeProfile
+	NodeProfiles  map[string]nodeProfile
 }
 
 type deprecatedConfig struct {
 	Clusters map[string]deprCluster
 }
 
+type nodeProfile struct {
+	Controller    systemServiceInstance `json:"arvados-controller"`
+	Health        systemServiceInstance `json:"arvados-health"`
+	Keepbalance   systemServiceInstance `json:"keep-balance"`
+	Keepproxy     systemServiceInstance `json:"keepproxy"`
+	Keepstore     systemServiceInstance `json:"keepstore"`
+	Keepweb       systemServiceInstance `json:"keep-web"`
+	Nodemanager   systemServiceInstance `json:"arvados-node-manager"`
+	DispatchCloud systemServiceInstance `json:"arvados-dispatch-cloud"`
+	RailsAPI      systemServiceInstance `json:"arvados-api-server"`
+	Websocket     systemServiceInstance `json:"arvados-ws"`
+	Workbench1    systemServiceInstance `json:"arvados-workbench"`
+}
+
+type systemServiceInstance struct {
+	Listen   string
+	TLS      bool
+	Insecure bool
+}
+
 func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) error {
 	var dc deprecatedConfig
 	err := yaml.Unmarshal(configdata, &dc)
@@ -63,7 +83,7 @@ func applyDeprecatedConfig(cfg *arvados.Config, configdata []byte, log logger) e
 	return nil
 }
 
-func applyDeprecatedNodeProfile(hostname string, ssi arvados.SystemServiceInstance, svc *arvados.Service) {
+func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc *arvados.Service) {
 	scheme := "https"
 	if !ssi.TLS {
 		scheme = "http"
diff --git a/lib/controller/cmd.go b/lib/controller/cmd.go
index f0268091b..434537046 100644
--- a/lib/controller/cmd.go
+++ b/lib/controller/cmd.go
@@ -14,6 +14,6 @@ import (
 
 var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
 
-func newHandler(_ context.Context, cluster *arvados.Cluster, np *arvados.NodeProfile, _ string) service.Handler {
-	return &Handler{Cluster: cluster, NodeProfile: np}
+func newHandler(_ context.Context, cluster *arvados.Cluster, _ string) service.Handler {
+	return &Handler{Cluster: cluster}
 }
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index c4aa33c15..1c859cfc5 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -54,25 +54,22 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 	s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
 	c.Assert(s.remoteMock.Start(), check.IsNil)
 
-	nodeProfile := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
-	}
-	s.testHandler = &Handler{Cluster: &arvados.Cluster{
+	cluster := &arvados.Cluster{
 		ClusterID:  "zhome",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		NodeProfiles: map[string]arvados.NodeProfile{
-			"*": nodeProfile,
-		},
+		TLS:        arvados.TLS{Insecure: true},
 		API: arvados.API{
 			MaxItemsPerResponse:     1000,
 			MaxRequestAmplification: 4,
 		},
-	}, NodeProfile: &nodeProfile}
+	}
+	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
+	arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
+	s.testHandler = &Handler{Cluster: cluster}
 	s.testServer = newServerFromIntegrationTestEnv(c)
 	s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
 
-	s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+	cluster.RemoteClusters = map[string]arvados.RemoteCluster{
 		"zzzzz": {
 			Host:   s.remoteServer.Addr,
 			Proxy:  true,
@@ -318,16 +315,8 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https
 			Handler: h,
 		},
 	}
-
 	c.Assert(srv.Start(), check.IsNil)
-
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-			TLS: false, Insecure: true}}
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
-
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
 	return srv
 }
 
@@ -338,13 +327,8 @@ func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server
 }
 
 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-			TLS: true, Insecure: true}}
 	s.testHandler.Cluster.ClusterID = "zzzzz"
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
 	// HTTP GET
 
@@ -416,12 +400,7 @@ func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-			TLS: true, Insecure: true}}
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
 	req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 	req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
@@ -505,12 +484,7 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-			TLS: true, Insecure: true}}
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
 	req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 	req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
@@ -526,12 +500,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-			TLS: true, Insecure: true}}
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
 	req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 	req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
@@ -616,13 +585,8 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
 	req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 	req.Header.Set("Content-type", "application/json")
 
-	np := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-			TLS: true, Insecure: true}}
+	arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	s.testHandler.Cluster.ClusterID = "zzzzz"
-	s.testHandler.Cluster.NodeProfiles["*"] = np
-	s.testHandler.NodeProfile = &np
 
 	resp := s.testRequest(req)
 	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 35734d780..2c3ce1d4f 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -8,7 +8,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
-	"net"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
@@ -22,8 +22,7 @@ import (
 )
 
 type Handler struct {
-	Cluster     *arvados.Cluster
-	NodeProfile *arvados.NodeProfile
+	Cluster *arvados.Cluster
 
 	setupOnce      sync.Once
 	handlerStack   http.Handler
@@ -61,7 +60,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 func (h *Handler) CheckHealth() error {
 	h.setupOnce.Do(h.setup)
-	_, _, err := findRailsAPI(h.Cluster, h.NodeProfile)
+	_, _, err := findRailsAPI(h.Cluster)
 	return err
 }
 
@@ -127,7 +126,7 @@ func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
 }
 
 func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error) {
-	urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
+	urlOut, insecure, err := findRailsAPI(h.Cluster)
 	if err != nil {
 		return nil, err
 	}
@@ -153,22 +152,19 @@ func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next h
 	}
 }
 
-// For now, findRailsAPI always uses the rails API running on this
-// node.
-func findRailsAPI(cluster *arvados.Cluster, np *arvados.NodeProfile) (*url.URL, bool, error) {
-	hostport := np.RailsAPI.Listen
-	if len(hostport) > 1 && hostport[0] == ':' && strings.TrimRight(hostport[1:], "0123456789") == "" {
-		// ":12345" => connect to indicated port on localhost
-		hostport = "localhost" + hostport
-	} else if _, _, err := net.SplitHostPort(hostport); err == nil {
-		// "[::1]:12345" => connect to indicated address & port
-	} else {
-		return nil, false, err
+// Use a localhost entry from Services.RailsAPI.InternalURLs if one is
+// present, otherwise choose an arbitrary entry.
+func findRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) {
+	var best *url.URL
+	for target := range cluster.Services.RailsAPI.InternalURLs {
+		target := url.URL(target)
+		best = &target
+		if strings.HasPrefix(target.Host, "localhost:") || strings.HasPrefix(target.Host, "127.0.0.1:") || strings.HasPrefix(target.Host, "[::1]:") {
+			break
+		}
 	}
-	proto := "http"
-	if np.RailsAPI.TLS {
-		proto = "https"
+	if best == nil {
+		return nil, false, fmt.Errorf("Services.RailsAPI.InternalURLs is empty")
 	}
-	url, err := url.Parse(proto + "://" + hostport)
-	return url, np.RailsAPI.Insecure, err
+	return best, cluster.TLS.Insecure, nil
 }
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 01544a2b0..a1efaacdd 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -42,15 +42,11 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
 	s.cluster = &arvados.Cluster{
 		ClusterID:  "zzzzz",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		NodeProfiles: map[string]arvados.NodeProfile{
-			"*": {
-				Controller: arvados.SystemServiceInstance{Listen: ":"},
-				RailsAPI:   arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true},
-			},
-		},
+		TLS:        arvados.TLS{Insecure: true},
 	}
-	node := s.cluster.NodeProfiles["*"]
-	s.handler = newHandler(s.ctx, s.cluster, &node, "")
+	arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
+	s.handler = newHandler(s.ctx, s.cluster, "")
 }
 
 func (s *HandlerSuite) TearDownTest(c *check.C) {
diff --git a/lib/controller/server_test.go b/lib/controller/server_test.go
index ae89c3d7e..a398af97b 100644
--- a/lib/controller/server_test.go
+++ b/lib/controller/server_test.go
@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	check "gopkg.in/check.v1"
@@ -32,23 +33,19 @@ func integrationTestCluster() *arvados.Cluster {
 func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
 	log := ctxlog.TestLogger(c)
 
-	nodeProfile := arvados.NodeProfile{
-		Controller: arvados.SystemServiceInstance{Listen: ":"},
-		RailsAPI:   arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"), TLS: true, Insecure: true},
-	}
 	handler := &Handler{Cluster: &arvados.Cluster{
 		ClusterID:  "zzzzz",
 		PostgreSQL: integrationTestCluster().PostgreSQL,
-		NodeProfiles: map[string]arvados.NodeProfile{
-			"*": nodeProfile,
-		},
-	}, NodeProfile: &nodeProfile}
+		TLS:        arvados.TLS{Insecure: true},
+	}}
+	arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+	arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
 
 	srv := &httpserver.Server{
 		Server: http.Server{
 			Handler: httpserver.AddRequestIDs(httpserver.LogRequests(log, handler)),
 		},
-		Addr: nodeProfile.Controller.Listen,
+		Addr: ":",
 	}
 	return srv
 }
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index 22ceb8aeb..ae6ac70e9 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -15,10 +15,10 @@ import (
 
 var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
 
-func newHandler(ctx context.Context, cluster *arvados.Cluster, np *arvados.NodeProfile, token string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler {
 	ac, err := arvados.NewClientFromConfig(cluster)
 	if err != nil {
-		return service.ErrorHandler(ctx, cluster, np, fmt.Errorf("error initializing client from cluster config: %s", err))
+		return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
 	}
 	d := &dispatcher{
 		Cluster:   cluster,
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 024459ca0..cfc40778f 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -10,9 +10,12 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"strings"
 
 	"git.curoverse.com/arvados.git/lib/cmd"
 	"git.curoverse.com/arvados.git/lib/config"
@@ -28,7 +31,7 @@ type Handler interface {
 	CheckHealth() error
 }
 
-type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, token string) Handler
+type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler
 
 type command struct {
 	newHandler NewHandlerFunc
@@ -62,7 +65,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	flags := flag.NewFlagSet("", flag.ContinueOnError)
 	flags.SetOutput(stderr)
 	configFile := flags.String("config", arvados.DefaultConfigFile, "Site configuration `file`")
-	nodeProfile := flags.String("node-profile", "", "`Name` of NodeProfiles config entry to use (if blank, use $ARVADOS_NODE_PROFILE or hostname reported by OS)")
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -83,19 +85,10 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	})
 	ctx := ctxlog.Context(c.ctx, log)
 
-	profileName := *nodeProfile
-	if profileName == "" {
-		profileName = os.Getenv("ARVADOS_NODE_PROFILE")
-	}
-	profile, err := cluster.GetNodeProfile(profileName)
+	listen, err := getListenAddr(cluster.Services, c.svcName)
 	if err != nil {
 		return 1
 	}
-	listen := profile.ServicePorts()[c.svcName]
-	if listen == "" {
-		err = fmt.Errorf("configuration does not enable the %s service on this host", c.svcName)
-		return 1
-	}
 
 	if cluster.SystemRootToken == "" {
 		log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
@@ -114,7 +107,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 		}
 	}
 
-	handler := c.newHandler(ctx, cluster, profile, cluster.SystemRootToken)
+	handler := c.newHandler(ctx, cluster, cluster.SystemRootToken)
 	if err = handler.CheckHealth(); err != nil {
 		return 1
 	}
@@ -147,3 +140,32 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 }
 
 const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) {
+	svc, ok := map[arvados.ServiceName]arvados.Service{
+		arvados.ServiceNameController:    svcs.Controller,
+		arvados.ServiceNameDispatchCloud: svcs.DispatchCloud,
+		arvados.ServiceNameHealth:        svcs.Health,
+		arvados.ServiceNameKeepbalance:   svcs.Keepbalance,
+		arvados.ServiceNameKeepproxy:     svcs.Keepproxy,
+		arvados.ServiceNameKeepstore:     svcs.Keepstore,
+		arvados.ServiceNameKeepweb:       svcs.WebDAV,
+		arvados.ServiceNameWebsocket:     svcs.Websocket,
+	}[prog]
+	if !ok {
+		return "", fmt.Errorf("unknown service name %q", prog)
+	}
+	for url := range svc.InternalURLs {
+		if strings.HasPrefix(url.Host, "localhost:") {
+			return url.Host, nil
+		}
+		listener, err := net.Listen("tcp", url.Host)
+		if err == nil {
+			listener.Close()
+			return url.Host, nil
+		}
+		log.Print(err)
+
+	}
+	return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
+}
diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index 62960dc31..bb7c5c51d 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -38,7 +38,7 @@ func (*Suite) TestCommand(c *check.C) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, token string) Handler {
+	cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string) Handler {
 		c.Check(ctx.Value("foo"), check.Equals, "bar")
 		c.Check(token, check.Equals, "abcde")
 		return &testHandler{ctx: ctx, healthCheck: healthCheck}
diff --git a/lib/service/error.go b/lib/service/error.go
index 895521091..1ca5c5f44 100644
--- a/lib/service/error.go
+++ b/lib/service/error.go
@@ -17,7 +17,7 @@ import (
 // responds 500 to all requests.  ErrorHandler itself logs the given
 // error once, and the handler logs it again for each incoming
 // request.
-func ErrorHandler(ctx context.Context, _ *arvados.Cluster, _ *arvados.NodeProfile, err error) Handler {
+func ErrorHandler(ctx context.Context, _ *arvados.Cluster, err error) Handler {
 	logger := ctxlog.FromContext(ctx)
 	logger.WithError(err).Error("unhealthy service")
 	return errorHandler{err, logger}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b25164c3d..4936aa270 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -9,7 +9,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"os"
 
 	"git.curoverse.com/arvados.git/sdk/go/config"
 )
@@ -62,7 +61,6 @@ type Cluster struct {
 	ManagementToken string
 	SystemRootToken string
 	Services        Services
-	NodeProfiles    map[string]NodeProfile
 	InstanceTypes   InstanceTypeMap
 	Containers      ContainersConfig
 	RemoteClusters  map[string]RemoteCluster
@@ -234,51 +232,16 @@ func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-// GetNodeProfile returns a NodeProfile for the given hostname. An
-// error is returned if the appropriate configuration can't be
-// determined (e.g., this does not appear to be a system node). If
-// node is empty, use the OS-reported hostname.
-func (cc *Cluster) GetNodeProfile(node string) (*NodeProfile, error) {
-	if node == "" {
-		hostname, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-		node = hostname
-	}
-	if cfg, ok := cc.NodeProfiles[node]; ok {
-		return &cfg, nil
-	}
-	// If node is not listed, but "*" gives a default system node
-	// config, use the default config.
-	if cfg, ok := cc.NodeProfiles["*"]; ok {
-		return &cfg, nil
-	}
-	return nil, fmt.Errorf("config does not provision host %q as a system node", node)
-}
-
-type NodeProfile struct {
-	Controller    SystemServiceInstance `json:"arvados-controller"`
-	Health        SystemServiceInstance `json:"arvados-health"`
-	Keepbalance   SystemServiceInstance `json:"keep-balance"`
-	Keepproxy     SystemServiceInstance `json:"keepproxy"`
-	Keepstore     SystemServiceInstance `json:"keepstore"`
-	Keepweb       SystemServiceInstance `json:"keep-web"`
-	Nodemanager   SystemServiceInstance `json:"arvados-node-manager"`
-	DispatchCloud SystemServiceInstance `json:"arvados-dispatch-cloud"`
-	RailsAPI      SystemServiceInstance `json:"arvados-api-server"`
-	Websocket     SystemServiceInstance `json:"arvados-ws"`
-	Workbench     SystemServiceInstance `json:"arvados-workbench"`
-}
-
 type ServiceName string
 
 const (
 	ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 	ServiceNameController    ServiceName = "arvados-controller"
 	ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
+	ServiceNameHealth        ServiceName = "arvados-health"
 	ServiceNameNodemanager   ServiceName = "arvados-node-manager"
-	ServiceNameWorkbench     ServiceName = "arvados-workbench"
+	ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
+	ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 	ServiceNameWebsocket     ServiceName = "arvados-ws"
 	ServiceNameKeepbalance   ServiceName = "keep-balance"
 	ServiceNameKeepweb       ServiceName = "keep-web"
@@ -288,27 +251,23 @@ const (
 
 // ServicePorts returns the configured listening address (or "" if
 // disabled) for each service on the node.
-func (np *NodeProfile) ServicePorts() map[ServiceName]string {
-	return map[ServiceName]string{
-		ServiceNameRailsAPI:      np.RailsAPI.Listen,
-		ServiceNameController:    np.Controller.Listen,
-		ServiceNameDispatchCloud: np.DispatchCloud.Listen,
-		ServiceNameNodemanager:   np.Nodemanager.Listen,
-		ServiceNameWorkbench:     np.Workbench.Listen,
-		ServiceNameWebsocket:     np.Websocket.Listen,
-		ServiceNameKeepbalance:   np.Keepbalance.Listen,
-		ServiceNameKeepweb:       np.Keepweb.Listen,
-		ServiceNameKeepproxy:     np.Keepproxy.Listen,
-		ServiceNameKeepstore:     np.Keepstore.Listen,
+func (svcs Services) Map() map[ServiceName]Service {
+	return map[ServiceName]Service{
+		ServiceNameRailsAPI:      svcs.RailsAPI,
+		ServiceNameController:    svcs.Controller,
+		ServiceNameDispatchCloud: svcs.DispatchCloud,
+		ServiceNameHealth:        svcs.Health,
+		ServiceNameNodemanager:   svcs.Nodemanager,
+		ServiceNameWorkbench1:    svcs.Workbench1,
+		ServiceNameWorkbench2:    svcs.Workbench2,
+		ServiceNameWebsocket:     svcs.Websocket,
+		ServiceNameKeepbalance:   svcs.Keepbalance,
+		ServiceNameKeepweb:       svcs.WebDAV,
+		ServiceNameKeepproxy:     svcs.Keepproxy,
+		ServiceNameKeepstore:     svcs.Keepstore,
 	}
 }
 
-type SystemServiceInstance struct {
-	Listen   string
-	TLS      bool
-	Insecure bool
-}
-
 type TLS struct {
 	Certificate string
 	Key         string
diff --git a/sdk/go/arvadostest/stub.go b/sdk/go/arvadostest/stub.go
index 89925a957..6b24a38fd 100644
--- a/sdk/go/arvadostest/stub.go
+++ b/sdk/go/arvadostest/stub.go
@@ -6,6 +6,9 @@ package arvadostest
 
 import (
 	"net/http"
+	"net/url"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 )
 
 // StubResponse struct with response status and body
@@ -37,3 +40,15 @@ func (stub *ServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 		resp.Write([]byte(``))
 	}
 }
+
+// SetServiceURL overrides the given service config/discovery with the
+// given internalURL.
+//
+// SetServiceURL panics on errors.
+func SetServiceURL(service *arvados.Service, internalURL string) {
+	u, err := url.Parse(internalURL)
+	if err != nil {
+		panic(err)
+	}
+	service.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL(*u): {}}
+}
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 564331327..d8c0a4abf 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -9,8 +9,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
+	"net/url"
 	"sync"
 	"time"
 
@@ -113,46 +113,41 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 
 	mtx := sync.Mutex{}
 	wg := sync.WaitGroup{}
-	for profileName, profile := range cluster.NodeProfiles {
-		for svc, addr := range profile.ServicePorts() {
-			// Ensure svc is listed in resp.Services.
-			mtx.Lock()
-			if _, ok := resp.Services[svc]; !ok {
-				resp.Services[svc] = ServiceHealth{Health: "ERROR"}
-			}
-			mtx.Unlock()
-
-			if addr == "" {
-				// svc is not expected on this node.
-				continue
-			}
+	for svcName, svc := range cluster.Services.Map() {
+		// Ensure svc is listed in resp.Services.
+		mtx.Lock()
+		if _, ok := resp.Services[svcName]; !ok {
+			resp.Services[svcName] = ServiceHealth{Health: "ERROR"}
+		}
+		mtx.Unlock()
 
+		for addr := range svc.InternalURLs {
 			wg.Add(1)
-			go func(profileName string, svc arvados.ServiceName, addr string) {
+			go func(svcName arvados.ServiceName, addr arvados.URL) {
 				defer wg.Done()
 				var result CheckResult
-				url, err := agg.pingURL(profileName, addr)
+				pingURL, err := agg.pingURL(addr)
 				if err != nil {
 					result = CheckResult{
 						Health: "ERROR",
 						Error:  err.Error(),
 					}
 				} else {
-					result = agg.ping(url, cluster)
+					result = agg.ping(pingURL, cluster)
 				}
 
 				mtx.Lock()
 				defer mtx.Unlock()
-				resp.Checks[fmt.Sprintf("%s+%s", svc, url)] = result
+				resp.Checks[fmt.Sprintf("%s+%s", svcName, pingURL)] = result
 				if result.Health == "OK" {
-					h := resp.Services[svc]
+					h := resp.Services[svcName]
 					h.N++
 					h.Health = "OK"
-					resp.Services[svc] = h
+					resp.Services[svcName] = h
 				} else {
 					resp.Health = "ERROR"
 				}
-			}(profileName, svc, addr)
+			}(svcName, addr)
 		}
 	}
 	wg.Wait()
@@ -168,12 +163,12 @@ func (agg *Aggregator) ClusterHealth(cluster *arvados.Cluster) ClusterHealthResp
 	return resp
 }
 
-func (agg *Aggregator) pingURL(node, addr string) (string, error) {
-	_, port, err := net.SplitHostPort(addr)
-	return "http://" + node + ":" + port + "/_health/ping", err
+func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) {
+	base := url.URL(svcURL)
+	return base.Parse("/_health/ping")
 }
 
-func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckResult) {
+func (agg *Aggregator) ping(target *url.URL, cluster *arvados.Cluster) (result CheckResult) {
 	t0 := time.Now()
 
 	var err error
@@ -186,7 +181,7 @@ func (agg *Aggregator) ping(url string, cluster *arvados.Cluster) (result CheckR
 		}
 	}()
 
-	req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest("GET", target.String(), nil)
 	if err != nil {
 		return
 	}

commit 1167d410254ff8babfec9f2fdaae1958b368a1b9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu May 23 11:38:09 2019 -0400

    15003: Add dispatch-cloud configs to default/template file.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 363d7eb02..4d4937c47 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -21,11 +21,7 @@ Clusters:
     Services:
       RailsAPI:
         InternalURLs: {}
-      GitHTTP:
-        InternalURLs: {}
-        ExternalURL: ""
-      Keepstore:
-        InternalURLs: {}
+        ExternalURL: "-"
       Controller:
         InternalURLs: {}
         ExternalURL: ""
@@ -34,6 +30,7 @@ Clusters:
         ExternalURL: ""
       Keepbalance:
         InternalURLs: {}
+        ExternalURL: "-"
       GitHTTP:
         InternalURLs: {}
         ExternalURL: ""
@@ -41,6 +38,7 @@ Clusters:
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
+        ExternalURL: "-"
       SSO:
         ExternalURL: ""
       Keepproxy:
@@ -54,6 +52,7 @@ Clusters:
         ExternalURL: ""
       Keepstore:
         InternalURLs: {}
+        ExternalURL: "-"
       Composer:
         ExternalURL: ""
       WebShell:
@@ -63,6 +62,13 @@ Clusters:
         ExternalURL: ""
       Workbench2:
         ExternalURL: ""
+      Nodemanager:
+        InternalURLs: {}
+        ExternalURL: "-"
+      Health:
+        InternalURLs: {}
+        ExternalURL: "-"
+
     PostgreSQL:
       # max concurrent connections per arvados server daemon
       ConnectionPool: 32
@@ -118,6 +124,9 @@ Clusters:
       # site secret. It should be at least 50 characters.
       RailsSessionSecretToken: ""
 
+      # Maximum wall clock time to spend handling an incoming request.
+      RequestTimeout: 5m
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want
@@ -185,6 +194,14 @@ Clusters:
       UnloggedAttributes: []
 
     SystemLogs:
+
+      # Logging threshold: panic, fatal, error, warn, info, debug, or
+      # trace
+      LogLevel: info
+
+      # Logging format: json or text
+      Format: json
+
       # Maximum characters of (JSON-encoded) query parameters to include
       # in each request log entry. When params exceed this size, they will
       # be JSON-encoded, truncated to this size, and logged as
@@ -271,6 +288,8 @@ Clusters:
       Repositories: /var/lib/arvados/git/repositories
 
     TLS:
+      Certificate: ""
+      Key: ""
       Insecure: false
 
     Containers:
@@ -323,6 +342,16 @@ Clusters:
       # troubleshooting purposes.
       LogReuseDecisions: false
 
+      # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
+      # (experimental) cloud dispatcher for executing containers on
+      # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+      # and ends with "\n-----END RSA PRIVATE KEY-----\n".
+      DispatchPrivateKey: none
+
+      # Maximum time to wait for workers to come up before abandoning
+      # stale locks from a previous dispatch process.
+      StaleLockTimeout: 1m
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -445,6 +474,111 @@ Clusters:
         # original job reuse behavior, and is still the default).
         ReuseJobIfOutputsDiffer: false
 
+      CloudVMs:
+        # Enable the cloud scheduler (experimental).
+        Enable: false
+
+        # Name/number of port where workers' SSH services listen.
+        SSHPort: "22"
+
+        # Interval between queue polls.
+        PollInterval: 10s
+
+        # Shell command to execute on each worker to determine whether
+        # the worker is booted and ready to run containers. It should
+        # exit zero if the worker is ready.
+        BootProbeCommand: "docker ps"
+
+        # Minimum interval between consecutive probes to a single
+        # worker.
+        ProbeInterval: 10s
+
+        # Maximum probes per second, across all workers in a pool.
+        MaxProbesPerSecond: 10
+
+        # Time before repeating SIGTERM when killing a container.
+        TimeoutSignal: 5s
+
+        # Time to give up on SIGTERM and write off the worker.
+        TimeoutTERM: 2m
+
+        # Maximum create/destroy-instance operations per second (0 =
+        # unlimited).
+        MaxCloudOpsPerSecond: 0
+
+        # Interval between cloud provider syncs/updates ("list all
+        # instances").
+        SyncInterval: 1m
+
+        # Time to leave an idle worker running (in case new containers
+        # appear in the queue that it can run) before shutting it
+        # down.
+        TimeoutIdle: 1m
+
+        # Time to wait for a new worker to boot (i.e., pass
+        # BootProbeCommand) before giving up and shutting it down.
+        TimeoutBooting: 10m
+
+        # Maximum time a worker can stay alive with no successful
+        # probes before being automatically shut down.
+        TimeoutProbe: 10m
+
+        # Time after shutting down a worker to retry the
+        # shutdown/destroy operation.
+        TimeoutShutdown: 10s
+
+        # Worker VM image ID.
+        ImageID: ami-01234567890abcdef
+
+        # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS).
+        Driver: ec2
+
+        # Cloud-specific driver parameters.
+        DriverParameters:
+
+          # (ec2) Credentials.
+          AccessKeyID: ""
+          SecretAccessKey: ""
+
+          # (ec2) Instance configuration.
+          SecurityGroupIDs:
+            - ""
+          SubnetID: ""
+          Region: ""
+          EBSVolumeType: gp2
+          AdminUsername: debian
+
+          # (azure) Credentials.
+          SubscriptionID: ""
+          ClientID: ""
+          ClientSecret: ""
+          TenantID: ""
+
+          # (azure) Instance configuration.
+          CloudEnvironment: AzurePublicCloud
+          ResourceGroup: ""
+          Location: centralus
+          Network: ""
+          Subnet: ""
+          StorageAccount: ""
+          BlobContainer: ""
+          DeleteDanglingResourcesAfter: 20s
+          AdminUsername: arvados
+
+    InstanceTypes:
+
+      # Use the instance type name as the key (in place of "SAMPLE" in
+      # this sample entry).
+      SAMPLE:
+        # Cloud provider's instance type. Defaults to the configured type name.
+        ProviderType: ""
+        VCPUs: 1
+        RAM: 128MiB
+        IncludedScratch: 16GB
+        AddedScratch: 0
+        Price: 0.1
+        Preemptible: false
+
     Mail:
       MailchimpAPIKey: ""
       MailchimpListID: ""
@@ -455,7 +589,10 @@ Clusters:
       EmailFrom: ""
     RemoteClusters:
       "*":
+        Host: ""
         Proxy: false
+        Scheme: https
+        Insecure: false
         ActivateUsers: false
       SAMPLE:
         Host: sample.arvadosapi.com
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index e26f5f41a..4f89166ac 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -27,11 +27,7 @@ Clusters:
     Services:
       RailsAPI:
         InternalURLs: {}
-      GitHTTP:
-        InternalURLs: {}
-        ExternalURL: ""
-      Keepstore:
-        InternalURLs: {}
+        ExternalURL: "-"
       Controller:
         InternalURLs: {}
         ExternalURL: ""
@@ -40,6 +36,7 @@ Clusters:
         ExternalURL: ""
       Keepbalance:
         InternalURLs: {}
+        ExternalURL: "-"
       GitHTTP:
         InternalURLs: {}
         ExternalURL: ""
@@ -47,6 +44,7 @@ Clusters:
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
+        ExternalURL: "-"
       SSO:
         ExternalURL: ""
       Keepproxy:
@@ -60,6 +58,7 @@ Clusters:
         ExternalURL: ""
       Keepstore:
         InternalURLs: {}
+        ExternalURL: "-"
       Composer:
         ExternalURL: ""
       WebShell:
@@ -69,6 +68,13 @@ Clusters:
         ExternalURL: ""
       Workbench2:
         ExternalURL: ""
+      Nodemanager:
+        InternalURLs: {}
+        ExternalURL: "-"
+      Health:
+        InternalURLs: {}
+        ExternalURL: "-"
+
     PostgreSQL:
       # max concurrent connections per arvados server daemon
       ConnectionPool: 32
@@ -124,6 +130,9 @@ Clusters:
       # site secret. It should be at least 50 characters.
       RailsSessionSecretToken: ""
 
+      # Maximum wall clock time to spend handling an incoming request.
+      RequestTimeout: 5m
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want
@@ -191,6 +200,14 @@ Clusters:
       UnloggedAttributes: []
 
     SystemLogs:
+
+      # Logging threshold: panic, fatal, error, warn, info, debug, or
+      # trace
+      LogLevel: info
+
+      # Logging format: json or text
+      Format: json
+
       # Maximum characters of (JSON-encoded) query parameters to include
       # in each request log entry. When params exceed this size, they will
       # be JSON-encoded, truncated to this size, and logged as
@@ -277,6 +294,8 @@ Clusters:
       Repositories: /var/lib/arvados/git/repositories
 
     TLS:
+      Certificate: ""
+      Key: ""
       Insecure: false
 
     Containers:
@@ -329,6 +348,16 @@ Clusters:
       # troubleshooting purposes.
       LogReuseDecisions: false
 
+      # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
+      # (experimental) cloud dispatcher for executing containers on
+      # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+      # and ends with "\n-----END RSA PRIVATE KEY-----\n".
+      DispatchPrivateKey: none
+
+      # Maximum time to wait for workers to come up before abandoning
+      # stale locks from a previous dispatch process.
+      StaleLockTimeout: 1m
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -451,6 +480,111 @@ Clusters:
         # original job reuse behavior, and is still the default).
         ReuseJobIfOutputsDiffer: false
 
+      CloudVMs:
+        # Enable the cloud scheduler (experimental).
+        Enable: false
+
+        # Name/number of port where workers' SSH services listen.
+        SSHPort: "22"
+
+        # Interval between queue polls.
+        PollInterval: 10s
+
+        # Shell command to execute on each worker to determine whether
+        # the worker is booted and ready to run containers. It should
+        # exit zero if the worker is ready.
+        BootProbeCommand: "docker ps"
+
+        # Minimum interval between consecutive probes to a single
+        # worker.
+        ProbeInterval: 10s
+
+        # Maximum probes per second, across all workers in a pool.
+        MaxProbesPerSecond: 10
+
+        # Time before repeating SIGTERM when killing a container.
+        TimeoutSignal: 5s
+
+        # Time to give up on SIGTERM and write off the worker.
+        TimeoutTERM: 2m
+
+        # Maximum create/destroy-instance operations per second (0 =
+        # unlimited).
+        MaxCloudOpsPerSecond: 0
+
+        # Interval between cloud provider syncs/updates ("list all
+        # instances").
+        SyncInterval: 1m
+
+        # Time to leave an idle worker running (in case new containers
+        # appear in the queue that it can run) before shutting it
+        # down.
+        TimeoutIdle: 1m
+
+        # Time to wait for a new worker to boot (i.e., pass
+        # BootProbeCommand) before giving up and shutting it down.
+        TimeoutBooting: 10m
+
+        # Maximum time a worker can stay alive with no successful
+        # probes before being automatically shut down.
+        TimeoutProbe: 10m
+
+        # Time after shutting down a worker to retry the
+        # shutdown/destroy operation.
+        TimeoutShutdown: 10s
+
+        # Worker VM image ID.
+        ImageID: ami-01234567890abcdef
+
+        # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS).
+        Driver: ec2
+
+        # Cloud-specific driver parameters.
+        DriverParameters:
+
+          # (ec2) Credentials.
+          AccessKeyID: ""
+          SecretAccessKey: ""
+
+          # (ec2) Instance configuration.
+          SecurityGroupIDs:
+            - ""
+          SubnetID: ""
+          Region: ""
+          EBSVolumeType: gp2
+          AdminUsername: debian
+
+          # (azure) Credentials.
+          SubscriptionID: ""
+          ClientID: ""
+          ClientSecret: ""
+          TenantID: ""
+
+          # (azure) Instance configuration.
+          CloudEnvironment: AzurePublicCloud
+          ResourceGroup: ""
+          Location: centralus
+          Network: ""
+          Subnet: ""
+          StorageAccount: ""
+          BlobContainer: ""
+          DeleteDanglingResourcesAfter: 20s
+          AdminUsername: arvados
+
+    InstanceTypes:
+
+      # Use the instance type name as the key (in place of "SAMPLE" in
+      # this sample entry).
+      SAMPLE:
+        # Cloud provider's instance type. Defaults to the configured type name.
+        ProviderType: ""
+        VCPUs: 1
+        RAM: 128MiB
+        IncludedScratch: 16GB
+        AddedScratch: 0
+        Price: 0.1
+        Preemptible: false
+
     Mail:
       MailchimpAPIKey: ""
       MailchimpListID: ""
@@ -461,7 +595,10 @@ Clusters:
       EmailFrom: ""
     RemoteClusters:
       "*":
+        Host: ""
         Proxy: false
+        Scheme: https
+        Insecure: false
         ActivateUsers: false
       SAMPLE:
         Host: sample.arvadosapi.com
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index bbcc45a3f..ed1dd1bdf 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -97,6 +97,20 @@ Clusters:
 	c.Check(logs, check.HasLen, 2)
 }
 
+func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
+	var logbuf bytes.Buffer
+	logger := logrus.New()
+	logger.Out = &logbuf
+	cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"zzzzz":{}}}`), logger)
+	c.Assert(err, check.IsNil)
+	yaml, err := yaml.Marshal(cfg)
+	c.Assert(err, check.IsNil)
+	cfgDumped, err := Load(bytes.NewBuffer(yaml), logger)
+	c.Assert(err, check.IsNil)
+	c.Check(cfg, check.DeepEquals, cfgDumped)
+	c.Check(logbuf.String(), check.Equals, "")
+}
+
 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 	_, err := Load(bytes.NewBufferString(`
 Clusters:
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 775d29034..35734d780 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -50,8 +50,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			req.URL.Path = strings.Replace(req.URL.Path, "//", "/", -1)
 		}
 	}
-	if h.Cluster.HTTPRequestTimeout > 0 {
-		ctx, cancel := context.WithDeadline(req.Context(), time.Now().Add(time.Duration(h.Cluster.HTTPRequestTimeout)))
+	if h.Cluster.API.RequestTimeout > 0 {
+		ctx, cancel := context.WithDeadline(req.Context(), time.Now().Add(time.Duration(h.Cluster.API.RequestTimeout)))
 		req = req.WithContext(ctx)
 		defer cancel()
 	}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 96110ea85..01544a2b0 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -72,7 +72,7 @@ func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 }
 
 func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
-	s.cluster.HTTPRequestTimeout = arvados.Duration(time.Nanosecond)
+	s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
 	req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 	resp := httptest.NewRecorder()
 	s.handler.ServeHTTP(resp, req)
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 4b7341d72..024459ca0 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -78,7 +78,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	if err != nil {
 		return 1
 	}
-	log = ctxlog.New(stderr, cluster.Logging.Format, cluster.Logging.Level).WithFields(logrus.Fields{
+	log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
 		"PID": os.Getpid(),
 	})
 	ctx := ctxlog.Context(c.ctx, log)
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6b3150c6f..b25164c3d 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -54,23 +54,22 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
 type API struct {
 	MaxItemsPerResponse     int
 	MaxRequestAmplification int
+	RequestTimeout          Duration
 }
 
 type Cluster struct {
-	ClusterID          string `json:"-"`
-	ManagementToken    string
-	SystemRootToken    string
-	Services           Services
-	NodeProfiles       map[string]NodeProfile
-	InstanceTypes      InstanceTypeMap
-	CloudVMs           CloudVMs
-	Dispatch           Dispatch
-	HTTPRequestTimeout Duration
-	RemoteClusters     map[string]RemoteCluster
-	PostgreSQL         PostgreSQL
-	API                API
-	Logging            Logging
-	TLS                TLS
+	ClusterID       string `json:"-"`
+	ManagementToken string
+	SystemRootToken string
+	Services        Services
+	NodeProfiles    map[string]NodeProfile
+	InstanceTypes   InstanceTypeMap
+	Containers      ContainersConfig
+	RemoteClusters  map[string]RemoteCluster
+	PostgreSQL      PostgreSQL
+	API             API
+	SystemLogs      SystemLogs
+	TLS             TLS
 }
 
 type Services struct {
@@ -89,7 +88,7 @@ type Services struct {
 }
 
 type Service struct {
-	InternalURLs map[URL]ServiceInstance
+	InternalURLs map[URL]ServiceInstance `json:",omitempty"`
 	ExternalURL  URL
 }
 
@@ -112,9 +111,10 @@ func (su URL) MarshalText() ([]byte, error) {
 
 type ServiceInstance struct{}
 
-type Logging struct {
-	Level  string
-	Format string
+type SystemLogs struct {
+	LogLevel                string
+	Format                  string
+	MaxRequestLogParamsSize int
 }
 
 type PostgreSQL struct {
@@ -148,59 +148,29 @@ type InstanceType struct {
 	Preemptible     bool
 }
 
-type Dispatch struct {
-	// PEM encoded SSH key (RSA, DSA, or ECDSA) able to log in to
-	// cloud VMs.
-	PrivateKey string
-
-	// Max time for workers to come up before abandoning stale
-	// locks from previous run
-	StaleLockTimeout Duration
-
-	// Interval between queue polls
-	PollInterval Duration
-
-	// Interval between probes to each worker
-	ProbeInterval Duration
-
-	// Maximum total worker probes per second
-	MaxProbesPerSecond int
-
-	// Time before repeating SIGTERM when killing a container
-	TimeoutSignal Duration
-
-	// Time to give up on SIGTERM and write off the worker
-	TimeoutTERM Duration
+type ContainersConfig struct {
+	CloudVMs           CloudVMsConfig
+	DispatchPrivateKey string
+	StaleLockTimeout   Duration
 }
 
-type CloudVMs struct {
-	// Shell command that exits zero IFF the VM is fully booted
-	// and ready to run containers, e.g., "mount | grep
-	// /encrypted-tmp"
-	BootProbeCommand string
-
-	// Listening port (name or number) of SSH servers on worker
-	// VMs
-	SSHPort string
+type CloudVMsConfig struct {
+	Enable bool
 
-	SyncInterval Duration
-
-	// Maximum idle time before automatic shutdown
-	TimeoutIdle Duration
-
-	// Maximum booting time before automatic shutdown
-	TimeoutBooting Duration
-
-	// Maximum time with no successful probes before automatic shutdown
-	TimeoutProbe Duration
-
-	// Time after shutdown to retry shutdown
-	TimeoutShutdown Duration
-
-	// Maximum create/destroy-instance operations per second
+	BootProbeCommand     string
+	ImageID              string
 	MaxCloudOpsPerSecond int
-
-	ImageID string
+	MaxProbesPerSecond   int
+	PollInterval         Duration
+	ProbeInterval        Duration
+	SSHPort              string
+	SyncInterval         Duration
+	TimeoutBooting       Duration
+	TimeoutIdle          Duration
+	TimeoutProbe         Duration
+	TimeoutShutdown      Duration
+	TimeoutSignal        Duration
+	TimeoutTERM          Duration
 
 	Driver           string
 	DriverParameters json.RawMessage
diff --git a/sdk/go/arvados/duration.go b/sdk/go/arvados/duration.go
index 25eed010f..d3e11c7a5 100644
--- a/sdk/go/arvados/duration.go
+++ b/sdk/go/arvados/duration.go
@@ -23,7 +23,7 @@ func (d *Duration) UnmarshalJSON(data []byte) error {
 }
 
 // MarshalJSON implements json.Marshaler.
-func (d *Duration) MarshalJSON() ([]byte, error) {
+func (d Duration) MarshalJSON() ([]byte, error) {
 	return json.Marshal(d.String())
 }
 
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 79767c2fa..c2ad7c892 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -414,7 +414,8 @@ def run_controller():
 Clusters:
   zzzzz:
     ManagementToken: e687950a23c3a9bceec28c6223a06c79
-    HTTPRequestTimeout: 30s
+    API:
+      RequestTimeout: 30s
     PostgreSQL:
       ConnectionPool: 32
       Connection:

commit c43d9e862d65065c05ee1ef4e6f2ba7d7ec28ed9
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu May 23 00:24:14 2019 -0400

    15003: Fix warnings about site-specific keys.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 16d6c61b5..363d7eb02 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -457,3 +457,9 @@ Clusters:
       "*":
         Proxy: false
         ActivateUsers: false
+      SAMPLE:
+        Host: sample.arvadosapi.com
+        Proxy: false
+        Scheme: https
+        Insecure: false
+        ActivateUsers: false
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 5ee62ee82..e26f5f41a 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -463,4 +463,10 @@ Clusters:
       "*":
         Proxy: false
         ActivateUsers: false
+      SAMPLE:
+        Host: sample.arvadosapi.com
+        Proxy: false
+        Scheme: https
+        Insecure: false
+        ActivateUsers: false
 `)
diff --git a/lib/config/load.go b/lib/config/load.go
index 526a050fb..3ed2b9928 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -79,6 +79,7 @@ func load(rdr io.Reader, log logger, useDeprecated bool) (*arvados.Config, error
 		return nil, fmt.Errorf("loading config data: %s", err)
 	}
 	logExtraKeys(log, merged, src, "")
+	removeSampleKeys(merged)
 	err = mergo.Merge(&merged, src, mergo.WithOverride)
 	if err != nil {
 		return nil, fmt.Errorf("merging config data: %s", err)
@@ -129,14 +130,32 @@ func checkKeyConflict(label string, m map[string]string) error {
 	return nil
 }
 
+func removeSampleKeys(m map[string]interface{}) {
+	delete(m, "SAMPLE")
+	for _, v := range m {
+		if v, _ := v.(map[string]interface{}); v != nil {
+			removeSampleKeys(v)
+		}
+	}
+}
+
 func logExtraKeys(log logger, expected, supplied map[string]interface{}, prefix string) {
 	if log == nil {
 		return
 	}
+	allowed := map[string]interface{}{}
+	for k, v := range expected {
+		allowed[strings.ToLower(k)] = v
+	}
 	for k, vsupp := range supplied {
-		if vexp, ok := expected[k]; !ok {
+		vexp, ok := allowed[strings.ToLower(k)]
+		if !ok && expected["SAMPLE"] != nil {
+			vexp = expected["SAMPLE"]
+		} else if !ok {
 			log.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
-		} else if vsupp, ok := vsupp.(map[string]interface{}); !ok {
+			continue
+		}
+		if vsupp, ok := vsupp.(map[string]interface{}); !ok {
 			// if vsupp is a map but vexp isn't map, this
 			// will be caught elsewhere; see TestBadType.
 			continue
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 2bf341fe1..bbcc45a3f 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -9,10 +9,13 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 	"testing"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"github.com/ghodss/yaml"
+	"github.com/sirupsen/logrus"
 	check "gopkg.in/check.v1"
 )
 
@@ -42,6 +45,23 @@ func (s *LoadSuite) TestNoConfigs(c *check.C) {
 	c.Check(cc.API.MaxItemsPerResponse, check.Equals, 1000)
 }
 
+func (s *LoadSuite) TestSampleKeys(c *check.C) {
+	for _, yaml := range []string{
+		`{"Clusters":{"z1111":{}}}`,
+		`{"Clusters":{"z1111":{"InstanceTypes":{"Foo":{"RAM": "12345M"}}}}}`,
+	} {
+		cfg, err := Load(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
+		c.Assert(err, check.IsNil)
+		cc, err := cfg.GetCluster("z1111")
+		_, hasSample := cc.InstanceTypes["SAMPLE"]
+		c.Check(hasSample, check.Equals, false)
+		if strings.Contains(yaml, "Foo") {
+			c.Check(cc.InstanceTypes["Foo"].RAM, check.Equals, arvados.ByteSize(12345000000))
+			c.Check(cc.InstanceTypes["Foo"].Price, check.Equals, 0.0)
+		}
+	}
+}
+
 func (s *LoadSuite) TestMultipleClusters(c *check.C) {
 	cfg, err := Load(bytes.NewBufferString(`{"Clusters":{"z1111":{},"z2222":{}}}`), ctxlog.TestLogger(c))
 	c.Assert(err, check.IsNil)
@@ -53,6 +73,30 @@ func (s *LoadSuite) TestMultipleClusters(c *check.C) {
 	c.Check(c2.ClusterID, check.Equals, "z2222")
 }
 
+func (s *LoadSuite) TestDeprecatedOrUnknownWarning(c *check.C) {
+	var logbuf bytes.Buffer
+	logger := logrus.New()
+	logger.Out = &logbuf
+	_, err := Load(bytes.NewBufferString(`
+Clusters:
+  zzzzz:
+    postgresql: {}
+    BadKey: {}
+    Containers: {}
+    RemoteClusters:
+      z2222:
+        Host: z2222.arvadosapi.com
+        Proxy: true
+        BadKey: badValue
+`), logger)
+	c.Assert(err, check.IsNil)
+	logs := strings.Split(strings.TrimSuffix(logbuf.String(), "\n"), "\n")
+	for _, log := range logs {
+		c.Check(log, check.Matches, `.*deprecated or unknown config entry:.*BadKey.*`)
+	}
+	c.Check(logs, check.HasLen, 2)
+}
+
 func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
 	_, err := Load(bytes.NewBufferString(`
 Clusters:

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list