[ARVADOS] updated: 1.3.0-1455-g8654bb498

Git user git at public.curoverse.com
Tue Aug 20 13:27:40 UTC 2019


Summary of changes:
 lib/config/cmd_test.go                  |   2 +-
 lib/config/config.default.yml           |   5 +-
 lib/config/deprecated_keepstore.go      |  76 +++++++++-
 lib/config/deprecated_keepstore_test.go |  45 ++++++
 lib/config/generated_config.go          |   5 +-
 lib/dispatchcloud/cmd.go                |   4 +-
 lib/dispatchcloud/dispatcher.go         |  11 +-
 lib/service/cmd.go                      |  62 +++++---
 lib/service/tls.go                      |  80 ++++++++++
 sdk/go/arvados/config.go                |   4 +
 sdk/go/ctxlog/log.go                    |   2 +-
 sdk/go/httpserver/httpserver.go         |   7 +-
 services/keepstore/azure_blob_volume.go | 249 ++++++++++++++------------------
 services/keepstore/bufferpool.go        |  11 +-
 services/keepstore/command.go           | 177 +++++++++--------------
 services/keepstore/config.go            |  84 -----------
 services/keepstore/handlers.go          |  92 ++++++------
 services/keepstore/keepstore.go         |  21 +--
 services/keepstore/perms.go             |  12 +-
 services/keepstore/proxy_remote.go      |   4 +-
 services/keepstore/pull_worker.go       |  50 +++----
 services/keepstore/s3_volume.go         | 122 +++++++---------
 services/keepstore/server.go            |  78 ----------
 services/keepstore/trash_worker.go      |  19 +--
 services/keepstore/unix_volume.go       | 150 ++++---------------
 services/keepstore/volume.go            |  82 ++++++-----
 26 files changed, 665 insertions(+), 789 deletions(-)
 create mode 100644 lib/service/tls.go
 delete mode 100644 services/keepstore/config.go
 delete mode 100644 services/keepstore/server.go

  discards  587f6f4239087291f617159cc1daad468b949345 (commit)
       via  8654bb498573256ac622d513bbfa3509f0b09e9d (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (587f6f4239087291f617159cc1daad468b949345)
            \
             N -- N -- N (8654bb498573256ac622d513bbfa3509f0b09e9d)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8654bb498573256ac622d513bbfa3509f0b09e9d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Aug 19 17:15:40 2019 -0400

    13647: Use cluster config instead of custom keepstore config.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index af7c57120..019d5edd0 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -85,7 +85,7 @@ func (s *CommandSuite) TestCheckOldKeepstoreConfigFile(c *check.C) {
 	c.Assert(err, check.IsNil)
 	defer os.Remove(f.Name())
 
-	io.WriteString(f, "Debug: true\n")
+	io.WriteString(f, "Listen: :12345\nDebug: true\n")
 
 	var stdout, stderr bytes.Buffer
 	in := `
@@ -97,7 +97,7 @@ Clusters:
 	code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-", "-legacy-keepstore-config", f.Name()}, bytes.NewBufferString(in), &stdout, &stderr)
 	c.Check(code, check.Equals, 1)
 	c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*LogLevel: info\n\+ +LogLevel: debug\n.*`)
-	c.Check(stderr.String(), check.Matches, `.*you should remove the legacy keepstore config file.*\n`)
+	c.Check(stderr.String(), check.Matches, `(?ms).*you should remove the legacy keepstore config file.*\n`)
 }
 
 func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 9ac4aeeb9..17556fdde 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -176,6 +176,15 @@ Clusters:
       # parameter higher than this value, this value is used instead.
       MaxItemsPerResponse: 1000
 
+      # Maximum number of concurrent requests to accept in a single
+      # service process, or 0 for no limit. Currently supported only
+      # by keepstore.
+      MaxConcurrentRequests: 0
+
+      # Maximum number of 64MiB memory buffers per keepstore server
+      # process, or 0 for no limit.
+      MaxKeepBlockBuffers: 128
+
       # API methods to disable. Disabled methods are not listed in the
       # discovery document, and respond 404 to all requests.
       # Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -313,15 +322,44 @@ Clusters:
 
       # BlobSigningKey is a string of alphanumeric characters used to
       # generate permission signatures for Keep locators. It must be
-      # identical to the permission key given to Keep. IMPORTANT: This is
-      # a site secret. It should be at least 50 characters.
+      # identical to the permission key given to Keep. IMPORTANT: This
+      # is a site secret. It should be at least 50 characters.
       #
-      # Modifying blob_signing_key will invalidate all existing
+      # Modifying BlobSigningKey will invalidate all existing
       # signatures, which can cause programs to fail (e.g., arv-put,
-      # arv-get, and Crunch jobs).  To avoid errors, rotate keys only when
-      # no such processes are running.
+      # arv-get, and Crunch jobs).  To avoid errors, rotate keys only
+      # when no such processes are running.
       BlobSigningKey: ""
 
+      # Enable garbage collection of unreferenced blobs in Keep.
+      BlobTrash: true
+
+      # Time to leave unreferenced blobs in "trashed" state before
+      # deleting them, or 0 to skip the "trashed" state entirely and
+      # delete unreferenced blobs.
+      #
+      # If you use any Amazon S3 buckets as storage volumes, this
+      # must be at least 24h to avoid occasional data loss.
+      BlobTrashLifetime: 336h
+
+      # How often to check for (and delete) trashed blocks whose
+      # BlobTrashLifetime has expired.
+      BlobTrashCheckInterval: 24h
+
+      # Maximum number of concurrent "trash blob" and "delete trashed
+      # blob" operations conducted by a single keepstore process. Each
+      # of these can be set to 0 to disable the respective operation.
+      #
+      # If BlobTrashLifetime is zero, "trash" and "delete trash"
+      # happen at once, so only the lower of these two values is used.
+      BlobTrashConcurrency: 4
+      BlobDeleteConcurrency: 4
+
+      # Maximum number of concurrent "create additional replica of
+      # existing blob" operations conducted by a single keepstore
+      # process.
+      BlobReplicateConcurrency: 4
+
       # Default replication level for collections. This is used when a
       # collection's replication_desired attribute is nil.
       DefaultReplication: 2
@@ -730,6 +768,47 @@ Clusters:
         Price: 0.1
         Preemptible: false
 
+    Volumes:
+      SAMPLE:
+        AccessViaHosts:
+          SAMPLE:
+            ReadOnly: false
+        ReadOnly: false
+        Replication: 1
+        StorageClasses:
+          default: true
+          SAMPLE: true
+        Driver: s3
+        DriverParameters:
+
+          # for s3 driver
+          AccessKey: aaaaa
+          SecretKey: aaaaa
+          Endpoint: ""
+          Region: us-east-1a
+          Bucket: aaaaa
+          LocationConstraint: false
+          IndexPageSize: 1000
+          ConnectTimeout: 1m
+          ReadTimeout: 10m
+          RaceWindow: 24h
+          UnsafeDelete: false
+
+          # for azure driver
+          StorageAccountName: aaaaa
+          StorageAccountKey: aaaaa
+          StorageBaseURL: core.windows.net
+          ContainerName: aaaaa
+          RequestTimeout: 30s
+          ListBlobsRetryDelay: 10s
+          ListBlobsMaxAttempts: 10
+          MaxGetBytes: 0
+          WriteRaceInterval: 15s
+          WriteRacePollTime: 1s
+
+          # for local directory driver
+          Root: /var/lib/arvados/keep-data
+
     Mail:
       MailchimpAPIKey: ""
       MailchimpListID: ""
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 12581ddff..10dfc98ae 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -102,12 +102,6 @@ func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc
 	svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
 }
 
-const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
-
-type oldKeepstoreConfig struct {
-	Debug *bool
-}
-
 func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
 	if path == "" {
 		return nil
@@ -126,35 +120,6 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
 	return nil
 }
 
-// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
-	if ldr.KeepstorePath == "" {
-		return nil
-	}
-	var oc oldKeepstoreConfig
-	err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
-	if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	cluster, err := cfg.GetCluster("")
-	if err != nil {
-		return err
-	}
-
-	if v := oc.Debug; v == nil {
-	} else if *v && cluster.SystemLogs.LogLevel != "debug" {
-		cluster.SystemLogs.LogLevel = "debug"
-	} else if !*v && cluster.SystemLogs.LogLevel != "info" {
-		cluster.SystemLogs.LogLevel = "info"
-	}
-
-	cfg.Clusters[cluster.ClusterID] = *cluster
-	return nil
-}
-
 type oldCrunchDispatchSlurmConfig struct {
 	Client *arvados.Client
 
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
new file mode 100644
index 000000000..faefca569
--- /dev/null
+++ b/lib/config/deprecated_keepstore.go
@@ -0,0 +1,544 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"github.com/sirupsen/logrus"
+)
+
+const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
+
+type oldKeepstoreConfig struct {
+	Debug  *bool
+	Listen *string
+
+	LogFormat *string
+
+	PIDFile *string
+
+	MaxBuffers  *int
+	MaxRequests *int
+
+	BlobSignatureTTL    *arvados.Duration
+	BlobSigningKeyFile  *string
+	RequireSignatures   *bool
+	SystemAuthTokenFile *string
+	EnableDelete        *bool
+	TrashLifetime       *arvados.Duration
+	TrashCheckInterval  *arvados.Duration
+	PullWorkers         *int
+	TrashWorkers        *int
+	EmptyTrashWorkers   *int
+	TLSCertificateFile  *string
+	TLSKeyFile          *string
+
+	Volumes *oldKeepstoreVolumeList
+
+	ManagementToken *string
+
+	DiscoverVolumesFromMountsFile string // not a real legacy config -- just useful for tests
+}
+
+type oldKeepstoreVolumeList []oldKeepstoreVolume
+
+type oldKeepstoreVolume struct {
+	arvados.Volume
+	Type string `json:",omitempty"`
+
+	// Azure driver configs
+	StorageAccountName    string           `json:",omitempty"`
+	StorageAccountKeyFile string           `json:",omitempty"`
+	StorageBaseURL        string           `json:",omitempty"`
+	ContainerName         string           `json:",omitempty"`
+	AzureReplication      int              `json:",omitempty"`
+	RequestTimeout        arvados.Duration `json:",omitempty"`
+	ListBlobsRetryDelay   arvados.Duration `json:",omitempty"`
+	ListBlobsMaxAttempts  int              `json:",omitempty"`
+
+	// S3 driver configs
+	AccessKeyFile      string           `json:",omitempty"`
+	SecretKeyFile      string           `json:",omitempty"`
+	Endpoint           string           `json:",omitempty"`
+	Region             string           `json:",omitempty"`
+	Bucket             string           `json:",omitempty"`
+	LocationConstraint bool             `json:",omitempty"`
+	IndexPageSize      int              `json:",omitempty"`
+	S3Replication      int              `json:",omitempty"`
+	ConnectTimeout     arvados.Duration `json:",omitempty"`
+	ReadTimeout        arvados.Duration `json:",omitempty"`
+	RaceWindow         arvados.Duration `json:",omitempty"`
+	UnsafeDelete       bool             `json:",omitempty"`
+
+	// Directory driver configs
+	Root                 string
+	DirectoryReplication int
+	Serialize            bool
+
+	// Common configs
+	ReadOnly       bool     `json:",omitempty"`
+	StorageClasses []string `json:",omitempty"`
+}
+
+// update config using values from an old-style keepstore config file.
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return fmt.Errorf("getting hostname: %s", err)
+	}
+
+	var oc oldKeepstoreConfig
+	err = ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
+	if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
+	}
+
+	myURL := arvados.URL{Scheme: "http"}
+	if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
+		myURL.Scheme = "https"
+	}
+
+	if v := oc.Debug; v == nil {
+	} else if *v && cluster.SystemLogs.LogLevel != "debug" {
+		cluster.SystemLogs.LogLevel = "debug"
+	} else if !*v && cluster.SystemLogs.LogLevel != "info" {
+		cluster.SystemLogs.LogLevel = "info"
+	}
+
+	if v := oc.TLSCertificateFile; v != nil && "file://"+*v != cluster.TLS.Certificate {
+		cluster.TLS.Certificate = "file://" + *v
+	}
+	if v := oc.TLSKeyFile; v != nil && "file://"+*v != cluster.TLS.Key {
+		cluster.TLS.Key = "file://" + *v
+	}
+	if v := oc.Listen; v != nil {
+		if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+			// already listed
+			myURL.Host = *v
+		} else if len(*v) > 1 && (*v)[0] == ':' {
+			myURL.Host = net.JoinHostPort(hostname, (*v)[1:])
+			cluster.Services.Keepstore.InternalURLs[myURL] = arvados.ServiceInstance{}
+		} else {
+			return fmt.Errorf("unable to migrate Listen value %q from legacy keepstore config file -- remove after configuring Services.Keepstore.InternalURLs.", *v)
+		}
+	} else {
+		for url := range cluster.Services.Keepstore.InternalURLs {
+			if host, _, _ := net.SplitHostPort(url.Host); host == hostname {
+				myURL = url
+				break
+			}
+		}
+		if myURL.Host == "" {
+			return fmt.Errorf("unable to migrate legacy keepstore config: no 'Listen' key, and hostname %q does not match an entry in Services.Keepstore.InternalURLs", hostname)
+		}
+	}
+
+	if v := oc.LogFormat; v != nil && *v != cluster.SystemLogs.Format {
+		cluster.SystemLogs.Format = *v
+	}
+	if v := oc.MaxBuffers; v != nil && *v != cluster.API.MaxKeepBlockBuffers {
+		cluster.API.MaxKeepBlockBuffers = *v
+	}
+	if v := oc.MaxRequests; v != nil && *v != cluster.API.MaxConcurrentRequests {
+		cluster.API.MaxConcurrentRequests = *v
+	}
+	if v := oc.BlobSignatureTTL; v != nil && *v != cluster.Collections.BlobSigningTTL {
+		cluster.Collections.BlobSigningTTL = *v
+	}
+	if v := oc.BlobSigningKeyFile; v != nil {
+		buf, err := ioutil.ReadFile(*v)
+		if err != nil {
+			return fmt.Errorf("error reading BlobSigningKeyFile: %s", err)
+		}
+		if key := strings.TrimSpace(string(buf)); key != cluster.Collections.BlobSigningKey {
+			cluster.Collections.BlobSigningKey = key
+		}
+	}
+	if v := oc.RequireSignatures; v != nil && *v != cluster.Collections.BlobSigning {
+		cluster.Collections.BlobSigning = *v
+	}
+	if v := oc.SystemAuthTokenFile; v != nil {
+		f, err := os.Open(*v)
+		if err != nil {
+			return fmt.Errorf("error opening SystemAuthTokenFile: %s", err)
+		}
+		defer f.Close()
+		buf, err := ioutil.ReadAll(f)
+		if err != nil {
+			return fmt.Errorf("error reading SystemAuthTokenFile: %s", err)
+		}
+		if key := strings.TrimSpace(string(buf)); key != cluster.SystemRootToken {
+			cluster.SystemRootToken = key
+		}
+	}
+	if v := oc.EnableDelete; v != nil && *v != cluster.Collections.BlobTrash {
+		cluster.Collections.BlobTrash = *v
+	}
+	if v := oc.TrashLifetime; v != nil && *v != cluster.Collections.BlobTrashLifetime {
+		cluster.Collections.BlobTrashLifetime = *v
+	}
+	if v := oc.TrashCheckInterval; v != nil && *v != cluster.Collections.BlobTrashCheckInterval {
+		cluster.Collections.BlobTrashCheckInterval = *v
+	}
+	if v := oc.TrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+		cluster.Collections.BlobTrashConcurrency = *v
+	}
+	if v := oc.EmptyTrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+		cluster.Collections.BlobDeleteConcurrency = *v
+	}
+	if v := oc.PullWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+		cluster.Collections.BlobReplicateConcurrency = *v
+	}
+	if v := oc.Volumes; v == nil {
+		ldr.Logger.Warn("no volumes in legacy config; discovering local directory volumes")
+		err := ldr.discoverLocalVolumes(cluster, oc.DiscoverVolumesFromMountsFile, myURL)
+		if err != nil {
+			return fmt.Errorf("error discovering local directory volumes: %s", err)
+		}
+	} else {
+		for i, oldvol := range *v {
+			var accessViaHosts map[arvados.URL]arvados.VolumeAccess
+			oldUUID, found := ldr.alreadyMigrated(oldvol, cluster.Volumes, myURL)
+			if found {
+				accessViaHosts = cluster.Volumes[oldUUID].AccessViaHosts
+				writers := false
+				for _, va := range accessViaHosts {
+					if !va.ReadOnly {
+						writers = true
+					}
+				}
+				if writers || len(accessViaHosts) == 0 {
+					ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+					if len(accessViaHosts) > 0 {
+						cluster.Volumes[oldUUID].AccessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+					}
+					continue
+				}
+			}
+			var newvol arvados.Volume
+			if found {
+				ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+				newvol = cluster.Volumes[oldUUID]
+				// Remove the old entry. It will be
+				// added back below, possibly with a
+				// new UUID.
+				delete(cluster.Volumes, oldUUID)
+			} else {
+				var params interface{}
+				switch oldvol.Type {
+				case "S3":
+					accesskeydata, err := ioutil.ReadFile(oldvol.AccessKeyFile)
+					if err != nil && oldvol.AccessKeyFile != "" {
+						return fmt.Errorf("error reading AccessKeyFile: %s", err)
+					}
+					secretkeydata, err := ioutil.ReadFile(oldvol.SecretKeyFile)
+					if err != nil && oldvol.SecretKeyFile != "" {
+						return fmt.Errorf("error reading SecretKeyFile: %s", err)
+					}
+					newvol = arvados.Volume{
+						Driver:         "S3",
+						ReadOnly:       oldvol.ReadOnly,
+						Replication:    oldvol.S3Replication,
+						StorageClasses: array2boolmap(oldvol.StorageClasses),
+					}
+					params = arvados.S3VolumeDriverParameters{
+						AccessKey:          string(bytes.TrimSpace(accesskeydata)),
+						SecretKey:          string(bytes.TrimSpace(secretkeydata)),
+						Endpoint:           oldvol.Endpoint,
+						Region:             oldvol.Region,
+						Bucket:             oldvol.Bucket,
+						LocationConstraint: oldvol.LocationConstraint,
+						IndexPageSize:      oldvol.IndexPageSize,
+						ConnectTimeout:     oldvol.ConnectTimeout,
+						ReadTimeout:        oldvol.ReadTimeout,
+						RaceWindow:         oldvol.RaceWindow,
+						UnsafeDelete:       oldvol.UnsafeDelete,
+					}
+				case "Azure":
+					keydata, err := ioutil.ReadFile(oldvol.StorageAccountKeyFile)
+					if err != nil && oldvol.StorageAccountKeyFile != "" {
+						return fmt.Errorf("error reading StorageAccountKeyFile: %s", err)
+					}
+					newvol = arvados.Volume{
+						Driver:         "Azure",
+						ReadOnly:       oldvol.ReadOnly,
+						Replication:    oldvol.AzureReplication,
+						StorageClasses: array2boolmap(oldvol.StorageClasses),
+					}
+					params = arvados.AzureVolumeDriverParameters{
+						StorageAccountName:   oldvol.StorageAccountName,
+						StorageAccountKey:    string(bytes.TrimSpace(keydata)),
+						StorageBaseURL:       oldvol.StorageBaseURL,
+						ContainerName:        oldvol.ContainerName,
+						RequestTimeout:       oldvol.RequestTimeout,
+						ListBlobsRetryDelay:  oldvol.ListBlobsRetryDelay,
+						ListBlobsMaxAttempts: oldvol.ListBlobsMaxAttempts,
+					}
+				case "Directory":
+					newvol = arvados.Volume{
+						Driver:         "Directory",
+						ReadOnly:       oldvol.ReadOnly,
+						Replication:    oldvol.DirectoryReplication,
+						StorageClasses: array2boolmap(oldvol.StorageClasses),
+					}
+					params = arvados.DirectoryVolumeDriverParameters{
+						Root:      oldvol.Root,
+						Serialize: oldvol.Serialize,
+					}
+				default:
+					return fmt.Errorf("unsupported volume type %q", oldvol.Type)
+				}
+				dp, err := json.Marshal(params)
+				if err != nil {
+					return err
+				}
+				newvol.DriverParameters = json.RawMessage(dp)
+				if newvol.Replication < 1 {
+					newvol.Replication = 1
+				}
+			}
+			if accessViaHosts == nil {
+				accessViaHosts = make(map[arvados.URL]arvados.VolumeAccess, 1)
+			}
+			accessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+			newvol.AccessViaHosts = accessViaHosts
+
+			volUUID := oldUUID
+			if oldvol.ReadOnly {
+			} else if oc.Listen == nil {
+				ldr.Logger.Warn("cannot find optimal volume UUID because Listen address is not given in legacy keepstore config")
+			} else if uuid, _, err := findKeepServicesItem(cluster, *oc.Listen); err != nil {
+				ldr.Logger.WithError(err).Warn("cannot find optimal volume UUID: failed to find a matching keep_service listing for this legacy keepstore config")
+			} else if len(uuid) != 27 {
+				ldr.Logger.WithField("UUID", uuid).Warn("cannot find optimal volume UUID: keep_service UUID does not have expected format")
+			} else {
+				rendezvousUUID := cluster.ClusterID + "-nyw5e-" + uuid[12:]
+				if _, ok := cluster.Volumes[rendezvousUUID]; ok {
+					ldr.Logger.Warn("suggesting a random volume UUID because the volume ID matching our keep_service UUID is already in use")
+				} else {
+					volUUID = rendezvousUUID
+				}
+			}
+			if volUUID == "" {
+				volUUID = newUUID(cluster.ClusterID, "nyw5e")
+				ldr.Logger.WithField("UUID", volUUID).Infof("suggesting a random volume UUID for volume #%d in legacy config", i)
+			}
+			cluster.Volumes[volUUID] = newvol
+		}
+	}
+
+	cfg.Clusters[cluster.ClusterID] = *cluster
+	return nil
+}
+
+func (ldr *Loader) alreadyMigrated(oldvol oldKeepstoreVolume, newvols map[string]arvados.Volume, myURL arvados.URL) (string, bool) {
+	for uuid, newvol := range newvols {
+		if oldvol.Type != newvol.Driver {
+			continue
+		}
+		switch oldvol.Type {
+		case "S3":
+			var params arvados.S3VolumeDriverParameters
+			if err := json.Unmarshal(newvol.DriverParameters, &params); err == nil &&
+				oldvol.Endpoint == params.Endpoint &&
+				oldvol.Region == params.Region &&
+				oldvol.Bucket == params.Bucket &&
+				oldvol.LocationConstraint == params.LocationConstraint {
+				return uuid, true
+			}
+		case "Azure":
+			var params arvados.AzureVolumeDriverParameters
+			if err := json.Unmarshal(newvol.DriverParameters, &params); err == nil &&
+				oldvol.StorageAccountName == params.StorageAccountName &&
+				oldvol.StorageBaseURL == params.StorageBaseURL &&
+				oldvol.ContainerName == params.ContainerName {
+				return uuid, true
+			}
+		case "Directory":
+			var params arvados.DirectoryVolumeDriverParameters
+			if err := json.Unmarshal(newvol.DriverParameters, &params); err == nil &&
+				oldvol.Root == params.Root {
+				if _, ok := newvol.AccessViaHosts[myURL]; ok {
+					return uuid, true
+				}
+			}
+		}
+	}
+	return "", false
+}
+
+func (ldr *Loader) discoverLocalVolumes(cluster *arvados.Cluster, mountsFile string, myURL arvados.URL) error {
+	if mountsFile == "" {
+		mountsFile = "/proc/mounts"
+	}
+	f, err := os.Open(mountsFile)
+	if err != nil {
+		return fmt.Errorf("error opening %s: %s", mountsFile, err)
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		args := strings.Fields(scanner.Text())
+		dev, mount := args[0], args[1]
+		if mount == "/" {
+			continue
+		}
+		if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
+			continue
+		}
+		keepdir := mount + "/keep"
+		if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
+			continue
+		}
+
+		ro := false
+		for _, fsopt := range strings.Split(args[3], ",") {
+			if fsopt == "ro" {
+				ro = true
+			}
+		}
+
+		uuid := newUUID(cluster.ClusterID, "nyw5e")
+		ldr.Logger.WithFields(logrus.Fields{
+			"UUID":                       uuid,
+			"Driver":                     "Directory",
+			"DriverParameters.Root":      keepdir,
+			"DriverParameters.Serialize": false,
+			"ReadOnly":                   ro,
+			"Replication":                1,
+		}).Warn("adding local directory volume")
+
+		p, err := json.Marshal(arvados.DirectoryVolumeDriverParameters{
+			Root:      keepdir,
+			Serialize: false,
+		})
+		if err != nil {
+			panic(err)
+		}
+		cluster.Volumes[uuid] = arvados.Volume{
+			Driver:           "Directory",
+			DriverParameters: p,
+			ReadOnly:         ro,
+			Replication:      1,
+			AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
+				myURL: {ReadOnly: ro},
+			},
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("reading %s: %s", mountsFile, err)
+	}
+	return nil
+}
+
+func array2boolmap(keys []string) map[string]bool {
+	m := map[string]bool{}
+	for _, k := range keys {
+		m[k] = true
+	}
+	return m
+}
+
+func newUUID(clusterID, infix string) string {
+	randint, err := rand.Int(rand.Reader, big.NewInt(0).Exp(big.NewInt(36), big.NewInt(15), big.NewInt(0)))
+	if err != nil {
+		panic(err)
+	}
+	randstr := randint.Text(36)
+	for len(randstr) < 15 {
+		randstr = "0" + randstr
+	}
+	return fmt.Sprintf("%s-%s-%s", clusterID, infix, randstr)
+}
+
+// Return the UUID and URL for the controller's keep_services listing
+// corresponding to this host/process.
+func findKeepServicesItem(cluster *arvados.Cluster, listen string) (uuid string, url arvados.URL, err error) {
+	client, err := arvados.NewClientFromConfig(cluster)
+	if err != nil {
+		return
+	}
+	client.AuthToken = cluster.SystemRootToken
+	var svcList arvados.KeepServiceList
+	err = client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+	if err != nil {
+		return
+	}
+	hostname, err := os.Hostname()
+	if err != nil {
+		err = fmt.Errorf("error getting hostname: %s", err)
+		return
+	}
+	for _, ks := range svcList.Items {
+		if ks.ServiceType != "proxy" && keepServiceIsMe(ks, hostname, listen) {
+			url := arvados.URL{
+				Scheme: "http",
+				Host:   net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+			}
+			if ks.ServiceSSLFlag {
+				url.Scheme = "https"
+			}
+			return ks.UUID, url, nil
+		}
+	}
+	err = errors.New("failed to find a keep_services entry that matches the current host/port")
+	return
+}
+
+var localhostOrAllInterfaces = map[string]bool{
+	"localhost": true,
+	"127.0.0.1": true,
+	"::1":       true,
+	"::":        true,
+	"":          true,
+}
+
+// Return true if the given KeepService entry matches the given
+// hostname and (keepstore config file) listen address.
+//
+// If the KeepService host is some variant of "localhost", we assume
+// this is a testing or single-node environment, ignore the given
+// hostname, and return true if the port numbers match.
+//
+// The hostname isn't assumed to be a FQDN: a hostname "foo.bar" will
+// match a KeepService host "foo.bar", but also "foo.bar.example",
+// "foo.bar.example.org", etc.
+func keepServiceIsMe(ks arvados.KeepService, hostname string, listen string) bool {
+	// Extract the port name/number from listen, and resolve it to
+	// a port number to compare with ks.ServicePort.
+	_, listenport, err := net.SplitHostPort(listen)
+	if err != nil && strings.HasPrefix(listen, ":") {
+		listenport = listen[1:]
+	}
+	if lp, err := net.LookupPort("tcp", listenport); err != nil {
+		return false
+	} else if !(lp == ks.ServicePort ||
+		(lp == 0 && ks.ServicePort == 80)) {
+		return false
+	}
+
+	kshost := strings.ToLower(ks.ServiceHost)
+	return localhostOrAllInterfaces[kshost] || strings.HasPrefix(kshost+".", strings.ToLower(hostname)+".")
+}
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
new file mode 100644
index 000000000..be4c643f4
--- /dev/null
+++ b/lib/config/deprecated_keepstore_test.go
@@ -0,0 +1,685 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	check "gopkg.in/check.v1"
+)
+
+type KeepstoreMigrationSuite struct {
+	hostname string // blank = use test system's hostname
+}
+
+var _ = check.Suite(&KeepstoreMigrationSuite{})
+
+func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
+	keepstorefile, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(keepstorefile.Name())
+	_, err = io.WriteString(keepstorefile, keepstoreconfig)
+	c.Assert(err, check.IsNil)
+	err = keepstorefile.Close()
+	c.Assert(err, check.IsNil)
+
+	gotldr := testLoader(c, clusterconfig, nil)
+	gotldr.KeepstorePath = keepstorefile.Name()
+	expectedldr := testLoader(c, expectedconfig, nil)
+	checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
+	keyfile, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(keyfile.Name())
+	io.WriteString(keyfile, "blobsigningkey\n")
+
+	hostname, err := os.Hostname()
+	c.Assert(err, check.IsNil)
+
+	s.checkEquivalentWithKeepstoreConfig(c, `
+Listen: ":12345"
+Debug: true
+LogFormat: text
+MaxBuffers: 1234
+MaxRequests: 2345
+BlobSignatureTTL: 123m
+BlobSigningKeyFile: `+keyfile.Name()+`
+`, `
+Clusters:
+  z1111:
+    {}
+`, `
+Clusters:
+  z1111:
+    Services:
+      Keepstore:
+        InternalURLs:
+          "http://`+hostname+`:12345": {}
+    SystemLogs:
+      Format: text
+      LogLevel: debug
+    API:
+      MaxKeepBlockBuffers: 1234
+      MaxConcurrentRequests: 2345
+    Collections:
+      BlobSigningTTL: 123m
+      BlobSigningKey: blobsigningkey
+`)
+}
+
+func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
+	tmpd, err := ioutil.TempDir("", "")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpd)
+	err = os.Mkdir(tmpd+"/keep", 0777)
+	c.Assert(err, check.IsNil)
+
+	tmpf, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(tmpf.Name())
+
+	// read/write
+	_, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
+	c.Assert(err, check.IsNil)
+
+	s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+		Driver:      "Directory",
+		ReadOnly:    false,
+		Replication: 1,
+	}, &arvados.DirectoryVolumeDriverParameters{
+		Root:      tmpd + "/keep",
+		Serialize: false,
+	}, &arvados.DirectoryVolumeDriverParameters{})
+
+	// read-only
+	tmpf.Seek(0, os.SEEK_SET)
+	tmpf.Truncate(0)
+	_, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
+	c.Assert(err, check.IsNil)
+
+	s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+		Driver:      "Directory",
+		ReadOnly:    true,
+		Replication: 1,
+	}, &arvados.DirectoryVolumeDriverParameters{
+		Root:      tmpd + "/keep",
+		Serialize: false,
+	}, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
+	accesskeyfile, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(accesskeyfile.Name())
+	io.WriteString(accesskeyfile, "accesskeydata\n")
+
+	secretkeyfile, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(secretkeyfile.Name())
+	io.WriteString(secretkeyfile, "secretkeydata\n")
+
+	// s3, empty/default
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+`, arvados.Volume{
+		Driver:      "S3",
+		Replication: 1,
+	}, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
+
+	// s3, fully configured
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+  AccessKeyFile: `+accesskeyfile.Name()+`
+  SecretKeyFile: `+secretkeyfile.Name()+`
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: testbucket
+  LocationConstraint: true
+  IndexPageSize: 1234
+  S3Replication: 4
+  ConnectTimeout: 3m
+  ReadTimeout: 4m
+  RaceWindow: 5m
+  UnsafeDelete: true
+`, arvados.Volume{
+		Driver:      "S3",
+		Replication: 4,
+	}, &arvados.S3VolumeDriverParameters{
+		AccessKey:          "accesskeydata",
+		SecretKey:          "secretkeydata",
+		Endpoint:           "https://storage.googleapis.com",
+		Region:             "us-east-1z",
+		Bucket:             "testbucket",
+		LocationConstraint: true,
+		IndexPageSize:      1234,
+		ConnectTimeout:     arvados.Duration(time.Minute * 3),
+		ReadTimeout:        arvados.Duration(time.Minute * 4),
+		RaceWindow:         arvados.Duration(time.Minute * 5),
+		UnsafeDelete:       true,
+	}, &arvados.S3VolumeDriverParameters{})
+
+	// azure, empty/default
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+`, arvados.Volume{
+		Driver:      "Azure",
+		Replication: 1,
+	}, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
+
+	// azure, fully configured
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+  ReadOnly: true
+  StorageAccountName: storageacctname
+  StorageAccountKeyFile: `+secretkeyfile.Name()+`
+  StorageBaseURL: https://example.example
+  ContainerName: testctr
+  LocationConstraint: true
+  AzureReplication: 4
+  RequestTimeout: 3m
+  ListBlobsRetryDelay: 4m
+  ListBlobsMaxAttempts: 5
+`, arvados.Volume{
+		Driver:      "Azure",
+		ReadOnly:    true,
+		Replication: 4,
+	}, &arvados.AzureVolumeDriverParameters{
+		StorageAccountName:   "storageacctname",
+		StorageAccountKey:    "secretkeydata",
+		StorageBaseURL:       "https://example.example",
+		ContainerName:        "testctr",
+		RequestTimeout:       arvados.Duration(time.Minute * 3),
+		ListBlobsRetryDelay:  arvados.Duration(time.Minute * 4),
+		ListBlobsMaxAttempts: 5,
+	}, &arvados.AzureVolumeDriverParameters{})
+
+	// directory, empty/default
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+  Root: /tmp/xyzzy
+`, arvados.Volume{
+		Driver:      "Directory",
+		Replication: 1,
+	}, &arvados.DirectoryVolumeDriverParameters{
+		Root: "/tmp/xyzzy",
+	}, &arvados.DirectoryVolumeDriverParameters{})
+
+	// directory, fully configured
+	s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+  ReadOnly: true
+  Root: /tmp/xyzzy
+  DirectoryReplication: 4
+  Serialize: true
+`, arvados.Volume{
+		Driver:      "Directory",
+		ReadOnly:    true,
+		Replication: 4,
+	}, &arvados.DirectoryVolumeDriverParameters{
+		Root:      "/tmp/xyzzy",
+		Serialize: true,
+	}, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
+	hostname := s.hostname
+	if hostname == "" {
+		h, err := os.Hostname()
+		c.Assert(err, check.IsNil)
+		hostname = h
+	}
+
+	oldconfig, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(oldconfig.Name())
+	io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
+	if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
+		// Prevent tests from looking at the real /proc/mounts on the test host.
+		io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
+	}
+
+	ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
+	ldr.KeepstorePath = oldconfig.Name()
+	cfg, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+	cc := cfg.Clusters["z1111"]
+	c.Check(cc.Volumes, check.HasLen, 1)
+	for uuid, v := range cc.Volumes {
+		c.Check(uuid, check.HasLen, 27)
+		c.Check(v.Driver, check.Equals, expectvol.Driver)
+		c.Check(v.Replication, check.Equals, expectvol.Replication)
+
+		avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+		c.Check(ok, check.Equals, true)
+		c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
+
+		err := json.Unmarshal(v.DriverParameters, paramsdst)
+		c.Check(err, check.IsNil)
+		c.Check(paramsdst, check.DeepEquals, expectparams)
+	}
+}
+
+// How we handle a volume from a legacy keepstore config file depends
+// on whether it's writable, whether a volume using the same cloud
+// backend already exists in the cluster config, and (if so) whether
+// it already has an AccessViaHosts entry for this host.
+//
+// In all cases, we should end up with an AccessViaHosts entry for
+// this host, to indicate that the current host's volumes have been
+// migrated.
+
+// Same backend already referenced in cluster config, this host
+// already listed in AccessViaHosts --> no change, except possibly
+// updating the ReadOnly flag on the AccessViaHosts entry.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: alreadymigrated
+  S3Replication: 3
+`)
+	checkEqualYAML(c, after, before)
+}
+
+// Writable volume, same cloud backend already referenced in cluster
+// config --> change UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
+	port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: readonlyonother
+  S3Replication: 3
+`)
+	c.Check(after, check.HasLen, len(before))
+	newuuids := s.findAddedVolumes(c, before, after, 1)
+	newvol := after[newuuids[0]]
+
+	var params arvados.S3VolumeDriverParameters
+	json.Unmarshal(newvol.DriverParameters, &params)
+	c.Check(params.Bucket, check.Equals, "readonlyonother")
+	c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same cloud backend not yet referenced --> add a
+// new volume, with UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
+	port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: bucket-to-migrate
+  S3Replication: 3
+`)
+	newuuids := s.findAddedVolumes(c, before, after, 1)
+	newvol := after[newuuids[0]]
+
+	var params arvados.S3VolumeDriverParameters
+	json.Unmarshal(newvol.DriverParameters, &params)
+	c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
+	c.Check(newvol.Replication, check.Equals, 3)
+
+	c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, but this host isn't in AccessViaHosts --> add a new
+// volume, with UUID to match this keepstore's UUID (filesystem-backed
+// volumes are assumed to be different on different hosts, even if
+// paths are the same).
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+  Root: /data/sdd
+  DirectoryReplication: 2
+`)
+	newuuids := s.findAddedVolumes(c, before, after, 1)
+	newvol := after[newuuids[0]]
+
+	var params arvados.DirectoryVolumeDriverParameters
+	json.Unmarshal(newvol.DriverParameters, &params)
+	c.Check(params.Root, check.Equals, "/data/sdd")
+	c.Check(newvol.Replication, check.Equals, 2)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, and this host is already listed in AccessViaHosts
+// --> already migrated, don't change anything.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+  Root: /data/sde
+  DirectoryReplication: 2
+`)
+	checkEqualYAML(c, after, before)
+}
+
+// Multiple writable cloud-backed volumes --> one of them will get a
+// UUID matching this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
+	port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: first-bucket-to-migrate
+  S3Replication: 3
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: second-bucket-to-migrate
+  S3Replication: 3
+`)
+	newuuids := s.findAddedVolumes(c, before, after, 2)
+	// Sort by bucket name (so "first" comes before "second")
+	params := map[string]arvados.S3VolumeDriverParameters{}
+	for _, uuid := range newuuids {
+		var p arvados.S3VolumeDriverParameters
+		json.Unmarshal(after[uuid].DriverParameters, &p)
+		params[uuid] = p
+	}
+	sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
+	newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
+	params0, params1 := params[newuuids[0]], params[newuuids[1]]
+
+	c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
+	c.Check(newvol0.Replication, check.Equals, 3)
+
+	c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
+	c.Check(newvol1.Replication, check.Equals, 3)
+
+	// Don't care which one gets the special UUID
+	if newuuids[0] != expectUUID {
+		c.Check(newuuids[1], check.Equals, expectUUID)
+	}
+}
+
+// Non-writable volume, same cloud backend already referenced in
+// cluster config --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
+	port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: readonlyonother
+  S3Replication: 3
+  ReadOnly: true
+`)
+	hostname, err := os.Hostname()
+	c.Assert(err, check.IsNil)
+	url := arvados.URL{
+		Scheme: "http",
+		Host:   fmt.Sprintf("%s:%d", hostname, port),
+	}
+	_, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+	c.Check(ok, check.Equals, false)
+	_, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+	c.Check(ok, check.Equals, true)
+}
+
+// Writable volume, same cloud backend already writable by another
+// keepstore server --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
+	port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: writableonother
+  S3Replication: 3
+  ReadOnly: false
+`)
+	hostname, err := os.Hostname()
+	c.Assert(err, check.IsNil)
+	url := arvados.URL{
+		Scheme: "http",
+		Host:   fmt.Sprintf("%s:%d", hostname, port),
+	}
+	_, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+	c.Check(ok, check.Equals, false)
+	_, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+	c.Check(ok, check.Equals, true)
+}
+
+// Non-writable volume, same cloud backend not already referenced in
+// cluster config --> assign a new random volume UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
+	port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+	before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+  Endpoint: https://storage.googleapis.com
+  Region: us-east-1z
+  Bucket: differentbucket
+  S3Replication: 3
+`)
+	newuuids := s.findAddedVolumes(c, before, after, 1)
+	newvol := after[newuuids[0]]
+
+	var params arvados.S3VolumeDriverParameters
+	json.Unmarshal(newvol.DriverParameters, &params)
+	c.Check(params.Bucket, check.Equals, "differentbucket")
+
+	hostname, err := os.Hostname()
+	c.Assert(err, check.IsNil)
+	_, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+	c.Check(ok, check.Equals, true)
+}
+
+const clusterConfigForKeepstoreMigrationTest = `
+Clusters:
+  zzzzz:
+    SystemRootToken: ` + arvadostest.AdminToken + `
+    Services:
+      Keepstore:
+        InternalURLs:
+          "http://{{.hostname}}:12345": {}
+      Controller:
+        ExternalURL: "https://{{.controller}}"
+    TLS:
+      Insecure: true
+    Volumes:
+
+      zzzzz-nyw5e-alreadymigrated:
+        AccessViaHosts:
+          "http://{{.hostname}}:12345": {}
+        Driver: S3
+        DriverParameters:
+          Endpoint: https://storage.googleapis.com
+          Region: us-east-1z
+          Bucket: alreadymigrated
+        Replication: 3
+
+      zzzzz-nyw5e-readonlyonother:
+        AccessViaHosts:
+          "http://other.host.example:12345": {ReadOnly: true}
+        Driver: S3
+        DriverParameters:
+          Endpoint: https://storage.googleapis.com
+          Region: us-east-1z
+          Bucket: readonlyonother
+        Replication: 3
+
+      zzzzz-nyw5e-writableonother:
+        AccessViaHosts:
+          "http://other.host.example:12345": {}
+        Driver: S3
+        DriverParameters:
+          Endpoint: https://storage.googleapis.com
+          Region: us-east-1z
+          Bucket: writableonother
+        Replication: 3
+
+      zzzzz-nyw5e-localfilesystem:
+        Driver: Directory
+        DriverParameters:
+          Root: /data/sdd
+        Replication: 1
+
+      zzzzz-nyw5e-localismigrated:
+        AccessViaHosts:
+          "http://{{.hostname}}:12345": {}
+        Driver: Directory
+        DriverParameters:
+          Root: /data/sde
+        Replication: 1
+`
+
+// Determine the effect of combining the given legacy keepstore config
+// YAML (just the "Volumes" entries of an old keepstore config file)
+// with the example clusterConfigForKeepstoreMigrationTest config.
+//
+// Return two Volumes configs -- one without loading
+// keepstoreconfigdata ("before") and one with ("after") -- for the
+// caller to compare.
+func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreVolumesYAML string) (before, after map[string]arvados.Volume) {
+	ldr := testLoader(c, s.clusterConfigYAML(c), nil)
+	cBefore, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+
+	keepstoreconfig, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(keepstoreconfig.Name())
+	io.WriteString(keepstoreconfig, keepstoreVolumesYAML)
+
+	ldr = testLoader(c, s.clusterConfigYAML(c), nil)
+	ldr.KeepstorePath = keepstoreconfig.Name()
+	cAfter, err := ldr.Load()
+	c.Assert(err, check.IsNil)
+
+	return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
+}
+
+func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
+	hostname, err := os.Hostname()
+	c.Assert(err, check.IsNil)
+
+	tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
+
+	var clusterconfigdata bytes.Buffer
+	err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
+		"hostname":   hostname,
+		"controller": os.Getenv("ARVADOS_API_HOST"),
+	})
+	c.Assert(err, check.IsNil)
+
+	return clusterconfigdata.String()
+}
+
+// Return the uuids of volumes that appear in "after" but not
+// "before".
+//
+// Assert the returned slice has at least minAdded entries.
+func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
+	for uuid := range after {
+		if _, ok := before[uuid]; !ok {
+			uuids = append(uuids, uuid)
+		}
+	}
+	if len(uuids) < minAdded {
+		c.Assert(uuids, check.HasLen, minAdded)
+	}
+	return
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
+	for uuid, port := range s.getTestKeepstorePorts(c) {
+		c.Assert(uuid, check.HasLen, 27)
+		return port, "zzzzz-nyw5e-" + uuid[12:]
+	}
+	c.Fatal("getTestKeepstorePorts() returned nothing")
+	return 0, ""
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePorts(c *check.C) map[string]int {
+	client := arvados.NewClientFromEnv()
+	var svcList arvados.KeepServiceList
+	err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+	c.Assert(err, check.IsNil)
+	ports := map[string]int{}
+	for _, ks := range svcList.Items {
+		if ks.ServiceType == "disk" {
+			ports[ks.UUID] = ks.ServicePort
+		}
+	}
+	return ports
+}
+
+func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
+	for i, trial := range []struct {
+		match       bool
+		hostname    string
+		listen      string
+		serviceHost string
+		servicePort int
+	}{
+		{true, "keep0", "keep0", "keep0", 80},
+		{true, "keep0", "[::1]:http", "keep0", 80},
+		{true, "keep0", "[::]:http", "keep0", 80},
+		{true, "keep0", "keep0:25107", "keep0", 25107},
+		{true, "keep0", ":25107", "keep0", 25107},
+		{true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
+		{true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
+		{true, "keep0", ":25107", "keep0.domain.example", 25107},
+		{true, "keep0", ":25107", "Keep0.domain.example", 25107},
+		{true, "keep0", ":http", "keep0.domain.example", 80},
+		{true, "keep0", ":25107", "localhost", 25107},
+		{true, "keep0", ":25107", "::1", 25107},
+		{false, "keep0", ":25107", "keep0", 1111},              // different port
+		{false, "keep0", ":25107", "localhost", 1111},          // different port
+		{false, "keep0", ":http", "keep0.domain.example", 443}, // different port
+		{false, "keep0", ":bogussss", "keep0", 25107},          // unresolvable port
+		{false, "keep0", ":25107", "keep1", 25107},             // different hostname
+		{false, "keep1", ":25107", "keep10", 25107},            // different hostname (prefix, but not on a "." boundary)
+	} {
+		c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
+	}
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 308b0cc35..55bb2193b 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -13,7 +13,7 @@ import (
 func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
 	hostname, err := os.Hostname()
 	c.Assert(err, check.IsNil)
-	s.checkEquivalent(c, `
+	checkEquivalent(c, `
 Clusters:
  z1111:
   NodeProfiles:
diff --git a/lib/config/export.go b/lib/config/export.go
index b125d7dc9..a349db516 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -63,8 +63,10 @@ var whitelist = map[string]bool{
 	"API":                                          true,
 	"API.AsyncPermissionsUpdateInterval":           false,
 	"API.DisabledAPIs":                             false,
+	"API.MaxConcurrentRequests":                    false,
 	"API.MaxIndexDatabaseRead":                     false,
 	"API.MaxItemsPerResponse":                      true,
+	"API.MaxKeepBlockBuffers":                      false,
 	"API.MaxRequestAmplification":                  false,
 	"API.MaxRequestSize":                           true,
 	"API.RailsSessionSecretToken":                  false,
@@ -80,6 +82,12 @@ var whitelist = map[string]bool{
 	"Collections.BlobSigning":                      true,
 	"Collections.BlobSigningKey":                   false,
 	"Collections.BlobSigningTTL":                   true,
+	"Collections.BlobTrash":                        false,
+	"Collections.BlobTrashLifetime":                false,
+	"Collections.BlobTrashConcurrency":             false,
+	"Collections.BlobTrashCheckInterval":           false,
+	"Collections.BlobDeleteConcurrency":            false,
+	"Collections.BlobReplicateConcurrency":         false,
 	"Collections.CollectionVersioning":             false,
 	"Collections.DefaultReplication":               true,
 	"Collections.DefaultTrashLifetime":             true,
@@ -153,6 +161,16 @@ var whitelist = map[string]bool{
 	"Users.NewUsersAreActive":                      false,
 	"Users.UserNotifierEmailFrom":                  false,
 	"Users.UserProfileNotificationAddress":         false,
+	"Volumes":                                      true,
+	"Volumes.*":                                    true,
+	"Volumes.*.*":                                  false,
+	"Volumes.*.AccessViaHosts":                     true,
+	"Volumes.*.AccessViaHosts.*":                   true,
+	"Volumes.*.AccessViaHosts.*.ReadOnly":          true,
+	"Volumes.*.ReadOnly":                           true,
+	"Volumes.*.Replication":                        true,
+	"Volumes.*.StorageClasses":                     true,
+	"Volumes.*.StorageClasses.*":                   false,
 	"Workbench":                                    true,
 	"Workbench.ActivationContactLink":              false,
 	"Workbench.APIClientConnectTimeout":            true,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 602f30e1d..92ceb86bd 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -182,6 +182,15 @@ Clusters:
       # parameter higher than this value, this value is used instead.
       MaxItemsPerResponse: 1000
 
+      # Maximum number of concurrent requests to accept in a single
+      # service process, or 0 for no limit. Currently supported only
+      # by keepstore.
+      MaxConcurrentRequests: 0
+
+      # Maximum number of 64MiB memory buffers per keepstore server
+      # process, or 0 for no limit.
+      MaxKeepBlockBuffers: 128
+
       # API methods to disable. Disabled methods are not listed in the
       # discovery document, and respond 404 to all requests.
       # Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -319,15 +328,44 @@ Clusters:
 
       # BlobSigningKey is a string of alphanumeric characters used to
       # generate permission signatures for Keep locators. It must be
-      # identical to the permission key given to Keep. IMPORTANT: This is
-      # a site secret. It should be at least 50 characters.
+      # identical to the permission key given to Keep. IMPORTANT: This
+      # is a site secret. It should be at least 50 characters.
       #
-      # Modifying blob_signing_key will invalidate all existing
+      # Modifying BlobSigningKey will invalidate all existing
       # signatures, which can cause programs to fail (e.g., arv-put,
-      # arv-get, and Crunch jobs).  To avoid errors, rotate keys only when
-      # no such processes are running.
+      # arv-get, and Crunch jobs).  To avoid errors, rotate keys only
+      # when no such processes are running.
       BlobSigningKey: ""
 
+      # Enable garbage collection of unreferenced blobs in Keep.
+      BlobTrash: true
+
+      # Time to leave unreferenced blobs in "trashed" state before
+      # deleting them, or 0 to skip the "trashed" state entirely and
+      # delete unreferenced blobs.
+      #
+      # If you use any Amazon S3 buckets as storage volumes, this
+      # must be at least 24h to avoid occasional data loss.
+      BlobTrashLifetime: 336h
+
+      # How often to check for (and delete) trashed blocks whose
+      # BlobTrashLifetime has expired.
+      BlobTrashCheckInterval: 24h
+
+      # Maximum number of concurrent "trash blob" and "delete trashed
+      # blob" operations conducted by a single keepstore process. Each
+      # of these can be set to 0 to disable the respective operation.
+      #
+      # If BlobTrashLifetime is zero, "trash" and "delete trash"
+      # happen at once, so only the lower of these two values is used.
+      BlobTrashConcurrency: 4
+      BlobDeleteConcurrency: 4
+
+      # Maximum number of concurrent "create additional replica of
+      # existing blob" operations conducted by a single keepstore
+      # process.
+      BlobReplicateConcurrency: 4
+
       # Default replication level for collections. This is used when a
       # collection's replication_desired attribute is nil.
       DefaultReplication: 2
@@ -736,6 +774,47 @@ Clusters:
         Price: 0.1
         Preemptible: false
 
+    Volumes:
+      SAMPLE:
+        AccessViaHosts:
+          SAMPLE:
+            ReadOnly: false
+        ReadOnly: false
+        Replication: 1
+        StorageClasses:
+          default: true
+          SAMPLE: true
+        Driver: s3
+        DriverParameters:
+
+          # for s3 driver
+          AccessKey: aaaaa
+          SecretKey: aaaaa
+          Endpoint: ""
+          Region: us-east-1a
+          Bucket: aaaaa
+          LocationConstraint: false
+          IndexPageSize: 1000
+          ConnectTimeout: 1m
+          ReadTimeout: 10m
+          RaceWindow: 24h
+          UnsafeDelete: false
+
+          # for azure driver
+          StorageAccountName: aaaaa
+          StorageAccountKey: aaaaa
+          StorageBaseURL: core.windows.net
+          ContainerName: aaaaa
+          RequestTimeout: 30s
+          ListBlobsRetryDelay: 10s
+          ListBlobsMaxAttempts: 10
+          MaxGetBytes: 0
+          WriteRaceInterval: 15s
+          WriteRacePollTime: 1s
+
+          # for local directory driver
+          Root: /var/lib/arvados/keep-data
+
     Mail:
       MailchimpAPIKey: ""
       MailchimpListID: ""
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index c7289350e..17e0af7ba 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -321,7 +321,7 @@ Clusters:
 }
 
 func (s *LoadSuite) TestMovedKeys(c *check.C) {
-	s.checkEquivalent(c, `# config has old keys only
+	checkEquivalent(c, `# config has old keys only
 Clusters:
  zzzzz:
   RequestLimits:
@@ -334,7 +334,7 @@ Clusters:
    MaxRequestAmplification: 3
    MaxItemsPerResponse: 999
 `)
-	s.checkEquivalent(c, `# config has both old and new keys; old values win
+	checkEquivalent(c, `# config has both old and new keys; old values win
 Clusters:
  zzzzz:
   RequestLimits:
@@ -352,30 +352,45 @@ Clusters:
 `)
 }
 
-func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
-	got, err := testLoader(c, goty, nil).Load()
+func checkEquivalent(c *check.C, goty, expectedy string) {
+	gotldr := testLoader(c, goty, nil)
+	expectedldr := testLoader(c, expectedy, nil)
+	checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func checkEqualYAML(c *check.C, got, expected interface{}) {
+	expectedyaml, err := yaml.Marshal(expected)
 	c.Assert(err, check.IsNil)
-	expected, err := testLoader(c, expectedy, nil).Load()
+	gotyaml, err := yaml.Marshal(got)
 	c.Assert(err, check.IsNil)
-	if !c.Check(got, check.DeepEquals, expected) {
+	if !bytes.Equal(gotyaml, expectedyaml) {
 		cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
-		for _, obj := range []interface{}{expected, got} {
-			y, _ := yaml.Marshal(obj)
+		for _, y := range [][]byte{expectedyaml, gotyaml} {
 			pr, pw, err := os.Pipe()
 			c.Assert(err, check.IsNil)
 			defer pr.Close()
-			go func() {
-				io.Copy(pw, bytes.NewBuffer(y))
+			go func(data []byte) {
+				pw.Write(data)
 				pw.Close()
-			}()
+			}(y)
 			cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
 		}
 		diff, err := cmd.CombinedOutput()
+		// diff should report differences and exit non-zero.
+		c.Check(err, check.NotNil)
 		c.Log(string(diff))
-		c.Check(err, check.IsNil)
+		c.Error("got != expected; see diff (-expected +got) above")
 	}
 }
 
+func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
+	got, err := gotldr.Load()
+	c.Assert(err, check.IsNil)
+	expected, err := expectedldr.Load()
+	c.Assert(err, check.IsNil)
+	checkEqualYAML(c, got, expected)
+}
+
 func checkListKeys(path string, x interface{}) (err error) {
 	v := reflect.Indirect(reflect.ValueOf(x))
 	switch v.Kind() {
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index ae6ac70e9..7ab38c6ca 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -11,11 +11,12 @@ import (
 	"git.curoverse.com/arvados.git/lib/cmd"
 	"git.curoverse.com/arvados.git/lib/service"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
 
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
 	ac, err := arvados.NewClientFromConfig(cluster)
 	if err != nil {
 		return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
@@ -25,6 +26,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) ser
 		Context:   ctx,
 		ArvClient: ac,
 		AuthToken: token,
+		Registry:  reg,
 	}
 	go d.Start()
 	return d
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 731c6d25d..f0aa83c2e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -48,10 +48,10 @@ type dispatcher struct {
 	Context       context.Context
 	ArvClient     *arvados.Client
 	AuthToken     string
+	Registry      *prometheus.Registry
 	InstanceSetID cloud.InstanceSetID
 
 	logger      logrus.FieldLogger
-	reg         *prometheus.Registry
 	instanceSet cloud.InstanceSet
 	pool        pool
 	queue       scheduler.ContainerQueue
@@ -132,14 +132,13 @@ func (disp *dispatcher) initialize() {
 		disp.sshKey = key
 	}
 
-	disp.reg = prometheus.NewRegistry()
-	instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.reg)
+	instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.Registry)
 	if err != nil {
 		disp.logger.Fatalf("error initializing driver: %s", err)
 	}
 	disp.instanceSet = instanceSet
-	disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.reg, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
-	disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, disp.ArvClient)
+	disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.Registry, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
+	disp.queue = container.NewQueue(disp.logger, disp.Registry, disp.typeChooser, disp.ArvClient)
 
 	if disp.Cluster.ManagementToken == "" {
 		disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -154,7 +153,7 @@ func (disp *dispatcher) initialize() {
 		mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/drain", disp.apiInstanceDrain)
 		mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/run", disp.apiInstanceRun)
 		mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/kill", disp.apiInstanceKill)
-		metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+		metricsH := promhttp.HandlerFor(disp.Registry, promhttp.HandlerOpts{
 			ErrorLog: disp.logger,
 		})
 		mux.Handler("GET", "/metrics", metricsH)
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index b6737bc55..f490f83c1 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -22,6 +22,7 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	"github.com/coreos/go-systemd/daemon"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -30,7 +31,7 @@ type Handler interface {
 	CheckHealth() error
 }
 
-type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler
+type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string, registry *prometheus.Registry) Handler
 
 type command struct {
 	newHandler NewHandlerFunc
@@ -68,7 +69,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	loader := config.NewLoader(stdin, log)
 	loader.SetupFlags(flags)
 	versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-
 	err = flags.Parse(args)
 	if err == flag.ErrHelp {
 		err = nil
@@ -87,22 +87,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 	if err != nil {
 		return 1
 	}
-	log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+
+	// Now that we've read the config, replace the bootstrap
+	// logger with a new one according to the logging config.
+	log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
+	logger := log.WithFields(logrus.Fields{
 		"PID": os.Getpid(),
 	})
-	ctx := ctxlog.Context(c.ctx, log)
+	ctx := ctxlog.Context(c.ctx, logger)
 
-	listen, err := getListenAddr(cluster.Services, c.svcName)
+	listenURL, err := getListenAddr(cluster.Services, c.svcName)
 	if err != nil {
 		return 1
 	}
+	ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
 
 	if cluster.SystemRootToken == "" {
-		log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+		logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
 		cluster.SystemRootToken = os.Getenv("ARVADOS_API_TOKEN")
 	}
 	if cluster.Services.Controller.ExternalURL.Host == "" {
-		log.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+		logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
 		u, err := url.Parse("https://" + os.Getenv("ARVADOS_API_HOST"))
 		if err != nil {
 			err = fmt.Errorf("ARVADOS_API_HOST: %s", err)
@@ -114,27 +119,42 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 		}
 	}
 
-	handler := c.newHandler(ctx, cluster, cluster.SystemRootToken)
+	reg := prometheus.NewRegistry()
+	handler := c.newHandler(ctx, cluster, cluster.SystemRootToken, reg)
 	if err = handler.CheckHealth(); err != nil {
 		return 1
 	}
+
+	instrumented := httpserver.Instrument(reg, log,
+		httpserver.HandlerWithContext(ctx,
+			httpserver.AddRequestIDs(
+				httpserver.LogRequests(
+					httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler)))))
 	srv := &httpserver.Server{
 		Server: http.Server{
-			Handler: httpserver.HandlerWithContext(ctx,
-				httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
+			Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
 		},
-		Addr: listen,
+		Addr: listenURL.Host,
+	}
+	if listenURL.Scheme == "https" {
+		tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger)
+		if err != nil {
+			logger.WithError(err).Error("cannot start %s service on %s", c.svcName, listenURL.String())
+			return 1
+		}
+		srv.TLSConfig = tlsconfig
 	}
 	err = srv.Start()
 	if err != nil {
 		return 1
 	}
-	log.WithFields(logrus.Fields{
+	logger.WithFields(logrus.Fields{
+		"URL":     listenURL,
 		"Listen":  srv.Addr,
 		"Service": c.svcName,
 	}).Info("listening")
 	if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
-		log.WithError(err).Errorf("error notifying init daemon")
+		logger.WithError(err).Errorf("error notifying init daemon")
 	}
 	go func() {
 		<-ctx.Done()
@@ -149,20 +169,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 
 const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
 
-func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) {
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (arvados.URL, error) {
 	svc, ok := svcs.Map()[prog]
 	if !ok {
-		return "", fmt.Errorf("unknown service name %q", prog)
+		return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
 	}
 	for url := range svc.InternalURLs {
 		if strings.HasPrefix(url.Host, "localhost:") {
-			return url.Host, nil
+			return url, nil
 		}
 		listener, err := net.Listen("tcp", url.Host)
 		if err == nil {
 			listener.Close()
-			return url.Host, nil
+			return url, nil
 		}
 	}
-	return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
+	return arvados.URL{}, fmt.Errorf("configuration does not enable the %s service on this host", prog)
+}
+
+type contextKeyURL struct{}
+
+func URLFromContext(ctx context.Context) (arvados.URL, bool) {
+	u, ok := ctx.Value(contextKeyURL{}).(arvados.URL)
+	return u, ok
 }
diff --git a/lib/service/tls.go b/lib/service/tls.go
new file mode 100644
index 000000000..8fef46158
--- /dev/null
+++ b/lib/service/tls.go
@@ -0,0 +1,80 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package service
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"github.com/sirupsen/logrus"
+)
+
+func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+	currentCert := make(chan *tls.Certificate, 1)
+	loaded := false
+
+	key, cert := cluster.TLS.Key, cluster.TLS.Certificate
+	if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
+		return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+	}
+
+	update := func() error {
+		cert, err := tls.LoadX509KeyPair(cert, key)
+		if err != nil {
+			return fmt.Errorf("error loading X509 key pair: %s", err)
+		}
+		if loaded {
+			// Throw away old cert
+			<-currentCert
+		}
+		currentCert <- &cert
+		loaded = true
+		return nil
+	}
+	err := update()
+	if err != nil {
+		return nil, err
+	}
+
+	go func() {
+		reload := make(chan os.Signal, 1)
+		signal.Notify(reload, syscall.SIGHUP)
+		for range reload {
+			err := update()
+			if err != nil {
+				logger.WithError(err).Warn("error updating TLS certificate")
+			}
+		}
+	}()
+
+	// https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
+	return &tls.Config{
+		PreferServerCipherSuites: true,
+		CurvePreferences: []tls.CurveID{
+			tls.CurveP256,
+			tls.X25519,
+		},
+		MinVersion: tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+		},
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := <-currentCert
+			currentCert <- cert
+			return cert, nil
+		},
+	}, nil
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index f6b736d58..74a08246a 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -72,6 +72,8 @@ type Cluster struct {
 		DisabledAPIs                   StringSet
 		MaxIndexDatabaseRead           int
 		MaxItemsPerResponse            int
+		MaxConcurrentRequests          int
+		MaxKeepBlockBuffers            int
 		MaxRequestAmplification        int
 		MaxRequestSize                 int
 		RailsSessionSecretToken        string
@@ -86,13 +88,19 @@ type Cluster struct {
 		UnloggedAttributes StringSet
 	}
 	Collections struct {
-		BlobSigning          bool
-		BlobSigningKey       string
-		BlobSigningTTL       Duration
-		CollectionVersioning bool
-		DefaultTrashLifetime Duration
-		DefaultReplication   int
-		ManagedProperties    map[string]struct {
+		BlobSigning              bool
+		BlobSigningKey           string
+		BlobSigningTTL           Duration
+		BlobTrash                bool
+		BlobTrashLifetime        Duration
+		BlobTrashCheckInterval   Duration
+		BlobTrashConcurrency     int
+		BlobDeleteConcurrency    int
+		BlobReplicateConcurrency int
+		CollectionVersioning     bool
+		DefaultTrashLifetime     Duration
+		DefaultReplication       int
+		ManagedProperties        map[string]struct {
 			Value     interface{}
 			Function  string
 			Protected bool
@@ -143,6 +151,7 @@ type Cluster struct {
 		UserNotifierEmailFrom                 string
 		UserProfileNotificationAddress        string
 	}
+	Volumes   map[string]Volume
 	Workbench struct {
 		ActivationContactLink            string
 		APIClientConnectTimeout          Duration
@@ -182,6 +191,48 @@ type Cluster struct {
 	EnableBetaController14287 bool
 }
 
+type Volume struct {
+	AccessViaHosts   map[URL]VolumeAccess
+	ReadOnly         bool
+	Replication      int
+	StorageClasses   map[string]bool
+	Driver           string
+	DriverParameters json.RawMessage
+}
+
+type S3VolumeDriverParameters struct {
+	AccessKey          string
+	SecretKey          string
+	Endpoint           string
+	Region             string
+	Bucket             string
+	LocationConstraint bool
+	IndexPageSize      int
+	ConnectTimeout     Duration
+	ReadTimeout        Duration
+	RaceWindow         Duration
+	UnsafeDelete       bool
+}
+
+type AzureVolumeDriverParameters struct {
+	StorageAccountName   string
+	StorageAccountKey    string
+	StorageBaseURL       string
+	ContainerName        string
+	RequestTimeout       Duration
+	ListBlobsRetryDelay  Duration
+	ListBlobsMaxAttempts int
+}
+
+type DirectoryVolumeDriverParameters struct {
+	Root      string
+	Serialize bool
+}
+
+type VolumeAccess struct {
+	ReadOnly bool
+}
+
 type Services struct {
 	Composer       Service
 	Controller     Service
@@ -225,6 +276,10 @@ func (su URL) MarshalText() ([]byte, error) {
 	return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 }
 
+func (su URL) String() string {
+	return (*url.URL)(&su).String()
+}
+
 type ServiceInstance struct{}
 
 type PostgreSQL struct {
diff --git a/sdk/go/ctxlog/log.go b/sdk/go/ctxlog/log.go
index e66eeadee..f7a7af64d 100644
--- a/sdk/go/ctxlog/log.go
+++ b/sdk/go/ctxlog/log.go
@@ -41,7 +41,7 @@ func FromContext(ctx context.Context) logrus.FieldLogger {
 
 // New returns a new logger with the indicated format and
 // level.
-func New(out io.Writer, format, level string) logrus.FieldLogger {
+func New(out io.Writer, format, level string) *logrus.Logger {
 	logger := logrus.New()
 	logger.Out = out
 	setFormat(logger, format)
diff --git a/sdk/go/httpserver/httpserver.go b/sdk/go/httpserver/httpserver.go
index a94146f85..627e04f0b 100644
--- a/sdk/go/httpserver/httpserver.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -43,7 +43,12 @@ func (srv *Server) Start() error {
 	srv.cond = sync.NewCond(mutex.RLocker())
 	srv.running = true
 	go func() {
-		err = srv.Serve(tcpKeepAliveListener{srv.listener})
+		lnr := tcpKeepAliveListener{srv.listener}
+		if srv.TLSConfig != nil {
+			err = srv.ServeTLS(lnr, "", "")
+		} else {
+			err = srv.Serve(lnr)
+		}
 		if !srv.wantDown {
 			srv.err = err
 		}
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 3c17b3bd0..66aa6c583 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -7,8 +7,8 @@ package main
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -22,99 +22,85 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
 	"github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
-const (
-	azureDefaultRequestTimeout       = arvados.Duration(10 * time.Minute)
-	azureDefaultListBlobsMaxAttempts = 12
-	azureDefaultListBlobsRetryDelay  = arvados.Duration(10 * time.Second)
-)
-
-var (
-	azureMaxGetBytes           int
-	azureStorageAccountName    string
-	azureStorageAccountKeyFile string
-	azureStorageReplication    int
-	azureWriteRaceInterval     = 15 * time.Second
-	azureWriteRacePollTime     = time.Second
-)
+func init() {
+	driver["Azure"] = newAzureBlobVolume
+}
 
-func readKeyFromFile(file string) (string, error) {
-	buf, err := ioutil.ReadFile(file)
+func newAzureBlobVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+	v := AzureBlobVolume{
+		StorageBaseURL:    storage.DefaultBaseURL,
+		RequestTimeout:    azureDefaultRequestTimeout,
+		WriteRaceInterval: azureDefaultWriteRaceInterval,
+		WriteRacePollTime: azureDefaultWriteRacePollTime,
+		cluster:           cluster,
+		volume:            volume,
+		logger:            logger,
+	}
+	err := json.Unmarshal(volume.DriverParameters, &v)
 	if err != nil {
-		return "", errors.New("reading key from " + file + ": " + err.Error())
+		return nil, err
 	}
-	accountKey := strings.TrimSpace(string(buf))
-	if accountKey == "" {
-		return "", errors.New("empty account key in " + file)
+	if v.ListBlobsRetryDelay == 0 {
+		v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
+	}
+	if v.ListBlobsMaxAttempts == 0 {
+		v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
+	}
+	if v.ContainerName == "" || v.StorageAccountName == "" || v.StorageAccountKey == "" {
+		return nil, errors.New("DriverParameters: ContainerName, StorageAccountName, and StorageAccountKey must be provided")
+	}
+	v.azClient, err = storage.NewClient(v.StorageAccountName, v.StorageAccountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
+	if err != nil {
+		return nil, fmt.Errorf("creating Azure storage client: %s", err)
+	}
+	v.azClient.Sender = &singleSender{}
+	v.azClient.HTTPClient = &http.Client{
+		Timeout: time.Duration(v.RequestTimeout),
+	}
+	bs := v.azClient.GetBlobService()
+	v.container = &azureContainer{
+		ctr: bs.GetContainerReference(v.ContainerName),
 	}
-	return accountKey, nil
-}
-
-type azureVolumeAdder struct {
-	*Config
-}
-
-// String implements flag.Value
-func (s *azureVolumeAdder) String() string {
-	return "-"
-}
 
-func (s *azureVolumeAdder) Set(containerName string) error {
-	s.Config.Volumes = append(s.Config.Volumes, &AzureBlobVolume{
-		ContainerName:         containerName,
-		StorageAccountName:    azureStorageAccountName,
-		StorageAccountKeyFile: azureStorageAccountKeyFile,
-		AzureReplication:      azureStorageReplication,
-		ReadOnly:              deprecated.flagReadonly,
-	})
-	return nil
+	if ok, err := v.container.Exists(); err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, fmt.Errorf("Azure container %q does not exist: %s", v.ContainerName, err)
+	}
+	return &v, nil
 }
 
-func init() {
-	VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &AzureBlobVolume{} })
-
-	flag.Var(&azureVolumeAdder{theConfig},
-		"azure-storage-container-volume",
-		"Use the given container as a storage volume. Can be given multiple times.")
-	flag.StringVar(
-		&azureStorageAccountName,
-		"azure-storage-account-name",
-		"",
-		"Azure storage account name used for subsequent --azure-storage-container-volume arguments.")
-	flag.StringVar(
-		&azureStorageAccountKeyFile,
-		"azure-storage-account-key-file",
-		"",
-		"`File` containing the account key used for subsequent --azure-storage-container-volume arguments.")
-	flag.IntVar(
-		&azureStorageReplication,
-		"azure-storage-replication",
-		3,
-		"Replication level to report to clients when data is stored in an Azure container.")
-	flag.IntVar(
-		&azureMaxGetBytes,
-		"azure-max-get-bytes",
-		BlockSize,
-		fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
-}
+const (
+	azureDefaultRequestTimeout       = arvados.Duration(10 * time.Minute)
+	azureDefaultListBlobsMaxAttempts = 12
+	azureDefaultListBlobsRetryDelay  = arvados.Duration(10 * time.Second)
+	azureDefaultWriteRaceInterval    = arvados.Duration(15 * time.Second)
+	azureDefaultWriteRacePollTime    = arvados.Duration(time.Second)
+)
 
 // An AzureBlobVolume stores and retrieves blocks in an Azure Blob
 // container.
 type AzureBlobVolume struct {
-	StorageAccountName    string
-	StorageAccountKeyFile string
-	StorageBaseURL        string // "" means default, "core.windows.net"
-	ContainerName         string
-	AzureReplication      int
-	ReadOnly              bool
-	RequestTimeout        arvados.Duration
-	StorageClasses        []string
-	ListBlobsRetryDelay   arvados.Duration
-	ListBlobsMaxAttempts  int
-
+	StorageAccountName   string
+	StorageAccountKey    string
+	StorageBaseURL       string // "" means default, "core.windows.net"
+	ContainerName        string
+	RequestTimeout       arvados.Duration
+	ListBlobsRetryDelay  arvados.Duration
+	ListBlobsMaxAttempts int
+	MaxGetBytes          int
+	WriteRaceInterval    arvados.Duration
+	WriteRacePollTime    arvados.Duration
+
+	cluster   *arvados.Cluster
+	volume    arvados.Volume
+	logger    logrus.FieldLogger
 	azClient  storage.Client
 	container *azureContainer
 }
@@ -127,27 +113,6 @@ func (*singleSender) Send(c *storage.Client, req *http.Request) (resp *http.Resp
 	return c.HTTPClient.Do(req)
 }
 
-// Examples implements VolumeWithExamples.
-func (*AzureBlobVolume) Examples() []Volume {
-	return []Volume{
-		&AzureBlobVolume{
-			StorageAccountName:    "example-account-name",
-			StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
-			ContainerName:         "example-container-name",
-			AzureReplication:      3,
-			RequestTimeout:        azureDefaultRequestTimeout,
-		},
-		&AzureBlobVolume{
-			StorageAccountName:    "cn-account-name",
-			StorageAccountKeyFile: "/etc/azure_cn_storage_account_key.txt",
-			StorageBaseURL:        "core.chinacloudapi.cn",
-			ContainerName:         "cn-container-name",
-			AzureReplication:      3,
-			RequestTimeout:        azureDefaultRequestTimeout,
-		},
-	}
-}
-
 // Type implements Volume.
 func (v *AzureBlobVolume) Type() string {
 	return "Azure"
@@ -155,56 +120,15 @@ func (v *AzureBlobVolume) Type() string {
 
 // Start implements Volume.
 func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
-	if v.ListBlobsRetryDelay == 0 {
-		v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
-	}
-	if v.ListBlobsMaxAttempts == 0 {
-		v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
-	}
-	if v.ContainerName == "" {
-		return errors.New("no container name given")
-	}
-	if v.StorageAccountName == "" || v.StorageAccountKeyFile == "" {
-		return errors.New("StorageAccountName and StorageAccountKeyFile must be given")
-	}
-	accountKey, err := readKeyFromFile(v.StorageAccountKeyFile)
-	if err != nil {
-		return err
-	}
-	if v.StorageBaseURL == "" {
-		v.StorageBaseURL = storage.DefaultBaseURL
-	}
-	v.azClient, err = storage.NewClient(v.StorageAccountName, accountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
-	if err != nil {
-		return fmt.Errorf("creating Azure storage client: %s", err)
-	}
-	v.azClient.Sender = &singleSender{}
-
-	if v.RequestTimeout == 0 {
-		v.RequestTimeout = azureDefaultRequestTimeout
-	}
-	v.azClient.HTTPClient = &http.Client{
-		Timeout: time.Duration(v.RequestTimeout),
-	}
-	bs := v.azClient.GetBlobService()
-	v.container = &azureContainer{
-		ctr: bs.GetContainerReference(v.ContainerName),
-	}
-
-	if ok, err := v.container.Exists(); err != nil {
-		return err
-	} else if !ok {
-		return fmt.Errorf("Azure container %q does not exist", v.ContainerName)
-	}
 	// Set up prometheus metrics
-	lbls := prometheus.Labels{"device_id": v.DeviceID()}
+	lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
 	v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(lbls)
 
 	return nil
 }
 
-// DeviceID returns a globally unique ID for the storage container.
-func (v *AzureBlobVolume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage container.
+func (v *AzureBlobVolume) GetDeviceID() string {
 	return "azure://" + v.StorageBaseURL + "/" + v.StorageAccountName + "/" + v.ContainerName
 }
 
@@ -245,14 +169,14 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
 		if !haveDeadline {
 			t, err := v.Mtime(loc)
 			if err != nil {
-				log.Print("Got empty block (possible race) but Mtime failed: ", err)
+				ctxlog.FromContext(ctx).Print("Got empty block (possible race) but Mtime failed: ", err)
 				break
 			}
-			deadline = t.Add(azureWriteRaceInterval)
+			deadline = t.Add(v.WriteRaceInterval.Duration())
 			if time.Now().After(deadline) {
 				break
 			}
-			log.Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
+			ctxlog.FromContext(ctx).Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
 			haveDeadline = true
 		} else if time.Now().After(deadline) {
 			break
@@ -260,12 +184,12 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
 		select {
 		case <-ctx.Done():
 			return 0, ctx.Err()
-		case <-time.After(azureWriteRacePollTime):
+		case <-time.After(v.WriteRacePollTime.Duration()):
 		}
 		size, err = v.get(ctx, loc, buf)
 	}
 	if haveDeadline {
-		log.Printf("Race ended with size==%d", size)
+		ctxlog.FromContext(ctx).Printf("Race ended with size==%d", size)
 	}
 	return size, err
 }
@@ -273,8 +197,15 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
 func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
+
+	pieceSize := BlockSize
+	if v.MaxGetBytes > 0 && v.MaxGetBytes < BlockSize {
+		pieceSize = v.MaxGetBytes
+	}
+
+	pieces := 1
 	expectSize := len(buf)
-	if azureMaxGetBytes < BlockSize {
+	if pieceSize < BlockSize {
 		// Unfortunately the handler doesn't tell us how long the blob
 		// is expected to be, so we have to ask Azure.
 		props, err := v.container.GetBlobProperties(loc)
@@ -285,6 +216,7 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
 			return 0, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
 		}
 		expectSize = int(props.ContentLength)
+		pieces = (expectSize + pieceSize - 1) / pieceSize
 	}
 
 	if expectSize == 0 {
@@ -293,7 +225,6 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
 
 	// We'll update this actualSize if/when we get the last piece.
 	actualSize := -1
-	pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
 	errors := make(chan error, pieces)
 	var wg sync.WaitGroup
 	wg.Add(pieces)
@@ -308,8 +239,8 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
 		// interrupted as a result.
 		go func(p int) {
 			defer wg.Done()
-			startPos := p * azureMaxGetBytes
-			endPos := startPos + azureMaxGetBytes
+			startPos := p * pieceSize
+			endPos := startPos + pieceSize
 			if endPos > expectSize {
 				endPos = expectSize
 			}
@@ -412,7 +343,7 @@ func (v *AzureBlobVolume) Compare(ctx context.Context, loc string, expect []byte
 
 // Put stores a Keep block as a block blob in the container.
 func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	// Send the block data through a pipe, so that (if we need to)
@@ -441,7 +372,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
 	}()
 	select {
 	case <-ctx.Done():
-		theConfig.debugLogf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
+		ctxlog.FromContext(ctx).Debugf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
 		// Our pipe might be stuck in Write(), waiting for
 		// io.Copy() to read. If so, un-stick it. This means
 		// CreateBlockBlobFromReader will get corrupt data,
@@ -450,7 +381,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
 		go io.Copy(ioutil.Discard, bufr)
 		// CloseWithError() will return once pending I/O is done.
 		bufw.CloseWithError(ctx.Err())
-		theConfig.debugLogf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
+		ctxlog.FromContext(ctx).Debugf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
 		return ctx.Err()
 	case err := <-errChan:
 		return err
@@ -459,7 +390,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
 
 // Touch updates the last-modified property of a block blob.
 func (v *AzureBlobVolume) Touch(loc string) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	trashed, metadata, err := v.checkTrashed(loc)
@@ -508,7 +439,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
 				continue
 			}
 			modtime := time.Time(b.Properties.LastModified)
-			if b.Properties.ContentLength == 0 && modtime.Add(azureWriteRaceInterval).After(time.Now()) {
+			if b.Properties.ContentLength == 0 && modtime.Add(v.WriteRaceInterval.Duration()).After(time.Now()) {
 				// A new zero-length blob is probably
 				// just a new non-empty blob that
 				// hasn't committed its data yet (see
@@ -535,7 +466,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
 		resp, err = v.container.ListBlobs(params)
 		err = v.translateError(err)
 		if err == VolumeBusyError {
-			log.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
+			v.logger.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
 			time.Sleep(time.Duration(v.ListBlobsRetryDelay))
 			continue
 		} else {
@@ -547,7 +478,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
 
 // Trash a Keep block.
 func (v *AzureBlobVolume) Trash(loc string) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 
@@ -562,12 +493,12 @@ func (v *AzureBlobVolume) Trash(loc string) error {
 	}
 	if t, err := v.Mtime(loc); err != nil {
 		return err
-	} else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+	} else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
 		return nil
 	}
 
 	// If TrashLifetime == 0, just delete it
-	if theConfig.TrashLifetime == 0 {
+	if v.cluster.Collections.BlobTrashLifetime == 0 {
 		return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{
 			IfMatch: props.Etag,
 		})
@@ -575,7 +506,7 @@ func (v *AzureBlobVolume) Trash(loc string) error {
 
 	// Otherwise, mark as trash
 	return v.container.SetBlobMetadata(loc, storage.BlobMetadata{
-		"expires_at": fmt.Sprintf("%d", time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()),
+		"expires_at": fmt.Sprintf("%d", time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()),
 	}, &storage.SetBlobMetadataOptions{
 		IfMatch: props.Etag,
 	})
@@ -613,23 +544,6 @@ func (v *AzureBlobVolume) String() string {
 	return fmt.Sprintf("azure-storage-container:%+q", v.ContainerName)
 }
 
-// Writable returns true, unless the -readonly flag was on when the
-// volume was added.
-func (v *AzureBlobVolume) Writable() bool {
-	return !v.ReadOnly
-}
-
-// Replication returns the replication level of the container, as
-// specified by the -azure-storage-replication argument.
-func (v *AzureBlobVolume) Replication() int {
-	return v.AzureReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *AzureBlobVolume) GetStorageClasses() []string {
-	return v.StorageClasses
-}
-
 // If possible, translate an Azure SDK error to a recognizable error
 // like os.ErrNotExist.
 func (v *AzureBlobVolume) translateError(err error) error {
@@ -656,6 +570,10 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool {
 // EmptyTrash looks for trashed blocks that exceeded TrashLifetime
 // and deletes them from the volume.
 func (v *AzureBlobVolume) EmptyTrash() {
+	if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+		return
+	}
+
 	var bytesDeleted, bytesInTrash int64
 	var blocksDeleted, blocksInTrash int64
 
@@ -670,7 +588,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
 
 		expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
 		if err != nil {
-			log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
+			v.logger.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
 			return
 		}
 
@@ -682,7 +600,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
 			IfMatch: b.Properties.Etag,
 		})
 		if err != nil {
-			log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+			v.logger.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
 			return
 		}
 		atomic.AddInt64(&blocksDeleted, 1)
@@ -690,8 +608,8 @@ func (v *AzureBlobVolume) EmptyTrash() {
 	}
 
 	var wg sync.WaitGroup
-	todo := make(chan storage.Blob, theConfig.EmptyTrashWorkers)
-	for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+	todo := make(chan storage.Blob, v.cluster.Collections.BlobDeleteConcurrency)
+	for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -705,7 +623,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
 	for page := 1; ; page++ {
 		resp, err := v.listBlobs(page, params)
 		if err != nil {
-			log.Printf("EmptyTrash: ListBlobs: %v", err)
+			v.logger.Printf("EmptyTrash: ListBlobs: %v", err)
 			break
 		}
 		for _, b := range resp.Blobs {
@@ -719,7 +637,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
 	close(todo)
 	wg.Wait()
 
-	log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+	v.logger.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
 }
 
 // InternalStats returns bucket I/O and API call counters.
@@ -748,7 +666,6 @@ func (s *azureBlobStats) TickErr(err error) {
 	if err, ok := err.(storage.AzureStorageServiceError); ok {
 		errType = errType + fmt.Sprintf(" %d (%s)", err.StatusCode, err.Code)
 	}
-	log.Printf("errType %T, err %s", err, err)
 	s.statsTicker.TickErr(err, errType)
 }
 
diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go
index d2e7c9ebd..c05d00412 100644
--- a/services/keepstore/bufferpool.go
+++ b/services/keepstore/bufferpool.go
@@ -8,9 +8,12 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/sirupsen/logrus"
 )
 
 type bufferPool struct {
+	log logrus.FieldLogger
 	// limiter has a "true" placeholder for each in-use buffer.
 	limiter chan bool
 	// allocated is the number of bytes currently allocated to buffers.
@@ -19,7 +22,7 @@ type bufferPool struct {
 	sync.Pool
 }
 
-func newBufferPool(count int, bufSize int) *bufferPool {
+func newBufferPool(log logrus.FieldLogger, count int, bufSize int) *bufferPool {
 	p := bufferPool{}
 	p.New = func() interface{} {
 		atomic.AddUint64(&p.allocated, uint64(bufSize))
@@ -34,13 +37,13 @@ func (p *bufferPool) Get(size int) []byte {
 	case p.limiter <- true:
 	default:
 		t0 := time.Now()
-		log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
+		p.log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
 		p.limiter <- true
-		log.Printf("waited %v for a buffer", time.Since(t0))
+		p.log.Printf("waited %v for a buffer", time.Since(t0))
 	}
 	buf := p.Pool.Get().([]byte)
 	if cap(buf) < size {
-		log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
+		p.log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
 	}
 	return buf[:size]
 }
diff --git a/services/keepstore/command.go b/services/keepstore/command.go
new file mode 100644
index 000000000..7ffb10996
--- /dev/null
+++ b/services/keepstore/command.go
@@ -0,0 +1,191 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"os"
+	"sync"
+
+	"git.curoverse.com/arvados.git/lib/service"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	version = "dev"
+	Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler)
+)
+
+func main() {
+	os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background()))
+	if !ok {
+		return 2
+	}
+	return Command.RunCommand(prog, args, stdin, stdout, stderr)
+}
+
+// Parse keepstore command line flags, and return equivalent
+// service.Command flags. The second return value ("ok") is true if
+// all provided flags were successfully converted.
+func convertKeepstoreFlagsToServiceFlags(args []string, lgr logrus.FieldLogger) ([]string, bool) {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.String("listen", "", "Services.Keepstore.InternalURLs")
+	flags.Int("max-buffers", 0, "API.MaxKeepBlockBuffers")
+	flags.Int("max-requests", 0, "API.MaxConcurrentRequests")
+	flags.Bool("never-delete", false, "Collections.BlobTrash")
+	flags.Bool("enforce-permissions", false, "Collections.BlobSigning")
+	flags.String("permission-key-file", "", "Collections.BlobSigningKey")
+	flags.String("blob-signing-key-file", "", "Collections.BlobSigningKey")
+	flags.String("data-manager-token-file", "", "SystemRootToken")
+	flags.Int("permission-ttl", 0, "Collections.BlobSigningTTL")
+	flags.Int("blob-signature-ttl", 0, "Collections.BlobSigningTTL")
+	flags.String("trash-lifetime", "", "Collections.BlobTrashLifetime")
+	flags.Bool("serialize", false, "Volumes.*.DriverParameters.Serialize")
+	flags.Bool("readonly", false, "Volumes.*.ReadOnly")
+	flags.String("pid", "", "-")
+	flags.String("trash-check-interval", "", "Collections.BlobTrashCheckInterval")
+
+	flags.String("azure-storage-container-volume", "", "Volumes.*.Driver")
+	flags.String("azure-storage-account-name", "", "Volumes.*.DriverParameters.StorageAccountName")
+	flags.String("azure-storage-account-key-file", "", "Volumes.*.DriverParameters.StorageAccountKey")
+	flags.String("azure-storage-replication", "", "Volumes.*.Replication")
+	flags.String("azure-max-get-bytes", "", "Volumes.*.DriverParameters.MaxDataReadSize")
+
+	flag.String("s3-bucket-volume", "", "Volumes.*.DriverParameters.Bucket")
+	flag.String("s3-region", "", "Volumes.*.DriverParameters.Region")
+	flag.String("s3-endpoint", "", "Volumes.*.DriverParameters.Endpoint")
+	flag.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKey")
+	flag.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretKey")
+	flag.String("s3-race-window", "", "Volumes.*.DriverParameters.RaceWindow")
+	flag.String("s3-replication", "", "Volumes.*.Replication")
+	flag.String("s3-unsafe-delete", "", "Volumes.*.DriverParameters.UnsafeDelete")
+
+	err := flags.Parse(args)
+	if err == flag.ErrHelp {
+		return []string{"-help"}, true
+	} else if err != nil {
+		return nil, false
+	}
+
+	args = nil
+	ok := true
+	flags.Visit(func(f *flag.Flag) {
+		if f.Name == "config" {
+			args = []string{"-legacy-keepstore-config-file", f.Value.String()}
+		} else if f.Usage == "-" {
+			ok = false
+			lgr.Errorf("command line flag -%s is no longer supported", f.Name)
+		} else {
+			ok = false
+			lgr.Errorf("command line flag -%s is no longer supported -- use Clusters.*.%s in cluster config file instead", f.Name, f.Usage)
+		}
+	})
+	return args, ok
+}
+
+type handler struct {
+	http.Handler
+	Cluster *arvados.Cluster
+	Logger  logrus.FieldLogger
+
+	volmgr     *RRVolumeManager
+	keepClient *keepclient.KeepClient
+
+	err       error
+	setupOnce sync.Once
+}
+
+func (h *handler) CheckHealth() error {
+	return h.err
+}
+
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+	var h handler
+	err := h.setup(ctx, cluster, token, reg)
+	if err != nil {
+		return service.ErrorHandler(ctx, cluster, err)
+	}
+	return &h
+}
+
+func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) error {
+	h.Cluster = cluster
+	h.Logger = ctxlog.FromContext(ctx)
+	serviceURL, ok := service.URLFromContext(ctx)
+	if !ok {
+		return errors.New("BUG: no URL from service.URLFromContext")
+	}
+	if h.Cluster.API.MaxKeepBlockBuffers <= 0 {
+		return fmt.Errorf("MaxBuffers must be greater than zero")
+	}
+	bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlockBuffers, BlockSize)
+
+	if h.Cluster.API.MaxConcurrentRequests < 1 {
+		h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlockBuffers * 2
+		h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlockBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
+	}
+
+	if h.Cluster.Collections.BlobSigningKey != "" {
+	} else if h.Cluster.Collections.BlobSigning {
+		return errors.New("cannot enable Collections.BlobSigning with no Collections.BlobSigningKey")
+	} else {
+		h.Logger.Warn("Running without a blob signing key. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions. To fix this, configure Collections.BlobSigning and Collections.BlobSigningKey.")
+	}
+
+	if len(h.Cluster.Volumes) == 0 {
+		return errors.New("no volumes configured")
+	}
+
+	h.Logger.Printf("keepstore %s starting, pid %d", version, os.Getpid())
+	defer h.Logger.Println("keepstore exiting, pid", os.Getpid())
+
+	// Start a round-robin VolumeManager with the volumes we have found.
+	h.volmgr = MakeRRVolumeManager(h.Cluster, serviceURL, newVolumeMetricsVecs(reg))
+	if len(h.volmgr.readables) == 0 {
+		return fmt.Errorf("no volumes configured for %s", serviceURL)
+	}
+
+	// Middleware/handler stack
+	h.Handler = MakeRESTRouter(ctx, cluster, reg)
+
+	// Initialize keepclient for pull workers
+	h.keepClient = &keepclient.KeepClient{
+		Arvados:       &arvadosclient.ArvadosClient{ApiToken: fmt.Sprintf("%x", rand.Int63())},
+		Want_replicas: 1,
+	}
+
+	// Initialize the pullq and workers
+	pullq = NewWorkQueue()
+	for i := 0; i < 1 || i < h.Cluster.Collections.BlobReplicateConcurrency; i++ {
+		go h.runPullWorker(pullq)
+	}
+
+	// Initialize the trashq and workers
+	trashq = NewWorkQueue()
+	for i := 0; i < 1 || i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
+		go RunTrashWorker(h.Cluster, trashq)
+	}
+
+	if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 {
+		go emptyTrash(h.volmgr.writables, d)
+	}
+
+	return nil
+}
diff --git a/services/keepstore/command_test.go b/services/keepstore/command_test.go
new file mode 100644
index 000000000..a7a403da7
--- /dev/null
+++ b/services/keepstore/command_test.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+
+	"gopkg.in/check.v1"
+)
+
+var _ = Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (*CommandSuite) TestLegacyConfigPath(c *check.C) {
+	var stdin, stdout, stderr bytes.Buffer
+	tmp, err := ioutil.TempFile("", "")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(tmp.Name())
+	tmp.Write([]byte("Listen: \"1.2.3.4.5:invalidport\"\n"))
+	tmp.Close()
+	exited := runCommand("keepstore", []string{"-config", tmp.Name()}, &stdin, &stdout, &stderr)
+	c.Check(exited, check.Equals, 1)
+	c.Check(stderr.String(), check.Matches, `(?ms).*unable to migrate Listen value.*`)
+}
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
deleted file mode 100644
index 43a219111..000000000
--- a/services/keepstore/config.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"strings"
-	"time"
-
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-)
-
-type Config struct {
-	Debug  bool
-	Listen string
-
-	LogFormat string
-
-	PIDFile string
-
-	MaxBuffers  int
-	MaxRequests int
-
-	BlobSignatureTTL    arvados.Duration
-	BlobSigningKeyFile  string
-	RequireSignatures   bool
-	SystemAuthTokenFile string
-	EnableDelete        bool
-	TrashLifetime       arvados.Duration
-	TrashCheckInterval  arvados.Duration
-	PullWorkers         int
-	TrashWorkers        int
-	EmptyTrashWorkers   int
-	TLSCertificateFile  string
-	TLSKeyFile          string
-
-	Volumes VolumeList
-
-	blobSigningKey  []byte
-	systemAuthToken string
-	debugLogf       func(string, ...interface{})
-
-	ManagementToken string
-}
-
-var (
-	theConfig = DefaultConfig()
-	formatter = map[string]logrus.Formatter{
-		"text": &logrus.TextFormatter{
-			FullTimestamp:   true,
-			TimestampFormat: rfc3339NanoFixed,
-		},
-		"json": &logrus.JSONFormatter{
-			TimestampFormat: rfc3339NanoFixed,
-		},
-	}
-	log = logrus.StandardLogger()
-)
-
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
-// DefaultConfig returns the default configuration.
-func DefaultConfig() *Config {
-	return &Config{
-		Listen:             ":25107",
-		LogFormat:          "json",
-		MaxBuffers:         128,
-		RequireSignatures:  true,
-		BlobSignatureTTL:   arvados.Duration(14 * 24 * time.Hour),
-		TrashLifetime:      arvados.Duration(14 * 24 * time.Hour),
-		TrashCheckInterval: arvados.Duration(24 * time.Hour),
-		Volumes:            []Volume{},
-	}
-}
-
-// Start should be called exactly once: after setting all public
-// fields, and before using the config.
-func (cfg *Config) Start(reg *prometheus.Registry) error {
-	if cfg.Debug {
-		log.Level = logrus.DebugLevel
-		cfg.debugLogf = log.Printf
-		cfg.debugLogf("debugging enabled")
-	} else {
-		log.Level = logrus.InfoLevel
-		cfg.debugLogf = func(string, ...interface{}) {}
-	}
-
-	f := formatter[strings.ToLower(cfg.LogFormat)]
-	if f == nil {
-		return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
-	}
-	log.Formatter = f
-
-	if cfg.MaxBuffers < 0 {
-		return fmt.Errorf("MaxBuffers must be greater than zero")
-	}
-	bufs = newBufferPool(cfg.MaxBuffers, BlockSize)
-
-	if cfg.MaxRequests < 1 {
-		cfg.MaxRequests = cfg.MaxBuffers * 2
-		log.Printf("MaxRequests <1 or not specified; defaulting to MaxBuffers * 2 == %d", cfg.MaxRequests)
-	}
-
-	if cfg.BlobSigningKeyFile != "" {
-		buf, err := ioutil.ReadFile(cfg.BlobSigningKeyFile)
-		if err != nil {
-			return fmt.Errorf("reading blob signing key file: %s", err)
-		}
-		cfg.blobSigningKey = bytes.TrimSpace(buf)
-		if len(cfg.blobSigningKey) == 0 {
-			return fmt.Errorf("blob signing key file %q is empty", cfg.BlobSigningKeyFile)
-		}
-	} else if cfg.RequireSignatures {
-		return fmt.Errorf("cannot enable RequireSignatures (-enforce-permissions) without a blob signing key")
-	} else {
-		log.Println("Running without a blob signing key. Block locators " +
-			"returned by this server will not be signed, and will be rejected " +
-			"by a server that enforces permissions.")
-		log.Println("To fix this, use the BlobSigningKeyFile config entry.")
-	}
-
-	if fn := cfg.SystemAuthTokenFile; fn != "" {
-		buf, err := ioutil.ReadFile(fn)
-		if err != nil {
-			return fmt.Errorf("cannot read system auth token file %q: %s", fn, err)
-		}
-		cfg.systemAuthToken = strings.TrimSpace(string(buf))
-	}
-
-	if cfg.EnableDelete {
-		log.Print("Trash/delete features are enabled. WARNING: this has not " +
-			"been extensively tested. You should disable this unless you can afford to lose data.")
-	}
-
-	if len(cfg.Volumes) == 0 {
-		if (&unixVolumeAdder{cfg}).Discover() == 0 {
-			return fmt.Errorf("no volumes found")
-		}
-	}
-	vm := newVolumeMetricsVecs(reg)
-	for _, v := range cfg.Volumes {
-		if err := v.Start(vm); err != nil {
-			return fmt.Errorf("volume %s: %s", v, err)
-		}
-		log.Printf("Using volume %v (writable=%v)", v, v.Writable())
-	}
-	return nil
-}
-
-// VolumeTypes is built up by init() funcs in the source files that
-// define the volume types.
-var VolumeTypes = []func() VolumeWithExamples{}
-
-type VolumeList []Volume
-
-// UnmarshalJSON -- given an array of objects -- deserializes each
-// object as the volume type indicated by the object's Type field.
-func (vl *VolumeList) UnmarshalJSON(data []byte) error {
-	typeMap := map[string]func() VolumeWithExamples{}
-	for _, factory := range VolumeTypes {
-		t := factory().Type()
-		if _, ok := typeMap[t]; ok {
-			log.Fatalf("volume type %+q is claimed by multiple VolumeTypes", t)
-		}
-		typeMap[t] = factory
-	}
-
-	var mapList []map[string]interface{}
-	err := json.Unmarshal(data, &mapList)
-	if err != nil {
-		return err
-	}
-	for _, mapIn := range mapList {
-		typeIn, ok := mapIn["Type"].(string)
-		if !ok {
-			return fmt.Errorf("invalid volume type %+v", mapIn["Type"])
-		}
-		factory, ok := typeMap[typeIn]
-		if !ok {
-			return fmt.Errorf("unsupported volume type %+q", typeIn)
-		}
-		data, err := json.Marshal(mapIn)
-		if err != nil {
-			return err
-		}
-		vol := factory()
-		err = json.Unmarshal(data, vol)
-		if err != nil {
-			return err
-		}
-		*vl = append(*vl, vol)
-	}
-	return nil
-}
-
-// MarshalJSON adds a "Type" field to each volume corresponding to its
-// Type().
-func (vl *VolumeList) MarshalJSON() ([]byte, error) {
-	data := []byte{'['}
-	for _, vs := range *vl {
-		j, err := json.Marshal(vs)
-		if err != nil {
-			return nil, err
-		}
-		if len(data) > 1 {
-			data = append(data, byte(','))
-		}
-		t, err := json.Marshal(vs.Type())
-		if err != nil {
-			panic(err)
-		}
-		data = append(data, j[0])
-		data = append(data, []byte(`"Type":`)...)
-		data = append(data, t...)
-		data = append(data, byte(','))
-		data = append(data, j[1:]...)
-	}
-	return append(data, byte(']')), nil
-}
diff --git a/services/keepstore/deprecated.go b/services/keepstore/deprecated.go
index d1377978a..1fd8a0668 100644
--- a/services/keepstore/deprecated.go
+++ b/services/keepstore/deprecated.go
@@ -3,45 +3,3 @@
 // SPDX-License-Identifier: AGPL-3.0
 
 package main
-
-import (
-	"flag"
-	"time"
-
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
-)
-
-type deprecatedOptions struct {
-	flagSerializeIO     bool
-	flagReadonly        bool
-	neverDelete         bool
-	signatureTTLSeconds int
-}
-
-var deprecated = deprecatedOptions{
-	neverDelete:         !theConfig.EnableDelete,
-	signatureTTLSeconds: int(theConfig.BlobSignatureTTL.Duration() / time.Second),
-}
-
-func (depr *deprecatedOptions) beforeFlagParse(cfg *Config) {
-	flag.StringVar(&cfg.Listen, "listen", cfg.Listen, "see Listen configuration")
-	flag.IntVar(&cfg.MaxBuffers, "max-buffers", cfg.MaxBuffers, "see MaxBuffers configuration")
-	flag.IntVar(&cfg.MaxRequests, "max-requests", cfg.MaxRequests, "see MaxRequests configuration")
-	flag.BoolVar(&depr.neverDelete, "never-delete", depr.neverDelete, "see EnableDelete configuration")
-	flag.BoolVar(&cfg.RequireSignatures, "enforce-permissions", cfg.RequireSignatures, "see RequireSignatures configuration")
-	flag.StringVar(&cfg.BlobSigningKeyFile, "permission-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
-	flag.StringVar(&cfg.BlobSigningKeyFile, "blob-signing-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
-	flag.StringVar(&cfg.SystemAuthTokenFile, "data-manager-token-file", cfg.SystemAuthTokenFile, "see SystemAuthToken`File` configuration")
-	flag.IntVar(&depr.signatureTTLSeconds, "permission-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
-	flag.IntVar(&depr.signatureTTLSeconds, "blob-signature-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
-	flag.Var(&cfg.TrashLifetime, "trash-lifetime", "see TrashLifetime configuration")
-	flag.BoolVar(&depr.flagSerializeIO, "serialize", depr.flagSerializeIO, "serialize read and write operations on the following volumes.")
-	flag.BoolVar(&depr.flagReadonly, "readonly", depr.flagReadonly, "do not write, delete, or touch anything on the following volumes.")
-	flag.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "see `PIDFile` configuration")
-	flag.Var(&cfg.TrashCheckInterval, "trash-check-interval", "see TrashCheckInterval configuration")
-}
-
-func (depr *deprecatedOptions) afterFlagParse(cfg *Config) {
-	cfg.BlobSignatureTTL = arvados.Duration(depr.signatureTTLSeconds) * arvados.Duration(time.Second)
-	cfg.EnableDelete = !depr.neverDelete
-}
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 72088e2b5..926fe3044 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
 	"regexp"
@@ -26,22 +27,25 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
 type router struct {
 	*mux.Router
 	limiter     httpserver.RequestCounter
 	cluster     *arvados.Cluster
+	logger      logrus.FieldLogger
 	remoteProxy remoteProxy
 	metrics     *nodeMetrics
 }
 
 // MakeRESTRouter returns a new router that forwards all Keep requests
 // to the appropriate handlers.
-func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
+func MakeRESTRouter(ctx context.Context, cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
 	rtr := &router{
 		Router:  mux.NewRouter(),
 		cluster: cluster,
+		logger:  ctxlog.FromContext(ctx),
 		metrics: &nodeMetrics{reg: reg},
 	}
 
@@ -52,12 +56,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
 		rtr.handleGET).Methods("GET", "HEAD")
 
 	rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handlePUT).Methods("PUT")
-	rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
+	rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleDELETE).Methods("DELETE")
 	// List all blocks stored here. Privileged client only.
-	rtr.HandleFunc(`/index`, rtr.IndexHandler).Methods("GET", "HEAD")
+	rtr.HandleFunc(`/index`, rtr.handleIndex).Methods("GET", "HEAD")
 	// List blocks stored here whose hash has the given prefix.
 	// Privileged client only.
-	rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.IndexHandler).Methods("GET", "HEAD")
+	rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
 
 	// Internals/debugging info (runtime.MemStats)
 	rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
@@ -67,20 +71,20 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
 
 	// List mounts: UUID, readonly, tier, device ID, ...
 	rtr.HandleFunc(`/mounts`, rtr.MountsHandler).Methods("GET")
-	rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.IndexHandler).Methods("GET")
-	rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.IndexHandler).Methods("GET")
+	rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.handleIndex).Methods("GET")
+	rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.handleIndex).Methods("GET")
 
 	// Replace the current pull queue.
-	rtr.HandleFunc(`/pull`, PullHandler).Methods("PUT")
+	rtr.HandleFunc(`/pull`, rtr.handlePull).Methods("PUT")
 
 	// Replace the current trash queue.
-	rtr.HandleFunc(`/trash`, TrashHandler).Methods("PUT")
+	rtr.HandleFunc(`/trash`, rtr.handleTrash).Methods("PUT")
 
 	// Untrash moves blocks from trash back into store
-	rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, UntrashHandler).Methods("PUT")
+	rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, rtr.handleUntrash).Methods("PUT")
 
 	rtr.Handle("/_health/{check}", &health.Handler{
-		Token:  theConfig.ManagementToken,
+		Token:  cluster.ManagementToken,
 		Prefix: "/_health/",
 	}).Methods("GET")
 
@@ -88,17 +92,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
 	// 400 Bad Request.
 	rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
 
-	rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
 	rtr.metrics.setupBufferPoolMetrics(bufs)
 	rtr.metrics.setupWorkQueueMetrics(pullq, "pull")
 	rtr.metrics.setupWorkQueueMetrics(trashq, "trash")
 	rtr.metrics.setupRequestMetrics(rtr.limiter)
 
-	instrumented := httpserver.Instrument(rtr.metrics.reg, log,
-		httpserver.HandlerWithContext(
-			ctxlog.Context(context.Background(), log),
-			httpserver.AddRequestIDs(httpserver.LogRequests(rtr.limiter))))
-	return instrumented.ServeAPI(theConfig.ManagementToken, instrumented)
+	return rtr
 }
 
 // BadRequestHandler is a HandleFunc to address bad requests.
@@ -116,9 +115,9 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	if theConfig.RequireSignatures {
+	if rtr.cluster.Collections.BlobSigning {
 		locator := req.URL.Path[1:] // strip leading slash
-		if err := VerifySignature(locator, GetAPIToken(req)); err != nil {
+		if err := VerifySignature(rtr.cluster, locator, GetAPIToken(req)); err != nil {
 			http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
 			return
 		}
@@ -160,7 +159,6 @@ func contextForResponse(parent context.Context, resp http.ResponseWriter) (conte
 		go func(c <-chan bool) {
 			select {
 			case <-c:
-				theConfig.debugLogf("cancel context")
 				cancel()
 			case <-ctx.Done():
 			}
@@ -244,9 +242,9 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
 	// return it to the client.
 	returnHash := fmt.Sprintf("%s+%d", hash, req.ContentLength)
 	apiToken := GetAPIToken(req)
-	if theConfig.blobSigningKey != nil && apiToken != "" {
-		expiry := time.Now().Add(theConfig.BlobSignatureTTL.Duration())
-		returnHash = SignLocator(returnHash, apiToken, expiry)
+	if rtr.cluster.Collections.BlobSigningKey != "" && apiToken != "" {
+		expiry := time.Now().Add(rtr.cluster.Collections.BlobSigningTTL.Duration())
+		returnHash = SignLocator(rtr.cluster, returnHash, apiToken, expiry)
 	}
 	resp.Header().Set("X-Keep-Replicas-Stored", strconv.Itoa(replication))
 	resp.Write([]byte(returnHash + "\n"))
@@ -254,8 +252,8 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
 
 // IndexHandler responds to "/index", "/index/{prefix}", and
 // "/mounts/{uuid}/blocks" requests.
-func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
-	if !IsSystemAuth(GetAPIToken(req)) {
+func (rtr *router) handleIndex(resp http.ResponseWriter, req *http.Request) {
+	if !rtr.isSystemAuth(GetAPIToken(req)) {
 		http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
 		return
 	}
@@ -268,14 +266,14 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
 
 	uuid := mux.Vars(req)["uuid"]
 
-	var vols []Volume
+	var vols []*VolumeMount
 	if uuid == "" {
 		vols = KeepVM.AllReadable()
 	} else if v := KeepVM.Lookup(uuid, false); v == nil {
 		http.Error(resp, "mount not found", http.StatusNotFound)
 		return
 	} else {
-		vols = []Volume{v}
+		vols = []*VolumeMount{v}
 	}
 
 	for _, v := range vols {
@@ -375,7 +373,7 @@ func (rtr *router) readNodeStatus(st *NodeStatus) {
 	st.Volumes = st.Volumes[:0]
 	for _, vol := range vols {
 		var internalStats interface{}
-		if vol, ok := vol.(InternalStatser); ok {
+		if vol, ok := vol.Volume.(InternalStatser); ok {
 			internalStats = vol.InternalStats()
 		}
 		st.Volumes = append(st.Volumes, &volumeStatusEnt{
@@ -407,7 +405,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
 	return q.Status()
 }
 
-// DeleteHandler processes DELETE requests.
+// handleDELETE processes DELETE requests.
 //
 // DELETE /{hash:[0-9a-f]{32} will delete the block with the specified hash
 // from all connected volumes.
@@ -418,7 +416,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
 // a PermissionError.
 //
 // Upon receiving a valid request from an authorized user,
-// DeleteHandler deletes all copies of the specified block on local
+// handleDELETE deletes all copies of the specified block on local
 // writable volumes.
 //
 // Response format:
@@ -434,17 +432,17 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
 // where d and f are integers representing the number of blocks that
 // were successfully and unsuccessfully deleted.
 //
-func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleDELETE(resp http.ResponseWriter, req *http.Request) {
 	hash := mux.Vars(req)["hash"]
 
 	// Confirm that this user is an admin and has a token with unlimited scope.
 	var tok = GetAPIToken(req)
-	if tok == "" || !CanDelete(tok) {
+	if tok == "" || !rtr.canDelete(tok) {
 		http.Error(resp, PermissionError.Error(), PermissionError.HTTPCode)
 		return
 	}
 
-	if !theConfig.EnableDelete {
+	if !rtr.cluster.Collections.BlobTrash {
 		http.Error(resp, MethodDisabledError.Error(), MethodDisabledError.HTTPCode)
 		return
 	}
@@ -530,9 +528,9 @@ type PullRequest struct {
 }
 
 // PullHandler processes "PUT /pull" requests for the data manager.
-func PullHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handlePull(resp http.ResponseWriter, req *http.Request) {
 	// Reject unauthorized requests.
-	if !IsSystemAuth(GetAPIToken(req)) {
+	if !rtr.isSystemAuth(GetAPIToken(req)) {
 		http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
 		return
 	}
@@ -569,9 +567,9 @@ type TrashRequest struct {
 }
 
 // TrashHandler processes /trash requests.
-func TrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleTrash(resp http.ResponseWriter, req *http.Request) {
 	// Reject unauthorized requests.
-	if !IsSystemAuth(GetAPIToken(req)) {
+	if !rtr.isSystemAuth(GetAPIToken(req)) {
 		http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
 		return
 	}
@@ -599,9 +597,9 @@ func TrashHandler(resp http.ResponseWriter, req *http.Request) {
 }
 
 // UntrashHandler processes "PUT /untrash/{hash:[0-9a-f]{32}}" requests for the data manager.
-func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleUntrash(resp http.ResponseWriter, req *http.Request) {
 	// Reject unauthorized requests.
-	if !IsSystemAuth(GetAPIToken(req)) {
+	if !rtr.isSystemAuth(GetAPIToken(req)) {
 		http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
 		return
 	}
@@ -759,7 +757,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
 	// If this volume fails, try all of the volumes in order.
 	if vol := KeepVM.NextWritable(); vol != nil {
 		if err := vol.Put(ctx, hash, block); err == nil {
-			return vol.Replication(), nil // success!
+			return vol.Replication, nil // success!
 		}
 		if ctx.Err() != nil {
 			return 0, ErrClientDisconnect
@@ -779,7 +777,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
 			return 0, ErrClientDisconnect
 		}
 		if err == nil {
-			return vol.Replication(), nil // success!
+			return vol.Replication, nil // success!
 		}
 		if err != FullError {
 			// The volume is not full but the
@@ -834,7 +832,7 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
 			continue
 		}
 		// Compare and Touch both worked --> done.
-		return vol.Replication(), nil
+		return vol.Replication, nil
 	}
 	return 0, bestErr
 }
@@ -875,15 +873,15 @@ func IsExpired(timestampHex string) bool {
 	return time.Unix(ts, 0).Before(time.Now())
 }
 
-// CanDelete returns true if the user identified by apiToken is
+// canDelete returns true if the user identified by apiToken is
 // allowed to delete blocks.
-func CanDelete(apiToken string) bool {
+func (rtr *router) canDelete(apiToken string) bool {
 	if apiToken == "" {
 		return false
 	}
 	// Blocks may be deleted only when Keep has been configured with a
 	// data manager.
-	if IsSystemAuth(apiToken) {
+	if rtr.isSystemAuth(apiToken) {
 		return true
 	}
 	// TODO(twp): look up apiToken with the API server
@@ -892,8 +890,8 @@ func CanDelete(apiToken string) bool {
 	return false
 }
 
-// IsSystemAuth returns true if the given token is allowed to perform
+// isSystemAuth returns true if the given token is allowed to perform
 // system level actions like deleting data.
-func IsSystemAuth(token string) bool {
-	return token != "" && token == theConfig.systemAuthToken
+func (rtr *router) isSystemAuth(token string) bool {
+	return token != "" && token == rtr.cluster.SystemRootToken
 }
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index fcbdddacb..e1f830bd6 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -5,24 +5,9 @@
 package main
 
 import (
-	"flag"
-	"fmt"
-	"net"
-	"os"
-	"os/signal"
-	"syscall"
 	"time"
-
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/config"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
-	"github.com/coreos/go-systemd/daemon"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
-var version = "dev"
-
 // A Keep "block" is 64MB.
 const BlockSize = 64 * 1024 * 1024
 
@@ -30,9 +15,6 @@ const BlockSize = 64 * 1024 * 1024
 // in order to permit writes.
 const MinFreeKilobytes = BlockSize / 1024
 
-// ProcMounts /proc/mounts
-var ProcMounts = "/proc/mounts"
-
 var bufs *bufferPool
 
 // KeepError types.
@@ -87,162 +69,11 @@ var KeepVM VolumeManager
 var pullq *WorkQueue
 var trashq *WorkQueue
 
-func main() {
-	deprecated.beforeFlagParse(theConfig)
-
-	dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
-	getVersion := flag.Bool("version", false, "Print version information and exit.")
-
-	defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
-	var configPath string
-	flag.StringVar(
-		&configPath,
-		"config",
-		defaultConfigPath,
-		"YAML or JSON configuration file `path`")
-	flag.Usage = usage
-	flag.Parse()
-
-	// Print version information if requested
-	if *getVersion {
-		fmt.Printf("keepstore %s\n", version)
-		return
-	}
-
-	deprecated.afterFlagParse(theConfig)
-
-	err := config.LoadFile(theConfig, configPath)
-	if err != nil && (!os.IsNotExist(err) || configPath != defaultConfigPath) {
-		log.Fatal(err)
-	}
-
-	if *dumpConfig {
-		log.Fatal(config.DumpAndExit(theConfig))
-	}
-
-	log.Printf("keepstore %s started", version)
-
-	metricsRegistry := prometheus.NewRegistry()
-
-	err = theConfig.Start(metricsRegistry)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if pidfile := theConfig.PIDFile; pidfile != "" {
-		f, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0777)
-		if err != nil {
-			log.Fatalf("open pidfile (%s): %s", pidfile, err)
-		}
-		defer f.Close()
-		err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
-		if err != nil {
-			log.Fatalf("flock pidfile (%s): %s", pidfile, err)
-		}
-		defer os.Remove(pidfile)
-		err = f.Truncate(0)
-		if err != nil {
-			log.Fatalf("truncate pidfile (%s): %s", pidfile, err)
-		}
-		_, err = fmt.Fprint(f, os.Getpid())
-		if err != nil {
-			log.Fatalf("write pidfile (%s): %s", pidfile, err)
-		}
-		err = f.Sync()
-		if err != nil {
-			log.Fatalf("sync pidfile (%s): %s", pidfile, err)
-		}
-	}
-
-	var cluster *arvados.Cluster
-	cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
-	if err != nil && os.IsNotExist(err) {
-		log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
-		cluster = &arvados.Cluster{
-			ClusterID: "xxxxx",
-		}
-	} else if err != nil {
-		log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
-	} else {
-		cluster, err = cfg.GetCluster("")
-		if err != nil {
-			log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
-		}
-	}
-
-	log.Println("keepstore starting, pid", os.Getpid())
-	defer log.Println("keepstore exiting, pid", os.Getpid())
-
-	// Start a round-robin VolumeManager with the volumes we have found.
-	KeepVM = MakeRRVolumeManager(theConfig.Volumes)
-
-	// Middleware/handler stack
-	router := MakeRESTRouter(cluster, metricsRegistry)
-
-	// Set up a TCP listener.
-	listener, err := net.Listen("tcp", theConfig.Listen)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// Initialize keepclient for pull workers
-	keepClient := &keepclient.KeepClient{
-		Arvados:       &arvadosclient.ArvadosClient{},
-		Want_replicas: 1,
-	}
-
-	// Initialize the pullq and workers
-	pullq = NewWorkQueue()
-	for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
-		go RunPullWorker(pullq, keepClient)
-	}
-
-	// Initialize the trashq and workers
-	trashq = NewWorkQueue()
-	for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
-		go RunTrashWorker(trashq)
-	}
-
-	// Start emptyTrash goroutine
-	doneEmptyingTrash := make(chan bool)
-	go emptyTrash(doneEmptyingTrash, theConfig.TrashCheckInterval.Duration())
-
-	// Shut down the server gracefully (by closing the listener)
-	// if SIGTERM is received.
-	term := make(chan os.Signal, 1)
-	go func(sig <-chan os.Signal) {
-		s := <-sig
-		log.Println("caught signal:", s)
-		doneEmptyingTrash <- true
-		listener.Close()
-	}(term)
-	signal.Notify(term, syscall.SIGTERM)
-	signal.Notify(term, syscall.SIGINT)
-
-	if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
-		log.Printf("Error notifying init daemon: %v", err)
-	}
-	log.Println("listening at", listener.Addr())
-	srv := &server{}
-	srv.Handler = router
-	srv.Serve(listener)
-}
-
 // Periodically (once per interval) invoke EmptyTrash on all volumes.
-func emptyTrash(done <-chan bool, interval time.Duration) {
-	ticker := time.NewTicker(interval)
-
-	for {
-		select {
-		case <-ticker.C:
-			for _, v := range theConfig.Volumes {
-				if v.Writable() {
-					v.EmptyTrash()
-				}
-			}
-		case <-done:
-			ticker.Stop()
-			return
+func emptyTrash(volumes []Volume, interval time.Duration) {
+	for range time.NewTicker(interval).C {
+		for _, v := range volumes {
+			v.EmptyTrash()
 		}
 	}
 }
diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 8b448e72c..728c6fded 100644
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -6,7 +6,6 @@
 Description=Arvados Keep Storage Daemon
 Documentation=https://doc.arvados.org/
 After=network.target
-AssertPathExists=/etc/arvados/keepstore/keepstore.yml
 
 # systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
diff --git a/services/keepstore/perms.go b/services/keepstore/perms.go
index 49a231685..e2155f94f 100644
--- a/services/keepstore/perms.go
+++ b/services/keepstore/perms.go
@@ -5,14 +5,16 @@
 package main
 
 import (
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 )
 
 // SignLocator takes a blobLocator, an apiToken and an expiry time, and
 // returns a signed locator string.
-func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
-	return keepclient.SignLocator(blobLocator, apiToken, expiry, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func SignLocator(cluster *arvados.Cluster, blobLocator, apiToken string, expiry time.Time) string {
+	return keepclient.SignLocator(blobLocator, apiToken, expiry, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
 }
 
 // VerifySignature returns nil if the signature on the signedLocator
@@ -20,8 +22,8 @@ func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
 // either ExpiredError (if the timestamp has expired, which is
 // something the client could have figured out independently) or
 // PermissionError.
-func VerifySignature(signedLocator, apiToken string) error {
-	err := keepclient.VerifySignature(signedLocator, apiToken, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func VerifySignature(cluster *arvados.Cluster, signedLocator, apiToken string) error {
+	err := keepclient.VerifySignature(signedLocator, apiToken, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
 	if err == keepclient.ErrSignatureExpired {
 		return ExpiredError
 	} else if err != nil {
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
index 1f82f3f4f..589bedc96 100644
--- a/services/keepstore/proxy_remote.go
+++ b/services/keepstore/proxy_remote.go
@@ -148,6 +148,7 @@ type remoteResponseCacher struct {
 	Locator string
 	Token   string
 	Buffer  []byte
+	Cluster *arvados.Cluster
 	Context context.Context
 	http.ResponseWriter
 	statusCode int
@@ -193,7 +194,8 @@ func (rrc *remoteResponseCacher) Close() error {
 	}
 
 	unsigned := localOrRemoteSignature.ReplaceAllLiteralString(rrc.Locator, "")
-	signed := SignLocator(unsigned, rrc.Token, time.Now().Add(theConfig.BlobSignatureTTL.Duration()))
+	expiry := time.Now().Add(rrc.Cluster.Collections.BlobSigningTTL.Duration())
+	signed := SignLocator(rrc.Cluster, unsigned, rrc.Token, expiry)
 	if signed == unsigned {
 		err = errors.New("could not sign locator")
 		http.Error(rrc.ResponseWriter, err.Error(), http.StatusInternalServerError)
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 42b5d5889..5e6315e14 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -6,27 +6,27 @@ package main
 
 import (
 	"context"
-	"crypto/rand"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"github.com/sirupsen/logrus"
 )
 
 // RunPullWorker receives PullRequests from pullq, invokes
 // PullItemAndProcess on each one. After each PR, it logs a message
 // indicating whether the pull was successful.
-func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
+func (h *handler) runPullWorker(pullq *WorkQueue) {
 	for item := range pullq.NextItem {
 		pr := item.(PullRequest)
-		err := PullItemAndProcess(pr, keepClient)
+		err := h.pullItemAndProcess(pr)
 		pullq.DoneItem <- struct{}{}
 		if err == nil {
-			log.Printf("Pull %s success", pr)
+			h.Logger.Printf("Pull %s success", pr)
 		} else {
-			log.Printf("Pull %s error: %s", pr, err)
+			h.Logger.Printf("Pull %s error: %s", pr, err)
 		}
 	}
 }
@@ -39,8 +39,8 @@ func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
 // only attempt to write the data to the corresponding
 // volume. Otherwise it writes to any local volume, as a PUT request
 // would.
-func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClient) error {
-	var vol Volume
+func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
+	var vol *VolumeMount
 	if uuid := pullRequest.MountUUID; uuid != "" {
 		vol = KeepVM.Lookup(pullRequest.MountUUID, true)
 		if vol == nil {
@@ -48,19 +48,19 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
 		}
 	}
 
-	keepClient.Arvados.ApiToken = randomToken
-
+	// Make a private copy of keepClient so we can set
+	// ServiceRoots to the source servers specified in the pull
+	// request.
+	keepClient := *h.keepClient
 	serviceRoots := make(map[string]string)
 	for _, addr := range pullRequest.Servers {
 		serviceRoots[addr] = addr
 	}
 	keepClient.SetServiceRoots(serviceRoots, nil, nil)
 
-	// Generate signature with a random token
-	expiresAt := time.Now().Add(60 * time.Second)
-	signedLocator := SignLocator(pullRequest.Locator, randomToken, expiresAt)
+	signedLocator := SignLocator(h.Cluster, pullRequest.Locator, keepClient.Arvados.ApiToken, time.Now().Add(time.Minute))
 
-	reader, contentLen, _, err := GetContent(signedLocator, keepClient)
+	reader, contentLen, _, err := GetContent(signedLocator, &keepClient)
 	if err != nil {
 		return err
 	}
@@ -78,8 +78,7 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
 		return fmt.Errorf("Content not found for: %s", signedLocator)
 	}
 
-	writePulledBlock(vol, readContent, pullRequest.Locator)
-	return nil
+	return writePulledBlock(vol, readContent, pullRequest.Locator, h.Logger)
 }
 
 // Fetch the content for the given locator using keepclient.
@@ -87,24 +86,11 @@ var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (
 	return keepClient.Get(signedLocator)
 }
 
-var writePulledBlock = func(volume Volume, data []byte, locator string) {
-	var err error
+var writePulledBlock = func(volume Volume, data []byte, locator string, log logrus.FieldLogger) error {
 	if volume != nil {
-		err = volume.Put(context.Background(), locator, data)
+		return volume.Put(context.Background(), locator, data)
 	} else {
-		_, err = PutBlock(context.Background(), data, locator)
-	}
-	if err != nil {
-		log.Printf("error writing pulled block %q: %s", locator, err)
+		_, err := PutBlock(context.Background(), data, locator)
+		return err
 	}
 }
-
-var randomToken = func() string {
-	const alphaNumeric = "0123456789abcdefghijklmnopqrstuvwxyz"
-	var bytes = make([]byte, 36)
-	rand.Read(bytes)
-	for i, b := range bytes {
-		bytes[i] = alphaNumeric[b%byte(len(alphaNumeric))]
-	}
-	return (string(bytes))
-}()
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 4c39dcd5c..c7a86d9e4 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -10,10 +10,12 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
-	"flag"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"regexp"
@@ -26,8 +28,31 @@ import (
 	"github.com/AdRoll/goamz/aws"
 	"github.com/AdRoll/goamz/s3"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
+func init() {
+	driver["S3"] = newS3Volume
+}
+
+func newS3Volume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+	v := S3Volume{cluster: cluster, volume: volume, logger: logger}
+	err := json.Unmarshal(volume.DriverParameters, &v)
+	if err != nil {
+		return nil, err
+	}
+	if v.Bucket == "" || v.AccessKey == "" || v.SecretKey == "" {
+		return nil, errors.New("DriverParameters: Bucket, AccessKey, and SecretKey must be provided")
+	}
+	if v.IndexPageSize == 0 {
+		v.IndexPageSize = 1000
+	}
+	if v.RaceWindow < 0 {
+		return nil, errors.New("DriverParameters: RaceWindow must not be negative")
+	}
+	return &v, nil
+}
+
 const (
 	s3DefaultReadTimeout    = arvados.Duration(10 * time.Minute)
 	s3DefaultConnectTimeout = arvados.Duration(time.Minute)
@@ -38,14 +63,6 @@ var (
 	// is impossible with the current config.
 	ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
 
-	s3AccessKeyFile string
-	s3SecretKeyFile string
-	s3RegionName    string
-	s3Endpoint      string
-	s3Replication   int
-	s3UnsafeDelete  bool
-	s3RaceWindow    time.Duration
-
 	s3ACL = s3.Private
 
 	zeroTime time.Time
@@ -56,40 +73,6 @@ const (
 	nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
 )
 
-type s3VolumeAdder struct {
-	*Config
-}
-
-// String implements flag.Value
-func (s *s3VolumeAdder) String() string {
-	return "-"
-}
-
-func (s *s3VolumeAdder) Set(bucketName string) error {
-	if bucketName == "" {
-		return fmt.Errorf("no container name given")
-	}
-	if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
-		return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
-	}
-	if deprecated.flagSerializeIO {
-		log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
-	}
-	s.Config.Volumes = append(s.Config.Volumes, &S3Volume{
-		Bucket:        bucketName,
-		AccessKeyFile: s3AccessKeyFile,
-		SecretKeyFile: s3SecretKeyFile,
-		Endpoint:      s3Endpoint,
-		Region:        s3RegionName,
-		RaceWindow:    arvados.Duration(s3RaceWindow),
-		S3Replication: s3Replication,
-		UnsafeDelete:  s3UnsafeDelete,
-		ReadOnly:      deprecated.flagReadonly,
-		IndexPageSize: 1000,
-	})
-	return nil
-}
-
 func s3regions() (okList []string) {
 	for r := range aws.Regions {
 		okList = append(okList, r)
@@ -97,117 +80,38 @@ func s3regions() (okList []string) {
 	return
 }
 
-func init() {
-	VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &S3Volume{} })
-
-	flag.Var(&s3VolumeAdder{theConfig},
-		"s3-bucket-volume",
-		"Use the given bucket as a storage volume. Can be given multiple times.")
-	flag.StringVar(
-		&s3RegionName,
-		"s3-region",
-		"",
-		fmt.Sprintf("AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are %+q.", s3regions()))
-	flag.StringVar(
-		&s3Endpoint,
-		"s3-endpoint",
-		"",
-		"Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use \"https://storage.googleapis.com\".")
-	flag.StringVar(
-		&s3AccessKeyFile,
-		"s3-access-key-file",
-		"",
-		"`File` containing the access key used for subsequent -s3-bucket-volume arguments.")
-	flag.StringVar(
-		&s3SecretKeyFile,
-		"s3-secret-key-file",
-		"",
-		"`File` containing the secret key used for subsequent -s3-bucket-volume arguments.")
-	flag.DurationVar(
-		&s3RaceWindow,
-		"s3-race-window",
-		24*time.Hour,
-		"Maximum eventual consistency latency for subsequent -s3-bucket-volume arguments.")
-	flag.IntVar(
-		&s3Replication,
-		"s3-replication",
-		2,
-		"Replication level reported to clients for subsequent -s3-bucket-volume arguments.")
-	flag.BoolVar(
-		&s3UnsafeDelete,
-		"s3-unsafe-delete",
-		false,
-		"EXPERIMENTAL. Enable deletion (garbage collection) even when trash lifetime is zero, even though there are known race conditions that can cause data loss.")
-}
-
 // S3Volume implements Volume using an S3 bucket.
 type S3Volume struct {
-	AccessKeyFile      string
-	SecretKeyFile      string
+	AccessKey          string
+	SecretKey          string
 	Endpoint           string
 	Region             string
 	Bucket             string
 	LocationConstraint bool
 	IndexPageSize      int
-	S3Replication      int
 	ConnectTimeout     arvados.Duration
 	ReadTimeout        arvados.Duration
 	RaceWindow         arvados.Duration
-	ReadOnly           bool
 	UnsafeDelete       bool
-	StorageClasses     []string
-
-	bucket *s3bucket
 
+	logger    logrus.FieldLogger
+	cluster   *arvados.Cluster
+	volume    arvados.Volume
+	bucket    *s3bucket
 	startOnce sync.Once
 }
 
-// Examples implements VolumeWithExamples.
-func (*S3Volume) Examples() []Volume {
-	return []Volume{
-		&S3Volume{
-			AccessKeyFile:  "/etc/aws_s3_access_key.txt",
-			SecretKeyFile:  "/etc/aws_s3_secret_key.txt",
-			Endpoint:       "",
-			Region:         "us-east-1",
-			Bucket:         "example-bucket-name",
-			IndexPageSize:  1000,
-			S3Replication:  2,
-			RaceWindow:     arvados.Duration(24 * time.Hour),
-			ConnectTimeout: arvados.Duration(time.Minute),
-			ReadTimeout:    arvados.Duration(5 * time.Minute),
-		},
-		&S3Volume{
-			AccessKeyFile:  "/etc/gce_s3_access_key.txt",
-			SecretKeyFile:  "/etc/gce_s3_secret_key.txt",
-			Endpoint:       "https://storage.googleapis.com",
-			Region:         "",
-			Bucket:         "example-bucket-name",
-			IndexPageSize:  1000,
-			S3Replication:  2,
-			RaceWindow:     arvados.Duration(24 * time.Hour),
-			ConnectTimeout: arvados.Duration(time.Minute),
-			ReadTimeout:    arvados.Duration(5 * time.Minute),
-		},
-	}
-}
-
-// Type implements Volume.
-func (*S3Volume) Type() string {
-	return "S3"
-}
-
 // Start populates private fields and verifies the configuration is
 // valid.
 func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
 	region, ok := aws.Regions[v.Region]
 	if v.Endpoint == "" {
 		if !ok {
-			return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region)
+			return fmt.Errorf("unrecognized region %+q; try specifying endpoint instead", v.Region)
 		}
 	} else if ok {
 		return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
-			"specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint)
+			"specify empty endpoint or use a different region name", v.Region, v.Endpoint)
 	} else {
 		region = aws.Region{
 			Name:                 v.Region,
@@ -216,15 +120,9 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
 		}
 	}
 
-	var err error
-	var auth aws.Auth
-	auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile)
-	if err != nil {
-		return err
-	}
-	auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile)
-	if err != nil {
-		return err
+	auth := aws.Auth{
+		AccessKey: v.AccessKey,
+		SecretKey: v.SecretKey,
 	}
 
 	// Zero timeouts mean "wait forever", which is a bad
@@ -250,14 +148,14 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
 		},
 	}
 	// Set up prometheus metrics
-	lbls := prometheus.Labels{"device_id": v.DeviceID()}
+	lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
 	v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls)
 
 	return nil
 }
 
-// DeviceID returns a globally unique ID for the storage bucket.
-func (v *S3Volume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage bucket.
+func (v *S3Volume) GetDeviceID() string {
 	return "s3://" + v.Endpoint + "/" + v.Bucket
 }
 
@@ -271,7 +169,7 @@ func (v *S3Volume) getReaderWithContext(ctx context.Context, loc string) (rdr io
 	case <-ready:
 		return
 	case <-ctx.Done():
-		theConfig.debugLogf("s3: abandoning getReader(): %s", ctx.Err())
+		v.logger.Debugf("s3: abandoning getReader(): %s", ctx.Err())
 		go func() {
 			<-ready
 			if err == nil {
@@ -339,11 +237,11 @@ func (v *S3Volume) Get(ctx context.Context, loc string, buf []byte) (int, error)
 	}()
 	select {
 	case <-ctx.Done():
-		theConfig.debugLogf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
+		v.logger.Debugf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
 		rdr.Close()
 		// Must wait for ReadFull to return, to ensure it
 		// doesn't write to buf after we return.
-		theConfig.debugLogf("s3: waiting for ReadFull() to fail")
+		v.logger.Debug("s3: waiting for ReadFull() to fail")
 		<-ready
 		return 0, ctx.Err()
 	case <-ready:
@@ -397,7 +295,7 @@ func (v *S3Volume) Compare(ctx context.Context, loc string, expect []byte) error
 
 // Put writes a block.
 func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	var opts s3.Options
@@ -433,7 +331,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
 	go func() {
 		defer func() {
 			if ctx.Err() != nil {
-				theConfig.debugLogf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
+				v.logger.Debugf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
 			}
 		}()
 		defer close(ready)
@@ -445,7 +343,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
 	}()
 	select {
 	case <-ctx.Done():
-		theConfig.debugLogf("%s: taking PutReader's input away: %s", v, ctx.Err())
+		v.logger.Debugf("%s: taking PutReader's input away: %s", v, ctx.Err())
 		// Our pipe might be stuck in Write(), waiting for
 		// PutReader() to read. If so, un-stick it. This means
 		// PutReader will get corrupt data, but that's OK: the
@@ -453,7 +351,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
 		go io.Copy(ioutil.Discard, bufr)
 		// CloseWithError() will return once pending I/O is done.
 		bufw.CloseWithError(ctx.Err())
-		theConfig.debugLogf("%s: abandoning PutReader goroutine", v)
+		v.logger.Debugf("%s: abandoning PutReader goroutine", v)
 		return ctx.Err()
 	case <-ready:
 		// Unblock pipe in case PutReader did not consume it.
@@ -464,7 +362,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
 
 // Touch sets the timestamp for the given locator to the current time.
 func (v *S3Volume) Touch(loc string) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	_, err := v.bucket.Head(loc, nil)
@@ -571,16 +469,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
 
 // Trash a Keep block.
 func (v *S3Volume) Trash(loc string) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	if t, err := v.Mtime(loc); err != nil {
 		return err
-	} else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+	} else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
 		return nil
 	}
-	if theConfig.TrashLifetime == 0 {
-		if !s3UnsafeDelete {
+	if v.cluster.Collections.BlobTrashLifetime == 0 {
+		if !v.UnsafeDelete {
 			return ErrS3TrashDisabled
 		}
 		return v.translateError(v.bucket.Del(loc))
@@ -615,7 +513,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
 		// Can't parse timestamp
 		return err
 	}
-	safeWindow := t.Add(theConfig.TrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
+	safeWindow := t.Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
 	if safeWindow <= 0 {
 		// We can't count on "touch trash/X" to prolong
 		// trash/X's lifetime. The new timestamp might not
@@ -698,23 +596,6 @@ func (v *S3Volume) String() string {
 	return fmt.Sprintf("s3-bucket:%+q", v.Bucket)
 }
 
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *S3Volume) Writable() bool {
-	return !v.ReadOnly
-}
-
-// Replication returns the storage redundancy of the underlying
-// device. Configured via command line flag.
-func (v *S3Volume) Replication() int {
-	return v.S3Replication
-}
-
-// GetStorageClasses implements Volume
-func (v *S3Volume) GetStorageClasses() []string {
-	return v.StorageClasses
-}
-
 var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
 
 func (v *S3Volume) isKeepBlock(s string) bool {
@@ -751,13 +632,13 @@ func (v *S3Volume) fixRace(loc string) bool {
 	}
 
 	ageWhenTrashed := trashTime.Sub(recentTime)
-	if ageWhenTrashed >= theConfig.BlobSignatureTTL.Duration() {
+	if ageWhenTrashed >= v.cluster.Collections.BlobSigningTTL.Duration() {
 		// No evidence of a race: block hasn't been written
 		// since it became eligible for Trash. No fix needed.
 		return false
 	}
 
-	log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, theConfig.BlobSignatureTTL)
+	log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, v.cluster.Collections.BlobSigningTTL)
 	log.Printf("notice: fixRace: copying %q to %q to recover from race between Put/Touch and Trash", "recent/"+loc, loc)
 	err = v.safeCopy(loc, "trash/"+loc)
 	if err != nil {
@@ -785,6 +666,10 @@ func (v *S3Volume) translateError(err error) error {
 // EmptyTrash looks for trashed blocks that exceeded TrashLifetime
 // and deletes them from the volume.
 func (v *S3Volume) EmptyTrash() {
+	if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+		return
+	}
+
 	var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
 
 	// Define "ready to delete" as "...when EmptyTrash started".
@@ -820,8 +705,8 @@ func (v *S3Volume) EmptyTrash() {
 			log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
 			return
 		}
-		if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
-			if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
+		if trashT.Sub(recentT) < v.cluster.Collections.BlobSigningTTL.Duration() {
+			if age := startT.Sub(recentT); age >= v.cluster.Collections.BlobSigningTTL.Duration()-time.Duration(v.RaceWindow) {
 				// recent/loc is too old to protect
 				// loc from being Trashed again during
 				// the raceWindow that starts if we
@@ -845,7 +730,7 @@ func (v *S3Volume) EmptyTrash() {
 				return
 			}
 		}
-		if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
+		if startT.Sub(trashT) < v.cluster.Collections.BlobTrashLifetime.Duration() {
 			return
 		}
 		err = v.bucket.Del(trash.Key)
@@ -872,8 +757,8 @@ func (v *S3Volume) EmptyTrash() {
 	}
 
 	var wg sync.WaitGroup
-	todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers)
-	for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+	todo := make(chan *s3.Key, v.cluster.Collections.BlobDeleteConcurrency)
+	for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
diff --git a/services/keepstore/server.go b/services/keepstore/server.go
deleted file mode 100644
index 3f6727712..000000000
--- a/services/keepstore/server.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
-	"crypto/tls"
-	"net"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-)
-
-type server struct {
-	http.Server
-
-	// channel (size=1) with the current keypair
-	currentCert chan *tls.Certificate
-}
-
-func (srv *server) Serve(l net.Listener) error {
-	if theConfig.TLSCertificateFile == "" && theConfig.TLSKeyFile == "" {
-		return srv.Server.Serve(l)
-	}
-	// https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
-	srv.TLSConfig = &tls.Config{
-		GetCertificate:           srv.getCertificate,
-		PreferServerCipherSuites: true,
-		CurvePreferences: []tls.CurveID{
-			tls.CurveP256,
-			tls.X25519,
-		},
-		MinVersion: tls.VersionTLS12,
-		CipherSuites: []uint16{
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
-			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
-			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-		},
-	}
-	srv.currentCert = make(chan *tls.Certificate, 1)
-	go srv.refreshCertificate(theConfig.TLSCertificateFile, theConfig.TLSKeyFile)
-	return srv.Server.ServeTLS(l, "", "")
-}
-
-func (srv *server) refreshCertificate(certfile, keyfile string) {
-	cert, err := tls.LoadX509KeyPair(certfile, keyfile)
-	if err != nil {
-		log.WithError(err).Fatal("error loading X509 key pair")
-	}
-	srv.currentCert <- &cert
-
-	reload := make(chan os.Signal, 1)
-	signal.Notify(reload, syscall.SIGHUP)
-	for range reload {
-		cert, err := tls.LoadX509KeyPair(certfile, keyfile)
-		if err != nil {
-			log.WithError(err).Warn("error loading X509 key pair")
-			continue
-		}
-		// Throw away old cert and start using new one
-		<-srv.currentCert
-		srv.currentCert <- &cert
-	}
-}
-
-func (srv *server) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-	if srv.currentCert == nil {
-		panic("srv.currentCert not initialized")
-	}
-	cert := <-srv.currentCert
-	srv.currentCert <- cert
-	return cert, nil
-}
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 8a9fedfb7..57301b814 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -6,6 +6,7 @@ package main
 
 import (
 	"errors"
+	"log"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -17,35 +18,35 @@ import (
 //      Delete the block indicated by the trash request Locator
 //		Repeat
 //
-func RunTrashWorker(trashq *WorkQueue) {
+func RunTrashWorker(cluster *arvados.Cluster, trashq *WorkQueue) {
 	for item := range trashq.NextItem {
 		trashRequest := item.(TrashRequest)
-		TrashItem(trashRequest)
+		TrashItem(cluster, trashRequest)
 		trashq.DoneItem <- struct{}{}
 	}
 }
 
 // TrashItem deletes the indicated block from every writable volume.
-func TrashItem(trashRequest TrashRequest) {
+func TrashItem(cluster *arvados.Cluster, trashRequest TrashRequest) {
 	reqMtime := time.Unix(0, trashRequest.BlockMtime)
-	if time.Since(reqMtime) < theConfig.BlobSignatureTTL.Duration() {
+	if time.Since(reqMtime) < cluster.Collections.BlobSigningTTL.Duration() {
 		log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
 			arvados.Duration(time.Since(reqMtime)),
 			trashRequest.Locator,
 			trashRequest.BlockMtime,
 			reqMtime,
-			theConfig.BlobSignatureTTL)
+			cluster.Collections.BlobSigningTTL)
 		return
 	}
 
-	var volumes []Volume
+	var volumes []*VolumeMount
 	if uuid := trashRequest.MountUUID; uuid == "" {
 		volumes = KeepVM.AllWritable()
 	} else if v := KeepVM.Lookup(uuid, true); v == nil {
 		log.Printf("warning: trash request for nonexistent mount: %v", trashRequest)
 		return
 	} else {
-		volumes = []Volume{v}
+		volumes = []*VolumeMount{v}
 	}
 
 	for _, volume := range volumes {
@@ -59,8 +60,8 @@ func TrashItem(trashRequest TrashRequest) {
 			continue
 		}
 
-		if !theConfig.EnableDelete {
-			err = errors.New("skipping because EnableDelete is false")
+		if !cluster.Collections.BlobTrash {
+			err = errors.New("skipping because Collections.BlobTrash is false")
 		} else {
 			err = volume.Trash(trashRequest.Locator)
 		}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 4d9e798ac..de953bd74 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -5,12 +5,13 @@
 package main
 
 import (
-	"bufio"
 	"context"
-	"flag"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -22,98 +23,34 @@ import (
 	"syscall"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
-type unixVolumeAdder struct {
-	*Config
-}
-
-// String implements flag.Value
-func (vs *unixVolumeAdder) String() string {
-	return "-"
-}
-
-func (vs *unixVolumeAdder) Set(path string) error {
-	if dirs := strings.Split(path, ","); len(dirs) > 1 {
-		log.Print("DEPRECATED: using comma-separated volume list.")
-		for _, dir := range dirs {
-			if err := vs.Set(dir); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	vs.Config.Volumes = append(vs.Config.Volumes, &UnixVolume{
-		Root:      path,
-		ReadOnly:  deprecated.flagReadonly,
-		Serialize: deprecated.flagSerializeIO,
-	})
-	return nil
-}
-
 func init() {
-	VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &UnixVolume{} })
-
-	flag.Var(&unixVolumeAdder{theConfig}, "volumes", "see Volumes configuration")
-	flag.Var(&unixVolumeAdder{theConfig}, "volume", "see Volumes configuration")
+	driver["Directory"] = newDirectoryVolume
 }
 
-// Discover adds a UnixVolume for every directory named "keep" that is
-// located at the top level of a device- or tmpfs-backed mount point
-// other than "/". It returns the number of volumes added.
-func (vs *unixVolumeAdder) Discover() int {
-	added := 0
-	f, err := os.Open(ProcMounts)
+func newDirectoryVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+	v := UnixVolume{volume: volume, logger: logger}
+	err := json.Unmarshal(volume.DriverParameters, &v)
 	if err != nil {
-		log.Fatalf("opening %s: %s", ProcMounts, err)
+		return nil, err
 	}
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		args := strings.Fields(scanner.Text())
-		if err := scanner.Err(); err != nil {
-			log.Fatalf("reading %s: %s", ProcMounts, err)
-		}
-		dev, mount := args[0], args[1]
-		if mount == "/" {
-			continue
-		}
-		if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
-			continue
-		}
-		keepdir := mount + "/keep"
-		if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
-			continue
-		}
-		// Set the -readonly flag (but only for this volume)
-		// if the filesystem is mounted readonly.
-		flagReadonlyWas := deprecated.flagReadonly
-		for _, fsopt := range strings.Split(args[3], ",") {
-			if fsopt == "ro" {
-				deprecated.flagReadonly = true
-				break
-			}
-			if fsopt == "rw" {
-				break
-			}
-		}
-		if err := vs.Set(keepdir); err != nil {
-			log.Printf("adding %q: %s", keepdir, err)
-		} else {
-			added++
-		}
-		deprecated.flagReadonly = flagReadonlyWas
+	if v.Root == "" {
+		return nil, errors.New("DriverParameters: Root must be provided")
 	}
-	return added
+	return &v, nil
 }
 
 // A UnixVolume stores and retrieves blocks in a local directory.
 type UnixVolume struct {
-	Root                 string // path to the volume's root directory
-	ReadOnly             bool
-	Serialize            bool
-	DirectoryReplication int
-	StorageClasses       []string
+	Root      string // path to the volume's root directory
+	Serialize bool
+
+	volume arvados.Volume
+	logger logrus.FieldLogger
 
 	// something to lock during IO, typically a sync.Mutex (or nil
 	// to skip locking)
@@ -122,12 +59,12 @@ type UnixVolume struct {
 	os osWithStats
 }
 
-// DeviceID returns a globally unique ID for the volume's root
+// GetDeviceID returns a globally unique ID for the volume's root
 // directory, consisting of the filesystem's UUID and the path from
 // filesystem root to storage directory, joined by "/". For example,
-// the DeviceID for a local directory "/mnt/xvda1/keep" might be
+// the device ID for a local directory "/mnt/xvda1/keep" might be
 // "fa0b6166-3b55-4994-bd3f-92f4e00a1bb0/keep".
-func (v *UnixVolume) DeviceID() string {
+func (v *UnixVolume) GetDeviceID() string {
 	giveup := func(f string, args ...interface{}) string {
 		log.Printf(f+"; using blank DeviceID for volume %s", append(args, v)...)
 		return ""
@@ -198,27 +135,6 @@ func (v *UnixVolume) DeviceID() string {
 	return giveup("could not find entry in %q matching %q", udir, dev)
 }
 
-// Examples implements VolumeWithExamples.
-func (*UnixVolume) Examples() []Volume {
-	return []Volume{
-		&UnixVolume{
-			Root:                 "/mnt/local-disk",
-			Serialize:            true,
-			DirectoryReplication: 1,
-		},
-		&UnixVolume{
-			Root:                 "/mnt/network-disk",
-			Serialize:            false,
-			DirectoryReplication: 2,
-		},
-	}
-}
-
-// Type implements Volume
-func (v *UnixVolume) Type() string {
-	return "Directory"
-}
-
 // Start implements Volume
 func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
 	if v.Serialize {
@@ -227,11 +143,8 @@ func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
 	if !strings.HasPrefix(v.Root, "/") {
 		return fmt.Errorf("volume root does not start with '/': %q", v.Root)
 	}
-	if v.DirectoryReplication == 0 {
-		v.DirectoryReplication = 1
-	}
 	// Set up prometheus metrics
-	lbls := prometheus.Labels{"device_id": v.DeviceID()}
+	lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
 	v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = vm.getCounterVecsFor(lbls)
 
 	_, err := v.os.Stat(v.Root)
@@ -241,7 +154,7 @@ func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
 
 // Touch sets the timestamp for the given locator to the current time
 func (v *UnixVolume) Touch(loc string) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	p := v.blockPath(loc)
@@ -349,7 +262,7 @@ func (v *UnixVolume) Put(ctx context.Context, loc string, block []byte) error {
 
 // WriteBlock implements BlockWriter.
 func (v *UnixVolume) WriteBlock(ctx context.Context, loc string, rdr io.Reader) error {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	if v.IsFull() {
@@ -516,7 +429,7 @@ func (v *UnixVolume) Trash(loc string) error {
 	// Trash() will read the correct up-to-date timestamp and choose not to
 	// trash the file.
 
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 	if err := v.lock(context.TODO()); err != nil {
@@ -555,7 +468,7 @@ func (v *UnixVolume) Trash(loc string) error {
 // Look for path/{loc}.trash.{deadline} in storage,
 // and rename the first such file as path/{loc}
 func (v *UnixVolume) Untrash(loc string) (err error) {
-	if v.ReadOnly {
+	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
 
@@ -650,23 +563,12 @@ func (v *UnixVolume) String() string {
 	return fmt.Sprintf("[UnixVolume %s]", v.Root)
 }
 
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *UnixVolume) Writable() bool {
-	return !v.ReadOnly
-}
-
 // Replication returns the number of replicas promised by the
 // underlying device (as specified in configuration).
 func (v *UnixVolume) Replication() int {
 	return v.DirectoryReplication
 }
 
-// GetStorageClasses implements Volume
-func (v *UnixVolume) GetStorageClasses() []string {
-	return v.StorageClasses
-}
-
 // InternalStats returns I/O and filesystem ops counters.
 func (v *UnixVolume) InternalStats() interface{} {
 	return &v.os.stats
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
deleted file mode 100644
index 8e83f6ce5..000000000
--- a/services/keepstore/usage.go
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"sort"
-	"strings"
-
-	"github.com/ghodss/yaml"
-)
-
-func usage() {
-	c := DefaultConfig()
-	knownTypes := []string{}
-	for _, vt := range VolumeTypes {
-		c.Volumes = append(c.Volumes, vt().Examples()...)
-		knownTypes = append(knownTypes, vt().Type())
-	}
-	exampleConfigFile, err := yaml.Marshal(c)
-	if err != nil {
-		panic(err)
-	}
-	sort.Strings(knownTypes)
-	knownTypeList := strings.Join(knownTypes, ", ")
-	fmt.Fprintf(os.Stderr, `
-
-keepstore provides a content-addressed data store backed by a local filesystem or networked storage.
-
-Usage: keepstore -config path/to/keepstore.yml
-       keepstore [OPTIONS] -dump-config
-
-NOTE: All options (other than -config) are deprecated in favor of YAML
-      configuration. Use -dump-config to translate existing
-      configurations to YAML format.
-
-Options:
-`)
-	flag.PrintDefaults()
-	fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Listen:
-
-    Local port to listen on. Can be "address:port" or ":port", where
-    "address" is a host IP address or name and "port" is a port number
-    or name.
-
-LogFormat:
-
-    Format of request/response and error logs: "json" or "text".
-
-PIDFile:
-
-   Path to write PID file during startup. This file is kept open and
-   locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
-   one way to shut down. Exit immediately if there is an error
-   opening, locking, or writing the PID file.
-
-MaxBuffers:
-
-    Maximum RAM to use for data buffers, given in multiples of block
-    size (64 MiB). When this limit is reached, HTTP requests requiring
-    buffers (like GET and PUT) will wait for buffer space to be
-    released.
-
-MaxRequests:
-
-    Maximum concurrent requests. When this limit is reached, new
-    requests will receive 503 responses. Note: this limit does not
-    include idle connections from clients using HTTP keepalive, so it
-    does not strictly limit the number of concurrent connections. If
-    omitted or zero, the default is 2 * MaxBuffers.
-
-BlobSigningKeyFile:
-
-    Local file containing the secret blob signing key (used to
-    generate and verify blob signatures).  This key should be
-    identical to the API server's blob_signing_key configuration
-    entry.
-
-RequireSignatures:
-
-    Honor read requests only if a valid signature is provided.  This
-    should be true, except for development use and when migrating from
-    a very old version.
-
-BlobSignatureTTL:
-
-    Duration for which new permission signatures (returned in PUT
-    responses) will be valid.  This should be equal to the API
-    server's blob_signature_ttl configuration entry.
-
-SystemAuthTokenFile:
-
-    Local file containing the Arvados API token used by keep-balance
-    or data manager.  Delete, trash, and index requests are honored
-    only for this token.
-
-EnableDelete:
-
-    Enable trash and delete features. If false, trash lists will be
-    accepted but blocks will not be trashed or deleted.
-
-TrashLifetime:
-
-    Time duration after a block is trashed during which it can be
-    recovered using an /untrash request.
-
-TrashCheckInterval:
-
-    How often to check for (and delete) trashed blocks whose
-    TrashLifetime has expired.
-
-TrashWorkers:
-
-    Maximum number of concurrent trash operations. Default is 1, i.e.,
-    trash lists are processed serially.
-
-EmptyTrashWorkers:
-
-    Maximum number of concurrent block deletion operations (per
-    volume) when emptying trash. Default is 1.
-
-PullWorkers:
-
-    Maximum number of concurrent pull operations. Default is 1, i.e.,
-    pull lists are processed serially.
-
-TLSCertificateFile:
-
-    Path to server certificate file in X509 format. Enables TLS mode.
-
-    Example: /var/lib/acme/live/keep0.example.com/fullchain
-
-TLSKeyFile:
-
-    Path to server key file in X509 format. Enables TLS mode.
-
-    The key pair is read from disk during startup, and whenever SIGHUP
-    is received.
-
-    Example: /var/lib/acme/live/keep0.example.com/privkey
-
-Volumes:
-
-    List of storage volumes. If omitted or empty, the default is to
-    use all directories named "keep" that exist in the top level
-    directory of a mount point at startup time.
-
-    Volume types: %s
-
-    (See volume configuration examples above.)
-
-`, exampleConfigFile, knownTypeList)
-}
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 52b9b1b24..3d5eb4325 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"github.com/sirupsen/logrus"
 )
 
 type BlockWriter interface {
@@ -28,14 +29,12 @@ type BlockReader interface {
 	ReadBlock(ctx context.Context, loc string, w io.Writer) error
 }
 
+var driver = map[string]func(*arvados.Cluster, arvados.Volume, logrus.FieldLogger) (Volume, error){}
+
 // A Volume is an interface representing a Keep back-end storage unit:
 // for example, a single mounted disk, a RAID array, an Amazon S3 volume,
 // etc.
 type Volume interface {
-	// Volume type as specified in config file. Examples: "S3",
-	// "Directory".
-	Type() string
-
 	// Do whatever private setup tasks and configuration checks
 	// are needed. Return non-nil if the volume is unusable (e.g.,
 	// invalid config).
@@ -222,29 +221,13 @@ type Volume interface {
 	// secrets.
 	String() string
 
-	// Writable returns false if all future Put, Mtime, and Delete
-	// calls are expected to fail.
-	//
-	// If the volume is only temporarily unwritable -- or if Put
-	// will fail because it is full, but Mtime or Delete can
-	// succeed -- then Writable should return false.
-	Writable() bool
-
-	// Replication returns the storage redundancy of the
-	// underlying device. It will be passed on to clients in
-	// responses to PUT requests.
-	Replication() int
-
 	// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
 	// and deletes them from the volume.
 	EmptyTrash()
 
 	// Return a globally unique ID of the underlying storage
 	// device if possible, otherwise "".
-	DeviceID() string
-
-	// Get the storage classes associated with this volume
-	GetStorageClasses() []string
+	GetDeviceID() string
 }
 
 // A VolumeWithExamples provides example configs to display in the
@@ -260,24 +243,24 @@ type VolumeManager interface {
 	// Mounts returns all mounts (volume attachments).
 	Mounts() []*VolumeMount
 
-	// Lookup returns the volume under the given mount
-	// UUID. Returns nil if the mount does not exist. If
-	// write==true, returns nil if the volume is not writable.
-	Lookup(uuid string, write bool) Volume
+	// Lookup returns the mount with the given UUID. Returns nil
+	// if the mount does not exist. If write==true, returns nil if
+	// the mount is not writable.
+	Lookup(uuid string, write bool) *VolumeMount
 
-	// AllReadable returns all volumes.
-	AllReadable() []Volume
+	// AllReadable returns all mounts.
+	AllReadable() []*VolumeMount
 
-	// AllWritable returns all volumes that aren't known to be in
+	// AllWritable returns all mounts that aren't known to be in
 	// a read-only state. (There is no guarantee that a write to
 	// one will succeed, though.)
-	AllWritable() []Volume
+	AllWritable() []*VolumeMount
 
 	// NextWritable returns the volume where the next new block
 	// should be written. A VolumeManager can select a volume in
 	// order to distribute activity across spindles, fill up disks
 	// with more free space, etc.
-	NextWritable() Volume
+	NextWritable() *VolumeMount
 
 	// VolumeStats returns the ioStats used for tracking stats for
 	// the given Volume.
@@ -290,7 +273,7 @@ type VolumeManager interface {
 // A VolumeMount is an attachment of a Volume to a VolumeManager.
 type VolumeMount struct {
 	arvados.KeepMount
-	volume Volume
+	Volume
 }
 
 // Generate a UUID the way API server would for a "KeepVolumeMount"
@@ -321,22 +304,43 @@ type RRVolumeManager struct {
 }
 
 // MakeRRVolumeManager initializes RRVolumeManager
-func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
+func MakeRRVolumeManager(cluster *arvados.Cluster, myURL arvados.URL, metrics *volumeMetricsVecs) *RRVolumeManager {
 	vm := &RRVolumeManager{
 		iostats: make(map[Volume]*ioStats),
 	}
 	vm.mountMap = make(map[string]*VolumeMount)
-	for _, v := range volumes {
-		sc := v.GetStorageClasses()
+	for uuid, v := range cluster.Volumes {
+		va, ok := v.AccessViaHosts[myURL]
+		if !ok && len(v.AccessViaHosts) > 0 {
+			continue
+		}
+		dri, ok := driver[cfgvol.Driver]
+		if !ok {
+			h.err = fmt.Errorf("volume %s: invalid driver %q", uuid, cfgvol.Driver)
+			return
+		}
+		vol, err := dri(h.Cluster, cfgvol)
+		if err != nil {
+			h.err = fmt.Errorf("error initializing volume %s: %s", uuid, err)
+			return
+		}
+		err = vol.Start(metrics)
+		if err != nil {
+			h.err = fmt.Errorf("error starting volume %s (%s): %s", uuid, vol, err)
+			return
+		}
+		h.Logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+
+		sc := v.StorageClasses
 		if len(sc) == 0 {
 			sc = []string{"default"}
 		}
 		mnt := &VolumeMount{
 			KeepMount: arvados.KeepMount{
 				UUID:           (*VolumeMount)(nil).generateUUID(),
-				DeviceID:       v.DeviceID(),
-				ReadOnly:       !v.Writable(),
-				Replication:    v.Replication(),
+				DeviceID:       v.GetDeviceID(),
+				ReadOnly:       v.ReadOnly || va.ReadOnly,
+				Replication:    v.Replication,
 				StorageClasses: sc,
 			},
 			volume: v,
@@ -345,7 +349,7 @@ func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
 		vm.mounts = append(vm.mounts, mnt)
 		vm.mountMap[mnt.UUID] = mnt
 		vm.readables = append(vm.readables, v)
-		if v.Writable() {
+		if !mnt.KeepMount.ReadOnly {
 			vm.writables = append(vm.writables, v)
 		}
 	}
@@ -358,7 +362,7 @@ func (vm *RRVolumeManager) Mounts() []*VolumeMount {
 
 func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) Volume {
 	if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) {
-		return mnt.volume
+		return mnt.Volume
 	} else {
 		return nil
 	}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list