[ARVADOS] updated: d10dcd346a4f64940986a2f30408814f17786fdc

Git user git at public.curoverse.com
Thu Sep 29 22:28:26 EDT 2016


Summary of changes:
 .../test/integration/application_layout_test.rb    |   2 +-
 apps/workbench/test/integration/download_test.rb   |   4 +-
 crunch_scripts/cwl-runner                          |   7 +-
 sdk/go/arvadosclient/arvadosclient.go              |  18 ++--
 sdk/go/arvadosclient/pool.go                       |   2 +-
 sdk/go/crunchrunner/crunchrunner.go                |   2 +-
 sdk/go/dispatch/dispatch.go                        |   2 +-
 sdk/go/keepclient/collectionreader_test.go         |   4 +-
 sdk/go/keepclient/discover_test.go                 |   6 +-
 sdk/go/keepclient/keepclient_test.go               |  62 ++++++-------
 sdk/go/logger/logger.go                            |   6 +-
 sdk/go/streamer/streamer.go                        |   6 ++
 sdk/go/util/util.go                                |   4 +-
 services/api/app/models/container.rb               |  35 ++++++-
 services/api/app/models/container_request.rb       |  20 +++-
 services/api/test/fixtures/containers.yml          |  10 +-
 services/api/test/unit/container_request_test.rb   |  15 ++-
 services/api/test/unit/container_test.rb           |  18 ++--
 services/api/test/unit/repository_test.rb          |   8 +-
 .../crunch-dispatch-local_test.go                  |   2 +-
 .../crunch-dispatch-slurm_test.go                  |   2 +-
 services/crunch-run/crunchrun.go                   |  27 ++++--
 services/crunch-run/crunchrun_test.go              |  10 ++
 services/datamanager/collection/collection.go      |   2 +-
 services/datamanager/collection/collection_test.go |   2 +-
 services/datamanager/datamanager.go                |   6 +-
 services/datamanager/datamanager_test.go           |   4 +-
 services/datamanager/keep/keep.go                  |   8 +-
 services/datamanager/keep/keep_test.go             |  16 ++--
 services/keep-balance/integration_test.go          |   2 +-
 services/keep-web/server_test.go                   |   4 +-
 services/keepproxy/keepproxy_test.go               |   4 +-
 services/keepstore/azure_blob_volume.go            |  17 ++--
 services/keepstore/config.go                       | 102 +++++++++++++++++++++
 services/keepstore/keepstore.go                    |  51 +++++------
 services/keepstore/keepstore_test.go               |  22 ++---
 services/keepstore/pull_worker_integration_test.go |   2 +-
 services/keepstore/pull_worker_test.go             |   2 +-
 services/keepstore/s3_volume.go                    |  80 ++++++++--------
 services/keepstore/s3_volume_test.go               |  86 +++++++++--------
 services/keepstore/usage.go                        |  19 +---
 services/keepstore/volume.go                       |   5 +
 services/keepstore/volume_test.go                  |   9 +-
 services/keepstore/volume_unix.go                  |  74 +++++++++------
 services/keepstore/volume_unix_test.go             |  24 ++---
 45 files changed, 510 insertions(+), 303 deletions(-)
 create mode 100644 services/keepstore/config.go

  discards  171554df860a3f95314f665736ac93a92dd8167b (commit)
       via  d10dcd346a4f64940986a2f30408814f17786fdc (commit)
       via  c1481e3977513b599750a6c03518d654229191de (commit)
       via  4786aea0e1b0e370a7210a9a404d8a9f83e01595 (commit)
       via  482dc81f2b7533043bc5195bded942f970f163d8 (commit)
       via  e2f3466f502865b79da7f97af08943db5af4ff45 (commit)
       via  6613ec1e9c705fb5b950611fd160d4a2babed251 (commit)
       via  17b7538373558196516676c9c72839d308966f86 (commit)
       via  7314917d65573b0e9d55f7b6522463c470356fba (commit)
       via  a642669b7faa1b04350c451bf2e85692e730abba (commit)
       via  e9587a4ba9e13719473ae222b10291ce58fb5560 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (171554df860a3f95314f665736ac93a92dd8167b)
            \
             N -- N -- N (d10dcd346a4f64940986a2f30408814f17786fdc)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d10dcd346a4f64940986a2f30408814f17786fdc
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 29 22:28:20 2016 -0400

    9956: Load volume config from YAML file

diff --git a/sdk/go/streamer/streamer.go b/sdk/go/streamer/streamer.go
index 2217dd3..0c4d208 100644
--- a/sdk/go/streamer/streamer.go
+++ b/sdk/go/streamer/streamer.go
@@ -37,8 +37,11 @@ package streamer
 
 import (
 	"io"
+	"errors"
 )
 
+var ErrAlreadyClosed = errors.New("cannot close a stream twice")
+
 type AsyncStream struct {
 	buffer            []byte
 	requests          chan sliceRequest
@@ -115,6 +118,9 @@ func (this *StreamReader) WriteTo(dest io.Writer) (written int64, err error) {
 
 // Close the responses channel
 func (this *StreamReader) Close() error {
+	if this.stream == nil {
+		return ErrAlreadyClosed
+	}
 	this.stream.subtract_reader <- true
 	close(this.responses)
 	this.stream = nil
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 48cb026..8632769 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -40,7 +40,12 @@ func readKeyFromFile(file string) (string, error) {
 }
 
 type azureVolumeAdder struct {
-	*volumeSet
+	*Config
+}
+
+// String implements flag.Value
+func (s *azureVolumeAdder) String() string {
+	return "-"
 }
 
 func (s *azureVolumeAdder) Set(containerName string) error {
@@ -66,15 +71,15 @@ func (s *azureVolumeAdder) Set(containerName string) error {
 		log.Print("Notice: -serialize is not supported by azure-blob-container volumes.")
 	}
 	v := NewAzureBlobVolume(azClient, containerName, flagReadonly, azureStorageReplication)
-	if err := v.Check(); err != nil {
+	if err := v.Start(); err != nil {
 		return err
 	}
-	*s.volumeSet = append(*s.volumeSet, v)
+	s.Config.Volumes = append(s.Config.Volumes, v)
 	return nil
 }
 
 func init() {
-	flag.Var(&azureVolumeAdder{&volumes},
+	flag.Var(&azureVolumeAdder{theConfig},
 		"azure-storage-container-volume",
 		"Use the given container as a storage volume. Can be given multiple times.")
 	flag.StringVar(
@@ -122,8 +127,10 @@ func NewAzureBlobVolume(client storage.Client, containerName string, readonly bo
 	}
 }
 
-// Check returns nil if the volume is usable.
-func (v *AzureBlobVolume) Check() error {
+func (v *AzureBlobVolume) Type() string { return "Azure" }
+
+// Start implements Volume.
+func (v *AzureBlobVolume) Start() error {
 	ok, err := v.bsClient.ContainerExists(v.containerName)
 	if err != nil {
 		return err
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
new file mode 100644
index 0000000..244b71c
--- /dev/null
+++ b/services/keepstore/config.go
@@ -0,0 +1,102 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+type Config struct {
+	Listen        string
+	TrashLifetime arvados.Duration
+	Volumes       VolumeConfigs
+}
+
+var theConfig = DefaultConfig()
+
+// DefaultConfig returns the default configuration.
+func DefaultConfig() *Config {
+	return &Config{
+		Listen:        ":25107",
+		TrashLifetime: arvados.Duration(14 * 24 * time.Hour),
+		Volumes:       []Volume{},
+	}
+}
+
+func (cfg *Config) Start() error {
+	if len(cfg.Volumes) == 0 {
+		if (&unixVolumeAdder{cfg}).Discover() == 0 {
+			return fmt.Errorf("no volumes found")
+		}
+
+	}
+	for _, v := range cfg.Volumes {
+		if err := v.Start(); err != nil {
+			return fmt.Errorf("volume %s: %s", v, err)
+		}
+	}
+	return nil
+}
+
+var VolumeTypes = []func() Volume{}
+
+type VolumeConfigs []Volume
+
+// UnmarshalJSON implements json.Unmarshaler
+func (vsl *VolumeConfigs) UnmarshalJSON(data []byte) error {
+	var mapList []map[string]interface{}
+	err := json.Unmarshal(data, &mapList)
+	if err != nil {
+		return err
+	}
+LIST:
+	for _, v := range mapList {
+		t, ok := v["Type"].(string)
+		if !ok {
+			return fmt.Errorf("invalid volume Type: %+v", v["Type"])
+		}
+		for _, factory := range VolumeTypes {
+			v := factory()
+			if v.Type() == t {
+				data, err := json.Marshal(v)
+				if err != nil {
+					return err
+				}
+				err = json.Unmarshal(data, v)
+				if err != nil {
+					return err
+				}
+				*vsl = append(*vsl, v)
+				continue LIST
+			}
+		}
+		return fmt.Errorf("unsupported volume Type: %+v", v["Type"])
+	}
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler
+func (vsl *VolumeConfigs) MarshalJSON() ([]byte, error) {
+	data := []byte{'['}
+	for _, vs := range *vsl {
+		j, err := json.Marshal(vs)
+		if err != nil {
+			return nil, err
+		}
+		if len(data) > 1 {
+			data = append(data, byte(','))
+		}
+		t, err := json.Marshal(vs.Type())
+		if err != nil {
+			panic(err)
+		}
+		data = append(data, j[0])
+		data = append(data, []byte(`"Type":`)...)
+		data = append(data, t...)
+		data = append(data, byte(','))
+		data = append(data, j[1:]...)
+	}
+	return append(data, byte(']')), nil
+}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 48b83de..f417753 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/config"
 	"git.curoverse.com/arvados.git/sdk/go/httpserver"
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"io/ioutil"
@@ -18,16 +19,6 @@ import (
 	"time"
 )
 
-// ======================
-// Configuration settings
-//
-// TODO(twp): make all of these configurable via command line flags
-// and/or configuration file settings.
-
-// Default TCP address on which to listen for requests.
-// Initialized by the --listen flag.
-const DefaultAddr = ":25107"
-
 // A Keep "block" is 64MB.
 const BlockSize = 64 * 1024 * 1024
 
@@ -126,7 +117,6 @@ type volumeSet []Volume
 var (
 	flagSerializeIO bool
 	flagReadonly    bool
-	volumes         volumeSet
 )
 
 func (vs *volumeSet) String() string {
@@ -144,7 +134,6 @@ func main() {
 
 	var (
 		dataManagerTokenFile string
-		listen               string
 		blobSigningKeyFile   string
 		permissionTTLSec     int
 		pidfile              string
@@ -162,9 +151,9 @@ func main() {
 		false,
 		"Enforce permission signatures on requests.")
 	flag.StringVar(
-		&listen,
+		&theConfig.Listen,
 		"listen",
-		DefaultAddr,
+		theConfig.Listen,
 		"Listening address, in the form \"host:port\". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces.")
 	flag.IntVar(
 		&maxRequests,
@@ -231,8 +220,23 @@ func main() {
 		24*time.Hour,
 		"Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day.")
 
+	defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
+	var configPath string
+	flag.StringVar(
+		&configPath,
+		"config",
+		defaultConfigPath,
+		"YAML or JSON configuration file `path`")
+	flag.Usage = usage
 	flag.Parse()
 
+	// TODO: Load config
+	err := config.LoadFile(theConfig, configPath)
+	if err != nil && (!os.IsNotExist(err) || configPath != defaultConfigPath) {
+		log.Fatal(err)
+	}
+	err = theConfig.Start()
+
 	if maxBuffers < 0 {
 		log.Fatal("-max-buffers must be greater than zero.")
 	}
@@ -263,13 +267,7 @@ func main() {
 		defer os.Remove(pidfile)
 	}
 
-	if len(volumes) == 0 {
-		if (&unixVolumeAdder{&volumes}).Discover() == 0 {
-			log.Fatal("No volumes found.")
-		}
-	}
-
-	for _, v := range volumes {
+	for _, v := range theConfig.Volumes {
 		log.Printf("Using volume %v (writable=%v)", v, v.Writable())
 	}
 
@@ -317,7 +315,7 @@ func main() {
 	}
 
 	// Start a round-robin VolumeManager with the volumes we have found.
-	KeepVM = MakeRRVolumeManager(volumes)
+	KeepVM = MakeRRVolumeManager(theConfig.Volumes)
 
 	// Middleware stack: logger, maxRequests limiter, method handlers
 	http.Handle("/", &LoggingRESTRouter{
@@ -326,7 +324,7 @@ func main() {
 	})
 
 	// Set up a TCP listener.
-	listener, err := net.Listen("tcp", listen)
+	listener, err := net.Listen("tcp", theConfig.Listen)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -362,8 +360,8 @@ func main() {
 	signal.Notify(term, syscall.SIGTERM)
 	signal.Notify(term, syscall.SIGINT)
 
-	log.Println("listening at", listen)
-	srv := &http.Server{Addr: listen}
+	log.Println("listening at", listener.Addr)
+	srv := &http.Server{}
 	srv.Serve(listener)
 }
 
@@ -374,7 +372,7 @@ func emptyTrash(doneEmptyingTrash chan bool, trashCheckInterval time.Duration) {
 	for {
 		select {
 		case <-ticker.C:
-			for _, v := range volumes {
+			for _, v := range theConfig.Volumes {
 				if v.Writable() {
 					v.EmptyTrash()
 				}
diff --git a/services/keepstore/keepstore_test.go b/services/keepstore/keepstore_test.go
index c0adbc0..f0a072a 100644
--- a/services/keepstore/keepstore_test.go
+++ b/services/keepstore/keepstore_test.go
@@ -341,23 +341,23 @@ func TestDiscoverTmpfs(t *testing.T) {
 	f.Close()
 	ProcMounts = f.Name()
 
-	resultVols := volumeSet{}
-	added := (&unixVolumeAdder{&resultVols}).Discover()
+	cfg := &Config{}
+	added := (&unixVolumeAdder{cfg}).Discover()
 
-	if added != len(resultVols) {
+	if added != len(cfg.Volumes) {
 		t.Errorf("Discover returned %d, but added %d volumes",
-			added, len(resultVols))
+			added, len(cfg.Volumes))
 	}
 	if added != len(tempVols) {
 		t.Errorf("Discover returned %d but we set up %d volumes",
 			added, len(tempVols))
 	}
 	for i, tmpdir := range tempVols {
-		if tmpdir != resultVols[i].(*UnixVolume).root {
+		if tmpdir != cfg.Volumes[i].(*UnixVolume).Root {
 			t.Errorf("Discover returned %s, expected %s\n",
-				resultVols[i].(*UnixVolume).root, tmpdir)
+				cfg.Volumes[i].(*UnixVolume).Root, tmpdir)
 		}
-		if expectReadonly := i%2 == 1; expectReadonly != resultVols[i].(*UnixVolume).readonly {
+		if expectReadonly := i%2 == 1; expectReadonly != cfg.Volumes[i].(*UnixVolume).ReadOnly {
 			t.Errorf("Discover added %s with readonly=%v, should be %v",
 				tmpdir, !expectReadonly, expectReadonly)
 		}
@@ -381,10 +381,10 @@ func TestDiscoverNone(t *testing.T) {
 	f.Close()
 	ProcMounts = f.Name()
 
-	resultVols := volumeSet{}
-	added := (&unixVolumeAdder{&resultVols}).Discover()
-	if added != 0 || len(resultVols) != 0 {
-		t.Fatalf("got %d, %v; expected 0, []", added, resultVols)
+	cfg := &Config{}
+	added := (&unixVolumeAdder{cfg}).Discover()
+	if added != 0 || len(cfg.Volumes) != 0 {
+		t.Fatalf("got %d, %v; expected 0, []", added, cfg.Volumes)
 	}
 }
 
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 1a2a47b..630eb46 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -11,8 +11,10 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/AdRoll/goamz/aws"
 	"github.com/AdRoll/goamz/s3"
 )
@@ -39,7 +41,12 @@ const (
 )
 
 type s3VolumeAdder struct {
-	*volumeSet
+	*Config
+}
+
+// String implements flag.Value
+func (s *s3VolumeAdder) String() string {
+	return "-"
 }
 
 func (s *s3VolumeAdder) Set(bucketName string) error {
@@ -49,39 +56,21 @@ func (s *s3VolumeAdder) Set(bucketName string) error {
 	if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
 		return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
 	}
-	region, ok := aws.Regions[s3RegionName]
-	if s3Endpoint == "" {
-		if !ok {
-			return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", s3RegionName)
-		}
-	} else {
-		if ok {
-			return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
-				"specify empty endpoint (\"-s3-endpoint=\") or use a different region name", s3RegionName, s3Endpoint)
-		}
-		region = aws.Region{
-			Name:       s3RegionName,
-			S3Endpoint: s3Endpoint,
-		}
-	}
-	var err error
-	var auth aws.Auth
-	auth.AccessKey, err = readKeyFromFile(s3AccessKeyFile)
-	if err != nil {
-		return err
-	}
-	auth.SecretKey, err = readKeyFromFile(s3SecretKeyFile)
-	if err != nil {
-		return err
-	}
 	if flagSerializeIO {
 		log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
 	}
-	v := NewS3Volume(auth, region, bucketName, s3RaceWindow, flagReadonly, s3Replication)
-	if err := v.Check(); err != nil {
-		return err
-	}
-	*s.volumeSet = append(*s.volumeSet, v)
+	s.Config.Volumes = append(s.Config.Volumes, &S3Volume{
+		Bucket:        bucketName,
+		AccessKeyFile: s3AccessKeyFile,
+		SecretKeyFile: s3SecretKeyFile,
+		Endpoint:      s3Endpoint,
+		Region:        s3RegionName,
+		RaceWindow:    arvados.Duration(s3RaceWindow),
+		S3Replication: s3Replication,
+		UnsafeDelete:  s3UnsafeDelete,
+		ReadOnly:      flagReadonly,
+		IndexPageSize: 1000,
+	})
 	return nil
 }
 
@@ -93,7 +82,9 @@ func s3regions() (okList []string) {
 }
 
 func init() {
-	flag.Var(&s3VolumeAdder{&volumes},
+	VolumeTypes = append(VolumeTypes, func() Volume { return &S3Volume{} })
+
+	flag.Var(&s3VolumeAdder{theConfig},
 		"s3-bucket-volume",
 		"Use the given bucket as a storage volume. Can be given multiple times.")
 	flag.StringVar(
@@ -135,32 +126,61 @@ func init() {
 
 // S3Volume implements Volume using an S3 bucket.
 type S3Volume struct {
-	*s3.Bucket
-	raceWindow    time.Duration
-	readonly      bool
-	replication   int
-	indexPageSize int
-}
-
-// NewS3Volume returns a new S3Volume using the given auth, region,
-// and bucket name. The replication argument specifies the replication
-// level to report when writing data.
-func NewS3Volume(auth aws.Auth, region aws.Region, bucket string, raceWindow time.Duration, readonly bool, replication int) *S3Volume {
-	return &S3Volume{
-		Bucket: &s3.Bucket{
-			S3:   s3.New(auth, region),
-			Name: bucket,
-		},
-		raceWindow:    raceWindow,
-		readonly:      readonly,
-		replication:   replication,
-		indexPageSize: 1000,
-	}
-}
-
-// Check returns an error if the volume is inaccessible (e.g., config
-// error).
-func (v *S3Volume) Check() error {
+	AccessKeyFile      string
+	SecretKeyFile      string
+	Endpoint           string
+	Region             string
+	Bucket             string
+	LocationConstraint bool
+	IndexPageSize      int
+	S3Replication      int
+	RaceWindow         arvados.Duration
+	ReadOnly           bool
+	UnsafeDelete       bool
+
+	bucket *s3.Bucket
+
+	startOnce sync.Once
+}
+
+// Type implements Volume.
+func (*S3Volume) Type() string {
+	return "S3"
+}
+
+// Start populates private fields and verifies the configuration is
+// valid.
+func (v *S3Volume) Start() error {
+	region, ok := aws.Regions[v.Region]
+	if v.Endpoint == "" {
+		if !ok {
+			return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region)
+		}
+	} else if ok {
+		return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
+			"specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint)
+	} else {
+		region = aws.Region{
+			Name:                 v.Region,
+			S3Endpoint:           v.Endpoint,
+			S3LocationConstraint: v.LocationConstraint,
+		}
+	}
+
+	var err error
+	var auth aws.Auth
+	auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile)
+	if err != nil {
+		return err
+	}
+	auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile)
+	if err != nil {
+		return err
+	}
+	v.bucket = &s3.Bucket{
+		S3:   s3.New(auth, region),
+		Name: v.Bucket,
+	}
 	return nil
 }
 
@@ -170,12 +190,12 @@ func (v *S3Volume) Check() error {
 // disappeared in a Trash race, getReader calls fixRace to recover the
 // data, and tries again.
 func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) {
-	rdr, err = v.Bucket.GetReader(loc)
+	rdr, err = v.bucket.GetReader(loc)
 	err = v.translateError(err)
 	if err == nil || !os.IsNotExist(err) {
 		return
 	}
-	_, err = v.Bucket.Head("recent/"+loc, nil)
+	_, err = v.bucket.Head("recent/"+loc, nil)
 	err = v.translateError(err)
 	if err != nil {
 		// If we can't read recent/X, there's no point in
@@ -186,7 +206,7 @@ func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) {
 		err = os.ErrNotExist
 		return
 	}
-	rdr, err = v.Bucket.GetReader(loc)
+	rdr, err = v.bucket.GetReader(loc)
 	if err != nil {
 		log.Printf("warning: reading %s after successful fixRace: %s", loc, err)
 		err = v.translateError(err)
@@ -223,7 +243,7 @@ func (v *S3Volume) Compare(loc string, expect []byte) error {
 
 // Put writes a block.
 func (v *S3Volume) Put(loc string, block []byte) error {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 	var opts s3.Options
@@ -234,20 +254,20 @@ func (v *S3Volume) Put(loc string, block []byte) error {
 		}
 		opts.ContentMD5 = base64.StdEncoding.EncodeToString(md5)
 	}
-	err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, opts)
+	err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, opts)
 	if err != nil {
 		return v.translateError(err)
 	}
-	err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
 	return v.translateError(err)
 }
 
 // Touch sets the timestamp for the given locator to the current time.
 func (v *S3Volume) Touch(loc string) error {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
-	_, err := v.Bucket.Head(loc, nil)
+	_, err := v.bucket.Head(loc, nil)
 	err = v.translateError(err)
 	if os.IsNotExist(err) && v.fixRace(loc) {
 		// The data object got trashed in a race, but fixRace
@@ -255,27 +275,27 @@ func (v *S3Volume) Touch(loc string) error {
 	} else if err != nil {
 		return err
 	}
-	err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
 	return v.translateError(err)
 }
 
 // Mtime returns the stored timestamp for the given locator.
 func (v *S3Volume) Mtime(loc string) (time.Time, error) {
-	_, err := v.Bucket.Head(loc, nil)
+	_, err := v.bucket.Head(loc, nil)
 	if err != nil {
 		return zeroTime, v.translateError(err)
 	}
-	resp, err := v.Bucket.Head("recent/"+loc, nil)
+	resp, err := v.bucket.Head("recent/"+loc, nil)
 	err = v.translateError(err)
 	if os.IsNotExist(err) {
 		// The data object X exists, but recent/X is missing.
-		err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+		err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
 		if err != nil {
 			log.Printf("error: creating %q: %s", "recent/"+loc, err)
 			return zeroTime, v.translateError(err)
 		}
 		log.Printf("info: created %q to migrate existing block to new storage scheme", "recent/"+loc)
-		resp, err = v.Bucket.Head("recent/"+loc, nil)
+		resp, err = v.bucket.Head("recent/"+loc, nil)
 		if err != nil {
 			log.Printf("error: created %q but HEAD failed: %s", "recent/"+loc, err)
 			return zeroTime, v.translateError(err)
@@ -292,14 +312,14 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) {
 func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
 	// Use a merge sort to find matching sets of X and recent/X.
 	dataL := s3Lister{
-		Bucket:   v.Bucket,
+		Bucket:   v.bucket,
 		Prefix:   prefix,
-		PageSize: v.indexPageSize,
+		PageSize: v.IndexPageSize,
 	}
 	recentL := s3Lister{
-		Bucket:   v.Bucket,
+		Bucket:   v.bucket,
 		Prefix:   "recent/" + prefix,
-		PageSize: v.indexPageSize,
+		PageSize: v.IndexPageSize,
 	}
 	for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() {
 		if data.Key >= "g" {
@@ -346,7 +366,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
 
 // Trash a Keep block.
 func (v *S3Volume) Trash(loc string) error {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 	if t, err := v.Mtime(loc); err != nil {
@@ -358,7 +378,7 @@ func (v *S3Volume) Trash(loc string) error {
 		if !s3UnsafeDelete {
 			return ErrS3TrashDisabled
 		}
-		return v.Bucket.Del(loc)
+		return v.bucket.Del(loc)
 	}
 	err := v.checkRaceWindow(loc)
 	if err != nil {
@@ -368,13 +388,13 @@ func (v *S3Volume) Trash(loc string) error {
 	if err != nil {
 		return err
 	}
-	return v.translateError(v.Bucket.Del(loc))
+	return v.translateError(v.bucket.Del(loc))
 }
 
 // checkRaceWindow returns a non-nil error if trash/loc is, or might
 // be, in the race window (i.e., it's not safe to trash loc).
 func (v *S3Volume) checkRaceWindow(loc string) error {
-	resp, err := v.Bucket.Head("trash/"+loc, nil)
+	resp, err := v.bucket.Head("trash/"+loc, nil)
 	err = v.translateError(err)
 	if os.IsNotExist(err) {
 		// OK, trash/X doesn't exist so we're not in the race
@@ -390,7 +410,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
 		// Can't parse timestamp
 		return err
 	}
-	safeWindow := t.Add(trashLifetime).Sub(time.Now().Add(v.raceWindow))
+	safeWindow := t.Add(trashLifetime).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
 	if safeWindow <= 0 {
 		// We can't count on "touch trash/X" to prolong
 		// trash/X's lifetime. The new timestamp might not
@@ -408,10 +428,10 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
 // (PutCopy returns 200 OK if the request was received, even if the
 // copy failed).
 func (v *S3Volume) safeCopy(dst, src string) error {
-	resp, err := v.Bucket.PutCopy(dst, s3ACL, s3.CopyOptions{
+	resp, err := v.bucket.PutCopy(dst, s3ACL, s3.CopyOptions{
 		ContentType:       "application/octet-stream",
 		MetadataDirective: "REPLACE",
-	}, v.Bucket.Name+"/"+src)
+	}, v.bucket.Name+"/"+src)
 	err = v.translateError(err)
 	if err != nil {
 		return err
@@ -446,7 +466,7 @@ func (v *S3Volume) Untrash(loc string) error {
 	if err != nil {
 		return err
 	}
-	err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{})
 	return v.translateError(err)
 }
 
@@ -463,19 +483,19 @@ func (v *S3Volume) Status() *VolumeStatus {
 
 // String implements fmt.Stringer.
 func (v *S3Volume) String() string {
-	return fmt.Sprintf("s3-bucket:%+q", v.Bucket.Name)
+	return fmt.Sprintf("s3-bucket:%+q", v.bucket.Name)
 }
 
 // Writable returns false if all future Put, Mtime, and Delete calls
 // are expected to fail.
 func (v *S3Volume) Writable() bool {
-	return !v.readonly
+	return !v.ReadOnly
 }
 
 // Replication returns the storage redundancy of the underlying
 // device. Configured via command line flag.
 func (v *S3Volume) Replication() int {
-	return v.replication
+	return v.S3Replication
 }
 
 var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
@@ -489,7 +509,7 @@ func (v *S3Volume) isKeepBlock(s string) bool {
 // there was a race between Put and Trash, fixRace recovers from the
 // race by Untrashing the block.
 func (v *S3Volume) fixRace(loc string) bool {
-	trash, err := v.Bucket.Head("trash/"+loc, nil)
+	trash, err := v.bucket.Head("trash/"+loc, nil)
 	if err != nil {
 		if !os.IsNotExist(v.translateError(err)) {
 			log.Printf("error: fixRace: HEAD %q: %s", "trash/"+loc, err)
@@ -502,7 +522,7 @@ func (v *S3Volume) fixRace(loc string) bool {
 		return false
 	}
 
-	recent, err := v.Bucket.Head("recent/"+loc, nil)
+	recent, err := v.bucket.Head("recent/"+loc, nil)
 	if err != nil {
 		log.Printf("error: fixRace: HEAD %q: %s", "recent/"+loc, err)
 		return false
@@ -552,9 +572,9 @@ func (v *S3Volume) EmptyTrash() {
 
 	// Use a merge sort to find matching sets of trash/X and recent/X.
 	trashL := s3Lister{
-		Bucket:   v.Bucket,
+		Bucket:   v.bucket,
 		Prefix:   "trash/",
-		PageSize: v.indexPageSize,
+		PageSize: v.IndexPageSize,
 	}
 	// Define "ready to delete" as "...when EmptyTrash started".
 	startT := time.Now()
@@ -571,7 +591,7 @@ func (v *S3Volume) EmptyTrash() {
 			log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err)
 			continue
 		}
-		recent, err := v.Bucket.Head("recent/"+loc, nil)
+		recent, err := v.bucket.Head("recent/"+loc, nil)
 		if err != nil && os.IsNotExist(v.translateError(err)) {
 			log.Printf("warning: %s: EmptyTrash: found trash marker %q but no %q (%s); calling Untrash", v, trash.Key, "recent/"+loc, err)
 			err = v.Untrash(loc)
@@ -589,7 +609,7 @@ func (v *S3Volume) EmptyTrash() {
 			continue
 		}
 		if trashT.Sub(recentT) < blobSignatureTTL {
-			if age := startT.Sub(recentT); age >= blobSignatureTTL-v.raceWindow {
+			if age := startT.Sub(recentT); age >= blobSignatureTTL-time.Duration(v.RaceWindow) {
 				// recent/loc is too old to protect
 				// loc from being Trashed again during
 				// the raceWindow that starts if we
@@ -602,7 +622,7 @@ func (v *S3Volume) EmptyTrash() {
 				v.fixRace(loc)
 				v.Touch(loc)
 				continue
-			} else if _, err := v.Bucket.Head(loc, nil); os.IsNotExist(err) {
+			} else if _, err := v.bucket.Head(loc, nil); os.IsNotExist(err) {
 				log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc)
 				v.fixRace(loc)
 				continue
@@ -614,7 +634,7 @@ func (v *S3Volume) EmptyTrash() {
 		if startT.Sub(trashT) < trashLifetime {
 			continue
 		}
-		err = v.Bucket.Del(trash.Key)
+		err = v.bucket.Del(trash.Key)
 		if err != nil {
 			log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err)
 			continue
@@ -622,9 +642,9 @@ func (v *S3Volume) EmptyTrash() {
 		bytesDeleted += trash.Size
 		blocksDeleted++
 
-		_, err = v.Bucket.Head(loc, nil)
+		_, err = v.bucket.Head(loc, nil)
 		if os.IsNotExist(err) {
-			err = v.Bucket.Del("recent/" + loc)
+			err = v.bucket.Del("recent/" + loc)
 			if err != nil {
 				log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err)
 			}
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 6ba3904..3e4dbcb 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -4,23 +4,17 @@ import (
 	"bytes"
 	"crypto/md5"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"time"
 
-	"github.com/AdRoll/goamz/aws"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"github.com/AdRoll/goamz/s3"
 	"github.com/AdRoll/goamz/s3/s3test"
 	check "gopkg.in/check.v1"
 )
 
-type TestableS3Volume struct {
-	*S3Volume
-	server      *s3test.Server
-	c           *check.C
-	serverClock *fakeClock
-}
-
 const (
 	TestBucketName = "testbucket"
 )
@@ -42,30 +36,6 @@ func init() {
 	s3UnsafeDelete = true
 }
 
-func NewTestableS3Volume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume {
-	clock := &fakeClock{}
-	srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
-	c.Assert(err, check.IsNil)
-	auth := aws.Auth{}
-	region := aws.Region{
-		Name:                 "test-region-1",
-		S3Endpoint:           srv.URL(),
-		S3LocationConstraint: true,
-	}
-	bucket := &s3.Bucket{
-		S3:   s3.New(auth, region),
-		Name: TestBucketName,
-	}
-	err = bucket.PutBucket(s3.ACL("private"))
-	c.Assert(err, check.IsNil)
-
-	return &TestableS3Volume{
-		S3Volume:    NewS3Volume(auth, region, TestBucketName, raceWindow, readonly, replication),
-		server:      srv,
-		serverClock: clock,
-	}
-}
-
 var _ = check.Suite(&StubbedS3Suite{})
 
 type StubbedS3Suite struct {
@@ -76,19 +46,19 @@ func (s *StubbedS3Suite) TestGeneric(c *check.C) {
 	DoGenericVolumeTests(c, func(t TB) TestableVolume {
 		// Use a negative raceWindow so s3test's 1-second
 		// timestamp precision doesn't confuse fixRace.
-		return NewTestableS3Volume(c, -2*time.Second, false, 2)
+		return s.newTestableVolume(c, -2*time.Second, false, 2)
 	})
 }
 
 func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
 	DoGenericVolumeTests(c, func(t TB) TestableVolume {
-		return NewTestableS3Volume(c, -2*time.Second, true, 2)
+		return s.newTestableVolume(c, -2*time.Second, true, 2)
 	})
 }
 
 func (s *StubbedS3Suite) TestIndex(c *check.C) {
-	v := NewTestableS3Volume(c, 0, false, 2)
-	v.indexPageSize = 3
+	v := s.newTestableVolume(c, 0, false, 2)
+	v.IndexPageSize = 3
 	for i := 0; i < 256; i++ {
 		v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111})
 	}
@@ -119,7 +89,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
 	trashLifetime = time.Hour
 	blobSignatureTTL = time.Hour
 
-	v := NewTestableS3Volume(c, 5*time.Minute, false, 2)
+	v := s.newTestableVolume(c, 5*time.Minute, false, 2)
 	var none time.Time
 
 	putS3Obj := func(t time.Time, key string, data []byte) {
@@ -127,7 +97,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
 			return
 		}
 		v.serverClock.now = &t
-		v.Bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{})
+		v.bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{})
 	}
 
 	t0 := time.Now()
@@ -286,7 +256,7 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
 		// freshAfterEmpty
 		loc, blk = setupScenario()
 		v.EmptyTrash()
-		_, err = v.Bucket.Head("trash/"+loc, nil)
+		_, err = v.bucket.Head("trash/"+loc, nil)
 		c.Check(err == nil, check.Equals, scenario.haveTrashAfterEmpty)
 		if scenario.freshAfterEmpty {
 			t, err := v.Mtime(loc)
@@ -307,9 +277,51 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
 	}
 }
 
+type TestableS3Volume struct {
+	*S3Volume
+	server      *s3test.Server
+	c           *check.C
+	serverClock *fakeClock
+}
+
+func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume {
+	clock := &fakeClock{}
+	srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
+	c.Assert(err, check.IsNil)
+
+	tmp, err := ioutil.TempFile("", "keepstore")
+	c.Assert(err, check.IsNil)
+	defer os.Remove(tmp.Name())
+	_, err = tmp.Write([]byte("xxx\n"))
+	c.Assert(err, check.IsNil)
+	c.Assert(tmp.Close(), check.IsNil)
+
+	v := &TestableS3Volume{
+		S3Volume: &S3Volume{
+			Bucket:             TestBucketName,
+			AccessKeyFile:      tmp.Name(),
+			SecretKeyFile:      tmp.Name(),
+			Endpoint:           srv.URL(),
+			Region:             "test-region-1",
+			LocationConstraint: true,
+			RaceWindow:         arvados.Duration(raceWindow),
+			S3Replication:      replication,
+			UnsafeDelete:       s3UnsafeDelete,
+			ReadOnly:           readonly,
+			IndexPageSize:      1000,
+		},
+		server:      srv,
+		serverClock: clock,
+	}
+	c.Assert(v.Start(), check.IsNil)
+	err = v.bucket.PutBucket(s3.ACL("private"))
+	c.Assert(err, check.IsNil)
+	return v
+}
+
 // PutRaw skips the ContentMD5 test
 func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
-	err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
+	err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
 	if err != nil {
 		log.Printf("PutRaw: %+v", err)
 	}
@@ -320,7 +332,7 @@ func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
 // while we do this.
 func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) {
 	v.serverClock.now = &lastPut
-	err := v.Bucket.Put("recent/"+locator, nil, "application/octet-stream", s3ACL, s3.Options{})
+	err := v.bucket.Put("recent/"+locator, nil, "application/octet-stream", s3ACL, s3.Options{})
 	if err != nil {
 		panic(err)
 	}
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
new file mode 100644
index 0000000..073b3b7
--- /dev/null
+++ b/services/keepstore/usage.go
@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/ghodss/yaml"
+)
+
+func usage() {
+	c := DefaultConfig()
+	exampleConfigFile, err := yaml.Marshal(c)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Fprintf(os.Stderr, `
+
+keepstore provides a content-addressed data store backed by a local filesystem or networked storage.
+
+Usage: keepstore -config path/to/keepstore.yml
+
+Options:
+`)
+	flag.PrintDefaults()
+	fmt.Fprintf(os.Stderr, `
+Example config file:
+
+%s
+
+Listen:
+
+    Local port to listen on. Can be "address", "address:port", or
+    ":port", where "address" is a host IP address or name and "port"
+    is a port number or name.
+
+TrashLifetime:
+
+    Minimum time a block can spend in the trash before being deleted
+    outright.
+
+`, exampleConfigFile)
+}
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 8ae6660..4a2a1b1 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -10,6 +10,15 @@ import (
 // for example, a single mounted disk, a RAID array, an Amazon S3 volume,
 // etc.
 type Volume interface {
+	// Volume type as specified in config file. Examples: "S3",
+	// "Directory".
+	Type() string
+
+	// Do whatever private setup tasks and configuration checks
+	// are needed. Return non-nil if the volume is unusable (e.g.,
+	// invalid config).
+	Start() error
+
 	// Get a block: copy the block data into buf, and return the
 	// number of bytes copied.
 	//
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 5671b8d..d2f3347 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -198,7 +198,14 @@ func (v *MockVolume) Trash(loc string) error {
 	return os.ErrNotExist
 }
 
-// TBD
+func (v *MockVolume) Type() string {
+	return "Mock"
+}
+
+func (v *MockVolume) Start() error {
+	return nil
+}
+
 func (v *MockVolume) Untrash(loc string) error {
 	return nil
 }
diff --git a/services/keepstore/volume_unix.go b/services/keepstore/volume_unix.go
index 5982fb0..7be4882 100644
--- a/services/keepstore/volume_unix.go
+++ b/services/keepstore/volume_unix.go
@@ -19,11 +19,16 @@ import (
 )
 
 type unixVolumeAdder struct {
-	*volumeSet
+	*Config
 }
 
-func (vs *unixVolumeAdder) Set(value string) error {
-	if dirs := strings.Split(value, ","); len(dirs) > 1 {
+// String implements flag.Value
+func (s *unixVolumeAdder) String() string {
+	return "-"
+}
+
+func (vs *unixVolumeAdder) Set(path string) error {
+	if dirs := strings.Split(path, ","); len(dirs) > 1 {
 		log.Print("DEPRECATED: using comma-separated volume list.")
 		for _, dir := range dirs {
 			if err := vs.Set(dir); err != nil {
@@ -32,31 +37,31 @@ func (vs *unixVolumeAdder) Set(value string) error {
 		}
 		return nil
 	}
-	if len(value) == 0 || value[0] != '/' {
+	if len(path) == 0 || path[0] != '/' {
 		return errors.New("Invalid volume: must begin with '/'.")
 	}
-	if _, err := os.Stat(value); err != nil {
+	if _, err := os.Stat(path); err != nil {
 		return err
 	}
 	var locker sync.Locker
 	if flagSerializeIO {
 		locker = &sync.Mutex{}
 	}
-	*vs.volumeSet = append(*vs.volumeSet, &UnixVolume{
-		root:     value,
+	vs.Config.Volumes = append(vs.Config.Volumes, &UnixVolume{
+		Root:     path,
+		ReadOnly: flagReadonly,
 		locker:   locker,
-		readonly: flagReadonly,
 	})
 	return nil
 }
 
 func init() {
 	flag.Var(
-		&unixVolumeAdder{&volumes},
+		&unixVolumeAdder{theConfig},
 		"volumes",
 		"Deprecated synonym for -volume.")
 	flag.Var(
-		&unixVolumeAdder{&volumes},
+		&unixVolumeAdder{theConfig},
 		"volume",
 		"Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named \"keep\" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead.")
 }
@@ -111,17 +116,28 @@ func (vs *unixVolumeAdder) Discover() int {
 
 // A UnixVolume stores and retrieves blocks in a local directory.
 type UnixVolume struct {
-	// path to the volume's root directory
-	root string
+	Root     string // path to the volume's root directory
+	ReadOnly bool
+
 	// something to lock during IO, typically a sync.Mutex (or nil
 	// to skip locking)
-	locker   sync.Locker
-	readonly bool
+	locker sync.Locker
+}
+
+// Type implements Volume
+func (v *UnixVolume) Type() string {
+	return "Directory"
+}
+
+// Start implements Volume
+func (v *UnixVolume) Start() error {
+	_, err := os.Stat(v.Root)
+	return err
 }
 
 // Touch sets the timestamp for the given locator to the current time
 func (v *UnixVolume) Touch(loc string) error {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 	p := v.blockPath(loc)
@@ -218,7 +234,7 @@ func (v *UnixVolume) Compare(loc string, expect []byte) error {
 // returns a FullError.  If the write fails due to some other error,
 // that error is returned.
 func (v *UnixVolume) Put(loc string, block []byte) error {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 	if v.IsFull() {
@@ -268,14 +284,14 @@ func (v *UnixVolume) Status() *VolumeStatus {
 	var fs syscall.Statfs_t
 	var devnum uint64
 
-	if fi, err := os.Stat(v.root); err == nil {
+	if fi, err := os.Stat(v.Root); err == nil {
 		devnum = fi.Sys().(*syscall.Stat_t).Dev
 	} else {
 		log.Printf("%s: os.Stat: %s\n", v, err)
 		return nil
 	}
 
-	err := syscall.Statfs(v.root, &fs)
+	err := syscall.Statfs(v.Root, &fs)
 	if err != nil {
 		log.Printf("%s: statfs: %s\n", v, err)
 		return nil
@@ -285,7 +301,7 @@ func (v *UnixVolume) Status() *VolumeStatus {
 	// uses fs.Blocks - fs.Bfree.
 	free := fs.Bavail * uint64(fs.Bsize)
 	used := (fs.Blocks - fs.Bfree) * uint64(fs.Bsize)
-	return &VolumeStatus{v.root, devnum, free, used}
+	return &VolumeStatus{v.Root, devnum, free, used}
 }
 
 var blockDirRe = regexp.MustCompile(`^[0-9a-f]+$`)
@@ -307,7 +323,7 @@ var blockFileRe = regexp.MustCompile(`^[0-9a-f]{32}$`)
 //
 func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error {
 	var lastErr error
-	rootdir, err := os.Open(v.root)
+	rootdir, err := os.Open(v.Root)
 	if err != nil {
 		return err
 	}
@@ -326,7 +342,7 @@ func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error {
 		if !blockDirRe.MatchString(names[0]) {
 			continue
 		}
-		blockdirpath := filepath.Join(v.root, names[0])
+		blockdirpath := filepath.Join(v.Root, names[0])
 		blockdir, err := os.Open(blockdirpath)
 		if err != nil {
 			log.Print("Error reading ", blockdirpath, ": ", err)
@@ -372,7 +388,7 @@ func (v *UnixVolume) Trash(loc string) error {
 	// Trash() will read the correct up-to-date timestamp and choose not to
 	// trash the file.
 
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 	if v.locker != nil {
@@ -411,7 +427,7 @@ func (v *UnixVolume) Trash(loc string) error {
 // Look for path/{loc}.trash.{deadline} in storage,
 // and rename the first such file as path/{loc}
 func (v *UnixVolume) Untrash(loc string) (err error) {
-	if v.readonly {
+	if v.ReadOnly {
 		return MethodDisabledError
 	}
 
@@ -446,7 +462,7 @@ func (v *UnixVolume) Untrash(loc string) (err error) {
 // blockDir returns the fully qualified directory name for the directory
 // where loc is (or would be) stored on this volume.
 func (v *UnixVolume) blockDir(loc string) string {
-	return filepath.Join(v.root, loc[0:3])
+	return filepath.Join(v.Root, loc[0:3])
 }
 
 // blockPath returns the fully qualified pathname for the path to loc
@@ -459,7 +475,7 @@ func (v *UnixVolume) blockPath(loc string) string {
 // MinFreeKilobytes.
 //
 func (v *UnixVolume) IsFull() (isFull bool) {
-	fullSymlink := v.root + "/full"
+	fullSymlink := v.Root + "/full"
 
 	// Check if the volume has been marked as full in the last hour.
 	if link, err := os.Readlink(fullSymlink); err == nil {
@@ -491,7 +507,7 @@ func (v *UnixVolume) IsFull() (isFull bool) {
 //
 func (v *UnixVolume) FreeDiskSpace() (free uint64, err error) {
 	var fs syscall.Statfs_t
-	err = syscall.Statfs(v.root, &fs)
+	err = syscall.Statfs(v.Root, &fs)
 	if err == nil {
 		// Statfs output is not guaranteed to measure free
 		// space in terms of 1K blocks.
@@ -501,13 +517,13 @@ func (v *UnixVolume) FreeDiskSpace() (free uint64, err error) {
 }
 
 func (v *UnixVolume) String() string {
-	return fmt.Sprintf("[UnixVolume %s]", v.root)
+	return fmt.Sprintf("[UnixVolume %s]", v.Root)
 }
 
 // Writable returns false if all future Put, Mtime, and Delete calls
 // are expected to fail.
 func (v *UnixVolume) Writable() bool {
-	return !v.readonly
+	return !v.ReadOnly
 }
 
 // Replication returns the number of replicas promised by the
@@ -546,7 +562,7 @@ func (v *UnixVolume) EmptyTrash() {
 	var bytesDeleted, bytesInTrash int64
 	var blocksDeleted, blocksInTrash int
 
-	err := filepath.Walk(v.root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(v.Root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			log.Printf("EmptyTrash: filepath.Walk: %v: %v", path, err)
 			return nil
diff --git a/services/keepstore/volume_unix_test.go b/services/keepstore/volume_unix_test.go
index c95538b..ac0a492 100644
--- a/services/keepstore/volume_unix_test.go
+++ b/services/keepstore/volume_unix_test.go
@@ -30,9 +30,9 @@ func NewTestableUnixVolume(t TB, serialize bool, readonly bool) *TestableUnixVol
 	}
 	return &TestableUnixVolume{
 		UnixVolume: UnixVolume{
-			root:     d,
+			Root:     d,
+			ReadOnly: readonly,
 			locker:   locker,
-			readonly: readonly,
 		},
 		t: t,
 	}
@@ -42,9 +42,9 @@ func NewTestableUnixVolume(t TB, serialize bool, readonly bool) *TestableUnixVol
 // the volume is readonly.
 func (v *TestableUnixVolume) PutRaw(locator string, data []byte) {
 	defer func(orig bool) {
-		v.readonly = orig
-	}(v.readonly)
-	v.readonly = false
+		v.ReadOnly = orig
+	}(v.ReadOnly)
+	v.ReadOnly = false
 	err := v.Put(locator, data)
 	if err != nil {
 		v.t.Fatal(err)
@@ -59,7 +59,7 @@ func (v *TestableUnixVolume) TouchWithDate(locator string, lastPut time.Time) {
 }
 
 func (v *TestableUnixVolume) Teardown() {
-	if err := os.RemoveAll(v.root); err != nil {
+	if err := os.RemoveAll(v.Root); err != nil {
 		v.t.Fatal(err)
 	}
 }
@@ -126,7 +126,7 @@ func TestPut(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	p := fmt.Sprintf("%s/%s/%s", v.root, TestHash[:3], TestHash)
+	p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
 	if buf, err := ioutil.ReadFile(p); err != nil {
 		t.Error(err)
 	} else if bytes.Compare(buf, TestBlock) != 0 {
@@ -139,7 +139,7 @@ func TestPutBadVolume(t *testing.T) {
 	v := NewTestableUnixVolume(t, false, false)
 	defer v.Teardown()
 
-	os.Chmod(v.root, 000)
+	os.Chmod(v.Root, 000)
 	err := v.Put(TestHash, TestBlock)
 	if err == nil {
 		t.Error("Write should have failed")
@@ -178,7 +178,7 @@ func TestIsFull(t *testing.T) {
 	v := NewTestableUnixVolume(t, false, false)
 	defer v.Teardown()
 
-	fullPath := v.root + "/full"
+	fullPath := v.Root + "/full"
 	now := fmt.Sprintf("%d", time.Now().Unix())
 	os.Symlink(now, fullPath)
 	if !v.IsFull() {
@@ -200,8 +200,8 @@ func TestNodeStatus(t *testing.T) {
 
 	// Get node status and make a basic sanity check.
 	volinfo := v.Status()
-	if volinfo.MountPoint != v.root {
-		t.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.root)
+	if volinfo.MountPoint != v.Root {
+		t.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.Root)
 	}
 	if volinfo.DeviceNum == 0 {
 		t.Errorf("uninitialized device_num in %v", volinfo)
@@ -301,7 +301,7 @@ func TestUnixVolumeCompare(t *testing.T) {
 		t.Errorf("Got err %q, expected %q", err, DiskHashError)
 	}
 
-	p := fmt.Sprintf("%s/%s/%s", v.root, TestHash[:3], TestHash)
+	p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
 	os.Chmod(p, 000)
 	err = v.Compare(TestHash, TestBlock)
 	if err == nil || strings.Index(err.Error(), "permission denied") < 0 {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list