[ARVADOS] updated: 1.3.0-1563-gc01708aa5
Git user
git at public.curoverse.com
Fri Sep 6 17:25:08 UTC 2019
Summary of changes:
lib/config/deprecated_keepstore.go | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
discards c4b2a2eb8ba9dc7387fb23673c18d3b2c9dc171c (commit)
via c01708aa52bd78d76fe9372ff8c218ce447ef404 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (c4b2a2eb8ba9dc7387fb23673c18d3b2c9dc171c)
\
N -- N -- N (c01708aa52bd78d76fe9372ff8c218ce447ef404)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c01708aa52bd78d76fe9372ff8c218ce447ef404
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Sep 6 13:24:57 2019 -0400
13647: Use cluster config instead of custom keepstore config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d70722272..f374fc53e 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -470,6 +470,7 @@ stop_services() {
&& python sdk/python/tests/run_test_server.py stop \
&& all_services_stopped=1
deactivate
+ unset ARVADOS_CONFIG
}
interrupt() {
@@ -633,15 +634,12 @@ install_env() {
for d in \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
"$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -h "$d" ]] && rm "$d"
[[ -d "$d" ]] && rmdir "$d"
done
fi
- for d in \
- "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
- "$GOPATH/src/git.curoverse.com/arvados.git"; do
- [[ -h "$d" ]] && rm "$d"
- done
ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v github.com/kardianos/govendor
cd "$GOPATH/src/git.curoverse.com/arvados.git"
@@ -712,8 +710,6 @@ retry() {
}
do_test() {
- check_arvados_config "$1"
-
case "${1}" in
apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
suite=apps/workbench
@@ -733,11 +729,14 @@ do_test() {
case "${1}" in
services/api)
stop_services
+ check_arvados_config "$1"
;;
gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ check_arvados_config "$1"
# don't care whether services are running
;;
*)
+ check_arvados_config "$1"
if ! start_services; then
title "test $1 -- failed to start services"
return 1
@@ -830,16 +829,17 @@ check_arvados_config() {
install_env
fi
. "$VENVDIR/bin/activate"
+ cd "$WORKSPACE"
eval $(python sdk/python/tests/run_test_server.py setup_config)
deactivate
fi
}
do_install() {
- check_arvados_config "$1"
if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then
return 0
fi
+ check_arvados_config "$1"
retry do_install_once ${@}
}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index af7c57120..019d5edd0 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -85,7 +85,7 @@ func (s *CommandSuite) TestCheckOldKeepstoreConfigFile(c *check.C) {
c.Assert(err, check.IsNil)
defer os.Remove(f.Name())
- io.WriteString(f, "Debug: true\n")
+ io.WriteString(f, "Listen: :12345\nDebug: true\n")
var stdout, stderr bytes.Buffer
in := `
@@ -97,7 +97,7 @@ Clusters:
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-", "-legacy-keepstore-config", f.Name()}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 1)
c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*LogLevel: info\n\+ +LogLevel: debug\n.*`)
- c.Check(stderr.String(), check.Matches, `.*you should remove the legacy keepstore config file.*\n`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*you should remove the legacy keepstore config file.*\n`)
}
func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 24b2e450e..b10a56e53 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -176,6 +176,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -316,15 +325,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
# Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -741,6 +779,48 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+ Serialize: false
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 9eb8c40c1..0a030fb04 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -102,12 +102,6 @@ func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc
svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
}
-const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
-
-type oldKeepstoreConfig struct {
- Debug *bool
-}
-
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
if path == "" {
return nil
@@ -126,35 +120,6 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
return nil
}
-// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
- if ldr.KeepstorePath == "" {
- return nil
- }
- var oc oldKeepstoreConfig
- err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
- if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
- return nil
- } else if err != nil {
- return err
- }
-
- cluster, err := cfg.GetCluster("")
- if err != nil {
- return err
- }
-
- if v := oc.Debug; v == nil {
- } else if *v && cluster.SystemLogs.LogLevel != "debug" {
- cluster.SystemLogs.LogLevel = "debug"
- } else if !*v && cluster.SystemLogs.LogLevel != "info" {
- cluster.SystemLogs.LogLevel = "info"
- }
-
- cfg.Clusters[cluster.ClusterID] = *cluster
- return nil
-}
-
type oldCrunchDispatchSlurmConfig struct {
Client *arvados.Client
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
new file mode 100644
index 000000000..d9d3dc5ba
--- /dev/null
+++ b/lib/config/deprecated_keepstore.go
@@ -0,0 +1,551 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/rand"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "os"
+ "strconv"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
+
+type oldKeepstoreConfig struct {
+ Debug *bool
+ Listen *string
+
+ LogFormat *string
+
+ PIDFile *string
+
+ MaxBuffers *int
+ MaxRequests *int
+
+ BlobSignatureTTL *arvados.Duration
+ BlobSigningKeyFile *string
+ RequireSignatures *bool
+ SystemAuthTokenFile *string
+ EnableDelete *bool
+ TrashLifetime *arvados.Duration
+ TrashCheckInterval *arvados.Duration
+ PullWorkers *int
+ TrashWorkers *int
+ EmptyTrashWorkers *int
+ TLSCertificateFile *string
+ TLSKeyFile *string
+
+ Volumes *oldKeepstoreVolumeList
+
+ ManagementToken *string
+
+ DiscoverVolumesFromMountsFile string // not a real legacy config -- just useful for tests
+}
+
+type oldKeepstoreVolumeList []oldKeepstoreVolume
+
+type oldKeepstoreVolume struct {
+ arvados.Volume
+ Type string `json:",omitempty"`
+
+ // Azure driver configs
+ StorageAccountName string `json:",omitempty"`
+ StorageAccountKeyFile string `json:",omitempty"`
+ StorageBaseURL string `json:",omitempty"`
+ ContainerName string `json:",omitempty"`
+ AzureReplication int `json:",omitempty"`
+ RequestTimeout arvados.Duration `json:",omitempty"`
+ ListBlobsRetryDelay arvados.Duration `json:",omitempty"`
+ ListBlobsMaxAttempts int `json:",omitempty"`
+
+ // S3 driver configs
+ AccessKeyFile string `json:",omitempty"`
+ SecretKeyFile string `json:",omitempty"`
+ Endpoint string `json:",omitempty"`
+ Region string `json:",omitempty"`
+ Bucket string `json:",omitempty"`
+ LocationConstraint bool `json:",omitempty"`
+ IndexPageSize int `json:",omitempty"`
+ S3Replication int `json:",omitempty"`
+ ConnectTimeout arvados.Duration `json:",omitempty"`
+ ReadTimeout arvados.Duration `json:",omitempty"`
+ RaceWindow arvados.Duration `json:",omitempty"`
+ UnsafeDelete bool `json:",omitempty"`
+
+ // Directory driver configs
+ Root string
+ DirectoryReplication int
+ Serialize bool
+
+ // Common configs
+ ReadOnly bool `json:",omitempty"`
+ StorageClasses []string `json:",omitempty"`
+}
+
+// update config using values from an old-style keepstore config file.
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+ if ldr.KeepstorePath == "" {
+ return nil
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ return fmt.Errorf("getting hostname: %s", err)
+ }
+
+ var oc oldKeepstoreConfig
+ err = ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
+ if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ myURL := arvados.URL{Scheme: "http"}
+ if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
+ myURL.Scheme = "https"
+ }
+
+ if v := oc.Debug; v == nil {
+ } else if *v && cluster.SystemLogs.LogLevel != "debug" {
+ cluster.SystemLogs.LogLevel = "debug"
+ } else if !*v && cluster.SystemLogs.LogLevel != "info" {
+ cluster.SystemLogs.LogLevel = "info"
+ }
+
+ if v := oc.TLSCertificateFile; v != nil && "file://"+*v != cluster.TLS.Certificate {
+ cluster.TLS.Certificate = "file://" + *v
+ }
+ if v := oc.TLSKeyFile; v != nil && "file://"+*v != cluster.TLS.Key {
+ cluster.TLS.Key = "file://" + *v
+ }
+ if v := oc.Listen; v != nil {
+ if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+ // already listed
+ myURL.Host = *v
+ } else if len(*v) > 1 && (*v)[0] == ':' {
+ myURL.Host = net.JoinHostPort(hostname, (*v)[1:])
+ cluster.Services.Keepstore.InternalURLs[myURL] = arvados.ServiceInstance{}
+ } else {
+ return fmt.Errorf("unable to migrate Listen value %q from legacy keepstore config file -- remove after configuring Services.Keepstore.InternalURLs.", *v)
+ }
+ } else {
+ for url := range cluster.Services.Keepstore.InternalURLs {
+ if host, _, _ := net.SplitHostPort(url.Host); host == hostname {
+ myURL = url
+ break
+ }
+ }
+ if myURL.Host == "" {
+ return fmt.Errorf("unable to migrate legacy keepstore config: no 'Listen' key, and hostname %q does not match an entry in Services.Keepstore.InternalURLs", hostname)
+ }
+ }
+
+ if v := oc.LogFormat; v != nil && *v != cluster.SystemLogs.Format {
+ cluster.SystemLogs.Format = *v
+ }
+ if v := oc.MaxBuffers; v != nil && *v != cluster.API.MaxKeepBlockBuffers {
+ cluster.API.MaxKeepBlockBuffers = *v
+ }
+ if v := oc.MaxRequests; v != nil && *v != cluster.API.MaxConcurrentRequests {
+ cluster.API.MaxConcurrentRequests = *v
+ }
+ if v := oc.BlobSignatureTTL; v != nil && *v != cluster.Collections.BlobSigningTTL {
+ cluster.Collections.BlobSigningTTL = *v
+ }
+ if v := oc.BlobSigningKeyFile; v != nil {
+ buf, err := ioutil.ReadFile(*v)
+ if err != nil {
+ return fmt.Errorf("error reading BlobSigningKeyFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.Collections.BlobSigningKey {
+ cluster.Collections.BlobSigningKey = key
+ }
+ }
+ if v := oc.RequireSignatures; v != nil && *v != cluster.Collections.BlobSigning {
+ cluster.Collections.BlobSigning = *v
+ }
+ if v := oc.SystemAuthTokenFile; v != nil {
+ f, err := os.Open(*v)
+ if err != nil {
+ return fmt.Errorf("error opening SystemAuthTokenFile: %s", err)
+ }
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ if err != nil {
+ return fmt.Errorf("error reading SystemAuthTokenFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.SystemRootToken {
+ cluster.SystemRootToken = key
+ }
+ }
+ if v := oc.EnableDelete; v != nil && *v != cluster.Collections.BlobTrash {
+ cluster.Collections.BlobTrash = *v
+ }
+ if v := oc.TrashLifetime; v != nil && *v != cluster.Collections.BlobTrashLifetime {
+ cluster.Collections.BlobTrashLifetime = *v
+ }
+ if v := oc.TrashCheckInterval; v != nil && *v != cluster.Collections.BlobTrashCheckInterval {
+ cluster.Collections.BlobTrashCheckInterval = *v
+ }
+ if v := oc.TrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobTrashConcurrency = *v
+ }
+ if v := oc.EmptyTrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobDeleteConcurrency = *v
+ }
+ if v := oc.PullWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobReplicateConcurrency = *v
+ }
+ if v := oc.Volumes; v == nil {
+ ldr.Logger.Warn("no volumes in legacy config; discovering local directory volumes")
+ err := ldr.discoverLocalVolumes(cluster, oc.DiscoverVolumesFromMountsFile, myURL)
+ if err != nil {
+ return fmt.Errorf("error discovering local directory volumes: %s", err)
+ }
+ } else {
+ for i, oldvol := range *v {
+ var accessViaHosts map[arvados.URL]arvados.VolumeAccess
+ oldUUID, found := ldr.alreadyMigrated(oldvol, cluster.Volumes, myURL)
+ if found {
+ accessViaHosts = cluster.Volumes[oldUUID].AccessViaHosts
+ writers := false
+ for _, va := range accessViaHosts {
+ if !va.ReadOnly {
+ writers = true
+ }
+ }
+ if writers || len(accessViaHosts) == 0 {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ if len(accessViaHosts) > 0 {
+ cluster.Volumes[oldUUID].AccessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ }
+ continue
+ }
+ }
+ var newvol arvados.Volume
+ if found {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ newvol = cluster.Volumes[oldUUID]
+ // Remove the old entry. It will be
+ // added back below, possibly with a
+ // new UUID.
+ delete(cluster.Volumes, oldUUID)
+ } else {
+ var params interface{}
+ switch oldvol.Type {
+ case "S3":
+ accesskeydata, err := ioutil.ReadFile(oldvol.AccessKeyFile)
+ if err != nil && oldvol.AccessKeyFile != "" {
+ return fmt.Errorf("error reading AccessKeyFile: %s", err)
+ }
+ secretkeydata, err := ioutil.ReadFile(oldvol.SecretKeyFile)
+ if err != nil && oldvol.SecretKeyFile != "" {
+ return fmt.Errorf("error reading SecretKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "S3",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.S3Replication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.S3VolumeDriverParameters{
+ AccessKey: string(bytes.TrimSpace(accesskeydata)),
+ SecretKey: string(bytes.TrimSpace(secretkeydata)),
+ Endpoint: oldvol.Endpoint,
+ Region: oldvol.Region,
+ Bucket: oldvol.Bucket,
+ LocationConstraint: oldvol.LocationConstraint,
+ IndexPageSize: oldvol.IndexPageSize,
+ ConnectTimeout: oldvol.ConnectTimeout,
+ ReadTimeout: oldvol.ReadTimeout,
+ RaceWindow: oldvol.RaceWindow,
+ UnsafeDelete: oldvol.UnsafeDelete,
+ }
+ case "Azure":
+ keydata, err := ioutil.ReadFile(oldvol.StorageAccountKeyFile)
+ if err != nil && oldvol.StorageAccountKeyFile != "" {
+ return fmt.Errorf("error reading StorageAccountKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.AzureReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.AzureVolumeDriverParameters{
+ StorageAccountName: oldvol.StorageAccountName,
+ StorageAccountKey: string(bytes.TrimSpace(keydata)),
+ StorageBaseURL: oldvol.StorageBaseURL,
+ ContainerName: oldvol.ContainerName,
+ RequestTimeout: oldvol.RequestTimeout,
+ ListBlobsRetryDelay: oldvol.ListBlobsRetryDelay,
+ ListBlobsMaxAttempts: oldvol.ListBlobsMaxAttempts,
+ }
+ case "Directory":
+ newvol = arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.DirectoryReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.DirectoryVolumeDriverParameters{
+ Root: oldvol.Root,
+ Serialize: oldvol.Serialize,
+ }
+ default:
+ return fmt.Errorf("unsupported volume type %q", oldvol.Type)
+ }
+ dp, err := json.Marshal(params)
+ if err != nil {
+ return err
+ }
+ newvol.DriverParameters = json.RawMessage(dp)
+ if newvol.Replication < 1 {
+ newvol.Replication = 1
+ }
+ }
+ if accessViaHosts == nil {
+ accessViaHosts = make(map[arvados.URL]arvados.VolumeAccess, 1)
+ }
+ accessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ newvol.AccessViaHosts = accessViaHosts
+
+ volUUID := oldUUID
+ if oldvol.ReadOnly {
+ } else if oc.Listen == nil {
+ ldr.Logger.Warn("cannot find optimal volume UUID because Listen address is not given in legacy keepstore config")
+ } else if uuid, _, err := findKeepServicesItem(cluster, *oc.Listen); err != nil {
+ ldr.Logger.WithError(err).Warn("cannot find optimal volume UUID: failed to find a matching keep_service listing for this legacy keepstore config")
+ } else if len(uuid) != 27 {
+ ldr.Logger.WithField("UUID", uuid).Warn("cannot find optimal volume UUID: keep_service UUID does not have expected format")
+ } else {
+ rendezvousUUID := cluster.ClusterID + "-nyw5e-" + uuid[12:]
+ if _, ok := cluster.Volumes[rendezvousUUID]; ok {
+ ldr.Logger.Warn("suggesting a random volume UUID because the volume ID matching our keep_service UUID is already in use")
+ } else {
+ volUUID = rendezvousUUID
+ }
+ }
+ if volUUID == "" {
+ volUUID = newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithField("UUID", volUUID).Infof("suggesting a random volume UUID for volume #%d in legacy config", i)
+ }
+ cluster.Volumes[volUUID] = newvol
+ }
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+func (ldr *Loader) alreadyMigrated(oldvol oldKeepstoreVolume, newvols map[string]arvados.Volume, myURL arvados.URL) (string, bool) {
+ for uuid, newvol := range newvols {
+ if oldvol.Type != newvol.Driver {
+ continue
+ }
+ switch oldvol.Type {
+ case "S3":
+ var params arvados.S3VolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Endpoint == params.Endpoint &&
+ oldvol.Region == params.Region &&
+ oldvol.Bucket == params.Bucket &&
+ oldvol.LocationConstraint == params.LocationConstraint {
+ return uuid, true
+ }
+ case "Azure":
+ var params arvados.AzureVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.StorageAccountName == params.StorageAccountName &&
+ oldvol.StorageBaseURL == params.StorageBaseURL &&
+ oldvol.ContainerName == params.ContainerName {
+ return uuid, true
+ }
+ case "Directory":
+ var params arvados.DirectoryVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Root == params.Root {
+ if _, ok := newvol.AccessViaHosts[myURL]; ok {
+ return uuid, true
+ }
+ }
+ }
+ }
+ return "", false
+}
+
+func (ldr *Loader) discoverLocalVolumes(cluster *arvados.Cluster, mountsFile string, myURL arvados.URL) error {
+ if mountsFile == "" {
+ mountsFile = "/proc/mounts"
+ }
+ f, err := os.Open(mountsFile)
+ if err != nil {
+ return fmt.Errorf("error opening %s: %s", mountsFile, err)
+ }
+ defer f.Close()
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ args := strings.Fields(scanner.Text())
+ dev, mount := args[0], args[1]
+ if mount == "/" {
+ continue
+ }
+ if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
+ continue
+ }
+ keepdir := mount + "/keep"
+ if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
+ continue
+ }
+
+ ro := false
+ for _, fsopt := range strings.Split(args[3], ",") {
+ if fsopt == "ro" {
+ ro = true
+ }
+ }
+
+ uuid := newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithFields(logrus.Fields{
+ "UUID": uuid,
+ "Driver": "Directory",
+ "DriverParameters.Root": keepdir,
+ "DriverParameters.Serialize": false,
+ "ReadOnly": ro,
+ "Replication": 1,
+ }).Warn("adding local directory volume")
+
+ p, err := json.Marshal(arvados.DirectoryVolumeDriverParameters{
+ Root: keepdir,
+ Serialize: false,
+ })
+ if err != nil {
+ panic(err)
+ }
+ cluster.Volumes[uuid] = arvados.Volume{
+ Driver: "Directory",
+ DriverParameters: p,
+ ReadOnly: ro,
+ Replication: 1,
+ AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
+ myURL: {ReadOnly: ro},
+ },
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("reading %s: %s", mountsFile, err)
+ }
+ return nil
+}
+
+func array2boolmap(keys []string) map[string]bool {
+ m := map[string]bool{}
+ for _, k := range keys {
+ m[k] = true
+ }
+ return m
+}
+
+func newUUID(clusterID, infix string) string {
+ randint, err := rand.Int(rand.Reader, big.NewInt(0).Exp(big.NewInt(36), big.NewInt(15), big.NewInt(0)))
+ if err != nil {
+ panic(err)
+ }
+ randstr := randint.Text(36)
+ for len(randstr) < 15 {
+ randstr = "0" + randstr
+ }
+ return fmt.Sprintf("%s-%s-%s", clusterID, infix, randstr)
+}
+
+// Return the UUID and URL for the controller's keep_services listing
+// corresponding to this host/process.
+func findKeepServicesItem(cluster *arvados.Cluster, listen string) (uuid string, url arvados.URL, err error) {
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return
+ }
+ client.AuthToken = cluster.SystemRootToken
+ var svcList arvados.KeepServiceList
+ err = client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ if err != nil {
+ return
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("error getting hostname: %s", err)
+ return
+ }
+ var tried []string
+ for _, ks := range svcList.Items {
+ if ks.ServiceType == "proxy" {
+ continue
+ } else if keepServiceIsMe(ks, hostname, listen) {
+ url := arvados.URL{
+ Scheme: "http",
+ Host: net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+ }
+ if ks.ServiceSSLFlag {
+ url.Scheme = "https"
+ }
+ return ks.UUID, url, nil
+ } else {
+ tried = append(tried, fmt.Sprintf("%s:%d", ks.ServiceHost, ks.ServicePort))
+ }
+ }
+ err = fmt.Errorf("listen address %q does not match any of the non-proxy keep_services entries %q", listen, tried)
+ return
+}
+
+var localhostOrAllInterfaces = map[string]bool{
+ "localhost": true,
+ "127.0.0.1": true,
+ "::1": true,
+ "::": true,
+ "": true,
+}
+
+// Return true if the given KeepService entry matches the given
+// hostname and (keepstore config file) listen address.
+//
+// If the KeepService host is some variant of "localhost", we assume
+// this is a testing or single-node environment, ignore the given
+// hostname, and return true if the port numbers match.
+//
+// The hostname isn't assumed to be a FQDN: a hostname "foo.bar" will
+// match a KeepService host "foo.bar", but also "foo.bar.example",
+// "foo.bar.example.org", etc.
+func keepServiceIsMe(ks arvados.KeepService, hostname string, listen string) bool {
+ // Extract the port name/number from listen, and resolve it to
+ // a port number to compare with ks.ServicePort.
+ _, listenport, err := net.SplitHostPort(listen)
+ if err != nil && strings.HasPrefix(listen, ":") {
+ listenport = listen[1:]
+ }
+ if lp, err := net.LookupPort("tcp", listenport); err != nil {
+ return false
+ } else if !(lp == ks.ServicePort ||
+ (lp == 0 && ks.ServicePort == 80)) {
+ return false
+ }
+
+ kshost := strings.ToLower(ks.ServiceHost)
+ return localhostOrAllInterfaces[kshost] || strings.HasPrefix(kshost+".", strings.ToLower(hostname)+".")
+}
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
new file mode 100644
index 000000000..be4c643f4
--- /dev/null
+++ b/lib/config/deprecated_keepstore_test.go
@@ -0,0 +1,685 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "text/template"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+type KeepstoreMigrationSuite struct {
+ hostname string // blank = use test system's hostname
+}
+
+var _ = check.Suite(&KeepstoreMigrationSuite{})
+
+func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
+ keepstorefile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstorefile.Name())
+ _, err = io.WriteString(keepstorefile, keepstoreconfig)
+ c.Assert(err, check.IsNil)
+ err = keepstorefile.Close()
+ c.Assert(err, check.IsNil)
+
+ gotldr := testLoader(c, clusterconfig, nil)
+ gotldr.KeepstorePath = keepstorefile.Name()
+ expectedldr := testLoader(c, expectedconfig, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
+ keyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keyfile.Name())
+ io.WriteString(keyfile, "blobsigningkey\n")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ s.checkEquivalentWithKeepstoreConfig(c, `
+Listen: ":12345"
+Debug: true
+LogFormat: text
+MaxBuffers: 1234
+MaxRequests: 2345
+BlobSignatureTTL: 123m
+BlobSigningKeyFile: `+keyfile.Name()+`
+`, `
+Clusters:
+ z1111:
+ {}
+`, `
+Clusters:
+ z1111:
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://`+hostname+`:12345": {}
+ SystemLogs:
+ Format: text
+ LogLevel: debug
+ API:
+ MaxKeepBlockBuffers: 1234
+ MaxConcurrentRequests: 2345
+ Collections:
+ BlobSigningTTL: 123m
+ BlobSigningKey: blobsigningkey
+`)
+}
+
+func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
+ tmpd, err := ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tmpd)
+ err = os.Mkdir(tmpd+"/keep", 0777)
+ c.Assert(err, check.IsNil)
+
+ tmpf, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmpf.Name())
+
+ // read/write
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: false,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // read-only
+ tmpf.Seek(0, os.SEEK_SET)
+ tmpf.Truncate(0)
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
+ accesskeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(accesskeyfile.Name())
+ io.WriteString(accesskeyfile, "accesskeydata\n")
+
+ secretkeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(secretkeyfile.Name())
+ io.WriteString(secretkeyfile, "secretkeydata\n")
+
+ // s3, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 1,
+ }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
+
+ // s3, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+ AccessKeyFile: `+accesskeyfile.Name()+`
+ SecretKeyFile: `+secretkeyfile.Name()+`
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: testbucket
+ LocationConstraint: true
+ IndexPageSize: 1234
+ S3Replication: 4
+ ConnectTimeout: 3m
+ ReadTimeout: 4m
+ RaceWindow: 5m
+ UnsafeDelete: true
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 4,
+ }, &arvados.S3VolumeDriverParameters{
+ AccessKey: "accesskeydata",
+ SecretKey: "secretkeydata",
+ Endpoint: "https://storage.googleapis.com",
+ Region: "us-east-1z",
+ Bucket: "testbucket",
+ LocationConstraint: true,
+ IndexPageSize: 1234,
+ ConnectTimeout: arvados.Duration(time.Minute * 3),
+ ReadTimeout: arvados.Duration(time.Minute * 4),
+ RaceWindow: arvados.Duration(time.Minute * 5),
+ UnsafeDelete: true,
+ }, &arvados.S3VolumeDriverParameters{})
+
+ // azure, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+`, arvados.Volume{
+ Driver: "Azure",
+ Replication: 1,
+ }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
+
+ // azure, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+ ReadOnly: true
+ StorageAccountName: storageacctname
+ StorageAccountKeyFile: `+secretkeyfile.Name()+`
+ StorageBaseURL: https://example.example
+ ContainerName: testctr
+ LocationConstraint: true
+ AzureReplication: 4
+ RequestTimeout: 3m
+ ListBlobsRetryDelay: 4m
+ ListBlobsMaxAttempts: 5
+`, arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.AzureVolumeDriverParameters{
+ StorageAccountName: "storageacctname",
+ StorageAccountKey: "secretkeydata",
+ StorageBaseURL: "https://example.example",
+ ContainerName: "testctr",
+ RequestTimeout: arvados.Duration(time.Minute * 3),
+ ListBlobsRetryDelay: arvados.Duration(time.Minute * 4),
+ ListBlobsMaxAttempts: 5,
+ }, &arvados.AzureVolumeDriverParameters{})
+
+ // directory, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ Root: /tmp/xyzzy
+`, arvados.Volume{
+ Driver: "Directory",
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // directory, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ ReadOnly: true
+ Root: /tmp/xyzzy
+ DirectoryReplication: 4
+ Serialize: true
+`, arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ Serialize: true,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
+ hostname := s.hostname
+ if hostname == "" {
+ h, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ hostname = h
+ }
+
+ oldconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(oldconfig.Name())
+ io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
+ if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
+ // Prevent tests from looking at the real /proc/mounts on the test host.
+ io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
+ }
+
+ ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
+ ldr.KeepstorePath = oldconfig.Name()
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc := cfg.Clusters["z1111"]
+ c.Check(cc.Volumes, check.HasLen, 1)
+ for uuid, v := range cc.Volumes {
+ c.Check(uuid, check.HasLen, 27)
+ c.Check(v.Driver, check.Equals, expectvol.Driver)
+ c.Check(v.Replication, check.Equals, expectvol.Replication)
+
+ avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+ c.Check(ok, check.Equals, true)
+ c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
+
+ err := json.Unmarshal(v.DriverParameters, paramsdst)
+ c.Check(err, check.IsNil)
+ c.Check(paramsdst, check.DeepEquals, expectparams)
+ }
+}
+
+// How we handle a volume from a legacy keepstore config file depends
+// on whether it's writable, whether a volume using the same cloud
+// backend already exists in the cluster config, and (if so) whether
+// it already has an AccessViaHosts entry for this host.
+//
+// In all cases, we should end up with an AccessViaHosts entry for
+// this host, to indicate that the current host's volumes have been
+// migrated.
+
+// Same backend already referenced in cluster config, this host
+// already listed in AccessViaHosts --> no change, except possibly
+// updating the ReadOnly flag on the AccessViaHosts entry.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ S3Replication: 3
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Writable volume, same cloud backend already referenced in cluster
+// config --> change UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+`)
+ c.Check(after, check.HasLen, len(before))
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "readonlyonother")
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same cloud backend not yet referenced --> add a
+// new volume, with UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
+ c.Check(newvol.Replication, check.Equals, 3)
+
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, but this host isn't in AccessViaHosts --> add a new
+// volume, with UUID to match this keepstore's UUID (filesystem-backed
+// volumes are assumed to be different on different hosts, even if
+// paths are the same).
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sdd
+ DirectoryReplication: 2
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.DirectoryVolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Root, check.Equals, "/data/sdd")
+ c.Check(newvol.Replication, check.Equals, 2)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, and this host is already listed in AccessViaHosts
+// --> already migrated, don't change anything.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sde
+ DirectoryReplication: 2
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Multiple writable cloud-backed volumes --> one of them will get a
+// UUID matching this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: first-bucket-to-migrate
+ S3Replication: 3
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: second-bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 2)
+ // Sort by bucket name (so "first" comes before "second")
+ params := map[string]arvados.S3VolumeDriverParameters{}
+ for _, uuid := range newuuids {
+ var p arvados.S3VolumeDriverParameters
+ json.Unmarshal(after[uuid].DriverParameters, &p)
+ params[uuid] = p
+ }
+ sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
+ newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
+ params0, params1 := params[newuuids[0]], params[newuuids[1]]
+
+ c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
+ c.Check(newvol0.Replication, check.Equals, 3)
+
+ c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
+ c.Check(newvol1.Replication, check.Equals, 3)
+
+ // Don't care which one gets the special UUID
+ if newuuids[0] != expectUUID {
+ c.Check(newuuids[1], check.Equals, expectUUID)
+ }
+}
+
+// Non-writable volume, same cloud backend already referenced in
+// cluster config --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+ ReadOnly: true
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Writable volume, same cloud backend already writable by another
+// keepstore server --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ S3Replication: 3
+ ReadOnly: false
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Non-writable volume, same cloud backend not already referenced in
+// cluster config --> assign a new random volume UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: differentbucket
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "differentbucket")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+ c.Check(ok, check.Equals, true)
+}
+
+const clusterConfigForKeepstoreMigrationTest = `
+Clusters:
+ zzzzz:
+ SystemRootToken: ` + arvadostest.AdminToken + `
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://{{.hostname}}:12345": {}
+ Controller:
+ ExternalURL: "https://{{.controller}}"
+ TLS:
+ Insecure: true
+ Volumes:
+
+ zzzzz-nyw5e-alreadymigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ Replication: 3
+
+ zzzzz-nyw5e-readonlyonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {ReadOnly: true}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ Replication: 3
+
+ zzzzz-nyw5e-writableonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ Replication: 3
+
+ zzzzz-nyw5e-localfilesystem:
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sdd
+ Replication: 1
+
+ zzzzz-nyw5e-localismigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sde
+ Replication: 1
+`
+
+// Determine the effect of combining the given legacy keepstore config
+// YAML (just the "Volumes" entries of an old keepstore config file)
+// with the example clusterConfigForKeepstoreMigrationTest config.
+//
+// Return two Volumes configs -- one without loading
+// keepstoreconfigdata ("before") and one with ("after") -- for the
+// caller to compare.
+func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreVolumesYAML string) (before, after map[string]arvados.Volume) {
+ ldr := testLoader(c, s.clusterConfigYAML(c), nil)
+ cBefore, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ keepstoreconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstoreconfig.Name())
+ io.WriteString(keepstoreconfig, keepstoreVolumesYAML)
+
+ ldr = testLoader(c, s.clusterConfigYAML(c), nil)
+ ldr.KeepstorePath = keepstoreconfig.Name()
+ cAfter, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
+}
+
+func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
+
+ var clusterconfigdata bytes.Buffer
+ err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
+ "hostname": hostname,
+ "controller": os.Getenv("ARVADOS_API_HOST"),
+ })
+ c.Assert(err, check.IsNil)
+
+ return clusterconfigdata.String()
+}
+
+// Return the uuids of volumes that appear in "after" but not
+// "before".
+//
+// Assert the returned slice has at least minAdded entries.
+func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
+ for uuid := range after {
+ if _, ok := before[uuid]; !ok {
+ uuids = append(uuids, uuid)
+ }
+ }
+ if len(uuids) < minAdded {
+ c.Assert(uuids, check.HasLen, minAdded)
+ }
+ return
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
+ for uuid, port := range s.getTestKeepstorePorts(c) {
+ c.Assert(uuid, check.HasLen, 27)
+ return port, "zzzzz-nyw5e-" + uuid[12:]
+ }
+ c.Fatal("getTestKeepstorePorts() returned nothing")
+ return 0, ""
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePorts(c *check.C) map[string]int {
+ client := arvados.NewClientFromEnv()
+ var svcList arvados.KeepServiceList
+ err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ c.Assert(err, check.IsNil)
+ ports := map[string]int{}
+ for _, ks := range svcList.Items {
+ if ks.ServiceType == "disk" {
+ ports[ks.UUID] = ks.ServicePort
+ }
+ }
+ return ports
+}
+
+func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
+ for i, trial := range []struct {
+ match bool
+ hostname string
+ listen string
+ serviceHost string
+ servicePort int
+ }{
+ {true, "keep0", "keep0", "keep0", 80},
+ {true, "keep0", "[::1]:http", "keep0", 80},
+ {true, "keep0", "[::]:http", "keep0", 80},
+ {true, "keep0", "keep0:25107", "keep0", 25107},
+ {true, "keep0", ":25107", "keep0", 25107},
+ {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "Keep0.domain.example", 25107},
+ {true, "keep0", ":http", "keep0.domain.example", 80},
+ {true, "keep0", ":25107", "localhost", 25107},
+ {true, "keep0", ":25107", "::1", 25107},
+ {false, "keep0", ":25107", "keep0", 1111}, // different port
+ {false, "keep0", ":25107", "localhost", 1111}, // different port
+ {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
+ {false, "keep0", ":bogussss", "keep0", 25107}, // unresolvable port
+ {false, "keep0", ":25107", "keep1", 25107}, // different hostname
+ {false, "keep1", ":25107", "keep10", 25107}, // different hostname (prefix, but not on a "." boundary)
+ } {
+ c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
+ }
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 5dda0ba94..ea9b50d03 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -47,7 +47,7 @@ func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvado
func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
- s.checkEquivalent(c, `
+ checkEquivalent(c, `
Clusters:
z1111:
NodeProfiles:
diff --git a/lib/config/export.go b/lib/config/export.go
index 6eb4fbe5f..57f62fa83 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -63,8 +63,10 @@ var whitelist = map[string]bool{
"API": true,
"API.AsyncPermissionsUpdateInterval": false,
"API.DisabledAPIs": false,
+ "API.MaxConcurrentRequests": false,
"API.MaxIndexDatabaseRead": false,
"API.MaxItemsPerResponse": true,
+ "API.MaxKeepBlockBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
"API.RailsSessionSecretToken": false,
@@ -81,6 +83,12 @@ var whitelist = map[string]bool{
"Collections.BlobSigning": true,
"Collections.BlobSigningKey": false,
"Collections.BlobSigningTTL": true,
+ "Collections.BlobTrash": false,
+ "Collections.BlobTrashLifetime": false,
+ "Collections.BlobTrashConcurrency": false,
+ "Collections.BlobTrashCheckInterval": false,
+ "Collections.BlobDeleteConcurrency": false,
+ "Collections.BlobReplicateConcurrency": false,
"Collections.CollectionVersioning": false,
"Collections.DefaultReplication": true,
"Collections.DefaultTrashLifetime": true,
@@ -150,6 +158,16 @@ var whitelist = map[string]bool{
"Users.NewUsersAreActive": false,
"Users.UserNotifierEmailFrom": false,
"Users.UserProfileNotificationAddress": false,
+ "Volumes": true,
+ "Volumes.*": true,
+ "Volumes.*.*": false,
+ "Volumes.*.AccessViaHosts": true,
+ "Volumes.*.AccessViaHosts.*": true,
+ "Volumes.*.AccessViaHosts.*.ReadOnly": true,
+ "Volumes.*.ReadOnly": true,
+ "Volumes.*.Replication": true,
+ "Volumes.*.StorageClasses": true,
+ "Volumes.*.StorageClasses.*": false,
"Workbench": true,
"Workbench.ActivationContactLink": false,
"Workbench.APIClientConnectTimeout": true,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8a5b4610c..c6be59ecf 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -182,6 +182,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -322,15 +331,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
# Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -747,6 +785,48 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+ Serialize: false
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index c7289350e..17e0af7ba 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -321,7 +321,7 @@ Clusters:
}
func (s *LoadSuite) TestMovedKeys(c *check.C) {
- s.checkEquivalent(c, `# config has old keys only
+ checkEquivalent(c, `# config has old keys only
Clusters:
zzzzz:
RequestLimits:
@@ -334,7 +334,7 @@ Clusters:
MaxRequestAmplification: 3
MaxItemsPerResponse: 999
`)
- s.checkEquivalent(c, `# config has both old and new keys; old values win
+ checkEquivalent(c, `# config has both old and new keys; old values win
Clusters:
zzzzz:
RequestLimits:
@@ -352,30 +352,45 @@ Clusters:
`)
}
-func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
- got, err := testLoader(c, goty, nil).Load()
+func checkEquivalent(c *check.C, goty, expectedy string) {
+ gotldr := testLoader(c, goty, nil)
+ expectedldr := testLoader(c, expectedy, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func checkEqualYAML(c *check.C, got, expected interface{}) {
+ expectedyaml, err := yaml.Marshal(expected)
c.Assert(err, check.IsNil)
- expected, err := testLoader(c, expectedy, nil).Load()
+ gotyaml, err := yaml.Marshal(got)
c.Assert(err, check.IsNil)
- if !c.Check(got, check.DeepEquals, expected) {
+ if !bytes.Equal(gotyaml, expectedyaml) {
cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
- for _, obj := range []interface{}{expected, got} {
- y, _ := yaml.Marshal(obj)
+ for _, y := range [][]byte{expectedyaml, gotyaml} {
pr, pw, err := os.Pipe()
c.Assert(err, check.IsNil)
defer pr.Close()
- go func() {
- io.Copy(pw, bytes.NewBuffer(y))
+ go func(data []byte) {
+ pw.Write(data)
pw.Close()
- }()
+ }(y)
cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
}
diff, err := cmd.CombinedOutput()
+ // diff should report differences and exit non-zero.
+ c.Check(err, check.NotNil)
c.Log(string(diff))
- c.Check(err, check.IsNil)
+ c.Error("got != expected; see diff (-expected +got) above")
}
}
+func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
+ got, err := gotldr.Load()
+ c.Assert(err, check.IsNil)
+ expected, err := expectedldr.Load()
+ c.Assert(err, check.IsNil)
+ checkEqualYAML(c, got, expected)
+}
+
func checkListKeys(path string, x interface{}) (err error) {
v := reflect.Indirect(reflect.ValueOf(x))
switch v.Kind() {
diff --git a/lib/controller/cmd.go b/lib/controller/cmd.go
index 434537046..b9af60fe9 100644
--- a/lib/controller/cmd.go
+++ b/lib/controller/cmd.go
@@ -10,10 +10,11 @@ import (
"git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/prometheus/client_golang/prometheus"
)
var Command cmd.Handler = service.Command(arvados.ServiceNameController, newHandler)
-func newHandler(_ context.Context, cluster *arvados.Cluster, _ string) service.Handler {
+func newHandler(_ context.Context, cluster *arvados.Cluster, _ string, _ *prometheus.Registry) service.Handler {
return &Handler{Cluster: cluster}
}
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 4b49e4615..d34df7f2c 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -19,6 +19,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
@@ -52,7 +53,7 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
- s.handler = newHandler(s.ctx, s.cluster, "")
+ s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry())
}
func (s *HandlerSuite) TearDownTest(c *check.C) {
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index ae6ac70e9..7ab38c6ca 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -11,11 +11,12 @@ import (
"git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/prometheus/client_golang/prometheus"
)
var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
ac, err := arvados.NewClientFromConfig(cluster)
if err != nil {
return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
@@ -25,6 +26,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) ser
Context: ctx,
ArvClient: ac,
AuthToken: token,
+ Registry: reg,
}
go d.Start()
return d
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 731c6d25d..f0aa83c2e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -48,10 +48,10 @@ type dispatcher struct {
Context context.Context
ArvClient *arvados.Client
AuthToken string
+ Registry *prometheus.Registry
InstanceSetID cloud.InstanceSetID
logger logrus.FieldLogger
- reg *prometheus.Registry
instanceSet cloud.InstanceSet
pool pool
queue scheduler.ContainerQueue
@@ -132,14 +132,13 @@ func (disp *dispatcher) initialize() {
disp.sshKey = key
}
- disp.reg = prometheus.NewRegistry()
- instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.reg)
+ instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.Registry)
if err != nil {
disp.logger.Fatalf("error initializing driver: %s", err)
}
disp.instanceSet = instanceSet
- disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.reg, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
- disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, disp.ArvClient)
+ disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.Registry, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
+ disp.queue = container.NewQueue(disp.logger, disp.Registry, disp.typeChooser, disp.ArvClient)
if disp.Cluster.ManagementToken == "" {
disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -154,7 +153,7 @@ func (disp *dispatcher) initialize() {
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/drain", disp.apiInstanceDrain)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/run", disp.apiInstanceRun)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/kill", disp.apiInstanceKill)
- metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+ metricsH := promhttp.HandlerFor(disp.Registry, promhttp.HandlerOpts{
ErrorLog: disp.logger,
})
mux.Handler("GET", "/metrics", metricsH)
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 4a5ca3017..1972468f4 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -19,6 +19,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/prometheus/client_golang/prometheus"
"golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
)
@@ -91,6 +92,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
Context: s.ctx,
ArvClient: arvClient,
AuthToken: arvadostest.AdminToken,
+ Registry: prometheus.NewRegistry(),
}
// Test cases can modify s.cluster before calling
// initialize(), and then modify private state before calling
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index b6737bc55..c410e5368 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -22,6 +22,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-systemd/daemon"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -30,7 +31,7 @@ type Handler interface {
CheckHealth() error
}
-type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler
+type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string, registry *prometheus.Registry) Handler
type command struct {
newHandler NewHandlerFunc
@@ -68,7 +69,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -87,22 +87,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
if err != nil {
return 1
}
- log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+
+ // Now that we've read the config, replace the bootstrap
+ // logger with a new one according to the logging config.
+ log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
+ logger := log.WithFields(logrus.Fields{
"PID": os.Getpid(),
})
- ctx := ctxlog.Context(c.ctx, log)
+ ctx := ctxlog.Context(c.ctx, logger)
- listen, err := getListenAddr(cluster.Services, c.svcName)
+ listenURL, err := getListenAddr(cluster.Services, c.svcName)
if err != nil {
return 1
}
+ ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
if cluster.SystemRootToken == "" {
- log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
cluster.SystemRootToken = os.Getenv("ARVADOS_API_TOKEN")
}
if cluster.Services.Controller.ExternalURL.Host == "" {
- log.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
u, err := url.Parse("https://" + os.Getenv("ARVADOS_API_HOST"))
if err != nil {
err = fmt.Errorf("ARVADOS_API_HOST: %s", err)
@@ -114,27 +119,42 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
}
}
- handler := c.newHandler(ctx, cluster, cluster.SystemRootToken)
+ reg := prometheus.NewRegistry()
+ handler := c.newHandler(ctx, cluster, cluster.SystemRootToken, reg)
if err = handler.CheckHealth(); err != nil {
return 1
}
+
+ instrumented := httpserver.Instrument(reg, log,
+ httpserver.HandlerWithContext(ctx,
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler, reg)))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.HandlerWithContext(ctx,
- httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
+ Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
},
- Addr: listen,
+ Addr: listenURL.Host,
+ }
+ if listenURL.Scheme == "https" {
+ tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger)
+ if err != nil {
+ logger.WithError(err).Errorf("cannot start %s service on %s", c.svcName, listenURL.String())
+ return 1
+ }
+ srv.TLSConfig = tlsconfig
}
err = srv.Start()
if err != nil {
return 1
}
- log.WithFields(logrus.Fields{
+ logger.WithFields(logrus.Fields{
+ "URL": listenURL,
"Listen": srv.Addr,
"Service": c.svcName,
}).Info("listening")
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.WithError(err).Errorf("error notifying init daemon")
+ logger.WithError(err).Errorf("error notifying init daemon")
}
go func() {
<-ctx.Done()
@@ -149,20 +169,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) {
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
- return "", fmt.Errorf("unknown service name %q", prog)
+ return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
}
for url := range svc.InternalURLs {
if strings.HasPrefix(url.Host, "localhost:") {
- return url.Host, nil
+ return url, nil
}
listener, err := net.Listen("tcp", url.Host)
if err == nil {
listener.Close()
- return url.Host, nil
+ return url, nil
}
}
- return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
+ return arvados.URL{}, fmt.Errorf("configuration does not enable the %s service on this host", prog)
+}
+
+type contextKeyURL struct{}
+
+func URLFromContext(ctx context.Context) (arvados.URL, bool) {
+ u, ok := ctx.Value(contextKeyURL{}).(arvados.URL)
+ return u, ok
}
diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index bb7c5c51d..ef047bc9d 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -8,14 +8,17 @@ package service
import (
"bytes"
"context"
+ "crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"os"
"testing"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
@@ -38,7 +41,7 @@ func (*Suite) TestCommand(c *check.C) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string) Handler {
+ cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
c.Check(ctx.Value("foo"), check.Equals, "bar")
c.Check(token, check.Equals, "abcde")
return &testHandler{ctx: ctx, healthCheck: healthCheck}
@@ -62,12 +65,77 @@ func (*Suite) TestCommand(c *check.C) {
c.Check(stderr.String(), check.Matches, `(?ms).*"msg":"CheckHealth called".*`)
}
+func (*Suite) TestTLS(c *check.C) {
+ cwd, err := os.Getwd()
+ c.Assert(err, check.IsNil)
+
+ stdin := bytes.NewBufferString(`
+Clusters:
+ zzzzz:
+ SystemRootToken: abcde
+ Services:
+ Controller:
+ ExternalURL: "https://localhost:12345"
+ InternalURLs: {"https://localhost:12345": {}}
+ TLS:
+ Key: file://` + cwd + `/../../services/api/tmp/self-signed.key
+ Certificate: file://` + cwd + `/../../services/api/tmp/self-signed.pem
+`)
+
+ called := make(chan bool)
+ cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
+ return &testHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte("ok"))
+ close(called)
+ })}
+ })
+
+ exited := make(chan bool)
+ var stdout, stderr bytes.Buffer
+ go func() {
+ cmd.RunCommand("arvados-controller", []string{"-config", "-"}, stdin, &stdout, &stderr)
+ close(exited)
+ }()
+ got := make(chan bool)
+ go func() {
+ defer close(got)
+ client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+ for range time.NewTicker(time.Millisecond).C {
+ resp, err := client.Get("https://localhost:12345")
+ if err != nil {
+ c.Log(err)
+ continue
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Logf("status %d, body %s", resp.StatusCode, string(body))
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ break
+ }
+ }()
+ select {
+ case <-called:
+ case <-exited:
+ c.Error("command exited without calling handler")
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
+ select {
+ case <-got:
+ case <-exited:
+ c.Error("command exited before client received response")
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
+ c.Log(stderr.String())
+}
+
type testHandler struct {
ctx context.Context
+ handler http.Handler
healthCheck chan bool
}
-func (th *testHandler) ServeHTTP(http.ResponseWriter, *http.Request) {}
+func (th *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { th.handler.ServeHTTP(w, r) }
func (th *testHandler) CheckHealth() error {
ctxlog.FromContext(th.ctx).Info("CheckHealth called")
select {
diff --git a/lib/service/tls.go b/lib/service/tls.go
new file mode 100644
index 000000000..5f14bc5e8
--- /dev/null
+++ b/lib/service/tls.go
@@ -0,0 +1,81 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package service
+
+import (
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+ currentCert := make(chan *tls.Certificate, 1)
+ loaded := false
+
+ key, cert := cluster.TLS.Key, cluster.TLS.Certificate
+ if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
+ return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+ }
+ key, cert = key[7:], cert[7:]
+
+ update := func() error {
+ cert, err := tls.LoadX509KeyPair(cert, key)
+ if err != nil {
+ return fmt.Errorf("error loading X509 key pair: %s", err)
+ }
+ if loaded {
+ // Throw away old cert
+ <-currentCert
+ }
+ currentCert <- &cert
+ loaded = true
+ return nil
+ }
+ err := update()
+ if err != nil {
+ return nil, err
+ }
+
+ go func() {
+ reload := make(chan os.Signal, 1)
+ signal.Notify(reload, syscall.SIGHUP)
+ for range reload {
+ err := update()
+ if err != nil {
+ logger.WithError(err).Warn("error updating TLS certificate")
+ }
+ }
+ }()
+
+ // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
+ return &tls.Config{
+ PreferServerCipherSuites: true,
+ CurvePreferences: []tls.CurveID{
+ tls.CurveP256,
+ tls.X25519,
+ },
+ MinVersion: tls.VersionTLS12,
+ CipherSuites: []uint16{
+ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ },
+ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ cert := <-currentCert
+ currentCert <- cert
+ return cert, nil
+ },
+ }, nil
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 5a18972f5..c51f609f8 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -81,6 +81,8 @@ type Cluster struct {
DisabledAPIs StringSet
MaxIndexDatabaseRead int
MaxItemsPerResponse int
+ MaxConcurrentRequests int
+ MaxKeepBlockBuffers int
MaxRequestAmplification int
MaxRequestSize int
RailsSessionSecretToken string
@@ -96,13 +98,19 @@ type Cluster struct {
UnloggedAttributes StringSet
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]struct {
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ BlobTrash bool
+ BlobTrashLifetime Duration
+ BlobTrashCheckInterval Duration
+ BlobTrashConcurrency int
+ BlobDeleteConcurrency int
+ BlobReplicateConcurrency int
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
Value interface{}
Function string
Protected bool
@@ -157,6 +165,7 @@ type Cluster struct {
UserNotifierEmailFrom string
UserProfileNotificationAddress string
}
+ Volumes map[string]Volume
Workbench struct {
ActivationContactLink string
APIClientConnectTimeout Duration
@@ -196,6 +205,48 @@ type Cluster struct {
EnableBetaController14287 bool
}
+type Volume struct {
+ AccessViaHosts map[URL]VolumeAccess
+ ReadOnly bool
+ Replication int
+ StorageClasses map[string]bool
+ Driver string
+ DriverParameters json.RawMessage
+}
+
+type S3VolumeDriverParameters struct {
+ AccessKey string
+ SecretKey string
+ Endpoint string
+ Region string
+ Bucket string
+ LocationConstraint bool
+ IndexPageSize int
+ ConnectTimeout Duration
+ ReadTimeout Duration
+ RaceWindow Duration
+ UnsafeDelete bool
+}
+
+type AzureVolumeDriverParameters struct {
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string
+ ContainerName string
+ RequestTimeout Duration
+ ListBlobsRetryDelay Duration
+ ListBlobsMaxAttempts int
+}
+
+type DirectoryVolumeDriverParameters struct {
+ Root string
+ Serialize bool
+}
+
+type VolumeAccess struct {
+ ReadOnly bool
+}
+
type Services struct {
Composer Service
Controller Service
@@ -239,6 +290,10 @@ func (su URL) MarshalText() ([]byte, error) {
return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
}
+func (su URL) String() string {
+ return (*url.URL)(&su).String()
+}
+
type ServiceInstance struct{}
type PostgreSQL struct {
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 0c866354a..e0ae1758d 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -23,11 +23,11 @@ type KeepService struct {
}
type KeepMount struct {
- UUID string `json:"uuid"`
- DeviceID string `json:"device_id"`
- ReadOnly bool `json:"read_only"`
- Replication int `json:"replication"`
- StorageClasses []string `json:"storage_classes"`
+ UUID string `json:"uuid"`
+ DeviceID string `json:"device_id"`
+ ReadOnly bool `json:"read_only"`
+ Replication int `json:"replication"`
+ StorageClasses map[string]bool `json:"storage_classes"`
}
// KeepServiceList is an arvados#keepServiceList record
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index 490a7f3e0..5b01db5c4 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -111,16 +111,16 @@ func StopAPI() {
}
// StartKeep starts the given number of keep servers,
-// optionally with -enforce-permissions enabled.
-// Use numKeepServers = 2 and enforcePermissions = false under all normal circumstances.
-func StartKeep(numKeepServers int, enforcePermissions bool) {
+// optionally with --keep-blob-signing enabled.
+// Use numKeepServers = 2 and blobSigning = false under all normal circumstances.
+func StartKeep(numKeepServers int, blobSigning bool) {
cwd, _ := os.Getwd()
defer os.Chdir(cwd)
chdirToPythonTests()
cmdArgs := []string{"run_test_server.py", "start_keep", "--num-keep-servers", strconv.Itoa(numKeepServers)}
- if enforcePermissions {
- cmdArgs = append(cmdArgs, "--keep-enforce-permissions")
+ if blobSigning {
+ cmdArgs = append(cmdArgs, "--keep-blob-signing")
}
bgRun(exec.Command("python", cmdArgs...))
diff --git a/sdk/go/ctxlog/log.go b/sdk/go/ctxlog/log.go
index e66eeadee..a17ad8d83 100644
--- a/sdk/go/ctxlog/log.go
+++ b/sdk/go/ctxlog/log.go
@@ -11,7 +11,6 @@ import (
"os"
"github.com/sirupsen/logrus"
- check "gopkg.in/check.v1"
)
var (
@@ -41,7 +40,7 @@ func FromContext(ctx context.Context) logrus.FieldLogger {
// New returns a new logger with the indicated format and
// level.
-func New(out io.Writer, format, level string) logrus.FieldLogger {
+func New(out io.Writer, format, level string) *logrus.Logger {
logger := logrus.New()
logger.Out = out
setFormat(logger, format)
@@ -49,7 +48,7 @@ func New(out io.Writer, format, level string) logrus.FieldLogger {
return logger
}
-func TestLogger(c *check.C) logrus.FieldLogger {
+func TestLogger(c interface{ Log(...interface{}) }) *logrus.Logger {
logger := logrus.New()
logger.Out = &logWriter{c.Log}
setFormat(logger, "text")
diff --git a/sdk/go/httpserver/httpserver.go b/sdk/go/httpserver/httpserver.go
index a94146f85..627e04f0b 100644
--- a/sdk/go/httpserver/httpserver.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -43,7 +43,12 @@ func (srv *Server) Start() error {
srv.cond = sync.NewCond(mutex.RLocker())
srv.running = true
go func() {
- err = srv.Serve(tcpKeepAliveListener{srv.listener})
+ lnr := tcpKeepAliveListener{srv.listener}
+ if srv.TLSConfig != nil {
+ err = srv.ServeTLS(lnr, "", "")
+ } else {
+ err = srv.Serve(lnr)
+ }
if !srv.wantDown {
srv.err = err
}
diff --git a/sdk/go/httpserver/request_limiter.go b/sdk/go/httpserver/request_limiter.go
index e7192d5b4..23e6e016d 100644
--- a/sdk/go/httpserver/request_limiter.go
+++ b/sdk/go/httpserver/request_limiter.go
@@ -6,6 +6,9 @@ package httpserver
import (
"net/http"
+ "sync/atomic"
+
+ "github.com/prometheus/client_golang/prometheus"
)
// RequestCounter is an http.Handler that tracks the number of
@@ -24,19 +27,45 @@ type RequestCounter interface {
type limiterHandler struct {
requests chan struct{}
handler http.Handler
+ count int64 // only used if cap(requests)==0
}
// NewRequestLimiter returns a RequestCounter that delegates up to
// maxRequests at a time to the given handler, and responds 503 to all
// incoming requests beyond that limit.
-func NewRequestLimiter(maxRequests int, handler http.Handler) RequestCounter {
- return &limiterHandler{
+//
+// "concurrent_requests" and "max_concurrent_requests" metrics are
+// registered with the given reg, if reg is not nil.
+func NewRequestLimiter(maxRequests int, handler http.Handler, reg *prometheus.Registry) RequestCounter {
+ h := &limiterHandler{
requests: make(chan struct{}, maxRequests),
handler: handler,
}
+ if reg != nil {
+ reg.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "concurrent_requests",
+ Help: "Number of requests in progress",
+ },
+ func() float64 { return float64(h.Current()) },
+ ))
+ reg.MustRegister(prometheus.NewGaugeFunc(
+ prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Name: "max_concurrent_requests",
+ Help: "Maximum number of concurrent requests",
+ },
+ func() float64 { return float64(h.Max()) },
+ ))
+ }
+ return h
}
func (h *limiterHandler) Current() int {
+ if cap(h.requests) == 0 {
+ return int(atomic.LoadInt64(&h.count))
+ }
return len(h.requests)
}
@@ -45,6 +74,11 @@ func (h *limiterHandler) Max() int {
}
func (h *limiterHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if cap(h.requests) == 0 {
+ atomic.AddInt64(&h.count, 1)
+ h.handler.ServeHTTP(resp, req)
+ atomic.AddInt64(&h.count, -1)
+ }
select {
case h.requests <- struct{}{}:
default:
diff --git a/sdk/go/httpserver/request_limiter_test.go b/sdk/go/httpserver/request_limiter_test.go
index afa4e3faa..64d1f3d4c 100644
--- a/sdk/go/httpserver/request_limiter_test.go
+++ b/sdk/go/httpserver/request_limiter_test.go
@@ -31,7 +31,7 @@ func newTestHandler(maxReqs int) *testHandler {
func TestRequestLimiter1(t *testing.T) {
h := newTestHandler(10)
- l := NewRequestLimiter(1, h)
+ l := NewRequestLimiter(1, h, nil)
var wg sync.WaitGroup
resps := make([]*httptest.ResponseRecorder, 10)
for i := 0; i < 10; i++ {
@@ -91,7 +91,7 @@ func TestRequestLimiter1(t *testing.T) {
func TestRequestLimiter10(t *testing.T) {
h := newTestHandler(10)
- l := NewRequestLimiter(10, h)
+ l := NewRequestLimiter(10, h, nil)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 34342059f..a71064f19 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -399,9 +399,9 @@ def get_config():
with open(os.environ["ARVADOS_CONFIG"]) as f:
return yaml.safe_load(f)
-def internal_port_from_config(service):
+def internal_port_from_config(service, idx=0):
return int(urlparse(
- list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys())[0]).
+ sorted(list(get_config()["Clusters"]["zzzzz"]["Services"][service]["InternalURLs"].keys()))[idx]).
netloc.split(":")[1])
def external_port_from_config(service):
@@ -444,47 +444,42 @@ def stop_ws():
return
kill_server_pid(_pidfile('ws'))
-def _start_keep(n, keep_args):
- keep0 = tempfile.mkdtemp()
- port = find_available_port()
- keep_cmd = ["keepstore",
- "-volume={}".format(keep0),
- "-listen=:{}".format(port),
- "-pid="+_pidfile('keep{}'.format(n))]
-
- for arg, val in keep_args.items():
- keep_cmd.append("{}={}".format(arg, val))
-
+def _start_keep(n, blob_signing=False):
+ datadir = os.path.join(TEST_TMPDIR, "keep%d.data"%n)
+ if os.path.exists(datadir):
+ shutil.rmtree(datadir)
+ os.mkdir(datadir)
+ port = internal_port_from_config("Keepstore", idx=n)
+ conf = os.path.join(TEST_TMPDIR, "keep%d.yaml"%n)
+ confdata = get_config()
+ confdata['Clusters']['zzzzz']['Services']['Keepstore']['InternalURLs'] = {"http://127.0.0.1:%d"%port: {}}
+ confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing
+ with open(conf, 'w') as f:
+ yaml.safe_dump(confdata, f)
+ f.truncate()
+ legacyconf = os.path.join(TEST_TMPDIR, "keep%d-legacy.yaml"%n)
+ with open(legacyconf, 'w') as f:
+ yaml.safe_dump({"Listen": "127.0.0.1:%d"%port}, f)
+ f.truncate()
+ keep_cmd = ["keepstore", "-config", conf, "-legacy-keepstore-config", legacyconf]
+
+ print('logfilename is %s'%_logfilename('keep{}'.format(n)), file=sys.stderr)
with open(_logfilename('keep{}'.format(n)), 'a') as logf:
with open('/dev/null') as _stdin:
- kp0 = subprocess.Popen(
+ child = subprocess.Popen(
keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+ print('child.pid is %d'%child.pid, file=sys.stderr)
with open(_pidfile('keep{}'.format(n)), 'w') as f:
- f.write(str(kp0.pid))
-
- with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
- f.write(keep0)
+ f.write(str(child.pid))
_wait_until_port_listens(port)
return port
-def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
+def run_keep(num_servers=2, **kwargs):
stop_keep(num_servers)
- keep_args = {}
- if not blob_signing_key:
- blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
- with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
- keep_args['-blob-signing-key-file'] = f.name
- f.write(blob_signing_key)
- keep_args['-enforce-permissions'] = str(enforce_permissions).lower()
- with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
- keep_args['-data-manager-token-file'] = f.name
- f.write(auth_token('data_manager'))
- keep_args['-never-delete'] = 'false'
-
api = arvados.api(
version='v1',
host=os.environ['ARVADOS_API_HOST'],
@@ -497,7 +492,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
api.keep_disks().delete(uuid=d['uuid']).execute()
for d in range(0, num_servers):
- port = _start_keep(d, keep_args)
+ port = _start_keep(d, **kwargs)
svc = api.keep_services().create(body={'keep_service': {
'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
'service_host': 'localhost',
@@ -522,12 +517,6 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2):
def _stop_keep(n):
kill_server_pid(_pidfile('keep{}'.format(n)))
- if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
- with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
- shutil.rmtree(r.read(), True)
- os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
- if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
- os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
def stop_keep(num_servers=2):
for n in range(0, num_servers):
@@ -663,6 +652,10 @@ def setup_config():
git_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
keepproxy_external_port = find_available_port()
+ keepstore0_port = find_available_port()
+ keepstore1_port = find_available_port()
+ keepstore2_port = find_available_port()
+ keepstore3_port = find_available_port()
keep_web_port = find_available_port()
keep_web_external_port = find_available_port()
keep_web_dl_port = find_available_port()
@@ -678,45 +671,53 @@ def setup_config():
services = {
"RailsAPI": {
"InternalURLs": {
- "https://%s:%s"%(localhost, rails_api_port): {}
- }
+ "https://%s:%s"%(localhost, rails_api_port): {},
+ },
},
"Controller": {
"ExternalURL": "https://%s:%s" % (localhost, controller_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, controller_port): {}
- }
+ "http://%s:%s"%(localhost, controller_port): {},
+ },
},
"Websocket": {
"ExternalURL": "wss://%s:%s/websocket" % (localhost, websocket_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, websocket_port): {}
- }
+ "http://%s:%s"%(localhost, websocket_port): {},
+ },
},
"GitHTTP": {
"ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
"InternalURLs": {
"http://%s:%s"%(localhost, git_httpd_port): {}
- }
+ },
+ },
+ "Keepstore": {
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, keepstore0_port): {},
+ "http://%s:%s"%(localhost, keepstore1_port): {},
+ "http://%s:%s"%(localhost, keepstore2_port): {},
+ "http://%s:%s"%(localhost, keepstore3_port): {},
+ },
},
"Keepproxy": {
"ExternalURL": "https://%s:%s" % (localhost, keepproxy_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keepproxy_port): {}
- }
+ "http://%s:%s"%(localhost, keepproxy_port): {},
+ },
},
"WebDAV": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_port): {}
- }
+ "http://%s:%s"%(localhost, keep_web_port): {},
+ },
},
"WebDAVDownload": {
"ExternalURL": "https://%s:%s" % (localhost, keep_web_dl_external_port),
"InternalURLs": {
- "http://%s:%s"%(localhost, keep_web_dl_port): {}
- }
- }
+ "http://%s:%s"%(localhost, keep_web_dl_port): {},
+ },
+ },
}
config = {
@@ -724,30 +725,70 @@ def setup_config():
"zzzzz": {
"EnableBetaController14287": ('14287' in os.environ.get('ARVADOS_EXPERIMENTAL', '')),
"ManagementToken": "e687950a23c3a9bceec28c6223a06c79",
+ "SystemRootToken": auth_token('data_manager'),
"API": {
- "RequestTimeout": "30s"
+ "RequestTimeout": "30s",
},
"SystemLogs": {
- "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug')
+ "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
},
"PostgreSQL": {
"Connection": pgconnection,
},
"TLS": {
- "Insecure": True
+ "Insecure": True,
},
"Services": services,
"Users": {
- "AnonymousUserToken": auth_token('anonymous')
+ "AnonymousUserToken": auth_token('anonymous'),
},
"Collections": {
- "TrustAllContent": True
+ "BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
+ "TrustAllContent": True,
},
"Git": {
- "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
- }
- }
- }
+ "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git'),
+ },
+ "Volumes": {
+ "zzzzz-nyw5e-000000000000000": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore0_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep0.data"),
+ },
+ },
+ "zzzzz-nyw5e-111111111111111": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore1_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep1.data"),
+ },
+ },
+ "zzzzz-nyw5e-222222222222222": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore2_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep2.data"),
+ },
+ },
+ "zzzzz-nyw5e-333333333333333": {
+ "AccessViaHosts": {
+ "http://%s:%s" % (localhost, keepstore3_port): {},
+ },
+ "Driver": "Directory",
+ "DriverParameters": {
+ "Root": os.path.join(TEST_TMPDIR, "keep3.data"),
+ },
+ },
+ },
+ },
+ },
}
conf = os.path.join(TEST_TMPDIR, 'arvados.yml')
@@ -864,7 +905,7 @@ if __name__ == "__main__":
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
parser.add_argument('--num-keep-servers', metavar='int', type=int, default=2, help="Number of keep servers desired")
- parser.add_argument('--keep-enforce-permissions', action="store_true", help="Enforce keep permissions")
+ parser.add_argument('--keep-blob-signing', action="store_true", help="Enable blob signing for keepstore servers")
args = parser.parse_args()
@@ -895,7 +936,7 @@ if __name__ == "__main__":
elif args.action == 'stop_controller':
stop_controller()
elif args.action == 'start_keep':
- run_keep(enforce_permissions=args.keep_enforce_permissions, num_servers=args.num_keep_servers)
+ run_keep(blob_signing=args.keep_blob_signing, num_servers=args.num_keep_servers)
elif args.action == 'stop_keep':
stop_keep(num_servers=args.num_keep_servers)
elif args.action == 'start_keep_proxy':
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 42adf2450..a8c4a853c 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -852,27 +852,8 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers,
class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
ArvadosBaseTestCase):
- def _getKeepServerConfig():
- for config_file, mandatory in [
- ['application.yml', False], ['application.default.yml', True]]:
- path = os.path.join(run_test_server.SERVICES_SRC_DIR,
- "api", "config", config_file)
- if not mandatory and not os.path.exists(path):
- continue
- with open(path) as f:
- rails_config = yaml.safe_load(f.read())
- for config_section in ['test', 'common']:
- try:
- key = rails_config[config_section]["blob_signing_key"]
- except (KeyError, TypeError):
- pass
- else:
- return {'blob_signing_key': key,
- 'enforce_permissions': True}
- return {'blog_signing_key': None, 'enforce_permissions': False}
-
MAIN_SERVER = {}
- KEEP_SERVER = _getKeepServerConfig()
+ KEEP_SERVER = {'blob_signing': True}
PROJECT_UUID = run_test_server.fixture('groups')['aproject']['uuid']
@classmethod
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index d6b3a2a12..80e6987b3 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -130,8 +130,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
- KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
- 'enforce_permissions': True}
+ KEEP_SERVER = {'blob_signing': True}
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
@@ -173,70 +172,6 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
unsigned_bar_locator)
-# KeepOptionalPermission: starts Keep with --permission-key-file
-# but not --enforce-permissions (i.e. generate signatures on PUT
-# requests, but do not require them for GET requests)
-#
-# All of these requests should succeed when permissions are optional:
-# * authenticated request, signed locator
-# * authenticated request, unsigned locator
-# * unauthenticated request, signed locator
-# * unauthenticated request, unsigned locator
-class KeepOptionalPermission(run_test_server.TestCaseWithServers):
- MAIN_SERVER = {}
- KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
- 'enforce_permissions': False}
-
- @classmethod
- def setUpClass(cls):
- super(KeepOptionalPermission, cls).setUpClass()
- run_test_server.authorize_with("admin")
- cls.api_client = arvados.api('v1')
-
- def setUp(self):
- super(KeepOptionalPermission, self).setUp()
- self.keep_client = arvados.KeepClient(api_client=self.api_client,
- proxy='', local_store='')
-
- def _put_foo_and_check(self):
- signed_locator = self.keep_client.put('foo')
- self.assertRegex(
- signed_locator,
- r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
- 'invalid locator from Keep.put("foo"): ' + signed_locator)
- return signed_locator
-
- def test_KeepAuthenticatedSignedTest(self):
- signed_locator = self._put_foo_and_check()
- self.assertEqual(self.keep_client.get(signed_locator),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepAuthenticatedUnsignedTest(self):
- signed_locator = self._put_foo_and_check()
- self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepUnauthenticatedSignedTest(self):
- # Check that signed GET requests work even when permissions
- # enforcement is off.
- signed_locator = self._put_foo_and_check()
- self.keep_client.api_token = ''
- self.assertEqual(self.keep_client.get(signed_locator),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
- def test_KeepUnauthenticatedUnsignedTest(self):
- # Since --enforce-permissions is not in effect, GET requests
- # need not be authenticated.
- signed_locator = self._put_foo_and_check()
- self.keep_client.api_token = ''
- self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
- b'foo',
- 'wrong content from Keep.get(md5("foo"))')
-
-
class KeepProxyTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
KEEP_SERVER = {}
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 7072eae68..89b39dbc8 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -61,7 +61,7 @@ class IntegrationTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
run_test_server.run()
- run_test_server.run_keep(enforce_permissions=True, num_servers=2)
+ run_test_server.run_keep(blob_signing=True, num_servers=2)
@classmethod
def tearDownClass(cls):
diff --git a/services/fuse/tests/test_exec.py b/services/fuse/tests/test_exec.py
index 192b1b5dc..c90e7fd8d 100644
--- a/services/fuse/tests/test_exec.py
+++ b/services/fuse/tests/test_exec.py
@@ -38,7 +38,7 @@ class ExecMode(unittest.TestCase):
@classmethod
def setUpClass(cls):
run_test_server.run()
- run_test_server.run_keep(enforce_permissions=True, num_servers=2)
+ run_test_server.run_keep(blob_signing=True, num_servers=2)
run_test_server.authorize_with('active')
@classmethod
diff --git a/services/health/main.go b/services/health/main.go
index a06a4f884..039954a00 100644
--- a/services/health/main.go
+++ b/services/health/main.go
@@ -12,6 +12,7 @@ import (
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
+ "github.com/prometheus/client_golang/prometheus"
)
var (
@@ -19,7 +20,7 @@ var (
command cmd.Handler = service.Command(arvados.ServiceNameHealth, newHandler)
)
-func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, _ string, _ *prometheus.Registry) service.Handler {
return &health.Aggregator{Cluster: cluster}
}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 08a6c5881..9f814a20d 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -523,7 +523,7 @@ func (bal *Balancer) setupLookupTables() {
bal.mountsByClass["default"][mnt] = true
continue
}
- for _, class := range mnt.StorageClasses {
+ for class := range mnt.StorageClasses {
if mbc := bal.mountsByClass[class]; mbc == nil {
bal.classes = append(bal.classes, class)
bal.mountsByClass[class] = map[*KeepMount]bool{mnt: true}
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 2259b3d8c..e372d3784 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -524,7 +524,7 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
bal.srvs[9].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special"},
+ StorageClasses: map[string]bool{"special": true},
UUID: "zzzzz-mount-special00000009",
DeviceID: "9-special",
},
@@ -532,7 +532,7 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
}, {
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special", "special2"},
+ StorageClasses: map[string]bool{"special": true, "special2": true},
UUID: "zzzzz-mount-special20000009",
DeviceID: "9-special-and-special2",
},
@@ -544,7 +544,7 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
bal.srvs[13].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special2"},
+ StorageClasses: map[string]bool{"special2": true},
UUID: "zzzzz-mount-special2000000d",
DeviceID: "13-special2",
},
@@ -552,7 +552,7 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
}, {
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"default"},
+ StorageClasses: map[string]bool{"default": true},
UUID: "zzzzz-mount-00000000000000d",
DeviceID: "13-default",
},
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 3c17b3bd0..b52b706b2 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -7,8 +7,8 @@ package main
import (
"bytes"
"context"
+ "encoding/json"
"errors"
- "flag"
"fmt"
"io"
"io/ioutil"
@@ -22,99 +22,94 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-const (
- azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
- azureDefaultListBlobsMaxAttempts = 12
- azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
-)
-
-var (
- azureMaxGetBytes int
- azureStorageAccountName string
- azureStorageAccountKeyFile string
- azureStorageReplication int
- azureWriteRaceInterval = 15 * time.Second
- azureWriteRacePollTime = time.Second
-)
+func init() {
+ driver["Azure"] = newAzureBlobVolume
+}
-func readKeyFromFile(file string) (string, error) {
- buf, err := ioutil.ReadFile(file)
+func newAzureBlobVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &AzureBlobVolume{
+ StorageBaseURL: storage.DefaultBaseURL,
+ RequestTimeout: azureDefaultRequestTimeout,
+ WriteRaceInterval: azureDefaultWriteRaceInterval,
+ WriteRacePollTime: azureDefaultWriteRacePollTime,
+ cluster: cluster,
+ volume: volume,
+ logger: logger,
+ metrics: metrics,
+ }
+ err := json.Unmarshal(volume.DriverParameters, &v)
if err != nil {
- return "", errors.New("reading key from " + file + ": " + err.Error())
+ return nil, err
}
- accountKey := strings.TrimSpace(string(buf))
- if accountKey == "" {
- return "", errors.New("empty account key in " + file)
+ if v.ListBlobsRetryDelay == 0 {
+ v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
+ }
+ if v.ListBlobsMaxAttempts == 0 {
+ v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
+ }
+ if v.ContainerName == "" || v.StorageAccountName == "" || v.StorageAccountKey == "" {
+ return nil, errors.New("DriverParameters: ContainerName, StorageAccountName, and StorageAccountKey must be provided")
+ }
+ azc, err := storage.NewClient(v.StorageAccountName, v.StorageAccountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
+ if err != nil {
+ return nil, fmt.Errorf("creating Azure storage client: %s", err)
+ }
+ v.azClient = azc
+ v.azClient.Sender = &singleSender{}
+ v.azClient.HTTPClient = &http.Client{
+ Timeout: time.Duration(v.RequestTimeout),
+ }
+ bs := v.azClient.GetBlobService()
+ v.container = &azureContainer{
+ ctr: bs.GetContainerReference(v.ContainerName),
}
- return accountKey, nil
-}
-
-type azureVolumeAdder struct {
- *Config
-}
-// String implements flag.Value
-func (s *azureVolumeAdder) String() string {
- return "-"
+ if ok, err := v.container.Exists(); err != nil {
+ return nil, err
+ } else if !ok {
+ return nil, fmt.Errorf("Azure container %q does not exist: %s", v.ContainerName, err)
+ }
+ return v, v.check()
}
-func (s *azureVolumeAdder) Set(containerName string) error {
- s.Config.Volumes = append(s.Config.Volumes, &AzureBlobVolume{
- ContainerName: containerName,
- StorageAccountName: azureStorageAccountName,
- StorageAccountKeyFile: azureStorageAccountKeyFile,
- AzureReplication: azureStorageReplication,
- ReadOnly: deprecated.flagReadonly,
- })
+func (v *AzureBlobVolume) check() error {
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
return nil
}
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &AzureBlobVolume{} })
-
- flag.Var(&azureVolumeAdder{theConfig},
- "azure-storage-container-volume",
- "Use the given container as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &azureStorageAccountName,
- "azure-storage-account-name",
- "",
- "Azure storage account name used for subsequent --azure-storage-container-volume arguments.")
- flag.StringVar(
- &azureStorageAccountKeyFile,
- "azure-storage-account-key-file",
- "",
- "`File` containing the account key used for subsequent --azure-storage-container-volume arguments.")
- flag.IntVar(
- &azureStorageReplication,
- "azure-storage-replication",
- 3,
- "Replication level to report to clients when data is stored in an Azure container.")
- flag.IntVar(
- &azureMaxGetBytes,
- "azure-max-get-bytes",
- BlockSize,
- fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
-}
+const (
+ azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+ azureDefaultListBlobsMaxAttempts = 12
+ azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
+ azureDefaultWriteRaceInterval = arvados.Duration(15 * time.Second)
+ azureDefaultWriteRacePollTime = arvados.Duration(time.Second)
+)
// An AzureBlobVolume stores and retrieves blocks in an Azure Blob
// container.
type AzureBlobVolume struct {
- StorageAccountName string
- StorageAccountKeyFile string
- StorageBaseURL string // "" means default, "core.windows.net"
- ContainerName string
- AzureReplication int
- ReadOnly bool
- RequestTimeout arvados.Duration
- StorageClasses []string
- ListBlobsRetryDelay arvados.Duration
- ListBlobsMaxAttempts int
-
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string // "" means default, "core.windows.net"
+ ContainerName string
+ RequestTimeout arvados.Duration
+ ListBlobsRetryDelay arvados.Duration
+ ListBlobsMaxAttempts int
+ MaxGetBytes int
+ WriteRaceInterval arvados.Duration
+ WriteRacePollTime arvados.Duration
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
azClient storage.Client
container *azureContainer
}
@@ -127,84 +122,13 @@ func (*singleSender) Send(c *storage.Client, req *http.Request) (resp *http.Resp
return c.HTTPClient.Do(req)
}
-// Examples implements VolumeWithExamples.
-func (*AzureBlobVolume) Examples() []Volume {
- return []Volume{
- &AzureBlobVolume{
- StorageAccountName: "example-account-name",
- StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
- ContainerName: "example-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- &AzureBlobVolume{
- StorageAccountName: "cn-account-name",
- StorageAccountKeyFile: "/etc/azure_cn_storage_account_key.txt",
- StorageBaseURL: "core.chinacloudapi.cn",
- ContainerName: "cn-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- }
-}
-
// Type implements Volume.
func (v *AzureBlobVolume) Type() string {
return "Azure"
}
-// Start implements Volume.
-func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
- if v.ListBlobsRetryDelay == 0 {
- v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
- }
- if v.ListBlobsMaxAttempts == 0 {
- v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
- }
- if v.ContainerName == "" {
- return errors.New("no container name given")
- }
- if v.StorageAccountName == "" || v.StorageAccountKeyFile == "" {
- return errors.New("StorageAccountName and StorageAccountKeyFile must be given")
- }
- accountKey, err := readKeyFromFile(v.StorageAccountKeyFile)
- if err != nil {
- return err
- }
- if v.StorageBaseURL == "" {
- v.StorageBaseURL = storage.DefaultBaseURL
- }
- v.azClient, err = storage.NewClient(v.StorageAccountName, accountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
- if err != nil {
- return fmt.Errorf("creating Azure storage client: %s", err)
- }
- v.azClient.Sender = &singleSender{}
-
- if v.RequestTimeout == 0 {
- v.RequestTimeout = azureDefaultRequestTimeout
- }
- v.azClient.HTTPClient = &http.Client{
- Timeout: time.Duration(v.RequestTimeout),
- }
- bs := v.azClient.GetBlobService()
- v.container = &azureContainer{
- ctr: bs.GetContainerReference(v.ContainerName),
- }
-
- if ok, err := v.container.Exists(); err != nil {
- return err
- } else if !ok {
- return fmt.Errorf("Azure container %q does not exist", v.ContainerName)
- }
- // Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(lbls)
-
- return nil
-}
-
-// DeviceID returns a globally unique ID for the storage container.
-func (v *AzureBlobVolume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage container.
+func (v *AzureBlobVolume) GetDeviceID() string {
return "azure://" + v.StorageBaseURL + "/" + v.StorageAccountName + "/" + v.ContainerName
}
@@ -245,14 +169,14 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
if !haveDeadline {
t, err := v.Mtime(loc)
if err != nil {
- log.Print("Got empty block (possible race) but Mtime failed: ", err)
+ ctxlog.FromContext(ctx).Print("Got empty block (possible race) but Mtime failed: ", err)
break
}
- deadline = t.Add(azureWriteRaceInterval)
+ deadline = t.Add(v.WriteRaceInterval.Duration())
if time.Now().After(deadline) {
break
}
- log.Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
+ ctxlog.FromContext(ctx).Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
haveDeadline = true
} else if time.Now().After(deadline) {
break
@@ -260,12 +184,12 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
select {
case <-ctx.Done():
return 0, ctx.Err()
- case <-time.After(azureWriteRacePollTime):
+ case <-time.After(v.WriteRacePollTime.Duration()):
}
size, err = v.get(ctx, loc, buf)
}
if haveDeadline {
- log.Printf("Race ended with size==%d", size)
+ ctxlog.FromContext(ctx).Printf("Race ended with size==%d", size)
}
return size, err
}
@@ -273,8 +197,15 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
+
+ pieceSize := BlockSize
+ if v.MaxGetBytes > 0 && v.MaxGetBytes < BlockSize {
+ pieceSize = v.MaxGetBytes
+ }
+
+ pieces := 1
expectSize := len(buf)
- if azureMaxGetBytes < BlockSize {
+ if pieceSize < BlockSize {
// Unfortunately the handler doesn't tell us how long the blob
// is expected to be, so we have to ask Azure.
props, err := v.container.GetBlobProperties(loc)
@@ -285,6 +216,7 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
return 0, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
}
expectSize = int(props.ContentLength)
+ pieces = (expectSize + pieceSize - 1) / pieceSize
}
if expectSize == 0 {
@@ -293,7 +225,6 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// We'll update this actualSize if/when we get the last piece.
actualSize := -1
- pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
errors := make(chan error, pieces)
var wg sync.WaitGroup
wg.Add(pieces)
@@ -308,8 +239,8 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// interrupted as a result.
go func(p int) {
defer wg.Done()
- startPos := p * azureMaxGetBytes
- endPos := startPos + azureMaxGetBytes
+ startPos := p * pieceSize
+ endPos := startPos + pieceSize
if endPos > expectSize {
endPos = expectSize
}
@@ -412,7 +343,7 @@ func (v *AzureBlobVolume) Compare(ctx context.Context, loc string, expect []byte
// Put stores a Keep block as a block blob in the container.
func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
// Send the block data through a pipe, so that (if we need to)
@@ -441,7 +372,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
+ ctxlog.FromContext(ctx).Debugf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// io.Copy() to read. If so, un-stick it. This means
// CreateBlockBlobFromReader will get corrupt data,
@@ -450,7 +381,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
+ ctxlog.FromContext(ctx).Debugf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
return ctx.Err()
case err := <-errChan:
return err
@@ -459,7 +390,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
// Touch updates the last-modified property of a block blob.
func (v *AzureBlobVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
trashed, metadata, err := v.checkTrashed(loc)
@@ -508,7 +439,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
continue
}
modtime := time.Time(b.Properties.LastModified)
- if b.Properties.ContentLength == 0 && modtime.Add(azureWriteRaceInterval).After(time.Now()) {
+ if b.Properties.ContentLength == 0 && modtime.Add(v.WriteRaceInterval.Duration()).After(time.Now()) {
// A new zero-length blob is probably
// just a new non-empty blob that
// hasn't committed its data yet (see
@@ -535,7 +466,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
resp, err = v.container.ListBlobs(params)
err = v.translateError(err)
if err == VolumeBusyError {
- log.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
+ v.logger.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
time.Sleep(time.Duration(v.ListBlobsRetryDelay))
continue
} else {
@@ -547,7 +478,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
// Trash a Keep block.
func (v *AzureBlobVolume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -562,12 +493,12 @@ func (v *AzureBlobVolume) Trash(loc string) error {
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
// If TrashLifetime == 0, just delete it
- if theConfig.TrashLifetime == 0 {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{
IfMatch: props.Etag,
})
@@ -575,7 +506,7 @@ func (v *AzureBlobVolume) Trash(loc string) error {
// Otherwise, mark as trash
return v.container.SetBlobMetadata(loc, storage.BlobMetadata{
- "expires_at": fmt.Sprintf("%d", time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()),
+ "expires_at": fmt.Sprintf("%d", time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()),
}, &storage.SetBlobMetadataOptions{
IfMatch: props.Etag,
})
@@ -613,23 +544,6 @@ func (v *AzureBlobVolume) String() string {
return fmt.Sprintf("azure-storage-container:%+q", v.ContainerName)
}
-// Writable returns true, unless the -readonly flag was on when the
-// volume was added.
-func (v *AzureBlobVolume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the replication level of the container, as
-// specified by the -azure-storage-replication argument.
-func (v *AzureBlobVolume) Replication() int {
- return v.AzureReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *AzureBlobVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// If possible, translate an Azure SDK error to a recognizable error
// like os.ErrNotExist.
func (v *AzureBlobVolume) translateError(err error) error {
@@ -656,6 +570,10 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
@@ -670,7 +588,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
if err != nil {
- log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
+ v.logger.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
return
}
@@ -682,7 +600,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
IfMatch: b.Properties.Etag,
})
if err != nil {
- log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+ v.logger.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
return
}
atomic.AddInt64(&blocksDeleted, 1)
@@ -690,8 +608,8 @@ func (v *AzureBlobVolume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan storage.Blob, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan storage.Blob, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -705,7 +623,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
for page := 1; ; page++ {
resp, err := v.listBlobs(page, params)
if err != nil {
- log.Printf("EmptyTrash: ListBlobs: %v", err)
+ v.logger.Printf("EmptyTrash: ListBlobs: %v", err)
break
}
for _, b := range resp.Blobs {
@@ -719,7 +637,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
close(todo)
wg.Wait()
- log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+ v.logger.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
}
// InternalStats returns bucket I/O and API call counters.
@@ -748,7 +666,6 @@ func (s *azureBlobStats) TickErr(err error) {
if err, ok := err.(storage.AzureStorageServiceError); ok {
errType = errType + fmt.Sprintf(" %d (%s)", err.StatusCode, err.Code)
}
- log.Printf("errType %T, err %s", err, err)
s.statsTicker.TickErr(err, errType)
}
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 8d02def14..3539c3067 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -24,13 +24,13 @@ import (
"strconv"
"strings"
"sync"
- "testing"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/Azure/azure-sdk-for-go/storage"
- "github.com/ghodss/yaml"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -66,14 +66,16 @@ type azBlob struct {
type azStubHandler struct {
sync.Mutex
+ logger logrus.FieldLogger
blobs map[string]*azBlob
race chan chan struct{}
didlist503 bool
}
-func newAzStubHandler() *azStubHandler {
+func newAzStubHandler(c *check.C) *azStubHandler {
return &azStubHandler{
- blobs: make(map[string]*azBlob),
+ blobs: make(map[string]*azBlob),
+ logger: ctxlog.TestLogger(c),
}
}
@@ -117,7 +119,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
h.Lock()
defer h.Unlock()
if azureTestDebug {
- defer log.Printf("azStubHandler: %+v", r)
+ defer h.logger.Printf("azStubHandler: %+v", r)
}
path := strings.Split(r.URL.Path, "/")
@@ -128,7 +130,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
}
if err := r.ParseForm(); err != nil {
- log.Printf("azStubHandler(%+v): %s", r, err)
+ h.logger.Printf("azStubHandler(%+v): %s", r, err)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -184,13 +186,13 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
case r.Method == "PUT" && r.Form.Get("comp") == "block":
// "Put Block" API
if !blobExists {
- log.Printf("Got block for nonexistent blob: %+v", r)
+ h.logger.Printf("Got block for nonexistent blob: %+v", r)
rw.WriteHeader(http.StatusBadRequest)
return
}
blockID, err := base64.StdEncoding.DecodeString(r.Form.Get("blockid"))
if err != nil || len(blockID) == 0 {
- log.Printf("Invalid blockid: %+q", r.Form.Get("blockid"))
+ h.logger.Printf("Invalid blockid: %+q", r.Form.Get("blockid"))
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -200,14 +202,14 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// "Put Block List" API
bl := &blockListRequestBody{}
if err := xml.Unmarshal(body, bl); err != nil {
- log.Printf("xml Unmarshal: %s", err)
+ h.logger.Printf("xml Unmarshal: %s", err)
rw.WriteHeader(http.StatusBadRequest)
return
}
for _, encBlockID := range bl.Uncommitted {
blockID, err := base64.StdEncoding.DecodeString(encBlockID)
if err != nil || len(blockID) == 0 || blob.Uncommitted[string(blockID)] == nil {
- log.Printf("Invalid blockid: %+q", encBlockID)
+ h.logger.Printf("Invalid blockid: %+q", encBlockID)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -223,7 +225,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// sets metadata headers only as a way to bump Etag
// and Last-Modified.
if !blobExists {
- log.Printf("Got metadata for nonexistent blob: %+v", r)
+ h.logger.Printf("Got metadata for nonexistent blob: %+v", r)
rw.WriteHeader(http.StatusBadRequest)
return
}
@@ -269,7 +271,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
rw.Header().Set("Content-Length", strconv.Itoa(len(data)))
if r.Method == "GET" {
if _, err := rw.Write(data); err != nil {
- log.Printf("write %+q: %s", data, err)
+ h.logger.Printf("write %+q: %s", data, err)
}
}
h.unlockAndRace()
@@ -333,12 +335,12 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
}
buf, err := xml.Marshal(resp)
if err != nil {
- log.Print(err)
+ h.logger.Error(err)
rw.WriteHeader(http.StatusInternalServerError)
}
rw.Write(buf)
default:
- log.Printf("azStubHandler: not implemented: %+v Body:%+q", r, body)
+ h.logger.Printf("azStubHandler: not implemented: %+v Body:%+q", r, body)
rw.WriteHeader(http.StatusNotImplemented)
}
}
@@ -347,6 +349,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// tries to connect to "devstoreaccount1.blob.127.0.0.1:46067", and
// in such cases transparently dials "127.0.0.1:46067" instead.
type azStubDialer struct {
+ logger logrus.FieldLogger
net.Dialer
}
@@ -355,7 +358,7 @@ var localHostPortRe = regexp.MustCompile(`(127\.0\.0\.1|localhost|\[::1\]):\d+`)
func (d *azStubDialer) Dial(network, address string) (net.Conn, error) {
if hp := localHostPortRe.FindString(address); hp != "" {
if azureTestDebug {
- log.Println("azStubDialer: dial", hp, "instead of", address)
+ d.logger.Debug("azStubDialer: dial", hp, "instead of", address)
}
address = hp
}
@@ -369,29 +372,24 @@ type TestableAzureBlobVolume struct {
t TB
}
-func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableAzureBlobVolume {
- azHandler := newAzStubHandler()
+func (s *StubbedAzureBlobSuite) newTestableAzureBlobVolume(t TB, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs) *TestableAzureBlobVolume {
+ azHandler := newAzStubHandler(t.(*check.C))
azStub := httptest.NewServer(azHandler)
var azClient storage.Client
+ var err error
container := azureTestContainer
if container == "" {
// Connect to stub instead of real Azure storage service
stubURLBase := strings.Split(azStub.URL, "://")[1]
- var err error
if azClient, err = storage.NewClient(fakeAccountName, fakeAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil {
t.Fatal(err)
}
container = "fakecontainername"
} else {
// Connect to real Azure storage service
- accountKey, err := readKeyFromFile(azureStorageAccountKeyFile)
- if err != nil {
- t.Fatal(err)
- }
- azClient, err = storage.NewBasicClient(azureStorageAccountName, accountKey)
- if err != nil {
+ if azClient, err = storage.NewBasicClient(os.Getenv("ARVADOS_TEST_AZURE_ACCOUNT_NAME"), os.Getenv("ARVADOS_TEST_AZURE_ACCOUNT_KEY")); err != nil {
t.Fatal(err)
}
}
@@ -400,12 +398,19 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
bs := azClient.GetBlobService()
v := &AzureBlobVolume{
ContainerName: container,
- ReadOnly: readonly,
- AzureReplication: replication,
+ WriteRaceInterval: arvados.Duration(time.Millisecond),
+ WriteRacePollTime: arvados.Duration(time.Nanosecond),
ListBlobsMaxAttempts: 2,
ListBlobsRetryDelay: arvados.Duration(time.Millisecond),
azClient: azClient,
container: &azureContainer{ctr: bs.GetContainerReference(container)},
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(t),
+ metrics: metrics,
+ }
+ if err = v.check(); err != nil {
+ t.Fatal(err)
}
return &TestableAzureBlobVolume{
@@ -419,84 +424,45 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
var _ = check.Suite(&StubbedAzureBlobSuite{})
type StubbedAzureBlobSuite struct {
- volume *TestableAzureBlobVolume
origHTTPTransport http.RoundTripper
}
func (s *StubbedAzureBlobSuite) SetUpTest(c *check.C) {
s.origHTTPTransport = http.DefaultTransport
http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
+ Dial: (&azStubDialer{logger: ctxlog.TestLogger(c)}).Dial,
}
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
-
- s.volume = NewTestableAzureBlobVolume(c, false, 3)
}
func (s *StubbedAzureBlobSuite) TearDownTest(c *check.C) {
- s.volume.Teardown()
http.DefaultTransport = s.origHTTPTransport
}
-func TestAzureBlobVolumeWithGeneric(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeWithGeneric(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableAzureBlobVolume(t, cluster, volume, metrics)
})
}
-func TestAzureBlobVolumeConcurrentRanges(t *testing.T) {
- defer func(b int) {
- azureMaxGetBytes = b
- }(azureMaxGetBytes)
-
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeConcurrentRanges(c *check.C) {
// Test (BlockSize mod azureMaxGetBytes)==0 and !=0 cases
- for _, azureMaxGetBytes = range []int{2 << 22, 2<<22 - 1} {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+ for _, b := range []int{2 << 22, 2<<22 - 1} {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ v := s.newTestableAzureBlobVolume(t, cluster, volume, metrics)
+ v.MaxGetBytes = b
+ return v
})
}
}
-func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
- azureWriteRaceInterval = time.Millisecond
- azureWriteRacePollTime = time.Nanosecond
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableAzureBlobVolume(t, true, azureStorageReplication)
+func (s *StubbedAzureBlobSuite) TestReadonlyAzureBlobVolumeWithGeneric(c *check.C) {
+ DoGenericVolumeTests(c, false, func(c TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableAzureBlobVolume(c, cluster, volume, metrics)
})
}
-func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeRangeFenceposts(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
for _, size := range []int{
@@ -514,47 +480,27 @@ func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
hash := fmt.Sprintf("%x", md5.Sum(data))
err := v.Put(context.Background(), hash, data)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
gotData := make([]byte, len(data))
gotLen, err := v.Get(context.Background(), hash, gotData)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
gotHash := fmt.Sprintf("%x", md5.Sum(gotData))
if gotLen != size {
- t.Errorf("length mismatch: got %d != %d", gotLen, size)
+ c.Errorf("length mismatch: got %d != %d", gotLen, size)
}
if gotHash != hash {
- t.Errorf("hash mismatch: got %s != %s", gotHash, hash)
- }
- }
-}
-
-func TestAzureBlobVolumeReplication(t *testing.T) {
- for r := 1; r <= 4; r++ {
- v := NewTestableAzureBlobVolume(t, false, r)
- defer v.Teardown()
- if n := v.Replication(); n != r {
- t.Errorf("Got replication %d, expected %d", n, r)
+ c.Errorf("hash mismatch: got %s != %s", gotHash, hash)
}
}
}
-func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeCreateBlobRace(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
- azureWriteRaceInterval = time.Second
- azureWriteRacePollTime = time.Millisecond
-
var wg sync.WaitGroup
v.azHandler.race = make(chan chan struct{})
@@ -564,7 +510,7 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
defer wg.Done()
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
}()
continuePut := make(chan struct{})
@@ -576,7 +522,7 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
buf := make([]byte, len(TestBlock))
_, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
}()
// Wait for the stub's Get to get the empty blob
@@ -588,26 +534,18 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
wg.Wait()
}
-func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeCreateBlobRaceDeadline(c *check.C) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
+ v.AzureBlobVolume.WriteRaceInterval.Set("2s")
+ v.AzureBlobVolume.WriteRacePollTime.Set("5ms")
defer v.Teardown()
- azureWriteRaceInterval = 2 * time.Second
- azureWriteRacePollTime = 5 * time.Millisecond
-
v.PutRaw(TestHash, nil)
buf := new(bytes.Buffer)
v.IndexTo("", buf)
if buf.Len() != 0 {
- t.Errorf("Index %+q should be empty", buf.Bytes())
+ c.Errorf("Index %+q should be empty", buf.Bytes())
}
v.TouchWithDate(TestHash, time.Now().Add(-1982*time.Millisecond))
@@ -618,56 +556,49 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
buf := make([]byte, BlockSize)
n, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Error(err)
+ c.Error(err)
return
}
if n != 0 {
- t.Errorf("Got %+q, expected empty buf", buf[:n])
+ c.Errorf("Got %+q, expected empty buf", buf[:n])
}
}()
select {
case <-allDone:
case <-time.After(time.Second):
- t.Error("Get should have stopped waiting for race when block was 2s old")
+ c.Error("Get should have stopped waiting for race when block was 2s old")
}
buf.Reset()
v.IndexTo("", buf)
if !bytes.HasPrefix(buf.Bytes(), []byte(TestHash+"+0")) {
- t.Errorf("Index %+q should have %+q", buf.Bytes(), TestHash+"+0")
+ c.Errorf("Index %+q should have %+q", buf.Bytes(), TestHash+"+0")
}
}
-func TestAzureBlobVolumeContextCancelGet(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelGet(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
v.PutRaw(TestHash, TestBlock)
_, err := v.Get(ctx, TestHash, make([]byte, BlockSize))
return err
})
}
-func TestAzureBlobVolumeContextCancelPut(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelPut(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
return v.Put(ctx, TestHash, make([]byte, BlockSize))
})
}
-func TestAzureBlobVolumeContextCancelCompare(t *testing.T) {
- testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error {
+func (s *StubbedAzureBlobSuite) TestAzureBlobVolumeContextCancelCompare(c *check.C) {
+ s.testAzureBlobVolumeContextCancel(c, func(ctx context.Context, v *TestableAzureBlobVolume) error {
v.PutRaw(TestHash, TestBlock)
return v.Compare(ctx, TestHash, TestBlock2)
})
}
-func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Context, *TestableAzureBlobVolume) error) {
- defer func(t http.RoundTripper) {
- http.DefaultTransport = t
- }(http.DefaultTransport)
- http.DefaultTransport = &http.Transport{
- Dial: (&azStubDialer{}).Dial,
- }
-
- v := NewTestableAzureBlobVolume(t, false, 3)
+func (s *StubbedAzureBlobSuite) testAzureBlobVolumeContextCancel(c *check.C, testFunc func(context.Context, *TestableAzureBlobVolume) error) {
+ v := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
defer v.Teardown()
v.azHandler.race = make(chan chan struct{})
@@ -677,15 +608,15 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
defer close(allDone)
err := testFunc(ctx, v)
if err != context.Canceled {
- t.Errorf("got %T %q, expected %q", err, err, context.Canceled)
+ c.Errorf("got %T %q, expected %q", err, err, context.Canceled)
}
}()
releaseHandler := make(chan struct{})
select {
case <-allDone:
- t.Error("testFunc finished without waiting for v.azHandler.race")
+ c.Error("testFunc finished without waiting for v.azHandler.race")
case <-time.After(10 * time.Second):
- t.Error("timed out waiting to enter handler")
+ c.Error("timed out waiting to enter handler")
case v.azHandler.race <- releaseHandler:
}
@@ -693,7 +624,7 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
select {
case <-time.After(10 * time.Second):
- t.Error("timed out waiting to cancel")
+ c.Error("timed out waiting to cancel")
case <-allDone:
}
@@ -703,8 +634,11 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
}
func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
+ volume := s.newTestableAzureBlobVolume(c, testCluster(c), arvados.Volume{Replication: 3}, newVolumeMetricsVecs(prometheus.NewRegistry()))
+ defer volume.Teardown()
+
stats := func() string {
- buf, err := json.Marshal(s.volume.InternalStats())
+ buf, err := json.Marshal(volume.InternalStats())
c.Check(err, check.IsNil)
return string(buf)
}
@@ -713,37 +647,25 @@ func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
c.Check(stats(), check.Matches, `.*"Errors":0,.*`)
loc := "acbd18db4cc2f85cedef654fccc4a4d8"
- _, err := s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err := volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
c.Check(stats(), check.Matches, `.*"storage\.AzureStorageServiceError 404 \(404 Not Found\)":[^0].*`)
c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
- err = s.volume.Put(context.Background(), loc, []byte("foo"))
+ err = volume.Put(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = volume.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"InBytes":6,.*`)
}
-func (s *StubbedAzureBlobSuite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: Azure
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
-
func (v *TestableAzureBlobVolume) PutRaw(locator string, data []byte) {
v.azHandler.PutRaw(v.ContainerName, locator, data)
}
@@ -760,17 +682,6 @@ func (v *TestableAzureBlobVolume) ReadWriteOperationLabelValues() (r, w string)
return "get", "create"
}
-func (v *TestableAzureBlobVolume) DeviceID() string {
- // Dummy device id for testing purposes
- return "azure://azure_blob_volume_test"
-}
-
-func (v *TestableAzureBlobVolume) Start(vm *volumeMetricsVecs) error {
- // Override original Start() to be able to assign CounterVecs with a dummy DeviceID
- v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(prometheus.Labels{"device_id": v.DeviceID()})
- return nil
-}
-
func makeEtag() string {
return fmt.Sprintf("0x%x", rand.Int63())
}
diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go
index d2e7c9ebd..623693cd1 100644
--- a/services/keepstore/bufferpool.go
+++ b/services/keepstore/bufferpool.go
@@ -8,9 +8,12 @@ import (
"sync"
"sync/atomic"
"time"
+
+ "github.com/sirupsen/logrus"
)
type bufferPool struct {
+ log logrus.FieldLogger
// limiter has a "true" placeholder for each in-use buffer.
limiter chan bool
// allocated is the number of bytes currently allocated to buffers.
@@ -19,9 +22,9 @@ type bufferPool struct {
sync.Pool
}
-func newBufferPool(count int, bufSize int) *bufferPool {
- p := bufferPool{}
- p.New = func() interface{} {
+func newBufferPool(log logrus.FieldLogger, count int, bufSize int) *bufferPool {
+ p := bufferPool{log: log}
+ p.Pool.New = func() interface{} {
atomic.AddUint64(&p.allocated, uint64(bufSize))
return make([]byte, bufSize)
}
@@ -34,13 +37,13 @@ func (p *bufferPool) Get(size int) []byte {
case p.limiter <- true:
default:
t0 := time.Now()
- log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
+ p.log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
p.limiter <- true
- log.Printf("waited %v for a buffer", time.Since(t0))
+ p.log.Printf("waited %v for a buffer", time.Since(t0))
}
buf := p.Pool.Get().([]byte)
if cap(buf) < size {
- log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
+ p.log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
}
return buf[:size]
}
diff --git a/services/keepstore/bufferpool_test.go b/services/keepstore/bufferpool_test.go
index 21b03edd4..2afa0ddb8 100644
--- a/services/keepstore/bufferpool_test.go
+++ b/services/keepstore/bufferpool_test.go
@@ -5,8 +5,10 @@
package main
import (
+ "context"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
. "gopkg.in/check.v1"
)
@@ -17,16 +19,16 @@ type BufferPoolSuite struct{}
// Initialize a default-sized buffer pool for the benefit of test
// suites that don't run main().
func init() {
- bufs = newBufferPool(theConfig.MaxBuffers, BlockSize)
+ bufs = newBufferPool(ctxlog.FromContext(context.Background()), 12, BlockSize)
}
// Restore sane default after bufferpool's own tests
func (s *BufferPoolSuite) TearDownTest(c *C) {
- bufs = newBufferPool(theConfig.MaxBuffers, BlockSize)
+ bufs = newBufferPool(ctxlog.FromContext(context.Background()), 12, BlockSize)
}
func (s *BufferPoolSuite) TestBufferPoolBufSize(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
b1 := bufs.Get(1)
bufs.Get(2)
bufs.Put(b1)
@@ -35,14 +37,14 @@ func (s *BufferPoolSuite) TestBufferPoolBufSize(c *C) {
}
func (s *BufferPoolSuite) TestBufferPoolUnderLimit(c *C) {
- bufs := newBufferPool(3, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 3, 10)
b1 := bufs.Get(10)
bufs.Get(10)
testBufferPoolRace(c, bufs, b1, "Get")
}
func (s *BufferPoolSuite) TestBufferPoolAtLimit(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
b1 := bufs.Get(10)
bufs.Get(10)
testBufferPoolRace(c, bufs, b1, "Put")
@@ -66,7 +68,7 @@ func testBufferPoolRace(c *C, bufs *bufferPool, unused []byte, expectWin string)
}
func (s *BufferPoolSuite) TestBufferPoolReuse(c *C) {
- bufs := newBufferPool(2, 10)
+ bufs := newBufferPool(ctxlog.TestLogger(c), 2, 10)
bufs.Get(10)
last := bufs.Get(10)
// The buffer pool is allowed to throw away unused buffers
diff --git a/services/keepstore/command.go b/services/keepstore/command.go
new file mode 100644
index 000000000..df279687d
--- /dev/null
+++ b/services/keepstore/command.go
@@ -0,0 +1,219 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/lib/config"
+ "git.curoverse.com/arvados.git/lib/service"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
+)
+
+var (
+ version = "dev"
+ Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler)
+)
+
+func main() {
+ os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background()))
+ if !ok {
+ return 2
+ }
+ return Command.RunCommand(prog, args, stdin, stdout, stderr)
+}
+
+// Parse keepstore command line flags, and return equivalent
+// service.Command flags. The second return value ("ok") is true if
+// all provided flags were successfully converted.
+func convertKeepstoreFlagsToServiceFlags(args []string, lgr logrus.FieldLogger) ([]string, bool) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.String("listen", "", "Services.Keepstore.InternalURLs")
+ flags.Int("max-buffers", 0, "API.MaxKeepBlockBuffers")
+ flags.Int("max-requests", 0, "API.MaxConcurrentRequests")
+ flags.Bool("never-delete", false, "Collections.BlobTrash")
+ flags.Bool("enforce-permissions", false, "Collections.BlobSigning")
+ flags.String("permission-key-file", "", "Collections.BlobSigningKey")
+ flags.String("blob-signing-key-file", "", "Collections.BlobSigningKey")
+ flags.String("data-manager-token-file", "", "SystemRootToken")
+ flags.Int("permission-ttl", 0, "Collections.BlobSigningTTL")
+ flags.Int("blob-signature-ttl", 0, "Collections.BlobSigningTTL")
+ flags.String("trash-lifetime", "", "Collections.BlobTrashLifetime")
+ flags.Bool("serialize", false, "Volumes.*.DriverParameters.Serialize")
+ flags.Bool("readonly", false, "Volumes.*.ReadOnly")
+ flags.String("pid", "", "-")
+ flags.String("trash-check-interval", "", "Collections.BlobTrashCheckInterval")
+
+ flags.String("azure-storage-container-volume", "", "Volumes.*.Driver")
+ flags.String("azure-storage-account-name", "", "Volumes.*.DriverParameters.StorageAccountName")
+ flags.String("azure-storage-account-key-file", "", "Volumes.*.DriverParameters.StorageAccountKey")
+ flags.String("azure-storage-replication", "", "Volumes.*.Replication")
+ flags.String("azure-max-get-bytes", "", "Volumes.*.DriverParameters.MaxDataReadSize")
+
+ flags.String("s3-bucket-volume", "", "Volumes.*.DriverParameters.Bucket")
+ flags.String("s3-region", "", "Volumes.*.DriverParameters.Region")
+ flags.String("s3-endpoint", "", "Volumes.*.DriverParameters.Endpoint")
+ flags.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKey")
+ flags.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretKey")
+ flags.String("s3-race-window", "", "Volumes.*.DriverParameters.RaceWindow")
+ flags.String("s3-replication", "", "Volumes.*.Replication")
+ flags.String("s3-unsafe-delete", "", "Volumes.*.DriverParameters.UnsafeDelete")
+
+ flags.String("volume", "", "Volumes")
+
+ flags.Bool("version", false, "")
+ flags.String("config", "", "")
+ flags.String("legacy-keepstore-config", "", "")
+
+ err := flags.Parse(args)
+ if err == flag.ErrHelp {
+ return []string{"-help"}, true
+ } else if err != nil {
+ return nil, false
+ }
+
+ args = nil
+ ok := true
+ flags.Visit(func(f *flag.Flag) {
+ if f.Name == "config" || f.Name == "legacy-keepstore-config" || f.Name == "version" {
+ args = append(args, "-"+f.Name, f.Value.String())
+ } else if f.Usage == "-" {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported", f.Name)
+ } else {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported -- use Clusters.*.%s in cluster config file instead", f.Name, f.Usage)
+ }
+ })
+ if !ok {
+ return nil, false
+ }
+
+ flags = flag.NewFlagSet("", flag.ExitOnError)
+ loader := config.NewLoader(nil, lgr)
+ loader.SetupFlags(flags)
+ return loader.MungeLegacyConfigArgs(lgr, args, "-legacy-keepstore-config"), true
+}
+
+type handler struct {
+ http.Handler
+ Cluster *arvados.Cluster
+ Logger logrus.FieldLogger
+
+ pullq *WorkQueue
+ trashq *WorkQueue
+ volmgr *RRVolumeManager
+ keepClient *keepclient.KeepClient
+
+ err error
+ setupOnce sync.Once
+}
+
+func (h *handler) CheckHealth() error {
+ return h.err
+}
+
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ var h handler
+ serviceURL, ok := service.URLFromContext(ctx)
+ if !ok {
+ return service.ErrorHandler(ctx, cluster, errors.New("BUG: no URL from service.URLFromContext"))
+ }
+ err := h.setup(ctx, cluster, token, reg, serviceURL)
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, err)
+ }
+ return &h
+}
+
+func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry, serviceURL arvados.URL) error {
+ h.Cluster = cluster
+ h.Logger = ctxlog.FromContext(ctx)
+ if h.Cluster.API.MaxKeepBlockBuffers <= 0 {
+ return fmt.Errorf("MaxBuffers must be greater than zero")
+ }
+ bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlockBuffers, BlockSize)
+
+ if h.Cluster.API.MaxConcurrentRequests < 1 {
+ h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlockBuffers * 2
+ h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlockBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
+ }
+
+ if h.Cluster.Collections.BlobSigningKey != "" {
+ } else if h.Cluster.Collections.BlobSigning {
+ return errors.New("cannot enable Collections.BlobSigning with no Collections.BlobSigningKey")
+ } else {
+ h.Logger.Warn("Running without a blob signing key. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions. To fix this, configure Collections.BlobSigning and Collections.BlobSigningKey.")
+ }
+
+ if len(h.Cluster.Volumes) == 0 {
+ return errors.New("no volumes configured")
+ }
+
+ h.Logger.Printf("keepstore %s starting, pid %d", version, os.Getpid())
+
+ // Start a round-robin VolumeManager with the configured volumes.
+ vm, err := makeRRVolumeManager(h.Logger, h.Cluster, serviceURL, newVolumeMetricsVecs(reg))
+ if err != nil {
+ return err
+ }
+ if len(vm.readables) == 0 {
+ return fmt.Errorf("no volumes configured for %s", serviceURL)
+ }
+ h.volmgr = vm
+
+ // Initialize the pullq and workers
+ h.pullq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobReplicateConcurrency; i++ {
+ go h.runPullWorker(h.pullq)
+ }
+
+ // Initialize the trashq and workers
+ h.trashq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
+ go RunTrashWorker(h.volmgr, h.Cluster, h.trashq)
+ }
+
+ // Set up routes and metrics
+ h.Handler = MakeRESTRouter(ctx, cluster, reg, vm, h.pullq, h.trashq)
+
+ // Initialize keepclient for pull workers
+ c, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return err
+ }
+ ac, err := arvadosclient.New(c)
+ if err != nil {
+ return err
+ }
+ h.keepClient = &keepclient.KeepClient{
+ Arvados: ac,
+ Want_replicas: 1,
+ }
+ h.keepClient.Arvados.ApiToken = fmt.Sprintf("%x", rand.Int63())
+
+ if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 {
+ go emptyTrash(h.volmgr.writables, d)
+ }
+
+ return nil
+}
diff --git a/services/keepstore/command_test.go b/services/keepstore/command_test.go
new file mode 100644
index 000000000..ad2aa0957
--- /dev/null
+++ b/services/keepstore/command_test.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (*CommandSuite) TestLegacyConfigPath(c *check.C) {
+ var stdin, stdout, stderr bytes.Buffer
+ tmp, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmp.Name())
+ tmp.Write([]byte("Listen: \"1.2.3.4.5:invalidport\"\n"))
+ tmp.Close()
+ exited := runCommand("keepstore", []string{"-config", tmp.Name()}, &stdin, &stdout, &stderr)
+ c.Check(exited, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `(?ms).*unable to migrate Listen value.*`)
+}
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
deleted file mode 100644
index 43a219111..000000000
--- a/services/keepstore/config.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "strings"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
-)
-
-type Config struct {
- Debug bool
- Listen string
-
- LogFormat string
-
- PIDFile string
-
- MaxBuffers int
- MaxRequests int
-
- BlobSignatureTTL arvados.Duration
- BlobSigningKeyFile string
- RequireSignatures bool
- SystemAuthTokenFile string
- EnableDelete bool
- TrashLifetime arvados.Duration
- TrashCheckInterval arvados.Duration
- PullWorkers int
- TrashWorkers int
- EmptyTrashWorkers int
- TLSCertificateFile string
- TLSKeyFile string
-
- Volumes VolumeList
-
- blobSigningKey []byte
- systemAuthToken string
- debugLogf func(string, ...interface{})
-
- ManagementToken string
-}
-
-var (
- theConfig = DefaultConfig()
- formatter = map[string]logrus.Formatter{
- "text": &logrus.TextFormatter{
- FullTimestamp: true,
- TimestampFormat: rfc3339NanoFixed,
- },
- "json": &logrus.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- },
- }
- log = logrus.StandardLogger()
-)
-
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
-// DefaultConfig returns the default configuration.
-func DefaultConfig() *Config {
- return &Config{
- Listen: ":25107",
- LogFormat: "json",
- MaxBuffers: 128,
- RequireSignatures: true,
- BlobSignatureTTL: arvados.Duration(14 * 24 * time.Hour),
- TrashLifetime: arvados.Duration(14 * 24 * time.Hour),
- TrashCheckInterval: arvados.Duration(24 * time.Hour),
- Volumes: []Volume{},
- }
-}
-
-// Start should be called exactly once: after setting all public
-// fields, and before using the config.
-func (cfg *Config) Start(reg *prometheus.Registry) error {
- if cfg.Debug {
- log.Level = logrus.DebugLevel
- cfg.debugLogf = log.Printf
- cfg.debugLogf("debugging enabled")
- } else {
- log.Level = logrus.InfoLevel
- cfg.debugLogf = func(string, ...interface{}) {}
- }
-
- f := formatter[strings.ToLower(cfg.LogFormat)]
- if f == nil {
- return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
- }
- log.Formatter = f
-
- if cfg.MaxBuffers < 0 {
- return fmt.Errorf("MaxBuffers must be greater than zero")
- }
- bufs = newBufferPool(cfg.MaxBuffers, BlockSize)
-
- if cfg.MaxRequests < 1 {
- cfg.MaxRequests = cfg.MaxBuffers * 2
- log.Printf("MaxRequests <1 or not specified; defaulting to MaxBuffers * 2 == %d", cfg.MaxRequests)
- }
-
- if cfg.BlobSigningKeyFile != "" {
- buf, err := ioutil.ReadFile(cfg.BlobSigningKeyFile)
- if err != nil {
- return fmt.Errorf("reading blob signing key file: %s", err)
- }
- cfg.blobSigningKey = bytes.TrimSpace(buf)
- if len(cfg.blobSigningKey) == 0 {
- return fmt.Errorf("blob signing key file %q is empty", cfg.BlobSigningKeyFile)
- }
- } else if cfg.RequireSignatures {
- return fmt.Errorf("cannot enable RequireSignatures (-enforce-permissions) without a blob signing key")
- } else {
- log.Println("Running without a blob signing key. Block locators " +
- "returned by this server will not be signed, and will be rejected " +
- "by a server that enforces permissions.")
- log.Println("To fix this, use the BlobSigningKeyFile config entry.")
- }
-
- if fn := cfg.SystemAuthTokenFile; fn != "" {
- buf, err := ioutil.ReadFile(fn)
- if err != nil {
- return fmt.Errorf("cannot read system auth token file %q: %s", fn, err)
- }
- cfg.systemAuthToken = strings.TrimSpace(string(buf))
- }
-
- if cfg.EnableDelete {
- log.Print("Trash/delete features are enabled. WARNING: this has not " +
- "been extensively tested. You should disable this unless you can afford to lose data.")
- }
-
- if len(cfg.Volumes) == 0 {
- if (&unixVolumeAdder{cfg}).Discover() == 0 {
- return fmt.Errorf("no volumes found")
- }
- }
- vm := newVolumeMetricsVecs(reg)
- for _, v := range cfg.Volumes {
- if err := v.Start(vm); err != nil {
- return fmt.Errorf("volume %s: %s", v, err)
- }
- log.Printf("Using volume %v (writable=%v)", v, v.Writable())
- }
- return nil
-}
-
-// VolumeTypes is built up by init() funcs in the source files that
-// define the volume types.
-var VolumeTypes = []func() VolumeWithExamples{}
-
-type VolumeList []Volume
-
-// UnmarshalJSON -- given an array of objects -- deserializes each
-// object as the volume type indicated by the object's Type field.
-func (vl *VolumeList) UnmarshalJSON(data []byte) error {
- typeMap := map[string]func() VolumeWithExamples{}
- for _, factory := range VolumeTypes {
- t := factory().Type()
- if _, ok := typeMap[t]; ok {
- log.Fatalf("volume type %+q is claimed by multiple VolumeTypes", t)
- }
- typeMap[t] = factory
- }
-
- var mapList []map[string]interface{}
- err := json.Unmarshal(data, &mapList)
- if err != nil {
- return err
- }
- for _, mapIn := range mapList {
- typeIn, ok := mapIn["Type"].(string)
- if !ok {
- return fmt.Errorf("invalid volume type %+v", mapIn["Type"])
- }
- factory, ok := typeMap[typeIn]
- if !ok {
- return fmt.Errorf("unsupported volume type %+q", typeIn)
- }
- data, err := json.Marshal(mapIn)
- if err != nil {
- return err
- }
- vol := factory()
- err = json.Unmarshal(data, vol)
- if err != nil {
- return err
- }
- *vl = append(*vl, vol)
- }
- return nil
-}
-
-// MarshalJSON adds a "Type" field to each volume corresponding to its
-// Type().
-func (vl *VolumeList) MarshalJSON() ([]byte, error) {
- data := []byte{'['}
- for _, vs := range *vl {
- j, err := json.Marshal(vs)
- if err != nil {
- return nil, err
- }
- if len(data) > 1 {
- data = append(data, byte(','))
- }
- t, err := json.Marshal(vs.Type())
- if err != nil {
- panic(err)
- }
- data = append(data, j[0])
- data = append(data, []byte(`"Type":`)...)
- data = append(data, t...)
- data = append(data, byte(','))
- data = append(data, j[1:]...)
- }
- return append(data, byte(']')), nil
-}
diff --git a/services/keepstore/config_test.go b/services/keepstore/config_test.go
deleted file mode 100644
index e3b0ffc22..000000000
--- a/services/keepstore/config_test.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "github.com/sirupsen/logrus"
-)
-
-func init() {
- log.Level = logrus.DebugLevel
- theConfig.debugLogf = log.Printf
-}
diff --git a/services/keepstore/deprecated.go b/services/keepstore/deprecated.go
deleted file mode 100644
index d1377978a..000000000
--- a/services/keepstore/deprecated.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
-)
-
-type deprecatedOptions struct {
- flagSerializeIO bool
- flagReadonly bool
- neverDelete bool
- signatureTTLSeconds int
-}
-
-var deprecated = deprecatedOptions{
- neverDelete: !theConfig.EnableDelete,
- signatureTTLSeconds: int(theConfig.BlobSignatureTTL.Duration() / time.Second),
-}
-
-func (depr *deprecatedOptions) beforeFlagParse(cfg *Config) {
- flag.StringVar(&cfg.Listen, "listen", cfg.Listen, "see Listen configuration")
- flag.IntVar(&cfg.MaxBuffers, "max-buffers", cfg.MaxBuffers, "see MaxBuffers configuration")
- flag.IntVar(&cfg.MaxRequests, "max-requests", cfg.MaxRequests, "see MaxRequests configuration")
- flag.BoolVar(&depr.neverDelete, "never-delete", depr.neverDelete, "see EnableDelete configuration")
- flag.BoolVar(&cfg.RequireSignatures, "enforce-permissions", cfg.RequireSignatures, "see RequireSignatures configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "permission-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "blob-signing-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.SystemAuthTokenFile, "data-manager-token-file", cfg.SystemAuthTokenFile, "see SystemAuthToken`File` configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "permission-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "blob-signature-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.Var(&cfg.TrashLifetime, "trash-lifetime", "see TrashLifetime configuration")
- flag.BoolVar(&depr.flagSerializeIO, "serialize", depr.flagSerializeIO, "serialize read and write operations on the following volumes.")
- flag.BoolVar(&depr.flagReadonly, "readonly", depr.flagReadonly, "do not write, delete, or touch anything on the following volumes.")
- flag.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "see `PIDFile` configuration")
- flag.Var(&cfg.TrashCheckInterval, "trash-check-interval", "see TrashCheckInterval configuration")
-}
-
-func (depr *deprecatedOptions) afterFlagParse(cfg *Config) {
- cfg.BlobSignatureTTL = arvados.Duration(depr.signatureTTLSeconds) * arvados.Duration(time.Second)
- cfg.EnableDelete = !depr.neverDelete
-}
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index ad907ef10..251ad0a1d 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -23,16 +23,49 @@ import (
"os"
"regexp"
"strings"
- "testing"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
-var testCluster = &arvados.Cluster{
- ClusterID: "zzzzz",
+var testServiceURL = func() arvados.URL {
+ return arvados.URL{Host: "localhost:12345", Scheme: "http"}
+}()
+
+func testCluster(t TB) *arvados.Cluster {
+ cfg, err := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(t)).Load()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cluster.SystemRootToken = arvadostest.DataManagerToken
+ cluster.ManagementToken = arvadostest.ManagementToken
+ cluster.Collections.BlobSigning = false
+ return cluster
+}
+
+var _ = check.Suite(&HandlerSuite{})
+
+type HandlerSuite struct {
+ cluster *arvados.Cluster
+ handler *handler
+}
+
+func (s *HandlerSuite) SetUpTest(c *check.C) {
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Replication: 1, Driver: "mock"},
+ "zzzzz-nyw5e-111111111111111": {Replication: 1, Driver: "mock"},
+ }
+ s.handler = &handler{}
}
// A RequestTester represents the parameters for an HTTP request to
@@ -52,46 +85,41 @@ type RequestTester struct {
// - permissions on, authenticated request, expired locator
// - permissions on, authenticated request, signed locator, transient error from backend
//
-func TestGetHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestGetHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
- if err := vols[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
- }
+ vols := s.handler.volmgr.AllWritable()
+ err := vols[0].Put(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.IsNil)
// Create locators for testing.
// Turn on permission settings so we can generate signed locators.
- theConfig.RequireSignatures = true
- theConfig.blobSigningKey = []byte(knownKey)
- theConfig.BlobSignatureTTL.Set("5m")
+ s.cluster.Collections.BlobSigning = true
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
var (
unsignedLocator = "/" + TestHash
- validTimestamp = time.Now().Add(theConfig.BlobSignatureTTL.Duration())
+ validTimestamp = time.Now().Add(s.cluster.Collections.BlobSigningTTL.Duration())
expiredTimestamp = time.Now().Add(-time.Hour)
- signedLocator = "/" + SignLocator(TestHash, knownToken, validTimestamp)
- expiredLocator = "/" + SignLocator(TestHash, knownToken, expiredTimestamp)
+ signedLocator = "/" + SignLocator(s.cluster, TestHash, knownToken, validTimestamp)
+ expiredLocator = "/" + SignLocator(s.cluster, TestHash, knownToken, expiredTimestamp)
)
// -----------------
// Test unauthenticated request with permissions off.
- theConfig.RequireSignatures = false
+ s.cluster.Collections.BlobSigning = false
// Unauthenticated request, unsigned locator
// => OK
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "GET",
uri: unsignedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, unsigned locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, unsigned locator",
string(TestBlock),
response)
@@ -99,58 +127,58 @@ func TestGetHandler(t *testing.T) {
receivedLen := response.Header().Get("Content-Length")
expectedLen := fmt.Sprintf("%d", len(TestBlock))
if receivedLen != expectedLen {
- t.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
+ c.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
}
// ----------------
// Permissions: on.
- theConfig.RequireSignatures = true
+ s.cluster.Collections.BlobSigning = true
// Authenticated request, signed locator
// => OK
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated request, signed locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Authenticated request, signed locator", string(TestBlock), response)
receivedLen = response.Header().Get("Content-Length")
expectedLen = fmt.Sprintf("%d", len(TestBlock))
if receivedLen != expectedLen {
- t.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
+ c.Errorf("expected Content-Length %s, got %s", expectedLen, receivedLen)
}
// Authenticated request, unsigned locator
// => PermissionError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: unsignedLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t, "unsigned locator", PermissionError.HTTPCode, response)
+ ExpectStatusCode(c, "unsigned locator", PermissionError.HTTPCode, response)
// Unauthenticated request, signed locator
// => PermissionError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, signed locator",
PermissionError.HTTPCode, response)
// Authenticated request, expired locator
// => ExpiredError
- response = IssueRequest(&RequestTester{
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: expiredLocator,
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated request, expired locator",
ExpiredError.HTTPCode, response)
@@ -158,16 +186,16 @@ func TestGetHandler(t *testing.T) {
// => 503 Server busy (transient error)
// Set up the block owning volume to respond with errors
- vols[0].(*MockVolume).Bad = true
- vols[0].(*MockVolume).BadVolumeError = VolumeBusyError
- response = IssueRequest(&RequestTester{
+ vols[0].Volume.(*MockVolume).Bad = true
+ vols[0].Volume.(*MockVolume).BadVolumeError = VolumeBusyError
+ response = IssueRequest(s.handler, &RequestTester{
method: "GET",
uri: signedLocator,
apiToken: knownToken,
})
// A transient error from one volume while the other doesn't find the block
// should make the service return a 503 so that clients can retry.
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Volume backend busy",
503, response)
}
@@ -177,44 +205,42 @@ func TestGetHandler(t *testing.T) {
// - with server key, authenticated request, unsigned locator
// - with server key, unauthenticated request, unsigned locator
//
-func TestPutHandler(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// --------------
// No server key.
+ s.cluster.Collections.BlobSigningKey = ""
+
// Unauthenticated request, no server key
// => OK (unsigned response)
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, no server key", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, no server key",
TestHashPutResp, response)
// ------------------
// With a server key.
- theConfig.blobSigningKey = []byte(knownKey)
- theConfig.BlobSignatureTTL.Set("5m")
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
// When a permission key is available, the locator returned
// from an authenticated PUT request will be signed.
// Authenticated PUT, signed locator
// => OK (signed response)
- response = IssueRequest(
+ response = IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
@@ -222,76 +248,72 @@ func TestPutHandler(t *testing.T) {
apiToken: knownToken,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Authenticated PUT, signed locator, with server key",
http.StatusOK, response)
responseLocator := strings.TrimSpace(response.Body.String())
- if VerifySignature(responseLocator, knownToken) != nil {
- t.Errorf("Authenticated PUT, signed locator, with server key:\n"+
+ if VerifySignature(s.cluster, responseLocator, knownToken) != nil {
+ c.Errorf("Authenticated PUT, signed locator, with server key:\n"+
"response '%s' does not contain a valid signature",
responseLocator)
}
// Unauthenticated PUT, unsigned locator
// => OK
- response = IssueRequest(
+ response = IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated PUT, unsigned locator, with server key",
http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated PUT, unsigned locator, with server key",
TestHashPutResp, response)
}
-func TestPutAndDeleteSkipReadonlyVolumes(t *testing.T) {
- defer teardown()
- theConfig.systemAuthToken = "fake-data-manager-token"
- vols := []*MockVolume{CreateMockVolume(), CreateMockVolume()}
- vols[0].Readonly = true
- KeepVM = MakeRRVolumeManager([]Volume{vols[0], vols[1]})
- defer KeepVM.Close()
- IssueRequest(
+func (s *HandlerSuite) TestPutAndDeleteSkipReadonlyVolumes(c *check.C) {
+ s.cluster.Volumes["zzzzz-nyw5e-000000000000000"] = arvados.Volume{Driver: "mock", ReadOnly: true}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+
+ s.cluster.SystemRootToken = "fake-data-manager-token"
+ IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
- defer func(orig bool) {
- theConfig.EnableDelete = orig
- }(theConfig.EnableDelete)
- theConfig.EnableDelete = true
- IssueRequest(
+
+ s.cluster.Collections.BlobTrash = true
+ IssueRequest(s.handler,
&RequestTester{
method: "DELETE",
uri: "/" + TestHash,
requestBody: TestBlock,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
})
type expect struct {
- volnum int
+ volid string
method string
callcount int
}
for _, e := range []expect{
- {0, "Get", 0},
- {0, "Compare", 0},
- {0, "Touch", 0},
- {0, "Put", 0},
- {0, "Delete", 0},
- {1, "Get", 0},
- {1, "Compare", 1},
- {1, "Touch", 1},
- {1, "Put", 1},
- {1, "Delete", 1},
+ {"zzzzz-nyw5e-000000000000000", "Get", 0},
+ {"zzzzz-nyw5e-000000000000000", "Compare", 0},
+ {"zzzzz-nyw5e-000000000000000", "Touch", 0},
+ {"zzzzz-nyw5e-000000000000000", "Put", 0},
+ {"zzzzz-nyw5e-000000000000000", "Delete", 0},
+ {"zzzzz-nyw5e-111111111111111", "Get", 0},
+ {"zzzzz-nyw5e-111111111111111", "Compare", 1},
+ {"zzzzz-nyw5e-111111111111111", "Touch", 1},
+ {"zzzzz-nyw5e-111111111111111", "Put", 1},
+ {"zzzzz-nyw5e-111111111111111", "Delete", 1},
} {
- if calls := vols[e.volnum].CallCount(e.method); calls != e.callcount {
- t.Errorf("Got %d %s() on vol %d, expect %d", calls, e.method, e.volnum, e.callcount)
+ if calls := s.handler.volmgr.mountMap[e.volid].Volume.(*MockVolume).CallCount(e.method); calls != e.callcount {
+ c.Errorf("Got %d %s() on vol %s, expect %d", calls, e.method, e.volid, e.callcount)
}
}
}
@@ -307,22 +329,18 @@ func TestPutAndDeleteSkipReadonlyVolumes(t *testing.T) {
// The only /index requests that should succeed are those issued by the
// superuser. They should pass regardless of the value of RequireSignatures.
//
-func TestIndexHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestIndexHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- // Set up Keep volumes and populate them.
// Include multiple blocks on different volumes, and
// some metadata files (which should be omitted from index listings)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
vols[1].Put(context.Background(), TestHash2, TestBlock2)
vols[0].Put(context.Background(), TestHash+".meta", []byte("metadata"))
vols[1].Put(context.Background(), TestHash2+".meta", []byte("metadata"))
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
unauthenticatedReq := &RequestTester{
method: "GET",
@@ -336,7 +354,7 @@ func TestIndexHandler(t *testing.T) {
superuserReq := &RequestTester{
method: "GET",
uri: "/index",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
unauthPrefixReq := &RequestTester{
method: "GET",
@@ -350,76 +368,76 @@ func TestIndexHandler(t *testing.T) {
superuserPrefixReq := &RequestTester{
method: "GET",
uri: "/index/" + TestHash[0:3],
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserNoSuchPrefixReq := &RequestTester{
method: "GET",
uri: "/index/abcd",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserInvalidPrefixReq := &RequestTester{
method: "GET",
uri: "/index/xyz",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
// -------------------------------------------------------------
// Only the superuser should be allowed to issue /index requests.
// ---------------------------
- // RequireSignatures enabled
+ // BlobSigning enabled
// This setting should not affect tests passing.
- theConfig.RequireSignatures = true
+ s.cluster.Collections.BlobSigning = true
// unauthenticated /index request
// => UnauthorizedError
- response := IssueRequest(unauthenticatedReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, unauthenticatedReq)
+ ExpectStatusCode(c,
"RequireSignatures on, unauthenticated request",
UnauthorizedError.HTTPCode,
response)
// unauthenticated /index/prefix request
// => UnauthorizedError
- response = IssueRequest(unauthPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, unauthPrefixReq)
+ ExpectStatusCode(c,
"permissions on, unauthenticated /index/prefix request",
UnauthorizedError.HTTPCode,
response)
// authenticated /index request, non-superuser
// => UnauthorizedError
- response = IssueRequest(authenticatedReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, authenticatedReq)
+ ExpectStatusCode(c,
"permissions on, authenticated request, non-superuser",
UnauthorizedError.HTTPCode,
response)
// authenticated /index/prefix request, non-superuser
// => UnauthorizedError
- response = IssueRequest(authPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, authPrefixReq)
+ ExpectStatusCode(c,
"permissions on, authenticated /index/prefix request, non-superuser",
UnauthorizedError.HTTPCode,
response)
// superuser /index request
// => OK
- response = IssueRequest(superuserReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
// ----------------------------
- // RequireSignatures disabled
+ // BlobSigning disabled
// Valid Request should still pass.
- theConfig.RequireSignatures = false
+ s.cluster.Collections.BlobSigning = false
// superuser /index request
// => OK
- response = IssueRequest(superuserReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
@@ -428,15 +446,15 @@ func TestIndexHandler(t *testing.T) {
TestHash2 + `\+\d+ \d+\n\n$`
match, _ := regexp.MatchString(expected, response.Body.String())
if !match {
- t.Errorf(
+ c.Errorf(
"permissions on, superuser request: expected %s, got:\n%s",
expected, response.Body.String())
}
// superuser /index/prefix request
// => OK
- response = IssueRequest(superuserPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
@@ -444,27 +462,27 @@ func TestIndexHandler(t *testing.T) {
expected = `^` + TestHash + `\+\d+ \d+\n\n$`
match, _ = regexp.MatchString(expected, response.Body.String())
if !match {
- t.Errorf(
+ c.Errorf(
"permissions on, superuser /index/prefix request: expected %s, got:\n%s",
expected, response.Body.String())
}
// superuser /index/{no-such-prefix} request
// => OK
- response = IssueRequest(superuserNoSuchPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserNoSuchPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusOK,
response)
if "\n" != response.Body.String() {
- t.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
+ c.Errorf("Expected empty response for %s. Found %s", superuserNoSuchPrefixReq.uri, response.Body.String())
}
// superuser /index/{invalid-prefix} request
// => StatusBadRequest
- response = IssueRequest(superuserInvalidPrefixReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserInvalidPrefixReq)
+ ExpectStatusCode(c,
"permissions on, superuser request",
http.StatusBadRequest,
response)
@@ -496,27 +514,21 @@ func TestIndexHandler(t *testing.T) {
// (test for 200 OK, response with copies_deleted=0, copies_failed=1,
// confirm block not deleted)
//
-func TestDeleteHandler(t *testing.T) {
- defer teardown()
-
- // Set up Keep volumes and populate them.
- // Include multiple blocks on different volumes, and
- // some metadata files (which should be omitted from index listings)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestDeleteHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
// Explicitly set the BlobSignatureTTL to 0 for these
// tests, to ensure the MockVolume deletes the blocks
// even though they have just been created.
- theConfig.BlobSignatureTTL = arvados.Duration(0)
+ s.cluster.Collections.BlobSigningTTL = arvados.Duration(0)
var userToken = "NOT DATA MANAGER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
- theConfig.EnableDelete = true
+ s.cluster.Collections.BlobTrash = true
unauthReq := &RequestTester{
method: "DELETE",
@@ -532,26 +544,26 @@ func TestDeleteHandler(t *testing.T) {
superuserExistingBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
superuserNonexistentBlockReq := &RequestTester{
method: "DELETE",
uri: "/" + TestHash2,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
// Unauthenticated request returns PermissionError.
var response *httptest.ResponseRecorder
- response = IssueRequest(unauthReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, unauthReq)
+ ExpectStatusCode(c,
"unauthenticated request",
PermissionError.HTTPCode,
response)
// Authenticated non-admin request returns PermissionError.
- response = IssueRequest(userReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, userReq)
+ ExpectStatusCode(c,
"authenticated non-admin request",
PermissionError.HTTPCode,
response)
@@ -563,24 +575,24 @@ func TestDeleteHandler(t *testing.T) {
}
var responseDc, expectedDc deletecounter
- response = IssueRequest(superuserNonexistentBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserNonexistentBlockReq)
+ ExpectStatusCode(c,
"data manager request, nonexistent block",
http.StatusNotFound,
response)
- // Authenticated admin request for existing block while EnableDelete is false.
- theConfig.EnableDelete = false
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ // Authenticated admin request for existing block while BlobTrash is false.
+ s.cluster.Collections.BlobTrash = false
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"authenticated request, existing block, method disabled",
MethodDisabledError.HTTPCode,
response)
- theConfig.EnableDelete = true
+ s.cluster.Collections.BlobTrash = true
// Authenticated admin request for existing block.
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"data manager request, existing block",
http.StatusOK,
response)
@@ -588,7 +600,7 @@ func TestDeleteHandler(t *testing.T) {
expectedDc = deletecounter{1, 0}
json.NewDecoder(response.Body).Decode(&responseDc)
if responseDc != expectedDc {
- t.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
+ c.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
expectedDc, responseDc)
}
// Confirm the block has been deleted
@@ -596,16 +608,16 @@ func TestDeleteHandler(t *testing.T) {
_, err := vols[0].Get(context.Background(), TestHash, buf)
var blockDeleted = os.IsNotExist(err)
if !blockDeleted {
- t.Error("superuserExistingBlockReq: block not deleted")
+ c.Error("superuserExistingBlockReq: block not deleted")
}
// A DELETE request on a block newer than BlobSignatureTTL
// should return success but leave the block on the volume.
vols[0].Put(context.Background(), TestHash, TestBlock)
- theConfig.BlobSignatureTTL = arvados.Duration(time.Hour)
+ s.cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour)
- response = IssueRequest(superuserExistingBlockReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, superuserExistingBlockReq)
+ ExpectStatusCode(c,
"data manager request, existing block",
http.StatusOK,
response)
@@ -613,13 +625,13 @@ func TestDeleteHandler(t *testing.T) {
expectedDc = deletecounter{1, 0}
json.NewDecoder(response.Body).Decode(&responseDc)
if responseDc != expectedDc {
- t.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
+ c.Errorf("superuserExistingBlockReq\nexpected: %+v\nreceived: %+v",
expectedDc, responseDc)
}
// Confirm the block has NOT been deleted.
_, err = vols[0].Get(context.Background(), TestHash, buf)
if err != nil {
- t.Errorf("testing delete on new block: %s\n", err)
+ c.Errorf("testing delete on new block: %s\n", err)
}
}
@@ -650,29 +662,33 @@ func TestDeleteHandler(t *testing.T) {
// pull list simultaneously. Make sure that none of them return 400
// Bad Request and that pullq.GetList() returns a valid list.
//
-func TestPullHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestPullHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- var userToken = "USER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ // Replace the router's pullq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ pullq := NewWorkQueue()
+ s.handler.Handler.(*router).pullq = pullq
- pullq = NewWorkQueue()
+ var userToken = "USER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
goodJSON := []byte(`[
{
- "locator":"locator_with_two_servers",
+ "locator":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+12345",
"servers":[
- "server1",
- "server2"
+ "http://server1",
+ "http://server2"
]
},
{
- "locator":"locator_with_no_servers",
+ "locator":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+12345",
"servers":[]
},
{
- "locator":"",
- "servers":["empty_locator"]
+ "locator":"cccccccccccccccccccccccccccccccc+12345",
+ "servers":["http://server1"]
}
]`)
@@ -699,34 +715,39 @@ func TestPullHandler(t *testing.T) {
},
{
"Valid pull request from the data manager",
- RequestTester{"/pull", theConfig.systemAuthToken, "PUT", goodJSON},
+ RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 pull requests\n",
},
{
"Invalid pull request from the data manager",
- RequestTester{"/pull", theConfig.systemAuthToken, "PUT", badJSON},
+ RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
}
for _, tst := range testcases {
- response := IssueRequest(&tst.req)
- ExpectStatusCode(t, tst.name, tst.responseCode, response)
- ExpectBody(t, tst.name, tst.responseBody, response)
+ response := IssueRequest(s.handler, &tst.req)
+ ExpectStatusCode(c, tst.name, tst.responseCode, response)
+ ExpectBody(c, tst.name, tst.responseBody, response)
}
// The Keep pull manager should have received one good list with 3
// requests on it.
for i := 0; i < 3; i++ {
- item := <-pullq.NextItem
+ var item interface{}
+ select {
+ case item = <-pullq.NextItem:
+ case <-time.After(time.Second):
+ c.Error("timed out")
+ }
if _, ok := item.(PullRequest); !ok {
- t.Errorf("item %v could not be parsed as a PullRequest", item)
+ c.Errorf("item %v could not be parsed as a PullRequest", item)
}
}
- expectChannelEmpty(t, pullq.NextItem)
+ expectChannelEmpty(c, pullq.NextItem)
}
// TestTrashHandler
@@ -756,13 +777,16 @@ func TestPullHandler(t *testing.T) {
// pull list simultaneously. Make sure that none of them return 400
// Bad Request and that replica.Dump() returns a valid list.
//
-func TestTrashHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestTrashHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ // Replace the router's trashq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ trashq := NewWorkQueue()
+ s.handler.Handler.(*router).trashq = trashq
var userToken = "USER TOKEN"
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
-
- trashq = NewWorkQueue()
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
goodJSON := []byte(`[
{
@@ -803,22 +827,22 @@ func TestTrashHandler(t *testing.T) {
},
{
"Valid trash list from the data manager",
- RequestTester{"/trash", theConfig.systemAuthToken, "PUT", goodJSON},
+ RequestTester{"/trash", s.cluster.SystemRootToken, "PUT", goodJSON},
http.StatusOK,
"Received 3 trash requests\n",
},
{
"Invalid trash list from the data manager",
- RequestTester{"/trash", theConfig.systemAuthToken, "PUT", badJSON},
+ RequestTester{"/trash", s.cluster.SystemRootToken, "PUT", badJSON},
http.StatusBadRequest,
"",
},
}
for _, tst := range testcases {
- response := IssueRequest(&tst.req)
- ExpectStatusCode(t, tst.name, tst.responseCode, response)
- ExpectBody(t, tst.name, tst.responseBody, response)
+ response := IssueRequest(s.handler, &tst.req)
+ ExpectStatusCode(c, tst.name, tst.responseCode, response)
+ ExpectBody(c, tst.name, tst.responseBody, response)
}
// The trash collector should have received one good list with 3
@@ -826,11 +850,11 @@ func TestTrashHandler(t *testing.T) {
for i := 0; i < 3; i++ {
item := <-trashq.NextItem
if _, ok := item.(TrashRequest); !ok {
- t.Errorf("item %v could not be parsed as a TrashRequest", item)
+ c.Errorf("item %v could not be parsed as a TrashRequest", item)
}
}
- expectChannelEmpty(t, trashq.NextItem)
+ expectChannelEmpty(c, trashq.NextItem)
}
// ====================
@@ -839,75 +863,71 @@ func TestTrashHandler(t *testing.T) {
// IssueTestRequest executes an HTTP request described by rt, to a
// REST router. It returns the HTTP response to the request.
-func IssueRequest(rt *RequestTester) *httptest.ResponseRecorder {
+func IssueRequest(handler http.Handler, rt *RequestTester) *httptest.ResponseRecorder {
response := httptest.NewRecorder()
body := bytes.NewReader(rt.requestBody)
req, _ := http.NewRequest(rt.method, rt.uri, body)
if rt.apiToken != "" {
req.Header.Set("Authorization", "OAuth2 "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter(testCluster, prometheus.NewRegistry())
- loggingRouter.ServeHTTP(response, req)
+ handler.ServeHTTP(response, req)
return response
}
-func IssueHealthCheckRequest(rt *RequestTester) *httptest.ResponseRecorder {
+func IssueHealthCheckRequest(handler http.Handler, rt *RequestTester) *httptest.ResponseRecorder {
response := httptest.NewRecorder()
body := bytes.NewReader(rt.requestBody)
req, _ := http.NewRequest(rt.method, rt.uri, body)
if rt.apiToken != "" {
req.Header.Set("Authorization", "Bearer "+rt.apiToken)
}
- loggingRouter := MakeRESTRouter(testCluster, prometheus.NewRegistry())
- loggingRouter.ServeHTTP(response, req)
+ handler.ServeHTTP(response, req)
return response
}
// ExpectStatusCode checks whether a response has the specified status code,
// and reports a test failure if not.
func ExpectStatusCode(
- t *testing.T,
+ c *check.C,
testname string,
expectedStatus int,
response *httptest.ResponseRecorder) {
if response.Code != expectedStatus {
- t.Errorf("%s: expected status %d, got %+v",
+ c.Errorf("%s: expected status %d, got %+v",
testname, expectedStatus, response)
}
}
func ExpectBody(
- t *testing.T,
+ c *check.C,
testname string,
expectedBody string,
response *httptest.ResponseRecorder) {
if expectedBody != "" && response.Body.String() != expectedBody {
- t.Errorf("%s: expected response body '%s', got %+v",
+ c.Errorf("%s: expected response body '%s', got %+v",
testname, expectedBody, response)
}
}
// See #7121
-func TestPutNeedsOnlyOneBuffer(t *testing.T) {
- defer teardown()
- KeepVM = MakeTestVolumeManager(1)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutNeedsOnlyOneBuffer(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
defer func(orig *bufferPool) {
bufs = orig
}(bufs)
- bufs = newBufferPool(1, BlockSize)
+ bufs = newBufferPool(ctxlog.TestLogger(c), 1, BlockSize)
ok := make(chan struct{})
go func() {
for i := 0; i < 2; i++ {
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"TestPutNeedsOnlyOneBuffer", http.StatusOK, response)
}
ok <- struct{}{}
@@ -916,34 +936,30 @@ func TestPutNeedsOnlyOneBuffer(t *testing.T) {
select {
case <-ok:
case <-time.After(time.Second):
- t.Fatal("PUT deadlocks with MaxBuffers==1")
+ c.Fatal("PUT deadlocks with MaxBuffers==1")
}
}
// Invoke the PutBlockHandler a bunch of times to test for bufferpool resource
// leak.
-func TestPutHandlerNoBufferleak(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestPutHandlerNoBufferleak(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
ok := make(chan bool)
go func() {
- for i := 0; i < theConfig.MaxBuffers+1; i++ {
+ for i := 0; i < s.cluster.API.MaxKeepBlockBuffers+1; i++ {
// Unauthenticated request, no server key
// => OK (unsigned response)
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "PUT",
uri: unsignedLocator,
requestBody: TestBlock,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"TestPutHandlerBufferleak", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"TestPutHandlerBufferleak",
TestHashPutResp, response)
}
@@ -952,7 +968,7 @@ func TestPutHandlerNoBufferleak(t *testing.T) {
select {
case <-time.After(20 * time.Second):
// If the buffer pool leaks, the test goroutine hangs.
- t.Fatal("test did not finish, assuming pool leaked")
+ c.Fatal("test did not finish, assuming pool leaked")
case <-ok:
}
}
@@ -966,23 +982,18 @@ func (r *notifyingResponseRecorder) CloseNotify() <-chan bool {
return r.closer
}
-func TestGetHandlerClientDisconnect(t *testing.T) {
- defer func(was bool) {
- theConfig.RequireSignatures = was
- }(theConfig.RequireSignatures)
- theConfig.RequireSignatures = false
+func (s *HandlerSuite) TestGetHandlerClientDisconnect(c *check.C) {
+ s.cluster.Collections.BlobSigning = false
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
defer func(orig *bufferPool) {
bufs = orig
}(bufs)
- bufs = newBufferPool(1, BlockSize)
+ bufs = newBufferPool(ctxlog.TestLogger(c), 1, BlockSize)
defer bufs.Put(bufs.Get(BlockSize))
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- if err := KeepVM.AllWritable()[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
+ if err := s.handler.volmgr.AllWritable()[0].Put(context.Background(), TestHash, TestBlock); err != nil {
+ c.Error(err)
}
resp := ¬ifyingResponseRecorder{
@@ -990,7 +1001,7 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
closer: make(chan bool, 1),
}
if _, ok := http.ResponseWriter(resp).(http.CloseNotifier); !ok {
- t.Fatal("notifyingResponseRecorder is broken")
+ c.Fatal("notifyingResponseRecorder is broken")
}
// If anyone asks, the client has disconnected.
resp.closer <- true
@@ -998,52 +1009,48 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
ok := make(chan struct{})
go func() {
req, _ := http.NewRequest("GET", fmt.Sprintf("/%s+%d", TestHash, len(TestBlock)), nil)
- MakeRESTRouter(testCluster, prometheus.NewRegistry()).ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
ok <- struct{}{}
}()
select {
case <-time.After(20 * time.Second):
- t.Fatal("request took >20s, close notifier must be broken")
+ c.Fatal("request took >20s, close notifier must be broken")
case <-ok:
}
- ExpectStatusCode(t, "client disconnect", http.StatusServiceUnavailable, resp.ResponseRecorder)
- for i, v := range KeepVM.AllWritable() {
- if calls := v.(*MockVolume).called["GET"]; calls != 0 {
- t.Errorf("volume %d got %d calls, expected 0", i, calls)
+ ExpectStatusCode(c, "client disconnect", http.StatusServiceUnavailable, resp.ResponseRecorder)
+ for i, v := range s.handler.volmgr.AllWritable() {
+ if calls := v.Volume.(*MockVolume).called["GET"]; calls != 0 {
+ c.Errorf("volume %d got %d calls, expected 0", i, calls)
}
}
}
// Invoke the GetBlockHandler a bunch of times to test for bufferpool resource
// leak.
-func TestGetHandlerNoBufferLeak(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) TestGetHandlerNoBufferLeak(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
if err := vols[0].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
+ c.Error(err)
}
ok := make(chan bool)
go func() {
- for i := 0; i < theConfig.MaxBuffers+1; i++ {
+ for i := 0; i < s.cluster.API.MaxKeepBlockBuffers+1; i++ {
// Unauthenticated request, unsigned locator
// => OK
unsignedLocator := "/" + TestHash
- response := IssueRequest(
+ response := IssueRequest(s.handler,
&RequestTester{
method: "GET",
uri: unsignedLocator,
})
- ExpectStatusCode(t,
+ ExpectStatusCode(c,
"Unauthenticated request, unsigned locator", http.StatusOK, response)
- ExpectBody(t,
+ ExpectBody(c,
"Unauthenticated request, unsigned locator",
string(TestBlock),
response)
@@ -1053,45 +1060,41 @@ func TestGetHandlerNoBufferLeak(t *testing.T) {
select {
case <-time.After(20 * time.Second):
// If the buffer pool leaks, the test goroutine hangs.
- t.Fatal("test did not finish, assuming pool leaked")
+ c.Fatal("test did not finish, assuming pool leaked")
case <-ok:
}
}
-func TestPutReplicationHeader(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestPutReplicationHeader(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- resp := IssueRequest(&RequestTester{
+ resp := IssueRequest(s.handler, &RequestTester{
method: "PUT",
uri: "/" + TestHash,
requestBody: TestBlock,
})
if r := resp.Header().Get("X-Keep-Replicas-Stored"); r != "1" {
- t.Errorf("Got X-Keep-Replicas-Stored: %q, expected %q", r, "1")
+ c.Logf("%#v", resp)
+ c.Errorf("Got X-Keep-Replicas-Stored: %q, expected %q", r, "1")
}
}
-func TestUntrashHandler(t *testing.T) {
- defer teardown()
+func (s *HandlerSuite) TestUntrashHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// Set up Keep volumes
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
- vols := KeepVM.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+ s.cluster.SystemRootToken = "DATA MANAGER TOKEN"
// unauthenticatedReq => UnauthorizedError
unauthenticatedReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
}
- response := IssueRequest(unauthenticatedReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, unauthenticatedReq)
+ ExpectStatusCode(c,
"Unauthenticated request",
UnauthorizedError.HTTPCode,
response)
@@ -1103,8 +1106,8 @@ func TestUntrashHandler(t *testing.T) {
apiToken: knownToken,
}
- response = IssueRequest(notDataManagerReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, notDataManagerReq)
+ ExpectStatusCode(c,
"Non-datamanager token",
UnauthorizedError.HTTPCode,
response)
@@ -1113,10 +1116,10 @@ func TestUntrashHandler(t *testing.T) {
datamanagerWithBadHashReq := &RequestTester{
method: "PUT",
uri: "/untrash/thisisnotalocator",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerWithBadHashReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerWithBadHashReq)
+ ExpectStatusCode(c,
"Bad locator in untrash request",
http.StatusBadRequest,
response)
@@ -1125,10 +1128,10 @@ func TestUntrashHandler(t *testing.T) {
datamanagerWrongMethodReq := &RequestTester{
method: "GET",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerWrongMethodReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerWrongMethodReq)
+ ExpectStatusCode(c,
"Only PUT method is supported for untrash",
http.StatusMethodNotAllowed,
response)
@@ -1137,60 +1140,57 @@ func TestUntrashHandler(t *testing.T) {
datamanagerReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response = IssueRequest(datamanagerReq)
- ExpectStatusCode(t,
+ response = IssueRequest(s.handler, datamanagerReq)
+ ExpectStatusCode(c,
"",
http.StatusOK,
response)
expected := "Successfully untrashed on: [MockVolume],[MockVolume]"
if response.Body.String() != expected {
- t.Errorf(
+ c.Errorf(
"Untrash response mismatched: expected %s, got:\n%s",
expected, response.Body.String())
}
}
-func TestUntrashHandlerWithNoWritableVolumes(t *testing.T) {
- defer teardown()
-
- // Set up readonly Keep volumes
- vols := []*MockVolume{CreateMockVolume(), CreateMockVolume()}
- vols[0].Readonly = true
- vols[1].Readonly = true
- KeepVM = MakeRRVolumeManager([]Volume{vols[0], vols[1]})
- defer KeepVM.Close()
-
- theConfig.systemAuthToken = "DATA MANAGER TOKEN"
+func (s *HandlerSuite) TestUntrashHandlerWithNoWritableVolumes(c *check.C) {
+ // Change all volumes to read-only
+ for uuid, v := range s.cluster.Volumes {
+ v.ReadOnly = true
+ s.cluster.Volumes[uuid] = v
+ }
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// datamanagerReq => StatusOK
datamanagerReq := &RequestTester{
method: "PUT",
uri: "/untrash/" + TestHash,
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
}
- response := IssueRequest(datamanagerReq)
- ExpectStatusCode(t,
+ response := IssueRequest(s.handler, datamanagerReq)
+ ExpectStatusCode(c,
"No writable volumes",
http.StatusNotFound,
response)
}
-func TestHealthCheckPing(t *testing.T) {
- theConfig.ManagementToken = arvadostest.ManagementToken
+func (s *HandlerSuite) TestHealthCheckPing(c *check.C) {
+ s.cluster.ManagementToken = arvadostest.ManagementToken
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
pingReq := &RequestTester{
method: "GET",
uri: "/_health/ping",
apiToken: arvadostest.ManagementToken,
}
- response := IssueHealthCheckRequest(pingReq)
- ExpectStatusCode(t,
+ response := IssueHealthCheckRequest(s.handler, pingReq)
+ ExpectStatusCode(c,
"",
http.StatusOK,
response)
want := `{"health":"OK"}`
if !strings.Contains(response.Body.String(), want) {
- t.Errorf("expected response to include %s: got %s", want, response.Body.String())
+ c.Errorf("expected response to include %s: got %s", want, response.Body.String())
}
}
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 72088e2b5..86504422d 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,23 +27,31 @@ import (
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
type router struct {
*mux.Router
- limiter httpserver.RequestCounter
cluster *arvados.Cluster
+ logger logrus.FieldLogger
remoteProxy remoteProxy
metrics *nodeMetrics
+ volmgr *RRVolumeManager
+ pullq *WorkQueue
+ trashq *WorkQueue
}
// MakeRESTRouter returns a new router that forwards all Keep requests
// to the appropriate handlers.
-func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
+func MakeRESTRouter(ctx context.Context, cluster *arvados.Cluster, reg *prometheus.Registry, volmgr *RRVolumeManager, pullq, trashq *WorkQueue) http.Handler {
rtr := &router{
Router: mux.NewRouter(),
cluster: cluster,
+ logger: ctxlog.FromContext(ctx),
metrics: &nodeMetrics{reg: reg},
+ volmgr: volmgr,
+ pullq: pullq,
+ trashq: trashq,
}
rtr.HandleFunc(
@@ -52,12 +61,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
rtr.handleGET).Methods("GET", "HEAD")
rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handlePUT).Methods("PUT")
- rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleDELETE).Methods("DELETE")
// List all blocks stored here. Privileged client only.
- rtr.HandleFunc(`/index`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index`, rtr.handleIndex).Methods("GET", "HEAD")
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
- rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
// Internals/debugging info (runtime.MemStats)
rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
@@ -67,20 +76,20 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// List mounts: UUID, readonly, tier, device ID, ...
rtr.HandleFunc(`/mounts`, rtr.MountsHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.IndexHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.IndexHandler).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.handleIndex).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.handleIndex).Methods("GET")
// Replace the current pull queue.
- rtr.HandleFunc(`/pull`, PullHandler).Methods("PUT")
+ rtr.HandleFunc(`/pull`, rtr.handlePull).Methods("PUT")
// Replace the current trash queue.
- rtr.HandleFunc(`/trash`, TrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/trash`, rtr.handleTrash).Methods("PUT")
// Untrash moves blocks from trash back into store
- rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, UntrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, rtr.handleUntrash).Methods("PUT")
rtr.Handle("/_health/{check}", &health.Handler{
- Token: theConfig.ManagementToken,
+ Token: cluster.ManagementToken,
Prefix: "/_health/",
}).Methods("GET")
@@ -88,17 +97,11 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// 400 Bad Request.
rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
- rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
rtr.metrics.setupBufferPoolMetrics(bufs)
- rtr.metrics.setupWorkQueueMetrics(pullq, "pull")
- rtr.metrics.setupWorkQueueMetrics(trashq, "trash")
- rtr.metrics.setupRequestMetrics(rtr.limiter)
-
- instrumented := httpserver.Instrument(rtr.metrics.reg, log,
- httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), log),
- httpserver.AddRequestIDs(httpserver.LogRequests(rtr.limiter))))
- return instrumented.ServeAPI(theConfig.ManagementToken, instrumented)
+ rtr.metrics.setupWorkQueueMetrics(rtr.pullq, "pull")
+ rtr.metrics.setupWorkQueueMetrics(rtr.trashq, "trash")
+
+ return rtr
}
// BadRequestHandler is a HandleFunc to address bad requests.
@@ -112,13 +115,13 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
locator := req.URL.Path[1:]
if strings.Contains(locator, "+R") && !strings.Contains(locator, "+A") {
- rtr.remoteProxy.Get(ctx, resp, req, rtr.cluster)
+ rtr.remoteProxy.Get(ctx, resp, req, rtr.cluster, rtr.volmgr)
return
}
- if theConfig.RequireSignatures {
+ if rtr.cluster.Collections.BlobSigning {
locator := req.URL.Path[1:] // strip leading slash
- if err := VerifySignature(locator, GetAPIToken(req)); err != nil {
+ if err := VerifySignature(rtr.cluster, locator, GetAPIToken(req)); err != nil {
http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
return
}
@@ -138,7 +141,7 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
}
defer bufs.Put(buf)
- size, err := GetBlock(ctx, mux.Vars(req)["hash"], buf, resp)
+ size, err := GetBlock(ctx, rtr.volmgr, mux.Vars(req)["hash"], buf, resp)
if err != nil {
code := http.StatusInternalServerError
if err, ok := err.(*KeepError); ok {
@@ -160,7 +163,6 @@ func contextForResponse(parent context.Context, resp http.ResponseWriter) (conte
go func(c <-chan bool) {
select {
case <-c:
- theConfig.debugLogf("cancel context")
cancel()
case <-ctx.Done():
}
@@ -210,7 +212,7 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
return
}
- if len(KeepVM.AllWritable()) == 0 {
+ if len(rtr.volmgr.AllWritable()) == 0 {
http.Error(resp, FullError.Error(), FullError.HTTPCode)
return
}
@@ -228,7 +230,7 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
return
}
- replication, err := PutBlock(ctx, buf, hash)
+ replication, err := PutBlock(ctx, rtr.volmgr, buf, hash)
bufs.Put(buf)
if err != nil {
@@ -244,9 +246,9 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// return it to the client.
returnHash := fmt.Sprintf("%s+%d", hash, req.ContentLength)
apiToken := GetAPIToken(req)
- if theConfig.blobSigningKey != nil && apiToken != "" {
- expiry := time.Now().Add(theConfig.BlobSignatureTTL.Duration())
- returnHash = SignLocator(returnHash, apiToken, expiry)
+ if rtr.cluster.Collections.BlobSigningKey != "" && apiToken != "" {
+ expiry := time.Now().Add(rtr.cluster.Collections.BlobSigningTTL.Duration())
+ returnHash = SignLocator(rtr.cluster, returnHash, apiToken, expiry)
}
resp.Header().Set("X-Keep-Replicas-Stored", strconv.Itoa(replication))
resp.Write([]byte(returnHash + "\n"))
@@ -254,8 +256,8 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// IndexHandler responds to "/index", "/index/{prefix}", and
// "/mounts/{uuid}/blocks" requests.
-func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
- if !IsSystemAuth(GetAPIToken(req)) {
+func (rtr *router) handleIndex(resp http.ResponseWriter, req *http.Request) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -268,14 +270,14 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
uuid := mux.Vars(req)["uuid"]
- var vols []Volume
+ var vols []*VolumeMount
if uuid == "" {
- vols = KeepVM.AllReadable()
- } else if v := KeepVM.Lookup(uuid, false); v == nil {
+ vols = rtr.volmgr.AllReadable()
+ } else if mnt := rtr.volmgr.Lookup(uuid, false); mnt == nil {
http.Error(resp, "mount not found", http.StatusNotFound)
return
} else {
- vols = []Volume{v}
+ vols = []*VolumeMount{mnt}
}
for _, v := range vols {
@@ -303,9 +305,9 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
// MountsHandler responds to "GET /mounts" requests.
func (rtr *router) MountsHandler(resp http.ResponseWriter, req *http.Request) {
- err := json.NewEncoder(resp).Encode(KeepVM.Mounts())
+ err := json.NewEncoder(resp).Encode(rtr.volmgr.Mounts())
if err != nil {
- http.Error(resp, err.Error(), http.StatusInternalServerError)
+ httpserver.Error(resp, err.Error(), http.StatusInternalServerError)
}
}
@@ -368,32 +370,28 @@ func (rtr *router) StatusHandler(resp http.ResponseWriter, req *http.Request) {
// populate the given NodeStatus struct with current values.
func (rtr *router) readNodeStatus(st *NodeStatus) {
st.Version = version
- vols := KeepVM.AllReadable()
+ vols := rtr.volmgr.AllReadable()
if cap(st.Volumes) < len(vols) {
st.Volumes = make([]*volumeStatusEnt, len(vols))
}
st.Volumes = st.Volumes[:0]
for _, vol := range vols {
var internalStats interface{}
- if vol, ok := vol.(InternalStatser); ok {
+ if vol, ok := vol.Volume.(InternalStatser); ok {
internalStats = vol.InternalStats()
}
st.Volumes = append(st.Volumes, &volumeStatusEnt{
Label: vol.String(),
Status: vol.Status(),
InternalStats: internalStats,
- //VolumeStats: KeepVM.VolumeStats(vol),
+ //VolumeStats: rtr.volmgr.VolumeStats(vol),
})
}
st.BufferPool.Alloc = bufs.Alloc()
st.BufferPool.Cap = bufs.Cap()
st.BufferPool.Len = bufs.Len()
- st.PullQueue = getWorkQueueStatus(pullq)
- st.TrashQueue = getWorkQueueStatus(trashq)
- if rtr.limiter != nil {
- st.RequestsCurrent = rtr.limiter.Current()
- st.RequestsMax = rtr.limiter.Max()
- }
+ st.PullQueue = getWorkQueueStatus(rtr.pullq)
+ st.TrashQueue = getWorkQueueStatus(rtr.trashq)
}
// return a WorkQueueStatus for the given queue. If q is nil (which
@@ -407,7 +405,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
return q.Status()
}
-// DeleteHandler processes DELETE requests.
+// handleDELETE processes DELETE requests.
//
// DELETE /{hash:[0-9a-f]{32} will delete the block with the specified hash
// from all connected volumes.
@@ -418,7 +416,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// a PermissionError.
//
// Upon receiving a valid request from an authorized user,
-// DeleteHandler deletes all copies of the specified block on local
+// handleDELETE deletes all copies of the specified block on local
// writable volumes.
//
// Response format:
@@ -434,17 +432,17 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// where d and f are integers representing the number of blocks that
// were successfully and unsuccessfully deleted.
//
-func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleDELETE(resp http.ResponseWriter, req *http.Request) {
hash := mux.Vars(req)["hash"]
// Confirm that this user is an admin and has a token with unlimited scope.
var tok = GetAPIToken(req)
- if tok == "" || !CanDelete(tok) {
+ if tok == "" || !rtr.canDelete(tok) {
http.Error(resp, PermissionError.Error(), PermissionError.HTTPCode)
return
}
- if !theConfig.EnableDelete {
+ if !rtr.cluster.Collections.BlobTrash {
http.Error(resp, MethodDisabledError.Error(), MethodDisabledError.HTTPCode)
return
}
@@ -456,7 +454,7 @@ func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
Deleted int `json:"copies_deleted"`
Failed int `json:"copies_failed"`
}
- for _, vol := range KeepVM.AllWritable() {
+ for _, vol := range rtr.volmgr.AllWritable() {
if err := vol.Trash(hash); err == nil {
result.Deleted++
} else if os.IsNotExist(err) {
@@ -530,9 +528,9 @@ type PullRequest struct {
}
// PullHandler processes "PUT /pull" requests for the data manager.
-func PullHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handlePull(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -556,7 +554,7 @@ func PullHandler(resp http.ResponseWriter, req *http.Request) {
for _, p := range pr {
plist.PushBack(p)
}
- pullq.ReplaceQueue(plist)
+ rtr.pullq.ReplaceQueue(plist)
}
// TrashRequest consists of a block locator and its Mtime
@@ -569,9 +567,9 @@ type TrashRequest struct {
}
// TrashHandler processes /trash requests.
-func TrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleTrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -595,27 +593,27 @@ func TrashHandler(resp http.ResponseWriter, req *http.Request) {
for _, t := range trash {
tlist.PushBack(t)
}
- trashq.ReplaceQueue(tlist)
+ rtr.trashq.ReplaceQueue(tlist)
}
// UntrashHandler processes "PUT /untrash/{hash:[0-9a-f]{32}}" requests for the data manager.
-func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleUntrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
hash := mux.Vars(req)["hash"]
- if len(KeepVM.AllWritable()) == 0 {
+ if len(rtr.volmgr.AllWritable()) == 0 {
http.Error(resp, "No writable volumes", http.StatusNotFound)
return
}
var untrashedOn, failedOn []string
var numNotFound int
- for _, vol := range KeepVM.AllWritable() {
+ for _, vol := range rtr.volmgr.AllWritable() {
err := vol.Untrash(hash)
if os.IsNotExist(err) {
@@ -629,12 +627,12 @@ func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
}
}
- if numNotFound == len(KeepVM.AllWritable()) {
+ if numNotFound == len(rtr.volmgr.AllWritable()) {
http.Error(resp, "Block not found on any of the writable volumes", http.StatusNotFound)
return
}
- if len(failedOn) == len(KeepVM.AllWritable()) {
+ if len(failedOn) == len(rtr.volmgr.AllWritable()) {
http.Error(resp, "Failed to untrash on all writable volumes", http.StatusInternalServerError)
} else {
respBody := "Successfully untrashed on: " + strings.Join(untrashedOn, ",")
@@ -664,11 +662,11 @@ func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
// If the block found does not have the correct MD5 hash, returns
// DiskHashError.
//
-func GetBlock(ctx context.Context, hash string, buf []byte, resp http.ResponseWriter) (int, error) {
+func GetBlock(ctx context.Context, volmgr *RRVolumeManager, hash string, buf []byte, resp http.ResponseWriter) (int, error) {
// Attempt to read the requested hash from a keep volume.
errorToCaller := NotFoundError
- for _, vol := range KeepVM.AllReadable() {
+ for _, vol := range volmgr.AllReadable() {
size, err := vol.Get(ctx, hash, buf)
select {
case <-ctx.Done():
@@ -738,7 +736,7 @@ func GetBlock(ctx context.Context, hash string, buf []byte, resp http.ResponseWr
// all writes failed). The text of the error message should
// provide as much detail as possible.
//
-func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
+func PutBlock(ctx context.Context, volmgr *RRVolumeManager, block []byte, hash string) (int, error) {
// Check that BLOCK's checksum matches HASH.
blockhash := fmt.Sprintf("%x", md5.Sum(block))
if blockhash != hash {
@@ -749,7 +747,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// If we already have this data, it's intact on disk, and we
// can update its timestamp, return success. If we have
// different data with the same hash, return failure.
- if n, err := CompareAndTouch(ctx, hash, block); err == nil || err == CollisionError {
+ if n, err := CompareAndTouch(ctx, volmgr, hash, block); err == nil || err == CollisionError {
return n, err
} else if ctx.Err() != nil {
return 0, ErrClientDisconnect
@@ -757,16 +755,16 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// Choose a Keep volume to write to.
// If this volume fails, try all of the volumes in order.
- if vol := KeepVM.NextWritable(); vol != nil {
- if err := vol.Put(ctx, hash, block); err == nil {
- return vol.Replication(), nil // success!
+ if mnt := volmgr.NextWritable(); mnt != nil {
+ if err := mnt.Put(ctx, hash, block); err == nil {
+ return mnt.Replication, nil // success!
}
if ctx.Err() != nil {
return 0, ErrClientDisconnect
}
}
- writables := KeepVM.AllWritable()
+ writables := volmgr.AllWritable()
if len(writables) == 0 {
log.Print("No writable volumes.")
return 0, FullError
@@ -779,7 +777,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
return 0, ErrClientDisconnect
}
if err == nil {
- return vol.Replication(), nil // success!
+ return vol.Replication, nil // success!
}
if err != FullError {
// The volume is not full but the
@@ -803,10 +801,10 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// the relevant block's modification time in order to protect it from
// premature garbage collection. Otherwise, it returns a non-nil
// error.
-func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error) {
+func CompareAndTouch(ctx context.Context, volmgr *RRVolumeManager, hash string, buf []byte) (int, error) {
var bestErr error = NotFoundError
- for _, vol := range KeepVM.AllWritable() {
- err := vol.Compare(ctx, hash, buf)
+ for _, mnt := range volmgr.AllWritable() {
+ err := mnt.Compare(ctx, hash, buf)
if ctx.Err() != nil {
return 0, ctx.Err()
} else if err == CollisionError {
@@ -815,7 +813,7 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
// to tell which one is wanted if we have
// both, so there's no point writing it even
// on a different volume.)
- log.Printf("%s: Compare(%s): %s", vol, hash, err)
+ log.Printf("%s: Compare(%s): %s", mnt.Volume, hash, err)
return 0, err
} else if os.IsNotExist(err) {
// Block does not exist. This is the only
@@ -825,16 +823,16 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
// Couldn't open file, data is corrupt on
// disk, etc.: log this abnormal condition,
// and try the next volume.
- log.Printf("%s: Compare(%s): %s", vol, hash, err)
+ log.Printf("%s: Compare(%s): %s", mnt.Volume, hash, err)
continue
}
- if err := vol.Touch(hash); err != nil {
- log.Printf("%s: Touch %s failed: %s", vol, hash, err)
+ if err := mnt.Touch(hash); err != nil {
+ log.Printf("%s: Touch %s failed: %s", mnt.Volume, hash, err)
bestErr = err
continue
}
// Compare and Touch both worked --> done.
- return vol.Replication(), nil
+ return mnt.Replication, nil
}
return 0, bestErr
}
@@ -875,15 +873,15 @@ func IsExpired(timestampHex string) bool {
return time.Unix(ts, 0).Before(time.Now())
}
-// CanDelete returns true if the user identified by apiToken is
+// canDelete returns true if the user identified by apiToken is
// allowed to delete blocks.
-func CanDelete(apiToken string) bool {
+func (rtr *router) canDelete(apiToken string) bool {
if apiToken == "" {
return false
}
// Blocks may be deleted only when Keep has been configured with a
// data manager.
- if IsSystemAuth(apiToken) {
+ if rtr.isSystemAuth(apiToken) {
return true
}
// TODO(twp): look up apiToken with the API server
@@ -892,8 +890,8 @@ func CanDelete(apiToken string) bool {
return false
}
-// IsSystemAuth returns true if the given token is allowed to perform
+// isSystemAuth returns true if the given token is allowed to perform
// system level actions like deleting data.
-func IsSystemAuth(token string) bool {
- return token != "" && token == theConfig.systemAuthToken
+func (rtr *router) isSystemAuth(token string) bool {
+ return token != "" && token == rtr.cluster.SystemRootToken
}
diff --git a/services/keepstore/handlers_with_generic_volume_test.go b/services/keepstore/handlers_with_generic_volume_test.go
deleted file mode 100644
index 4ffb7f8f1..000000000
--- a/services/keepstore/handlers_with_generic_volume_test.go
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
-)
-
-// A TestableVolumeManagerFactory creates a volume manager with at least two TestableVolume instances.
-// The factory function, and the TestableVolume instances it returns, can use "t" to write
-// logs, fail the current test, etc.
-type TestableVolumeManagerFactory func(t TB) (*RRVolumeManager, []TestableVolume)
-
-// DoHandlersWithGenericVolumeTests runs a set of handler tests with a
-// Volume Manager comprised of TestableVolume instances.
-// It calls factory to create a volume manager with TestableVolume
-// instances for each test case, to avoid leaking state between tests.
-func DoHandlersWithGenericVolumeTests(t TB, factory TestableVolumeManagerFactory) {
- testGetBlock(t, factory, TestHash, TestBlock)
- testGetBlock(t, factory, EmptyHash, EmptyBlock)
- testPutRawBadDataGetBlock(t, factory, TestHash, TestBlock, []byte("baddata"))
- testPutRawBadDataGetBlock(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
- testPutBlock(t, factory, TestHash, TestBlock)
- testPutBlock(t, factory, EmptyHash, EmptyBlock)
- testPutBlockCorrupt(t, factory, TestHash, TestBlock, []byte("baddata"))
- testPutBlockCorrupt(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
-}
-
-// Setup RRVolumeManager with TestableVolumes
-func setupHandlersWithGenericVolumeTest(t TB, factory TestableVolumeManagerFactory) []TestableVolume {
- vm, testableVolumes := factory(t)
- KeepVM = vm
-
- for _, v := range testableVolumes {
- defer v.Teardown()
- }
- defer KeepVM.Close()
-
- return testableVolumes
-}
-
-// Put a block using PutRaw in just one volume and Get it using GetBlock
-func testGetBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put testBlock in one volume
- testableVolumes[1].PutRaw(testHash, testBlock)
-
- // Get should pass
- buf := make([]byte, len(testBlock))
- n, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error while getting block %s", err)
- }
- if bytes.Compare(buf[:n], testBlock) != 0 {
- t.Errorf("Put succeeded but Get returned %+v, expected %+v", buf[:n], testBlock)
- }
-}
-
-// Put a bad block using PutRaw and get it.
-func testPutRawBadDataGetBlock(t TB, factory TestableVolumeManagerFactory,
- testHash string, testBlock []byte, badData []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put bad data for testHash in both volumes
- testableVolumes[0].PutRaw(testHash, badData)
- testableVolumes[1].PutRaw(testHash, badData)
-
- // Get should fail
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err == nil {
- t.Fatalf("Got %+q, expected error while getting corrupt block %v", buf[:size], testHash)
- }
-}
-
-// Invoke PutBlock twice to ensure CompareAndTouch path is tested.
-func testPutBlock(t TB, factory TestableVolumeManagerFactory, testHash string, testBlock []byte) {
- setupHandlersWithGenericVolumeTest(t, factory)
-
- // PutBlock
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock: %s", err)
- }
-
- // Check that PutBlock succeeds again even after CompareAndTouch
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock: %s", err)
- }
-
- // Check that PutBlock stored the data as expected
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
- } else if bytes.Compare(buf[:size], testBlock) != 0 {
- t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf[:size])
- }
-}
-
-// Put a bad block using PutRaw, overwrite it using PutBlock and get it.
-func testPutBlockCorrupt(t TB, factory TestableVolumeManagerFactory,
- testHash string, testBlock []byte, badData []byte) {
- testableVolumes := setupHandlersWithGenericVolumeTest(t, factory)
-
- // Put bad data for testHash in both volumes
- testableVolumes[0].PutRaw(testHash, badData)
- testableVolumes[1].PutRaw(testHash, badData)
-
- // Check that PutBlock with good data succeeds
- if _, err := PutBlock(context.Background(), testBlock, testHash); err != nil {
- t.Fatalf("Error during PutBlock for %q: %s", testHash, err)
- }
-
- // Put succeeded and overwrote the badData in one volume,
- // and Get should return the testBlock now, ignoring the bad data.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testHash, buf, nil)
- if err != nil {
- t.Fatalf("Error during GetBlock for %q: %s", testHash, err)
- } else if bytes.Compare(buf[:size], testBlock) != 0 {
- t.Errorf("Get response incorrect. Expected %q; found %q", testBlock, buf[:size])
- }
-}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index fcbdddacb..f2973b586 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -5,24 +5,9 @@
package main
import (
- "flag"
- "fmt"
- "net"
- "os"
- "os/signal"
- "syscall"
"time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/config"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "github.com/coreos/go-systemd/daemon"
- "github.com/prometheus/client_golang/prometheus"
)
-var version = "dev"
-
// A Keep "block" is 64MB.
const BlockSize = 64 * 1024 * 1024
@@ -30,9 +15,6 @@ const BlockSize = 64 * 1024 * 1024
// in order to permit writes.
const MinFreeKilobytes = BlockSize / 1024
-// ProcMounts /proc/mounts
-var ProcMounts = "/proc/mounts"
-
var bufs *bufferPool
// KeepError types.
@@ -65,184 +47,11 @@ func (e *KeepError) Error() string {
return e.ErrMsg
}
-// ========================
-// Internal data structures
-//
-// These global variables are used by multiple parts of the
-// program. They are good candidates for moving into their own
-// packages.
-
-// The Keep VolumeManager maintains a list of available volumes.
-// Initialized by the --volumes flag (or by FindKeepVolumes).
-var KeepVM VolumeManager
-
-// The pull list manager and trash queue are threadsafe queues which
-// support atomic update operations. The PullHandler and TrashHandler
-// store results from Data Manager /pull and /trash requests here.
-//
-// See the Keep and Data Manager design documents for more details:
-// https://arvados.org/projects/arvados/wiki/Keep_Design_Doc
-// https://arvados.org/projects/arvados/wiki/Data_Manager_Design_Doc
-//
-var pullq *WorkQueue
-var trashq *WorkQueue
-
-func main() {
- deprecated.beforeFlagParse(theConfig)
-
- dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flag.Bool("version", false, "Print version information and exit.")
-
- defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
- var configPath string
- flag.StringVar(
- &configPath,
- "config",
- defaultConfigPath,
- "YAML or JSON configuration file `path`")
- flag.Usage = usage
- flag.Parse()
-
- // Print version information if requested
- if *getVersion {
- fmt.Printf("keepstore %s\n", version)
- return
- }
-
- deprecated.afterFlagParse(theConfig)
-
- err := config.LoadFile(theConfig, configPath)
- if err != nil && (!os.IsNotExist(err) || configPath != defaultConfigPath) {
- log.Fatal(err)
- }
-
- if *dumpConfig {
- log.Fatal(config.DumpAndExit(theConfig))
- }
-
- log.Printf("keepstore %s started", version)
-
- metricsRegistry := prometheus.NewRegistry()
-
- err = theConfig.Start(metricsRegistry)
- if err != nil {
- log.Fatal(err)
- }
-
- if pidfile := theConfig.PIDFile; pidfile != "" {
- f, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0777)
- if err != nil {
- log.Fatalf("open pidfile (%s): %s", pidfile, err)
- }
- defer f.Close()
- err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
- if err != nil {
- log.Fatalf("flock pidfile (%s): %s", pidfile, err)
- }
- defer os.Remove(pidfile)
- err = f.Truncate(0)
- if err != nil {
- log.Fatalf("truncate pidfile (%s): %s", pidfile, err)
- }
- _, err = fmt.Fprint(f, os.Getpid())
- if err != nil {
- log.Fatalf("write pidfile (%s): %s", pidfile, err)
- }
- err = f.Sync()
- if err != nil {
- log.Fatalf("sync pidfile (%s): %s", pidfile, err)
- }
- }
-
- var cluster *arvados.Cluster
- cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
- if err != nil && os.IsNotExist(err) {
- log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
- cluster = &arvados.Cluster{
- ClusterID: "xxxxx",
- }
- } else if err != nil {
- log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
- } else {
- cluster, err = cfg.GetCluster("")
- if err != nil {
- log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
- }
- }
-
- log.Println("keepstore starting, pid", os.Getpid())
- defer log.Println("keepstore exiting, pid", os.Getpid())
-
- // Start a round-robin VolumeManager with the volumes we have found.
- KeepVM = MakeRRVolumeManager(theConfig.Volumes)
-
- // Middleware/handler stack
- router := MakeRESTRouter(cluster, metricsRegistry)
-
- // Set up a TCP listener.
- listener, err := net.Listen("tcp", theConfig.Listen)
- if err != nil {
- log.Fatal(err)
- }
-
- // Initialize keepclient for pull workers
- keepClient := &keepclient.KeepClient{
- Arvados: &arvadosclient.ArvadosClient{},
- Want_replicas: 1,
- }
-
- // Initialize the pullq and workers
- pullq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
- go RunPullWorker(pullq, keepClient)
- }
-
- // Initialize the trashq and workers
- trashq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
- go RunTrashWorker(trashq)
- }
-
- // Start emptyTrash goroutine
- doneEmptyingTrash := make(chan bool)
- go emptyTrash(doneEmptyingTrash, theConfig.TrashCheckInterval.Duration())
-
- // Shut down the server gracefully (by closing the listener)
- // if SIGTERM is received.
- term := make(chan os.Signal, 1)
- go func(sig <-chan os.Signal) {
- s := <-sig
- log.Println("caught signal:", s)
- doneEmptyingTrash <- true
- listener.Close()
- }(term)
- signal.Notify(term, syscall.SIGTERM)
- signal.Notify(term, syscall.SIGINT)
-
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
- }
- log.Println("listening at", listener.Addr())
- srv := &server{}
- srv.Handler = router
- srv.Serve(listener)
-}
-
// Periodically (once per interval) invoke EmptyTrash on all volumes.
-func emptyTrash(done <-chan bool, interval time.Duration) {
- ticker := time.NewTicker(interval)
-
- for {
- select {
- case <-ticker.C:
- for _, v := range theConfig.Volumes {
- if v.Writable() {
- v.EmptyTrash()
- }
- }
- case <-done:
- ticker.Stop()
- return
+func emptyTrash(mounts []*VolumeMount, interval time.Duration) {
+ for range time.NewTicker(interval).C {
+ for _, v := range mounts {
+ v.EmptyTrash()
}
}
}
diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 8b448e72c..728c6fded 100644
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -6,7 +6,6 @@
Description=Arvados Keep Storage Daemon
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/keepstore/keepstore.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
diff --git a/services/keepstore/keepstore_test.go b/services/keepstore/keepstore_test.go
deleted file mode 100644
index d1d380466..000000000
--- a/services/keepstore/keepstore_test.go
+++ /dev/null
@@ -1,456 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "io/ioutil"
- "os"
- "path"
- "regexp"
- "sort"
- "strings"
- "testing"
-
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-)
-
-var TestBlock = []byte("The quick brown fox jumps over the lazy dog.")
-var TestHash = "e4d909c290d0fb1ca068ffaddf22cbd0"
-var TestHashPutResp = "e4d909c290d0fb1ca068ffaddf22cbd0+44\n"
-
-var TestBlock2 = []byte("Pack my box with five dozen liquor jugs.")
-var TestHash2 = "f15ac516f788aec4f30932ffb6395c39"
-
-var TestBlock3 = []byte("Now is the time for all good men to come to the aid of their country.")
-var TestHash3 = "eed29bbffbc2dbe5e5ee0bb71888e61f"
-
-// BadBlock is used to test collisions and corruption.
-// It must not match any test hashes.
-var BadBlock = []byte("The magic words are squeamish ossifrage.")
-
-// Empty block
-var EmptyHash = "d41d8cd98f00b204e9800998ecf8427e"
-var EmptyBlock = []byte("")
-
-// TODO(twp): Tests still to be written
-//
-// * TestPutBlockFull
-// - test that PutBlock returns 503 Full if the filesystem is full.
-// (must mock FreeDiskSpace or Statfs? use a tmpfs?)
-//
-// * TestPutBlockWriteErr
-// - test the behavior when Write returns an error.
-// - Possible solutions: use a small tmpfs and a high
-// MIN_FREE_KILOBYTES to trick PutBlock into attempting
-// to write a block larger than the amount of space left
-// - use an interface to mock ioutil.TempFile with a File
-// object that always returns an error on write
-//
-// ========================================
-// GetBlock tests.
-// ========================================
-
-// TestGetBlock
-// Test that simple block reads succeed.
-//
-func TestGetBlock(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes. Our block is stored on the second volume.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- if err := vols[1].Put(context.Background(), TestHash, TestBlock); err != nil {
- t.Error(err)
- }
-
- // Check that GetBlock returns success.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != nil {
- t.Errorf("GetBlock error: %s", err)
- }
- if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Errorf("got %v, expected %v", buf[:size], TestBlock)
- }
-}
-
-// TestGetBlockMissing
-// GetBlock must return an error when the block is not found.
-//
-func TestGetBlockMissing(t *testing.T) {
- defer teardown()
-
- // Create two empty test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that GetBlock returns failure.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != NotFoundError {
- t.Errorf("Expected NotFoundError, got %v, err %v", buf[:size], err)
- }
-}
-
-// TestGetBlockCorrupt
-// GetBlock must return an error when a corrupted block is requested
-// (the contents of the file do not checksum to its hash).
-//
-func TestGetBlockCorrupt(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes and store a corrupt block in one.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- vols[0].Put(context.Background(), TestHash, BadBlock)
-
- // Check that GetBlock returns failure.
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != DiskHashError {
- t.Errorf("Expected DiskHashError, got %v (buf: %v)", err, buf[:size])
- }
-}
-
-// ========================================
-// PutBlock tests
-// ========================================
-
-// TestPutBlockOK
-// PutBlock can perform a simple block write and returns success.
-//
-func TestPutBlockOK(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that PutBlock stores the data as expected.
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
-
- vols := KeepVM.AllReadable()
- buf := make([]byte, BlockSize)
- n, err := vols[1].Get(context.Background(), TestHash, buf)
- if err != nil {
- t.Fatalf("Volume #0 Get returned error: %v", err)
- }
- if string(buf[:n]) != string(TestBlock) {
- t.Fatalf("PutBlock stored '%s', Get retrieved '%s'",
- string(TestBlock), string(buf[:n]))
- }
-}
-
-// TestPutBlockOneVol
-// PutBlock still returns success even when only one of the known
-// volumes is online.
-//
-func TestPutBlockOneVol(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes, but cripple one of them.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllWritable()
- vols[0].(*MockVolume).Bad = true
- vols[0].(*MockVolume).BadVolumeError = errors.New("Bad volume")
-
- // Check that PutBlock stores the data as expected.
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
-
- buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), TestHash, buf, nil)
- if err != nil {
- t.Fatalf("GetBlock: %v", err)
- }
- if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Fatalf("PutBlock stored %+q, GetBlock retrieved %+q",
- TestBlock, buf[:size])
- }
-}
-
-// TestPutBlockMD5Fail
-// Check that PutBlock returns an error if passed a block and hash that
-// do not match.
-//
-func TestPutBlockMD5Fail(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Check that PutBlock returns the expected error when the hash does
- // not match the block.
- if _, err := PutBlock(context.Background(), BadBlock, TestHash); err != RequestHashError {
- t.Errorf("Expected RequestHashError, got %v", err)
- }
-
- // Confirm that GetBlock fails to return anything.
- if result, err := GetBlock(context.Background(), TestHash, make([]byte, BlockSize), nil); err != NotFoundError {
- t.Errorf("GetBlock succeeded after a corrupt block store (result = %s, err = %v)",
- string(result), err)
- }
-}
-
-// TestPutBlockCorrupt
-// PutBlock should overwrite corrupt blocks on disk when given
-// a PUT request with a good block.
-//
-func TestPutBlockCorrupt(t *testing.T) {
- defer teardown()
-
- // Create two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Store a corrupted block under TestHash.
- vols := KeepVM.AllWritable()
- vols[0].Put(context.Background(), TestHash, BadBlock)
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Errorf("PutBlock: n %d err %v", n, err)
- }
-
- // The block on disk should now match TestBlock.
- buf := make([]byte, BlockSize)
- if size, err := GetBlock(context.Background(), TestHash, buf, nil); err != nil {
- t.Errorf("GetBlock: %v", err)
- } else if bytes.Compare(buf[:size], TestBlock) != 0 {
- t.Errorf("Got %+q, expected %+q", buf[:size], TestBlock)
- }
-}
-
-// TestPutBlockCollision
-// PutBlock returns a 400 Collision error when attempting to
-// store a block that collides with another block on disk.
-//
-func TestPutBlockCollision(t *testing.T) {
- defer teardown()
-
- // These blocks both hash to the MD5 digest cee9a457e790cf20d4bdaa6d69f01e41.
- b1 := arvadostest.MD5CollisionData[0]
- b2 := arvadostest.MD5CollisionData[1]
- locator := arvadostest.MD5CollisionMD5
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- // Store one block, then attempt to store the other. Confirm that
- // PutBlock reported a CollisionError.
- if _, err := PutBlock(context.Background(), b1, locator); err != nil {
- t.Error(err)
- }
- if _, err := PutBlock(context.Background(), b2, locator); err == nil {
- t.Error("PutBlock did not report a collision")
- } else if err != CollisionError {
- t.Errorf("PutBlock returned %v", err)
- }
-}
-
-// TestPutBlockTouchFails
-// When PutBlock is asked to PUT an existing block, but cannot
-// modify the timestamp, it should write a second block.
-//
-func TestPutBlockTouchFails(t *testing.T) {
- defer teardown()
-
- // Prepare two test Keep volumes.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
- vols := KeepVM.AllWritable()
-
- // Store a block and then make the underlying volume bad,
- // so a subsequent attempt to update the file timestamp
- // will fail.
- vols[0].Put(context.Background(), TestHash, BadBlock)
- oldMtime, err := vols[0].Mtime(TestHash)
- if err != nil {
- t.Fatalf("vols[0].Mtime(%s): %s\n", TestHash, err)
- }
-
- // vols[0].Touch will fail on the next call, so the volume
- // manager will store a copy on vols[1] instead.
- vols[0].(*MockVolume).Touchable = false
- if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {
- t.Fatalf("PutBlock: n %d err %v", n, err)
- }
- vols[0].(*MockVolume).Touchable = true
-
- // Now the mtime on the block on vols[0] should be unchanged, and
- // there should be a copy of the block on vols[1].
- newMtime, err := vols[0].Mtime(TestHash)
- if err != nil {
- t.Fatalf("vols[0].Mtime(%s): %s\n", TestHash, err)
- }
- if !newMtime.Equal(oldMtime) {
- t.Errorf("mtime was changed on vols[0]:\noldMtime = %v\nnewMtime = %v\n",
- oldMtime, newMtime)
- }
- buf := make([]byte, BlockSize)
- n, err := vols[1].Get(context.Background(), TestHash, buf)
- if err != nil {
- t.Fatalf("vols[1]: %v", err)
- }
- if bytes.Compare(buf[:n], TestBlock) != 0 {
- t.Errorf("new block does not match test block\nnew block = %v\n", buf[:n])
- }
-}
-
-func TestDiscoverTmpfs(t *testing.T) {
- var tempVols [4]string
- var err error
-
- // Create some directories suitable for using as keep volumes.
- for i := range tempVols {
- if tempVols[i], err = ioutil.TempDir("", "findvol"); err != nil {
- t.Fatal(err)
- }
- defer os.RemoveAll(tempVols[i])
- tempVols[i] = tempVols[i] + "/keep"
- if err = os.Mkdir(tempVols[i], 0755); err != nil {
- t.Fatal(err)
- }
- }
-
- // Set up a bogus ProcMounts file.
- f, err := ioutil.TempFile("", "keeptest")
- if err != nil {
- t.Fatal(err)
- }
- defer os.Remove(f.Name())
- for i, vol := range tempVols {
- // Add readonly mount points at odd indexes.
- var opts string
- switch i % 2 {
- case 0:
- opts = "rw,nosuid,nodev,noexec"
- case 1:
- opts = "nosuid,nodev,noexec,ro"
- }
- fmt.Fprintf(f, "tmpfs %s tmpfs %s 0 0\n", path.Dir(vol), opts)
- }
- f.Close()
- ProcMounts = f.Name()
-
- cfg := &Config{}
- added := (&unixVolumeAdder{cfg}).Discover()
-
- if added != len(cfg.Volumes) {
- t.Errorf("Discover returned %d, but added %d volumes",
- added, len(cfg.Volumes))
- }
- if added != len(tempVols) {
- t.Errorf("Discover returned %d but we set up %d volumes",
- added, len(tempVols))
- }
- for i, tmpdir := range tempVols {
- if tmpdir != cfg.Volumes[i].(*UnixVolume).Root {
- t.Errorf("Discover returned %s, expected %s\n",
- cfg.Volumes[i].(*UnixVolume).Root, tmpdir)
- }
- if expectReadonly := i%2 == 1; expectReadonly != cfg.Volumes[i].(*UnixVolume).ReadOnly {
- t.Errorf("Discover added %s with readonly=%v, should be %v",
- tmpdir, !expectReadonly, expectReadonly)
- }
- }
-}
-
-func TestDiscoverNone(t *testing.T) {
- defer teardown()
-
- // Set up a bogus ProcMounts file with no Keep vols.
- f, err := ioutil.TempFile("", "keeptest")
- if err != nil {
- t.Fatal(err)
- }
- defer os.Remove(f.Name())
- fmt.Fprintln(f, "rootfs / rootfs opts 0 0")
- fmt.Fprintln(f, "sysfs /sys sysfs opts 0 0")
- fmt.Fprintln(f, "proc /proc proc opts 0 0")
- fmt.Fprintln(f, "udev /dev devtmpfs opts 0 0")
- fmt.Fprintln(f, "devpts /dev/pts devpts opts 0 0")
- f.Close()
- ProcMounts = f.Name()
-
- cfg := &Config{}
- added := (&unixVolumeAdder{cfg}).Discover()
- if added != 0 || len(cfg.Volumes) != 0 {
- t.Fatalf("got %d, %v; expected 0, []", added, cfg.Volumes)
- }
-}
-
-// TestIndex
-// Test an /index request.
-func TestIndex(t *testing.T) {
- defer teardown()
-
- // Set up Keep volumes and populate them.
- // Include multiple blocks on different volumes, and
- // some metadata files.
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
-
- vols := KeepVM.AllReadable()
- vols[0].Put(context.Background(), TestHash, TestBlock)
- vols[1].Put(context.Background(), TestHash2, TestBlock2)
- vols[0].Put(context.Background(), TestHash3, TestBlock3)
- vols[0].Put(context.Background(), TestHash+".meta", []byte("metadata"))
- vols[1].Put(context.Background(), TestHash2+".meta", []byte("metadata"))
-
- buf := new(bytes.Buffer)
- vols[0].IndexTo("", buf)
- vols[1].IndexTo("", buf)
- indexRows := strings.Split(string(buf.Bytes()), "\n")
- sort.Strings(indexRows)
- sortedIndex := strings.Join(indexRows, "\n")
- expected := `^\n` + TestHash + `\+\d+ \d+\n` +
- TestHash3 + `\+\d+ \d+\n` +
- TestHash2 + `\+\d+ \d+$`
-
- match, err := regexp.MatchString(expected, sortedIndex)
- if err == nil {
- if !match {
- t.Errorf("IndexLocators returned:\n%s", string(buf.Bytes()))
- }
- } else {
- t.Errorf("regexp.MatchString: %s", err)
- }
-}
-
-// ========================================
-// Helper functions for unit tests.
-// ========================================
-
-// MakeTestVolumeManager returns a RRVolumeManager with the specified
-// number of MockVolumes.
-func MakeTestVolumeManager(numVolumes int) VolumeManager {
- vols := make([]Volume, numVolumes)
- for i := range vols {
- vols[i] = CreateMockVolume()
- }
- return MakeRRVolumeManager(vols)
-}
-
-// teardown cleans up after each test.
-func teardown() {
- theConfig.systemAuthToken = ""
- theConfig.RequireSignatures = false
- theConfig.blobSigningKey = nil
- KeepVM = nil
-}
diff --git a/services/keepstore/metrics.go b/services/keepstore/metrics.go
index 235c41891..b2f0aa663 100644
--- a/services/keepstore/metrics.go
+++ b/services/keepstore/metrics.go
@@ -7,7 +7,6 @@ package main
import (
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
)
@@ -66,27 +65,6 @@ func (m *nodeMetrics) setupWorkQueueMetrics(q *WorkQueue, qName string) {
))
}
-func (m *nodeMetrics) setupRequestMetrics(rc httpserver.RequestCounter) {
- m.reg.MustRegister(prometheus.NewGaugeFunc(
- prometheus.GaugeOpts{
- Namespace: "arvados",
- Subsystem: "keepstore",
- Name: "concurrent_requests",
- Help: "Number of requests in progress",
- },
- func() float64 { return float64(rc.Current()) },
- ))
- m.reg.MustRegister(prometheus.NewGaugeFunc(
- prometheus.GaugeOpts{
- Namespace: "arvados",
- Subsystem: "keepstore",
- Name: "max_concurrent_requests",
- Help: "Maximum number of concurrent requests",
- },
- func() float64 { return float64(rc.Max()) },
- ))
-}
-
type volumeMetricsVecs struct {
ioBytes *prometheus.CounterVec
errCounters *prometheus.CounterVec
diff --git a/services/keepstore/mounts_test.go b/services/keepstore/mounts_test.go
index 7c932ee02..9b5606b5c 100644
--- a/services/keepstore/mounts_test.go
+++ b/services/keepstore/mounts_test.go
@@ -12,59 +12,39 @@ import (
"net/http/httptest"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
-var _ = check.Suite(&MountsSuite{})
+func (s *HandlerSuite) TestMounts(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
-type MountsSuite struct {
- vm VolumeManager
- rtr http.Handler
-}
-
-func (s *MountsSuite) SetUpTest(c *check.C) {
- s.vm = MakeTestVolumeManager(2)
- KeepVM = s.vm
- theConfig = DefaultConfig()
- theConfig.systemAuthToken = arvadostest.DataManagerToken
- theConfig.ManagementToken = arvadostest.ManagementToken
- r := prometheus.NewRegistry()
- theConfig.Start(r)
- s.rtr = MakeRESTRouter(testCluster, r)
-}
-
-func (s *MountsSuite) TearDownTest(c *check.C) {
- s.vm.Close()
- KeepVM = nil
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
-}
-
-func (s *MountsSuite) TestMounts(c *check.C) {
- vols := s.vm.AllWritable()
+ vols := s.handler.volmgr.AllWritable()
vols[0].Put(context.Background(), TestHash, TestBlock)
vols[1].Put(context.Background(), TestHash2, TestBlock2)
resp := s.call("GET", "/mounts", "", nil)
c.Check(resp.Code, check.Equals, http.StatusOK)
var mntList []struct {
- UUID string `json:"uuid"`
- DeviceID string `json:"device_id"`
- ReadOnly bool `json:"read_only"`
- Replication int `json:"replication"`
- StorageClasses []string `json:"storage_classes"`
+ UUID string `json:"uuid"`
+ DeviceID string `json:"device_id"`
+ ReadOnly bool `json:"read_only"`
+ Replication int `json:"replication"`
+ StorageClasses map[string]bool `json:"storage_classes"`
}
+ c.Log(resp.Body.String())
err := json.Unmarshal(resp.Body.Bytes(), &mntList)
c.Assert(err, check.IsNil)
c.Assert(len(mntList), check.Equals, 2)
for _, m := range mntList {
c.Check(len(m.UUID), check.Equals, 27)
- c.Check(m.UUID[:12], check.Equals, "zzzzz-ivpuk-")
+ c.Check(m.UUID[:12], check.Equals, "zzzzz-nyw5e-")
c.Check(m.DeviceID, check.Equals, "mock-device-id")
c.Check(m.ReadOnly, check.Equals, false)
c.Check(m.Replication, check.Equals, 1)
- c.Check(m.StorageClasses, check.DeepEquals, []string{"default"})
+ c.Check(m.StorageClasses, check.DeepEquals, map[string]bool{"default": true})
}
c.Check(mntList[0].UUID, check.Not(check.Equals), mntList[1].UUID)
@@ -103,7 +83,12 @@ func (s *MountsSuite) TestMounts(c *check.C) {
c.Check(resp.Body.String(), check.Equals, "\n")
}
-func (s *MountsSuite) TestMetrics(c *check.C) {
+func (s *HandlerSuite) TestMetrics(c *check.C) {
+ reg := prometheus.NewRegistry()
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", reg, testServiceURL), check.IsNil)
+ instrumented := httpserver.Instrument(reg, ctxlog.TestLogger(c), s.handler.Handler)
+ s.handler.Handler = instrumented.ServeAPI(s.cluster.ManagementToken, instrumented)
+
s.call("PUT", "/"+TestHash, "", TestBlock)
s.call("PUT", "/"+TestHash2, "", TestBlock2)
resp := s.call("GET", "/metrics.json", "", nil)
@@ -145,8 +130,6 @@ func (s *MountsSuite) TestMetrics(c *check.C) {
}
}
}
- c.Check(found["request_duration_seconds"], check.Equals, true)
- c.Check(found["time_to_status_seconds"], check.Equals, true)
metricsNames := []string{
"arvados_keepstore_bufferpool_inuse_buffers",
@@ -154,25 +137,22 @@ func (s *MountsSuite) TestMetrics(c *check.C) {
"arvados_keepstore_bufferpool_allocated_bytes",
"arvados_keepstore_pull_queue_inprogress_entries",
"arvados_keepstore_pull_queue_pending_entries",
- "arvados_keepstore_concurrent_requests",
- "arvados_keepstore_max_concurrent_requests",
"arvados_keepstore_trash_queue_inprogress_entries",
"arvados_keepstore_trash_queue_pending_entries",
"request_duration_seconds",
- "time_to_status_seconds",
}
for _, m := range metricsNames {
_, ok := names[m]
- c.Check(ok, check.Equals, true)
+ c.Check(ok, check.Equals, true, check.Commentf("checking metric %q", m))
}
}
-func (s *MountsSuite) call(method, path, tok string, body []byte) *httptest.ResponseRecorder {
+func (s *HandlerSuite) call(method, path, tok string, body []byte) *httptest.ResponseRecorder {
resp := httptest.NewRecorder()
req, _ := http.NewRequest(method, path, bytes.NewReader(body))
if tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
return resp
}
diff --git a/services/keepstore/perms.go b/services/keepstore/perms.go
index 49a231685..e2155f94f 100644
--- a/services/keepstore/perms.go
+++ b/services/keepstore/perms.go
@@ -5,14 +5,16 @@
package main
import (
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
// SignLocator takes a blobLocator, an apiToken and an expiry time, and
// returns a signed locator string.
-func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
- return keepclient.SignLocator(blobLocator, apiToken, expiry, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func SignLocator(cluster *arvados.Cluster, blobLocator, apiToken string, expiry time.Time) string {
+ return keepclient.SignLocator(blobLocator, apiToken, expiry, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
}
// VerifySignature returns nil if the signature on the signedLocator
@@ -20,8 +22,8 @@ func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
// either ExpiredError (if the timestamp has expired, which is
// something the client could have figured out independently) or
// PermissionError.
-func VerifySignature(signedLocator, apiToken string) error {
- err := keepclient.VerifySignature(signedLocator, apiToken, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func VerifySignature(cluster *arvados.Cluster, signedLocator, apiToken string) error {
+ err := keepclient.VerifySignature(signedLocator, apiToken, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
if err == keepclient.ErrSignatureExpired {
return ExpiredError
} else if err != nil {
diff --git a/services/keepstore/perms_test.go b/services/keepstore/perms_test.go
index dd57faf27..6ec4887ce 100644
--- a/services/keepstore/perms_test.go
+++ b/services/keepstore/perms_test.go
@@ -6,10 +6,10 @@ package main
import (
"strconv"
- "testing"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ check "gopkg.in/check.v1"
)
const (
@@ -30,44 +30,34 @@ const (
knownSignedLocator = knownLocator + knownSigHint
)
-func TestSignLocator(t *testing.T) {
- defer func(b []byte) {
- theConfig.blobSigningKey = b
- }(theConfig.blobSigningKey)
-
+func (s *HandlerSuite) TestSignLocator(c *check.C) {
tsInt, err := strconv.ParseInt(knownTimestamp, 16, 0)
if err != nil {
- t.Fatal(err)
+ c.Fatal(err)
}
t0 := time.Unix(tsInt, 0)
- theConfig.BlobSignatureTTL = knownSignatureTTL
-
- theConfig.blobSigningKey = []byte(knownKey)
- if x := SignLocator(knownLocator, knownToken, t0); x != knownSignedLocator {
- t.Fatalf("Got %+q, expected %+q", x, knownSignedLocator)
+ s.cluster.Collections.BlobSigningTTL = knownSignatureTTL
+ s.cluster.Collections.BlobSigningKey = knownKey
+ if x := SignLocator(s.cluster, knownLocator, knownToken, t0); x != knownSignedLocator {
+ c.Fatalf("Got %+q, expected %+q", x, knownSignedLocator)
}
- theConfig.blobSigningKey = []byte("arbitrarykey")
- if x := SignLocator(knownLocator, knownToken, t0); x == knownSignedLocator {
- t.Fatalf("Got same signature %+q, even though blobSigningKey changed", x)
+ s.cluster.Collections.BlobSigningKey = "arbitrarykey"
+ if x := SignLocator(s.cluster, knownLocator, knownToken, t0); x == knownSignedLocator {
+ c.Fatalf("Got same signature %+q, even though blobSigningKey changed", x)
}
}
-func TestVerifyLocator(t *testing.T) {
- defer func(b []byte) {
- theConfig.blobSigningKey = b
- }(theConfig.blobSigningKey)
-
- theConfig.BlobSignatureTTL = knownSignatureTTL
-
- theConfig.blobSigningKey = []byte(knownKey)
- if err := VerifySignature(knownSignedLocator, knownToken); err != nil {
- t.Fatal(err)
+func (s *HandlerSuite) TestVerifyLocator(c *check.C) {
+ s.cluster.Collections.BlobSigningTTL = knownSignatureTTL
+ s.cluster.Collections.BlobSigningKey = knownKey
+ if err := VerifySignature(s.cluster, knownSignedLocator, knownToken); err != nil {
+ c.Fatal(err)
}
- theConfig.blobSigningKey = []byte("arbitrarykey")
- if err := VerifySignature(knownSignedLocator, knownToken); err == nil {
- t.Fatal("Verified signature even with wrong blobSigningKey")
+ s.cluster.Collections.BlobSigningKey = "arbitrarykey"
+ if err := VerifySignature(s.cluster, knownSignedLocator, knownToken); err == nil {
+ c.Fatal("Verified signature even with wrong blobSigningKey")
}
}
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
index 1f82f3f4f..fac9c542f 100644
--- a/services/keepstore/proxy_remote.go
+++ b/services/keepstore/proxy_remote.go
@@ -25,7 +25,7 @@ type remoteProxy struct {
mtx sync.Mutex
}
-func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.Request, cluster *arvados.Cluster) {
+func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.Request, cluster *arvados.Cluster, volmgr *RRVolumeManager) {
// Intervening proxies must not return a cached GET response
// to a prior request if a X-Keep-Signature request header has
// been added or changed.
@@ -49,6 +49,8 @@ func (rp *remoteProxy) Get(ctx context.Context, w http.ResponseWriter, r *http.R
Buffer: buf[:0],
ResponseWriter: w,
Context: ctx,
+ Cluster: cluster,
+ VolumeManager: volmgr,
}
defer rrc.Close()
w = rrc
@@ -145,10 +147,12 @@ var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^\+]*`)
// local volume, adds a response header with a locally-signed locator,
// and finally writes the data through.
type remoteResponseCacher struct {
- Locator string
- Token string
- Buffer []byte
- Context context.Context
+ Locator string
+ Token string
+ Buffer []byte
+ Context context.Context
+ Cluster *arvados.Cluster
+ VolumeManager *RRVolumeManager
http.ResponseWriter
statusCode int
}
@@ -173,7 +177,7 @@ func (rrc *remoteResponseCacher) Close() error {
rrc.ResponseWriter.Write(rrc.Buffer)
return nil
}
- _, err := PutBlock(rrc.Context, rrc.Buffer, rrc.Locator[:32])
+ _, err := PutBlock(rrc.Context, rrc.VolumeManager, rrc.Buffer, rrc.Locator[:32])
if rrc.Context.Err() != nil {
// If caller hung up, log that instead of subsequent/misleading errors.
http.Error(rrc.ResponseWriter, rrc.Context.Err().Error(), http.StatusGatewayTimeout)
@@ -193,7 +197,8 @@ func (rrc *remoteResponseCacher) Close() error {
}
unsigned := localOrRemoteSignature.ReplaceAllLiteralString(rrc.Locator, "")
- signed := SignLocator(unsigned, rrc.Token, time.Now().Add(theConfig.BlobSignatureTTL.Duration()))
+ expiry := time.Now().Add(rrc.Cluster.Collections.BlobSigningTTL.Duration())
+ signed := SignLocator(rrc.Cluster, unsigned, rrc.Token, expiry)
if signed == unsigned {
err = errors.New("could not sign locator")
http.Error(rrc.ResponseWriter, err.Error(), http.StatusInternalServerError)
diff --git a/services/keepstore/proxy_remote_test.go b/services/keepstore/proxy_remote_test.go
index 6c22d1d32..6483d6cf0 100644
--- a/services/keepstore/proxy_remote_test.go
+++ b/services/keepstore/proxy_remote_test.go
@@ -5,6 +5,7 @@
package main
import (
+ "context"
"crypto/md5"
"encoding/json"
"fmt"
@@ -28,8 +29,7 @@ var _ = check.Suite(&ProxyRemoteSuite{})
type ProxyRemoteSuite struct {
cluster *arvados.Cluster
- vm VolumeManager
- rtr http.Handler
+ handler *handler
remoteClusterID string
remoteBlobSigningKey []byte
@@ -87,7 +87,9 @@ func (s *ProxyRemoteSuite) SetUpTest(c *check.C) {
s.remoteKeepproxy = httptest.NewServer(http.HandlerFunc(s.remoteKeepproxyHandler))
s.remoteAPI = httptest.NewUnstartedServer(http.HandlerFunc(s.remoteAPIHandler))
s.remoteAPI.StartTLS()
- s.cluster = arvados.IntegrationTestCluster()
+ s.cluster = testCluster(c)
+ s.cluster.Collections.BlobSigningKey = knownKey
+ s.cluster.SystemRootToken = arvadostest.DataManagerToken
s.cluster.RemoteClusters = map[string]arvados.RemoteCluster{
s.remoteClusterID: arvados.RemoteCluster{
Host: strings.Split(s.remoteAPI.URL, "//")[1],
@@ -96,21 +98,12 @@ func (s *ProxyRemoteSuite) SetUpTest(c *check.C) {
Insecure: true,
},
}
- s.vm = MakeTestVolumeManager(2)
- KeepVM = s.vm
- theConfig = DefaultConfig()
- theConfig.systemAuthToken = arvadostest.DataManagerToken
- theConfig.blobSigningKey = []byte(knownKey)
- r := prometheus.NewRegistry()
- theConfig.Start(r)
- s.rtr = MakeRESTRouter(s.cluster, r)
+ s.cluster.Volumes = map[string]arvados.Volume{"zzzzz-nyw5e-000000000000000": {Driver: "mock"}}
+ s.handler = &handler{}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
}
func (s *ProxyRemoteSuite) TearDownTest(c *check.C) {
- s.vm.Close()
- KeepVM = nil
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
s.remoteAPI.Close()
s.remoteKeepproxy.Close()
}
@@ -191,7 +184,7 @@ func (s *ProxyRemoteSuite) TestProxyRemote(c *check.C) {
req.Header.Set("X-Keep-Signature", trial.xKeepSignature)
}
resp = httptest.NewRecorder()
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(s.remoteKeepRequests, check.Equals, trial.expectRemoteReqs)
c.Check(resp.Code, check.Equals, trial.expectCode)
if resp.Code == http.StatusOK {
@@ -210,13 +203,13 @@ func (s *ProxyRemoteSuite) TestProxyRemote(c *check.C) {
c.Check(locHdr, check.Not(check.Equals), "")
c.Check(locHdr, check.Not(check.Matches), `.*\+R.*`)
- c.Check(VerifySignature(locHdr, trial.token), check.IsNil)
+ c.Check(VerifySignature(s.cluster, locHdr, trial.token), check.IsNil)
// Ensure block can be requested using new signature
req = httptest.NewRequest("GET", "/"+locHdr, nil)
req.Header.Set("Authorization", "Bearer "+trial.token)
resp = httptest.NewRecorder()
- s.rtr.ServeHTTP(resp, req)
+ s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(s.remoteKeepRequests, check.Equals, trial.expectRemoteReqs)
}
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 42b5d5889..100d08838 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -6,7 +6,6 @@ package main
import (
"context"
- "crypto/rand"
"fmt"
"io"
"io/ioutil"
@@ -18,15 +17,15 @@ import (
// RunPullWorker receives PullRequests from pullq, invokes
// PullItemAndProcess on each one. After each PR, it logs a message
// indicating whether the pull was successful.
-func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
+func (h *handler) runPullWorker(pullq *WorkQueue) {
for item := range pullq.NextItem {
pr := item.(PullRequest)
- err := PullItemAndProcess(pr, keepClient)
+ err := h.pullItemAndProcess(pr)
pullq.DoneItem <- struct{}{}
if err == nil {
- log.Printf("Pull %s success", pr)
+ h.Logger.Printf("Pull %s success", pr)
} else {
- log.Printf("Pull %s error: %s", pr, err)
+ h.Logger.Printf("Pull %s error: %s", pr, err)
}
}
}
@@ -39,28 +38,28 @@ func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
// only attempt to write the data to the corresponding
// volume. Otherwise it writes to any local volume, as a PUT request
// would.
-func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClient) error {
- var vol Volume
+func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
+ var vol *VolumeMount
if uuid := pullRequest.MountUUID; uuid != "" {
- vol = KeepVM.Lookup(pullRequest.MountUUID, true)
+ vol = h.volmgr.Lookup(pullRequest.MountUUID, true)
if vol == nil {
return fmt.Errorf("pull req has nonexistent mount: %v", pullRequest)
}
}
- keepClient.Arvados.ApiToken = randomToken
-
+ // Make a private copy of keepClient so we can set
+ // ServiceRoots to the source servers specified in the pull
+ // request.
+ keepClient := *h.keepClient
serviceRoots := make(map[string]string)
for _, addr := range pullRequest.Servers {
serviceRoots[addr] = addr
}
keepClient.SetServiceRoots(serviceRoots, nil, nil)
- // Generate signature with a random token
- expiresAt := time.Now().Add(60 * time.Second)
- signedLocator := SignLocator(pullRequest.Locator, randomToken, expiresAt)
+ signedLocator := SignLocator(h.Cluster, pullRequest.Locator, keepClient.Arvados.ApiToken, time.Now().Add(time.Minute))
- reader, contentLen, _, err := GetContent(signedLocator, keepClient)
+ reader, contentLen, _, err := GetContent(signedLocator, &keepClient)
if err != nil {
return err
}
@@ -78,8 +77,7 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
return fmt.Errorf("Content not found for: %s", signedLocator)
}
- writePulledBlock(vol, readContent, pullRequest.Locator)
- return nil
+ return writePulledBlock(h.volmgr, vol, readContent, pullRequest.Locator)
}
// Fetch the content for the given locator using keepclient.
@@ -87,24 +85,11 @@ var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (
return keepClient.Get(signedLocator)
}
-var writePulledBlock = func(volume Volume, data []byte, locator string) {
- var err error
+var writePulledBlock = func(volmgr *RRVolumeManager, volume Volume, data []byte, locator string) error {
if volume != nil {
- err = volume.Put(context.Background(), locator, data)
+ return volume.Put(context.Background(), locator, data)
} else {
- _, err = PutBlock(context.Background(), data, locator)
- }
- if err != nil {
- log.Printf("error writing pulled block %q: %s", locator, err)
+ _, err := PutBlock(context.Background(), volmgr, data, locator)
+ return err
}
}
-
-var randomToken = func() string {
- const alphaNumeric = "0123456789abcdefghijklmnopqrstuvwxyz"
- var bytes = make([]byte, 36)
- rand.Read(bytes)
- for i, b := range bytes {
- bytes[i] = alphaNumeric[b%byte(len(alphaNumeric))]
- }
- return (string(bytes))
-}()
diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index 231a4c0ab..a35b744c5 100644
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -6,20 +6,18 @@ package main
import (
"bytes"
+ "context"
"errors"
"io"
"io/ioutil"
- "os"
"strings"
- "testing"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
-var keepClient *keepclient.KeepClient
-
type PullWorkIntegrationTestData struct {
Name string
Locator string
@@ -27,55 +25,31 @@ type PullWorkIntegrationTestData struct {
GetError string
}
-func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
- // start api and keep servers
- arvadostest.StartAPI()
+func (s *HandlerSuite) setupPullWorkerIntegrationTest(c *check.C, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
arvadostest.StartKeep(2, false)
-
- // make arvadosclient
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- t.Fatalf("Error creating arv: %s", err)
- }
-
- // keep client
- keepClient, err = keepclient.MakeKeepClient(arv)
- if err != nil {
- t.Fatalf("error creating KeepClient: %s", err)
- }
- keepClient.Want_replicas = 1
- keepClient.RefreshServiceDiscovery()
-
- // discover keep services
- var servers []string
- for _, host := range keepClient.LocalRoots() {
- servers = append(servers, host)
- }
-
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
// Put content if the test needs it
if wantData {
- locator, _, err := keepClient.PutB([]byte(testData.Content))
+ locator, _, err := s.handler.keepClient.PutB([]byte(testData.Content))
if err != nil {
- t.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
+ c.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
}
if locator == "" {
- t.Errorf("No locator found after putting test data")
+ c.Errorf("No locator found after putting test data")
}
}
// Create pullRequest for the test
pullRequest := PullRequest{
Locator: testData.Locator,
- Servers: servers,
}
return pullRequest
}
// Do a get on a block that is not existing in any of the keep servers.
// Expect "block not found" error.
-func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) {
+func (s *HandlerSuite) TestPullWorkerIntegration_GetNonExistingLocator(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
testData := PullWorkIntegrationTestData{
Name: "TestPullWorkerIntegration_GetLocator",
Locator: "5d41402abc4b2a76b9719d911017c592",
@@ -83,16 +57,17 @@ func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) {
GetError: "Block not found",
}
- pullRequest := SetupPullWorkerIntegrationTest(t, testData, false)
+ pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false)
defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
- performPullWorkerIntegrationTest(testData, pullRequest, t)
+ s.performPullWorkerIntegrationTest(testData, pullRequest, c)
}
// Do a get on a block that exists on one of the keep servers.
// The setup method will create this block before doing the get.
-func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) {
+func (s *HandlerSuite) TestPullWorkerIntegration_GetExistingLocator(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
testData := PullWorkIntegrationTestData{
Name: "TestPullWorkerIntegration_GetLocator",
Locator: "5d41402abc4b2a76b9719d911017c592",
@@ -100,24 +75,23 @@ func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) {
GetError: "",
}
- pullRequest := SetupPullWorkerIntegrationTest(t, testData, true)
+ pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true)
defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
- performPullWorkerIntegrationTest(testData, pullRequest, t)
+ s.performPullWorkerIntegrationTest(testData, pullRequest, c)
}
// Perform the test.
// The test directly invokes the "PullItemAndProcess" rather than
// putting an item on the pullq so that the errors can be verified.
-func performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, t *testing.T) {
+func (s *HandlerSuite) performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, c *check.C) {
// Override writePulledBlock to mock PutBlock functionality
- defer func(orig func(Volume, []byte, string)) { writePulledBlock = orig }(writePulledBlock)
- writePulledBlock = func(v Volume, content []byte, locator string) {
- if string(content) != testData.Content {
- t.Errorf("writePulledBlock invoked with unexpected data. Expected: %s; Found: %s", testData.Content, content)
- }
+ defer func(orig func(*RRVolumeManager, Volume, []byte, string) error) { writePulledBlock = orig }(writePulledBlock)
+ writePulledBlock = func(_ *RRVolumeManager, _ Volume, content []byte, _ string) error {
+ c.Check(string(content), check.Equals, testData.Content)
+ return nil
}
// Override GetContent to mock keepclient Get functionality
@@ -132,15 +106,15 @@ func performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pull
return rdr, int64(len(testData.Content)), "", nil
}
- err := PullItemAndProcess(pullRequest, keepClient)
+ err := s.handler.pullItemAndProcess(pullRequest)
if len(testData.GetError) > 0 {
if (err == nil) || (!strings.Contains(err.Error(), testData.GetError)) {
- t.Errorf("Got error %v, expected %v", err, testData.GetError)
+ c.Errorf("Got error %v, expected %v", err, testData.GetError)
}
} else {
if err != nil {
- t.Errorf("Got error %v, expected nil", err)
+ c.Errorf("Got error %v, expected nil", err)
}
}
}
diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go
index 8e667e048..6a7a0b7a8 100644
--- a/services/keepstore/pull_worker_test.go
+++ b/services/keepstore/pull_worker_test.go
@@ -6,21 +6,26 @@ package main
import (
"bytes"
+ "context"
"errors"
"io"
"io/ioutil"
"net/http"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"github.com/prometheus/client_golang/prometheus"
. "gopkg.in/check.v1"
+ check "gopkg.in/check.v1"
)
var _ = Suite(&PullWorkerTestSuite{})
type PullWorkerTestSuite struct {
+ cluster *arvados.Cluster
+ handler *handler
+
testPullLists map[string]string
readContent string
readError error
@@ -29,7 +34,16 @@ type PullWorkerTestSuite struct {
}
func (s *PullWorkerTestSuite) SetUpTest(c *C) {
- theConfig.systemAuthToken = "arbitrary data manager token"
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Driver: "mock"},
+ "zzzzz-nyw5e-111111111111111": {Driver: "mock"},
+ }
+ s.cluster.Collections.BlobReplicateConcurrency = 1
+
+ s.handler = &handler{}
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+
s.readContent = ""
s.readError = nil
s.putContent = []byte{}
@@ -39,27 +53,6 @@ func (s *PullWorkerTestSuite) SetUpTest(c *C) {
// This behavior is verified using these two maps in the
// "TestPullWorkerPullList_with_two_items_latest_replacing_old"
s.testPullLists = make(map[string]string)
-
- KeepVM = MakeTestVolumeManager(2)
-
- // Normally the pull queue and workers are started by main()
- // -- tests need to set up their own.
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, IsNil)
- keepClient, err := keepclient.MakeKeepClient(arv)
- c.Assert(err, IsNil)
- pullq = NewWorkQueue()
- go RunPullWorker(pullq, keepClient)
-}
-
-func (s *PullWorkerTestSuite) TearDownTest(c *C) {
- KeepVM.Close()
- KeepVM = nil
- pullq.Close()
- pullq = nil
- teardown()
- theConfig = DefaultConfig()
- theConfig.Start(prometheus.NewRegistry())
}
var firstPullList = []byte(`[
@@ -100,9 +93,10 @@ type PullWorkerTestData struct {
// Ensure MountUUID in a pull list is correctly translated to a Volume
// argument passed to writePulledBlock().
func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
- defer func(f func(Volume, []byte, string)) {
+ defer func(f func(*RRVolumeManager, Volume, []byte, string) error) {
writePulledBlock = f
}(writePulledBlock)
+ pullq := s.handler.Handler.(*router).pullq
for _, spec := range []struct {
sendUUID string
@@ -113,17 +107,18 @@ func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
expectVolume: nil,
},
{
- sendUUID: KeepVM.Mounts()[0].UUID,
- expectVolume: KeepVM.Mounts()[0].volume,
+ sendUUID: s.handler.volmgr.Mounts()[0].UUID,
+ expectVolume: s.handler.volmgr.Mounts()[0].Volume,
},
} {
- writePulledBlock = func(v Volume, _ []byte, _ string) {
+ writePulledBlock = func(_ *RRVolumeManager, v Volume, _ []byte, _ string) error {
c.Check(v, Equals, spec.expectVolume)
+ return nil
}
- resp := IssueRequest(&RequestTester{
+ resp := IssueRequest(s.handler, &RequestTester{
uri: "/pull",
- apiToken: theConfig.systemAuthToken,
+ apiToken: s.cluster.SystemRootToken,
method: "PUT",
requestBody: []byte(`[{
"locator":"acbd18db4cc2f85cedef654fccc4a4d8+3",
@@ -141,7 +136,7 @@ func (s *PullWorkerTestSuite) TestSpecifyMountUUID(c *C) {
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello",
@@ -155,7 +150,7 @@ func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_two_locators(c *C) {
func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorkerPullList_with_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hola",
@@ -169,7 +164,7 @@ func (s *PullWorkerTestSuite) TestPullWorkerPullList_with_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "unused",
@@ -183,7 +178,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_get_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "unused",
@@ -197,7 +192,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_one_locator",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", secondPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", secondPullList},
responseCode: http.StatusOK,
responseBody: "Received 1 pull requests\n",
readContent: "hello hello",
@@ -211,7 +206,7 @@ func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_two_locators(c *C) {
testData := PullWorkerTestData{
name: "TestPullWorker_error_on_put_two_locators",
- req: RequestTester{"/pull", theConfig.systemAuthToken, "PUT", firstPullList},
+ req: RequestTester{"/pull", s.cluster.SystemRootToken, "PUT", firstPullList},
responseCode: http.StatusOK,
responseBody: "Received 2 pull requests\n",
readContent: "hello again",
@@ -238,6 +233,8 @@ func (s *PullWorkerTestSuite) TestPullWorker_invalidToken(c *C) {
}
func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
+ pullq := s.handler.Handler.(*router).pullq
+
s.testPullLists[testData.name] = testData.responseBody
processedPullLists := make(map[string]string)
@@ -247,7 +244,7 @@ func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
GetContent = orig
}(GetContent)
GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (reader io.ReadCloser, contentLength int64, url string, err error) {
- c.Assert(getStatusItem("PullQueue", "InProgress"), Equals, float64(1))
+ c.Assert(getStatusItem(s.handler, "PullQueue", "InProgress"), Equals, float64(1))
processedPullLists[testData.name] = testData.responseBody
if testData.readError {
err = errors.New("Error getting data")
@@ -261,20 +258,21 @@ func (s *PullWorkerTestSuite) performTest(testData PullWorkerTestData, c *C) {
}
// Override writePulledBlock to mock PutBlock functionality
- defer func(orig func(Volume, []byte, string)) { writePulledBlock = orig }(writePulledBlock)
- writePulledBlock = func(v Volume, content []byte, locator string) {
+ defer func(orig func(*RRVolumeManager, Volume, []byte, string) error) { writePulledBlock = orig }(writePulledBlock)
+ writePulledBlock = func(_ *RRVolumeManager, v Volume, content []byte, locator string) error {
if testData.putError {
s.putError = errors.New("Error putting data")
- return
+ return s.putError
}
s.putContent = content
+ return nil
}
- c.Check(getStatusItem("PullQueue", "InProgress"), Equals, float64(0))
- c.Check(getStatusItem("PullQueue", "Queued"), Equals, float64(0))
- c.Check(getStatusItem("Version"), Not(Equals), "")
+ c.Check(getStatusItem(s.handler, "PullQueue", "InProgress"), Equals, float64(0))
+ c.Check(getStatusItem(s.handler, "PullQueue", "Queued"), Equals, float64(0))
+ c.Check(getStatusItem(s.handler, "Version"), Not(Equals), "")
- response := IssueRequest(&testData.req)
+ response := IssueRequest(s.handler, &testData.req)
c.Assert(response.Code, Equals, testData.responseCode)
c.Assert(response.Body.String(), Equals, testData.responseBody)
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 4c39dcd5c..22a38e208 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -10,10 +10,12 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,188 +28,41 @@ import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-const (
- s3DefaultReadTimeout = arvados.Duration(10 * time.Minute)
- s3DefaultConnectTimeout = arvados.Duration(time.Minute)
-)
-
-var (
- // ErrS3TrashDisabled is returned by Trash if that operation
- // is impossible with the current config.
- ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
-
- s3AccessKeyFile string
- s3SecretKeyFile string
- s3RegionName string
- s3Endpoint string
- s3Replication int
- s3UnsafeDelete bool
- s3RaceWindow time.Duration
-
- s3ACL = s3.Private
-
- zeroTime time.Time
-)
-
-const (
- maxClockSkew = 600 * time.Second
- nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
-)
-
-type s3VolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (s *s3VolumeAdder) String() string {
- return "-"
-}
-
-func (s *s3VolumeAdder) Set(bucketName string) error {
- if bucketName == "" {
- return fmt.Errorf("no container name given")
- }
- if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
- return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
- }
- if deprecated.flagSerializeIO {
- log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
- }
- s.Config.Volumes = append(s.Config.Volumes, &S3Volume{
- Bucket: bucketName,
- AccessKeyFile: s3AccessKeyFile,
- SecretKeyFile: s3SecretKeyFile,
- Endpoint: s3Endpoint,
- Region: s3RegionName,
- RaceWindow: arvados.Duration(s3RaceWindow),
- S3Replication: s3Replication,
- UnsafeDelete: s3UnsafeDelete,
- ReadOnly: deprecated.flagReadonly,
- IndexPageSize: 1000,
- })
- return nil
-}
-
-func s3regions() (okList []string) {
- for r := range aws.Regions {
- okList = append(okList, r)
- }
- return
-}
-
func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &S3Volume{} })
-
- flag.Var(&s3VolumeAdder{theConfig},
- "s3-bucket-volume",
- "Use the given bucket as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &s3RegionName,
- "s3-region",
- "",
- fmt.Sprintf("AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are %+q.", s3regions()))
- flag.StringVar(
- &s3Endpoint,
- "s3-endpoint",
- "",
- "Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use \"https://storage.googleapis.com\".")
- flag.StringVar(
- &s3AccessKeyFile,
- "s3-access-key-file",
- "",
- "`File` containing the access key used for subsequent -s3-bucket-volume arguments.")
- flag.StringVar(
- &s3SecretKeyFile,
- "s3-secret-key-file",
- "",
- "`File` containing the secret key used for subsequent -s3-bucket-volume arguments.")
- flag.DurationVar(
- &s3RaceWindow,
- "s3-race-window",
- 24*time.Hour,
- "Maximum eventual consistency latency for subsequent -s3-bucket-volume arguments.")
- flag.IntVar(
- &s3Replication,
- "s3-replication",
- 2,
- "Replication level reported to clients for subsequent -s3-bucket-volume arguments.")
- flag.BoolVar(
- &s3UnsafeDelete,
- "s3-unsafe-delete",
- false,
- "EXPERIMENTAL. Enable deletion (garbage collection) even when trash lifetime is zero, even though there are known race conditions that can cause data loss.")
+ driver["S3"] = newS3Volume
}
-// S3Volume implements Volume using an S3 bucket.
-type S3Volume struct {
- AccessKeyFile string
- SecretKeyFile string
- Endpoint string
- Region string
- Bucket string
- LocationConstraint bool
- IndexPageSize int
- S3Replication int
- ConnectTimeout arvados.Duration
- ReadTimeout arvados.Duration
- RaceWindow arvados.Duration
- ReadOnly bool
- UnsafeDelete bool
- StorageClasses []string
-
- bucket *s3bucket
-
- startOnce sync.Once
-}
-
-// Examples implements VolumeWithExamples.
-func (*S3Volume) Examples() []Volume {
- return []Volume{
- &S3Volume{
- AccessKeyFile: "/etc/aws_s3_access_key.txt",
- SecretKeyFile: "/etc/aws_s3_secret_key.txt",
- Endpoint: "",
- Region: "us-east-1",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
- &S3Volume{
- AccessKeyFile: "/etc/gce_s3_access_key.txt",
- SecretKeyFile: "/etc/gce_s3_secret_key.txt",
- Endpoint: "https://storage.googleapis.com",
- Region: "",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
+func newS3Volume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &S3Volume{cluster: cluster, volume: volume, logger: logger, metrics: metrics}
+ err := json.Unmarshal(volume.DriverParameters, &v)
+ if err != nil {
+ return nil, err
}
+ return v, v.check()
}
-// Type implements Volume.
-func (*S3Volume) Type() string {
- return "S3"
-}
+func (v *S3Volume) check() error {
+ if v.Bucket == "" || v.AccessKey == "" || v.SecretKey == "" {
+ return errors.New("DriverParameters: Bucket, AccessKey, and SecretKey must be provided")
+ }
+ if v.IndexPageSize == 0 {
+ v.IndexPageSize = 1000
+ }
+ if v.RaceWindow < 0 {
+ return errors.New("DriverParameters: RaceWindow must not be negative")
+ }
-// Start populates private fields and verifies the configuration is
-// valid.
-func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
region, ok := aws.Regions[v.Region]
if v.Endpoint == "" {
if !ok {
- return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region)
+ return fmt.Errorf("unrecognized region %+q; try specifying endpoint instead", v.Region)
}
} else if ok {
return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
- "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint)
+ "specify empty endpoint or use a different region name", v.Region, v.Endpoint)
} else {
region = aws.Region{
Name: v.Region,
@@ -216,15 +71,9 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
}
}
- var err error
- var auth aws.Auth
- auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile)
- if err != nil {
- return err
- }
- auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile)
- if err != nil {
- return err
+ auth := aws.Auth{
+ AccessKey: v.AccessKey,
+ SecretKey: v.SecretKey,
}
// Zero timeouts mean "wait forever", which is a bad
@@ -250,14 +99,63 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
},
}
// Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls)
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
return nil
}
-// DeviceID returns a globally unique ID for the storage bucket.
-func (v *S3Volume) DeviceID() string {
+const (
+ s3DefaultReadTimeout = arvados.Duration(10 * time.Minute)
+ s3DefaultConnectTimeout = arvados.Duration(time.Minute)
+)
+
+var (
+ // ErrS3TrashDisabled is returned by Trash if that operation
+ // is impossible with the current config.
+ ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
+
+ s3ACL = s3.Private
+
+ zeroTime time.Time
+)
+
+const (
+ maxClockSkew = 600 * time.Second
+ nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
+)
+
+func s3regions() (okList []string) {
+ for r := range aws.Regions {
+ okList = append(okList, r)
+ }
+ return
+}
+
+// S3Volume implements Volume using an S3 bucket.
+type S3Volume struct {
+ AccessKey string
+ SecretKey string
+ Endpoint string
+ Region string
+ Bucket string
+ LocationConstraint bool
+ IndexPageSize int
+ ConnectTimeout arvados.Duration
+ ReadTimeout arvados.Duration
+ RaceWindow arvados.Duration
+ UnsafeDelete bool
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ bucket *s3bucket
+ startOnce sync.Once
+}
+
+// GetDeviceID returns a globally unique ID for the storage bucket.
+func (v *S3Volume) GetDeviceID() string {
return "s3://" + v.Endpoint + "/" + v.Bucket
}
@@ -271,7 +169,7 @@ func (v *S3Volume) getReaderWithContext(ctx context.Context, loc string) (rdr io
case <-ready:
return
case <-ctx.Done():
- theConfig.debugLogf("s3: abandoning getReader(): %s", ctx.Err())
+ v.logger.Debugf("s3: abandoning getReader(): %s", ctx.Err())
go func() {
<-ready
if err == nil {
@@ -339,11 +237,11 @@ func (v *S3Volume) Get(ctx context.Context, loc string, buf []byte) (int, error)
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
+ v.logger.Debugf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
rdr.Close()
// Must wait for ReadFull to return, to ensure it
// doesn't write to buf after we return.
- theConfig.debugLogf("s3: waiting for ReadFull() to fail")
+ v.logger.Debug("s3: waiting for ReadFull() to fail")
<-ready
return 0, ctx.Err()
case <-ready:
@@ -397,7 +295,7 @@ func (v *S3Volume) Compare(ctx context.Context, loc string, expect []byte) error
// Put writes a block.
func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
var opts s3.Options
@@ -433,7 +331,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go func() {
defer func() {
if ctx.Err() != nil {
- theConfig.debugLogf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
+ v.logger.Debugf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
}
}()
defer close(ready)
@@ -445,7 +343,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking PutReader's input away: %s", v, ctx.Err())
+ v.logger.Debugf("%s: taking PutReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// PutReader() to read. If so, un-stick it. This means
// PutReader will get corrupt data, but that's OK: the
@@ -453,7 +351,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning PutReader goroutine", v)
+ v.logger.Debugf("%s: abandoning PutReader goroutine", v)
return ctx.Err()
case <-ready:
// Unblock pipe in case PutReader did not consume it.
@@ -464,7 +362,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
// Touch sets the timestamp for the given locator to the current time.
func (v *S3Volume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
_, err := v.bucket.Head(loc, nil)
@@ -571,16 +469,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
// Trash a Keep block.
func (v *S3Volume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
- if theConfig.TrashLifetime == 0 {
- if !s3UnsafeDelete {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
+ if !v.UnsafeDelete {
return ErrS3TrashDisabled
}
return v.translateError(v.bucket.Del(loc))
@@ -615,7 +513,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
// Can't parse timestamp
return err
}
- safeWindow := t.Add(theConfig.TrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
+ safeWindow := t.Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
if safeWindow <= 0 {
// We can't count on "touch trash/X" to prolong
// trash/X's lifetime. The new timestamp might not
@@ -698,23 +596,6 @@ func (v *S3Volume) String() string {
return fmt.Sprintf("s3-bucket:%+q", v.Bucket)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *S3Volume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the storage redundancy of the underlying
-// device. Configured via command line flag.
-func (v *S3Volume) Replication() int {
- return v.S3Replication
-}
-
-// GetStorageClasses implements Volume
-func (v *S3Volume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
func (v *S3Volume) isKeepBlock(s string) bool {
@@ -751,13 +632,13 @@ func (v *S3Volume) fixRace(loc string) bool {
}
ageWhenTrashed := trashTime.Sub(recentTime)
- if ageWhenTrashed >= theConfig.BlobSignatureTTL.Duration() {
+ if ageWhenTrashed >= v.cluster.Collections.BlobSigningTTL.Duration() {
// No evidence of a race: block hasn't been written
// since it became eligible for Trash. No fix needed.
return false
}
- log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, theConfig.BlobSignatureTTL)
+ log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, v.cluster.Collections.BlobSigningTTL)
log.Printf("notice: fixRace: copying %q to %q to recover from race between Put/Touch and Trash", "recent/"+loc, loc)
err = v.safeCopy(loc, "trash/"+loc)
if err != nil {
@@ -785,6 +666,10 @@ func (v *S3Volume) translateError(err error) error {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *S3Volume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
// Define "ready to delete" as "...when EmptyTrash started".
@@ -820,8 +705,8 @@ func (v *S3Volume) EmptyTrash() {
log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
return
}
- if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
- if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
+ if trashT.Sub(recentT) < v.cluster.Collections.BlobSigningTTL.Duration() {
+ if age := startT.Sub(recentT); age >= v.cluster.Collections.BlobSigningTTL.Duration()-time.Duration(v.RaceWindow) {
// recent/loc is too old to protect
// loc from being Trashed again during
// the raceWindow that starts if we
@@ -845,7 +730,7 @@ func (v *S3Volume) EmptyTrash() {
return
}
}
- if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
+ if startT.Sub(trashT) < v.cluster.Collections.BlobTrashLifetime.Duration() {
return
}
err = v.bucket.Del(trash.Key)
@@ -872,8 +757,8 @@ func (v *S3Volume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan *s3.Key, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 6377420ff..b8c4458a5 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -10,17 +10,19 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
- "io/ioutil"
+ "log"
"net/http"
"net/http/httptest"
"os"
+ "strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/s3/s3test"
- "github.com/ghodss/yaml"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -39,34 +41,41 @@ func (c *fakeClock) Now() time.Time {
return *c.now
}
-func init() {
- // Deleting isn't safe from races, but if it's turned on
- // anyway we do expect it to pass the generic volume tests.
- s3UnsafeDelete = true
-}
-
var _ = check.Suite(&StubbedS3Suite{})
type StubbedS3Suite struct {
- volumes []*TestableS3Volume
+ s3server *httptest.Server
+ cluster *arvados.Cluster
+ handler *handler
+ volumes []*TestableS3Volume
+}
+
+func (s *StubbedS3Suite) SetUpTest(c *check.C) {
+ s.s3server = nil
+ s.cluster = testCluster(c)
+ s.cluster.Volumes = map[string]arvados.Volume{
+ "zzzzz-nyw5e-000000000000000": {Driver: "S3"},
+ "zzzzz-nyw5e-111111111111111": {Driver: "S3"},
+ }
+ s.handler = &handler{}
}
func (s *StubbedS3Suite) TestGeneric(c *check.C) {
- DoGenericVolumeTests(c, func(t TB) TestableVolume {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
// Use a negative raceWindow so s3test's 1-second
// timestamp precision doesn't confuse fixRace.
- return s.newTestableVolume(c, -2*time.Second, false, 2)
+ return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second)
})
}
func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
- DoGenericVolumeTests(c, func(t TB) TestableVolume {
- return s.newTestableVolume(c, -2*time.Second, true, 2)
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableVolume(c, cluster, volume, metrics, -2*time.Second)
})
}
func (s *StubbedS3Suite) TestIndex(c *check.C) {
- v := s.newTestableVolume(c, 0, false, 2)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 0)
v.IndexPageSize = 3
for i := 0; i < 256; i++ {
v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111})
@@ -91,7 +100,7 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) {
}
func (s *StubbedS3Suite) TestStats(c *check.C) {
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
stats := func() string {
buf, err := json.Marshal(v.InternalStats())
c.Check(err, check.IsNil)
@@ -125,6 +134,11 @@ type blockingHandler struct {
}
func (h *blockingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "PUT" && !strings.Contains(strings.Trim(r.URL.Path, "/"), "/") {
+ // Accept PutBucket ("PUT /bucketname/"), called by
+ // newTestableVolume
+ return
+ }
if h.requested != nil {
h.requested <- r
}
@@ -164,15 +178,10 @@ func (s *StubbedS3Suite) TestPutContextCancel(c *check.C) {
func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Context, *TestableS3Volume) error) {
handler := &blockingHandler{}
- srv := httptest.NewServer(handler)
- defer srv.Close()
+ s.s3server = httptest.NewServer(handler)
+ defer s.s3server.Close()
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
- vol := *v.S3Volume
- vol.Endpoint = srv.URL
- v = &TestableS3Volume{S3Volume: &vol}
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics)
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
ctx, cancel := context.WithCancel(context.Background())
@@ -209,14 +218,10 @@ func (s *StubbedS3Suite) testContextCancel(c *check.C, testFunc func(context.Con
}
func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
- defer func(tl, bs arvados.Duration) {
- theConfig.TrashLifetime = tl
- theConfig.BlobSignatureTTL = bs
- }(theConfig.TrashLifetime, theConfig.BlobSignatureTTL)
- theConfig.TrashLifetime.Set("1h")
- theConfig.BlobSignatureTTL.Set("1h")
-
- v := s.newTestableVolume(c, 5*time.Minute, false, 2)
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
+ s.cluster.Collections.BlobSigningTTL.Set("1h")
+
+ v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
var none time.Time
putS3Obj := func(t time.Time, key string, data []byte) {
@@ -411,61 +416,42 @@ type TestableS3Volume struct {
serverClock *fakeClock
}
-func (s *StubbedS3Suite) newTestableVolume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume {
+func (s *StubbedS3Suite) newTestableVolume(c *check.C, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs, raceWindow time.Duration) *TestableS3Volume {
clock := &fakeClock{}
srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
c.Assert(err, check.IsNil)
+ endpoint := srv.URL()
+ if s.s3server != nil {
+ endpoint = s.s3server.URL
+ }
v := &TestableS3Volume{
S3Volume: &S3Volume{
+ AccessKey: "xxx",
+ SecretKey: "xxx",
Bucket: TestBucketName,
- Endpoint: srv.URL(),
+ Endpoint: endpoint,
Region: "test-region-1",
LocationConstraint: true,
- RaceWindow: arvados.Duration(raceWindow),
- S3Replication: replication,
- UnsafeDelete: s3UnsafeDelete,
- ReadOnly: readonly,
+ UnsafeDelete: true,
IndexPageSize: 1000,
+ cluster: cluster,
+ volume: volume,
+ logger: ctxlog.TestLogger(c),
+ metrics: metrics,
},
c: c,
server: srv,
serverClock: clock,
}
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- v.Start(metrics)
- err = v.bucket.PutBucket(s3.ACL("private"))
- c.Assert(err, check.IsNil)
+ c.Assert(v.S3Volume.check(), check.IsNil)
+ c.Assert(v.bucket.PutBucket(s3.ACL("private")), check.IsNil)
+ // We couldn't set RaceWindow until now because check()
+ // rejects negative values.
+ v.S3Volume.RaceWindow = arvados.Duration(raceWindow)
return v
}
-func (s *StubbedS3Suite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: S3
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
-
-func (v *TestableS3Volume) Start(vm *volumeMetricsVecs) error {
- tmp, err := ioutil.TempFile("", "keepstore")
- v.c.Assert(err, check.IsNil)
- defer os.Remove(tmp.Name())
- _, err = tmp.Write([]byte("xxx\n"))
- v.c.Assert(err, check.IsNil)
- v.c.Assert(tmp.Close(), check.IsNil)
-
- v.S3Volume.AccessKeyFile = tmp.Name()
- v.S3Volume.SecretKeyFile = tmp.Name()
-
- v.c.Assert(v.S3Volume.Start(vm), check.IsNil)
- return nil
-}
-
// PutRaw skips the ContentMD5 test
func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
diff --git a/services/keepstore/server.go b/services/keepstore/server.go
deleted file mode 100644
index 3f6727712..000000000
--- a/services/keepstore/server.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "crypto/tls"
- "net"
- "net/http"
- "os"
- "os/signal"
- "syscall"
-)
-
-type server struct {
- http.Server
-
- // channel (size=1) with the current keypair
- currentCert chan *tls.Certificate
-}
-
-func (srv *server) Serve(l net.Listener) error {
- if theConfig.TLSCertificateFile == "" && theConfig.TLSKeyFile == "" {
- return srv.Server.Serve(l)
- }
- // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
- srv.TLSConfig = &tls.Config{
- GetCertificate: srv.getCertificate,
- PreferServerCipherSuites: true,
- CurvePreferences: []tls.CurveID{
- tls.CurveP256,
- tls.X25519,
- },
- MinVersion: tls.VersionTLS12,
- CipherSuites: []uint16{
- tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
- tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
- },
- }
- srv.currentCert = make(chan *tls.Certificate, 1)
- go srv.refreshCertificate(theConfig.TLSCertificateFile, theConfig.TLSKeyFile)
- return srv.Server.ServeTLS(l, "", "")
-}
-
-func (srv *server) refreshCertificate(certfile, keyfile string) {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Fatal("error loading X509 key pair")
- }
- srv.currentCert <- &cert
-
- reload := make(chan os.Signal, 1)
- signal.Notify(reload, syscall.SIGHUP)
- for range reload {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Warn("error loading X509 key pair")
- continue
- }
- // Throw away old cert and start using new one
- <-srv.currentCert
- srv.currentCert <- &cert
- }
-}
-
-func (srv *server) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
- if srv.currentCert == nil {
- panic("srv.currentCert not initialized")
- }
- cert := <-srv.currentCert
- srv.currentCert <- cert
- return cert, nil
-}
diff --git a/services/keepstore/server_test.go b/services/keepstore/server_test.go
deleted file mode 100644
index 84adf3603..000000000
--- a/services/keepstore/server_test.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "context"
- "crypto/tls"
- "io/ioutil"
- "net"
- "net/http"
- "testing"
-)
-
-func TestTLS(t *testing.T) {
- defer func() {
- theConfig.TLSKeyFile = ""
- theConfig.TLSCertificateFile = ""
- }()
- theConfig.TLSKeyFile = "../api/tmp/self-signed.key"
- theConfig.TLSCertificateFile = "../api/tmp/self-signed.pem"
- srv := &server{}
- srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte("OK"))
- })
- l, err := net.Listen("tcp", ":")
- if err != nil {
- t.Fatal(err)
- }
- defer l.Close()
- go srv.Serve(l)
- defer srv.Shutdown(context.Background())
- c := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
- resp, err := c.Get("https://" + l.Addr().String() + "/")
- if err != nil {
- t.Fatal(err)
- }
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- t.Error(err)
- }
- if !bytes.Equal(body, []byte("OK")) {
- t.Errorf("expected OK, got %q", body)
- }
-}
diff --git a/services/keepstore/status_test.go b/services/keepstore/status_test.go
index dc6efb083..7bff2584e 100644
--- a/services/keepstore/status_test.go
+++ b/services/keepstore/status_test.go
@@ -14,8 +14,8 @@ import (
// getStatusItem("foo","bar","baz") retrieves /status.json, decodes
// the response body into resp, and returns resp["foo"]["bar"]["baz"].
-func getStatusItem(keys ...string) interface{} {
- resp := IssueRequest(&RequestTester{"/status.json", "", "GET", nil})
+func getStatusItem(h *handler, keys ...string) interface{} {
+ resp := IssueRequest(h, &RequestTester{"/status.json", "", "GET", nil})
var s interface{}
json.NewDecoder(resp.Body).Decode(&s)
for _, k := range keys {
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 8a9fedfb7..ba1455ac6 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -6,6 +6,7 @@ package main
import (
"errors"
+ "log"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -17,35 +18,35 @@ import (
// Delete the block indicated by the trash request Locator
// Repeat
//
-func RunTrashWorker(trashq *WorkQueue) {
+func RunTrashWorker(volmgr *RRVolumeManager, cluster *arvados.Cluster, trashq *WorkQueue) {
for item := range trashq.NextItem {
trashRequest := item.(TrashRequest)
- TrashItem(trashRequest)
+ TrashItem(volmgr, cluster, trashRequest)
trashq.DoneItem <- struct{}{}
}
}
// TrashItem deletes the indicated block from every writable volume.
-func TrashItem(trashRequest TrashRequest) {
+func TrashItem(volmgr *RRVolumeManager, cluster *arvados.Cluster, trashRequest TrashRequest) {
reqMtime := time.Unix(0, trashRequest.BlockMtime)
- if time.Since(reqMtime) < theConfig.BlobSignatureTTL.Duration() {
+ if time.Since(reqMtime) < cluster.Collections.BlobSigningTTL.Duration() {
log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
arvados.Duration(time.Since(reqMtime)),
trashRequest.Locator,
trashRequest.BlockMtime,
reqMtime,
- theConfig.BlobSignatureTTL)
+ cluster.Collections.BlobSigningTTL)
return
}
- var volumes []Volume
+ var volumes []*VolumeMount
if uuid := trashRequest.MountUUID; uuid == "" {
- volumes = KeepVM.AllWritable()
- } else if v := KeepVM.Lookup(uuid, true); v == nil {
+ volumes = volmgr.AllWritable()
+ } else if mnt := volmgr.Lookup(uuid, true); mnt == nil {
log.Printf("warning: trash request for nonexistent mount: %v", trashRequest)
return
} else {
- volumes = []Volume{v}
+ volumes = []*VolumeMount{mnt}
}
for _, volume := range volumes {
@@ -59,8 +60,8 @@ func TrashItem(trashRequest TrashRequest) {
continue
}
- if !theConfig.EnableDelete {
- err = errors.New("skipping because EnableDelete is false")
+ if !cluster.Collections.BlobTrash {
+ err = errors.New("skipping because Collections.BlobTrash is false")
} else {
err = volume.Trash(trashRequest.Locator)
}
diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index c5a410b06..bd3743090 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -7,8 +7,10 @@ package main
import (
"container/list"
"context"
- "testing"
"time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ check "gopkg.in/check.v1"
)
type TrashWorkerTestData struct {
@@ -36,8 +38,8 @@ type TrashWorkerTestData struct {
/* Delete block that does not exist in any of the keep volumes.
Expect no errors.
*/
-func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_GetNonExistingLocator(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: "5d41402abc4b2a76b9719d911017c592",
Block1: []byte("hello"),
@@ -52,14 +54,14 @@ func TestTrashWorkerIntegration_GetNonExistingLocator(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block that exists on volume 1 of the keep servers.
Expect the second locator in volume 2 to be unaffected.
*/
-func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInVolume1(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -74,14 +76,14 @@ func TestTrashWorkerIntegration_LocatorInVolume1(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block that exists on volume 2 of the keep servers.
Expect the first locator in volume 1 to be unaffected.
*/
-func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInVolume2(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -96,14 +98,14 @@ func TestTrashWorkerIntegration_LocatorInVolume2(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block with matching mtime for locator in both volumes.
Expect locator to be deleted from both volumes.
*/
-func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_LocatorInBothVolumes(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -118,14 +120,14 @@ func TestTrashWorkerIntegration_LocatorInBothVolumes(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Same locator with different Mtimes exists in both volumes.
Delete the second and expect the first to be still around.
*/
-func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -141,14 +143,14 @@ func TestTrashWorkerIntegration_MtimeMatchesForLocator1ButNotForLocator2(t *test
ExpectLocator1: true,
ExpectLocator2: false,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
// Delete a block that exists on both volumes with matching mtimes,
// but specify a MountUUID in the request so it only gets deleted from
// the first volume.
-func TestTrashWorkerIntegration_SpecifyMountUUID(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_SpecifyMountUUID(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -164,15 +166,15 @@ func TestTrashWorkerIntegration_SpecifyMountUUID(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Two different locators in volume 1.
Delete one of them.
Expect the other unaffected.
*/
-func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -188,14 +190,14 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
ExpectLocator1: false,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Allow default Trash Life time to be used. Thus, the newly created block
will not be deleted because its Mtime is within the trash life time.
*/
-func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
- theConfig.EnableDelete = true
+func (s *HandlerSuite) TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(c *check.C) {
+ s.cluster.Collections.BlobTrash = true
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -214,14 +216,14 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Delete a block with matching mtime for locator in both volumes, but EnableDelete is false,
so block won't be deleted.
*/
-func TestTrashWorkerIntegration_DisabledDelete(t *testing.T) {
- theConfig.EnableDelete = false
+func (s *HandlerSuite) TestTrashWorkerIntegration_DisabledDelete(c *check.C) {
+ s.cluster.Collections.BlobTrash = false
testData := TrashWorkerTestData{
Locator1: TestHash,
Block1: TestBlock,
@@ -236,31 +238,34 @@ func TestTrashWorkerIntegration_DisabledDelete(t *testing.T) {
ExpectLocator1: true,
ExpectLocator2: true,
}
- performTrashWorkerTest(testData, t)
+ s.performTrashWorkerTest(c, testData)
}
/* Perform the test */
-func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
- // Create Keep Volumes
- KeepVM = MakeTestVolumeManager(2)
- defer KeepVM.Close()
+func (s *HandlerSuite) performTrashWorkerTest(c *check.C, testData TrashWorkerTestData) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ // Replace the router's trashq -- which the worker goroutines
+ // started by setup() are now receiving from -- with a new
+ // one, so we can see what the handler sends to it.
+ trashq := NewWorkQueue()
+ s.handler.Handler.(*router).trashq = trashq
// Put test content
- vols := KeepVM.AllWritable()
+ mounts := s.handler.volmgr.AllWritable()
if testData.CreateData {
- vols[0].Put(context.Background(), testData.Locator1, testData.Block1)
- vols[0].Put(context.Background(), testData.Locator1+".meta", []byte("metadata"))
+ mounts[0].Put(context.Background(), testData.Locator1, testData.Block1)
+ mounts[0].Put(context.Background(), testData.Locator1+".meta", []byte("metadata"))
if testData.CreateInVolume1 {
- vols[0].Put(context.Background(), testData.Locator2, testData.Block2)
- vols[0].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
+ mounts[0].Put(context.Background(), testData.Locator2, testData.Block2)
+ mounts[0].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
} else {
- vols[1].Put(context.Background(), testData.Locator2, testData.Block2)
- vols[1].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
+ mounts[1].Put(context.Background(), testData.Locator2, testData.Block2)
+ mounts[1].Put(context.Background(), testData.Locator2+".meta", []byte("metadata"))
}
}
- oldBlockTime := time.Now().Add(-theConfig.BlobSignatureTTL.Duration() - time.Minute)
+ oldBlockTime := time.Now().Add(-s.cluster.Collections.BlobSigningTTL.Duration() - time.Minute)
// Create TrashRequest for the test
trashRequest := TrashRequest{
@@ -268,37 +273,35 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
BlockMtime: oldBlockTime.UnixNano(),
}
if testData.SpecifyMountUUID {
- trashRequest.MountUUID = KeepVM.Mounts()[0].UUID
+ trashRequest.MountUUID = s.handler.volmgr.Mounts()[0].UUID
}
// Run trash worker and put the trashRequest on trashq
trashList := list.New()
trashList.PushBack(trashRequest)
- trashq = NewWorkQueue()
- defer trashq.Close()
if !testData.UseTrashLifeTime {
// Trash worker would not delete block if its Mtime is
// within trash life time. Back-date the block to
// allow the deletion to succeed.
- for _, v := range vols {
- v.(*MockVolume).Timestamps[testData.DeleteLocator] = oldBlockTime
+ for _, mnt := range mounts {
+ mnt.Volume.(*MockVolume).Timestamps[testData.DeleteLocator] = oldBlockTime
if testData.DifferentMtimes {
oldBlockTime = oldBlockTime.Add(time.Second)
}
}
}
- go RunTrashWorker(trashq)
+ go RunTrashWorker(s.handler.volmgr, s.cluster, trashq)
// Install gate so all local operations block until we say go
gate := make(chan struct{})
- for _, v := range vols {
- v.(*MockVolume).Gate = gate
+ for _, mnt := range mounts {
+ mnt.Volume.(*MockVolume).Gate = gate
}
assertStatusItem := func(k string, expect float64) {
- if v := getStatusItem("TrashQueue", k); v != expect {
- t.Errorf("Got %s %v, expected %v", k, v, expect)
+ if v := getStatusItem(s.handler, "TrashQueue", k); v != expect {
+ c.Errorf("Got %s %v, expected %v", k, v, expect)
}
}
@@ -309,7 +312,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
trashq.ReplaceQueue(trashList)
// Wait for worker to take request(s)
- expectEqualWithin(t, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
+ expectEqualWithin(c, time.Second, listLen, func() interface{} { return trashq.Status().InProgress })
// Ensure status.json also reports work is happening
assertStatusItem("InProgress", float64(1))
@@ -319,31 +322,31 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
close(gate)
// Wait for worker to finish
- expectEqualWithin(t, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
+ expectEqualWithin(c, time.Second, 0, func() interface{} { return trashq.Status().InProgress })
// Verify Locator1 to be un/deleted as expected
buf := make([]byte, BlockSize)
- size, err := GetBlock(context.Background(), testData.Locator1, buf, nil)
+ size, err := GetBlock(context.Background(), s.handler.volmgr, testData.Locator1, buf, nil)
if testData.ExpectLocator1 {
if size == 0 || err != nil {
- t.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
+ c.Errorf("Expected Locator1 to be still present: %s", testData.Locator1)
}
} else {
if size > 0 || err == nil {
- t.Errorf("Expected Locator1 to be deleted: %s", testData.Locator1)
+ c.Errorf("Expected Locator1 to be deleted: %s", testData.Locator1)
}
}
// Verify Locator2 to be un/deleted as expected
if testData.Locator1 != testData.Locator2 {
- size, err = GetBlock(context.Background(), testData.Locator2, buf, nil)
+ size, err = GetBlock(context.Background(), s.handler.volmgr, testData.Locator2, buf, nil)
if testData.ExpectLocator2 {
if size == 0 || err != nil {
- t.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)
+ c.Errorf("Expected Locator2 to be still present: %s", testData.Locator2)
}
} else {
if size > 0 || err == nil {
- t.Errorf("Expected Locator2 to be deleted: %s", testData.Locator2)
+ c.Errorf("Expected Locator2 to be deleted: %s", testData.Locator2)
}
}
}
@@ -353,14 +356,12 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
// the trash request.
if testData.DifferentMtimes {
locatorFoundIn := 0
- for _, volume := range KeepVM.AllReadable() {
+ for _, volume := range s.handler.volmgr.AllReadable() {
buf := make([]byte, BlockSize)
if _, err := volume.Get(context.Background(), testData.Locator1, buf); err == nil {
locatorFoundIn = locatorFoundIn + 1
}
}
- if locatorFoundIn != 1 {
- t.Errorf("Found %d copies of %s, expected 1", locatorFoundIn, testData.Locator1)
- }
+ c.Check(locatorFoundIn, check.Equals, 1)
}
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 4d9e798ac..918555c2b 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -5,12 +5,13 @@
package main
import (
- "bufio"
"context"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"os"
"os/exec"
"path/filepath"
@@ -22,98 +23,52 @@ import (
"syscall"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-type unixVolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (vs *unixVolumeAdder) String() string {
- return "-"
+func init() {
+ driver["Directory"] = newDirectoryVolume
}
-func (vs *unixVolumeAdder) Set(path string) error {
- if dirs := strings.Split(path, ","); len(dirs) > 1 {
- log.Print("DEPRECATED: using comma-separated volume list.")
- for _, dir := range dirs {
- if err := vs.Set(dir); err != nil {
- return err
- }
- }
- return nil
+func newDirectoryVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
+ v := &UnixVolume{cluster: cluster, volume: volume, logger: logger, metrics: metrics}
+ err := json.Unmarshal(volume.DriverParameters, &v)
+ if err != nil {
+ return nil, err
}
- vs.Config.Volumes = append(vs.Config.Volumes, &UnixVolume{
- Root: path,
- ReadOnly: deprecated.flagReadonly,
- Serialize: deprecated.flagSerializeIO,
- })
- return nil
-}
-
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &UnixVolume{} })
-
- flag.Var(&unixVolumeAdder{theConfig}, "volumes", "see Volumes configuration")
- flag.Var(&unixVolumeAdder{theConfig}, "volume", "see Volumes configuration")
+ return v, v.check()
}
-// Discover adds a UnixVolume for every directory named "keep" that is
-// located at the top level of a device- or tmpfs-backed mount point
-// other than "/". It returns the number of volumes added.
-func (vs *unixVolumeAdder) Discover() int {
- added := 0
- f, err := os.Open(ProcMounts)
- if err != nil {
- log.Fatalf("opening %s: %s", ProcMounts, err)
+func (v *UnixVolume) check() error {
+ if v.Root == "" {
+ return errors.New("DriverParameters.Root was not provided")
}
- scanner := bufio.NewScanner(f)
- for scanner.Scan() {
- args := strings.Fields(scanner.Text())
- if err := scanner.Err(); err != nil {
- log.Fatalf("reading %s: %s", ProcMounts, err)
- }
- dev, mount := args[0], args[1]
- if mount == "/" {
- continue
- }
- if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
- continue
- }
- keepdir := mount + "/keep"
- if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
- continue
- }
- // Set the -readonly flag (but only for this volume)
- // if the filesystem is mounted readonly.
- flagReadonlyWas := deprecated.flagReadonly
- for _, fsopt := range strings.Split(args[3], ",") {
- if fsopt == "ro" {
- deprecated.flagReadonly = true
- break
- }
- if fsopt == "rw" {
- break
- }
- }
- if err := vs.Set(keepdir); err != nil {
- log.Printf("adding %q: %s", keepdir, err)
- } else {
- added++
- }
- deprecated.flagReadonly = flagReadonlyWas
+ if v.Serialize {
+ v.locker = &sync.Mutex{}
+ }
+ if !strings.HasPrefix(v.Root, "/") {
+ return fmt.Errorf("DriverParameters.Root %q does not start with '/'", v.Root)
}
- return added
+
+ // Set up prometheus metrics
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
+ v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = v.metrics.getCounterVecsFor(lbls)
+
+ _, err := v.os.Stat(v.Root)
+ return err
}
// A UnixVolume stores and retrieves blocks in a local directory.
type UnixVolume struct {
- Root string // path to the volume's root directory
- ReadOnly bool
- Serialize bool
- DirectoryReplication int
- StorageClasses []string
+ Root string // path to the volume's root directory
+ Serialize bool
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
// something to lock during IO, typically a sync.Mutex (or nil
// to skip locking)
@@ -122,12 +77,12 @@ type UnixVolume struct {
os osWithStats
}
-// DeviceID returns a globally unique ID for the volume's root
+// GetDeviceID returns a globally unique ID for the volume's root
// directory, consisting of the filesystem's UUID and the path from
// filesystem root to storage directory, joined by "/". For example,
-// the DeviceID for a local directory "/mnt/xvda1/keep" might be
+// the device ID for a local directory "/mnt/xvda1/keep" might be
// "fa0b6166-3b55-4994-bd3f-92f4e00a1bb0/keep".
-func (v *UnixVolume) DeviceID() string {
+func (v *UnixVolume) GetDeviceID() string {
giveup := func(f string, args ...interface{}) string {
log.Printf(f+"; using blank DeviceID for volume %s", append(args, v)...)
return ""
@@ -198,50 +153,9 @@ func (v *UnixVolume) DeviceID() string {
return giveup("could not find entry in %q matching %q", udir, dev)
}
-// Examples implements VolumeWithExamples.
-func (*UnixVolume) Examples() []Volume {
- return []Volume{
- &UnixVolume{
- Root: "/mnt/local-disk",
- Serialize: true,
- DirectoryReplication: 1,
- },
- &UnixVolume{
- Root: "/mnt/network-disk",
- Serialize: false,
- DirectoryReplication: 2,
- },
- }
-}
-
-// Type implements Volume
-func (v *UnixVolume) Type() string {
- return "Directory"
-}
-
-// Start implements Volume
-func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
- if v.Serialize {
- v.locker = &sync.Mutex{}
- }
- if !strings.HasPrefix(v.Root, "/") {
- return fmt.Errorf("volume root does not start with '/': %q", v.Root)
- }
- if v.DirectoryReplication == 0 {
- v.DirectoryReplication = 1
- }
- // Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
- v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = vm.getCounterVecsFor(lbls)
-
- _, err := v.os.Stat(v.Root)
-
- return err
-}
-
// Touch sets the timestamp for the given locator to the current time
func (v *UnixVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
p := v.blockPath(loc)
@@ -349,7 +263,7 @@ func (v *UnixVolume) Put(ctx context.Context, loc string, block []byte) error {
// WriteBlock implements BlockWriter.
func (v *UnixVolume) WriteBlock(ctx context.Context, loc string, rdr io.Reader) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if v.IsFull() {
@@ -516,7 +430,7 @@ func (v *UnixVolume) Trash(loc string) error {
// Trash() will read the correct up-to-date timestamp and choose not to
// trash the file.
- if v.ReadOnly {
+ if v.volume.ReadOnly || !v.cluster.Collections.BlobTrash {
return MethodDisabledError
}
if err := v.lock(context.TODO()); err != nil {
@@ -541,21 +455,21 @@ func (v *UnixVolume) Trash(loc string) error {
// anyway (because the permission signatures have expired).
if fi, err := v.os.Stat(p); err != nil {
return err
- } else if time.Since(fi.ModTime()) < time.Duration(theConfig.BlobSignatureTTL) {
+ } else if time.Since(fi.ModTime()) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
- if theConfig.TrashLifetime == 0 {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.os.Remove(p)
}
- return v.os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()))
+ return v.os.Rename(p, fmt.Sprintf("%v.trash.%d", p, time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()))
}
// Untrash moves block from trash back into store
// Look for path/{loc}.trash.{deadline} in storage,
// and rename the first such file as path/{loc}
func (v *UnixVolume) Untrash(loc string) (err error) {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -650,23 +564,6 @@ func (v *UnixVolume) String() string {
return fmt.Sprintf("[UnixVolume %s]", v.Root)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *UnixVolume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the number of replicas promised by the
-// underlying device (as specified in configuration).
-func (v *UnixVolume) Replication() int {
- return v.DirectoryReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *UnixVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// InternalStats returns I/O and filesystem ops counters.
func (v *UnixVolume) InternalStats() interface{} {
return &v.os.stats
@@ -739,6 +636,10 @@ var unixTrashLocRegexp = regexp.MustCompile(`/([0-9a-f]{32})\.trash\.(\d+)$`)
// EmptyTrash walks hierarchy looking for {hash}.trash.*
// and deletes those with deadline < now.
func (v *UnixVolume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
@@ -774,8 +675,8 @@ func (v *UnixVolume) EmptyTrash() {
info os.FileInfo
}
var wg sync.WaitGroup
- todo := make(chan dirent, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan dirent, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index 872f408cf..1ffc46513 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -16,11 +16,11 @@ import (
"strings"
"sync"
"syscall"
- "testing"
"time"
- "github.com/ghodss/yaml"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
check "gopkg.in/check.v1"
)
@@ -29,32 +29,13 @@ type TestableUnixVolume struct {
t TB
}
-func NewTestableUnixVolume(t TB, serialize bool, readonly bool) *TestableUnixVolume {
- d, err := ioutil.TempDir("", "volume_test")
- if err != nil {
- t.Fatal(err)
- }
- var locker sync.Locker
- if serialize {
- locker = &sync.Mutex{}
- }
- return &TestableUnixVolume{
- UnixVolume: UnixVolume{
- Root: d,
- ReadOnly: readonly,
- locker: locker,
- },
- t: t,
- }
-}
-
// PutRaw writes a Keep block directly into a UnixVolume, even if
// the volume is readonly.
func (v *TestableUnixVolume) PutRaw(locator string, data []byte) {
defer func(orig bool) {
- v.ReadOnly = orig
- }(v.ReadOnly)
- v.ReadOnly = false
+ v.volume.ReadOnly = orig
+ }(v.volume.ReadOnly)
+ v.volume.ReadOnly = false
err := v.Put(context.Background(), locator, data)
if err != nil {
v.t.Fatal(err)
@@ -70,7 +51,7 @@ func (v *TestableUnixVolume) TouchWithDate(locator string, lastPut time.Time) {
func (v *TestableUnixVolume) Teardown() {
if err := os.RemoveAll(v.Root); err != nil {
- v.t.Fatal(err)
+ v.t.Error(err)
}
}
@@ -78,59 +59,77 @@ func (v *TestableUnixVolume) ReadWriteOperationLabelValues() (r, w string) {
return "open", "create"
}
+var _ = check.Suite(&UnixVolumeSuite{})
+
+type UnixVolumeSuite struct {
+ cluster *arvados.Cluster
+ volumes []*TestableUnixVolume
+ metrics *volumeMetricsVecs
+}
+
+func (s *UnixVolumeSuite) SetUpTest(c *check.C) {
+ s.cluster = testCluster(c)
+ s.metrics = newVolumeMetricsVecs(prometheus.NewRegistry())
+}
+
+func (s *UnixVolumeSuite) TearDownTest(c *check.C) {
+ for _, v := range s.volumes {
+ v.Teardown()
+ }
+}
+
+func (s *UnixVolumeSuite) newTestableUnixVolume(c *check.C, cluster *arvados.Cluster, volume arvados.Volume, metrics *volumeMetricsVecs, serialize bool) *TestableUnixVolume {
+ d, err := ioutil.TempDir("", "volume_test")
+ c.Check(err, check.IsNil)
+ var locker sync.Locker
+ if serialize {
+ locker = &sync.Mutex{}
+ }
+ v := &TestableUnixVolume{
+ UnixVolume: UnixVolume{
+ Root: d,
+ locker: locker,
+ cluster: cluster,
+ volume: volume,
+ metrics: metrics,
+ },
+ t: c,
+ }
+ c.Check(v.check(), check.IsNil)
+ s.volumes = append(s.volumes, v)
+ return v
+}
+
// serialize = false; readonly = false
-func TestUnixVolumeWithGenericTests(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTests(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, false)
})
}
// serialize = false; readonly = true
-func TestUnixVolumeWithGenericTestsReadOnly(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, false, true)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTestsReadOnly(c *check.C) {
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, true)
})
}
// serialize = true; readonly = false
-func TestUnixVolumeWithGenericTestsSerialized(t *testing.T) {
- DoGenericVolumeTests(t, func(t TB) TestableVolume {
- return NewTestableUnixVolume(t, true, false)
+func (s *UnixVolumeSuite) TestUnixVolumeWithGenericTestsSerialized(c *check.C) {
+ DoGenericVolumeTests(c, false, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, false)
})
}
-// serialize = false; readonly = false
-func TestUnixVolumeHandlersWithGenericVolumeTests(t *testing.T) {
- DoHandlersWithGenericVolumeTests(t, func(t TB) (*RRVolumeManager, []TestableVolume) {
- vols := make([]Volume, 2)
- testableUnixVols := make([]TestableVolume, 2)
-
- for i := range vols {
- v := NewTestableUnixVolume(t, false, false)
- vols[i] = v
- testableUnixVols[i] = v
- }
-
- return MakeRRVolumeManager(vols), testableUnixVols
+// serialize = true; readonly = true
+func (s *UnixVolumeSuite) TestUnixVolumeHandlersWithGenericVolumeTests(c *check.C) {
+ DoGenericVolumeTests(c, true, func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume {
+ return s.newTestableUnixVolume(c, cluster, volume, metrics, true)
})
}
-func TestReplicationDefault1(t *testing.T) {
- v := &UnixVolume{
- Root: "/",
- ReadOnly: true,
- }
- metrics := newVolumeMetricsVecs(prometheus.NewRegistry())
- if err := v.Start(metrics); err != nil {
- t.Error(err)
- }
- if got := v.Replication(); got != 1 {
- t.Errorf("Replication() returned %d, expected 1 if no config given", got)
- }
-}
-
-func TestGetNotFound(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestGetNotFound(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -140,42 +139,42 @@ func TestGetNotFound(t *testing.T) {
case os.IsNotExist(err):
break
case err == nil:
- t.Errorf("Read should have failed, returned %+q", buf[:n])
+ c.Errorf("Read should have failed, returned %+q", buf[:n])
default:
- t.Errorf("Read expected ErrNotExist, got: %s", err)
+ c.Errorf("Read expected ErrNotExist, got: %s", err)
}
}
-func TestPut(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestPut(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Error(err)
+ c.Error(err)
}
p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
if buf, err := ioutil.ReadFile(p); err != nil {
- t.Error(err)
+ c.Error(err)
} else if bytes.Compare(buf, TestBlock) != 0 {
- t.Errorf("Write should have stored %s, did store %s",
+ c.Errorf("Write should have stored %s, did store %s",
string(TestBlock), string(buf))
}
}
-func TestPutBadVolume(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestPutBadVolume(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
os.Chmod(v.Root, 000)
err := v.Put(context.Background(), TestHash, TestBlock)
if err == nil {
- t.Error("Write should have failed")
+ c.Error("Write should have failed")
}
}
-func TestUnixVolumeReadonly(t *testing.T) {
- v := NewTestableUnixVolume(t, false, true)
+func (s *UnixVolumeSuite) TestUnixVolumeReadonly(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{ReadOnly: true, Replication: 1}, s.metrics, false)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -183,34 +182,34 @@ func TestUnixVolumeReadonly(t *testing.T) {
buf := make([]byte, BlockSize)
_, err := v.Get(context.Background(), TestHash, buf)
if err != nil {
- t.Errorf("got err %v, expected nil", err)
+ c.Errorf("got err %v, expected nil", err)
}
err = v.Put(context.Background(), TestHash, TestBlock)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
err = v.Touch(TestHash)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
err = v.Trash(TestHash)
if err != MethodDisabledError {
- t.Errorf("got err %v, expected MethodDisabledError", err)
+ c.Errorf("got err %v, expected MethodDisabledError", err)
}
}
-func TestIsFull(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestIsFull(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
fullPath := v.Root + "/full"
now := fmt.Sprintf("%d", time.Now().Unix())
os.Symlink(now, fullPath)
if !v.IsFull() {
- t.Errorf("%s: claims not to be full", v)
+ c.Errorf("%s: claims not to be full", v)
}
os.Remove(fullPath)
@@ -218,32 +217,32 @@ func TestIsFull(t *testing.T) {
expired := fmt.Sprintf("%d", time.Now().Unix()-3605)
os.Symlink(expired, fullPath)
if v.IsFull() {
- t.Errorf("%s: should no longer be full", v)
+ c.Errorf("%s: should no longer be full", v)
}
}
-func TestNodeStatus(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestNodeStatus(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
// Get node status and make a basic sanity check.
volinfo := v.Status()
if volinfo.MountPoint != v.Root {
- t.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.Root)
+ c.Errorf("GetNodeStatus mount_point %s, expected %s", volinfo.MountPoint, v.Root)
}
if volinfo.DeviceNum == 0 {
- t.Errorf("uninitialized device_num in %v", volinfo)
+ c.Errorf("uninitialized device_num in %v", volinfo)
}
if volinfo.BytesFree == 0 {
- t.Errorf("uninitialized bytes_free in %v", volinfo)
+ c.Errorf("uninitialized bytes_free in %v", volinfo)
}
if volinfo.BytesUsed == 0 {
- t.Errorf("uninitialized bytes_used in %v", volinfo)
+ c.Errorf("uninitialized bytes_used in %v", volinfo)
}
}
-func TestUnixVolumeGetFuncWorkerError(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncWorkerError(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -252,12 +251,12 @@ func TestUnixVolumeGetFuncWorkerError(t *testing.T) {
return mockErr
})
if err != mockErr {
- t.Errorf("Got %v, expected %v", err, mockErr)
+ c.Errorf("Got %v, expected %v", err, mockErr)
}
}
-func TestUnixVolumeGetFuncFileError(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncFileError(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
funcCalled := false
@@ -266,15 +265,15 @@ func TestUnixVolumeGetFuncFileError(t *testing.T) {
return nil
})
if err == nil {
- t.Errorf("Expected error opening non-existent file")
+ c.Errorf("Expected error opening non-existent file")
}
if funcCalled {
- t.Errorf("Worker func should not have been called")
+ c.Errorf("Worker func should not have been called")
}
}
-func TestUnixVolumeGetFuncWorkerWaitsOnMutex(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeGetFuncWorkerWaitsOnMutex(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
@@ -290,55 +289,55 @@ func TestUnixVolumeGetFuncWorkerWaitsOnMutex(t *testing.T) {
select {
case mtx.AllowLock <- struct{}{}:
case <-funcCalled:
- t.Fatal("Function was called before mutex was acquired")
+ c.Fatal("Function was called before mutex was acquired")
case <-time.After(5 * time.Second):
- t.Fatal("Timed out before mutex was acquired")
+ c.Fatal("Timed out before mutex was acquired")
}
select {
case <-funcCalled:
case mtx.AllowUnlock <- struct{}{}:
- t.Fatal("Mutex was released before function was called")
+ c.Fatal("Mutex was released before function was called")
case <-time.After(5 * time.Second):
- t.Fatal("Timed out waiting for funcCalled")
+ c.Fatal("Timed out waiting for funcCalled")
}
select {
case mtx.AllowUnlock <- struct{}{}:
case <-time.After(5 * time.Second):
- t.Fatal("Timed out waiting for getFunc() to release mutex")
+ c.Fatal("Timed out waiting for getFunc() to release mutex")
}
}
-func TestUnixVolumeCompare(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeCompare(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
v.Put(context.Background(), TestHash, TestBlock)
err := v.Compare(context.Background(), TestHash, TestBlock)
if err != nil {
- t.Errorf("Got err %q, expected nil", err)
+ c.Errorf("Got err %q, expected nil", err)
}
err = v.Compare(context.Background(), TestHash, []byte("baddata"))
if err != CollisionError {
- t.Errorf("Got err %q, expected %q", err, CollisionError)
+ c.Errorf("Got err %q, expected %q", err, CollisionError)
}
v.Put(context.Background(), TestHash, []byte("baddata"))
err = v.Compare(context.Background(), TestHash, TestBlock)
if err != DiskHashError {
- t.Errorf("Got err %q, expected %q", err, DiskHashError)
+ c.Errorf("Got err %q, expected %q", err, DiskHashError)
}
p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
os.Chmod(p, 000)
err = v.Compare(context.Background(), TestHash, TestBlock)
if err == nil || strings.Index(err.Error(), "permission denied") < 0 {
- t.Errorf("Got err %q, expected %q", err, "permission denied")
+ c.Errorf("Got err %q, expected %q", err, "permission denied")
}
}
-func TestUnixVolumeContextCancelPut(t *testing.T) {
- v := NewTestableUnixVolume(t, true, false)
+func (s *UnixVolumeSuite) TestUnixVolumeContextCancelPut(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, true)
defer v.Teardown()
v.locker.Lock()
ctx, cancel := context.WithCancel(context.Background())
@@ -350,19 +349,19 @@ func TestUnixVolumeContextCancelPut(t *testing.T) {
}()
err := v.Put(ctx, TestHash, TestBlock)
if err != context.Canceled {
- t.Errorf("Put() returned %s -- expected short read / canceled", err)
+ c.Errorf("Put() returned %s -- expected short read / canceled", err)
}
}
-func TestUnixVolumeContextCancelGet(t *testing.T) {
- v := NewTestableUnixVolume(t, false, false)
+func (s *UnixVolumeSuite) TestUnixVolumeContextCancelGet(c *check.C) {
+ v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
bpath := v.blockPath(TestHash)
v.PutRaw(TestHash, TestBlock)
os.Remove(bpath)
err := syscall.Mkfifo(bpath, 0600)
if err != nil {
- t.Fatalf("Mkfifo %s: %s", bpath, err)
+ c.Fatalf("Mkfifo %s: %s", bpath, err)
}
defer os.Remove(bpath)
ctx, cancel := context.WithCancel(context.Background())
@@ -373,35 +372,23 @@ func TestUnixVolumeContextCancelGet(t *testing.T) {
buf := make([]byte, len(TestBlock))
n, err := v.Get(ctx, TestHash, buf)
if n == len(TestBlock) || err != context.Canceled {
- t.Errorf("Get() returned %d, %s -- expected short read / canceled", n, err)
- }
-}
-
-var _ = check.Suite(&UnixVolumeSuite{})
-
-type UnixVolumeSuite struct {
- volume *TestableUnixVolume
-}
-
-func (s *UnixVolumeSuite) TearDownTest(c *check.C) {
- if s.volume != nil {
- s.volume.Teardown()
+ c.Errorf("Get() returned %d, %s -- expected short read / canceled", n, err)
}
}
func (s *UnixVolumeSuite) TestStats(c *check.C) {
- s.volume = NewTestableUnixVolume(c, false, false)
+ vol := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
stats := func() string {
- buf, err := json.Marshal(s.volume.InternalStats())
+ buf, err := json.Marshal(vol.InternalStats())
c.Check(err, check.IsNil)
return string(buf)
}
- c.Check(stats(), check.Matches, `.*"StatOps":0,.*`)
+ c.Check(stats(), check.Matches, `.*"StatOps":1,.*`) // (*UnixVolume)check() calls Stat() once
c.Check(stats(), check.Matches, `.*"Errors":0,.*`)
loc := "acbd18db4cc2f85cedef654fccc4a4d8"
- _, err := s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err := vol.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"StatOps":[^0],.*`)
c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
@@ -410,39 +397,27 @@ func (s *UnixVolumeSuite) TestStats(c *check.C) {
c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":0,.*`)
- err = s.volume.Put(context.Background(), loc, []byte("foo"))
+ err = vol.Put(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
c.Check(stats(), check.Matches, `.*"UtimesOps":0,.*`)
- err = s.volume.Touch(loc)
+ err = vol.Touch(loc)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":1,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":1,.*`)
c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
- _, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+ _, err = vol.Get(context.Background(), loc, make([]byte, 3))
c.Check(err, check.IsNil)
- err = s.volume.Compare(context.Background(), loc, []byte("foo"))
+ err = vol.Compare(context.Background(), loc, []byte("foo"))
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"InBytes":6,.*`)
c.Check(stats(), check.Matches, `.*"OpenOps":3,.*`)
- err = s.volume.Trash(loc)
+ err = vol.Trash(loc)
c.Check(err, check.IsNil)
c.Check(stats(), check.Matches, `.*"FlockOps":2,.*`)
}
-
-func (s *UnixVolumeSuite) TestConfig(c *check.C) {
- var cfg Config
- err := yaml.Unmarshal([]byte(`
-Volumes:
- - Type: Directory
- StorageClasses: ["class_a", "class_b"]
-`), &cfg)
-
- c.Check(err, check.IsNil)
- c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
-}
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
deleted file mode 100644
index 8e83f6ce5..000000000
--- a/services/keepstore/usage.go
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "sort"
- "strings"
-
- "github.com/ghodss/yaml"
-)
-
-func usage() {
- c := DefaultConfig()
- knownTypes := []string{}
- for _, vt := range VolumeTypes {
- c.Volumes = append(c.Volumes, vt().Examples()...)
- knownTypes = append(knownTypes, vt().Type())
- }
- exampleConfigFile, err := yaml.Marshal(c)
- if err != nil {
- panic(err)
- }
- sort.Strings(knownTypes)
- knownTypeList := strings.Join(knownTypes, ", ")
- fmt.Fprintf(os.Stderr, `
-
-keepstore provides a content-addressed data store backed by a local filesystem or networked storage.
-
-Usage: keepstore -config path/to/keepstore.yml
- keepstore [OPTIONS] -dump-config
-
-NOTE: All options (other than -config) are deprecated in favor of YAML
- configuration. Use -dump-config to translate existing
- configurations to YAML format.
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-LogFormat:
-
- Format of request/response and error logs: "json" or "text".
-
-PIDFile:
-
- Path to write PID file during startup. This file is kept open and
- locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
- one way to shut down. Exit immediately if there is an error
- opening, locking, or writing the PID file.
-
-MaxBuffers:
-
- Maximum RAM to use for data buffers, given in multiples of block
- size (64 MiB). When this limit is reached, HTTP requests requiring
- buffers (like GET and PUT) will wait for buffer space to be
- released.
-
-MaxRequests:
-
- Maximum concurrent requests. When this limit is reached, new
- requests will receive 503 responses. Note: this limit does not
- include idle connections from clients using HTTP keepalive, so it
- does not strictly limit the number of concurrent connections. If
- omitted or zero, the default is 2 * MaxBuffers.
-
-BlobSigningKeyFile:
-
- Local file containing the secret blob signing key (used to
- generate and verify blob signatures). This key should be
- identical to the API server's blob_signing_key configuration
- entry.
-
-RequireSignatures:
-
- Honor read requests only if a valid signature is provided. This
- should be true, except for development use and when migrating from
- a very old version.
-
-BlobSignatureTTL:
-
- Duration for which new permission signatures (returned in PUT
- responses) will be valid. This should be equal to the API
- server's blob_signature_ttl configuration entry.
-
-SystemAuthTokenFile:
-
- Local file containing the Arvados API token used by keep-balance
- or data manager. Delete, trash, and index requests are honored
- only for this token.
-
-EnableDelete:
-
- Enable trash and delete features. If false, trash lists will be
- accepted but blocks will not be trashed or deleted.
-
-TrashLifetime:
-
- Time duration after a block is trashed during which it can be
- recovered using an /untrash request.
-
-TrashCheckInterval:
-
- How often to check for (and delete) trashed blocks whose
- TrashLifetime has expired.
-
-TrashWorkers:
-
- Maximum number of concurrent trash operations. Default is 1, i.e.,
- trash lists are processed serially.
-
-EmptyTrashWorkers:
-
- Maximum number of concurrent block deletion operations (per
- volume) when emptying trash. Default is 1.
-
-PullWorkers:
-
- Maximum number of concurrent pull operations. Default is 1, i.e.,
- pull lists are processed serially.
-
-TLSCertificateFile:
-
- Path to server certificate file in X509 format. Enables TLS mode.
-
- Example: /var/lib/acme/live/keep0.example.com/fullchain
-
-TLSKeyFile:
-
- Path to server key file in X509 format. Enables TLS mode.
-
- The key pair is read from disk during startup, and whenever SIGHUP
- is received.
-
- Example: /var/lib/acme/live/keep0.example.com/privkey
-
-Volumes:
-
- List of storage volumes. If omitted or empty, the default is to
- use all directories named "keep" that exist in the top level
- directory of a mount point at startup time.
-
- Volume types: %s
-
- (See volume configuration examples above.)
-
-`, exampleConfigFile, knownTypeList)
-}
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 52b9b1b24..861435502 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -14,6 +14,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
)
type BlockWriter interface {
@@ -28,19 +29,12 @@ type BlockReader interface {
ReadBlock(ctx context.Context, loc string, w io.Writer) error
}
+var driver = map[string]func(*arvados.Cluster, arvados.Volume, logrus.FieldLogger, *volumeMetricsVecs) (Volume, error){}
+
// A Volume is an interface representing a Keep back-end storage unit:
// for example, a single mounted disk, a RAID array, an Amazon S3 volume,
// etc.
type Volume interface {
- // Volume type as specified in config file. Examples: "S3",
- // "Directory".
- Type() string
-
- // Do whatever private setup tasks and configuration checks
- // are needed. Return non-nil if the volume is unusable (e.g.,
- // invalid config).
- Start(vm *volumeMetricsVecs) error
-
// Get a block: copy the block data into buf, and return the
// number of bytes copied.
//
@@ -222,29 +216,13 @@ type Volume interface {
// secrets.
String() string
- // Writable returns false if all future Put, Mtime, and Delete
- // calls are expected to fail.
- //
- // If the volume is only temporarily unwritable -- or if Put
- // will fail because it is full, but Mtime or Delete can
- // succeed -- then Writable should return false.
- Writable() bool
-
- // Replication returns the storage redundancy of the
- // underlying device. It will be passed on to clients in
- // responses to PUT requests.
- Replication() int
-
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
EmptyTrash()
// Return a globally unique ID of the underlying storage
// device if possible, otherwise "".
- DeviceID() string
-
- // Get the storage classes associated with this volume
- GetStorageClasses() []string
+ GetDeviceID() string
}
// A VolumeWithExamples provides example configs to display in the
@@ -260,24 +238,24 @@ type VolumeManager interface {
// Mounts returns all mounts (volume attachments).
Mounts() []*VolumeMount
- // Lookup returns the volume under the given mount
- // UUID. Returns nil if the mount does not exist. If
- // write==true, returns nil if the volume is not writable.
- Lookup(uuid string, write bool) Volume
+ // Lookup returns the mount with the given UUID. Returns nil
+ // if the mount does not exist. If write==true, returns nil if
+ // the mount is not writable.
+ Lookup(uuid string, write bool) *VolumeMount
- // AllReadable returns all volumes.
- AllReadable() []Volume
+ // AllReadable returns all mounts.
+ AllReadable() []*VolumeMount
- // AllWritable returns all volumes that aren't known to be in
+ // AllWritable returns all mounts that aren't known to be in
// a read-only state. (There is no guarantee that a write to
// one will succeed, though.)
- AllWritable() []Volume
+ AllWritable() []*VolumeMount
// NextWritable returns the volume where the next new block
// should be written. A VolumeManager can select a volume in
// order to distribute activity across spindles, fill up disks
// with more free space, etc.
- NextWritable() Volume
+ NextWritable() *VolumeMount
// VolumeStats returns the ioStats used for tracking stats for
// the given Volume.
@@ -290,7 +268,7 @@ type VolumeManager interface {
// A VolumeMount is an attachment of a Volume to a VolumeManager.
type VolumeMount struct {
arvados.KeepMount
- volume Volume
+ Volume
}
// Generate a UUID the way API server would for a "KeepVolumeMount"
@@ -314,68 +292,85 @@ func (*VolumeMount) generateUUID() string {
type RRVolumeManager struct {
mounts []*VolumeMount
mountMap map[string]*VolumeMount
- readables []Volume
- writables []Volume
+ readables []*VolumeMount
+ writables []*VolumeMount
counter uint32
iostats map[Volume]*ioStats
}
-// MakeRRVolumeManager initializes RRVolumeManager
-func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
+func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, myURL arvados.URL, metrics *volumeMetricsVecs) (*RRVolumeManager, error) {
vm := &RRVolumeManager{
iostats: make(map[Volume]*ioStats),
}
vm.mountMap = make(map[string]*VolumeMount)
- for _, v := range volumes {
- sc := v.GetStorageClasses()
+ for uuid, cfgvol := range cluster.Volumes {
+ va, ok := cfgvol.AccessViaHosts[myURL]
+ if !ok && len(cfgvol.AccessViaHosts) > 0 {
+ continue
+ }
+ dri, ok := driver[cfgvol.Driver]
+ if !ok {
+ return nil, fmt.Errorf("volume %s: invalid driver %q", uuid, cfgvol.Driver)
+ }
+ vol, err := dri(cluster, cfgvol, logger, metrics)
+ if err != nil {
+ return nil, fmt.Errorf("error initializing volume %s: %s", uuid, err)
+ }
+ logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+
+ sc := cfgvol.StorageClasses
if len(sc) == 0 {
- sc = []string{"default"}
+ sc = map[string]bool{"default": true}
+ }
+ repl := cfgvol.Replication
+ if repl < 1 {
+ repl = 1
}
mnt := &VolumeMount{
KeepMount: arvados.KeepMount{
- UUID: (*VolumeMount)(nil).generateUUID(),
- DeviceID: v.DeviceID(),
- ReadOnly: !v.Writable(),
- Replication: v.Replication(),
+ UUID: uuid,
+ DeviceID: vol.GetDeviceID(),
+ ReadOnly: cfgvol.ReadOnly || va.ReadOnly,
+ Replication: repl,
StorageClasses: sc,
},
- volume: v,
+ Volume: vol,
}
- vm.iostats[v] = &ioStats{}
+ vm.iostats[vol] = &ioStats{}
vm.mounts = append(vm.mounts, mnt)
- vm.mountMap[mnt.UUID] = mnt
- vm.readables = append(vm.readables, v)
- if v.Writable() {
- vm.writables = append(vm.writables, v)
+ vm.mountMap[uuid] = mnt
+ vm.readables = append(vm.readables, mnt)
+ if !mnt.KeepMount.ReadOnly {
+ vm.writables = append(vm.writables, mnt)
}
}
- return vm
+ return vm, nil
}
func (vm *RRVolumeManager) Mounts() []*VolumeMount {
return vm.mounts
}
-func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) Volume {
+func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) *VolumeMount {
if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) {
- return mnt.volume
+ return mnt
} else {
return nil
}
}
// AllReadable returns an array of all readable volumes
-func (vm *RRVolumeManager) AllReadable() []Volume {
+func (vm *RRVolumeManager) AllReadable() []*VolumeMount {
return vm.readables
}
// AllWritable returns an array of all writable volumes
-func (vm *RRVolumeManager) AllWritable() []Volume {
+func (vm *RRVolumeManager) AllWritable() []*VolumeMount {
return vm.writables
}
// NextWritable returns the next writable
-func (vm *RRVolumeManager) NextWritable() Volume {
+func (vm *RRVolumeManager) NextWritable() *VolumeMount {
if len(vm.writables) == 0 {
return nil
}
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index d5a413693..683521c01 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -18,8 +18,10 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
+ "github.com/sirupsen/logrus"
)
type TB interface {
@@ -37,65 +39,96 @@ type TB interface {
// A TestableVolumeFactory returns a new TestableVolume. The factory
// function, and the TestableVolume it returns, can use "t" to write
// logs, fail the current test, etc.
-type TestableVolumeFactory func(t TB) TestableVolume
+type TestableVolumeFactory func(t TB, cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) TestableVolume
// DoGenericVolumeTests runs a set of tests that every TestableVolume
// is expected to pass. It calls factory to create a new TestableVolume
// for each test case, to avoid leaking state between tests.
-func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
- testGet(t, factory)
- testGetNoSuchBlock(t, factory)
+func DoGenericVolumeTests(t TB, readonly bool, factory TestableVolumeFactory) {
+ var s genericVolumeSuite
+ s.volume.ReadOnly = readonly
- testCompareNonexistent(t, factory)
- testCompareSameContent(t, factory, TestHash, TestBlock)
- testCompareSameContent(t, factory, EmptyHash, EmptyBlock)
- testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata"))
- testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock)
- testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock)
- testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata"))
- testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock)
- testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
+ s.testGet(t, factory)
+ s.testGetNoSuchBlock(t, factory)
- testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
- testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
- testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
- testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
- testPutMultipleBlocks(t, factory)
+ s.testCompareNonexistent(t, factory)
+ s.testCompareSameContent(t, factory, TestHash, TestBlock)
+ s.testCompareSameContent(t, factory, EmptyHash, EmptyBlock)
+ s.testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata"))
+ s.testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock)
+ s.testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock)
+ s.testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata"))
+ s.testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock)
+ s.testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
- testPutAndTouch(t, factory)
- testTouchNoSuchBlock(t, factory)
+ if !readonly {
+ s.testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
+ s.testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
+ s.testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
+ s.testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
+ s.testPutMultipleBlocks(t, factory)
- testMtimeNoSuchBlock(t, factory)
+ s.testPutAndTouch(t, factory)
+ }
+ s.testTouchNoSuchBlock(t, factory)
+
+ s.testMtimeNoSuchBlock(t, factory)
+
+ s.testIndexTo(t, factory)
- testIndexTo(t, factory)
+ if !readonly {
+ s.testDeleteNewBlock(t, factory)
+ s.testDeleteOldBlock(t, factory)
+ }
+ s.testDeleteNoSuchBlock(t, factory)
- testDeleteNewBlock(t, factory)
- testDeleteOldBlock(t, factory)
- testDeleteNoSuchBlock(t, factory)
+ s.testStatus(t, factory)
- testStatus(t, factory)
+ s.testMetrics(t, readonly, factory)
- testMetrics(t, factory)
+ s.testString(t, factory)
- testString(t, factory)
+ if readonly {
+ s.testUpdateReadOnly(t, factory)
+ }
- testUpdateReadOnly(t, factory)
+ s.testGetConcurrent(t, factory)
+ if !readonly {
+ s.testPutConcurrent(t, factory)
- testGetConcurrent(t, factory)
- testPutConcurrent(t, factory)
+ s.testPutFullBlock(t, factory)
+ }
- testPutFullBlock(t, factory)
+ s.testTrashUntrash(t, readonly, factory)
+ s.testTrashEmptyTrashUntrash(t, factory)
+}
+
+type genericVolumeSuite struct {
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ registry *prometheus.Registry
+}
+
+func (s *genericVolumeSuite) setup(t TB) {
+ s.cluster = testCluster(t)
+ s.logger = ctxlog.TestLogger(t)
+ s.registry = prometheus.NewRegistry()
+ s.metrics = newVolumeMetricsVecs(s.registry)
+}
- testTrashUntrash(t, factory)
- testTrashEmptyTrashUntrash(t, factory)
+func (s *genericVolumeSuite) newVolume(t TB, factory TestableVolumeFactory) TestableVolume {
+ return factory(t, s.cluster, s.volume, s.logger, s.metrics)
}
// Put a test block, get it and verify content
// Test should pass for both writable and read-only volumes
-func testGet(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGet(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -113,8 +146,9 @@ func testGet(t TB, factory TestableVolumeFactory) {
// Invoke get on a block that does not exist in volume; should result in error
// Test should pass for both writable and read-only volumes
-func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
buf := make([]byte, BlockSize)
@@ -126,8 +160,9 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Compare() should return os.ErrNotExist if the block does not exist.
// Otherwise, writing new data causes CompareAndTouch() to generate
// error logs even though everything is working fine.
-func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareNonexistent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
err := v.Compare(context.Background(), TestHash, TestBlock)
@@ -138,8 +173,9 @@ func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
// Put a test block and compare the locator with same content
// Test should pass for both writable and read-only volumes
-func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(testHash, testData)
@@ -156,8 +192,9 @@ func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string
// testHash = md5(testDataA).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(testHash, testDataA)
@@ -173,8 +210,9 @@ func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash stri
// corrupted. Requires testHash = md5(testDataA) != md5(testDataB).
//
// Test should pass for both writable and read-only volumes
-func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, testDataB)
@@ -187,14 +225,11 @@ func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testH
// Put a block and put again with same content
// Test is intended for only writable volumes
-func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
err := v.Put(context.Background(), testHash, testData)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -208,14 +243,11 @@ func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash s
// Put a block and put again with different content
// Test is intended for only writable volumes
-func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
v.PutRaw(testHash, testDataA)
putErr := v.Put(context.Background(), testHash, testDataB)
@@ -239,14 +271,11 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH
// Put and get multiple blocks
// Test is intended for only writable volumes
-func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
err := v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -295,14 +324,11 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
// Test that when applying PUT to a block that already exists,
// the block's modification time is updated.
// Test is intended for only writable volumes
-func testPutAndTouch(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutAndTouch(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
if err := v.Put(context.Background(), TestHash, TestBlock); err != nil {
t.Error(err)
}
@@ -337,8 +363,9 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) {
// Touching a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if err := v.Touch(TestHash); err == nil {
@@ -348,8 +375,9 @@ func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Invoking Mtime on a non-existing block should result in error.
// Test should pass for both writable and read-only volumes
-func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if _, err := v.Mtime("12345678901234567890123456789012"); err == nil {
@@ -362,8 +390,9 @@ func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
// * with a prefix
// * with no such prefix
// Test should pass for both writable and read-only volumes
-func testIndexTo(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testIndexTo(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
// minMtime and maxMtime are the minimum and maximum
@@ -437,14 +466,11 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block immediately after writing it (not old enough)
// should neither delete the data nor return an error.
// Test is intended for only writable volumes
-func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- theConfig.BlobSignatureTTL.Set("5m")
-
- if v.Writable() == false {
- return
- }
v.Put(context.Background(), TestHash, TestBlock)
@@ -463,17 +489,14 @@ func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block with a timestamp older than
// BlobSignatureTTL seconds in the past should delete the data.
// Test is intended for only writable volumes
-func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobSigningTTL.Set("5m")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- theConfig.BlobSignatureTTL.Set("5m")
-
- if v.Writable() == false {
- return
- }
v.Put(context.Background(), TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
if err := v.Trash(TestHash); err != nil {
t.Error(err)
@@ -507,8 +530,9 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
// Calling Delete() for a block that does not exist should result in error.
// Test should pass for both writable and read-only volumes
-func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if err := v.Trash(TestHash2); err == nil {
@@ -518,8 +542,9 @@ func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
// Invoke Status and verify that VolumeStatus is returned
// Test should pass for both writable and read-only volumes
-func testStatus(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testStatus(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
// Get node status and make a basic sanity check.
@@ -544,19 +569,14 @@ func getValueFrom(cv *prometheus.CounterVec, lbls prometheus.Labels) float64 {
return pb.GetCounter().GetValue()
}
-func testMetrics(t TB, factory TestableVolumeFactory) {
+func (s *genericVolumeSuite) testMetrics(t TB, readonly bool, factory TestableVolumeFactory) {
var err error
- v := factory(t)
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- reg := prometheus.NewRegistry()
- vm := newVolumeMetricsVecs(reg)
- err = v.Start(vm)
- if err != nil {
- t.Error("Failed Start(): ", err)
- }
- opsC, _, ioC := vm.getCounterVecsFor(prometheus.Labels{"device_id": v.DeviceID()})
+ opsC, _, ioC := s.metrics.getCounterVecsFor(prometheus.Labels{"device_id": v.GetDeviceID()})
if ioC == nil {
t.Error("ioBytes CounterVec is nil")
@@ -580,7 +600,7 @@ func testMetrics(t TB, factory TestableVolumeFactory) {
readOpCounter = getValueFrom(opsC, prometheus.Labels{"operation": readOpType})
// Test Put if volume is writable
- if v.Writable() {
+ if !readonly {
err = v.Put(context.Background(), TestHash, TestBlock)
if err != nil {
t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
@@ -617,8 +637,9 @@ func testMetrics(t TB, factory TestableVolumeFactory) {
// Invoke String for the volume; expect non-empty result
// Test should pass for both writable and read-only volumes
-func testString(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testString(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
if id := v.String(); len(id) == 0 {
@@ -628,14 +649,11 @@ func testString(t TB, factory TestableVolumeFactory) {
// Putting, updating, touching, and deleting blocks from a read-only volume result in error.
// Test is intended for only read-only volumes
-func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == true {
- return
- }
-
v.PutRaw(TestHash, TestBlock)
buf := make([]byte, BlockSize)
@@ -676,8 +694,9 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
// Launch concurrent Gets
// Test should pass for both writable and read-only volumes
-func testGetConcurrent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testGetConcurrent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
v.PutRaw(TestHash, TestBlock)
@@ -729,14 +748,11 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) {
// Launch concurrent Puts
// Test is intended for only writable volumes
-func testPutConcurrent(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutConcurrent(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if v.Writable() == false {
- return
- }
-
sem := make(chan int)
go func(sem chan int) {
err := v.Put(context.Background(), TestHash, TestBlock)
@@ -795,14 +811,11 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
}
// Write and read back a full size block
-func testPutFullBlock(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testPutFullBlock(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- if !v.Writable() {
- return
- }
-
wdata := make([]byte, BlockSize)
wdata[0] = 'a'
wdata[BlockSize-1] = 'z'
@@ -825,18 +838,15 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) {
// Trash an old block - which either raises ErrNotImplemented or succeeds
// Untrash - which either raises ErrNotImplemented or succeeds
// Get - which must succeed
-func testTrashUntrash(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTrashUntrash(t TB, readonly bool, factory TestableVolumeFactory) {
+ s.setup(t)
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
+ v := s.newVolume(t, factory)
defer v.Teardown()
- defer func() {
- theConfig.TrashLifetime = 0
- }()
-
- theConfig.TrashLifetime.Set("1h")
// put block and backdate it
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
buf := make([]byte, BlockSize)
n, err := v.Get(context.Background(), TestHash, buf)
@@ -849,7 +859,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
// Trash
err = v.Trash(TestHash)
- if v.Writable() == false {
+ if readonly {
if err != MethodDisabledError {
t.Fatal(err)
}
@@ -880,12 +890,10 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
}
}
-func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
- v := factory(t)
+func (s *genericVolumeSuite) testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
+ s.setup(t)
+ v := s.newVolume(t, factory)
defer v.Teardown()
- defer func(orig arvados.Duration) {
- theConfig.TrashLifetime = orig
- }(theConfig.TrashLifetime)
checkGet := func() error {
buf := make([]byte, BlockSize)
@@ -918,10 +926,10 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// First set: EmptyTrash before reaching the trash deadline.
- theConfig.TrashLifetime.Set("1h")
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
err := checkGet()
if err != nil {
@@ -966,7 +974,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
// Because we Touch'ed, need to backdate again for next set of tests
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// If the only block in the trash has already been untrashed,
// most volumes will fail a subsequent Untrash with a 404, but
@@ -984,11 +992,11 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
// Untrash might have updated the timestamp, so backdate again
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// Second set: EmptyTrash after the trash deadline has passed.
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
@@ -1013,7 +1021,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// Trash it again, and this time call EmptyTrash so it really
// goes away.
// (In Azure volumes, un/trash changes Mtime, so first backdate again)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
_ = v.Trash(TestHash)
err = checkGet()
if err == nil || !os.IsNotExist(err) {
@@ -1038,9 +1046,9 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// un-trashed copy doesn't get deleted along with it.
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
@@ -1051,7 +1059,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
}
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
// EmptyTrash should not delete the untrashed copy.
v.EmptyTrash()
@@ -1066,18 +1074,18 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
// untrash the block whose deadline is "C".
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1ns")
+ s.cluster.Collections.BlobTrashLifetime.Set("1ns")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
}
v.PutRaw(TestHash, TestBlock)
- v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
+ v.TouchWithDate(TestHash, time.Now().Add(-2*s.cluster.Collections.BlobSigningTTL.Duration()))
- theConfig.TrashLifetime.Set("1h")
+ s.cluster.Collections.BlobTrashLifetime.Set("1h")
err = v.Trash(TestHash)
if err != nil {
t.Fatal(err)
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 0b8af330f..62582d309 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -15,6 +15,28 @@ import (
"strings"
"sync"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+var (
+ TestBlock = []byte("The quick brown fox jumps over the lazy dog.")
+ TestHash = "e4d909c290d0fb1ca068ffaddf22cbd0"
+ TestHashPutResp = "e4d909c290d0fb1ca068ffaddf22cbd0+44\n"
+
+ TestBlock2 = []byte("Pack my box with five dozen liquor jugs.")
+ TestHash2 = "f15ac516f788aec4f30932ffb6395c39"
+
+ TestBlock3 = []byte("Now is the time for all good men to come to the aid of their country.")
+ TestHash3 = "eed29bbffbc2dbe5e5ee0bb71888e61f"
+
+ // BadBlock is used to test collisions and corruption.
+ // It must not match any test hashes.
+ BadBlock = []byte("The magic words are squeamish ossifrage.")
+
+ EmptyHash = "d41d8cd98f00b204e9800998ecf8427e"
+ EmptyBlock = []byte("")
)
// A TestableVolume allows test suites to manipulate the state of an
@@ -38,6 +60,10 @@ type TestableVolume interface {
Teardown()
}
+func init() {
+ driver["mock"] = newMockVolume
+}
+
// MockVolumes are test doubles for Volumes, used to test handlers.
type MockVolume struct {
Store map[string][]byte
@@ -51,10 +77,6 @@ type MockVolume struct {
// that has been Put().
Touchable bool
- // Readonly volumes return an error for Put, Delete, and
- // Touch.
- Readonly bool
-
// Gate is a "starting gate", allowing test cases to pause
// volume operations long enough to inspect state. Every
// operation (except Status) starts by receiving from
@@ -62,15 +84,19 @@ type MockVolume struct {
// channel unblocks all operations. By default, Gate is a
// closed channel, so all operations proceed without
// blocking. See trash_worker_test.go for an example.
- Gate chan struct{}
-
- called map[string]int
- mutex sync.Mutex
+ Gate chan struct{} `json:"-"`
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
+ metrics *volumeMetricsVecs
+ called map[string]int
+ mutex sync.Mutex
}
-// CreateMockVolume returns a non-Bad, non-Readonly, Touchable mock
+// newMockVolume returns a non-Bad, non-Readonly, Touchable mock
// volume.
-func CreateMockVolume() *MockVolume {
+func newMockVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) {
gate := make(chan struct{})
close(gate)
return &MockVolume{
@@ -78,10 +104,13 @@ func CreateMockVolume() *MockVolume {
Timestamps: make(map[string]time.Time),
Bad: false,
Touchable: true,
- Readonly: false,
called: map[string]int{},
Gate: gate,
- }
+ cluster: cluster,
+ volume: volume,
+ logger: logger,
+ metrics: metrics,
+ }, nil
}
// CallCount returns how many times the named method has been called.
@@ -141,7 +170,7 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
if v.Bad {
return v.BadVolumeError
}
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
v.Store[loc] = block
@@ -151,7 +180,7 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
func (v *MockVolume) Touch(loc string) error {
v.gotCall("Touch")
<-v.Gate
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if v.Touchable {
@@ -195,11 +224,11 @@ func (v *MockVolume) IndexTo(prefix string, w io.Writer) error {
func (v *MockVolume) Trash(loc string) error {
v.gotCall("Delete")
<-v.Gate
- if v.Readonly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if _, ok := v.Store[loc]; ok {
- if time.Since(v.Timestamps[loc]) < time.Duration(theConfig.BlobSignatureTTL) {
+ if time.Since(v.Timestamps[loc]) < time.Duration(v.cluster.Collections.BlobSigningTTL) {
return nil
}
delete(v.Store, loc)
@@ -208,18 +237,10 @@ func (v *MockVolume) Trash(loc string) error {
return os.ErrNotExist
}
-func (v *MockVolume) DeviceID() string {
+func (v *MockVolume) GetDeviceID() string {
return "mock-device-id"
}
-func (v *MockVolume) Type() string {
- return "Mock"
-}
-
-func (v *MockVolume) Start(vm *volumeMetricsVecs) error {
- return nil
-}
-
func (v *MockVolume) Untrash(loc string) error {
return nil
}
@@ -236,14 +257,6 @@ func (v *MockVolume) String() string {
return "[MockVolume]"
}
-func (v *MockVolume) Writable() bool {
- return !v.Readonly
-}
-
-func (v *MockVolume) Replication() int {
- return 1
-}
-
func (v *MockVolume) EmptyTrash() {
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list