[ARVADOS] updated: 1.3.0-1455-g8654bb498
Git user
git at public.curoverse.com
Tue Aug 20 13:27:40 UTC 2019
Summary of changes:
lib/config/cmd_test.go | 2 +-
lib/config/config.default.yml | 5 +-
lib/config/deprecated_keepstore.go | 76 +++++++++-
lib/config/deprecated_keepstore_test.go | 45 ++++++
lib/config/generated_config.go | 5 +-
lib/dispatchcloud/cmd.go | 4 +-
lib/dispatchcloud/dispatcher.go | 11 +-
lib/service/cmd.go | 62 +++++---
lib/service/tls.go | 80 ++++++++++
sdk/go/arvados/config.go | 4 +
sdk/go/ctxlog/log.go | 2 +-
sdk/go/httpserver/httpserver.go | 7 +-
services/keepstore/azure_blob_volume.go | 249 ++++++++++++++------------------
services/keepstore/bufferpool.go | 11 +-
services/keepstore/command.go | 177 +++++++++--------------
services/keepstore/config.go | 84 -----------
services/keepstore/handlers.go | 92 ++++++------
services/keepstore/keepstore.go | 21 +--
services/keepstore/perms.go | 12 +-
services/keepstore/proxy_remote.go | 4 +-
services/keepstore/pull_worker.go | 50 +++----
services/keepstore/s3_volume.go | 122 +++++++---------
services/keepstore/server.go | 78 ----------
services/keepstore/trash_worker.go | 19 +--
services/keepstore/unix_volume.go | 150 ++++---------------
services/keepstore/volume.go | 82 ++++++-----
26 files changed, 665 insertions(+), 789 deletions(-)
create mode 100644 lib/service/tls.go
delete mode 100644 services/keepstore/config.go
delete mode 100644 services/keepstore/server.go
discards 587f6f4239087291f617159cc1daad468b949345 (commit)
via 8654bb498573256ac622d513bbfa3509f0b09e9d (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (587f6f4239087291f617159cc1daad468b949345)
\
N -- N -- N (8654bb498573256ac622d513bbfa3509f0b09e9d)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8654bb498573256ac622d513bbfa3509f0b09e9d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Aug 19 17:15:40 2019 -0400
13647: Use cluster config instead of custom keepstore config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index af7c57120..019d5edd0 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -85,7 +85,7 @@ func (s *CommandSuite) TestCheckOldKeepstoreConfigFile(c *check.C) {
c.Assert(err, check.IsNil)
defer os.Remove(f.Name())
- io.WriteString(f, "Debug: true\n")
+ io.WriteString(f, "Listen: :12345\nDebug: true\n")
var stdout, stderr bytes.Buffer
in := `
@@ -97,7 +97,7 @@ Clusters:
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-", "-legacy-keepstore-config", f.Name()}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 1)
c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*LogLevel: info\n\+ +LogLevel: debug\n.*`)
- c.Check(stderr.String(), check.Matches, `.*you should remove the legacy keepstore config file.*\n`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*you should remove the legacy keepstore config file.*\n`)
}
func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 9ac4aeeb9..17556fdde 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -176,6 +176,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -313,15 +322,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
- # Modifying blob_signing_key will invalidate all existing
+ # Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -730,6 +768,47 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 12581ddff..10dfc98ae 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -102,12 +102,6 @@ func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc
svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
}
-const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
-
-type oldKeepstoreConfig struct {
- Debug *bool
-}
-
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
if path == "" {
return nil
@@ -126,35 +120,6 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
return nil
}
-// update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
- if ldr.KeepstorePath == "" {
- return nil
- }
- var oc oldKeepstoreConfig
- err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
- if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
- return nil
- } else if err != nil {
- return err
- }
-
- cluster, err := cfg.GetCluster("")
- if err != nil {
- return err
- }
-
- if v := oc.Debug; v == nil {
- } else if *v && cluster.SystemLogs.LogLevel != "debug" {
- cluster.SystemLogs.LogLevel = "debug"
- } else if !*v && cluster.SystemLogs.LogLevel != "info" {
- cluster.SystemLogs.LogLevel = "info"
- }
-
- cfg.Clusters[cluster.ClusterID] = *cluster
- return nil
-}
-
type oldCrunchDispatchSlurmConfig struct {
Client *arvados.Client
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
new file mode 100644
index 000000000..faefca569
--- /dev/null
+++ b/lib/config/deprecated_keepstore.go
@@ -0,0 +1,544 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "os"
+ "strconv"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+const defaultKeepstoreConfigPath = "/etc/arvados/keepstore/keepstore.yml"
+
+type oldKeepstoreConfig struct {
+ Debug *bool
+ Listen *string
+
+ LogFormat *string
+
+ PIDFile *string
+
+ MaxBuffers *int
+ MaxRequests *int
+
+ BlobSignatureTTL *arvados.Duration
+ BlobSigningKeyFile *string
+ RequireSignatures *bool
+ SystemAuthTokenFile *string
+ EnableDelete *bool
+ TrashLifetime *arvados.Duration
+ TrashCheckInterval *arvados.Duration
+ PullWorkers *int
+ TrashWorkers *int
+ EmptyTrashWorkers *int
+ TLSCertificateFile *string
+ TLSKeyFile *string
+
+ Volumes *oldKeepstoreVolumeList
+
+ ManagementToken *string
+
+ DiscoverVolumesFromMountsFile string // not a real legacy config -- just useful for tests
+}
+
+type oldKeepstoreVolumeList []oldKeepstoreVolume
+
+type oldKeepstoreVolume struct {
+ arvados.Volume
+ Type string `json:",omitempty"`
+
+ // Azure driver configs
+ StorageAccountName string `json:",omitempty"`
+ StorageAccountKeyFile string `json:",omitempty"`
+ StorageBaseURL string `json:",omitempty"`
+ ContainerName string `json:",omitempty"`
+ AzureReplication int `json:",omitempty"`
+ RequestTimeout arvados.Duration `json:",omitempty"`
+ ListBlobsRetryDelay arvados.Duration `json:",omitempty"`
+ ListBlobsMaxAttempts int `json:",omitempty"`
+
+ // S3 driver configs
+ AccessKeyFile string `json:",omitempty"`
+ SecretKeyFile string `json:",omitempty"`
+ Endpoint string `json:",omitempty"`
+ Region string `json:",omitempty"`
+ Bucket string `json:",omitempty"`
+ LocationConstraint bool `json:",omitempty"`
+ IndexPageSize int `json:",omitempty"`
+ S3Replication int `json:",omitempty"`
+ ConnectTimeout arvados.Duration `json:",omitempty"`
+ ReadTimeout arvados.Duration `json:",omitempty"`
+ RaceWindow arvados.Duration `json:",omitempty"`
+ UnsafeDelete bool `json:",omitempty"`
+
+ // Directory driver configs
+ Root string
+ DirectoryReplication int
+ Serialize bool
+
+ // Common configs
+ ReadOnly bool `json:",omitempty"`
+ StorageClasses []string `json:",omitempty"`
+}
+
+// update config using values from an old-style keepstore config file.
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+ hostname, err := os.Hostname()
+ if err != nil {
+ return fmt.Errorf("getting hostname: %s", err)
+ }
+
+ var oc oldKeepstoreConfig
+ err = ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
+ if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ myURL := arvados.URL{Scheme: "http"}
+ if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
+ myURL.Scheme = "https"
+ }
+
+ if v := oc.Debug; v == nil {
+ } else if *v && cluster.SystemLogs.LogLevel != "debug" {
+ cluster.SystemLogs.LogLevel = "debug"
+ } else if !*v && cluster.SystemLogs.LogLevel != "info" {
+ cluster.SystemLogs.LogLevel = "info"
+ }
+
+ if v := oc.TLSCertificateFile; v != nil && "file://"+*v != cluster.TLS.Certificate {
+ cluster.TLS.Certificate = "file://" + *v
+ }
+ if v := oc.TLSKeyFile; v != nil && "file://"+*v != cluster.TLS.Key {
+ cluster.TLS.Key = "file://" + *v
+ }
+ if v := oc.Listen; v != nil {
+ if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+ // already listed
+ myURL.Host = *v
+ } else if len(*v) > 1 && (*v)[0] == ':' {
+ myURL.Host = net.JoinHostPort(hostname, (*v)[1:])
+ cluster.Services.Keepstore.InternalURLs[myURL] = arvados.ServiceInstance{}
+ } else {
+ return fmt.Errorf("unable to migrate Listen value %q from legacy keepstore config file -- remove after configuring Services.Keepstore.InternalURLs.", *v)
+ }
+ } else {
+ for url := range cluster.Services.Keepstore.InternalURLs {
+ if host, _, _ := net.SplitHostPort(url.Host); host == hostname {
+ myURL = url
+ break
+ }
+ }
+ if myURL.Host == "" {
+ return fmt.Errorf("unable to migrate legacy keepstore config: no 'Listen' key, and hostname %q does not match an entry in Services.Keepstore.InternalURLs", hostname)
+ }
+ }
+
+ if v := oc.LogFormat; v != nil && *v != cluster.SystemLogs.Format {
+ cluster.SystemLogs.Format = *v
+ }
+ if v := oc.MaxBuffers; v != nil && *v != cluster.API.MaxKeepBlockBuffers {
+ cluster.API.MaxKeepBlockBuffers = *v
+ }
+ if v := oc.MaxRequests; v != nil && *v != cluster.API.MaxConcurrentRequests {
+ cluster.API.MaxConcurrentRequests = *v
+ }
+ if v := oc.BlobSignatureTTL; v != nil && *v != cluster.Collections.BlobSigningTTL {
+ cluster.Collections.BlobSigningTTL = *v
+ }
+ if v := oc.BlobSigningKeyFile; v != nil {
+ buf, err := ioutil.ReadFile(*v)
+ if err != nil {
+ return fmt.Errorf("error reading BlobSigningKeyFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.Collections.BlobSigningKey {
+ cluster.Collections.BlobSigningKey = key
+ }
+ }
+ if v := oc.RequireSignatures; v != nil && *v != cluster.Collections.BlobSigning {
+ cluster.Collections.BlobSigning = *v
+ }
+ if v := oc.SystemAuthTokenFile; v != nil {
+ f, err := os.Open(*v)
+ if err != nil {
+ return fmt.Errorf("error opening SystemAuthTokenFile: %s", err)
+ }
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ if err != nil {
+ return fmt.Errorf("error reading SystemAuthTokenFile: %s", err)
+ }
+ if key := strings.TrimSpace(string(buf)); key != cluster.SystemRootToken {
+ cluster.SystemRootToken = key
+ }
+ }
+ if v := oc.EnableDelete; v != nil && *v != cluster.Collections.BlobTrash {
+ cluster.Collections.BlobTrash = *v
+ }
+ if v := oc.TrashLifetime; v != nil && *v != cluster.Collections.BlobTrashLifetime {
+ cluster.Collections.BlobTrashLifetime = *v
+ }
+ if v := oc.TrashCheckInterval; v != nil && *v != cluster.Collections.BlobTrashCheckInterval {
+ cluster.Collections.BlobTrashCheckInterval = *v
+ }
+ if v := oc.TrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobTrashConcurrency = *v
+ }
+ if v := oc.EmptyTrashWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobDeleteConcurrency = *v
+ }
+ if v := oc.PullWorkers; v != nil && *v != cluster.Collections.BlobReplicateConcurrency {
+ cluster.Collections.BlobReplicateConcurrency = *v
+ }
+ if v := oc.Volumes; v == nil {
+ ldr.Logger.Warn("no volumes in legacy config; discovering local directory volumes")
+ err := ldr.discoverLocalVolumes(cluster, oc.DiscoverVolumesFromMountsFile, myURL)
+ if err != nil {
+ return fmt.Errorf("error discovering local directory volumes: %s", err)
+ }
+ } else {
+ for i, oldvol := range *v {
+ var accessViaHosts map[arvados.URL]arvados.VolumeAccess
+ oldUUID, found := ldr.alreadyMigrated(oldvol, cluster.Volumes, myURL)
+ if found {
+ accessViaHosts = cluster.Volumes[oldUUID].AccessViaHosts
+ writers := false
+ for _, va := range accessViaHosts {
+ if !va.ReadOnly {
+ writers = true
+ }
+ }
+ if writers || len(accessViaHosts) == 0 {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ if len(accessViaHosts) > 0 {
+ cluster.Volumes[oldUUID].AccessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ }
+ continue
+ }
+ }
+ var newvol arvados.Volume
+ if found {
+ ldr.Logger.Infof("ignoring volume #%d's parameters in legacy keepstore config: using matching entry in cluster config instead", i)
+ newvol = cluster.Volumes[oldUUID]
+ // Remove the old entry. It will be
+ // added back below, possibly with a
+ // new UUID.
+ delete(cluster.Volumes, oldUUID)
+ } else {
+ var params interface{}
+ switch oldvol.Type {
+ case "S3":
+ accesskeydata, err := ioutil.ReadFile(oldvol.AccessKeyFile)
+ if err != nil && oldvol.AccessKeyFile != "" {
+ return fmt.Errorf("error reading AccessKeyFile: %s", err)
+ }
+ secretkeydata, err := ioutil.ReadFile(oldvol.SecretKeyFile)
+ if err != nil && oldvol.SecretKeyFile != "" {
+ return fmt.Errorf("error reading SecretKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "S3",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.S3Replication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.S3VolumeDriverParameters{
+ AccessKey: string(bytes.TrimSpace(accesskeydata)),
+ SecretKey: string(bytes.TrimSpace(secretkeydata)),
+ Endpoint: oldvol.Endpoint,
+ Region: oldvol.Region,
+ Bucket: oldvol.Bucket,
+ LocationConstraint: oldvol.LocationConstraint,
+ IndexPageSize: oldvol.IndexPageSize,
+ ConnectTimeout: oldvol.ConnectTimeout,
+ ReadTimeout: oldvol.ReadTimeout,
+ RaceWindow: oldvol.RaceWindow,
+ UnsafeDelete: oldvol.UnsafeDelete,
+ }
+ case "Azure":
+ keydata, err := ioutil.ReadFile(oldvol.StorageAccountKeyFile)
+ if err != nil && oldvol.StorageAccountKeyFile != "" {
+ return fmt.Errorf("error reading StorageAccountKeyFile: %s", err)
+ }
+ newvol = arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.AzureReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.AzureVolumeDriverParameters{
+ StorageAccountName: oldvol.StorageAccountName,
+ StorageAccountKey: string(bytes.TrimSpace(keydata)),
+ StorageBaseURL: oldvol.StorageBaseURL,
+ ContainerName: oldvol.ContainerName,
+ RequestTimeout: oldvol.RequestTimeout,
+ ListBlobsRetryDelay: oldvol.ListBlobsRetryDelay,
+ ListBlobsMaxAttempts: oldvol.ListBlobsMaxAttempts,
+ }
+ case "Directory":
+ newvol = arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: oldvol.ReadOnly,
+ Replication: oldvol.DirectoryReplication,
+ StorageClasses: array2boolmap(oldvol.StorageClasses),
+ }
+ params = arvados.DirectoryVolumeDriverParameters{
+ Root: oldvol.Root,
+ Serialize: oldvol.Serialize,
+ }
+ default:
+ return fmt.Errorf("unsupported volume type %q", oldvol.Type)
+ }
+ dp, err := json.Marshal(params)
+ if err != nil {
+ return err
+ }
+ newvol.DriverParameters = json.RawMessage(dp)
+ if newvol.Replication < 1 {
+ newvol.Replication = 1
+ }
+ }
+ if accessViaHosts == nil {
+ accessViaHosts = make(map[arvados.URL]arvados.VolumeAccess, 1)
+ }
+ accessViaHosts[myURL] = arvados.VolumeAccess{ReadOnly: oldvol.ReadOnly}
+ newvol.AccessViaHosts = accessViaHosts
+
+ volUUID := oldUUID
+ if oldvol.ReadOnly {
+ } else if oc.Listen == nil {
+ ldr.Logger.Warn("cannot find optimal volume UUID because Listen address is not given in legacy keepstore config")
+ } else if uuid, _, err := findKeepServicesItem(cluster, *oc.Listen); err != nil {
+ ldr.Logger.WithError(err).Warn("cannot find optimal volume UUID: failed to find a matching keep_service listing for this legacy keepstore config")
+ } else if len(uuid) != 27 {
+ ldr.Logger.WithField("UUID", uuid).Warn("cannot find optimal volume UUID: keep_service UUID does not have expected format")
+ } else {
+ rendezvousUUID := cluster.ClusterID + "-nyw5e-" + uuid[12:]
+ if _, ok := cluster.Volumes[rendezvousUUID]; ok {
+ ldr.Logger.Warn("suggesting a random volume UUID because the volume ID matching our keep_service UUID is already in use")
+ } else {
+ volUUID = rendezvousUUID
+ }
+ }
+ if volUUID == "" {
+ volUUID = newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithField("UUID", volUUID).Infof("suggesting a random volume UUID for volume #%d in legacy config", i)
+ }
+ cluster.Volumes[volUUID] = newvol
+ }
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+func (ldr *Loader) alreadyMigrated(oldvol oldKeepstoreVolume, newvols map[string]arvados.Volume, myURL arvados.URL) (string, bool) {
+ for uuid, newvol := range newvols {
+ if oldvol.Type != newvol.Driver {
+ continue
+ }
+ switch oldvol.Type {
+ case "S3":
+ var params arvados.S3VolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Endpoint == params.Endpoint &&
+ oldvol.Region == params.Region &&
+ oldvol.Bucket == params.Bucket &&
+ oldvol.LocationConstraint == params.LocationConstraint {
+ return uuid, true
+ }
+ case "Azure":
+ var params arvados.AzureVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.StorageAccountName == params.StorageAccountName &&
+ oldvol.StorageBaseURL == params.StorageBaseURL &&
+ oldvol.ContainerName == params.ContainerName {
+ return uuid, true
+ }
+ case "Directory":
+ var params arvados.DirectoryVolumeDriverParameters
+ if err := json.Unmarshal(newvol.DriverParameters, ¶ms); err == nil &&
+ oldvol.Root == params.Root {
+ if _, ok := newvol.AccessViaHosts[myURL]; ok {
+ return uuid, true
+ }
+ }
+ }
+ }
+ return "", false
+}
+
+func (ldr *Loader) discoverLocalVolumes(cluster *arvados.Cluster, mountsFile string, myURL arvados.URL) error {
+ if mountsFile == "" {
+ mountsFile = "/proc/mounts"
+ }
+ f, err := os.Open(mountsFile)
+ if err != nil {
+ return fmt.Errorf("error opening %s: %s", mountsFile, err)
+ }
+ defer f.Close()
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ args := strings.Fields(scanner.Text())
+ dev, mount := args[0], args[1]
+ if mount == "/" {
+ continue
+ }
+ if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
+ continue
+ }
+ keepdir := mount + "/keep"
+ if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
+ continue
+ }
+
+ ro := false
+ for _, fsopt := range strings.Split(args[3], ",") {
+ if fsopt == "ro" {
+ ro = true
+ }
+ }
+
+ uuid := newUUID(cluster.ClusterID, "nyw5e")
+ ldr.Logger.WithFields(logrus.Fields{
+ "UUID": uuid,
+ "Driver": "Directory",
+ "DriverParameters.Root": keepdir,
+ "DriverParameters.Serialize": false,
+ "ReadOnly": ro,
+ "Replication": 1,
+ }).Warn("adding local directory volume")
+
+ p, err := json.Marshal(arvados.DirectoryVolumeDriverParameters{
+ Root: keepdir,
+ Serialize: false,
+ })
+ if err != nil {
+ panic(err)
+ }
+ cluster.Volumes[uuid] = arvados.Volume{
+ Driver: "Directory",
+ DriverParameters: p,
+ ReadOnly: ro,
+ Replication: 1,
+ AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
+ myURL: {ReadOnly: ro},
+ },
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("reading %s: %s", mountsFile, err)
+ }
+ return nil
+}
+
+func array2boolmap(keys []string) map[string]bool {
+ m := map[string]bool{}
+ for _, k := range keys {
+ m[k] = true
+ }
+ return m
+}
+
+func newUUID(clusterID, infix string) string {
+ randint, err := rand.Int(rand.Reader, big.NewInt(0).Exp(big.NewInt(36), big.NewInt(15), big.NewInt(0)))
+ if err != nil {
+ panic(err)
+ }
+ randstr := randint.Text(36)
+ for len(randstr) < 15 {
+ randstr = "0" + randstr
+ }
+ return fmt.Sprintf("%s-%s-%s", clusterID, infix, randstr)
+}
+
+// Return the UUID and URL for the controller's keep_services listing
+// corresponding to this host/process.
+func findKeepServicesItem(cluster *arvados.Cluster, listen string) (uuid string, url arvados.URL, err error) {
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return
+ }
+ client.AuthToken = cluster.SystemRootToken
+ var svcList arvados.KeepServiceList
+ err = client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ if err != nil {
+ return
+ }
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("error getting hostname: %s", err)
+ return
+ }
+ for _, ks := range svcList.Items {
+ if ks.ServiceType != "proxy" && keepServiceIsMe(ks, hostname, listen) {
+ url := arvados.URL{
+ Scheme: "http",
+ Host: net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+ }
+ if ks.ServiceSSLFlag {
+ url.Scheme = "https"
+ }
+ return ks.UUID, url, nil
+ }
+ }
+ err = errors.New("failed to find a keep_services entry that matches the current host/port")
+ return
+}
+
+var localhostOrAllInterfaces = map[string]bool{
+ "localhost": true,
+ "127.0.0.1": true,
+ "::1": true,
+ "::": true,
+ "": true,
+}
+
+// Return true if the given KeepService entry matches the given
+// hostname and (keepstore config file) listen address.
+//
+// If the KeepService host is some variant of "localhost", we assume
+// this is a testing or single-node environment, ignore the given
+// hostname, and return true if the port numbers match.
+//
+// The hostname isn't assumed to be a FQDN: a hostname "foo.bar" will
+// match a KeepService host "foo.bar", but also "foo.bar.example",
+// "foo.bar.example.org", etc.
+func keepServiceIsMe(ks arvados.KeepService, hostname string, listen string) bool {
+ // Extract the port name/number from listen, and resolve it to
+ // a port number to compare with ks.ServicePort.
+ _, listenport, err := net.SplitHostPort(listen)
+ if err != nil && strings.HasPrefix(listen, ":") {
+ listenport = listen[1:]
+ }
+ if lp, err := net.LookupPort("tcp", listenport); err != nil {
+ return false
+ } else if !(lp == ks.ServicePort ||
+ (lp == 0 && ks.ServicePort == 80)) {
+ return false
+ }
+
+ kshost := strings.ToLower(ks.ServiceHost)
+ return localhostOrAllInterfaces[kshost] || strings.HasPrefix(kshost+".", strings.ToLower(hostname)+".")
+}
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
new file mode 100644
index 000000000..be4c643f4
--- /dev/null
+++ b/lib/config/deprecated_keepstore_test.go
@@ -0,0 +1,685 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package config
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "text/template"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+type KeepstoreMigrationSuite struct {
+ hostname string // blank = use test system's hostname
+}
+
+var _ = check.Suite(&KeepstoreMigrationSuite{})
+
+func (s *KeepstoreMigrationSuite) checkEquivalentWithKeepstoreConfig(c *check.C, keepstoreconfig, clusterconfig, expectedconfig string) {
+ keepstorefile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstorefile.Name())
+ _, err = io.WriteString(keepstorefile, keepstoreconfig)
+ c.Assert(err, check.IsNil)
+ err = keepstorefile.Close()
+ c.Assert(err, check.IsNil)
+
+ gotldr := testLoader(c, clusterconfig, nil)
+ gotldr.KeepstorePath = keepstorefile.Name()
+ expectedldr := testLoader(c, expectedconfig, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
+ keyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keyfile.Name())
+ io.WriteString(keyfile, "blobsigningkey\n")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ s.checkEquivalentWithKeepstoreConfig(c, `
+Listen: ":12345"
+Debug: true
+LogFormat: text
+MaxBuffers: 1234
+MaxRequests: 2345
+BlobSignatureTTL: 123m
+BlobSigningKeyFile: `+keyfile.Name()+`
+`, `
+Clusters:
+ z1111:
+ {}
+`, `
+Clusters:
+ z1111:
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://`+hostname+`:12345": {}
+ SystemLogs:
+ Format: text
+ LogLevel: debug
+ API:
+ MaxKeepBlockBuffers: 1234
+ MaxConcurrentRequests: 2345
+ Collections:
+ BlobSigningTTL: 123m
+ BlobSigningKey: blobsigningkey
+`)
+}
+
+func (s *KeepstoreMigrationSuite) TestDiscoverLocalVolumes(c *check.C) {
+ tmpd, err := ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tmpd)
+ err = os.Mkdir(tmpd+"/keep", 0777)
+ c.Assert(err, check.IsNil)
+
+ tmpf, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmpf.Name())
+
+ // read/write
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 rw,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: false,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // read-only
+ tmpf.Seek(0, os.SEEK_SET)
+ tmpf.Truncate(0)
+ _, err = fmt.Fprintf(tmpf, "/dev/xvdb %s ext4 ro,noexec 0 0\n", tmpd)
+ c.Assert(err, check.IsNil)
+
+ s.testDeprecatedVolume(c, "DiscoverVolumesFromMountsFile: "+tmpf.Name(), arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: tmpd + "/keep",
+ Serialize: false,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) TestDeprecatedVolumes(c *check.C) {
+ accesskeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(accesskeyfile.Name())
+ io.WriteString(accesskeyfile, "accesskeydata\n")
+
+ secretkeyfile, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(secretkeyfile.Name())
+ io.WriteString(secretkeyfile, "secretkeydata\n")
+
+ // s3, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 1,
+ }, &arvados.S3VolumeDriverParameters{}, &arvados.S3VolumeDriverParameters{})
+
+ // s3, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: S3
+ AccessKeyFile: `+accesskeyfile.Name()+`
+ SecretKeyFile: `+secretkeyfile.Name()+`
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: testbucket
+ LocationConstraint: true
+ IndexPageSize: 1234
+ S3Replication: 4
+ ConnectTimeout: 3m
+ ReadTimeout: 4m
+ RaceWindow: 5m
+ UnsafeDelete: true
+`, arvados.Volume{
+ Driver: "S3",
+ Replication: 4,
+ }, &arvados.S3VolumeDriverParameters{
+ AccessKey: "accesskeydata",
+ SecretKey: "secretkeydata",
+ Endpoint: "https://storage.googleapis.com",
+ Region: "us-east-1z",
+ Bucket: "testbucket",
+ LocationConstraint: true,
+ IndexPageSize: 1234,
+ ConnectTimeout: arvados.Duration(time.Minute * 3),
+ ReadTimeout: arvados.Duration(time.Minute * 4),
+ RaceWindow: arvados.Duration(time.Minute * 5),
+ UnsafeDelete: true,
+ }, &arvados.S3VolumeDriverParameters{})
+
+ // azure, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+`, arvados.Volume{
+ Driver: "Azure",
+ Replication: 1,
+ }, &arvados.AzureVolumeDriverParameters{}, &arvados.AzureVolumeDriverParameters{})
+
+ // azure, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Azure
+ ReadOnly: true
+ StorageAccountName: storageacctname
+ StorageAccountKeyFile: `+secretkeyfile.Name()+`
+ StorageBaseURL: https://example.example
+ ContainerName: testctr
+ LocationConstraint: true
+ AzureReplication: 4
+ RequestTimeout: 3m
+ ListBlobsRetryDelay: 4m
+ ListBlobsMaxAttempts: 5
+`, arvados.Volume{
+ Driver: "Azure",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.AzureVolumeDriverParameters{
+ StorageAccountName: "storageacctname",
+ StorageAccountKey: "secretkeydata",
+ StorageBaseURL: "https://example.example",
+ ContainerName: "testctr",
+ RequestTimeout: arvados.Duration(time.Minute * 3),
+ ListBlobsRetryDelay: arvados.Duration(time.Minute * 4),
+ ListBlobsMaxAttempts: 5,
+ }, &arvados.AzureVolumeDriverParameters{})
+
+ // directory, empty/default
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ Root: /tmp/xyzzy
+`, arvados.Volume{
+ Driver: "Directory",
+ Replication: 1,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ }, &arvados.DirectoryVolumeDriverParameters{})
+
+ // directory, fully configured
+ s.testDeprecatedVolume(c, `
+Volumes:
+- Type: Directory
+ ReadOnly: true
+ Root: /tmp/xyzzy
+ DirectoryReplication: 4
+ Serialize: true
+`, arvados.Volume{
+ Driver: "Directory",
+ ReadOnly: true,
+ Replication: 4,
+ }, &arvados.DirectoryVolumeDriverParameters{
+ Root: "/tmp/xyzzy",
+ Serialize: true,
+ }, &arvados.DirectoryVolumeDriverParameters{})
+}
+
+func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata string, expectvol arvados.Volume, expectparams interface{}, paramsdst interface{}) {
+ hostname := s.hostname
+ if hostname == "" {
+ h, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ hostname = h
+ }
+
+ oldconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(oldconfig.Name())
+ io.WriteString(oldconfig, "Listen: :12345\n"+oldconfigdata)
+ if !strings.Contains(oldconfigdata, "DiscoverVolumesFromMountsFile") {
+ // Prevent tests from looking at the real /proc/mounts on the test host.
+ io.WriteString(oldconfig, "\nDiscoverVolumesFromMountsFile: /dev/null\n")
+ }
+
+ ldr := testLoader(c, "Clusters: {z1111: {}}", nil)
+ ldr.KeepstorePath = oldconfig.Name()
+ cfg, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+ cc := cfg.Clusters["z1111"]
+ c.Check(cc.Volumes, check.HasLen, 1)
+ for uuid, v := range cc.Volumes {
+ c.Check(uuid, check.HasLen, 27)
+ c.Check(v.Driver, check.Equals, expectvol.Driver)
+ c.Check(v.Replication, check.Equals, expectvol.Replication)
+
+ avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+ c.Check(ok, check.Equals, true)
+ c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
+
+ err := json.Unmarshal(v.DriverParameters, paramsdst)
+ c.Check(err, check.IsNil)
+ c.Check(paramsdst, check.DeepEquals, expectparams)
+ }
+}
+
+// How we handle a volume from a legacy keepstore config file depends
+// on whether it's writable, whether a volume using the same cloud
+// backend already exists in the cluster config, and (if so) whether
+// it already has an AccessViaHosts entry for this host.
+//
+// In all cases, we should end up with an AccessViaHosts entry for
+// this host, to indicate that the current host's volumes have been
+// migrated.
+
+// Same backend already referenced in cluster config, this host
+// already listed in AccessViaHosts --> no change, except possibly
+// updating the ReadOnly flag on the AccessViaHosts entry.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ S3Replication: 3
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Writable volume, same cloud backend already referenced in cluster
+// config --> change UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateUUID(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+`)
+ c.Check(after, check.HasLen, len(before))
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "readonlyonother")
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same cloud backend not yet referenced --> add a
+// new volume, with UUID to match this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddCloudVolume(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "bucket-to-migrate")
+ c.Check(newvol.Replication, check.Equals, 3)
+
+ c.Check(newuuids[0], check.Equals, expectUUID)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, but this host isn't in AccessViaHosts --> add a new
+// volume, with UUID to match this keepstore's UUID (filesystem-backed
+// volumes are assumed to be different on different hosts, even if
+// paths are the same).
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddLocalVolume(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sdd
+ DirectoryReplication: 2
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.DirectoryVolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Root, check.Equals, "/data/sdd")
+ c.Check(newvol.Replication, check.Equals, 2)
+}
+
+// Writable volume, same filesystem backend already referenced in
+// cluster config, and this host is already listed in AccessViaHosts
+// --> already migrated, don't change anything.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_LocalVolumeAlreadyMigrated(c *check.C) {
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :12345
+Volumes:
+- Type: Directory
+ Root: /data/sde
+ DirectoryReplication: 2
+`)
+ checkEqualYAML(c, after, before)
+}
+
+// Multiple writable cloud-backed volumes --> one of them will get a
+// UUID matching this keepstore's UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddMultipleCloudVolumes(c *check.C) {
+ port, expectUUID := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: first-bucket-to-migrate
+ S3Replication: 3
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: second-bucket-to-migrate
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 2)
+ // Sort by bucket name (so "first" comes before "second")
+ params := map[string]arvados.S3VolumeDriverParameters{}
+ for _, uuid := range newuuids {
+ var p arvados.S3VolumeDriverParameters
+ json.Unmarshal(after[uuid].DriverParameters, &p)
+ params[uuid] = p
+ }
+ sort.Slice(newuuids, func(i, j int) bool { return params[newuuids[i]].Bucket < params[newuuids[j]].Bucket })
+ newvol0, newvol1 := after[newuuids[0]], after[newuuids[1]]
+ params0, params1 := params[newuuids[0]], params[newuuids[1]]
+
+ c.Check(params0.Bucket, check.Equals, "first-bucket-to-migrate")
+ c.Check(newvol0.Replication, check.Equals, 3)
+
+ c.Check(params1.Bucket, check.Equals, "second-bucket-to-migrate")
+ c.Check(newvol1.Replication, check.Equals, 3)
+
+ // Don't care which one gets the special UUID
+ if newuuids[0] != expectUUID {
+ c.Check(newuuids[1], check.Equals, expectUUID)
+ }
+}
+
+// Non-writable volume, same cloud backend already referenced in
+// cluster config --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateWithReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ S3Replication: 3
+ ReadOnly: true
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Writable volume, same cloud backend already writable by another
+// keepstore server --> add this host to AccessViaHosts with
+// ReadOnly==true
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_UpdateAlreadyWritable(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ S3Replication: 3
+ ReadOnly: false
+`)
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ url := arvados.URL{
+ Scheme: "http",
+ Host: fmt.Sprintf("%s:%d", hostname, port),
+ }
+ _, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, false)
+ _, ok = after["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
+ c.Check(ok, check.Equals, true)
+}
+
+// Non-writable volume, same cloud backend not already referenced in
+// cluster config --> assign a new random volume UUID.
+func (s *KeepstoreMigrationSuite) TestIncrementalVolumeMigration_AddReadOnly(c *check.C) {
+ port, _ := s.getTestKeepstorePortAndMatchingVolumeUUID(c)
+ before, after := s.loadWithKeepstoreConfig(c, `
+Listen: :`+strconv.Itoa(port)+`
+Volumes:
+- Type: S3
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: differentbucket
+ S3Replication: 3
+`)
+ newuuids := s.findAddedVolumes(c, before, after, 1)
+ newvol := after[newuuids[0]]
+
+ var params arvados.S3VolumeDriverParameters
+ json.Unmarshal(newvol.DriverParameters, ¶ms)
+ c.Check(params.Bucket, check.Equals, "differentbucket")
+
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+ _, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+ c.Check(ok, check.Equals, true)
+}
+
+const clusterConfigForKeepstoreMigrationTest = `
+Clusters:
+ zzzzz:
+ SystemRootToken: ` + arvadostest.AdminToken + `
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://{{.hostname}}:12345": {}
+ Controller:
+ ExternalURL: "https://{{.controller}}"
+ TLS:
+ Insecure: true
+ Volumes:
+
+ zzzzz-nyw5e-alreadymigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: alreadymigrated
+ Replication: 3
+
+ zzzzz-nyw5e-readonlyonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {ReadOnly: true}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: readonlyonother
+ Replication: 3
+
+ zzzzz-nyw5e-writableonother:
+ AccessViaHosts:
+ "http://other.host.example:12345": {}
+ Driver: S3
+ DriverParameters:
+ Endpoint: https://storage.googleapis.com
+ Region: us-east-1z
+ Bucket: writableonother
+ Replication: 3
+
+ zzzzz-nyw5e-localfilesystem:
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sdd
+ Replication: 1
+
+ zzzzz-nyw5e-localismigrated:
+ AccessViaHosts:
+ "http://{{.hostname}}:12345": {}
+ Driver: Directory
+ DriverParameters:
+ Root: /data/sde
+ Replication: 1
+`
+
+// Determine the effect of combining the given legacy keepstore config
+// YAML (just the "Volumes" entries of an old keepstore config file)
+// with the example clusterConfigForKeepstoreMigrationTest config.
+//
+// Return two Volumes configs -- one without loading
+// keepstoreconfigdata ("before") and one with ("after") -- for the
+// caller to compare.
+func (s *KeepstoreMigrationSuite) loadWithKeepstoreConfig(c *check.C, keepstoreVolumesYAML string) (before, after map[string]arvados.Volume) {
+ ldr := testLoader(c, s.clusterConfigYAML(c), nil)
+ cBefore, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ keepstoreconfig, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(keepstoreconfig.Name())
+ io.WriteString(keepstoreconfig, keepstoreVolumesYAML)
+
+ ldr = testLoader(c, s.clusterConfigYAML(c), nil)
+ ldr.KeepstorePath = keepstoreconfig.Name()
+ cAfter, err := ldr.Load()
+ c.Assert(err, check.IsNil)
+
+ return cBefore.Clusters["zzzzz"].Volumes, cAfter.Clusters["zzzzz"].Volumes
+}
+
+func (s *KeepstoreMigrationSuite) clusterConfigYAML(c *check.C) string {
+ hostname, err := os.Hostname()
+ c.Assert(err, check.IsNil)
+
+ tmpl := template.Must(template.New("config").Parse(clusterConfigForKeepstoreMigrationTest))
+
+ var clusterconfigdata bytes.Buffer
+ err = tmpl.Execute(&clusterconfigdata, map[string]interface{}{
+ "hostname": hostname,
+ "controller": os.Getenv("ARVADOS_API_HOST"),
+ })
+ c.Assert(err, check.IsNil)
+
+ return clusterconfigdata.String()
+}
+
+// Return the uuids of volumes that appear in "after" but not
+// "before".
+//
+// Assert the returned slice has at least minAdded entries.
+func (s *KeepstoreMigrationSuite) findAddedVolumes(c *check.C, before, after map[string]arvados.Volume, minAdded int) (uuids []string) {
+ for uuid := range after {
+ if _, ok := before[uuid]; !ok {
+ uuids = append(uuids, uuid)
+ }
+ }
+ if len(uuids) < minAdded {
+ c.Assert(uuids, check.HasLen, minAdded)
+ }
+ return
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePortAndMatchingVolumeUUID(c *check.C) (int, string) {
+ for uuid, port := range s.getTestKeepstorePorts(c) {
+ c.Assert(uuid, check.HasLen, 27)
+ return port, "zzzzz-nyw5e-" + uuid[12:]
+ }
+ c.Fatal("getTestKeepstorePorts() returned nothing")
+ return 0, ""
+}
+
+func (s *KeepstoreMigrationSuite) getTestKeepstorePorts(c *check.C) map[string]int {
+ client := arvados.NewClientFromEnv()
+ var svcList arvados.KeepServiceList
+ err := client.RequestAndDecode(&svcList, "GET", "arvados/v1/keep_services", nil, nil)
+ c.Assert(err, check.IsNil)
+ ports := map[string]int{}
+ for _, ks := range svcList.Items {
+ if ks.ServiceType == "disk" {
+ ports[ks.UUID] = ks.ServicePort
+ }
+ }
+ return ports
+}
+
+func (s *KeepstoreMigrationSuite) TestKeepServiceIsMe(c *check.C) {
+ for i, trial := range []struct {
+ match bool
+ hostname string
+ listen string
+ serviceHost string
+ servicePort int
+ }{
+ {true, "keep0", "keep0", "keep0", 80},
+ {true, "keep0", "[::1]:http", "keep0", 80},
+ {true, "keep0", "[::]:http", "keep0", 80},
+ {true, "keep0", "keep0:25107", "keep0", 25107},
+ {true, "keep0", ":25107", "keep0", 25107},
+ {true, "keep0.domain", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0.domain.example", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "keep0.domain.example", 25107},
+ {true, "keep0", ":25107", "Keep0.domain.example", 25107},
+ {true, "keep0", ":http", "keep0.domain.example", 80},
+ {true, "keep0", ":25107", "localhost", 25107},
+ {true, "keep0", ":25107", "::1", 25107},
+ {false, "keep0", ":25107", "keep0", 1111}, // different port
+ {false, "keep0", ":25107", "localhost", 1111}, // different port
+ {false, "keep0", ":http", "keep0.domain.example", 443}, // different port
+ {false, "keep0", ":bogussss", "keep0", 25107}, // unresolvable port
+ {false, "keep0", ":25107", "keep1", 25107}, // different hostname
+ {false, "keep1", ":25107", "keep10", 25107}, // different hostname (prefix, but not on a "." boundary)
+ } {
+ c.Check(keepServiceIsMe(arvados.KeepService{ServiceHost: trial.serviceHost, ServicePort: trial.servicePort}, trial.hostname, trial.listen), check.Equals, trial.match, check.Commentf("trial #%d: %#v", i, trial))
+ }
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 308b0cc35..55bb2193b 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -13,7 +13,7 @@ import (
func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
- s.checkEquivalent(c, `
+ checkEquivalent(c, `
Clusters:
z1111:
NodeProfiles:
diff --git a/lib/config/export.go b/lib/config/export.go
index b125d7dc9..a349db516 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -63,8 +63,10 @@ var whitelist = map[string]bool{
"API": true,
"API.AsyncPermissionsUpdateInterval": false,
"API.DisabledAPIs": false,
+ "API.MaxConcurrentRequests": false,
"API.MaxIndexDatabaseRead": false,
"API.MaxItemsPerResponse": true,
+ "API.MaxKeepBlockBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
"API.RailsSessionSecretToken": false,
@@ -80,6 +82,12 @@ var whitelist = map[string]bool{
"Collections.BlobSigning": true,
"Collections.BlobSigningKey": false,
"Collections.BlobSigningTTL": true,
+ "Collections.BlobTrash": false,
+ "Collections.BlobTrashLifetime": false,
+ "Collections.BlobTrashConcurrency": false,
+ "Collections.BlobTrashCheckInterval": false,
+ "Collections.BlobDeleteConcurrency": false,
+ "Collections.BlobReplicateConcurrency": false,
"Collections.CollectionVersioning": false,
"Collections.DefaultReplication": true,
"Collections.DefaultTrashLifetime": true,
@@ -153,6 +161,16 @@ var whitelist = map[string]bool{
"Users.NewUsersAreActive": false,
"Users.UserNotifierEmailFrom": false,
"Users.UserProfileNotificationAddress": false,
+ "Volumes": true,
+ "Volumes.*": true,
+ "Volumes.*.*": false,
+ "Volumes.*.AccessViaHosts": true,
+ "Volumes.*.AccessViaHosts.*": true,
+ "Volumes.*.AccessViaHosts.*.ReadOnly": true,
+ "Volumes.*.ReadOnly": true,
+ "Volumes.*.Replication": true,
+ "Volumes.*.StorageClasses": true,
+ "Volumes.*.StorageClasses.*": false,
"Workbench": true,
"Workbench.ActivationContactLink": false,
"Workbench.APIClientConnectTimeout": true,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 602f30e1d..92ceb86bd 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -182,6 +182,15 @@ Clusters:
# parameter higher than this value, this value is used instead.
MaxItemsPerResponse: 1000
+ # Maximum number of concurrent requests to accept in a single
+ # service process, or 0 for no limit. Currently supported only
+ # by keepstore.
+ MaxConcurrentRequests: 0
+
+ # Maximum number of 64MiB memory buffers per keepstore server
+ # process, or 0 for no limit.
+ MaxKeepBlockBuffers: 128
+
# API methods to disable. Disabled methods are not listed in the
# discovery document, and respond 404 to all requests.
# Example: {"jobs.create":{}, "pipeline_instances.create": {}}
@@ -319,15 +328,44 @@ Clusters:
# BlobSigningKey is a string of alphanumeric characters used to
# generate permission signatures for Keep locators. It must be
- # identical to the permission key given to Keep. IMPORTANT: This is
- # a site secret. It should be at least 50 characters.
+ # identical to the permission key given to Keep. IMPORTANT: This
+ # is a site secret. It should be at least 50 characters.
#
- # Modifying blob_signing_key will invalidate all existing
+ # Modifying BlobSigningKey will invalidate all existing
# signatures, which can cause programs to fail (e.g., arv-put,
- # arv-get, and Crunch jobs). To avoid errors, rotate keys only when
- # no such processes are running.
+ # arv-get, and Crunch jobs). To avoid errors, rotate keys only
+ # when no such processes are running.
BlobSigningKey: ""
+ # Enable garbage collection of unreferenced blobs in Keep.
+ BlobTrash: true
+
+ # Time to leave unreferenced blobs in "trashed" state before
+ # deleting them, or 0 to skip the "trashed" state entirely and
+ # delete unreferenced blobs.
+ #
+ # If you use any Amazon S3 buckets as storage volumes, this
+ # must be at least 24h to avoid occasional data loss.
+ BlobTrashLifetime: 336h
+
+ # How often to check for (and delete) trashed blocks whose
+ # BlobTrashLifetime has expired.
+ BlobTrashCheckInterval: 24h
+
+ # Maximum number of concurrent "trash blob" and "delete trashed
+ # blob" operations conducted by a single keepstore process. Each
+ # of these can be set to 0 to disable the respective operation.
+ #
+ # If BlobTrashLifetime is zero, "trash" and "delete trash"
+ # happen at once, so only the lower of these two values is used.
+ BlobTrashConcurrency: 4
+ BlobDeleteConcurrency: 4
+
+ # Maximum number of concurrent "create additional replica of
+ # existing blob" operations conducted by a single keepstore
+ # process.
+ BlobReplicateConcurrency: 4
+
# Default replication level for collections. This is used when a
# collection's replication_desired attribute is nil.
DefaultReplication: 2
@@ -736,6 +774,47 @@ Clusters:
Price: 0.1
Preemptible: false
+ Volumes:
+ SAMPLE:
+ AccessViaHosts:
+ SAMPLE:
+ ReadOnly: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses:
+ default: true
+ SAMPLE: true
+ Driver: s3
+ DriverParameters:
+
+ # for s3 driver
+ AccessKey: aaaaa
+ SecretKey: aaaaa
+ Endpoint: ""
+ Region: us-east-1a
+ Bucket: aaaaa
+ LocationConstraint: false
+ IndexPageSize: 1000
+ ConnectTimeout: 1m
+ ReadTimeout: 10m
+ RaceWindow: 24h
+ UnsafeDelete: false
+
+ # for azure driver
+ StorageAccountName: aaaaa
+ StorageAccountKey: aaaaa
+ StorageBaseURL: core.windows.net
+ ContainerName: aaaaa
+ RequestTimeout: 30s
+ ListBlobsRetryDelay: 10s
+ ListBlobsMaxAttempts: 10
+ MaxGetBytes: 0
+ WriteRaceInterval: 15s
+ WriteRacePollTime: 1s
+
+ # for local directory driver
+ Root: /var/lib/arvados/keep-data
+
Mail:
MailchimpAPIKey: ""
MailchimpListID: ""
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index c7289350e..17e0af7ba 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -321,7 +321,7 @@ Clusters:
}
func (s *LoadSuite) TestMovedKeys(c *check.C) {
- s.checkEquivalent(c, `# config has old keys only
+ checkEquivalent(c, `# config has old keys only
Clusters:
zzzzz:
RequestLimits:
@@ -334,7 +334,7 @@ Clusters:
MaxRequestAmplification: 3
MaxItemsPerResponse: 999
`)
- s.checkEquivalent(c, `# config has both old and new keys; old values win
+ checkEquivalent(c, `# config has both old and new keys; old values win
Clusters:
zzzzz:
RequestLimits:
@@ -352,30 +352,45 @@ Clusters:
`)
}
-func (s *LoadSuite) checkEquivalent(c *check.C, goty, expectedy string) {
- got, err := testLoader(c, goty, nil).Load()
+func checkEquivalent(c *check.C, goty, expectedy string) {
+ gotldr := testLoader(c, goty, nil)
+ expectedldr := testLoader(c, expectedy, nil)
+ checkEquivalentLoaders(c, gotldr, expectedldr)
+}
+
+func checkEqualYAML(c *check.C, got, expected interface{}) {
+ expectedyaml, err := yaml.Marshal(expected)
c.Assert(err, check.IsNil)
- expected, err := testLoader(c, expectedy, nil).Load()
+ gotyaml, err := yaml.Marshal(got)
c.Assert(err, check.IsNil)
- if !c.Check(got, check.DeepEquals, expected) {
+ if !bytes.Equal(gotyaml, expectedyaml) {
cmd := exec.Command("diff", "-u", "--label", "expected", "--label", "got", "/dev/fd/3", "/dev/fd/4")
- for _, obj := range []interface{}{expected, got} {
- y, _ := yaml.Marshal(obj)
+ for _, y := range [][]byte{expectedyaml, gotyaml} {
pr, pw, err := os.Pipe()
c.Assert(err, check.IsNil)
defer pr.Close()
- go func() {
- io.Copy(pw, bytes.NewBuffer(y))
+ go func(data []byte) {
+ pw.Write(data)
pw.Close()
- }()
+ }(y)
cmd.ExtraFiles = append(cmd.ExtraFiles, pr)
}
diff, err := cmd.CombinedOutput()
+ // diff should report differences and exit non-zero.
+ c.Check(err, check.NotNil)
c.Log(string(diff))
- c.Check(err, check.IsNil)
+ c.Error("got != expected; see diff (-expected +got) above")
}
}
+func checkEquivalentLoaders(c *check.C, gotldr, expectedldr *Loader) {
+ got, err := gotldr.Load()
+ c.Assert(err, check.IsNil)
+ expected, err := expectedldr.Load()
+ c.Assert(err, check.IsNil)
+ checkEqualYAML(c, got, expected)
+}
+
func checkListKeys(path string, x interface{}) (err error) {
v := reflect.Indirect(reflect.ValueOf(x))
switch v.Kind() {
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index ae6ac70e9..7ab38c6ca 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -11,11 +11,12 @@ import (
"git.curoverse.com/arvados.git/lib/cmd"
"git.curoverse.com/arvados.git/lib/service"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/prometheus/client_golang/prometheus"
)
var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) service.Handler {
+func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
ac, err := arvados.NewClientFromConfig(cluster)
if err != nil {
return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
@@ -25,6 +26,7 @@ func newHandler(ctx context.Context, cluster *arvados.Cluster, token string) ser
Context: ctx,
ArvClient: ac,
AuthToken: token,
+ Registry: reg,
}
go d.Start()
return d
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 731c6d25d..f0aa83c2e 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -48,10 +48,10 @@ type dispatcher struct {
Context context.Context
ArvClient *arvados.Client
AuthToken string
+ Registry *prometheus.Registry
InstanceSetID cloud.InstanceSetID
logger logrus.FieldLogger
- reg *prometheus.Registry
instanceSet cloud.InstanceSet
pool pool
queue scheduler.ContainerQueue
@@ -132,14 +132,13 @@ func (disp *dispatcher) initialize() {
disp.sshKey = key
}
- disp.reg = prometheus.NewRegistry()
- instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.reg)
+ instanceSet, err := newInstanceSet(disp.Cluster, disp.InstanceSetID, disp.logger, disp.Registry)
if err != nil {
disp.logger.Fatalf("error initializing driver: %s", err)
}
disp.instanceSet = instanceSet
- disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.reg, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
- disp.queue = container.NewQueue(disp.logger, disp.reg, disp.typeChooser, disp.ArvClient)
+ disp.pool = worker.NewPool(disp.logger, disp.ArvClient, disp.Registry, disp.InstanceSetID, disp.instanceSet, disp.newExecutor, disp.sshKey.PublicKey(), disp.Cluster)
+ disp.queue = container.NewQueue(disp.logger, disp.Registry, disp.typeChooser, disp.ArvClient)
if disp.Cluster.ManagementToken == "" {
disp.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -154,7 +153,7 @@ func (disp *dispatcher) initialize() {
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/drain", disp.apiInstanceDrain)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/run", disp.apiInstanceRun)
mux.HandlerFunc("POST", "/arvados/v1/dispatch/instances/kill", disp.apiInstanceKill)
- metricsH := promhttp.HandlerFor(disp.reg, promhttp.HandlerOpts{
+ metricsH := promhttp.HandlerFor(disp.Registry, promhttp.HandlerOpts{
ErrorLog: disp.logger,
})
mux.Handler("GET", "/metrics", metricsH)
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index b6737bc55..f490f83c1 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -22,6 +22,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-systemd/daemon"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -30,7 +31,7 @@ type Handler interface {
CheckHealth() error
}
-type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string) Handler
+type NewHandlerFunc func(_ context.Context, _ *arvados.Cluster, token string, registry *prometheus.Registry) Handler
type command struct {
newHandler NewHandlerFunc
@@ -68,7 +69,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
-
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -87,22 +87,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
if err != nil {
return 1
}
- log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+
+ // Now that we've read the config, replace the bootstrap
+ // logger with a new one according to the logging config.
+ log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
+ logger := log.WithFields(logrus.Fields{
"PID": os.Getpid(),
})
- ctx := ctxlog.Context(c.ctx, log)
+ ctx := ctxlog.Context(c.ctx, logger)
- listen, err := getListenAddr(cluster.Services, c.svcName)
+ listenURL, err := getListenAddr(cluster.Services, c.svcName)
if err != nil {
return 1
}
+ ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
if cluster.SystemRootToken == "" {
- log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
cluster.SystemRootToken = os.Getenv("ARVADOS_API_TOKEN")
}
if cluster.Services.Controller.ExternalURL.Host == "" {
- log.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
u, err := url.Parse("https://" + os.Getenv("ARVADOS_API_HOST"))
if err != nil {
err = fmt.Errorf("ARVADOS_API_HOST: %s", err)
@@ -114,27 +119,42 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
}
}
- handler := c.newHandler(ctx, cluster, cluster.SystemRootToken)
+ reg := prometheus.NewRegistry()
+ handler := c.newHandler(ctx, cluster, cluster.SystemRootToken, reg)
if err = handler.CheckHealth(); err != nil {
return 1
}
+
+ instrumented := httpserver.Instrument(reg, log,
+ httpserver.HandlerWithContext(ctx,
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ httpserver.NewRequestLimiter(cluster.API.MaxConcurrentRequests, handler)))))
srv := &httpserver.Server{
Server: http.Server{
- Handler: httpserver.HandlerWithContext(ctx,
- httpserver.AddRequestIDs(httpserver.LogRequests(handler))),
+ Handler: instrumented.ServeAPI(cluster.ManagementToken, instrumented),
},
- Addr: listen,
+ Addr: listenURL.Host,
+ }
+ if listenURL.Scheme == "https" {
+ tlsconfig, err := tlsConfigWithCertUpdater(cluster, logger)
+ if err != nil {
+ logger.WithError(err).Error("cannot start %s service on %s", c.svcName, listenURL.String())
+ return 1
+ }
+ srv.TLSConfig = tlsconfig
}
err = srv.Start()
if err != nil {
return 1
}
- log.WithFields(logrus.Fields{
+ logger.WithFields(logrus.Fields{
+ "URL": listenURL,
"Listen": srv.Addr,
"Service": c.svcName,
}).Info("listening")
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.WithError(err).Errorf("error notifying init daemon")
+ logger.WithError(err).Errorf("error notifying init daemon")
}
go func() {
<-ctx.Done()
@@ -149,20 +169,27 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (string, error) {
+func getListenAddr(svcs arvados.Services, prog arvados.ServiceName) (arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
- return "", fmt.Errorf("unknown service name %q", prog)
+ return arvados.URL{}, fmt.Errorf("unknown service name %q", prog)
}
for url := range svc.InternalURLs {
if strings.HasPrefix(url.Host, "localhost:") {
- return url.Host, nil
+ return url, nil
}
listener, err := net.Listen("tcp", url.Host)
if err == nil {
listener.Close()
- return url.Host, nil
+ return url, nil
}
}
- return "", fmt.Errorf("configuration does not enable the %s service on this host", prog)
+ return arvados.URL{}, fmt.Errorf("configuration does not enable the %s service on this host", prog)
+}
+
+type contextKeyURL struct{}
+
+func URLFromContext(ctx context.Context) (arvados.URL, bool) {
+ u, ok := ctx.Value(contextKeyURL{}).(arvados.URL)
+ return u, ok
}
diff --git a/lib/service/tls.go b/lib/service/tls.go
new file mode 100644
index 000000000..8fef46158
--- /dev/null
+++ b/lib/service/tls.go
@@ -0,0 +1,80 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package service
+
+import (
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+)
+
+func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogger) (*tls.Config, error) {
+ currentCert := make(chan *tls.Certificate, 1)
+ loaded := false
+
+ key, cert := cluster.TLS.Key, cluster.TLS.Certificate
+ if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
+ return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+ }
+
+ update := func() error {
+ cert, err := tls.LoadX509KeyPair(cert, key)
+ if err != nil {
+ return fmt.Errorf("error loading X509 key pair: %s", err)
+ }
+ if loaded {
+ // Throw away old cert
+ <-currentCert
+ }
+ currentCert <- &cert
+ loaded = true
+ return nil
+ }
+ err := update()
+ if err != nil {
+ return nil, err
+ }
+
+ go func() {
+ reload := make(chan os.Signal, 1)
+ signal.Notify(reload, syscall.SIGHUP)
+ for range reload {
+ err := update()
+ if err != nil {
+ logger.WithError(err).Warn("error updating TLS certificate")
+ }
+ }
+ }()
+
+ // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
+ return &tls.Config{
+ PreferServerCipherSuites: true,
+ CurvePreferences: []tls.CurveID{
+ tls.CurveP256,
+ tls.X25519,
+ },
+ MinVersion: tls.VersionTLS12,
+ CipherSuites: []uint16{
+ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ },
+ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ cert := <-currentCert
+ currentCert <- cert
+ return cert, nil
+ },
+ }, nil
+}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index f6b736d58..74a08246a 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -72,6 +72,8 @@ type Cluster struct {
DisabledAPIs StringSet
MaxIndexDatabaseRead int
MaxItemsPerResponse int
+ MaxConcurrentRequests int
+ MaxKeepBlockBuffers int
MaxRequestAmplification int
MaxRequestSize int
RailsSessionSecretToken string
@@ -86,13 +88,19 @@ type Cluster struct {
UnloggedAttributes StringSet
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]struct {
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ BlobTrash bool
+ BlobTrashLifetime Duration
+ BlobTrashCheckInterval Duration
+ BlobTrashConcurrency int
+ BlobDeleteConcurrency int
+ BlobReplicateConcurrency int
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
Value interface{}
Function string
Protected bool
@@ -143,6 +151,7 @@ type Cluster struct {
UserNotifierEmailFrom string
UserProfileNotificationAddress string
}
+ Volumes map[string]Volume
Workbench struct {
ActivationContactLink string
APIClientConnectTimeout Duration
@@ -182,6 +191,48 @@ type Cluster struct {
EnableBetaController14287 bool
}
+type Volume struct {
+ AccessViaHosts map[URL]VolumeAccess
+ ReadOnly bool
+ Replication int
+ StorageClasses map[string]bool
+ Driver string
+ DriverParameters json.RawMessage
+}
+
+type S3VolumeDriverParameters struct {
+ AccessKey string
+ SecretKey string
+ Endpoint string
+ Region string
+ Bucket string
+ LocationConstraint bool
+ IndexPageSize int
+ ConnectTimeout Duration
+ ReadTimeout Duration
+ RaceWindow Duration
+ UnsafeDelete bool
+}
+
+type AzureVolumeDriverParameters struct {
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string
+ ContainerName string
+ RequestTimeout Duration
+ ListBlobsRetryDelay Duration
+ ListBlobsMaxAttempts int
+}
+
+type DirectoryVolumeDriverParameters struct {
+ Root string
+ Serialize bool
+}
+
+type VolumeAccess struct {
+ ReadOnly bool
+}
+
type Services struct {
Composer Service
Controller Service
@@ -225,6 +276,10 @@ func (su URL) MarshalText() ([]byte, error) {
return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
}
+func (su URL) String() string {
+ return (*url.URL)(&su).String()
+}
+
type ServiceInstance struct{}
type PostgreSQL struct {
diff --git a/sdk/go/ctxlog/log.go b/sdk/go/ctxlog/log.go
index e66eeadee..f7a7af64d 100644
--- a/sdk/go/ctxlog/log.go
+++ b/sdk/go/ctxlog/log.go
@@ -41,7 +41,7 @@ func FromContext(ctx context.Context) logrus.FieldLogger {
// New returns a new logger with the indicated format and
// level.
-func New(out io.Writer, format, level string) logrus.FieldLogger {
+func New(out io.Writer, format, level string) *logrus.Logger {
logger := logrus.New()
logger.Out = out
setFormat(logger, format)
diff --git a/sdk/go/httpserver/httpserver.go b/sdk/go/httpserver/httpserver.go
index a94146f85..627e04f0b 100644
--- a/sdk/go/httpserver/httpserver.go
+++ b/sdk/go/httpserver/httpserver.go
@@ -43,7 +43,12 @@ func (srv *Server) Start() error {
srv.cond = sync.NewCond(mutex.RLocker())
srv.running = true
go func() {
- err = srv.Serve(tcpKeepAliveListener{srv.listener})
+ lnr := tcpKeepAliveListener{srv.listener}
+ if srv.TLSConfig != nil {
+ err = srv.ServeTLS(lnr, "", "")
+ } else {
+ err = srv.Serve(lnr)
+ }
if !srv.wantDown {
srv.err = err
}
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 3c17b3bd0..66aa6c583 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -7,8 +7,8 @@ package main
import (
"bytes"
"context"
+ "encoding/json"
"errors"
- "flag"
"fmt"
"io"
"io/ioutil"
@@ -22,99 +22,85 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-const (
- azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
- azureDefaultListBlobsMaxAttempts = 12
- azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
-)
-
-var (
- azureMaxGetBytes int
- azureStorageAccountName string
- azureStorageAccountKeyFile string
- azureStorageReplication int
- azureWriteRaceInterval = 15 * time.Second
- azureWriteRacePollTime = time.Second
-)
+func init() {
+ driver["Azure"] = newAzureBlobVolume
+}
-func readKeyFromFile(file string) (string, error) {
- buf, err := ioutil.ReadFile(file)
+func newAzureBlobVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+ v := AzureBlobVolume{
+ StorageBaseURL: storage.DefaultBaseURL,
+ RequestTimeout: azureDefaultRequestTimeout,
+ WriteRaceInterval: azureDefaultWriteRaceInterval,
+ WriteRacePollTime: azureDefaultWriteRacePollTime,
+ cluster: cluster,
+ volume: volume,
+ logger: logger,
+ }
+ err := json.Unmarshal(volume.DriverParameters, &v)
if err != nil {
- return "", errors.New("reading key from " + file + ": " + err.Error())
+ return nil, err
}
- accountKey := strings.TrimSpace(string(buf))
- if accountKey == "" {
- return "", errors.New("empty account key in " + file)
+ if v.ListBlobsRetryDelay == 0 {
+ v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
+ }
+ if v.ListBlobsMaxAttempts == 0 {
+ v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
+ }
+ if v.ContainerName == "" || v.StorageAccountName == "" || v.StorageAccountKey == "" {
+ return nil, errors.New("DriverParameters: ContainerName, StorageAccountName, and StorageAccountKey must be provided")
+ }
+ v.azClient, err = storage.NewClient(v.StorageAccountName, v.StorageAccountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
+ if err != nil {
+ return nil, fmt.Errorf("creating Azure storage client: %s", err)
+ }
+ v.azClient.Sender = &singleSender{}
+ v.azClient.HTTPClient = &http.Client{
+ Timeout: time.Duration(v.RequestTimeout),
+ }
+ bs := v.azClient.GetBlobService()
+ v.container = &azureContainer{
+ ctr: bs.GetContainerReference(v.ContainerName),
}
- return accountKey, nil
-}
-
-type azureVolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (s *azureVolumeAdder) String() string {
- return "-"
-}
-func (s *azureVolumeAdder) Set(containerName string) error {
- s.Config.Volumes = append(s.Config.Volumes, &AzureBlobVolume{
- ContainerName: containerName,
- StorageAccountName: azureStorageAccountName,
- StorageAccountKeyFile: azureStorageAccountKeyFile,
- AzureReplication: azureStorageReplication,
- ReadOnly: deprecated.flagReadonly,
- })
- return nil
+ if ok, err := v.container.Exists(); err != nil {
+ return nil, err
+ } else if !ok {
+ return nil, fmt.Errorf("Azure container %q does not exist: %s", v.ContainerName, err)
+ }
+ return &v, nil
}
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &AzureBlobVolume{} })
-
- flag.Var(&azureVolumeAdder{theConfig},
- "azure-storage-container-volume",
- "Use the given container as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &azureStorageAccountName,
- "azure-storage-account-name",
- "",
- "Azure storage account name used for subsequent --azure-storage-container-volume arguments.")
- flag.StringVar(
- &azureStorageAccountKeyFile,
- "azure-storage-account-key-file",
- "",
- "`File` containing the account key used for subsequent --azure-storage-container-volume arguments.")
- flag.IntVar(
- &azureStorageReplication,
- "azure-storage-replication",
- 3,
- "Replication level to report to clients when data is stored in an Azure container.")
- flag.IntVar(
- &azureMaxGetBytes,
- "azure-max-get-bytes",
- BlockSize,
- fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
-}
+const (
+ azureDefaultRequestTimeout = arvados.Duration(10 * time.Minute)
+ azureDefaultListBlobsMaxAttempts = 12
+ azureDefaultListBlobsRetryDelay = arvados.Duration(10 * time.Second)
+ azureDefaultWriteRaceInterval = arvados.Duration(15 * time.Second)
+ azureDefaultWriteRacePollTime = arvados.Duration(time.Second)
+)
// An AzureBlobVolume stores and retrieves blocks in an Azure Blob
// container.
type AzureBlobVolume struct {
- StorageAccountName string
- StorageAccountKeyFile string
- StorageBaseURL string // "" means default, "core.windows.net"
- ContainerName string
- AzureReplication int
- ReadOnly bool
- RequestTimeout arvados.Duration
- StorageClasses []string
- ListBlobsRetryDelay arvados.Duration
- ListBlobsMaxAttempts int
-
+ StorageAccountName string
+ StorageAccountKey string
+ StorageBaseURL string // "" means default, "core.windows.net"
+ ContainerName string
+ RequestTimeout arvados.Duration
+ ListBlobsRetryDelay arvados.Duration
+ ListBlobsMaxAttempts int
+ MaxGetBytes int
+ WriteRaceInterval arvados.Duration
+ WriteRacePollTime arvados.Duration
+
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ logger logrus.FieldLogger
azClient storage.Client
container *azureContainer
}
@@ -127,27 +113,6 @@ func (*singleSender) Send(c *storage.Client, req *http.Request) (resp *http.Resp
return c.HTTPClient.Do(req)
}
-// Examples implements VolumeWithExamples.
-func (*AzureBlobVolume) Examples() []Volume {
- return []Volume{
- &AzureBlobVolume{
- StorageAccountName: "example-account-name",
- StorageAccountKeyFile: "/etc/azure_storage_account_key.txt",
- ContainerName: "example-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- &AzureBlobVolume{
- StorageAccountName: "cn-account-name",
- StorageAccountKeyFile: "/etc/azure_cn_storage_account_key.txt",
- StorageBaseURL: "core.chinacloudapi.cn",
- ContainerName: "cn-container-name",
- AzureReplication: 3,
- RequestTimeout: azureDefaultRequestTimeout,
- },
- }
-}
-
// Type implements Volume.
func (v *AzureBlobVolume) Type() string {
return "Azure"
@@ -155,56 +120,15 @@ func (v *AzureBlobVolume) Type() string {
// Start implements Volume.
func (v *AzureBlobVolume) Start(vm *volumeMetricsVecs) error {
- if v.ListBlobsRetryDelay == 0 {
- v.ListBlobsRetryDelay = azureDefaultListBlobsRetryDelay
- }
- if v.ListBlobsMaxAttempts == 0 {
- v.ListBlobsMaxAttempts = azureDefaultListBlobsMaxAttempts
- }
- if v.ContainerName == "" {
- return errors.New("no container name given")
- }
- if v.StorageAccountName == "" || v.StorageAccountKeyFile == "" {
- return errors.New("StorageAccountName and StorageAccountKeyFile must be given")
- }
- accountKey, err := readKeyFromFile(v.StorageAccountKeyFile)
- if err != nil {
- return err
- }
- if v.StorageBaseURL == "" {
- v.StorageBaseURL = storage.DefaultBaseURL
- }
- v.azClient, err = storage.NewClient(v.StorageAccountName, accountKey, v.StorageBaseURL, storage.DefaultAPIVersion, true)
- if err != nil {
- return fmt.Errorf("creating Azure storage client: %s", err)
- }
- v.azClient.Sender = &singleSender{}
-
- if v.RequestTimeout == 0 {
- v.RequestTimeout = azureDefaultRequestTimeout
- }
- v.azClient.HTTPClient = &http.Client{
- Timeout: time.Duration(v.RequestTimeout),
- }
- bs := v.azClient.GetBlobService()
- v.container = &azureContainer{
- ctr: bs.GetContainerReference(v.ContainerName),
- }
-
- if ok, err := v.container.Exists(); err != nil {
- return err
- } else if !ok {
- return fmt.Errorf("Azure container %q does not exist", v.ContainerName)
- }
// Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
v.container.stats.opsCounters, v.container.stats.errCounters, v.container.stats.ioBytes = vm.getCounterVecsFor(lbls)
return nil
}
-// DeviceID returns a globally unique ID for the storage container.
-func (v *AzureBlobVolume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage container.
+func (v *AzureBlobVolume) GetDeviceID() string {
return "azure://" + v.StorageBaseURL + "/" + v.StorageAccountName + "/" + v.ContainerName
}
@@ -245,14 +169,14 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
if !haveDeadline {
t, err := v.Mtime(loc)
if err != nil {
- log.Print("Got empty block (possible race) but Mtime failed: ", err)
+ ctxlog.FromContext(ctx).Print("Got empty block (possible race) but Mtime failed: ", err)
break
}
- deadline = t.Add(azureWriteRaceInterval)
+ deadline = t.Add(v.WriteRaceInterval.Duration())
if time.Now().After(deadline) {
break
}
- log.Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
+ ctxlog.FromContext(ctx).Printf("Race? Block %s is 0 bytes, %s old. Polling until %s", loc, time.Since(t), deadline)
haveDeadline = true
} else if time.Now().After(deadline) {
break
@@ -260,12 +184,12 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
select {
case <-ctx.Done():
return 0, ctx.Err()
- case <-time.After(azureWriteRacePollTime):
+ case <-time.After(v.WriteRacePollTime.Duration()):
}
size, err = v.get(ctx, loc, buf)
}
if haveDeadline {
- log.Printf("Race ended with size==%d", size)
+ ctxlog.FromContext(ctx).Printf("Race ended with size==%d", size)
}
return size, err
}
@@ -273,8 +197,15 @@ func (v *AzureBlobVolume) Get(ctx context.Context, loc string, buf []byte) (int,
func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
+
+ pieceSize := BlockSize
+ if v.MaxGetBytes > 0 && v.MaxGetBytes < BlockSize {
+ pieceSize = v.MaxGetBytes
+ }
+
+ pieces := 1
expectSize := len(buf)
- if azureMaxGetBytes < BlockSize {
+ if pieceSize < BlockSize {
// Unfortunately the handler doesn't tell us how long the blob
// is expected to be, so we have to ask Azure.
props, err := v.container.GetBlobProperties(loc)
@@ -285,6 +216,7 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
return 0, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
}
expectSize = int(props.ContentLength)
+ pieces = (expectSize + pieceSize - 1) / pieceSize
}
if expectSize == 0 {
@@ -293,7 +225,6 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// We'll update this actualSize if/when we get the last piece.
actualSize := -1
- pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
errors := make(chan error, pieces)
var wg sync.WaitGroup
wg.Add(pieces)
@@ -308,8 +239,8 @@ func (v *AzureBlobVolume) get(ctx context.Context, loc string, buf []byte) (int,
// interrupted as a result.
go func(p int) {
defer wg.Done()
- startPos := p * azureMaxGetBytes
- endPos := startPos + azureMaxGetBytes
+ startPos := p * pieceSize
+ endPos := startPos + pieceSize
if endPos > expectSize {
endPos = expectSize
}
@@ -412,7 +343,7 @@ func (v *AzureBlobVolume) Compare(ctx context.Context, loc string, expect []byte
// Put stores a Keep block as a block blob in the container.
func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
// Send the block data through a pipe, so that (if we need to)
@@ -441,7 +372,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
+ ctxlog.FromContext(ctx).Debugf("%s: taking CreateBlockBlobFromReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// io.Copy() to read. If so, un-stick it. This means
// CreateBlockBlobFromReader will get corrupt data,
@@ -450,7 +381,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
+ ctxlog.FromContext(ctx).Debugf("%s: abandoning CreateBlockBlobFromReader goroutine", v)
return ctx.Err()
case err := <-errChan:
return err
@@ -459,7 +390,7 @@ func (v *AzureBlobVolume) Put(ctx context.Context, loc string, block []byte) err
// Touch updates the last-modified property of a block blob.
func (v *AzureBlobVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
trashed, metadata, err := v.checkTrashed(loc)
@@ -508,7 +439,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
continue
}
modtime := time.Time(b.Properties.LastModified)
- if b.Properties.ContentLength == 0 && modtime.Add(azureWriteRaceInterval).After(time.Now()) {
+ if b.Properties.ContentLength == 0 && modtime.Add(v.WriteRaceInterval.Duration()).After(time.Now()) {
// A new zero-length blob is probably
// just a new non-empty blob that
// hasn't committed its data yet (see
@@ -535,7 +466,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
resp, err = v.container.ListBlobs(params)
err = v.translateError(err)
if err == VolumeBusyError {
- log.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
+ v.logger.Printf("ListBlobs: will retry page %d in %s after error: %s", page, v.ListBlobsRetryDelay, err)
time.Sleep(time.Duration(v.ListBlobsRetryDelay))
continue
} else {
@@ -547,7 +478,7 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
// Trash a Keep block.
func (v *AzureBlobVolume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -562,12 +493,12 @@ func (v *AzureBlobVolume) Trash(loc string) error {
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
// If TrashLifetime == 0, just delete it
- if theConfig.TrashLifetime == 0 {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
return v.container.DeleteBlob(loc, &storage.DeleteBlobOptions{
IfMatch: props.Etag,
})
@@ -575,7 +506,7 @@ func (v *AzureBlobVolume) Trash(loc string) error {
// Otherwise, mark as trash
return v.container.SetBlobMetadata(loc, storage.BlobMetadata{
- "expires_at": fmt.Sprintf("%d", time.Now().Add(theConfig.TrashLifetime.Duration()).Unix()),
+ "expires_at": fmt.Sprintf("%d", time.Now().Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Unix()),
}, &storage.SetBlobMetadataOptions{
IfMatch: props.Etag,
})
@@ -613,23 +544,6 @@ func (v *AzureBlobVolume) String() string {
return fmt.Sprintf("azure-storage-container:%+q", v.ContainerName)
}
-// Writable returns true, unless the -readonly flag was on when the
-// volume was added.
-func (v *AzureBlobVolume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the replication level of the container, as
-// specified by the -azure-storage-replication argument.
-func (v *AzureBlobVolume) Replication() int {
- return v.AzureReplication
-}
-
-// GetStorageClasses implements Volume
-func (v *AzureBlobVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// If possible, translate an Azure SDK error to a recognizable error
// like os.ErrNotExist.
func (v *AzureBlobVolume) translateError(err error) error {
@@ -656,6 +570,10 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
@@ -670,7 +588,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
expiresAt, err := strconv.ParseInt(b.Metadata["expires_at"], 10, 64)
if err != nil {
- log.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
+ v.logger.Printf("EmptyTrash: ParseInt(%v): %v", b.Metadata["expires_at"], err)
return
}
@@ -682,7 +600,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
IfMatch: b.Properties.Etag,
})
if err != nil {
- log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+ v.logger.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
return
}
atomic.AddInt64(&blocksDeleted, 1)
@@ -690,8 +608,8 @@ func (v *AzureBlobVolume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan storage.Blob, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan storage.Blob, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -705,7 +623,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
for page := 1; ; page++ {
resp, err := v.listBlobs(page, params)
if err != nil {
- log.Printf("EmptyTrash: ListBlobs: %v", err)
+ v.logger.Printf("EmptyTrash: ListBlobs: %v", err)
break
}
for _, b := range resp.Blobs {
@@ -719,7 +637,7 @@ func (v *AzureBlobVolume) EmptyTrash() {
close(todo)
wg.Wait()
- log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
+ v.logger.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
}
// InternalStats returns bucket I/O and API call counters.
@@ -748,7 +666,6 @@ func (s *azureBlobStats) TickErr(err error) {
if err, ok := err.(storage.AzureStorageServiceError); ok {
errType = errType + fmt.Sprintf(" %d (%s)", err.StatusCode, err.Code)
}
- log.Printf("errType %T, err %s", err, err)
s.statsTicker.TickErr(err, errType)
}
diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go
index d2e7c9ebd..c05d00412 100644
--- a/services/keepstore/bufferpool.go
+++ b/services/keepstore/bufferpool.go
@@ -8,9 +8,12 @@ import (
"sync"
"sync/atomic"
"time"
+
+ "github.com/sirupsen/logrus"
)
type bufferPool struct {
+ log logrus.FieldLogger
// limiter has a "true" placeholder for each in-use buffer.
limiter chan bool
// allocated is the number of bytes currently allocated to buffers.
@@ -19,7 +22,7 @@ type bufferPool struct {
sync.Pool
}
-func newBufferPool(count int, bufSize int) *bufferPool {
+func newBufferPool(log logrus.FieldLogger, count int, bufSize int) *bufferPool {
p := bufferPool{}
p.New = func() interface{} {
atomic.AddUint64(&p.allocated, uint64(bufSize))
@@ -34,13 +37,13 @@ func (p *bufferPool) Get(size int) []byte {
case p.limiter <- true:
default:
t0 := time.Now()
- log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
+ p.log.Printf("reached max buffers (%d), waiting", cap(p.limiter))
p.limiter <- true
- log.Printf("waited %v for a buffer", time.Since(t0))
+ p.log.Printf("waited %v for a buffer", time.Since(t0))
}
buf := p.Pool.Get().([]byte)
if cap(buf) < size {
- log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
+ p.log.Fatalf("bufferPool Get(size=%d) but max=%d", size, cap(buf))
}
return buf[:size]
}
diff --git a/services/keepstore/command.go b/services/keepstore/command.go
new file mode 100644
index 000000000..7ffb10996
--- /dev/null
+++ b/services/keepstore/command.go
@@ -0,0 +1,191 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "os"
+ "sync"
+
+ "git.curoverse.com/arvados.git/lib/service"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
+)
+
+var (
+ version = "dev"
+ Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler)
+)
+
+func main() {
+ os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
+
+func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background()))
+ if !ok {
+ return 2
+ }
+ return Command.RunCommand(prog, args, stdin, stdout, stderr)
+}
+
+// Parse keepstore command line flags, and return equivalent
+// service.Command flags. The second return value ("ok") is true if
+// all provided flags were successfully converted.
+func convertKeepstoreFlagsToServiceFlags(args []string, lgr logrus.FieldLogger) ([]string, bool) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.String("listen", "", "Services.Keepstore.InternalURLs")
+ flags.Int("max-buffers", 0, "API.MaxKeepBlockBuffers")
+ flags.Int("max-requests", 0, "API.MaxConcurrentRequests")
+ flags.Bool("never-delete", false, "Collections.BlobTrash")
+ flags.Bool("enforce-permissions", false, "Collections.BlobSigning")
+ flags.String("permission-key-file", "", "Collections.BlobSigningKey")
+ flags.String("blob-signing-key-file", "", "Collections.BlobSigningKey")
+ flags.String("data-manager-token-file", "", "SystemRootToken")
+ flags.Int("permission-ttl", 0, "Collections.BlobSigningTTL")
+ flags.Int("blob-signature-ttl", 0, "Collections.BlobSigningTTL")
+ flags.String("trash-lifetime", "", "Collections.BlobTrashLifetime")
+ flags.Bool("serialize", false, "Volumes.*.DriverParameters.Serialize")
+ flags.Bool("readonly", false, "Volumes.*.ReadOnly")
+ flags.String("pid", "", "-")
+ flags.String("trash-check-interval", "", "Collections.BlobTrashCheckInterval")
+
+ flags.String("azure-storage-container-volume", "", "Volumes.*.Driver")
+ flags.String("azure-storage-account-name", "", "Volumes.*.DriverParameters.StorageAccountName")
+ flags.String("azure-storage-account-key-file", "", "Volumes.*.DriverParameters.StorageAccountKey")
+ flags.String("azure-storage-replication", "", "Volumes.*.Replication")
+ flags.String("azure-max-get-bytes", "", "Volumes.*.DriverParameters.MaxDataReadSize")
+
+ flag.String("s3-bucket-volume", "", "Volumes.*.DriverParameters.Bucket")
+ flag.String("s3-region", "", "Volumes.*.DriverParameters.Region")
+ flag.String("s3-endpoint", "", "Volumes.*.DriverParameters.Endpoint")
+ flag.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKey")
+ flag.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretKey")
+ flag.String("s3-race-window", "", "Volumes.*.DriverParameters.RaceWindow")
+ flag.String("s3-replication", "", "Volumes.*.Replication")
+ flag.String("s3-unsafe-delete", "", "Volumes.*.DriverParameters.UnsafeDelete")
+
+ err := flags.Parse(args)
+ if err == flag.ErrHelp {
+ return []string{"-help"}, true
+ } else if err != nil {
+ return nil, false
+ }
+
+ args = nil
+ ok := true
+ flags.Visit(func(f *flag.Flag) {
+ if f.Name == "config" {
+ args = []string{"-legacy-keepstore-config-file", f.Value.String()}
+ } else if f.Usage == "-" {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported", f.Name)
+ } else {
+ ok = false
+ lgr.Errorf("command line flag -%s is no longer supported -- use Clusters.*.%s in cluster config file instead", f.Name, f.Usage)
+ }
+ })
+ return args, ok
+}
+
+type handler struct {
+ http.Handler
+ Cluster *arvados.Cluster
+ Logger logrus.FieldLogger
+
+ volmgr *RRVolumeManager
+ keepClient *keepclient.KeepClient
+
+ err error
+ setupOnce sync.Once
+}
+
+func (h *handler) CheckHealth() error {
+ return h.err
+}
+
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
+ var h handler
+ err := h.setup(ctx, cluster, token, reg)
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, err)
+ }
+ return &h
+}
+
+func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) error {
+ h.Cluster = cluster
+ h.Logger = ctxlog.FromContext(ctx)
+ serviceURL, ok := service.URLFromContext(ctx)
+ if !ok {
+ return errors.New("BUG: no URL from service.URLFromContext")
+ }
+ if h.Cluster.API.MaxKeepBlockBuffers <= 0 {
+ return fmt.Errorf("MaxBuffers must be greater than zero")
+ }
+ bufs = newBufferPool(h.Logger, h.Cluster.API.MaxKeepBlockBuffers, BlockSize)
+
+ if h.Cluster.API.MaxConcurrentRequests < 1 {
+ h.Cluster.API.MaxConcurrentRequests = h.Cluster.API.MaxKeepBlockBuffers * 2
+ h.Logger.Warnf("MaxRequests <1 or not specified; defaulting to MaxKeepBlockBuffers * 2 == %d", h.Cluster.API.MaxConcurrentRequests)
+ }
+
+ if h.Cluster.Collections.BlobSigningKey != "" {
+ } else if h.Cluster.Collections.BlobSigning {
+ return errors.New("cannot enable Collections.BlobSigning with no Collections.BlobSigningKey")
+ } else {
+ h.Logger.Warn("Running without a blob signing key. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions. To fix this, configure Collections.BlobSigning and Collections.BlobSigningKey.")
+ }
+
+ if len(h.Cluster.Volumes) == 0 {
+ return errors.New("no volumes configured")
+ }
+
+ h.Logger.Printf("keepstore %s starting, pid %d", version, os.Getpid())
+ defer h.Logger.Println("keepstore exiting, pid", os.Getpid())
+
+ // Start a round-robin VolumeManager with the volumes we have found.
+ h.volmgr = MakeRRVolumeManager(h.Cluster, serviceURL, newVolumeMetricsVecs(reg))
+ if len(h.volmgr.readables) == 0 {
+ return fmt.Errorf("no volumes configured for %s", serviceURL)
+ }
+
+ // Middleware/handler stack
+ h.Handler = MakeRESTRouter(ctx, cluster, reg)
+
+ // Initialize keepclient for pull workers
+ h.keepClient = &keepclient.KeepClient{
+ Arvados: &arvadosclient.ArvadosClient{ApiToken: fmt.Sprintf("%x", rand.Int63())},
+ Want_replicas: 1,
+ }
+
+ // Initialize the pullq and workers
+ pullq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobReplicateConcurrency; i++ {
+ go h.runPullWorker(pullq)
+ }
+
+ // Initialize the trashq and workers
+ trashq = NewWorkQueue()
+ for i := 0; i < 1 || i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
+ go RunTrashWorker(h.Cluster, trashq)
+ }
+
+ if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 {
+ go emptyTrash(h.volmgr.writables, d)
+ }
+
+ return nil
+}
diff --git a/services/keepstore/command_test.go b/services/keepstore/command_test.go
new file mode 100644
index 000000000..a7a403da7
--- /dev/null
+++ b/services/keepstore/command_test.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+
+ "gopkg.in/check.v1"
+)
+
+var _ = Suite(&CommandSuite{})
+
+type CommandSuite struct{}
+
+func (*CommandSuite) TestLegacyConfigPath(c *check.C) {
+ var stdin, stdout, stderr bytes.Buffer
+ tmp, err := ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(tmp.Name())
+ tmp.Write([]byte("Listen: \"1.2.3.4.5:invalidport\"\n"))
+ tmp.Close()
+ exited := runCommand("keepstore", []string{"-config", tmp.Name()}, &stdin, &stdout, &stderr)
+ c.Check(exited, check.Equals, 1)
+ c.Check(stderr.String(), check.Matches, `(?ms).*unable to migrate Listen value.*`)
+}
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
deleted file mode 100644
index 43a219111..000000000
--- a/services/keepstore/config.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "strings"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/sirupsen/logrus"
-)
-
-type Config struct {
- Debug bool
- Listen string
-
- LogFormat string
-
- PIDFile string
-
- MaxBuffers int
- MaxRequests int
-
- BlobSignatureTTL arvados.Duration
- BlobSigningKeyFile string
- RequireSignatures bool
- SystemAuthTokenFile string
- EnableDelete bool
- TrashLifetime arvados.Duration
- TrashCheckInterval arvados.Duration
- PullWorkers int
- TrashWorkers int
- EmptyTrashWorkers int
- TLSCertificateFile string
- TLSKeyFile string
-
- Volumes VolumeList
-
- blobSigningKey []byte
- systemAuthToken string
- debugLogf func(string, ...interface{})
-
- ManagementToken string
-}
-
-var (
- theConfig = DefaultConfig()
- formatter = map[string]logrus.Formatter{
- "text": &logrus.TextFormatter{
- FullTimestamp: true,
- TimestampFormat: rfc3339NanoFixed,
- },
- "json": &logrus.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- },
- }
- log = logrus.StandardLogger()
-)
-
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
-// DefaultConfig returns the default configuration.
-func DefaultConfig() *Config {
- return &Config{
- Listen: ":25107",
- LogFormat: "json",
- MaxBuffers: 128,
- RequireSignatures: true,
- BlobSignatureTTL: arvados.Duration(14 * 24 * time.Hour),
- TrashLifetime: arvados.Duration(14 * 24 * time.Hour),
- TrashCheckInterval: arvados.Duration(24 * time.Hour),
- Volumes: []Volume{},
- }
-}
-
-// Start should be called exactly once: after setting all public
-// fields, and before using the config.
-func (cfg *Config) Start(reg *prometheus.Registry) error {
- if cfg.Debug {
- log.Level = logrus.DebugLevel
- cfg.debugLogf = log.Printf
- cfg.debugLogf("debugging enabled")
- } else {
- log.Level = logrus.InfoLevel
- cfg.debugLogf = func(string, ...interface{}) {}
- }
-
- f := formatter[strings.ToLower(cfg.LogFormat)]
- if f == nil {
- return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
- }
- log.Formatter = f
-
- if cfg.MaxBuffers < 0 {
- return fmt.Errorf("MaxBuffers must be greater than zero")
- }
- bufs = newBufferPool(cfg.MaxBuffers, BlockSize)
-
- if cfg.MaxRequests < 1 {
- cfg.MaxRequests = cfg.MaxBuffers * 2
- log.Printf("MaxRequests <1 or not specified; defaulting to MaxBuffers * 2 == %d", cfg.MaxRequests)
- }
-
- if cfg.BlobSigningKeyFile != "" {
- buf, err := ioutil.ReadFile(cfg.BlobSigningKeyFile)
- if err != nil {
- return fmt.Errorf("reading blob signing key file: %s", err)
- }
- cfg.blobSigningKey = bytes.TrimSpace(buf)
- if len(cfg.blobSigningKey) == 0 {
- return fmt.Errorf("blob signing key file %q is empty", cfg.BlobSigningKeyFile)
- }
- } else if cfg.RequireSignatures {
- return fmt.Errorf("cannot enable RequireSignatures (-enforce-permissions) without a blob signing key")
- } else {
- log.Println("Running without a blob signing key. Block locators " +
- "returned by this server will not be signed, and will be rejected " +
- "by a server that enforces permissions.")
- log.Println("To fix this, use the BlobSigningKeyFile config entry.")
- }
-
- if fn := cfg.SystemAuthTokenFile; fn != "" {
- buf, err := ioutil.ReadFile(fn)
- if err != nil {
- return fmt.Errorf("cannot read system auth token file %q: %s", fn, err)
- }
- cfg.systemAuthToken = strings.TrimSpace(string(buf))
- }
-
- if cfg.EnableDelete {
- log.Print("Trash/delete features are enabled. WARNING: this has not " +
- "been extensively tested. You should disable this unless you can afford to lose data.")
- }
-
- if len(cfg.Volumes) == 0 {
- if (&unixVolumeAdder{cfg}).Discover() == 0 {
- return fmt.Errorf("no volumes found")
- }
- }
- vm := newVolumeMetricsVecs(reg)
- for _, v := range cfg.Volumes {
- if err := v.Start(vm); err != nil {
- return fmt.Errorf("volume %s: %s", v, err)
- }
- log.Printf("Using volume %v (writable=%v)", v, v.Writable())
- }
- return nil
-}
-
-// VolumeTypes is built up by init() funcs in the source files that
-// define the volume types.
-var VolumeTypes = []func() VolumeWithExamples{}
-
-type VolumeList []Volume
-
-// UnmarshalJSON -- given an array of objects -- deserializes each
-// object as the volume type indicated by the object's Type field.
-func (vl *VolumeList) UnmarshalJSON(data []byte) error {
- typeMap := map[string]func() VolumeWithExamples{}
- for _, factory := range VolumeTypes {
- t := factory().Type()
- if _, ok := typeMap[t]; ok {
- log.Fatalf("volume type %+q is claimed by multiple VolumeTypes", t)
- }
- typeMap[t] = factory
- }
-
- var mapList []map[string]interface{}
- err := json.Unmarshal(data, &mapList)
- if err != nil {
- return err
- }
- for _, mapIn := range mapList {
- typeIn, ok := mapIn["Type"].(string)
- if !ok {
- return fmt.Errorf("invalid volume type %+v", mapIn["Type"])
- }
- factory, ok := typeMap[typeIn]
- if !ok {
- return fmt.Errorf("unsupported volume type %+q", typeIn)
- }
- data, err := json.Marshal(mapIn)
- if err != nil {
- return err
- }
- vol := factory()
- err = json.Unmarshal(data, vol)
- if err != nil {
- return err
- }
- *vl = append(*vl, vol)
- }
- return nil
-}
-
-// MarshalJSON adds a "Type" field to each volume corresponding to its
-// Type().
-func (vl *VolumeList) MarshalJSON() ([]byte, error) {
- data := []byte{'['}
- for _, vs := range *vl {
- j, err := json.Marshal(vs)
- if err != nil {
- return nil, err
- }
- if len(data) > 1 {
- data = append(data, byte(','))
- }
- t, err := json.Marshal(vs.Type())
- if err != nil {
- panic(err)
- }
- data = append(data, j[0])
- data = append(data, []byte(`"Type":`)...)
- data = append(data, t...)
- data = append(data, byte(','))
- data = append(data, j[1:]...)
- }
- return append(data, byte(']')), nil
-}
diff --git a/services/keepstore/deprecated.go b/services/keepstore/deprecated.go
index d1377978a..1fd8a0668 100644
--- a/services/keepstore/deprecated.go
+++ b/services/keepstore/deprecated.go
@@ -3,45 +3,3 @@
// SPDX-License-Identifier: AGPL-3.0
package main
-
-import (
- "flag"
- "time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
-)
-
-type deprecatedOptions struct {
- flagSerializeIO bool
- flagReadonly bool
- neverDelete bool
- signatureTTLSeconds int
-}
-
-var deprecated = deprecatedOptions{
- neverDelete: !theConfig.EnableDelete,
- signatureTTLSeconds: int(theConfig.BlobSignatureTTL.Duration() / time.Second),
-}
-
-func (depr *deprecatedOptions) beforeFlagParse(cfg *Config) {
- flag.StringVar(&cfg.Listen, "listen", cfg.Listen, "see Listen configuration")
- flag.IntVar(&cfg.MaxBuffers, "max-buffers", cfg.MaxBuffers, "see MaxBuffers configuration")
- flag.IntVar(&cfg.MaxRequests, "max-requests", cfg.MaxRequests, "see MaxRequests configuration")
- flag.BoolVar(&depr.neverDelete, "never-delete", depr.neverDelete, "see EnableDelete configuration")
- flag.BoolVar(&cfg.RequireSignatures, "enforce-permissions", cfg.RequireSignatures, "see RequireSignatures configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "permission-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.BlobSigningKeyFile, "blob-signing-key-file", cfg.BlobSigningKeyFile, "see BlobSigningKey`File` configuration")
- flag.StringVar(&cfg.SystemAuthTokenFile, "data-manager-token-file", cfg.SystemAuthTokenFile, "see SystemAuthToken`File` configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "permission-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.IntVar(&depr.signatureTTLSeconds, "blob-signature-ttl", depr.signatureTTLSeconds, "signature TTL in seconds; see BlobSignatureTTL configuration")
- flag.Var(&cfg.TrashLifetime, "trash-lifetime", "see TrashLifetime configuration")
- flag.BoolVar(&depr.flagSerializeIO, "serialize", depr.flagSerializeIO, "serialize read and write operations on the following volumes.")
- flag.BoolVar(&depr.flagReadonly, "readonly", depr.flagReadonly, "do not write, delete, or touch anything on the following volumes.")
- flag.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "see `PIDFile` configuration")
- flag.Var(&cfg.TrashCheckInterval, "trash-check-interval", "see TrashCheckInterval configuration")
-}
-
-func (depr *deprecatedOptions) afterFlagParse(cfg *Config) {
- cfg.BlobSignatureTTL = arvados.Duration(depr.signatureTTLSeconds) * arvados.Duration(time.Second)
- cfg.EnableDelete = !depr.neverDelete
-}
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 72088e2b5..926fe3044 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,22 +27,25 @@ import (
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
type router struct {
*mux.Router
limiter httpserver.RequestCounter
cluster *arvados.Cluster
+ logger logrus.FieldLogger
remoteProxy remoteProxy
metrics *nodeMetrics
}
// MakeRESTRouter returns a new router that forwards all Keep requests
// to the appropriate handlers.
-func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
+func MakeRESTRouter(ctx context.Context, cluster *arvados.Cluster, reg *prometheus.Registry) http.Handler {
rtr := &router{
Router: mux.NewRouter(),
cluster: cluster,
+ logger: ctxlog.FromContext(ctx),
metrics: &nodeMetrics{reg: reg},
}
@@ -52,12 +56,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
rtr.handleGET).Methods("GET", "HEAD")
rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handlePUT).Methods("PUT")
- rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, DeleteHandler).Methods("DELETE")
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleDELETE).Methods("DELETE")
// List all blocks stored here. Privileged client only.
- rtr.HandleFunc(`/index`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index`, rtr.handleIndex).Methods("GET", "HEAD")
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
- rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.IndexHandler).Methods("GET", "HEAD")
+ rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
// Internals/debugging info (runtime.MemStats)
rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
@@ -67,20 +71,20 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// List mounts: UUID, readonly, tier, device ID, ...
rtr.HandleFunc(`/mounts`, rtr.MountsHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.IndexHandler).Methods("GET")
- rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.IndexHandler).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks`, rtr.handleIndex).Methods("GET")
+ rtr.HandleFunc(`/mounts/{uuid}/blocks/`, rtr.handleIndex).Methods("GET")
// Replace the current pull queue.
- rtr.HandleFunc(`/pull`, PullHandler).Methods("PUT")
+ rtr.HandleFunc(`/pull`, rtr.handlePull).Methods("PUT")
// Replace the current trash queue.
- rtr.HandleFunc(`/trash`, TrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/trash`, rtr.handleTrash).Methods("PUT")
// Untrash moves blocks from trash back into store
- rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, UntrashHandler).Methods("PUT")
+ rtr.HandleFunc(`/untrash/{hash:[0-9a-f]{32}}`, rtr.handleUntrash).Methods("PUT")
rtr.Handle("/_health/{check}", &health.Handler{
- Token: theConfig.ManagementToken,
+ Token: cluster.ManagementToken,
Prefix: "/_health/",
}).Methods("GET")
@@ -88,17 +92,12 @@ func MakeRESTRouter(cluster *arvados.Cluster, reg *prometheus.Registry) http.Han
// 400 Bad Request.
rtr.NotFoundHandler = http.HandlerFunc(BadRequestHandler)
- rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
rtr.metrics.setupBufferPoolMetrics(bufs)
rtr.metrics.setupWorkQueueMetrics(pullq, "pull")
rtr.metrics.setupWorkQueueMetrics(trashq, "trash")
rtr.metrics.setupRequestMetrics(rtr.limiter)
- instrumented := httpserver.Instrument(rtr.metrics.reg, log,
- httpserver.HandlerWithContext(
- ctxlog.Context(context.Background(), log),
- httpserver.AddRequestIDs(httpserver.LogRequests(rtr.limiter))))
- return instrumented.ServeAPI(theConfig.ManagementToken, instrumented)
+ return rtr
}
// BadRequestHandler is a HandleFunc to address bad requests.
@@ -116,9 +115,9 @@ func (rtr *router) handleGET(resp http.ResponseWriter, req *http.Request) {
return
}
- if theConfig.RequireSignatures {
+ if rtr.cluster.Collections.BlobSigning {
locator := req.URL.Path[1:] // strip leading slash
- if err := VerifySignature(locator, GetAPIToken(req)); err != nil {
+ if err := VerifySignature(rtr.cluster, locator, GetAPIToken(req)); err != nil {
http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
return
}
@@ -160,7 +159,6 @@ func contextForResponse(parent context.Context, resp http.ResponseWriter) (conte
go func(c <-chan bool) {
select {
case <-c:
- theConfig.debugLogf("cancel context")
cancel()
case <-ctx.Done():
}
@@ -244,9 +242,9 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// return it to the client.
returnHash := fmt.Sprintf("%s+%d", hash, req.ContentLength)
apiToken := GetAPIToken(req)
- if theConfig.blobSigningKey != nil && apiToken != "" {
- expiry := time.Now().Add(theConfig.BlobSignatureTTL.Duration())
- returnHash = SignLocator(returnHash, apiToken, expiry)
+ if rtr.cluster.Collections.BlobSigningKey != "" && apiToken != "" {
+ expiry := time.Now().Add(rtr.cluster.Collections.BlobSigningTTL.Duration())
+ returnHash = SignLocator(rtr.cluster, returnHash, apiToken, expiry)
}
resp.Header().Set("X-Keep-Replicas-Stored", strconv.Itoa(replication))
resp.Write([]byte(returnHash + "\n"))
@@ -254,8 +252,8 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
// IndexHandler responds to "/index", "/index/{prefix}", and
// "/mounts/{uuid}/blocks" requests.
-func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
- if !IsSystemAuth(GetAPIToken(req)) {
+func (rtr *router) handleIndex(resp http.ResponseWriter, req *http.Request) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -268,14 +266,14 @@ func (rtr *router) IndexHandler(resp http.ResponseWriter, req *http.Request) {
uuid := mux.Vars(req)["uuid"]
- var vols []Volume
+ var vols []*VolumeMount
if uuid == "" {
vols = KeepVM.AllReadable()
} else if v := KeepVM.Lookup(uuid, false); v == nil {
http.Error(resp, "mount not found", http.StatusNotFound)
return
} else {
- vols = []Volume{v}
+ vols = []*VolumeMount{v}
}
for _, v := range vols {
@@ -375,7 +373,7 @@ func (rtr *router) readNodeStatus(st *NodeStatus) {
st.Volumes = st.Volumes[:0]
for _, vol := range vols {
var internalStats interface{}
- if vol, ok := vol.(InternalStatser); ok {
+ if vol, ok := vol.Volume.(InternalStatser); ok {
internalStats = vol.InternalStats()
}
st.Volumes = append(st.Volumes, &volumeStatusEnt{
@@ -407,7 +405,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
return q.Status()
}
-// DeleteHandler processes DELETE requests.
+// handleDELETE processes DELETE requests.
//
// DELETE /{hash:[0-9a-f]{32} will delete the block with the specified hash
// from all connected volumes.
@@ -418,7 +416,7 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// a PermissionError.
//
// Upon receiving a valid request from an authorized user,
-// DeleteHandler deletes all copies of the specified block on local
+// handleDELETE deletes all copies of the specified block on local
// writable volumes.
//
// Response format:
@@ -434,17 +432,17 @@ func getWorkQueueStatus(q *WorkQueue) WorkQueueStatus {
// where d and f are integers representing the number of blocks that
// were successfully and unsuccessfully deleted.
//
-func DeleteHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleDELETE(resp http.ResponseWriter, req *http.Request) {
hash := mux.Vars(req)["hash"]
// Confirm that this user is an admin and has a token with unlimited scope.
var tok = GetAPIToken(req)
- if tok == "" || !CanDelete(tok) {
+ if tok == "" || !rtr.canDelete(tok) {
http.Error(resp, PermissionError.Error(), PermissionError.HTTPCode)
return
}
- if !theConfig.EnableDelete {
+ if !rtr.cluster.Collections.BlobTrash {
http.Error(resp, MethodDisabledError.Error(), MethodDisabledError.HTTPCode)
return
}
@@ -530,9 +528,9 @@ type PullRequest struct {
}
// PullHandler processes "PUT /pull" requests for the data manager.
-func PullHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handlePull(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -569,9 +567,9 @@ type TrashRequest struct {
}
// TrashHandler processes /trash requests.
-func TrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleTrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -599,9 +597,9 @@ func TrashHandler(resp http.ResponseWriter, req *http.Request) {
}
// UntrashHandler processes "PUT /untrash/{hash:[0-9a-f]{32}}" requests for the data manager.
-func UntrashHandler(resp http.ResponseWriter, req *http.Request) {
+func (rtr *router) handleUntrash(resp http.ResponseWriter, req *http.Request) {
// Reject unauthorized requests.
- if !IsSystemAuth(GetAPIToken(req)) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
return
}
@@ -759,7 +757,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
// If this volume fails, try all of the volumes in order.
if vol := KeepVM.NextWritable(); vol != nil {
if err := vol.Put(ctx, hash, block); err == nil {
- return vol.Replication(), nil // success!
+ return vol.Replication, nil // success!
}
if ctx.Err() != nil {
return 0, ErrClientDisconnect
@@ -779,7 +777,7 @@ func PutBlock(ctx context.Context, block []byte, hash string) (int, error) {
return 0, ErrClientDisconnect
}
if err == nil {
- return vol.Replication(), nil // success!
+ return vol.Replication, nil // success!
}
if err != FullError {
// The volume is not full but the
@@ -834,7 +832,7 @@ func CompareAndTouch(ctx context.Context, hash string, buf []byte) (int, error)
continue
}
// Compare and Touch both worked --> done.
- return vol.Replication(), nil
+ return vol.Replication, nil
}
return 0, bestErr
}
@@ -875,15 +873,15 @@ func IsExpired(timestampHex string) bool {
return time.Unix(ts, 0).Before(time.Now())
}
-// CanDelete returns true if the user identified by apiToken is
+// canDelete returns true if the user identified by apiToken is
// allowed to delete blocks.
-func CanDelete(apiToken string) bool {
+func (rtr *router) canDelete(apiToken string) bool {
if apiToken == "" {
return false
}
// Blocks may be deleted only when Keep has been configured with a
// data manager.
- if IsSystemAuth(apiToken) {
+ if rtr.isSystemAuth(apiToken) {
return true
}
// TODO(twp): look up apiToken with the API server
@@ -892,8 +890,8 @@ func CanDelete(apiToken string) bool {
return false
}
-// IsSystemAuth returns true if the given token is allowed to perform
+// isSystemAuth returns true if the given token is allowed to perform
// system level actions like deleting data.
-func IsSystemAuth(token string) bool {
- return token != "" && token == theConfig.systemAuthToken
+func (rtr *router) isSystemAuth(token string) bool {
+ return token != "" && token == rtr.cluster.SystemRootToken
}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index fcbdddacb..e1f830bd6 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -5,24 +5,9 @@
package main
import (
- "flag"
- "fmt"
- "net"
- "os"
- "os/signal"
- "syscall"
"time"
-
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/config"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "github.com/coreos/go-systemd/daemon"
- "github.com/prometheus/client_golang/prometheus"
)
-var version = "dev"
-
// A Keep "block" is 64MB.
const BlockSize = 64 * 1024 * 1024
@@ -30,9 +15,6 @@ const BlockSize = 64 * 1024 * 1024
// in order to permit writes.
const MinFreeKilobytes = BlockSize / 1024
-// ProcMounts /proc/mounts
-var ProcMounts = "/proc/mounts"
-
var bufs *bufferPool
// KeepError types.
@@ -87,162 +69,11 @@ var KeepVM VolumeManager
var pullq *WorkQueue
var trashq *WorkQueue
-func main() {
- deprecated.beforeFlagParse(theConfig)
-
- dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flag.Bool("version", false, "Print version information and exit.")
-
- defaultConfigPath := "/etc/arvados/keepstore/keepstore.yml"
- var configPath string
- flag.StringVar(
- &configPath,
- "config",
- defaultConfigPath,
- "YAML or JSON configuration file `path`")
- flag.Usage = usage
- flag.Parse()
-
- // Print version information if requested
- if *getVersion {
- fmt.Printf("keepstore %s\n", version)
- return
- }
-
- deprecated.afterFlagParse(theConfig)
-
- err := config.LoadFile(theConfig, configPath)
- if err != nil && (!os.IsNotExist(err) || configPath != defaultConfigPath) {
- log.Fatal(err)
- }
-
- if *dumpConfig {
- log.Fatal(config.DumpAndExit(theConfig))
- }
-
- log.Printf("keepstore %s started", version)
-
- metricsRegistry := prometheus.NewRegistry()
-
- err = theConfig.Start(metricsRegistry)
- if err != nil {
- log.Fatal(err)
- }
-
- if pidfile := theConfig.PIDFile; pidfile != "" {
- f, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0777)
- if err != nil {
- log.Fatalf("open pidfile (%s): %s", pidfile, err)
- }
- defer f.Close()
- err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
- if err != nil {
- log.Fatalf("flock pidfile (%s): %s", pidfile, err)
- }
- defer os.Remove(pidfile)
- err = f.Truncate(0)
- if err != nil {
- log.Fatalf("truncate pidfile (%s): %s", pidfile, err)
- }
- _, err = fmt.Fprint(f, os.Getpid())
- if err != nil {
- log.Fatalf("write pidfile (%s): %s", pidfile, err)
- }
- err = f.Sync()
- if err != nil {
- log.Fatalf("sync pidfile (%s): %s", pidfile, err)
- }
- }
-
- var cluster *arvados.Cluster
- cfg, err := arvados.GetConfig(arvados.DefaultConfigFile)
- if err != nil && os.IsNotExist(err) {
- log.Warnf("DEPRECATED: proceeding without cluster configuration file %q (%s)", arvados.DefaultConfigFile, err)
- cluster = &arvados.Cluster{
- ClusterID: "xxxxx",
- }
- } else if err != nil {
- log.Fatalf("load config %q: %s", arvados.DefaultConfigFile, err)
- } else {
- cluster, err = cfg.GetCluster("")
- if err != nil {
- log.Fatalf("config error in %q: %s", arvados.DefaultConfigFile, err)
- }
- }
-
- log.Println("keepstore starting, pid", os.Getpid())
- defer log.Println("keepstore exiting, pid", os.Getpid())
-
- // Start a round-robin VolumeManager with the volumes we have found.
- KeepVM = MakeRRVolumeManager(theConfig.Volumes)
-
- // Middleware/handler stack
- router := MakeRESTRouter(cluster, metricsRegistry)
-
- // Set up a TCP listener.
- listener, err := net.Listen("tcp", theConfig.Listen)
- if err != nil {
- log.Fatal(err)
- }
-
- // Initialize keepclient for pull workers
- keepClient := &keepclient.KeepClient{
- Arvados: &arvadosclient.ArvadosClient{},
- Want_replicas: 1,
- }
-
- // Initialize the pullq and workers
- pullq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.PullWorkers; i++ {
- go RunPullWorker(pullq, keepClient)
- }
-
- // Initialize the trashq and workers
- trashq = NewWorkQueue()
- for i := 0; i < 1 || i < theConfig.TrashWorkers; i++ {
- go RunTrashWorker(trashq)
- }
-
- // Start emptyTrash goroutine
- doneEmptyingTrash := make(chan bool)
- go emptyTrash(doneEmptyingTrash, theConfig.TrashCheckInterval.Duration())
-
- // Shut down the server gracefully (by closing the listener)
- // if SIGTERM is received.
- term := make(chan os.Signal, 1)
- go func(sig <-chan os.Signal) {
- s := <-sig
- log.Println("caught signal:", s)
- doneEmptyingTrash <- true
- listener.Close()
- }(term)
- signal.Notify(term, syscall.SIGTERM)
- signal.Notify(term, syscall.SIGINT)
-
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
- }
- log.Println("listening at", listener.Addr())
- srv := &server{}
- srv.Handler = router
- srv.Serve(listener)
-}
-
// Periodically (once per interval) invoke EmptyTrash on all volumes.
-func emptyTrash(done <-chan bool, interval time.Duration) {
- ticker := time.NewTicker(interval)
-
- for {
- select {
- case <-ticker.C:
- for _, v := range theConfig.Volumes {
- if v.Writable() {
- v.EmptyTrash()
- }
- }
- case <-done:
- ticker.Stop()
- return
+func emptyTrash(volumes []Volume, interval time.Duration) {
+ for range time.NewTicker(interval).C {
+ for _, v := range volumes {
+ v.EmptyTrash()
}
}
}
diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 8b448e72c..728c6fded 100644
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -6,7 +6,6 @@
Description=Arvados Keep Storage Daemon
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/keepstore/keepstore.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
diff --git a/services/keepstore/perms.go b/services/keepstore/perms.go
index 49a231685..e2155f94f 100644
--- a/services/keepstore/perms.go
+++ b/services/keepstore/perms.go
@@ -5,14 +5,16 @@
package main
import (
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
// SignLocator takes a blobLocator, an apiToken and an expiry time, and
// returns a signed locator string.
-func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
- return keepclient.SignLocator(blobLocator, apiToken, expiry, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func SignLocator(cluster *arvados.Cluster, blobLocator, apiToken string, expiry time.Time) string {
+ return keepclient.SignLocator(blobLocator, apiToken, expiry, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
}
// VerifySignature returns nil if the signature on the signedLocator
@@ -20,8 +22,8 @@ func SignLocator(blobLocator, apiToken string, expiry time.Time) string {
// either ExpiredError (if the timestamp has expired, which is
// something the client could have figured out independently) or
// PermissionError.
-func VerifySignature(signedLocator, apiToken string) error {
- err := keepclient.VerifySignature(signedLocator, apiToken, theConfig.BlobSignatureTTL.Duration(), theConfig.blobSigningKey)
+func VerifySignature(cluster *arvados.Cluster, signedLocator, apiToken string) error {
+ err := keepclient.VerifySignature(signedLocator, apiToken, cluster.Collections.BlobSigningTTL.Duration(), []byte(cluster.Collections.BlobSigningKey))
if err == keepclient.ErrSignatureExpired {
return ExpiredError
} else if err != nil {
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
index 1f82f3f4f..589bedc96 100644
--- a/services/keepstore/proxy_remote.go
+++ b/services/keepstore/proxy_remote.go
@@ -148,6 +148,7 @@ type remoteResponseCacher struct {
Locator string
Token string
Buffer []byte
+ Cluster *arvados.Cluster
Context context.Context
http.ResponseWriter
statusCode int
@@ -193,7 +194,8 @@ func (rrc *remoteResponseCacher) Close() error {
}
unsigned := localOrRemoteSignature.ReplaceAllLiteralString(rrc.Locator, "")
- signed := SignLocator(unsigned, rrc.Token, time.Now().Add(theConfig.BlobSignatureTTL.Duration()))
+ expiry := time.Now().Add(rrc.Cluster.Collections.BlobSigningTTL.Duration())
+ signed := SignLocator(rrc.Cluster, unsigned, rrc.Token, expiry)
if signed == unsigned {
err = errors.New("could not sign locator")
http.Error(rrc.ResponseWriter, err.Error(), http.StatusInternalServerError)
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 42b5d5889..5e6315e14 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -6,27 +6,27 @@ package main
import (
"context"
- "crypto/rand"
"fmt"
"io"
"io/ioutil"
"time"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/sirupsen/logrus"
)
// RunPullWorker receives PullRequests from pullq, invokes
// PullItemAndProcess on each one. After each PR, it logs a message
// indicating whether the pull was successful.
-func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
+func (h *handler) runPullWorker(pullq *WorkQueue) {
for item := range pullq.NextItem {
pr := item.(PullRequest)
- err := PullItemAndProcess(pr, keepClient)
+ err := h.pullItemAndProcess(pr)
pullq.DoneItem <- struct{}{}
if err == nil {
- log.Printf("Pull %s success", pr)
+ h.Logger.Printf("Pull %s success", pr)
} else {
- log.Printf("Pull %s error: %s", pr, err)
+ h.Logger.Printf("Pull %s error: %s", pr, err)
}
}
}
@@ -39,8 +39,8 @@ func RunPullWorker(pullq *WorkQueue, keepClient *keepclient.KeepClient) {
// only attempt to write the data to the corresponding
// volume. Otherwise it writes to any local volume, as a PUT request
// would.
-func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClient) error {
- var vol Volume
+func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
+ var vol *VolumeMount
if uuid := pullRequest.MountUUID; uuid != "" {
vol = KeepVM.Lookup(pullRequest.MountUUID, true)
if vol == nil {
@@ -48,19 +48,19 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
}
}
- keepClient.Arvados.ApiToken = randomToken
-
+ // Make a private copy of keepClient so we can set
+ // ServiceRoots to the source servers specified in the pull
+ // request.
+ keepClient := *h.keepClient
serviceRoots := make(map[string]string)
for _, addr := range pullRequest.Servers {
serviceRoots[addr] = addr
}
keepClient.SetServiceRoots(serviceRoots, nil, nil)
- // Generate signature with a random token
- expiresAt := time.Now().Add(60 * time.Second)
- signedLocator := SignLocator(pullRequest.Locator, randomToken, expiresAt)
+ signedLocator := SignLocator(h.Cluster, pullRequest.Locator, keepClient.Arvados.ApiToken, time.Now().Add(time.Minute))
- reader, contentLen, _, err := GetContent(signedLocator, keepClient)
+ reader, contentLen, _, err := GetContent(signedLocator, &keepClient)
if err != nil {
return err
}
@@ -78,8 +78,7 @@ func PullItemAndProcess(pullRequest PullRequest, keepClient *keepclient.KeepClie
return fmt.Errorf("Content not found for: %s", signedLocator)
}
- writePulledBlock(vol, readContent, pullRequest.Locator)
- return nil
+ return writePulledBlock(vol, readContent, pullRequest.Locator, h.Logger)
}
// Fetch the content for the given locator using keepclient.
@@ -87,24 +86,11 @@ var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (
return keepClient.Get(signedLocator)
}
-var writePulledBlock = func(volume Volume, data []byte, locator string) {
- var err error
+var writePulledBlock = func(volume Volume, data []byte, locator string, log logrus.FieldLogger) error {
if volume != nil {
- err = volume.Put(context.Background(), locator, data)
+ return volume.Put(context.Background(), locator, data)
} else {
- _, err = PutBlock(context.Background(), data, locator)
- }
- if err != nil {
- log.Printf("error writing pulled block %q: %s", locator, err)
+ _, err := PutBlock(context.Background(), data, locator)
+ return err
}
}
-
-var randomToken = func() string {
- const alphaNumeric = "0123456789abcdefghijklmnopqrstuvwxyz"
- var bytes = make([]byte, 36)
- rand.Read(bytes)
- for i, b := range bytes {
- bytes[i] = alphaNumeric[b%byte(len(alphaNumeric))]
- }
- return (string(bytes))
-}()
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 4c39dcd5c..c7a86d9e4 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -10,10 +10,12 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"net/http"
"os"
"regexp"
@@ -26,8 +28,31 @@ import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
+func init() {
+ driver["S3"] = newS3Volume
+}
+
+func newS3Volume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+ v := S3Volume{cluster: cluster, volume: volume, logger: logger}
+ err := json.Unmarshal(volume.DriverParameters, &v)
+ if err != nil {
+ return nil, err
+ }
+ if v.Bucket == "" || v.AccessKey == "" || v.SecretKey == "" {
+ return nil, errors.New("DriverParameters: Bucket, AccessKey, and SecretKey must be provided")
+ }
+ if v.IndexPageSize == 0 {
+ v.IndexPageSize = 1000
+ }
+ if v.RaceWindow < 0 {
+ return nil, errors.New("DriverParameters: RaceWindow must not be negative")
+ }
+ return &v, nil
+}
+
const (
s3DefaultReadTimeout = arvados.Duration(10 * time.Minute)
s3DefaultConnectTimeout = arvados.Duration(time.Minute)
@@ -38,14 +63,6 @@ var (
// is impossible with the current config.
ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false")
- s3AccessKeyFile string
- s3SecretKeyFile string
- s3RegionName string
- s3Endpoint string
- s3Replication int
- s3UnsafeDelete bool
- s3RaceWindow time.Duration
-
s3ACL = s3.Private
zeroTime time.Time
@@ -56,40 +73,6 @@ const (
nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT"
)
-type s3VolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (s *s3VolumeAdder) String() string {
- return "-"
-}
-
-func (s *s3VolumeAdder) Set(bucketName string) error {
- if bucketName == "" {
- return fmt.Errorf("no container name given")
- }
- if s3AccessKeyFile == "" || s3SecretKeyFile == "" {
- return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume")
- }
- if deprecated.flagSerializeIO {
- log.Print("Notice: -serialize is not supported by s3-bucket volumes.")
- }
- s.Config.Volumes = append(s.Config.Volumes, &S3Volume{
- Bucket: bucketName,
- AccessKeyFile: s3AccessKeyFile,
- SecretKeyFile: s3SecretKeyFile,
- Endpoint: s3Endpoint,
- Region: s3RegionName,
- RaceWindow: arvados.Duration(s3RaceWindow),
- S3Replication: s3Replication,
- UnsafeDelete: s3UnsafeDelete,
- ReadOnly: deprecated.flagReadonly,
- IndexPageSize: 1000,
- })
- return nil
-}
-
func s3regions() (okList []string) {
for r := range aws.Regions {
okList = append(okList, r)
@@ -97,117 +80,38 @@ func s3regions() (okList []string) {
return
}
-func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &S3Volume{} })
-
- flag.Var(&s3VolumeAdder{theConfig},
- "s3-bucket-volume",
- "Use the given bucket as a storage volume. Can be given multiple times.")
- flag.StringVar(
- &s3RegionName,
- "s3-region",
- "",
- fmt.Sprintf("AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are %+q.", s3regions()))
- flag.StringVar(
- &s3Endpoint,
- "s3-endpoint",
- "",
- "Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use \"https://storage.googleapis.com\".")
- flag.StringVar(
- &s3AccessKeyFile,
- "s3-access-key-file",
- "",
- "`File` containing the access key used for subsequent -s3-bucket-volume arguments.")
- flag.StringVar(
- &s3SecretKeyFile,
- "s3-secret-key-file",
- "",
- "`File` containing the secret key used for subsequent -s3-bucket-volume arguments.")
- flag.DurationVar(
- &s3RaceWindow,
- "s3-race-window",
- 24*time.Hour,
- "Maximum eventual consistency latency for subsequent -s3-bucket-volume arguments.")
- flag.IntVar(
- &s3Replication,
- "s3-replication",
- 2,
- "Replication level reported to clients for subsequent -s3-bucket-volume arguments.")
- flag.BoolVar(
- &s3UnsafeDelete,
- "s3-unsafe-delete",
- false,
- "EXPERIMENTAL. Enable deletion (garbage collection) even when trash lifetime is zero, even though there are known race conditions that can cause data loss.")
-}
-
// S3Volume implements Volume using an S3 bucket.
type S3Volume struct {
- AccessKeyFile string
- SecretKeyFile string
+ AccessKey string
+ SecretKey string
Endpoint string
Region string
Bucket string
LocationConstraint bool
IndexPageSize int
- S3Replication int
ConnectTimeout arvados.Duration
ReadTimeout arvados.Duration
RaceWindow arvados.Duration
- ReadOnly bool
UnsafeDelete bool
- StorageClasses []string
-
- bucket *s3bucket
+ logger logrus.FieldLogger
+ cluster *arvados.Cluster
+ volume arvados.Volume
+ bucket *s3bucket
startOnce sync.Once
}
-// Examples implements VolumeWithExamples.
-func (*S3Volume) Examples() []Volume {
- return []Volume{
- &S3Volume{
- AccessKeyFile: "/etc/aws_s3_access_key.txt",
- SecretKeyFile: "/etc/aws_s3_secret_key.txt",
- Endpoint: "",
- Region: "us-east-1",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
- &S3Volume{
- AccessKeyFile: "/etc/gce_s3_access_key.txt",
- SecretKeyFile: "/etc/gce_s3_secret_key.txt",
- Endpoint: "https://storage.googleapis.com",
- Region: "",
- Bucket: "example-bucket-name",
- IndexPageSize: 1000,
- S3Replication: 2,
- RaceWindow: arvados.Duration(24 * time.Hour),
- ConnectTimeout: arvados.Duration(time.Minute),
- ReadTimeout: arvados.Duration(5 * time.Minute),
- },
- }
-}
-
-// Type implements Volume.
-func (*S3Volume) Type() string {
- return "S3"
-}
-
// Start populates private fields and verifies the configuration is
// valid.
func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
region, ok := aws.Regions[v.Region]
if v.Endpoint == "" {
if !ok {
- return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region)
+ return fmt.Errorf("unrecognized region %+q; try specifying endpoint instead", v.Region)
}
} else if ok {
return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+
- "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint)
+ "specify empty endpoint or use a different region name", v.Region, v.Endpoint)
} else {
region = aws.Region{
Name: v.Region,
@@ -216,15 +120,9 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
}
}
- var err error
- var auth aws.Auth
- auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile)
- if err != nil {
- return err
- }
- auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile)
- if err != nil {
- return err
+ auth := aws.Auth{
+ AccessKey: v.AccessKey,
+ SecretKey: v.SecretKey,
}
// Zero timeouts mean "wait forever", which is a bad
@@ -250,14 +148,14 @@ func (v *S3Volume) Start(vm *volumeMetricsVecs) error {
},
}
// Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls)
return nil
}
-// DeviceID returns a globally unique ID for the storage bucket.
-func (v *S3Volume) DeviceID() string {
+// GetDeviceID returns a globally unique ID for the storage bucket.
+func (v *S3Volume) GetDeviceID() string {
return "s3://" + v.Endpoint + "/" + v.Bucket
}
@@ -271,7 +169,7 @@ func (v *S3Volume) getReaderWithContext(ctx context.Context, loc string) (rdr io
case <-ready:
return
case <-ctx.Done():
- theConfig.debugLogf("s3: abandoning getReader(): %s", ctx.Err())
+ v.logger.Debugf("s3: abandoning getReader(): %s", ctx.Err())
go func() {
<-ready
if err == nil {
@@ -339,11 +237,11 @@ func (v *S3Volume) Get(ctx context.Context, loc string, buf []byte) (int, error)
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
+ v.logger.Debugf("s3: interrupting ReadFull() with Close() because %s", ctx.Err())
rdr.Close()
// Must wait for ReadFull to return, to ensure it
// doesn't write to buf after we return.
- theConfig.debugLogf("s3: waiting for ReadFull() to fail")
+ v.logger.Debug("s3: waiting for ReadFull() to fail")
<-ready
return 0, ctx.Err()
case <-ready:
@@ -397,7 +295,7 @@ func (v *S3Volume) Compare(ctx context.Context, loc string, expect []byte) error
// Put writes a block.
func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
var opts s3.Options
@@ -433,7 +331,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go func() {
defer func() {
if ctx.Err() != nil {
- theConfig.debugLogf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
+ v.logger.Debugf("%s: abandoned PutReader goroutine finished with err: %s", v, err)
}
}()
defer close(ready)
@@ -445,7 +343,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
}()
select {
case <-ctx.Done():
- theConfig.debugLogf("%s: taking PutReader's input away: %s", v, ctx.Err())
+ v.logger.Debugf("%s: taking PutReader's input away: %s", v, ctx.Err())
// Our pipe might be stuck in Write(), waiting for
// PutReader() to read. If so, un-stick it. This means
// PutReader will get corrupt data, but that's OK: the
@@ -453,7 +351,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
go io.Copy(ioutil.Discard, bufr)
// CloseWithError() will return once pending I/O is done.
bufw.CloseWithError(ctx.Err())
- theConfig.debugLogf("%s: abandoning PutReader goroutine", v)
+ v.logger.Debugf("%s: abandoning PutReader goroutine", v)
return ctx.Err()
case <-ready:
// Unblock pipe in case PutReader did not consume it.
@@ -464,7 +362,7 @@ func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error {
// Touch sets the timestamp for the given locator to the current time.
func (v *S3Volume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
_, err := v.bucket.Head(loc, nil)
@@ -571,16 +469,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
// Trash a Keep block.
func (v *S3Volume) Trash(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if t, err := v.Mtime(loc); err != nil {
return err
- } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() {
+ } else if time.Since(t) < v.cluster.Collections.BlobSigningTTL.Duration() {
return nil
}
- if theConfig.TrashLifetime == 0 {
- if !s3UnsafeDelete {
+ if v.cluster.Collections.BlobTrashLifetime == 0 {
+ if !v.UnsafeDelete {
return ErrS3TrashDisabled
}
return v.translateError(v.bucket.Del(loc))
@@ -615,7 +513,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error {
// Can't parse timestamp
return err
}
- safeWindow := t.Add(theConfig.TrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
+ safeWindow := t.Add(v.cluster.Collections.BlobTrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow)))
if safeWindow <= 0 {
// We can't count on "touch trash/X" to prolong
// trash/X's lifetime. The new timestamp might not
@@ -698,23 +596,6 @@ func (v *S3Volume) String() string {
return fmt.Sprintf("s3-bucket:%+q", v.Bucket)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *S3Volume) Writable() bool {
- return !v.ReadOnly
-}
-
-// Replication returns the storage redundancy of the underlying
-// device. Configured via command line flag.
-func (v *S3Volume) Replication() int {
- return v.S3Replication
-}
-
-// GetStorageClasses implements Volume
-func (v *S3Volume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
func (v *S3Volume) isKeepBlock(s string) bool {
@@ -751,13 +632,13 @@ func (v *S3Volume) fixRace(loc string) bool {
}
ageWhenTrashed := trashTime.Sub(recentTime)
- if ageWhenTrashed >= theConfig.BlobSignatureTTL.Duration() {
+ if ageWhenTrashed >= v.cluster.Collections.BlobSigningTTL.Duration() {
// No evidence of a race: block hasn't been written
// since it became eligible for Trash. No fix needed.
return false
}
- log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, theConfig.BlobSignatureTTL)
+ log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, v.cluster.Collections.BlobSigningTTL)
log.Printf("notice: fixRace: copying %q to %q to recover from race between Put/Touch and Trash", "recent/"+loc, loc)
err = v.safeCopy(loc, "trash/"+loc)
if err != nil {
@@ -785,6 +666,10 @@ func (v *S3Volume) translateError(err error) error {
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
func (v *S3Volume) EmptyTrash() {
+ if v.cluster.Collections.BlobDeleteConcurrency < 1 {
+ return
+ }
+
var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
// Define "ready to delete" as "...when EmptyTrash started".
@@ -820,8 +705,8 @@ func (v *S3Volume) EmptyTrash() {
log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
return
}
- if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
- if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
+ if trashT.Sub(recentT) < v.cluster.Collections.BlobSigningTTL.Duration() {
+ if age := startT.Sub(recentT); age >= v.cluster.Collections.BlobSigningTTL.Duration()-time.Duration(v.RaceWindow) {
// recent/loc is too old to protect
// loc from being Trashed again during
// the raceWindow that starts if we
@@ -845,7 +730,7 @@ func (v *S3Volume) EmptyTrash() {
return
}
}
- if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
+ if startT.Sub(trashT) < v.cluster.Collections.BlobTrashLifetime.Duration() {
return
}
err = v.bucket.Del(trash.Key)
@@ -872,8 +757,8 @@ func (v *S3Volume) EmptyTrash() {
}
var wg sync.WaitGroup
- todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers)
- for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ {
+ todo := make(chan *s3.Key, v.cluster.Collections.BlobDeleteConcurrency)
+ for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/services/keepstore/server.go b/services/keepstore/server.go
deleted file mode 100644
index 3f6727712..000000000
--- a/services/keepstore/server.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "crypto/tls"
- "net"
- "net/http"
- "os"
- "os/signal"
- "syscall"
-)
-
-type server struct {
- http.Server
-
- // channel (size=1) with the current keypair
- currentCert chan *tls.Certificate
-}
-
-func (srv *server) Serve(l net.Listener) error {
- if theConfig.TLSCertificateFile == "" && theConfig.TLSKeyFile == "" {
- return srv.Server.Serve(l)
- }
- // https://blog.gopheracademy.com/advent-2016/exposing-go-on-the-internet/
- srv.TLSConfig = &tls.Config{
- GetCertificate: srv.getCertificate,
- PreferServerCipherSuites: true,
- CurvePreferences: []tls.CurveID{
- tls.CurveP256,
- tls.X25519,
- },
- MinVersion: tls.VersionTLS12,
- CipherSuites: []uint16{
- tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
- tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
- },
- }
- srv.currentCert = make(chan *tls.Certificate, 1)
- go srv.refreshCertificate(theConfig.TLSCertificateFile, theConfig.TLSKeyFile)
- return srv.Server.ServeTLS(l, "", "")
-}
-
-func (srv *server) refreshCertificate(certfile, keyfile string) {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Fatal("error loading X509 key pair")
- }
- srv.currentCert <- &cert
-
- reload := make(chan os.Signal, 1)
- signal.Notify(reload, syscall.SIGHUP)
- for range reload {
- cert, err := tls.LoadX509KeyPair(certfile, keyfile)
- if err != nil {
- log.WithError(err).Warn("error loading X509 key pair")
- continue
- }
- // Throw away old cert and start using new one
- <-srv.currentCert
- srv.currentCert <- &cert
- }
-}
-
-func (srv *server) getCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
- if srv.currentCert == nil {
- panic("srv.currentCert not initialized")
- }
- cert := <-srv.currentCert
- srv.currentCert <- cert
- return cert, nil
-}
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 8a9fedfb7..57301b814 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -6,6 +6,7 @@ package main
import (
"errors"
+ "log"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -17,35 +18,35 @@ import (
// Delete the block indicated by the trash request Locator
// Repeat
//
-func RunTrashWorker(trashq *WorkQueue) {
+func RunTrashWorker(cluster *arvados.Cluster, trashq *WorkQueue) {
for item := range trashq.NextItem {
trashRequest := item.(TrashRequest)
- TrashItem(trashRequest)
+ TrashItem(cluster, trashRequest)
trashq.DoneItem <- struct{}{}
}
}
// TrashItem deletes the indicated block from every writable volume.
-func TrashItem(trashRequest TrashRequest) {
+func TrashItem(cluster *arvados.Cluster, trashRequest TrashRequest) {
reqMtime := time.Unix(0, trashRequest.BlockMtime)
- if time.Since(reqMtime) < theConfig.BlobSignatureTTL.Duration() {
+ if time.Since(reqMtime) < cluster.Collections.BlobSigningTTL.Duration() {
log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
arvados.Duration(time.Since(reqMtime)),
trashRequest.Locator,
trashRequest.BlockMtime,
reqMtime,
- theConfig.BlobSignatureTTL)
+ cluster.Collections.BlobSigningTTL)
return
}
- var volumes []Volume
+ var volumes []*VolumeMount
if uuid := trashRequest.MountUUID; uuid == "" {
volumes = KeepVM.AllWritable()
} else if v := KeepVM.Lookup(uuid, true); v == nil {
log.Printf("warning: trash request for nonexistent mount: %v", trashRequest)
return
} else {
- volumes = []Volume{v}
+ volumes = []*VolumeMount{v}
}
for _, volume := range volumes {
@@ -59,8 +60,8 @@ func TrashItem(trashRequest TrashRequest) {
continue
}
- if !theConfig.EnableDelete {
- err = errors.New("skipping because EnableDelete is false")
+ if !cluster.Collections.BlobTrash {
+ err = errors.New("skipping because Collections.BlobTrash is false")
} else {
err = volume.Trash(trashRequest.Locator)
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index 4d9e798ac..de953bd74 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -5,12 +5,13 @@
package main
import (
- "bufio"
"context"
- "flag"
+ "encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
+ "log"
"os"
"os/exec"
"path/filepath"
@@ -22,98 +23,34 @@ import (
"syscall"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sirupsen/logrus"
)
-type unixVolumeAdder struct {
- *Config
-}
-
-// String implements flag.Value
-func (vs *unixVolumeAdder) String() string {
- return "-"
-}
-
-func (vs *unixVolumeAdder) Set(path string) error {
- if dirs := strings.Split(path, ","); len(dirs) > 1 {
- log.Print("DEPRECATED: using comma-separated volume list.")
- for _, dir := range dirs {
- if err := vs.Set(dir); err != nil {
- return err
- }
- }
- return nil
- }
- vs.Config.Volumes = append(vs.Config.Volumes, &UnixVolume{
- Root: path,
- ReadOnly: deprecated.flagReadonly,
- Serialize: deprecated.flagSerializeIO,
- })
- return nil
-}
-
func init() {
- VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &UnixVolume{} })
-
- flag.Var(&unixVolumeAdder{theConfig}, "volumes", "see Volumes configuration")
- flag.Var(&unixVolumeAdder{theConfig}, "volume", "see Volumes configuration")
+ driver["Directory"] = newDirectoryVolume
}
-// Discover adds a UnixVolume for every directory named "keep" that is
-// located at the top level of a device- or tmpfs-backed mount point
-// other than "/". It returns the number of volumes added.
-func (vs *unixVolumeAdder) Discover() int {
- added := 0
- f, err := os.Open(ProcMounts)
+func newDirectoryVolume(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger) (Volume, error) {
+ v := UnixVolume{volume: volume, logger: logger}
+ err := json.Unmarshal(volume.DriverParameters, &v)
if err != nil {
- log.Fatalf("opening %s: %s", ProcMounts, err)
+ return nil, err
}
- scanner := bufio.NewScanner(f)
- for scanner.Scan() {
- args := strings.Fields(scanner.Text())
- if err := scanner.Err(); err != nil {
- log.Fatalf("reading %s: %s", ProcMounts, err)
- }
- dev, mount := args[0], args[1]
- if mount == "/" {
- continue
- }
- if dev != "tmpfs" && !strings.HasPrefix(dev, "/dev/") {
- continue
- }
- keepdir := mount + "/keep"
- if st, err := os.Stat(keepdir); err != nil || !st.IsDir() {
- continue
- }
- // Set the -readonly flag (but only for this volume)
- // if the filesystem is mounted readonly.
- flagReadonlyWas := deprecated.flagReadonly
- for _, fsopt := range strings.Split(args[3], ",") {
- if fsopt == "ro" {
- deprecated.flagReadonly = true
- break
- }
- if fsopt == "rw" {
- break
- }
- }
- if err := vs.Set(keepdir); err != nil {
- log.Printf("adding %q: %s", keepdir, err)
- } else {
- added++
- }
- deprecated.flagReadonly = flagReadonlyWas
+ if v.Root == "" {
+ return nil, errors.New("DriverParameters: Root must be provided")
}
- return added
+ return &v, nil
}
// A UnixVolume stores and retrieves blocks in a local directory.
type UnixVolume struct {
- Root string // path to the volume's root directory
- ReadOnly bool
- Serialize bool
- DirectoryReplication int
- StorageClasses []string
+ Root string // path to the volume's root directory
+ Serialize bool
+
+ volume arvados.Volume
+ logger logrus.FieldLogger
// something to lock during IO, typically a sync.Mutex (or nil
// to skip locking)
@@ -122,12 +59,12 @@ type UnixVolume struct {
os osWithStats
}
-// DeviceID returns a globally unique ID for the volume's root
+// GetDeviceID returns a globally unique ID for the volume's root
// directory, consisting of the filesystem's UUID and the path from
// filesystem root to storage directory, joined by "/". For example,
-// the DeviceID for a local directory "/mnt/xvda1/keep" might be
+// the device ID for a local directory "/mnt/xvda1/keep" might be
// "fa0b6166-3b55-4994-bd3f-92f4e00a1bb0/keep".
-func (v *UnixVolume) DeviceID() string {
+func (v *UnixVolume) GetDeviceID() string {
giveup := func(f string, args ...interface{}) string {
log.Printf(f+"; using blank DeviceID for volume %s", append(args, v)...)
return ""
@@ -198,27 +135,6 @@ func (v *UnixVolume) DeviceID() string {
return giveup("could not find entry in %q matching %q", udir, dev)
}
-// Examples implements VolumeWithExamples.
-func (*UnixVolume) Examples() []Volume {
- return []Volume{
- &UnixVolume{
- Root: "/mnt/local-disk",
- Serialize: true,
- DirectoryReplication: 1,
- },
- &UnixVolume{
- Root: "/mnt/network-disk",
- Serialize: false,
- DirectoryReplication: 2,
- },
- }
-}
-
-// Type implements Volume
-func (v *UnixVolume) Type() string {
- return "Directory"
-}
-
// Start implements Volume
func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
if v.Serialize {
@@ -227,11 +143,8 @@ func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
if !strings.HasPrefix(v.Root, "/") {
return fmt.Errorf("volume root does not start with '/': %q", v.Root)
}
- if v.DirectoryReplication == 0 {
- v.DirectoryReplication = 1
- }
// Set up prometheus metrics
- lbls := prometheus.Labels{"device_id": v.DeviceID()}
+ lbls := prometheus.Labels{"device_id": v.GetDeviceID()}
v.os.stats.opsCounters, v.os.stats.errCounters, v.os.stats.ioBytes = vm.getCounterVecsFor(lbls)
_, err := v.os.Stat(v.Root)
@@ -241,7 +154,7 @@ func (v *UnixVolume) Start(vm *volumeMetricsVecs) error {
// Touch sets the timestamp for the given locator to the current time
func (v *UnixVolume) Touch(loc string) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
p := v.blockPath(loc)
@@ -349,7 +262,7 @@ func (v *UnixVolume) Put(ctx context.Context, loc string, block []byte) error {
// WriteBlock implements BlockWriter.
func (v *UnixVolume) WriteBlock(ctx context.Context, loc string, rdr io.Reader) error {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if v.IsFull() {
@@ -516,7 +429,7 @@ func (v *UnixVolume) Trash(loc string) error {
// Trash() will read the correct up-to-date timestamp and choose not to
// trash the file.
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
if err := v.lock(context.TODO()); err != nil {
@@ -555,7 +468,7 @@ func (v *UnixVolume) Trash(loc string) error {
// Look for path/{loc}.trash.{deadline} in storage,
// and rename the first such file as path/{loc}
func (v *UnixVolume) Untrash(loc string) (err error) {
- if v.ReadOnly {
+ if v.volume.ReadOnly {
return MethodDisabledError
}
@@ -650,23 +563,12 @@ func (v *UnixVolume) String() string {
return fmt.Sprintf("[UnixVolume %s]", v.Root)
}
-// Writable returns false if all future Put, Mtime, and Delete calls
-// are expected to fail.
-func (v *UnixVolume) Writable() bool {
- return !v.ReadOnly
-}
-
// Replication returns the number of replicas promised by the
// underlying device (as specified in configuration).
func (v *UnixVolume) Replication() int {
return v.DirectoryReplication
}
-// GetStorageClasses implements Volume
-func (v *UnixVolume) GetStorageClasses() []string {
- return v.StorageClasses
-}
-
// InternalStats returns I/O and filesystem ops counters.
func (v *UnixVolume) InternalStats() interface{} {
return &v.os.stats
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
deleted file mode 100644
index 8e83f6ce5..000000000
--- a/services/keepstore/usage.go
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "sort"
- "strings"
-
- "github.com/ghodss/yaml"
-)
-
-func usage() {
- c := DefaultConfig()
- knownTypes := []string{}
- for _, vt := range VolumeTypes {
- c.Volumes = append(c.Volumes, vt().Examples()...)
- knownTypes = append(knownTypes, vt().Type())
- }
- exampleConfigFile, err := yaml.Marshal(c)
- if err != nil {
- panic(err)
- }
- sort.Strings(knownTypes)
- knownTypeList := strings.Join(knownTypes, ", ")
- fmt.Fprintf(os.Stderr, `
-
-keepstore provides a content-addressed data store backed by a local filesystem or networked storage.
-
-Usage: keepstore -config path/to/keepstore.yml
- keepstore [OPTIONS] -dump-config
-
-NOTE: All options (other than -config) are deprecated in favor of YAML
- configuration. Use -dump-config to translate existing
- configurations to YAML format.
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-LogFormat:
-
- Format of request/response and error logs: "json" or "text".
-
-PIDFile:
-
- Path to write PID file during startup. This file is kept open and
- locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
- one way to shut down. Exit immediately if there is an error
- opening, locking, or writing the PID file.
-
-MaxBuffers:
-
- Maximum RAM to use for data buffers, given in multiples of block
- size (64 MiB). When this limit is reached, HTTP requests requiring
- buffers (like GET and PUT) will wait for buffer space to be
- released.
-
-MaxRequests:
-
- Maximum concurrent requests. When this limit is reached, new
- requests will receive 503 responses. Note: this limit does not
- include idle connections from clients using HTTP keepalive, so it
- does not strictly limit the number of concurrent connections. If
- omitted or zero, the default is 2 * MaxBuffers.
-
-BlobSigningKeyFile:
-
- Local file containing the secret blob signing key (used to
- generate and verify blob signatures). This key should be
- identical to the API server's blob_signing_key configuration
- entry.
-
-RequireSignatures:
-
- Honor read requests only if a valid signature is provided. This
- should be true, except for development use and when migrating from
- a very old version.
-
-BlobSignatureTTL:
-
- Duration for which new permission signatures (returned in PUT
- responses) will be valid. This should be equal to the API
- server's blob_signature_ttl configuration entry.
-
-SystemAuthTokenFile:
-
- Local file containing the Arvados API token used by keep-balance
- or data manager. Delete, trash, and index requests are honored
- only for this token.
-
-EnableDelete:
-
- Enable trash and delete features. If false, trash lists will be
- accepted but blocks will not be trashed or deleted.
-
-TrashLifetime:
-
- Time duration after a block is trashed during which it can be
- recovered using an /untrash request.
-
-TrashCheckInterval:
-
- How often to check for (and delete) trashed blocks whose
- TrashLifetime has expired.
-
-TrashWorkers:
-
- Maximum number of concurrent trash operations. Default is 1, i.e.,
- trash lists are processed serially.
-
-EmptyTrashWorkers:
-
- Maximum number of concurrent block deletion operations (per
- volume) when emptying trash. Default is 1.
-
-PullWorkers:
-
- Maximum number of concurrent pull operations. Default is 1, i.e.,
- pull lists are processed serially.
-
-TLSCertificateFile:
-
- Path to server certificate file in X509 format. Enables TLS mode.
-
- Example: /var/lib/acme/live/keep0.example.com/fullchain
-
-TLSKeyFile:
-
- Path to server key file in X509 format. Enables TLS mode.
-
- The key pair is read from disk during startup, and whenever SIGHUP
- is received.
-
- Example: /var/lib/acme/live/keep0.example.com/privkey
-
-Volumes:
-
- List of storage volumes. If omitted or empty, the default is to
- use all directories named "keep" that exist in the top level
- directory of a mount point at startup time.
-
- Volume types: %s
-
- (See volume configuration examples above.)
-
-`, exampleConfigFile, knownTypeList)
-}
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 52b9b1b24..3d5eb4325 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -14,6 +14,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
)
type BlockWriter interface {
@@ -28,14 +29,12 @@ type BlockReader interface {
ReadBlock(ctx context.Context, loc string, w io.Writer) error
}
+var driver = map[string]func(*arvados.Cluster, arvados.Volume, logrus.FieldLogger) (Volume, error){}
+
// A Volume is an interface representing a Keep back-end storage unit:
// for example, a single mounted disk, a RAID array, an Amazon S3 volume,
// etc.
type Volume interface {
- // Volume type as specified in config file. Examples: "S3",
- // "Directory".
- Type() string
-
// Do whatever private setup tasks and configuration checks
// are needed. Return non-nil if the volume is unusable (e.g.,
// invalid config).
@@ -222,29 +221,13 @@ type Volume interface {
// secrets.
String() string
- // Writable returns false if all future Put, Mtime, and Delete
- // calls are expected to fail.
- //
- // If the volume is only temporarily unwritable -- or if Put
- // will fail because it is full, but Mtime or Delete can
- // succeed -- then Writable should return false.
- Writable() bool
-
- // Replication returns the storage redundancy of the
- // underlying device. It will be passed on to clients in
- // responses to PUT requests.
- Replication() int
-
// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
// and deletes them from the volume.
EmptyTrash()
// Return a globally unique ID of the underlying storage
// device if possible, otherwise "".
- DeviceID() string
-
- // Get the storage classes associated with this volume
- GetStorageClasses() []string
+ GetDeviceID() string
}
// A VolumeWithExamples provides example configs to display in the
@@ -260,24 +243,24 @@ type VolumeManager interface {
// Mounts returns all mounts (volume attachments).
Mounts() []*VolumeMount
- // Lookup returns the volume under the given mount
- // UUID. Returns nil if the mount does not exist. If
- // write==true, returns nil if the volume is not writable.
- Lookup(uuid string, write bool) Volume
+ // Lookup returns the mount with the given UUID. Returns nil
+ // if the mount does not exist. If write==true, returns nil if
+ // the mount is not writable.
+ Lookup(uuid string, write bool) *VolumeMount
- // AllReadable returns all volumes.
- AllReadable() []Volume
+ // AllReadable returns all mounts.
+ AllReadable() []*VolumeMount
- // AllWritable returns all volumes that aren't known to be in
+ // AllWritable returns all mounts that aren't known to be in
// a read-only state. (There is no guarantee that a write to
// one will succeed, though.)
- AllWritable() []Volume
+ AllWritable() []*VolumeMount
// NextWritable returns the volume where the next new block
// should be written. A VolumeManager can select a volume in
// order to distribute activity across spindles, fill up disks
// with more free space, etc.
- NextWritable() Volume
+ NextWritable() *VolumeMount
// VolumeStats returns the ioStats used for tracking stats for
// the given Volume.
@@ -290,7 +273,7 @@ type VolumeManager interface {
// A VolumeMount is an attachment of a Volume to a VolumeManager.
type VolumeMount struct {
arvados.KeepMount
- volume Volume
+ Volume
}
// Generate a UUID the way API server would for a "KeepVolumeMount"
@@ -321,22 +304,43 @@ type RRVolumeManager struct {
}
// MakeRRVolumeManager initializes RRVolumeManager
-func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
+func MakeRRVolumeManager(cluster *arvados.Cluster, myURL arvados.URL, metrics *volumeMetricsVecs) *RRVolumeManager {
vm := &RRVolumeManager{
iostats: make(map[Volume]*ioStats),
}
vm.mountMap = make(map[string]*VolumeMount)
- for _, v := range volumes {
- sc := v.GetStorageClasses()
+ for uuid, v := range cluster.Volumes {
+ va, ok := v.AccessViaHosts[myURL]
+ if !ok && len(v.AccessViaHosts) > 0 {
+ continue
+ }
+ dri, ok := driver[cfgvol.Driver]
+ if !ok {
+ h.err = fmt.Errorf("volume %s: invalid driver %q", uuid, cfgvol.Driver)
+ return
+ }
+ vol, err := dri(h.Cluster, cfgvol)
+ if err != nil {
+ h.err = fmt.Errorf("error initializing volume %s: %s", uuid, err)
+ return
+ }
+ err = vol.Start(metrics)
+ if err != nil {
+ h.err = fmt.Errorf("error starting volume %s (%s): %s", uuid, vol, err)
+ return
+ }
+ h.Logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+
+ sc := v.StorageClasses
if len(sc) == 0 {
sc = []string{"default"}
}
mnt := &VolumeMount{
KeepMount: arvados.KeepMount{
UUID: (*VolumeMount)(nil).generateUUID(),
- DeviceID: v.DeviceID(),
- ReadOnly: !v.Writable(),
- Replication: v.Replication(),
+ DeviceID: v.GetDeviceID(),
+ ReadOnly: v.ReadOnly || va.ReadOnly,
+ Replication: v.Replication,
StorageClasses: sc,
},
volume: v,
@@ -345,7 +349,7 @@ func MakeRRVolumeManager(volumes []Volume) *RRVolumeManager {
vm.mounts = append(vm.mounts, mnt)
vm.mountMap[mnt.UUID] = mnt
vm.readables = append(vm.readables, v)
- if v.Writable() {
+ if !mnt.KeepMount.ReadOnly {
vm.writables = append(vm.writables, v)
}
}
@@ -358,7 +362,7 @@ func (vm *RRVolumeManager) Mounts() []*VolumeMount {
func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) Volume {
if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) {
- return mnt.volume
+ return mnt.Volume
} else {
return nil
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list