[ARVADOS] created: 1.3.0-1565-ge8637e52b
Git user
git at public.curoverse.com
Fri Sep 6 18:50:14 UTC 2019
at e8637e52b1c042ba6a83f1a19c5a903de689fc82 (commit)
commit e8637e52b1c042ba6a83f1a19c5a903de689fc82
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Fri Sep 6 14:39:39 2019 -0400
14714: Adds keep-balance to cluster config loading
Also adds a deprecated config loading test and fixes the service file
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 24b2e450e..ab2f4c4ef 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -347,6 +347,36 @@ Clusters:
# The default is 2 weeks.
BlobSigningTTL: 336h
+ # When running keep-balance, this is the destination filename for the
+ # list of lost block hashes if there are any, one per line. Updated atomically during
+ # each successful run.
+ BlobMissingReport: ""
+
+ # keep-balance operates periodically, i.e.: do a
+ # scan/balance operation, sleep, repeat.
+ #
+ # BalancePeriod determines the interval between start times of
+ # successive scan/balance operations. If a scan/balance operation
+ # takes longer than RunPeriod, the next one will follow it
+ # immediately.
+ #
+ # If SIGUSR1 is received during an idle period between operations,
+ # the next operation will start immediately.
+ BalancePeriod: 10m
+
+ # Limits the number of collections retrieved by keep-balance per
+ # API transaction. If this is zero, page size is
+ # determined by the API server's own page size limits (see
+ # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead).
+ BalanceCollectionBatch: 100000
+
+ # The size of keep-balance's internal queue of
+ # collections. Higher values use more memory and improve throughput
+ # by allowing keep-balance to fetch the next page of collections
+ # while the current page is still being processed. If this is zero
+ # or omitted, pages are processed serially.
+ BalanceCollectionBuffers: 1000
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 9eb8c40c1..c78fb9962 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -509,3 +509,71 @@ func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
cfg.Clusters[cluster.ClusterID] = *cluster
return nil
}
+
+const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
+
+type oldKeepBalanceConfig struct {
+ Client *arvados.Client
+ Listen *string
+ KeepServiceTypes *[]string
+ KeepServiceList *arvados.KeepServiceList
+ RunPeriod *arvados.Duration
+ CollectionBatchSize *int
+ CollectionBuffers *int
+ RequestTimeout *arvados.Duration
+ LostBlocksFile *string
+ ManagementToken *string
+}
+
+func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
+ if ldr.KeepBalancePath == "" {
+ return nil
+ }
+ var oc oldKeepBalanceConfig
+ err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
+ if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ if oc.Listen != nil {
+ cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+ if oc.RunPeriod != nil {
+ cluster.Collections.BalancePeriod = *oc.RunPeriod
+ }
+ if oc.LostBlocksFile != nil {
+ cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
+ }
+ if oc.CollectionBatchSize != nil {
+ cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
+ }
+ if oc.CollectionBuffers != nil {
+ cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
+ }
+ if oc.RequestTimeout != nil {
+ cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
+ }
+
+ msg := "To balance specfic keep services, please update to the cluster config."
+ if oc.KeepServiceTypes != nil && len(*oc.KeepServiceTypes) > 0 {
+ ldr.Logger.Warnf("The KeepServiceType configuration option is not longer supported and is being ignored. %s", msg)
+ }
+ if oc.KeepServiceList != nil {
+ return fmt.Errorf("The KeepServiceList configuration option is no longer supported. Please remove it from your configuration file. %s", msg)
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 5dda0ba94..db7e7b19a 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -216,3 +216,48 @@ func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
}
+
+func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
+ f := "-legacy-keepbalance-config"
+ content := []byte(fmtKeepBalanceConfig(""))
+ cluster, err := testLoadLegacyConfig(content, f, c)
+
+ c.Check(err, check.IsNil)
+ c.Check(cluster, check.NotNil)
+ c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
+ c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
+ c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
+ c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
+ c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
+ c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
+ c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
+
+ content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.IsNil)
+
+ content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.NotNil)
+}
+
+func fmtKeepBalanceConfig(param string) string {
+ return fmt.Sprintf(`
+{
+ "Client": {
+ "Scheme": "",
+ "APIHost": "example.com",
+ "AuthToken": "abcdefg",
+ "Insecure": false
+ },
+ "Listen": ":80",
+ %s
+ "RunPeriod": "10m",
+ "CollectionBatchSize": 100000,
+ "CollectionBuffers": 1000,
+ "RequestTimeout": "30m",
+ "ManagementToken": "xyzzy",
+ "LostBlocksFile": "testfile"
+}
+`, param)
+}
diff --git a/lib/config/export.go b/lib/config/export.go
index 6eb4fbe5f..e157671ea 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -91,6 +91,10 @@ var whitelist = map[string]bool{
"Collections.TrashSweepInterval": false,
"Collections.TrustAllContent": false,
"Collections.WebDAVCache": false,
+ "Collections.BalanceCollectionBatch": false,
+ "Collections.BalancePeriod": false,
+ "Collections.BlobMissingReport": false,
+ "Collections.BalanceCollectionBuffers": false,
"Containers": true,
"Containers.CloudVMs": false,
"Containers.CrunchRunCommand": false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8a5b4610c..5a241135c 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -353,6 +353,36 @@ Clusters:
# The default is 2 weeks.
BlobSigningTTL: 336h
+ # When running keep-balance, this is the destination filename for the
+ # list of lost block hashes if there are any, one per line. Updated atomically during
+ # each successful run.
+ BlobMissingReport: ""
+
+ # keep-balance operates periodically, i.e.: do a
+ # scan/balance operation, sleep, repeat.
+ #
+ # BalancePeriod determines the interval between start times of
+ # successive scan/balance operations. If a scan/balance operation
+ # takes longer than RunPeriod, the next one will follow it
+ # immediately.
+ #
+ # If SIGUSR1 is received during an idle period between operations,
+ # the next operation will start immediately.
+ BalancePeriod: 10m
+
+ # Limits the number of collections retrieved by keep-balance per
+ # API transaction. If this is zero, page size is
+ # determined by the API server's own page size limits (see
+ # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead).
+ BalanceCollectionBatch: 100000
+
+ # The size of keep-balance's internal queue of
+ # collections. Higher values use more memory and improve throughput
+ # by allowing keep-balance to fetch the next page of collections
+ # while the current page is still being processed. If this is zero
+ # or omitted, pages are processed serially.
+ BalanceCollectionBuffers: 1000
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
diff --git a/lib/config/load.go b/lib/config/load.go
index 7e4849393..d75734694 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -36,6 +36,7 @@ type Loader struct {
WebsocketPath string
KeepproxyPath string
GitHttpdPath string
+ KeepBalancePath string
configdata []byte
}
@@ -68,6 +69,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+ flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
@@ -148,6 +150,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
if legacyConfigArg != "-legacy-git-httpd-config" {
ldr.GitHttpdPath = ""
}
+ if legacyConfigArg != "-legacy-keepbalance-config" {
+ ldr.KeepBalancePath = ""
+ }
return munged
}
@@ -250,6 +255,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
ldr.loadOldWebsocketConfig(&cfg),
ldr.loadOldKeepproxyConfig(&cfg),
ldr.loadOldGitHttpdConfig(&cfg),
+ ldr.loadOldKeepBalanceConfig(&cfg),
} {
if err != nil {
return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 5a18972f5..3823d8034 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -111,6 +111,11 @@ type Cluster struct {
TrashSweepInterval Duration
TrustAllContent bool
+ BlobMissingReport string
+ BalancePeriod Duration
+ BalanceCollectionBatch int
+ BalanceCollectionBuffers int
+
WebDAVCache WebDAVCacheConfig
}
Git struct {
diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 563871607..1b71fb4e4 100644
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -6,7 +6,6 @@
Description=Arvados Keep Balance
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list