[ARVADOS] created: 1.3.0-1565-ge8637e52b

Git user git at public.curoverse.com
Fri Sep 6 18:50:14 UTC 2019


        at  e8637e52b1c042ba6a83f1a19c5a903de689fc82 (commit)


commit e8637e52b1c042ba6a83f1a19c5a903de689fc82
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date:   Fri Sep 6 14:39:39 2019 -0400

    14714: Adds keep-balance to cluster config loading
    
    Also adds a deprecated config loading test and fixes the service file
    
    Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 24b2e450e..ab2f4c4ef 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -347,6 +347,36 @@ Clusters:
       # The default is 2 weeks.
       BlobSigningTTL: 336h
 
+      # When running keep-balance, this is the destination filename for the
+      # list of lost block hashes if there are any, one per line. Updated atomically during
+      # each successful run.
+      BlobMissingReport: ""
+
+      # keep-balance operates periodically, i.e.: do a
+      # scan/balance operation, sleep, repeat.
+      #
+      # BalancePeriod determines the interval between start times of
+      # successive scan/balance operations. If a scan/balance operation
+      # takes longer than RunPeriod, the next one will follow it
+      # immediately.
+      #
+      # If SIGUSR1 is received during an idle period between operations,
+      # the next operation will start immediately.
+      BalancePeriod: 10m
+
+      # Limits the number of collections retrieved by keep-balance per
+      # API transaction. If this is zero, page size is
+      # determined by the API server's own page size limits (see
+      # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead).
+      BalanceCollectionBatch: 100000
+
+      # The size of keep-balance's internal queue of
+      # collections. Higher values use more memory and improve throughput
+      # by allowing keep-balance to fetch the next page of collections
+      # while the current page is still being processed. If this is zero
+      # or omitted, pages are processed serially.
+      BalanceCollectionBuffers: 1000
+
       # Default lifetime for ephemeral collections: 2 weeks. This must not
       # be less than BlobSigningTTL.
       DefaultTrashLifetime: 336h
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 9eb8c40c1..c78fb9962 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -509,3 +509,71 @@ func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
 	cfg.Clusters[cluster.ClusterID] = *cluster
 	return nil
 }
+
+const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
+
+type oldKeepBalanceConfig struct {
+	Client              *arvados.Client
+	Listen              *string
+	KeepServiceTypes    *[]string
+	KeepServiceList     *arvados.KeepServiceList
+	RunPeriod           *arvados.Duration
+	CollectionBatchSize *int
+	CollectionBuffers   *int
+	RequestTimeout      *arvados.Duration
+	LostBlocksFile      *string
+	ManagementToken     *string
+}
+
+func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
+	if ldr.KeepBalancePath == "" {
+		return nil
+	}
+	var oc oldKeepBalanceConfig
+	err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
+	if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return err
+	}
+
+	loadOldClientConfig(cluster, oc.Client)
+
+	if oc.Listen != nil {
+		cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+	}
+	if oc.ManagementToken != nil {
+		cluster.ManagementToken = *oc.ManagementToken
+	}
+	if oc.RunPeriod != nil {
+		cluster.Collections.BalancePeriod = *oc.RunPeriod
+	}
+	if oc.LostBlocksFile != nil {
+		cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
+	}
+	if oc.CollectionBatchSize != nil {
+		cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
+	}
+	if oc.CollectionBuffers != nil {
+		cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
+	}
+	if oc.RequestTimeout != nil {
+		cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
+	}
+
+	msg := "To balance specfic keep services, please update to the cluster config."
+	if oc.KeepServiceTypes != nil && len(*oc.KeepServiceTypes) > 0 {
+		ldr.Logger.Warnf("The KeepServiceType configuration option is not longer supported and is being ignored. %s", msg)
+	}
+	if oc.KeepServiceList != nil {
+		return fmt.Errorf("The KeepServiceList configuration option is no longer supported. Please remove it from your configuration file. %s", msg)
+	}
+
+	cfg.Clusters[cluster.ClusterID] = *cluster
+	return nil
+}
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 5dda0ba94..db7e7b19a 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -216,3 +216,48 @@ func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 	c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
 	c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
 }
+
+func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
+	f := "-legacy-keepbalance-config"
+	content := []byte(fmtKeepBalanceConfig(""))
+	cluster, err := testLoadLegacyConfig(content, f, c)
+
+	c.Check(err, check.IsNil)
+	c.Check(cluster, check.NotNil)
+	c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
+	c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
+	c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
+	c.Check(cluster.Collections.BalanceCollectionBatch, check.Equals, 100000)
+	c.Check(cluster.Collections.BalancePeriod.String(), check.Equals, "10m")
+	c.Check(cluster.Collections.BlobMissingReport, check.Equals, "testfile")
+	c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "30m")
+
+	content = []byte(fmtKeepBalanceConfig(`"KeepServiceTypes":["disk"],`))
+	_, err = testLoadLegacyConfig(content, f, c)
+	c.Check(err, check.IsNil)
+
+	content = []byte(fmtKeepBalanceConfig(`"KeepServiceList":{},`))
+	_, err = testLoadLegacyConfig(content, f, c)
+	c.Check(err, check.NotNil)
+}
+
+func fmtKeepBalanceConfig(param string) string {
+	return fmt.Sprintf(`
+{
+	"Client": {
+		"Scheme": "",
+		"APIHost": "example.com",
+		"AuthToken": "abcdefg",
+		"Insecure": false
+	},
+	"Listen": ":80",
+	%s
+	"RunPeriod": "10m",
+	"CollectionBatchSize": 100000,
+	"CollectionBuffers": 1000,
+	"RequestTimeout": "30m",
+	"ManagementToken": "xyzzy",
+	"LostBlocksFile": "testfile"
+}
+`, param)
+}
diff --git a/lib/config/export.go b/lib/config/export.go
index 6eb4fbe5f..e157671ea 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -91,6 +91,10 @@ var whitelist = map[string]bool{
 	"Collections.TrashSweepInterval":               false,
 	"Collections.TrustAllContent":                  false,
 	"Collections.WebDAVCache":                      false,
+	"Collections.BalanceCollectionBatch":           false,
+	"Collections.BalancePeriod":                    false,
+	"Collections.BlobMissingReport":                false,
+	"Collections.BalanceCollectionBuffers":         false,
 	"Containers":                                   true,
 	"Containers.CloudVMs":                          false,
 	"Containers.CrunchRunCommand":                  false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8a5b4610c..5a241135c 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -353,6 +353,36 @@ Clusters:
       # The default is 2 weeks.
       BlobSigningTTL: 336h
 
+      # When running keep-balance, this is the destination filename for the
+      # list of lost block hashes if there are any, one per line. Updated atomically during
+      # each successful run.
+      BlobMissingReport: ""
+
+      # keep-balance operates periodically, i.e.: do a
+      # scan/balance operation, sleep, repeat.
+      #
+      # BalancePeriod determines the interval between start times of
+      # successive scan/balance operations. If a scan/balance operation
+      # takes longer than RunPeriod, the next one will follow it
+      # immediately.
+      #
+      # If SIGUSR1 is received during an idle period between operations,
+      # the next operation will start immediately.
+      BalancePeriod: 10m
+
+      # Limits the number of collections retrieved by keep-balance per
+      # API transaction. If this is zero, page size is
+      # determined by the API server's own page size limits (see
+      # API.MaxItemsPerResponse and API.MaxIndexDatabaseRead).
+      BalanceCollectionBatch: 100000
+
+      # The size of keep-balance's internal queue of
+      # collections. Higher values use more memory and improve throughput
+      # by allowing keep-balance to fetch the next page of collections
+      # while the current page is still being processed. If this is zero
+      # or omitted, pages are processed serially.
+      BalanceCollectionBuffers: 1000
+
       # Default lifetime for ephemeral collections: 2 weeks. This must not
       # be less than BlobSigningTTL.
       DefaultTrashLifetime: 336h
diff --git a/lib/config/load.go b/lib/config/load.go
index 7e4849393..d75734694 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -36,6 +36,7 @@ type Loader struct {
 	WebsocketPath           string
 	KeepproxyPath           string
 	GitHttpdPath            string
+	KeepBalancePath         string
 
 	configdata []byte
 }
@@ -68,6 +69,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
 	flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
 	flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
 	flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+	flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
 	flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
 }
 
@@ -148,6 +150,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
 	if legacyConfigArg != "-legacy-git-httpd-config" {
 		ldr.GitHttpdPath = ""
 	}
+	if legacyConfigArg != "-legacy-keepbalance-config" {
+		ldr.KeepBalancePath = ""
+	}
 
 	return munged
 }
@@ -250,6 +255,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
 			ldr.loadOldWebsocketConfig(&cfg),
 			ldr.loadOldKeepproxyConfig(&cfg),
 			ldr.loadOldGitHttpdConfig(&cfg),
+			ldr.loadOldKeepBalanceConfig(&cfg),
 		} {
 			if err != nil {
 				return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 5a18972f5..3823d8034 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -111,6 +111,11 @@ type Cluster struct {
 		TrashSweepInterval    Duration
 		TrustAllContent       bool
 
+		BlobMissingReport        string
+		BalancePeriod            Duration
+		BalanceCollectionBatch   int
+		BalanceCollectionBuffers int
+
 		WebDAVCache WebDAVCacheConfig
 	}
 	Git struct {
diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index 563871607..1b71fb4e4 100644
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -6,7 +6,6 @@
 Description=Arvados Keep Balance
 Documentation=https://doc.arvados.org/
 After=network.target
-AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
 
 # systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list