[ARVADOS] updated: 1.3.0-1669-g52b7b2934
Git user
git at public.curoverse.com
Mon Sep 30 16:42:08 UTC 2019
Summary of changes:
lib/config/cmd_test.go | 6 +++
lib/config/deprecated.go | 28 +++++++++++++
lib/config/deprecated_keepstore.go | 8 +++-
lib/config/deprecated_keepstore_test.go | 4 ++
lib/config/load.go | 2 +-
lib/config/load_test.go | 6 +++
lib/service/cmd.go | 18 ---------
sdk/python/tests/run_test_server.py | 1 +
services/keep-balance/balance.go | 4 +-
services/keep-balance/balance_test.go | 2 +
services/keep-balance/main.go | 71 ++++++++++++++-------------------
services/keep-balance/server.go | 8 ----
12 files changed, 85 insertions(+), 73 deletions(-)
via 52b7b2934d5d74ee67ca13f8d1cc95f1379faddc (commit)
via 8db3a110976bbb9288214eeb3064d31cb12c1c82 (commit)
via fcb08f0ca7794b05c77a15ab8bd46b978cb07778 (commit)
via 22cff6bb354d6980c33a0f471d5860f968e853a6 (commit)
via 110580d079cdb0b0a773ecf1671c1f97f1736cc6 (commit)
via 7c0257925a75185937c4a84dfd077458a24c53f9 (commit)
from c8bfd534fc6e33b0b37f9fed1ee6232159edb631 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 52b7b2934d5d74ee67ca13f8d1cc95f1379faddc
Merge: 8db3a1109 fcb08f0ca
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Sep 30 12:31:13 2019 -0400
Merge remote-tracking branch 'origin/master' into 14714-keep-balance-config
refs #14714
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --cc lib/config/deprecated.go
index 593a9bdcb,d0e61dbca..e6b06c131
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@@ -475,77 -476,29 +476,104 @@@ func (ldr *Loader) loadOldGitHttpdConfi
return nil
}
+const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
+
+type oldKeepBalanceConfig struct {
+ Client *arvados.Client
+ Listen *string
+ KeepServiceTypes *[]string
+ KeepServiceList *arvados.KeepServiceList
+ RunPeriod *arvados.Duration
+ CollectionBatchSize *int
+ CollectionBuffers *int
+ RequestTimeout *arvados.Duration
+ LostBlocksFile *string
+ ManagementToken *string
+}
+
+func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
+ if ldr.KeepBalancePath == "" {
+ return nil
+ }
+ var oc oldKeepBalanceConfig
+ err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
+ if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ if oc.Listen != nil {
+ cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+ if oc.RunPeriod != nil {
+ cluster.Collections.BalancePeriod = *oc.RunPeriod
+ }
+ if oc.LostBlocksFile != nil {
+ cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
+ }
+ if oc.CollectionBatchSize != nil {
+ cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
+ }
+ if oc.CollectionBuffers != nil {
+ cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
+ }
+ if oc.RequestTimeout != nil {
+ cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
+ }
+
+ msg := "The %s configuration option is no longer supported. Please remove it from your configuration file. Keep-balance will operate on all configured volumes."
+
+ // If the keep service type provided is "disk" silently ignore it, since
+ // this is what ends up being done anyway.
+ if oc.KeepServiceTypes != nil {
+ numTypes := len(*oc.KeepServiceTypes)
+ if numTypes != 0 && !(numTypes == 1 && (*oc.KeepServiceTypes)[0] == "disk") {
+ return fmt.Errorf(msg, "KeepServiceType")
+ }
+ }
+
+ if oc.KeepServiceList != nil {
+ return fmt.Errorf(msg, "KeepServiceList")
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
++
+ func (ldr *Loader) loadOldEnvironmentVariables(cfg *arvados.Config) error {
+ if os.Getenv("ARVADOS_API_TOKEN") == "" && os.Getenv("ARVADOS_API_HOST") == "" {
+ return nil
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+ if tok := os.Getenv("ARVADOS_API_TOKEN"); tok != "" && cluster.SystemRootToken == "" {
+ ldr.Logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ cluster.SystemRootToken = tok
+ }
+ if apihost := os.Getenv("ARVADOS_API_HOST"); apihost != "" && cluster.Services.Controller.ExternalURL.Host == "" {
+ ldr.Logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ u, err := url.Parse("https://" + apihost)
+ if err != nil {
+ return fmt.Errorf("cannot parse ARVADOS_API_HOST: %s", err)
+ }
+ cluster.Services.Controller.ExternalURL = arvados.URL(*u)
+ if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
+ cluster.TLS.Insecure = true
+ }
+ }
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+ }
commit 8db3a110976bbb9288214eeb3064d31cb12c1c82
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Sep 30 12:26:20 2019 -0400
14714: Simplifies server startup and removes globals
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index a887c3c69..3471d50fe 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -446,7 +446,7 @@ func (bal *Balancer) addCollection(coll arvados.Collection) error {
if coll.ReplicationDesired != nil {
repl = *coll.ReplicationDesired
}
- debugf("%v: %d block x%d", coll.UUID, len(blkids), repl)
+ bal.Logger.Debugf("%v: %d block x%d", coll.UUID, len(blkids), repl)
// Pass pdh to IncreaseDesired only if LostBlocksFile is being
// written -- otherwise it's just a waste of memory.
pdh := ""
@@ -563,7 +563,7 @@ type balanceResult struct {
// balanceBlock compares current state to desired state for a single
// block, and makes the appropriate ChangeSet calls.
func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) balanceResult {
- debugf("balanceBlock: %v %+v", blkid, blk)
+ bal.Logger.Debugf("balanceBlock: %v %+v", blkid, blk)
type slot struct {
mnt *KeepMount // never nil
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index e372d3784..6cffa8ded 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -13,6 +13,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
)
@@ -69,6 +70,7 @@ func (bal *balancerSuite) SetUpSuite(c *check.C) {
}
bal.signatureTTL = 3600
+ bal.Logger = ctxlog.TestLogger(c)
}
func (bal *balancerSuite) SetUpTest(c *check.C) {
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 4379ae535..cf844ab05 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -9,7 +9,6 @@ import (
"flag"
"fmt"
"io"
- "log"
"os"
"git.curoverse.com/arvados.git/lib/config"
@@ -20,46 +19,6 @@ import (
"github.com/sirupsen/logrus"
)
-var (
- version = "dev"
- debugf = func(string, ...interface{}) {}
- command = service.Command(arvados.ServiceNameKeepbalance, newHandler)
- options RunOptions
-)
-
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, registry *prometheus.Registry) service.Handler {
- if !options.Once && cluster.Collections.BalancePeriod == arvados.Duration(0) {
- return service.ErrorHandler(ctx, cluster, fmt.Errorf("cannot start service: Collections.BalancePeriod is zero (if you want to run once and then exit, use the -once flag)"))
- "If using the legacy keep-balance.yml config, RunPeriod is the equivalant of Collections.BalancePeriod."))
- }
-
- ac, err := arvados.NewClientFromConfig(cluster)
- ac.AuthToken = token
- if err != nil {
- return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
- }
-
- if cluster.SystemLogs.LogLevel == "debug" {
- debugf = log.Printf
- }
-
- if options.Logger == nil {
- options.Logger = ctxlog.FromContext(ctx)
- }
-
- srv := &Server{
- Cluster: cluster,
- ArvClient: ac,
- RunOptions: options,
- Metrics: newMetrics(registry),
- Logger: options.Logger,
- Dumper: options.Dumper,
- }
-
- srv.Start()
- return srv
-}
-
func main() {
os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
@@ -67,6 +26,7 @@ func main() {
func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.FromContext(context.Background())
+ var options RunOptions
flags := flag.NewFlagSet(prog, flag.ExitOnError)
flags.BoolVar(&options.Once, "once", false,
"balance once and then exit")
@@ -98,5 +58,32 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
}
})
- return command.RunCommand(prog, args, stdin, stdout, stderr)
+ return service.Command(arvados.ServiceNameKeepbalance,
+ func(ctx context.Context, cluster *arvados.Cluster, token string, registry *prometheus.Registry) service.Handler {
+ if !options.Once && cluster.Collections.BalancePeriod == arvados.Duration(0) {
+ return service.ErrorHandler(ctx, cluster, fmt.Errorf("cannot start service: Collections.BalancePeriod is zero (if you want to run once and then exit, use the -once flag)"))
+ }
+
+ ac, err := arvados.NewClientFromConfig(cluster)
+ ac.AuthToken = token
+ if err != nil {
+ return service.ErrorHandler(ctx, cluster, fmt.Errorf("error initializing client from cluster config: %s", err))
+ }
+
+ if options.Logger == nil {
+ options.Logger = ctxlog.FromContext(ctx)
+ }
+
+ srv := &Server{
+ Cluster: cluster,
+ ArvClient: ac,
+ RunOptions: options,
+ Metrics: newMetrics(registry),
+ Logger: options.Logger,
+ Dumper: options.Dumper,
+ }
+
+ go srv.run()
+ return srv
+ }).RunCommand(prog, args, stdin, stdout, stderr)
}
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 3e665a30d..9f192d635 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -8,7 +8,6 @@ import (
"net/http"
"os"
"os/signal"
- "sync"
"syscall"
"time"
@@ -47,7 +46,6 @@ type Server struct {
Metrics *metrics
httpHandler http.Handler
- setupOnce sync.Once
Logger logrus.FieldLogger
Dumper logrus.FieldLogger
@@ -63,12 +61,6 @@ func (srv *Server) CheckHealth() error {
return nil
}
-// Start sets up and runs the balancer.
-func (srv *Server) Start() {
- srv.setupOnce.Do(srv.setup)
- go srv.run()
-}
-
func (srv *Server) setup() {
if srv.Cluster.ManagementToken == "" {
srv.httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list