[ARVADOS] created: 1.3.0-728-g386a9d0dc

Git user git at public.curoverse.com
Mon Apr 22 00:31:55 UTC 2019


        at  386a9d0dc47231f45b610585e47cf2bec6dd8354 (commit)


commit 386a9d0dc47231f45b610585e47cf2bec6dd8354
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sun Apr 21 20:25:28 2019 -0400

    15112: Stop processing collections on first error.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index ab500d2c6..eff99b733 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -332,7 +332,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 		defer wg.Done()
 		for coll := range collQ {
 			err := bal.addCollection(coll)
-			if err != nil {
+			if err != nil || len(errs) > 0 {
 				select {
 				case errs <- err:
 				default:

commit 579ebe94b9aae842e23c27bdcde6340dd787e239
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sun Apr 21 20:24:40 2019 -0400

    15112: Fix duplicate default.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 8031ff214..ab500d2c6 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -441,7 +441,7 @@ func (bal *Balancer) ComputeChangeSets() {
 
 func (bal *Balancer) setupLookupTables() {
 	bal.serviceRoots = make(map[string]string)
-	bal.classes = []string{"default"}
+	bal.classes = defaultClasses
 	bal.mountsByClass = map[string]map[*KeepMount]bool{"default": {}}
 	bal.mounts = 0
 	for _, srv := range bal.KeepServices {

commit 2762334aafe02d39a3ce7078b77bfb9da09fd9e7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sun Apr 21 20:24:12 2019 -0400

    15112: Error out right away on unusable manifest.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 346de30b6..8031ff214 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -383,10 +383,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 func (bal *Balancer) addCollection(coll arvados.Collection) error {
 	blkids, err := coll.SizedDigests()
 	if err != nil {
-		bal.mutex.Lock()
-		bal.errors = append(bal.errors, fmt.Errorf("%v: %v", coll.UUID, err))
-		bal.mutex.Unlock()
-		return nil
+		return fmt.Errorf("%v: %v", coll.UUID, err)
 	}
 	repl := bal.DefaultReplication
 	if coll.ReplicationDesired != nil {

commit fc4780dda89687f53ecf0a80758819d3996b4b64
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sun Apr 21 20:23:16 2019 -0400

    15112: Fix log check in test case.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index 8f5d08a19..a79779c7d 100644
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -71,11 +71,11 @@ func (s *integrationSuite) SetUpTest(c *check.C) {
 }
 
 func (s *integrationSuite) TestBalanceAPIFixtures(c *check.C) {
-	var logBuf *bytes.Buffer
+	var logBuf bytes.Buffer
 	for iter := 0; iter < 20; iter++ {
-		logBuf := &bytes.Buffer{}
+		logBuf.Reset()
 		logger := logrus.New()
-		logger.Out = logBuf
+		logger.Out = &logBuf
 		opts := RunOptions{
 			CommitPulls: true,
 			CommitTrash: true,

commit ec91dcd3f74046da500388f86e3fbcd533045b2a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sun Apr 21 20:14:16 2019 -0400

    15112: Show #collections and class->#desired map in dump.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index f47a99761..346de30b6 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -754,7 +754,7 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) ba
 		}
 	}
 	if bal.Dumper != nil {
-		bal.Dumper.Printf("%s have=%d want=%v %s", blkid, have, want, strings.Join(changes, " "))
+		bal.Dumper.Printf("%s refs=%d have=%d want=%v %v %v", blkid, blk.RefCount, have, want, blk.Desired, changes)
 	}
 	return balanceResult{
 		blk:        blk,

commit 201f954cf151bc8285d93223cebb2e4de3a12f18
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sat Apr 20 20:32:54 2019 -0400

    15112: Test multiple volumes with same device IDs.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 1ff18316e..423546c46 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -132,6 +132,75 @@ func (bal *balancerSuite) TestSkipReadonly(c *check.C) {
 		shouldPull: slots{2, 4}})
 }
 
+func (bal *balancerSuite) TestMultipleViewsReadOnly(c *check.C) {
+	bal.testMultipleViews(c, true)
+}
+
+func (bal *balancerSuite) TestMultipleViews(c *check.C) {
+	bal.testMultipleViews(c, false)
+}
+
+func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
+	for i, srv := range bal.srvs {
+		// Add a mount to each service
+		srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i)
+		srv.mounts = append(srv.mounts, &KeepMount{
+			KeepMount: arvados.KeepMount{
+				DeviceID:    fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
+				UUID:        fmt.Sprintf("zzzzz-mount-%015x", i<<16),
+				ReadOnly:    readonly,
+				Replication: 1,
+			},
+			KeepService: srv,
+		})
+	}
+	for i := 1; i < len(bal.srvs); i++ {
+		c.Logf("i=%d", i)
+		if i == 4 {
+			// Timestamps are all different, but one of
+			// the mounts on srv[4] has the same device ID
+			// where the non-deletable replica is stored
+			// on srv[3], so only one replica is safe to
+			// trash.
+			bal.try(c, tester{
+				desired:     map[string]int{"default": 1},
+				current:     slots{0, i, i},
+				shouldTrash: slots{i}})
+		} else if readonly {
+			// Timestamps are all different, and the third
+			// replica can't be trashed because it's on a
+			// read-only mount, so the first two replicas
+			// should be trashed.
+			bal.try(c, tester{
+				desired:     map[string]int{"default": 1},
+				current:     slots{0, i, i},
+				shouldTrash: slots{0, i}})
+		} else {
+			// Timestamps are all different, so both
+			// replicas on the non-optimal server should
+			// be trashed.
+			bal.try(c, tester{
+				desired:     map[string]int{"default": 1},
+				current:     slots{0, i, i},
+				shouldTrash: slots{i, i}})
+		}
+		// If the three replicas have identical timestamps,
+		// none of them can be trashed safely.
+		bal.try(c, tester{
+			desired:    map[string]int{"default": 1},
+			current:    slots{0, i, i},
+			timestamps: []int64{12345678, 12345678, 12345678}})
+		// If the first and third replicas have identical
+		// timestamps, only the second replica should be
+		// trashed.
+		bal.try(c, tester{
+			desired:     map[string]int{"default": 1},
+			current:     slots{0, i, i},
+			timestamps:  []int64{12345678, 12345679, 12345678},
+			shouldTrash: slots{i}})
+	}
+}
+
 func (bal *balancerSuite) TestFixUnbalanced(c *check.C) {
 	bal.try(c, tester{
 		desired:    map[string]int{"default": 2},

commit 606706fd1bda2d20f749b1d416daabd3614e56cc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sat Apr 20 20:29:42 2019 -0400

    15112: Fix mount IDs in test case.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 37be185dc..1ff18316e 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -162,9 +162,10 @@ func (bal *balancerSuite) TestFixUnbalanced(c *check.C) {
 }
 
 func (bal *balancerSuite) TestMultipleReplicasPerService(c *check.C) {
-	for _, srv := range bal.srvs {
+	for s, srv := range bal.srvs {
 		for i := 0; i < 3; i++ {
 			m := *(srv.mounts[0])
+			m.UUID = fmt.Sprintf("zzzzz-mount-%015x", (s<<10)+i)
 			srv.mounts = append(srv.mounts, &m)
 		}
 	}

commit a5be5ca4a71c6c087999ae52843a12df81c2a0ea
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sat Apr 20 20:26:44 2019 -0400

    15112: Use FieldLogger interface so tests can use ctxlog.TestLogger.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 9e8ca3a67..f47a99761 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -31,8 +31,8 @@ import (
 // BlobSignatureTTL; and all N existing replicas of a given data block
 // are in the N best positions in rendezvous probe order.
 type Balancer struct {
-	Logger  *logrus.Logger
-	Dumper  *logrus.Logger
+	Logger  logrus.FieldLogger
+	Dumper  logrus.FieldLogger
 	Metrics *metrics
 
 	*BlockStateMap
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 3316a1724..84516a821 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -76,9 +76,10 @@ func main() {
 		}
 	}
 	if *dumpFlag {
-		runOptions.Dumper = logrus.New()
-		runOptions.Dumper.Out = os.Stdout
-		runOptions.Dumper.Formatter = &logrus.TextFormatter{}
+		dumper := logrus.New()
+		dumper.Out = os.Stdout
+		dumper.Formatter = &logrus.TextFormatter{}
+		runOptions.Dumper = dumper
 	}
 	srv, err := NewServer(cfg, runOptions)
 	if err != nil {
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 613a2f7d3..c867d7b10 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -70,8 +70,8 @@ type RunOptions struct {
 	Once        bool
 	CommitPulls bool
 	CommitTrash bool
-	Logger      *logrus.Logger
-	Dumper      *logrus.Logger
+	Logger      logrus.FieldLogger
+	Dumper      logrus.FieldLogger
 
 	// SafeRendezvousState from the most recent balance operation,
 	// or "" if unknown. If this changes from one run to the next,
@@ -86,8 +86,8 @@ type Server struct {
 	metrics    *metrics
 	listening  string // for tests
 
-	Logger *logrus.Logger
-	Dumper *logrus.Logger
+	Logger logrus.FieldLogger
+	Dumper logrus.FieldLogger
 }
 
 // NewServer returns a new Server that runs Balancers using the given

commit 909988584655ba2cc765bcb72448781594680adc
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Sat Apr 20 17:22:41 2019 -0400

    15112: Report volumes without replicas as "none" in -dump output.
    
    ...instead of reporting as "stay" or "none" depending on
    whether *other* volumes have replicas.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 836be2e60..9e8ca3a67 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -732,17 +732,17 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) ba
 				From:        slot.mnt,
 			})
 			change = changeTrash
-		case len(blk.Replicas) == 0:
-			change = changeNone
-		case slot.repl == nil && slot.want && !slot.mnt.ReadOnly:
+		case len(blk.Replicas) > 0 && slot.repl == nil && slot.want && !slot.mnt.ReadOnly:
 			slot.mnt.KeepService.AddPull(Pull{
 				SizedDigest: blkid,
 				From:        blk.Replicas[0].KeepMount.KeepService,
 				To:          slot.mnt,
 			})
 			change = changePull
-		default:
+		case slot.repl != nil:
 			change = changeStay
+		default:
+			change = changeNone
 		}
 		if bal.Dumper != nil {
 			var mtime int64

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list