[ARVADOS] updated: 6717b69f19af5dd4489bcc97c3e328fc499fa311

[Git user]

Summary of changes:
 services/keep-balance/balance.go | 45 +++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

       via  6717b69f19af5dd4489bcc97c3e328fc499fa311 (commit)
       via  b32b8c95b5a8a1c6fa2e0a541354c46fc11d5055 (commit)
       via  0726bc8d809bc2012d28fac547e8902b89bad45c (commit)
      from  34df3cb56d837687c47931dcaba3acdad81fd1ad (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 6717b69f19af5dd4489bcc97c3e328fc499fa311
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue May 24 10:02:39 2016 -0400

    9162: Add replication level histogram
    
    Ported from 00a8ece1580a894dbbf9f756685eefc134e4d0d6 by jrandall

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index bb216a1..32a4b21 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"math"
 	"os"
 	"runtime"
 	"strings"
@@ -386,9 +387,11 @@ type balancerStats struct {
 	lost, overrep, unref, garbage, underrep, justright blocksNBytes
 	desired, current                                   blocksNBytes
 	pulls, trashes                                     int
+	replHistogram                                      []int
 }
 
 func (bal *Balancer) getStatistics() (s balancerStats) {
+	s.replHistogram = make([]int, 2)
 	bal.BlockStateMap.Apply(func(blkid arvados.SizedDigest, blk *BlockState) {
 		surplus := len(blk.Replicas) - blk.Desired
 		bytes := blkid.Size()
@@ -432,6 +435,11 @@ func (bal *Balancer) getStatistics() (s balancerStats) {
 			s.current.blocks++
 			s.current.bytes += bytes * int64(len(blk.Replicas))
 		}
+
+		for len(s.replHistogram) <= len(blk.Replicas) {
+			s.replHistogram = append(s.replHistogram, 0)
+		}
+		s.replHistogram[len(blk.Replicas)]++
 	})
 	for _, srv := range bal.KeepServices {
 		s.pulls += len(srv.ChangeSet.Pulls)
@@ -460,6 +468,25 @@ func (bal *Balancer) PrintStatistics() {
 		bal.logf("%s: %v\n", srv, &srv.ChangeSet)
 	}
 	bal.logf("===")
+	bal.printHistogram(s, 60)
+	bal.logf("===")
+}
+
+func (bal *Balancer) printHistogram(s balancerStats, hashColumns int) {
+	bal.logf("Replication level distribution (counting N replicas on a single server as N):")
+	maxCount := 0
+	for _, count := range s.replHistogram {
+		if maxCount < count {
+			maxCount = count
+		}
+	}
+	hashes := strings.Repeat("#", hashColumns)
+	countWidth := 1 + int(math.Log10(float64(maxCount+1)))
+	scaleCount := 10 * float64(hashColumns) / math.Floor(1+10*math.Log10(float64(maxCount+1)))
+	for repl, count := range s.replHistogram {
+		nHashes := int(scaleCount * math.Log10(float64(count+1)))
+		bal.logf("%2d: %*d %s", repl, countWidth, count, hashes[:nHashes])
+	}
 }
 
 // CheckSanityLate checks for configuration and runtime errors after

commit b32b8c95b5a8a1c6fa2e0a541354c46fc11d5055
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue May 24 09:55:51 2016 -0400

    9162: Tweak errs channel use

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 94230c0..bb216a1 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -152,7 +152,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 	bal.DefaultReplication = dd.DefaultCollectionReplication
 	bal.MinMtime = time.Now().Unix() - dd.BlobSignatureTTL
 
-	errs := make(chan error)
+	errs := make(chan error, 2+len(bal.KeepServices))
 	wg := sync.WaitGroup{}
 
 	for _, srv := range bal.KeepServices {
@@ -180,7 +180,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 			err := bal.addCollection(coll)
 			if err != nil {
 				errs <- err
-				for _ = range collQ {
+				for range collQ {
 				}
 				return
 			}
@@ -212,8 +212,11 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 	}()
 
 	go func() {
+		// Send a nil error when all goroutines finish. If
+		// this is the first error sent to errs, then
+		// everything worked.
 		wg.Wait()
-		close(errs)
+		errs <- nil
 	}()
 	return <-errs
 }

commit 0726bc8d809bc2012d28fac547e8902b89bad45c
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue May 24 08:33:53 2016 -0400

    9162: Adjust dump format for easier grep.

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index ffe24f7..94230c0 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -3,13 +3,14 @@ package main
 import (
 	"fmt"
 	"log"
-	"runtime"
 	"os"
+	"runtime"
+	"strings"
 	"sync"
 	"time"
 
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 )
 
 // Balancer compares the contents of keepstore servers with the
@@ -360,11 +361,11 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) {
 			change = changePull
 		}
 		if bal.Dumper != nil {
-			changes = append(changes, fmt.Sprintf("%s:%s:%d", changeName[change], uuid, repl.Mtime))
+			changes = append(changes, fmt.Sprintf("%s:%d=%s,%d", srv.ServiceHost, srv.ServicePort, changeName[change], repl.Mtime))
 		}
 	}
 	if bal.Dumper != nil {
-		bal.Dumper.Printf("%s have=%d want=%d %v", blkid, len(blk.Replicas), blk.Desired, changes)
+		bal.Dumper.Printf("%s have=%d want=%d %s", blkid, len(blk.Replicas), blk.Desired, strings.Join(changes, " "))
 	}
 }
 

-----------------------------------------------------------------------


hooks/post-receive
--