[ARVADOS] updated: 1.2.0-201-g9f6bbb975
Git user
git at public.curoverse.com
Mon Oct 15 17:02:34 EDT 2018
Summary of changes:
doc/_includes/_create_superuser_token.liquid | 14 +++++++++
doc/_includes/_install_compute_docker.liquid | 2 +-
.../install-dispatch.html.textile.liquid | 9 ++----
.../install-keep-balance.html.textile.liquid | 2 +-
doc/install/install-keepproxy.html.textile.liquid | 2 +-
doc/install/install-keepstore.html.textile.liquid | 2 +-
.../install-shell-server.html.textile.liquid | 2 +-
sdk/go/arvadostest/fixtures.go | 1 +
services/api/test/fixtures/collections.yml | 4 +--
services/keep-balance/balance.go | 27 +++++++++++++++++
services/keep-balance/balance_run_test.go | 5 +++-
services/keep-balance/block_state.go | 2 ++
services/keep-balance/metrics.go | 18 +++++++-----
services/keep-web/handler_test.go | 34 ++++++++++++++++++++++
services/keep-web/server_test.go | 1 +
15 files changed, 103 insertions(+), 22 deletions(-)
create mode 100644 doc/_includes/_create_superuser_token.liquid
via 9f6bbb975f5e2fa7a46266307318a0fc95349a29 (commit)
via f29266f2fd9d2713f6b2666d13d4f706630b7215 (commit)
via 70459cdae1f4e92789a14c1fecb66f5954a5aa7f (commit)
via c4ec117c9f3ed7108d7079603407b5c5c64a8c1d (commit)
via fd7b63d5d85d23e410f9d6f0c0a697f65b755c47 (commit)
via 3e0605a8286fb11df9e8dba739aea496b1c18418 (commit)
via c38cb3f4595b5e97c23db899b4d7c9993f089c08 (commit)
via a57d9687340458cb4f7861faace44da0e5b0dda1 (commit)
from 7ad5d87a9e2d67224ed440c9320f1850cbaf6ae1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9f6bbb975f5e2fa7a46266307318a0fc95349a29
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Oct 15 17:02:22 2018 -0400
14285: Add deduplication ratio to keep-balance metrics.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 333a4fbde..5f5f9afdc 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -780,6 +780,26 @@ type balancerStats struct {
trashes int
replHistogram []int
classStats map[string]replicationStats
+
+ // collectionBytes / collectionBlockBytes = deduplication ratio
+ collectionBytes int64 // sum(bytes in referenced blocks) across all collections
+ collectionBlockBytes int64 // sum(block size) across all blocks referenced by collections
+ collectionBlockRefs int64 // sum(number of blocks referenced) across all collections
+ collectionBlocks int64 // number of blocks referenced by any collection
+}
+
+func (s *balancerStats) dedupByteRatio() float64 {
+ if s.collectionBlockBytes == 0 {
+ return 0
+ }
+ return float64(s.collectionBytes) / float64(s.collectionBlockBytes)
+}
+
+func (s *balancerStats) dedupBlockRatio() float64 {
+ if s.collectionBlocks == 0 {
+ return 0
+ }
+ return float64(s.collectionBlockRefs) / float64(s.collectionBlocks)
}
type replicationStats struct {
@@ -803,6 +823,13 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
surplus := result.have - result.want
bytes := result.blkid.Size()
+ if rc := int64(result.blk.RefCount); rc > 0 {
+ s.collectionBytes += rc * bytes
+ s.collectionBlockBytes += bytes
+ s.collectionBlockRefs += rc
+ s.collectionBlocks++
+ }
+
for class, state := range result.classState {
cs := s.classStats[class]
if state.unachievable {
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index f42383297..607131545 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -184,7 +184,8 @@ func (s *stubServer) serveFooBarFileCollections() *reqTracker {
if strings.Contains(r.Form.Get("filters"), `modified_at`) {
io.WriteString(w, `{"items_available":0,"items":[]}`)
} else {
- io.WriteString(w, `{"items_available":2,"items":[
+ io.WriteString(w, `{"items_available":3,"items":[
+ {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
{"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
{"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
}
@@ -463,6 +464,8 @@ func (s *runSuite) TestCommit(c *check.C) {
c.Check(metrics, check.Matches, `(?ms).*\nkeep_total_bytes 15\n.*`)
c.Check(metrics, check.Matches, `(?ms).*\nkeepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
c.Check(metrics, check.Matches, `(?ms).*\nkeepbalance_changeset_compute_seconds_count 1\n.*`)
+ c.Check(metrics, check.Matches, `(?ms).*\nkeep_dedup_byte_ratio 1\.5\n.*`)
+ c.Check(metrics, check.Matches, `(?ms).*\nkeep_dedup_block_ratio 1\.5\n.*`)
}
func (s *runSuite) TestRunForever(c *check.C) {
diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index 22e89c019..46e69059c 100644
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -23,6 +23,7 @@ type Replica struct {
// replicas actually stored (according to the keepstore indexes we
// know about).
type BlockState struct {
+ RefCount int
Replicas []Replica
Desired map[string]int
// TODO: Support combinations of classes ("private + durable")
@@ -42,6 +43,7 @@ func (bs *BlockState) addReplica(r Replica) {
}
func (bs *BlockState) increaseDesired(classes []string, n int) {
+ bs.RefCount++
if len(classes) == 0 {
classes = defaultClasses
}
diff --git a/services/keep-balance/metrics.go b/services/keep-balance/metrics.go
index 96ee66ce5..d1613d8bf 100644
--- a/services/keep-balance/metrics.go
+++ b/services/keep-balance/metrics.go
@@ -57,12 +57,14 @@ func (m *metrics) UpdateStats(s balancerStats) {
Help string
}
s2g := map[string]gauge{
- "total": {s.current, "current backend storage usage"},
- "garbage": {s.garbage, "garbage (unreferenced, old)"},
- "transient": {s.unref, "transient (unreferenced, new)"},
- "overreplicated": {s.overrep, "overreplicated"},
- "underreplicated": {s.underrep, "underreplicated"},
- "lost": {s.lost, "lost"},
+ "total": {s.current, "current backend storage usage"},
+ "garbage": {s.garbage, "garbage (unreferenced, old)"},
+ "transient": {s.unref, "transient (unreferenced, new)"},
+ "overreplicated": {s.overrep, "overreplicated"},
+ "underreplicated": {s.underrep, "underreplicated"},
+ "lost": {s.lost, "lost"},
+ "dedup_byte_ratio": {s.dedupByteRatio(), "deduplication ratio, bytes referenced / bytes stored"},
+ "dedup_block_ratio": {s.dedupBlockRatio(), "deduplication ratio, blocks referenced / blocks stored"},
}
m.setupOnce.Do(func() {
// Register gauge(s) for each balancerStats field.
@@ -81,7 +83,7 @@ func (m *metrics) UpdateStats(s balancerStats) {
for _, sub := range []string{"blocks", "bytes", "replicas"} {
addGauge(name+"_"+sub, sub+" of "+gauge.Help)
}
- case int, int64:
+ case int, int64, float64:
addGauge(name, gauge.Help)
default:
panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
@@ -99,6 +101,8 @@ func (m *metrics) UpdateStats(s balancerStats) {
m.statsGauges[name].Set(float64(val))
case int64:
m.statsGauges[name].Set(float64(val))
+ case float64:
+ m.statsGauges[name].Set(float64(val))
default:
panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list