[ARVADOS] updated: 1.2.0-207-gfaeb978ad
Git user
git at public.curoverse.com
Tue Oct 16 11:49:23 EDT 2018
Summary of changes:
apps/workbench/app/views/layouts/body.html.erb | 4 ++++
apps/workbench/config/application.default.yml | 6 ++++++
services/dockercleaner/arvados-docker-cleaner.service | 2 +-
services/keep-balance/balance_run_test.go | 12 ++++++------
services/keep-balance/metrics.go | 2 ++
5 files changed, 19 insertions(+), 7 deletions(-)
discards 8b8e888029c273da3558308b6d965a89fd33f89e (commit)
discards 9f6bbb975f5e2fa7a46266307318a0fc95349a29 (commit)
via faeb978ad985f983ec8dc18292524e0bd7bbda03 (commit)
via c64cf92414315c09771770e5d079bc4ad8328f52 (commit)
via d6a21ab8a0c5a45bdcd7c0fbaa93f7096e56e831 (commit)
via d2fb3f8daccf2362ac5f5fb91eae0255674b0818 (commit)
via 6f81d7266a7db4d1be83e79e5bf2e5afc49b2e3c (commit)
via 10fcb91c287fb91d1c63eaebb9a477a098034dfb (commit)
via e45c5654ed1122792b177b9aa86b71c0e840f9f3 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (8b8e888029c273da3558308b6d965a89fd33f89e)
\
N -- N -- N (faeb978ad985f983ec8dc18292524e0bd7bbda03)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit faeb978ad985f983ec8dc18292524e0bd7bbda03
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Oct 15 17:36:03 2018 -0400
14285: Update admin/metrics docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/admin/metrics.html.textile.liquid b/doc/admin/metrics.html.textile.liquid
index d553263d5..893eac1c8 100644
--- a/doc/admin/metrics.html.textile.liquid
+++ b/doc/admin/metrics.html.textile.liquid
@@ -158,6 +158,8 @@ table(table table-bordered table-condensed).
|arvados_keep_overreplicated_{replicas,blocks,bytes}|gauge|overreplicated data (more replicas exist than are needed)|
|arvados_keep_underreplicated_{replicas,blocks,bytes}|gauge|underreplicated data (fewer replicas exist than are needed)|
|arvados_keep_lost_{replicas,blocks,bytes}|gauge|lost data (referenced by collections, but not found on any backend volume)|
+|arvados_keep_dedup_block_ratio|gauge|deduplication ratio (block references in collections ÷ distinct blocks referenced)|
+|arvados_keep_dedup_byte_ratio|gauge|deduplication ratio (block references in collections ÷ distinct blocks referenced, weighted by block size)|
|arvados_keepbalance_get_state_seconds|summary|time to get all collections and keepstore volume indexes for one iteration|
|arvados_keepbalance_changeset_compute_seconds|summary|time to compute changesets for one iteration|
|arvados_keepbalance_send_pull_list_seconds|summary|time to send pull lists to all keepstore servers for one iteration|
commit c64cf92414315c09771770e5d079bc4ad8328f52
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Oct 15 17:02:22 2018 -0400
14285: Add deduplication ratio to keep-balance metrics.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 333a4fbde..5f5f9afdc 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -780,6 +780,26 @@ type balancerStats struct {
trashes int
replHistogram []int
classStats map[string]replicationStats
+
+ // collectionBytes / collectionBlockBytes = deduplication ratio
+ collectionBytes int64 // sum(bytes in referenced blocks) across all collections
+ collectionBlockBytes int64 // sum(block size) across all blocks referenced by collections
+ collectionBlockRefs int64 // sum(number of blocks referenced) across all collections
+ collectionBlocks int64 // number of blocks referenced by any collection
+}
+
+func (s *balancerStats) dedupByteRatio() float64 {
+ if s.collectionBlockBytes == 0 {
+ return 0
+ }
+ return float64(s.collectionBytes) / float64(s.collectionBlockBytes)
+}
+
+func (s *balancerStats) dedupBlockRatio() float64 {
+ if s.collectionBlocks == 0 {
+ return 0
+ }
+ return float64(s.collectionBlockRefs) / float64(s.collectionBlocks)
}
type replicationStats struct {
@@ -803,6 +823,13 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
surplus := result.have - result.want
bytes := result.blkid.Size()
+ if rc := int64(result.blk.RefCount); rc > 0 {
+ s.collectionBytes += rc * bytes
+ s.collectionBlockBytes += bytes
+ s.collectionBlockRefs += rc
+ s.collectionBlocks++
+ }
+
for class, state := range result.classState {
cs := s.classStats[class]
if state.unachievable {
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index e1b92005f..923ea47c4 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -184,7 +184,8 @@ func (s *stubServer) serveFooBarFileCollections() *reqTracker {
if strings.Contains(r.Form.Get("filters"), `modified_at`) {
io.WriteString(w, `{"items_available":0,"items":[]}`)
} else {
- io.WriteString(w, `{"items_available":2,"items":[
+ io.WriteString(w, `{"items_available":3,"items":[
+ {"uuid":"zzzzz-4zz18-aaaaaaaaaaaaaaa","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
{"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
{"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
}
@@ -463,6 +464,8 @@ func (s *runSuite) TestCommit(c *check.C) {
c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_count 1\n.*`)
+ c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_byte_ratio 1\.5\n.*`)
+ c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_dedup_block_ratio 1\.5\n.*`)
}
func (s *runSuite) TestRunForever(c *check.C) {
diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index 22e89c019..46e69059c 100644
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -23,6 +23,7 @@ type Replica struct {
// replicas actually stored (according to the keepstore indexes we
// know about).
type BlockState struct {
+ RefCount int
Replicas []Replica
Desired map[string]int
// TODO: Support combinations of classes ("private + durable")
@@ -42,6 +43,7 @@ func (bs *BlockState) addReplica(r Replica) {
}
func (bs *BlockState) increaseDesired(classes []string, n int) {
+ bs.RefCount++
if len(classes) == 0 {
classes = defaultClasses
}
diff --git a/services/keep-balance/metrics.go b/services/keep-balance/metrics.go
index 0564d46cd..5f3c98723 100644
--- a/services/keep-balance/metrics.go
+++ b/services/keep-balance/metrics.go
@@ -58,12 +58,14 @@ func (m *metrics) UpdateStats(s balancerStats) {
Help string
}
s2g := map[string]gauge{
- "total": {s.current, "current backend storage usage"},
- "garbage": {s.garbage, "garbage (unreferenced, old)"},
- "transient": {s.unref, "transient (unreferenced, new)"},
- "overreplicated": {s.overrep, "overreplicated"},
- "underreplicated": {s.underrep, "underreplicated"},
- "lost": {s.lost, "lost"},
+ "total": {s.current, "current backend storage usage"},
+ "garbage": {s.garbage, "garbage (unreferenced, old)"},
+ "transient": {s.unref, "transient (unreferenced, new)"},
+ "overreplicated": {s.overrep, "overreplicated"},
+ "underreplicated": {s.underrep, "underreplicated"},
+ "lost": {s.lost, "lost"},
+ "dedup_byte_ratio": {s.dedupByteRatio(), "deduplication ratio, bytes referenced / bytes stored"},
+ "dedup_block_ratio": {s.dedupBlockRatio(), "deduplication ratio, blocks referenced / blocks stored"},
}
m.setupOnce.Do(func() {
// Register gauge(s) for each balancerStats field.
@@ -83,7 +85,7 @@ func (m *metrics) UpdateStats(s balancerStats) {
for _, sub := range []string{"blocks", "bytes", "replicas"} {
addGauge(name+"_"+sub, sub+" of "+gauge.Help)
}
- case int, int64:
+ case int, int64, float64:
addGauge(name, gauge.Help)
default:
panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
@@ -101,6 +103,8 @@ func (m *metrics) UpdateStats(s balancerStats) {
m.statsGauges[name].Set(float64(val))
case int64:
m.statsGauges[name].Set(float64(val))
+ case float64:
+ m.statsGauges[name].Set(float64(val))
default:
panic(fmt.Sprintf("bad gauge type %T", gauge.Value))
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list