[ARVADOS] updated: 2e3dfa6394ea17ab266b4c6e2346a469c63efb74
Git user
git at public.curoverse.com
Wed May 25 16:48:49 EDT 2016
Summary of changes:
services/keep-balance/balance_run_test.go | 39 +++++++++++++++++++++++++++++++
services/keep-balance/balance_test.go | 6 -----
services/keep-balance/collection.go | 8 +++----
services/keep-balance/main_test.go | 20 ++++++++++++++++
services/keep-balance/usage.go | 29 ++++++++++++-----------
5 files changed, 79 insertions(+), 23 deletions(-)
create mode 100644 services/keep-balance/main_test.go
via 2e3dfa6394ea17ab266b4c6e2346a469c63efb74 (commit)
via a43da81e3e7453bb73d69cca74b82f73a69286eb (commit)
from 1c470ee799e6b085929155a3942d2a3d47276954 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2e3dfa6394ea17ab266b4c6e2346a469c63efb74
Author: Tom Clegg <tom at curoverse.com>
Date: Wed May 25 16:48:38 2016 -0400
9162: Fix & test "count collections after fetching" sanity check.
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 9b7ea67..4b04f8a 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -118,6 +118,7 @@ func (s *stubServer) serveZeroCollections() *reqTracker {
func (s *stubServer) serveFooBarFileCollections() *reqTracker {
rt := &reqTracker{}
s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
+ r.ParseForm()
rt.Add(r)
if strings.Contains(r.Form.Get("filters"), `modified_at`) {
io.WriteString(w, `{"items_available":0,"items":[]}`)
@@ -130,6 +131,24 @@ func (s *stubServer) serveFooBarFileCollections() *reqTracker {
return rt
}
+func (s *stubServer) serveCollectionsButSkipOne() *reqTracker {
+ rt := &reqTracker{}
+ s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
+ r.ParseForm()
+ rt.Add(r)
+ if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003c="`) {
+ io.WriteString(w, `{"items_available":3,"items":[]}`)
+ } else if strings.Contains(r.Form.Get("filters"), `"modified_at","\u003e="`) {
+ io.WriteString(w, `{"items_available":0,"items":[]}`)
+ } else {
+ io.WriteString(w, `{"items_available":2,"items":[
+ {"uuid":"zzzzz-4zz18-ehbhgtheo8909or","portable_data_hash":"fa7aeb5140e2848d39b416daeef4ffc5+45","manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n","modified_at":"2014-02-03T17:22:54Z"},
+ {"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}]}`)
+ }
+ })
+ return rt
+}
+
func (s *stubServer) serveZeroKeepServices() *reqTracker {
return s.serveStatic("/arvados/v1/keep_services",
`{"items":[],"items_available":0}`)
@@ -258,6 +277,24 @@ func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
c.Check(pullReqs.Count(), check.Equals, 0)
}
+func (s *runSuite) TestDetectSkippedCollections(c *check.C) {
+ opts := RunOptions{
+ CommitPulls: true,
+ CommitTrash: true,
+ Logger: s.logger(c),
+ }
+ s.stub.serveCurrentUserAdmin()
+ s.stub.serveCollectionsButSkipOne()
+ s.stub.serveFourDiskKeepServices()
+ s.stub.serveKeepstoreIndexFoo4Bar1()
+ trashReqs := s.stub.serveKeepstoreTrash()
+ pullReqs := s.stub.serveKeepstorePull()
+ err := (&Balancer{}).Run(s.config, opts)
+ c.Check(err, check.ErrorMatches, `Retrieved 2 collections with modtime <= .* but server now reports there are 3 collections.*`)
+ c.Check(trashReqs.Count(), check.Equals, 4)
+ c.Check(pullReqs.Count(), check.Equals, 0)
+}
+
func (s *runSuite) TestDryRun(c *check.C) {
opts := RunOptions{
CommitPulls: false,
@@ -277,6 +314,8 @@ func (s *runSuite) TestDryRun(c *check.C) {
c.Check(pullReqs.Count(), check.Equals, 0)
stats := bal.getStatistics()
c.Check(stats.pulls, check.Not(check.Equals), 0)
+ c.Check(stats.underrep.replicas, check.Not(check.Equals), 0)
+ c.Check(stats.overrep.replicas, check.Not(check.Equals), 0)
}
func (s *runSuite) TestCommit(c *check.C) {
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index 782afe1..e6a1f08 100644
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -10,8 +10,8 @@ import (
func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int, error) {
var page arvados.CollectionList
var zero int
- err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil,
- arvados.ResourceListParams{Limit: &zero})
+ params.Limit = &zero
+ err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
return page.ItemsAvailable, err
}
@@ -84,11 +84,11 @@ func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progres
if checkCount, err := countCollections(c, arvados.ResourceListParams{Filters: []arvados.Filter{{
Attr: "modified_at",
- Operator: ">=",
+ Operator: "<=",
Operand: filterTime}}}); err != nil {
return err
} else if callCount < checkCount {
- return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, expectCount)
+ return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, checkCount)
}
return nil
commit a43da81e3e7453bb73d69cca74b82f73a69286eb
Author: Tom Clegg <tom at curoverse.com>
Date: Wed May 25 16:18:40 2016 -0400
9162: Test example config. Remove dead code.
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 13c8081..dad10b0 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -44,12 +44,6 @@ type tester struct {
shouldTrash slots
}
-type serviceByUUID []*KeepService
-
-func (s serviceByUUID) Len() int { return len(s) }
-func (s serviceByUUID) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-func (s serviceByUUID) Less(i, j int) bool { return s[i].UUID < s[j].UUID }
-
func (bal *balancerSuite) SetUpSuite(c *check.C) {
bal.knownRendezvous = nil
for _, str := range []string{
diff --git a/services/keep-balance/main_test.go b/services/keep-balance/main_test.go
new file mode 100644
index 0000000..5e1619a
--- /dev/null
+++ b/services/keep-balance/main_test.go
@@ -0,0 +1,20 @@
+package main
+
+import (
+ "encoding/json"
+ "time"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&mainSuite{})
+
+type mainSuite struct {}
+
+func (s *mainSuite) TestExampleJSON(c *check.C) {
+ var config Config
+ c.Check(json.Unmarshal(exampleConfigFile, &config), check.IsNil)
+ c.Check(config.KeepServiceTypes, check.DeepEquals, []string{"disk"})
+ c.Check(config.Client.AuthToken, check.Equals, "xyzzy")
+ c.Check(time.Duration(config.RunPeriod), check.Equals, 600 * time.Second)
+}
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index fab7d09..8361479 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -6,6 +6,19 @@ import (
"os"
)
+var exampleConfigFile = []byte(`
+ {
+ "Client": {
+ "APIHost": "zzzzz.arvadosapi.com:443",
+ "AuthToken": "xyzzy",
+ "Insecure": false
+ },
+ "KeepServiceTypes": [
+ "disk"
+ ],
+ "RunPeriod": "600s"
+ }`)
+
func usage() {
fmt.Fprintf(os.Stderr, `
keep-balance copies blocks to better positions so clients find them
@@ -19,18 +32,7 @@ Options:
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, `
Example config file:
-
- {
- "Client": {
- "APIHost": "zzzzz.arvadosapi.com:443",
- "AuthToken": "xyzzy",
- "Insecure": false
- },
- "KeepServiceTypes": [
- "disk"
- ],
- "RunPeriod": "600s"
- }
+%s
Client.AuthToken must be recognized by Arvados as an admin token,
and must be recognized by all Keep services as a "data manager
@@ -63,4 +65,5 @@ Committing:
Use the -commit-pull and -commit-trash flags to implement the
computed changes.
-`) }
+`, exampleConfigFile)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list