[ARVADOS] created: 177b3b66c3e876a5e1665e6067309fb15cabdf83
Git user
git at public.curoverse.com
Mon Jun 13 10:32:40 EDT 2016
at 177b3b66c3e876a5e1665e6067309fb15cabdf83 (commit)
commit 177b3b66c3e876a5e1665e6067309fb15cabdf83
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 13 10:31:39 2016 -0400
9395: Add CollectionBuffers config to keep-balance.
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 0ca6e71..b53d1fd 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -80,7 +80,7 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (err error) {
return
}
}
- if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize); err != nil {
+ if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize, config.CollectionBuffers); err != nil {
return
}
bal.ComputeChangeSets()
@@ -190,7 +190,7 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
// collection manifests in the database (API server).
//
// It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
+func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
defer timeMe(bal.Logger, "GetCurrentState")()
bal.BlockStateMap = NewBlockStateMap()
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 6e7d70d..364bb3f 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -31,6 +31,11 @@ type Config struct {
// Number of collections to request in each API call
CollectionBatchSize int
+
+ // Max collections to buffer in memory (bigger values consume
+ // more memory, but can reduce store-and-forward latency when
+ // fetching pages)
+ CollectionBuffers int
}
// RunOptions controls runtime behavior. The flags/options that belong
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index c536364..2273f3a 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -17,7 +17,8 @@ var exampleConfigFile = []byte(`
"disk"
],
"RunPeriod": "600s",
- "CollectionBatchSize": 100000
+ "CollectionBatchSize": 100000,
+ "CollectionBuffers": 1000
}`)
func usage() {
commit 2630fc7ca045639b3d1b83fec9fb0552b22a41a6
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jun 13 10:30:23 2016 -0400
9395: Add CollectionBatchSize config to keep-balance.
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 2a2480c..0ca6e71 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -80,7 +80,7 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (err error) {
return
}
}
- if err = bal.GetCurrentState(&config.Client); err != nil {
+ if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize); err != nil {
return
}
bal.ComputeChangeSets()
@@ -190,7 +190,7 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
// collection manifests in the database (API server).
//
// It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
+func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
defer timeMe(bal.Logger, "GetCurrentState")()
bal.BlockStateMap = NewBlockStateMap()
@@ -224,9 +224,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
// collQ buffers incoming collections so we can start fetching
// the next page without waiting for the current page to
- // finish processing. (1000 happens to match the page size
- // used by (*arvados.Client)EachCollection(), but it's OK if
- // they don't match.)
+ // finish processing.
collQ := make(chan arvados.Collection, 1000)
// Start a goroutine to process collections. (We could use a
@@ -252,7 +250,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
wg.Add(1)
go func() {
defer wg.Done()
- err = EachCollection(c,
+ err = EachCollection(c, pageSize,
func(coll arvados.Collection) error {
collQ <- coll
if len(errs) > 0 {
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index e6a1f08..f4fc721 100644
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -22,7 +22,10 @@ func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int
// The progress function is called periodically with done (number of
// times f has been called) and total (number of times f is expected
// to be called).
-func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progress func(done, total int)) error {
+//
+// If pageSize > 0 it is used as the maximum page size in each API
+// call; otherwise the maximum allowed page size is requested.
+func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
if progress == nil {
progress = func(_, _ int) {}
}
@@ -32,7 +35,11 @@ func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progres
return err
}
- limit := 1000
+ limit := pageSize
+ if limit <= 0 {
+ // Use the maximum page size the server allows
+ limit = 1<<31 - 1
+ }
params := arvados.ResourceListParams{
Limit: &limit,
Order: "modified_at, uuid",
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 42a8d63..6e7d70d 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -28,6 +28,9 @@ type Config struct {
// How often to check
RunPeriod arvados.Duration
+
+ // Number of collections to request in each API call
+ CollectionBatchSize int
}
// RunOptions controls runtime behavior. The flags/options that belong
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index eb9990c..c536364 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -16,7 +16,8 @@ var exampleConfigFile = []byte(`
"KeepServiceTypes": [
"disk"
],
- "RunPeriod": "600s"
+ "RunPeriod": "600s",
+ "CollectionBatchSize": 100000
}`)
func usage() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list