[ARVADOS] created: 177b3b66c3e876a5e1665e6067309fb15cabdf83

Git user git at public.curoverse.com
Mon Jun 13 10:32:40 EDT 2016


        at  177b3b66c3e876a5e1665e6067309fb15cabdf83 (commit)


commit 177b3b66c3e876a5e1665e6067309fb15cabdf83
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 13 10:31:39 2016 -0400

    9395: Add CollectionBuffers config to keep-balance.

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 0ca6e71..b53d1fd 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -80,7 +80,7 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (err error) {
 			return
 		}
 	}
-	if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize); err != nil {
+	if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize, config.CollectionBuffers); err != nil {
 		return
 	}
 	bal.ComputeChangeSets()
@@ -190,7 +190,7 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
 // collection manifests in the database (API server).
 //
 // It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
+func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
 	defer timeMe(bal.Logger, "GetCurrentState")()
 	bal.BlockStateMap = NewBlockStateMap()
 
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 6e7d70d..364bb3f 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -31,6 +31,11 @@ type Config struct {
 
 	// Number of collections to request in each API call
 	CollectionBatchSize int
+
+	// Max collections to buffer in memory (bigger values consume
+	// more memory, but can reduce store-and-forward latency when
+	// fetching pages)
+	CollectionBuffers int
 }
 
 // RunOptions controls runtime behavior. The flags/options that belong
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index c536364..2273f3a 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -17,7 +17,8 @@ var exampleConfigFile = []byte(`
 	    "disk"
 	],
 	"RunPeriod": "600s",
-	"CollectionBatchSize": 100000
+	"CollectionBatchSize": 100000,
+	"CollectionBuffers": 1000
     }`)
 
 func usage() {

commit 2630fc7ca045639b3d1b83fec9fb0552b22a41a6
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Jun 13 10:30:23 2016 -0400

    9395: Add CollectionBatchSize config to keep-balance.

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 2a2480c..0ca6e71 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -80,7 +80,7 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (err error) {
 			return
 		}
 	}
-	if err = bal.GetCurrentState(&config.Client); err != nil {
+	if err = bal.GetCurrentState(&config.Client, config.CollectionBatchSize); err != nil {
 		return
 	}
 	bal.ComputeChangeSets()
@@ -190,7 +190,7 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
 // collection manifests in the database (API server).
 //
 // It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
+func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize int) error {
 	defer timeMe(bal.Logger, "GetCurrentState")()
 	bal.BlockStateMap = NewBlockStateMap()
 
@@ -224,9 +224,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 
 	// collQ buffers incoming collections so we can start fetching
 	// the next page without waiting for the current page to
-	// finish processing. (1000 happens to match the page size
-	// used by (*arvados.Client)EachCollection(), but it's OK if
-	// they don't match.)
+	// finish processing.
 	collQ := make(chan arvados.Collection, 1000)
 
 	// Start a goroutine to process collections. (We could use a
@@ -252,7 +250,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		err = EachCollection(c,
+		err = EachCollection(c, pageSize,
 			func(coll arvados.Collection) error {
 				collQ <- coll
 				if len(errs) > 0 {
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index e6a1f08..f4fc721 100644
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -22,7 +22,10 @@ func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int
 // The progress function is called periodically with done (number of
 // times f has been called) and total (number of times f is expected
 // to be called).
-func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progress func(done, total int)) error {
+//
+// If pageSize > 0 it is used as the maximum page size in each API
+// call; otherwise the maximum allowed page size is requested.
+func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
 	if progress == nil {
 		progress = func(_, _ int) {}
 	}
@@ -32,7 +35,11 @@ func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progres
 		return err
 	}
 
-	limit := 1000
+	limit := pageSize
+	if limit <= 0 {
+		// Use the maximum page size the server allows
+		limit = 1<<31 - 1
+	}
 	params := arvados.ResourceListParams{
 		Limit:  &limit,
 		Order:  "modified_at, uuid",
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 42a8d63..6e7d70d 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -28,6 +28,9 @@ type Config struct {
 
 	// How often to check
 	RunPeriod arvados.Duration
+
+	// Number of collections to request in each API call
+	CollectionBatchSize int
 }
 
 // RunOptions controls runtime behavior. The flags/options that belong
diff --git a/services/keep-balance/usage.go b/services/keep-balance/usage.go
index eb9990c..c536364 100644
--- a/services/keep-balance/usage.go
+++ b/services/keep-balance/usage.go
@@ -16,7 +16,8 @@ var exampleConfigFile = []byte(`
 	"KeepServiceTypes": [
 	    "disk"
 	],
-	"RunPeriod": "600s"
+	"RunPeriod": "600s",
+	"CollectionBatchSize": 100000
     }`)
 
 func usage() {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list