[ARVADOS] updated: 595f455a425d23e79789c2346a22a4a23520a417
Git user
git at public.curoverse.com
Wed May 25 10:35:58 EDT 2016
Summary of changes:
sdk/go/arvados/collection.go | 86 ---------------------------------
services/keep-balance/balance.go | 2 +-
services/keep-balance/collection.go | 95 +++++++++++++++++++++++++++++++++++++
3 files changed, 96 insertions(+), 87 deletions(-)
create mode 100644 services/keep-balance/collection.go
via 595f455a425d23e79789c2346a22a4a23520a417 (commit)
from 30c9c2e679fe43d3a7d5e8770496beca4cf87282 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 595f455a425d23e79789c2346a22a4a23520a417
Author: Tom Clegg <tom at curoverse.com>
Date: Wed May 25 10:35:39 2016 -0400
9162: Move EachCollection from SDK to keep-balance.
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index b247d9c..71f5247 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -60,89 +60,3 @@ type CollectionList struct {
Offset int `json:"offset"`
Limit int `json:"limit"`
}
-
-func (c *Client) countCollections(params ResourceListParams) (int, error) {
- var page CollectionList
- var zero int
- err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, ResourceListParams{Limit: &zero})
- return page.ItemsAvailable, err
-}
-
-// EachCollection calls f once for every readable
-// collection. EachCollection stops if it encounters an error, such as
-// f returning a non-nil error.
-//
-// The progress function is called periodically with done (number of
-// times f has been called) and total (number of times f is expected
-// to be called).
-func (c *Client) EachCollection(f func(Collection) error, progress func(done, total int)) error {
- if progress == nil {
- progress = func(_, _ int) {}
- }
-
- expectCount, err := c.countCollections(ResourceListParams{})
- if err != nil {
- return err
- }
-
- limit := 1000
- params := ResourceListParams{
- Limit: &limit,
- Order: "modified_at, uuid",
- Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
- }
- var last Collection
- var filterTime time.Time
- callCount := 0
- for {
- progress(callCount, expectCount)
- var page CollectionList
- err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
- if err != nil {
- return err
- }
- for _, coll := range page.Items {
- if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
- continue
- }
- callCount++
- err = f(coll)
- if err != nil {
- return err
- }
- last = coll
- }
- if last.ModifiedAt == nil || *last.ModifiedAt == filterTime {
- if page.ItemsAvailable > len(page.Items) {
- // TODO: use "mtime=X && UUID>Y"
- // filters to get all collections with
- // this timestamp, then use "mtime>X"
- // to get the next timestamp.
- return fmt.Errorf("BUG: Received an entire page with the same modified_at timestamp (%v), cannot make progress", filterTime)
- }
- break
- }
- filterTime = *last.ModifiedAt
- params.Filters = []Filter{{
- Attr: "modified_at",
- Operator: ">=",
- Operand: filterTime,
- }, {
- Attr: "uuid",
- Operator: "!=",
- Operand: last.UUID,
- }}
- }
- progress(callCount, expectCount)
-
- if checkCount, err := c.countCollections(ResourceListParams{Filters: []Filter{{
- Attr: "modified_at",
- Operator: ">=",
- Operand: filterTime}}}); err != nil {
- return err
- } else if callCount < checkCount {
- return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, expectCount)
- }
-
- return nil
-}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 121dd7a..d077123 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -230,7 +230,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
wg.Add(1)
go func() {
defer wg.Done()
- err = c.EachCollection(
+ err = EachCollection(c,
func(coll arvados.Collection) error {
collQ <- coll
if len(errs) > 0 {
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
new file mode 100644
index 0000000..782afe1
--- /dev/null
+++ b/services/keep-balance/collection.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+ "fmt"
+ "time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int, error) {
+ var page arvados.CollectionList
+ var zero int
+ err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil,
+ arvados.ResourceListParams{Limit: &zero})
+ return page.ItemsAvailable, err
+}
+
+// EachCollection calls f once for every readable
+// collection. EachCollection stops if it encounters an error, such as
+// f returning a non-nil error.
+//
+// The progress function is called periodically with done (number of
+// times f has been called) and total (number of times f is expected
+// to be called).
+func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progress func(done, total int)) error {
+ if progress == nil {
+ progress = func(_, _ int) {}
+ }
+
+ expectCount, err := countCollections(c, arvados.ResourceListParams{})
+ if err != nil {
+ return err
+ }
+
+ limit := 1000
+ params := arvados.ResourceListParams{
+ Limit: &limit,
+ Order: "modified_at, uuid",
+ Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
+ }
+ var last arvados.Collection
+ var filterTime time.Time
+ callCount := 0
+ for {
+ progress(callCount, expectCount)
+ var page arvados.CollectionList
+ err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
+ if err != nil {
+ return err
+ }
+ for _, coll := range page.Items {
+ if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
+ continue
+ }
+ callCount++
+ err = f(coll)
+ if err != nil {
+ return err
+ }
+ last = coll
+ }
+ if last.ModifiedAt == nil || *last.ModifiedAt == filterTime {
+ if page.ItemsAvailable > len(page.Items) {
+ // TODO: use "mtime=X && UUID>Y"
+ // filters to get all collections with
+ // this timestamp, then use "mtime>X"
+ // to get the next timestamp.
+ return fmt.Errorf("BUG: Received an entire page with the same modified_at timestamp (%v), cannot make progress", filterTime)
+ }
+ break
+ }
+ filterTime = *last.ModifiedAt
+ params.Filters = []arvados.Filter{{
+ Attr: "modified_at",
+ Operator: ">=",
+ Operand: filterTime,
+ }, {
+ Attr: "uuid",
+ Operator: "!=",
+ Operand: last.UUID,
+ }}
+ }
+ progress(callCount, expectCount)
+
+ if checkCount, err := countCollections(c, arvados.ResourceListParams{Filters: []arvados.Filter{{
+ Attr: "modified_at",
+ Operator: ">=",
+ Operand: filterTime}}}); err != nil {
+ return err
+ } else if callCount < checkCount {
+ return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, expectCount)
+ }
+
+ return nil
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list