[ARVADOS] updated: 595f455a425d23e79789c2346a22a4a23520a417

Git user git at public.curoverse.com
Wed May 25 10:35:58 EDT 2016


Summary of changes:
 sdk/go/arvados/collection.go        | 86 ---------------------------------
 services/keep-balance/balance.go    |  2 +-
 services/keep-balance/collection.go | 95 +++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 87 deletions(-)
 create mode 100644 services/keep-balance/collection.go

       via  595f455a425d23e79789c2346a22a4a23520a417 (commit)
      from  30c9c2e679fe43d3a7d5e8770496beca4cf87282 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 595f455a425d23e79789c2346a22a4a23520a417
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed May 25 10:35:39 2016 -0400

    9162: Move EachCollection from SDK to keep-balance.

diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index b247d9c..71f5247 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -60,89 +60,3 @@ type CollectionList struct {
 	Offset         int          `json:"offset"`
 	Limit          int          `json:"limit"`
 }
-
-func (c *Client) countCollections(params ResourceListParams) (int, error) {
-	var page CollectionList
-	var zero int
-	err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, ResourceListParams{Limit: &zero})
-	return page.ItemsAvailable, err
-}
-
-// EachCollection calls f once for every readable
-// collection. EachCollection stops if it encounters an error, such as
-// f returning a non-nil error.
-//
-// The progress function is called periodically with done (number of
-// times f has been called) and total (number of times f is expected
-// to be called).
-func (c *Client) EachCollection(f func(Collection) error, progress func(done, total int)) error {
-	if progress == nil {
-		progress = func(_, _ int) {}
-	}
-
-	expectCount, err := c.countCollections(ResourceListParams{})
-	if err != nil {
-		return err
-	}
-
-	limit := 1000
-	params := ResourceListParams{
-		Limit:  &limit,
-		Order:  "modified_at, uuid",
-		Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
-	}
-	var last Collection
-	var filterTime time.Time
-	callCount := 0
-	for {
-		progress(callCount, expectCount)
-		var page CollectionList
-		err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
-		if err != nil {
-			return err
-		}
-		for _, coll := range page.Items {
-			if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
-				continue
-			}
-			callCount++
-			err = f(coll)
-			if err != nil {
-				return err
-			}
-			last = coll
-		}
-		if last.ModifiedAt == nil || *last.ModifiedAt == filterTime {
-			if page.ItemsAvailable > len(page.Items) {
-				// TODO: use "mtime=X && UUID>Y"
-				// filters to get all collections with
-				// this timestamp, then use "mtime>X"
-				// to get the next timestamp.
-				return fmt.Errorf("BUG: Received an entire page with the same modified_at timestamp (%v), cannot make progress", filterTime)
-			}
-			break
-		}
-		filterTime = *last.ModifiedAt
-		params.Filters = []Filter{{
-			Attr:     "modified_at",
-			Operator: ">=",
-			Operand:  filterTime,
-		}, {
-			Attr:     "uuid",
-			Operator: "!=",
-			Operand:  last.UUID,
-		}}
-	}
-	progress(callCount, expectCount)
-
-	if checkCount, err := c.countCollections(ResourceListParams{Filters: []Filter{{
-		Attr:     "modified_at",
-		Operator: ">=",
-		Operand:  filterTime}}}); err != nil {
-		return err
-	} else if callCount < checkCount {
-		return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, expectCount)
-	}
-
-	return nil
-}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 121dd7a..d077123 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -230,7 +230,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client) error {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		err = c.EachCollection(
+		err = EachCollection(c,
 			func(coll arvados.Collection) error {
 				collQ <- coll
 				if len(errs) > 0 {
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
new file mode 100644
index 0000000..782afe1
--- /dev/null
+++ b/services/keep-balance/collection.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int, error) {
+	var page arvados.CollectionList
+	var zero int
+	err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil,
+		arvados.ResourceListParams{Limit: &zero})
+	return page.ItemsAvailable, err
+}
+
+// EachCollection calls f once for every readable
+// collection. EachCollection stops if it encounters an error, such as
+// f returning a non-nil error.
+//
+// The progress function is called periodically with done (number of
+// times f has been called) and total (number of times f is expected
+// to be called).
+func EachCollection(c *arvados.Client, f func(arvados.Collection) error, progress func(done, total int)) error {
+	if progress == nil {
+		progress = func(_, _ int) {}
+	}
+
+	expectCount, err := countCollections(c, arvados.ResourceListParams{})
+	if err != nil {
+		return err
+	}
+
+	limit := 1000
+	params := arvados.ResourceListParams{
+		Limit:  &limit,
+		Order:  "modified_at, uuid",
+		Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
+	}
+	var last arvados.Collection
+	var filterTime time.Time
+	callCount := 0
+	for {
+		progress(callCount, expectCount)
+		var page arvados.CollectionList
+		err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
+		if err != nil {
+			return err
+		}
+		for _, coll := range page.Items {
+			if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
+				continue
+			}
+			callCount++
+			err = f(coll)
+			if err != nil {
+				return err
+			}
+			last = coll
+		}
+		if last.ModifiedAt == nil || *last.ModifiedAt == filterTime {
+			if page.ItemsAvailable > len(page.Items) {
+				// TODO: use "mtime=X && UUID>Y"
+				// filters to get all collections with
+				// this timestamp, then use "mtime>X"
+				// to get the next timestamp.
+				return fmt.Errorf("BUG: Received an entire page with the same modified_at timestamp (%v), cannot make progress", filterTime)
+			}
+			break
+		}
+		filterTime = *last.ModifiedAt
+		params.Filters = []arvados.Filter{{
+			Attr:     "modified_at",
+			Operator: ">=",
+			Operand:  filterTime,
+		}, {
+			Attr:     "uuid",
+			Operator: "!=",
+			Operand:  last.UUID,
+		}}
+	}
+	progress(callCount, expectCount)
+
+	if checkCount, err := countCollections(c, arvados.ResourceListParams{Filters: []arvados.Filter{{
+		Attr:     "modified_at",
+		Operator: ">=",
+		Operand:  filterTime}}}); err != nil {
+		return err
+	} else if callCount < checkCount {
+		return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, expectCount)
+	}
+
+	return nil
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list