[ARVADOS] updated: cdfc5f568046d2f39861cea65b62682bc9ddbce1
Git user
git at public.curoverse.com
Wed May 18 00:13:04 EDT 2016
Summary of changes:
sdk/go/x/arvados/collection.go | 2 +-
services/keep-balance/balance.go | 4 +
services/keep-balance/client.go | 136 ------------------------------
services/keep-balance/integration_test.go | 6 +-
services/keep-balance/main.go | 17 ++--
services/keep-balance/main_test.go | 67 ++++++++++++---
6 files changed, 72 insertions(+), 160 deletions(-)
delete mode 100644 services/keep-balance/client.go
via cdfc5f568046d2f39861cea65b62682bc9ddbce1 (commit)
from 6251cb2459021711d9d9c0c8aee47149b4ed336c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit cdfc5f568046d2f39861cea65b62682bc9ddbce1
Author: Tom Clegg <tom at curoverse.com>
Date: Wed May 18 00:13:01 2016 -0400
9162: Fix dry-run test by giving it something to not do.
diff --git a/sdk/go/x/arvados/collection.go b/sdk/go/x/arvados/collection.go
index e5b0922..97b5e09 100644
--- a/sdk/go/x/arvados/collection.go
+++ b/sdk/go/x/arvados/collection.go
@@ -103,7 +103,7 @@ func (c *Client) EachCollection(f func(Collection) error, progress func(done, to
params := ResourceListParams{
Limit: &limit,
Order: "modified_at, uuid",
- Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash"},
+ Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
}
var last Collection
var filterTime time.Time
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index b86bbcd..957f37f 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -92,6 +92,10 @@ func (bal *Balancer) CheckSanityLate() error {
return fmt.Errorf("zero blocks have desired replication>0")
}
+ if dr := bal.DefaultReplication; dr < 1 {
+ return fmt.Errorf("Default replication (%d) is less than 1", dr)
+ }
+
// TODO: no two services have identical indexes
// TODO: no collisions (same md5, different size)
return nil
diff --git a/services/keep-balance/client.go b/services/keep-balance/client.go
deleted file mode 100644
index aabaf93..0000000
--- a/services/keep-balance/client.go
+++ /dev/null
@@ -1,136 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "net/url"
- "os"
-)
-
-type Client struct {
- *http.Client
- APIHost string
- AuthToken string
- Insecure bool
-}
-
-func NewClientFromEnv() *Client {
- return &Client{
- APIHost: os.Getenv("ARVADOS_API_HOST"),
- AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
- Insecure: os.Getenv("ARVADOS_API_HOST_INSECURE") != "",
- }
-}
-
-// Do adds authentication headers and then calls (*http.Client)Do().
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
- if c.AuthToken != "" {
- req.Header.Add("Authorization", "OAuth2 "+c.AuthToken)
- }
- debugf("%#v", req)
- return c.httpClient().Do(req)
-}
-
-// DoAndDecode performs req and unmarshals the response (which must be
-// JSON) into dst. Use this instead of RequestAndDecode if you need
-// more control of the http.Request object.
-func (c *Client) DoAndDecode(dst interface{}, req *http.Request) error {
- resp, err := c.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- buf, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return err
- }
- if resp.StatusCode != 200 {
- return fmt.Errorf("request failed (%s): %s", req.URL, resp.Status)
- }
- if dst == nil {
- return nil
- }
- return json.Unmarshal(buf, dst)
-}
-
-// RequestAndDecode performs an API request and unmarshals the
-// response (which must be JSON) into dst. Method and body arguments
-// are the same as for http.NewRequest(). The given path is added to
-// the server's scheme/host/port to form the request URL. The given
-// params are passed via POST form or query string.
-//
-// path must not contain a query string.
-func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
- urlString := c.apiURL(path)
- var urlValues url.Values
- if v, ok := params.(url.Values); ok {
- urlValues = v
- } else if params != nil {
- // Convert an arbitrary struct to url.Values. For
- // example, Foo{Bar: []int{1,2,3}, Baz: "waz"} becomes
- // url.Values{`bar`:`{"a":[1,2,3]}`,`Baz`:`waz`}
- //
- // TODO: Do this more efficiently, possibly using
- // json.Decode/Encode, so the whole thing doesn't have
- // to get encoded, decoded, and re-encoded.
- j, err := json.Marshal(params)
- if err != nil {
- return err
- }
- var generic map[string]interface{}
- err = json.Unmarshal(j, &generic)
- if err != nil {
- return err
- }
- urlValues = url.Values{}
- for k, v := range generic {
- if v, ok := v.(string); ok {
- urlValues.Set(k, v)
- continue
- }
- j, err := json.Marshal(v)
- if err != nil {
- return err
- }
- urlValues.Set(k, string(j))
- }
- }
- if (method == "GET" || body != nil) && urlValues != nil {
- // FIXME: what if params don't fit in URL
- u, err := url.Parse(urlString)
- if err != nil {
- return err
- }
- u.RawQuery = urlValues.Encode()
- urlString = u.String()
- }
- req, err := http.NewRequest(method, urlString, body)
- if err != nil {
- return err
- }
- return c.DoAndDecode(dst, req)
-}
-
-func (c *Client) httpClient() *http.Client {
- if c.Client == nil {
- return http.DefaultClient
- }
- return c.Client
-}
-
-func (c *Client) apiURL(path string) string {
- return "https://" + c.APIHost + "/" + path
-}
-
-type DiscoveryDocument struct {
- DefaultCollectionReplication int `json:"defaultCollectionReplication"`
- BlobSignatureTTL int64 `json:"blobSignatureTtl"`
-}
-
-func (c *Client) DiscoveryDocument() (*DiscoveryDocument, error) {
- var dd DiscoveryDocument
- return &dd, c.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
-}
diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index e6365a0..a460f56 100644
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -6,6 +6,7 @@ import (
"net/http"
"os"
"strings"
+ "testing"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
@@ -24,6 +25,9 @@ type integrationSuite struct {
}
func (s *integrationSuite) SetUpSuite(c *check.C) {
+ if testing.Short() {
+ c.Skip("-short")
+ }
arvadostest.ResetEnv()
arvadostest.StartAPI()
arvadostest.StartKeep(4, true)
@@ -71,7 +75,7 @@ func (s *integrationSuite) TestBalanceAPIFixtures(c *check.C) {
CommitTrash: true,
Logger: log.New(&logBuf, "", log.LstdFlags),
}
- err := Run(s.config, opts)
+ _, err := Run(s.config, opts)
c.Check(err, check.IsNil)
if iter == 0 {
c.Check(logBuf.String(), check.Matches, `(?ms).*ChangeSet{Pulls:1.*`)
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 23dfaa2..615c47d 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -51,41 +51,40 @@ func main() {
log.Printf("config is %s", j)
}
}
- if err := Run(config, runOptions); err != nil {
+ if _, err := Run(config, runOptions); err != nil {
log.Fatal(err)
}
}
-func Run(config Config, runOptions RunOptions) error {
+func Run(config Config, runOptions RunOptions) (bal *Balancer, err error) {
if runOptions.Logger == nil {
runOptions.Logger = log.New(os.Stderr, "", log.LstdFlags)
}
- bal := Balancer{
+ bal = &Balancer{
Logger: runOptions.Logger,
ServiceTypes: config.ServiceTypes,
}
- var err error
if err = bal.CheckSanityEarly(&config.Client); err != nil {
- return err
+ return
}
if err = bal.GetCurrentState(&config.Client); err != nil {
- return err
+ return
}
bal.ComputeChangeSets()
bal.PrintStatistics()
if err = bal.CheckSanityLate(); err != nil {
- return err
+ return
}
if runOptions.CommitPulls {
err = bal.CommitPulls(&config.Client)
if err != nil {
// Skip trash if we can't pull. (Too cautious?)
- return err
+ return
}
}
if runOptions.CommitTrash {
err = bal.CommitTrash(&config.Client)
}
- return err
+ return
}
diff --git a/services/keep-balance/main_test.go b/services/keep-balance/main_test.go
index 73f92c8..fcb0d35 100644
--- a/services/keep-balance/main_test.go
+++ b/services/keep-balance/main_test.go
@@ -8,7 +8,9 @@ import (
"log"
"net/http"
"net/http/httptest"
+ "strings"
"sync"
+ "sync/atomic"
"git.curoverse.com/arvados.git/sdk/go/x/arvados"
@@ -33,7 +35,6 @@ func (s *stubServer) start() *http.Client {
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.mutex.Lock()
s.Requests = append(s.Requests, *r)
- s.logf("%+v", r)
s.mutex.Unlock()
w.Header().Set("Content-Type", "application/json")
s.mux.ServeHTTP(w, r)
@@ -59,7 +60,7 @@ func (s *stubServer) serveStatic(path, data string) *int {
var count int
s.mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
count++
- fmt.Fprintln(w, data)
+ io.WriteString(w, data)
})
return &count
}
@@ -76,7 +77,7 @@ func (s *stubServer) serveCurrentUserNotAdmin() *int {
func (s *stubServer) serveDiscoveryDoc() *int {
return s.serveStatic("/discovery/v1/apis/arvados/v1/rest",
- `{"default_collection_replication":2}`)
+ `{"defaultCollectionReplication":2}`)
}
func (s *stubServer) serveZeroCollections() *int {
@@ -84,11 +85,43 @@ func (s *stubServer) serveZeroCollections() *int {
`{"items":[],"items_available":0}`)
}
+func (s *stubServer) serveFooFileCollection() *int {
+ var count int
+ s.mux.HandleFunc("/arvados/v1/collections", func(w http.ResponseWriter, r *http.Request) {
+ count++
+ if strings.Contains(r.Form.Get("filters"), `modified_at`) {
+ io.WriteString(w, `{"items":[],"items_available":0}`)
+ } else {
+ io.WriteString(w, `{"items":[{"uuid":"zzzzz-4zz18-znfnqtbbv4spc3w","portable_data_hash":"1f4b0bc7583c2a7f9102c395f4ffc5e3+45","manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n","modified_at":"2014-02-03T17:22:54Z"}],"items_available":1}`)
+ }
+ })
+ return &count
+}
+
func (s *stubServer) serveZeroKeepServices() *int {
return s.serveStatic("/arvados/v1/keep_services",
`{"items":[],"items_available":0}`)
}
+func (s *stubServer) serveFourKeepServices() *int {
+ return s.serveStatic("/arvados/v1/keep_services",
+ `{"items":[
+ {"uuid":"zzzzz-bi6l4-6zhilxar6r8ey90","service_host":"keep0.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"},
+ {"uuid":"zzzzz-bi6l4-rsnj3c76ndxb7o0","service_host":"keep1.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"},
+ {"uuid":"zzzzz-bi6l4-6zhilxar6r8ey91","service_host":"keep2.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"},
+ {"uuid":"zzzzz-bi6l4-rsnj3c76ndxb7o1","service_host":"keep3.zzzzz.arvadosapi.com","service_port":25107,"service_ssl_flag":false,"service_type":"disk"}
+ ],"items_available":4}`)
+}
+
+func (s *stubServer) serveKeepstoreIndexFooBlock() *int64 {
+ var count int64
+ s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
+ count := atomic.AddInt64(&count, 1)
+ fmt.Fprintf(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 %d\n\n", 12345678+count)
+ })
+ return &count
+}
+
func (s *stubServer) serveKeepstoreTrash() *int {
return s.serveStatic("/trash", `{}`)
}
@@ -123,10 +156,12 @@ func (s *mainSuite) logger(c *check.C) *log.Logger {
}
func (s *mainSuite) SetUpTest(c *check.C) {
- s.config = Config{Client: arvados.Client{
- AuthToken: "xyzzy",
- APIHost: "zzzzz.arvadosapi.com",
- Client: s.stub.start()}}
+ s.config = Config{
+ Client: arvados.Client{
+ AuthToken: "xyzzy",
+ APIHost: "zzzzz.arvadosapi.com",
+ Client: s.stub.start()},
+ ServiceTypes: []string{"disk"}}
s.stub.serveDiscoveryDoc()
s.stub.logf = c.Logf
}
@@ -145,7 +180,7 @@ func (s *mainSuite) TestRefuseZeroCollections(c *check.C) {
countCollectionsList := s.stub.serveZeroCollections()
s.stub.serveZeroKeepServices()
countTrash := s.stub.serveKeepstoreTrash()
- err := Run(s.config, opts)
+ _, err := Run(s.config, opts)
c.Check(err, check.ErrorMatches, "received zero collections")
c.Check(*countCollectionsList, check.Equals, 2)
c.Check(*countTrash, check.Equals, 0)
@@ -162,7 +197,7 @@ func (s *mainSuite) TestRefuseNonAdmin(c *check.C) {
countKS := s.stub.serveZeroKeepServices()
countTrash := s.stub.serveKeepstoreTrash()
countPull := s.stub.serveKeepstorePull()
- err := Run(s.config, opts)
+ _, err := Run(s.config, opts)
c.Check(err, check.ErrorMatches, "Current user .* is not .* admin user")
c.Check(*countCurrentUser, check.Equals, 1)
c.Check(*countColl, check.Equals, 0)
@@ -178,14 +213,20 @@ func (s *mainSuite) TestDryRun(c *check.C) {
Logger: s.logger(c),
}
s.stub.serveCurrentUserAdmin()
- countColl := s.stub.serveZeroCollections()
- countKS := s.stub.serveZeroKeepServices()
+ s.stub.serveFooFileCollection()
+ countKS := s.stub.serveFourKeepServices()
+ countIndex := s.stub.serveKeepstoreIndexFooBlock()
countTrash := s.stub.serveKeepstoreTrash()
countPull := s.stub.serveKeepstorePull()
- err := Run(s.config, opts)
- c.Check(*countColl, check.Equals, 2)
+ bal, err := Run(s.config, opts)
c.Check(*countKS, check.Equals, 1)
+ c.Check(*countIndex, check.Equals, int64(4))
c.Check(err, check.IsNil)
c.Check(*countTrash, check.Equals, 0)
c.Check(*countPull, check.Equals, 0)
+ var todo int
+ for _, srv := range bal.KeepServices {
+ todo += len(srv.Pulls) + len(srv.Trashes)
+ }
+ c.Check(todo, check.Not(check.Equals), 0)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list