[ARVADOS] updated: f3b5ffc2a4409d4c1b676e40ba4e582ca2beeb61
git at public.curoverse.com
git at public.curoverse.com
Mon Oct 5 11:41:59 EDT 2015
Summary of changes:
sdk/go/keepclient/keepclient.go | 17 ++++++++++++++---
sdk/go/keepclient/support.go | 32 ++++++++++++++++++++++++++------
tools/keep-rsync/keep-rsync.go | 29 +++++++++++++++++++++++++----
tools/keep-rsync/keep-rsync_test.go | 28 ++++++++++++++++++++++++++++
4 files changed, 93 insertions(+), 13 deletions(-)
via f3b5ffc2a4409d4c1b676e40ba4e582ca2beeb61 (commit)
from 6823f2d65a48bc989b819b85b6428c230b223c82 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f3b5ffc2a4409d4c1b676e40ba4e582ca2beeb61
Author: radhika <radhika at curoverse.com>
Date: Mon Oct 5 11:41:39 2015 -0400
7167: Refactor MakeKeepClient and DiscoverKeepServers to allow making KeepClient from input JSON as well.
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 53dfb2b..05fad6e 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -54,9 +54,20 @@ type KeepClient struct {
replicasPerService int
}
-// Create a new KeepClient. This will contact the API server to discover Keep
-// servers.
+// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
func MakeKeepClient(arv *arvadosclient.ArvadosClient) (*KeepClient, error) {
+ kc := initKeepClient(arv)
+ return kc, kc.DiscoverKeepServers()
+}
+
+// MakeKeepClientFromJSON creates a new KeepClient using the given json to load keep servers.
+func MakeKeepClientFromJSON(arv *arvadosclient.ArvadosClient, svcJSON string) (*KeepClient, error) {
+ kc := initKeepClient(arv)
+ return kc, kc.DiscoverKeepServersFromJSON(svcJSON)
+}
+
+// Make a new KeepClient struct.
+func initKeepClient(arv *arvadosclient.ArvadosClient) *KeepClient {
var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
kc := &KeepClient{
@@ -66,7 +77,7 @@ func MakeKeepClient(arv *arvadosclient.ArvadosClient) (*KeepClient, error) {
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
}
- return kc, kc.DiscoverKeepServers()
+ return kc
}
// Put a block given the block hash, a reader, and the number of bytes
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 63800b1..8be1780 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -2,6 +2,7 @@ package keepclient
import (
"crypto/md5"
+ "encoding/json"
"errors"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/streamer"
@@ -76,19 +77,38 @@ func (this *KeepClient) setClientSettingsDisk() {
}
}
+type svcList struct {
+ Items []keepService `json:"items"`
+}
+
// DiscoverKeepServers gets list of available keep services from api server
func (this *KeepClient) DiscoverKeepServers() error {
- type svcList struct {
- Items []keepService `json:"items"`
- }
- var m svcList
+ var list svcList
// Get keep services from api server
- err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &m)
+ err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &list)
if err != nil {
return err
}
+ return this.loadKeepServers(list)
+}
+
+// DiscoverKeepServersFromJSON gets list of available keep services from given JSON
+func (this *KeepClient) DiscoverKeepServersFromJSON(services string) error {
+ var list svcList
+
+ // Load keep services from given json
+ dec := json.NewDecoder(strings.NewReader(services))
+ if err := dec.Decode(&list); err != nil {
+ return err
+ }
+
+ return this.loadKeepServers(list)
+}
+
+// loadKeepServers
+func (this *KeepClient) loadKeepServers(list svcList) error {
listed := make(map[string]bool)
localRoots := make(map[string]string)
gatewayRoots := make(map[string]string)
@@ -98,7 +118,7 @@ func (this *KeepClient) DiscoverKeepServers() error {
this.replicasPerService = 1
this.Using_proxy = false
- for _, service := range m.Items {
+ for _, service := range list.Items {
scheme := "http"
if service.SSL {
scheme = "https"
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index c9fd77a..3761cc7 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -120,22 +120,43 @@ var (
// Initializes keep-rsync using the config provided
func initializeKeepRsync() (err error) {
+ // arvSrc from srcConfig
arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig)
if err != nil {
return
}
+ // arvDst from dstConfig
arvDst, err = arvadosclient.MakeArvadosClientWithConfig(dstConfig)
if err != nil {
return
}
- kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
- if err != nil {
- return
+ // if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+ if srcKeepServicesJSON == "" {
+ kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
+ if err != nil {
+ return
+ }
+ } else {
+ kcSrc, err = keepclient.MakeKeepClientFromJSON(&arvSrc, srcKeepServicesJSON)
+ if err != nil {
+ return
+ }
}
- kcDst, err = keepclient.MakeKeepClient(&arvDst)
+ // if dstKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+ if dstKeepServicesJSON == "" {
+ kcDst, err = keepclient.MakeKeepClient(&arvDst)
+ if err != nil {
+ return
+ }
+ } else {
+ kcDst, err = keepclient.MakeKeepClientFromJSON(&arvDst, dstKeepServicesJSON)
+ if err != nil {
+ return
+ }
+ }
return
}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index c45fbf8..7636c2e 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -29,6 +29,8 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) {
func (s *ServerRequiredSuite) SetUpTest(c *C) {
arvadostest.ResetEnv()
+ srcKeepServicesJSON = ""
+ dstKeepServicesJSON = ""
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
@@ -115,3 +117,29 @@ func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDstShouldFail(c *C) {
_, _, _, err = kcDst.Get(hash)
c.Assert(err.Error(), Equals, "Block not found")
}
+
+// Test keep-rsync initialization, with srcKeepServicesJSON
+func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
+ srcKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
+
+ setupRsync(c)
+
+ localRoots := kcSrc.LocalRoots()
+ c.Check(localRoots != nil, Equals, true)
+
+ foundIt := false
+ for k, _ := range localRoots {
+ if k == "zzzzz-bi6l4-123456789012340" {
+ foundIt = true
+ }
+ }
+ c.Check(foundIt, Equals, true)
+
+ foundIt = false
+ for k, _ := range localRoots {
+ if k == "zzzzz-bi6l4-123456789012341" {
+ foundIt = true
+ }
+ }
+ c.Check(foundIt, Equals, true)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list