[ARVADOS] updated: 305cee61ba4a122f7d63a01f4a7b9b98737b8646
Git user
git at public.curoverse.com
Tue Jul 26 09:00:26 EDT 2016
Summary of changes:
sdk/go/arvados/client.go | 19 ++++++++++++---
sdk/go/arvadosclient/arvadosclient.go | 25 ++++++++++++++++---
sdk/go/keepclient/discover.go | 24 +++++++++++++++---
sdk/go/keepclient/discover_test.go | 46 ++++++++++++++++++++++++++++++++++-
sdk/go/keepclient/keepclient.go | 2 +-
5 files changed, 104 insertions(+), 12 deletions(-)
via 305cee61ba4a122f7d63a01f4a7b9b98737b8646 (commit)
via 5e8d67eff3adda469e4c72955b0f83e375291beb (commit)
via 2376a805daae84c5e899f190485d7d7ac96e3f20 (commit)
via cf5009fc7f67c726c82e331b579ae0d3880d926a (commit)
via 853cb66fe7d3c24e9936b4a49fdd775abc0ed0f4 (commit)
from 34f8647caace707dfb524a89d27bded9274058fb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 305cee61ba4a122f7d63a01f4a7b9b98737b8646
Merge: 34f8647 5e8d67e
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jul 26 08:59:48 2016 -0400
Merge branch '9550-keep-services-env'
closes #9550
commit 5e8d67eff3adda469e4c72955b0f83e375291beb
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 25 16:41:25 2016 -0400
9550: Ignore extra spaces in ARVADOS_KEEP_SERVICES, and ensure each entry is an absolute URI.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 57f81c2..aeb81f9 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -120,8 +120,16 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
External: external,
Retries: 2}
- if s := os.Getenv("ARVADOS_KEEP_SERVICES"); s != "" {
- ac.KeepServiceURIs = strings.Split(s, " ")
+ for _, s := range strings.Split(os.Getenv("ARVADOS_KEEP_SERVICES"), " ") {
+ if s == "" {
+ continue
+ }
+ if u, err := url.Parse(s); err != nil {
+ return ac, fmt.Errorf("ARVADOS_KEEP_SERVICES: %q: %s", s, err)
+ } else if !u.IsAbs() {
+ return ac, fmt.Errorf("ARVADOS_KEEP_SERVICES: %q: not an absolute URI", s)
+ }
+ ac.KeepServiceURIs = append(ac.KeepServiceURIs, s)
}
if ac.ApiServer == "" {
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index b145ee6..ee31ea0 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -42,7 +42,7 @@ func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
c.Assert(err, check.IsNil)
arv1.ApiToken = arvadostest.ActiveToken
- os.Setenv("ARVADOS_KEEP_SERVICES", ks[0].url+" "+ks[1].url)
+ os.Setenv("ARVADOS_KEEP_SERVICES", ks[0].url+" "+ks[1].url+" ")
arv2, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.IsNil)
arv2.ApiToken = arvadostest.ActiveToken
commit 2376a805daae84c5e899f190485d7d7ac96e3f20
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jul 25 15:54:41 2016 -0400
9550: Update comments.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 2befcd5..57f81c2 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -86,8 +86,10 @@ type ArvadosClient struct {
// the client is outside the cluster.
External bool
- // Use provided list of keep services, instead of using the
- // API to discover available services.
+ // Base URIs of Keep services, e.g., {"https://host1:8443",
+ // "https://host2:8443"}. If this is nil, Keep clients will
+ // use the arvados.v1.keep_services.accessible API to discover
+ // available services.
KeepServiceURIs []string
// Discovery document
@@ -99,9 +101,10 @@ type ArvadosClient struct {
Retries int
}
-// Create a new ArvadosClient, initialized with standard Arvados environment
-// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
-// ARVADOS_API_HOST_INSECURE.
+// MakeArvadosClient creates a new ArvadosClient using the standard
+// environment variables ARVADOS_API_HOST, ARVADOS_API_TOKEN,
+// ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT, and
+// ARVADOS_KEEP_SERVICES.
func MakeArvadosClient() (ac ArvadosClient, err error) {
var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 66eb316..2892031 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -31,14 +31,13 @@ func (this *KeepClient) DiscoverKeepServers() error {
return nil
}
+ // ArvadosClient did not provide a services list. Ask API
+ // server for a list of accessible services.
var list svcList
-
- // Get keep services from api server
err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &list)
if err != nil {
return err
}
-
return this.loadKeepServers(list)
}
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index ab7b2da..b145ee6 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -12,7 +12,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
)
-func ExampleRefreshServices() {
+func ExampleKeepClient_RefreshServices() {
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
panic(err)
commit cf5009fc7f67c726c82e331b579ae0d3880d926a
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jul 22 09:59:32 2016 -0400
9550: Add service discovery override field to new SDK.
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 3d8fa55..f95152b 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"os"
+ "strings"
"time"
)
@@ -33,6 +34,13 @@ type Client struct {
// Accept unverified certificates. This works only if the
// Client field is nil: otherwise, it has no effect.
Insecure bool
+
+ // Override keep service discovery with a list of base
+ // URIs. (Currently there are no Client methods for
+ // discovering keep services so this is just a convenience for
+ // callers who use a Client to initialize an
+ // arvadosclient.ArvadosClient.)
+ KeepServiceURIs []string
}
// The default http.Client used by a Client with Insecure==true and
@@ -51,10 +59,15 @@ var DefaultSecureClient = &http.Client{
// client with the API endpoint and credentials given by the
// ARVADOS_API_* environment variables.
func NewClientFromEnv() *Client {
+ var svcs []string
+ if s := os.Getenv("ARVADOS_KEEP_SERVICES"); s != "" {
+ svcs = strings.Split(s, " ")
+ }
return &Client{
- APIHost: os.Getenv("ARVADOS_API_HOST"),
- AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
- Insecure: os.Getenv("ARVADOS_API_HOST_INSECURE") != "",
+ APIHost: os.Getenv("ARVADOS_API_HOST"),
+ AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
+ Insecure: os.Getenv("ARVADOS_API_HOST_INSECURE") != "",
+ KeepServiceURIs: svcs,
}
}
commit 853cb66fe7d3c24e9936b4a49fdd775abc0ed0f4
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jul 22 09:41:47 2016 -0400
9550: Allow overriding keep services discovery with ARVADOS_KEEP_SERVICES env var.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 8cdfa48..2befcd5 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -86,6 +86,10 @@ type ArvadosClient struct {
// the client is outside the cluster.
External bool
+ // Use provided list of keep services, instead of using the
+ // API to discover available services.
+ KeepServiceURIs []string
+
// Discovery document
DiscoveryDoc Dict
@@ -113,6 +117,10 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
External: external,
Retries: 2}
+ if s := os.Getenv("ARVADOS_KEEP_SERVICES"); s != "" {
+ ac.KeepServiceURIs = strings.Split(s, " ")
+ }
+
if ac.ApiServer == "" {
return ac, MissingArvadosApiHost
}
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index f039c21..66eb316 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -12,8 +12,25 @@ import (
"time"
)
-// DiscoverKeepServers gets list of available keep services from api server
+// DiscoverKeepServers gets list of available keep services from the
+// API server.
+//
+// If a list of services is provided in the arvadosclient (e.g., from
+// an environment variable or local config), that list is used
+// instead.
func (this *KeepClient) DiscoverKeepServers() error {
+ if this.Arvados.KeepServiceURIs != nil {
+ this.foundNonDiskSvc = true
+ this.replicasPerService = 0
+ this.setClientSettingsNonDisk()
+ roots := make(map[string]string)
+ for i, uri := range this.Arvados.KeepServiceURIs {
+ roots[fmt.Sprintf("00000-bi6l4-%015d", i)] = uri
+ }
+ this.SetServiceRoots(roots, roots, roots)
+ return nil
+ }
+
var list svcList
// Get keep services from api server
diff --git a/sdk/go/keepclient/discover_test.go b/sdk/go/keepclient/discover_test.go
index 43a984e..ab7b2da 100644
--- a/sdk/go/keepclient/discover_test.go
+++ b/sdk/go/keepclient/discover_test.go
@@ -1,10 +1,15 @@
package keepclient
import (
+ "crypto/md5"
"fmt"
+ "gopkg.in/check.v1"
+ "net/http"
+ "os"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
)
func ExampleRefreshServices() {
@@ -19,3 +24,42 @@ func ExampleRefreshServices() {
go kc.RefreshServices(5*time.Minute, 3*time.Second)
fmt.Printf("LocalRoots: %#v\n", kc.LocalRoots())
}
+
+func (s *ServerRequiredSuite) TestOverrideDiscovery(c *check.C) {
+ defer os.Setenv("ARVADOS_KEEP_SERVICES", "")
+
+ hash := fmt.Sprintf("%x+3", md5.Sum([]byte("TestOverrideDiscovery")))
+ st := StubGetHandler{
+ c,
+ hash,
+ arvadostest.ActiveToken,
+ http.StatusOK,
+ []byte("TestOverrideDiscovery")}
+ ks := RunSomeFakeKeepServers(st, 2)
+
+ os.Setenv("ARVADOS_KEEP_SERVICES", "")
+ arv1, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv1.ApiToken = arvadostest.ActiveToken
+
+ os.Setenv("ARVADOS_KEEP_SERVICES", ks[0].url+" "+ks[1].url)
+ arv2, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv2.ApiToken = arvadostest.ActiveToken
+
+ // ARVADOS_KEEP_SERVICES was empty when we created arv1, but
+ // it pointed to our stub servers when we created
+ // arv2. Regardless of what it's set to now, a keepclient for
+ // arv2 should use our stub servers, but one created for arv1
+ // should not.
+
+ kc1, err := MakeKeepClient(&arv1)
+ c.Assert(err, check.IsNil)
+ kc2, err := MakeKeepClient(&arv2)
+ c.Assert(err, check.IsNil)
+
+ _, _, _, err = kc1.Get(hash)
+ c.Check(err, check.NotNil)
+ _, _, _, err = kc2.Get(hash)
+ c.Check(err, check.IsNil)
+}
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 26aa717..58f3ffb 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -352,7 +352,7 @@ func (kc *KeepClient) WritableLocalRoots() map[string]string {
// caller can reuse/modify them after SetServiceRoots returns, but
// they should not be modified by any other goroutine while
// SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals map[string]string, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
locals := make(map[string]string)
for uuid, root := range newLocals {
locals[uuid] = root
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list