[ARVADOS] updated: 2274bfc359dbe4d6b864704f4ef140cf44bf1963
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 30 18:13:12 EST 2015
Summary of changes:
sdk/go/blockdigest/blockdigest.go | 10 ---------
sdk/go/blockdigest/blockdigest_test.go | 24 +++++++++++++++++-----
sdk/go/keepclient/discover.go | 5 -----
sdk/go/keepclient/keepclient.go | 2 --
sdk/go/keepclient/keepclient_test.go | 8 +++-----
sdk/go/logger/logger.go | 16 ++++++++-------
sdk/go/manifest/manifest_test.go | 24 ++++++++++++++++------
services/datamanager/datamanager.go | 2 +-
services/datamanager/datamanager_test.go | 1 -
services/datamanager/keep/keep.go | 21 +++++++++++++++++--
services/datamanager/keep/keep_test.go | 2 +-
services/keepproxy/keepproxy_test.go | 3 +--
services/keepstore/keepstore.go | 1 -
services/keepstore/pull_worker_integration_test.go | 1 -
14 files changed, 71 insertions(+), 49 deletions(-)
via 2274bfc359dbe4d6b864704f4ef140cf44bf1963 (commit)
via c77b8d1cced9f2e1365d6718702e588e3d6f11b2 (commit)
via 4addff6dc8ecad6c92949c2f2eac2cc879b2edbf (commit)
via 128c2b5e228e1821384064ec50604a1463c29898 (commit)
via f04a6c74663185addff5564b47db0442679de78e (commit)
via a5ed26a2a1d01646ac511b925f56484be2e8819a (commit)
via 040a541c74913c01ee3517273a7be30c510cc620 (commit)
from 5f7c3a3a4a6926edb67ecb46d8aa3649a47a2643 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2274bfc359dbe4d6b864704f4ef140cf44bf1963
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 30 18:09:57 2015 -0500
7710: Remove KeepClient.Using_proxy and all it's references since we are no longer using it.
Update datamanager to use a configured supported service type instead of hardcoding "disk" type.
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 1095085..099c56f 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -84,7 +84,6 @@ func (this *KeepClient) loadKeepServers(list svcList) error {
// replicasPerService is 1 for disks; unknown or unlimited otherwise
this.replicasPerService = 1
- this.Using_proxy = false
for _, service := range list.Items {
scheme := "http"
@@ -100,10 +99,6 @@ func (this *KeepClient) loadKeepServers(list svcList) error {
listed[url] = true
localRoots[service.Uuid] = url
- if service.SvcType == "proxy" {
- this.Using_proxy = true
- }
-
if service.ReadOnly == false {
writableLocalRoots[service.Uuid] = url
if service.SvcType != "disk" {
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index ddb502c..f15a6b2 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -68,7 +68,6 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
- Using_proxy bool
localRoots *map[string]string
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
@@ -104,7 +103,6 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
kc := &KeepClient{
Arvados: arv,
Want_replicas: defaultReplicationLevel,
- Using_proxy: false,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
Retries: 2,
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 2c77e68..87b9b1d 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -895,7 +895,6 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 2
- kc.Using_proxy = true
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
@@ -928,7 +927,6 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
kc, _ := MakeKeepClient(&arv)
kc.Want_replicas = 3
- kc.Using_proxy = true
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
@@ -1271,21 +1269,21 @@ func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
for _, root := range kc.LocalRoots() {
c.Check(root, Matches, "http://localhost:\\d+")
}
- c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], NotNil)
+ c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
// verify kc.GatewayRoots
c.Check(len(kc.GatewayRoots()), Equals, 3)
for _, root := range kc.GatewayRoots() {
c.Check(root, Matches, "http://localhost:\\d+")
}
- c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], NotNil)
+ c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
// verify kc.WritableLocalRoots
c.Check(len(kc.WritableLocalRoots()), Equals, 3)
for _, root := range kc.WritableLocalRoots() {
c.Check(root, Matches, "http://localhost:\\d+")
}
- c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], NotNil)
+ c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
c.Assert(kc.replicasPerService, Equals, 0)
c.Assert(kc.foundNonDiskSvc, Equals, true)
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 28faf98..4203936 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -41,7 +41,6 @@ func SetupDataManagerTest(t *testing.T) {
keepClient = &keepclient.KeepClient{
Arvados: &arv,
Want_replicas: 2,
- Using_proxy: true,
Client: &http.Client{},
}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 7da74b1..c165120 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -6,6 +6,7 @@ import (
"bufio"
"encoding/json"
"errors"
+ "flag"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
@@ -76,6 +77,15 @@ type ServiceList struct {
KeepServers []ServerAddress `json:"items"`
}
+var serviceType string
+
+func init() {
+ flag.StringVar(&serviceType,
+ "service-type",
+ "disk",
+ "Supported keepservice type. Default is disk.")
+}
+
// String
// TODO(misha): Change this to include the UUID as well.
func (s ServerAddress) String() string {
@@ -122,12 +132,19 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
}
// Currently, only "disk" types are supported. Stop if any other service types are found.
+ foundSupportedServieType := false
for _, server := range sdkResponse.KeepServers {
- if server.ServiceType != "disk" {
- return results, fmt.Errorf("Unsupported service type %q found for: %v", server.ServiceType, server)
+ if server.ServiceType == serviceType {
+ foundSupportedServieType = true
+ } else {
+ log.Printf("Ignore unsupported service type: %v", server.ServiceType)
}
}
+ if !foundSupportedServieType {
+ return results, fmt.Errorf("Found no keepservices with the supported type %v", serviceType)
+ }
+
if params.Logger != nil {
params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := logger.GetOrCreateMap(p, "keep_info")
diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 0691e64..2286453 100644
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -101,7 +101,7 @@ type APITestData struct {
}
func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) {
- testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Unsupported service type")
+ testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Found no keepservices with the supported type disk")
}
func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) {
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 7c5b362..56c9bff 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -105,7 +105,6 @@ func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient
}
kc.SetServiceRoots(sr, sr, sr)
kc.Arvados.External = true
- kc.Using_proxy = true
return kc
}
@@ -474,7 +473,7 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- for _, f := range []func()error {
+ for _, f := range []func() error{
func() error {
_, _, err := kc.Ask(hash)
return err
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 7525441..96a887f 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -300,7 +300,6 @@ func main() {
keepClient := &keepclient.KeepClient{
Arvados: nil,
Want_replicas: 1,
- Using_proxy: true,
Client: &http.Client{},
}
diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go
index 52b59ec..e0613a2 100644
--- a/services/keepstore/pull_worker_integration_test.go
+++ b/services/keepstore/pull_worker_integration_test.go
@@ -39,7 +39,6 @@ func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTe
keepClient = &keepclient.KeepClient{
Arvados: &arv,
Want_replicas: 1,
- Using_proxy: true,
Client: &http.Client{},
}
commit c77b8d1cced9f2e1365d6718702e588e3d6f11b2
Merge: 4addff6 5f7c3a3
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 30 18:08:45 2015 -0500
Merge branch '7710-keepclient-all-service-types' of git.curoverse.com:arvados into 7710-keepclient-all-service-types
commit 4addff6dc8ecad6c92949c2f2eac2cc879b2edbf
Merge: 0a07e79 128c2b5
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 30 10:40:07 2015 -0500
Merge branch 'master' into 7710-keepclient-all-service-types
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list