[ARVADOS] updated: 4c9b3a85870572aa375329f39347fad58ab9d0e6
git at public.curoverse.com
git at public.curoverse.com
Tue Dec 1 09:28:31 EST 2015
Summary of changes:
services/datamanager/datamanager_test.go | 49 ++++++++++++++++++++++++++++++--
services/datamanager/keep/keep.go | 24 +++++++++-------
2 files changed, 59 insertions(+), 14 deletions(-)
via 4c9b3a85870572aa375329f39347fad58ab9d0e6 (commit)
from 2274bfc359dbe4d6b864704f4ef140cf44bf1963 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4c9b3a85870572aa375329f39347fad58ab9d0e6
Author: radhika <radhika at curoverse.com>
Date: Tue Dec 1 09:26:13 2015 -0500
7710: update serviceType argument handling to ignore any keepservices that are of a different type. Add a datamanager test with one extra unsupported-typed keepserver, which should hence be ignored by datamanager.
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 4203936..75950d1 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -624,15 +624,26 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
// Also, create stray block and backdate it.
// After datamanager run: expect blocks from the collection, but not the stray block.
func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
+ testManifestWithMultipleStreamsAndBlocks(t, 100, 10, "")
+}
+
+// Same test as TestManifestWithMultipleStreamsAndBlocks with an additional
+// keepstore of a service type other than "disk". Only the "disk" type services
+// will be indexed by datamanager and hence should work the same way.
+func TestManifestWithMultipleStreamsAndBlocks_WithOneUnsupportedKeepServer(t *testing.T) {
+ testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "testblobstore")
+}
+
+func testManifestWithMultipleStreamsAndBlocks(t *testing.T, numStreams, numBlocks int, createExtraKeepServerWithType string) {
defer TearDownDataManagerTest(t)
SetupDataManagerTest(t)
// create collection whose blocks will be backdated
- collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10)
+ collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", numStreams, numBlocks)
if collectionWithOldBlocks == "" {
- t.Fatalf("Failed to create collection with 1000 blocks")
+ t.Fatalf("Failed to create collection with %d blocks", numStreams*numBlocks)
}
- if len(oldBlocks) != 1000 {
+ if len(oldBlocks) != numStreams*numBlocks {
t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks))
}
@@ -649,9 +660,41 @@ func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
// also backdate the stray old block
backdateBlocks(t, []string{strayOldBlock})
+ // If requested, create an extra keepserver with the given type
+ // This should be ignored during indexing and hence not change the datamanager outcome
+ var extraKeepServerUUID string
+ if createExtraKeepServerWithType != "" {
+ extraKeepServerUUID = addExtraKeepServer(t, createExtraKeepServerWithType)
+ defer deleteExtraKeepServer(extraKeepServerUUID)
+ }
+
// run datamanager
dataManagerSingleRun(t)
// verify that strayOldBlock is not to be found, but the collections blocks are still there
verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
}
+
+// Add one more keepstore with the given service type
+func addExtraKeepServer(t *testing.T, serviceType string) string {
+ defer switchToken(arvadostest.AdminToken)()
+
+ extraKeepService := make(arvadosclient.Dict)
+ err := arv.Create("keep_services",
+ arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+ "service_host": "localhost",
+ "service_port": "21321",
+ "service_ssl_flag": false,
+ "service_type": serviceType}},
+ &extraKeepService)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return extraKeepService["uuid"].(string)
+}
+
+func deleteExtraKeepServer(uuid string) {
+ defer switchToken(arvadostest.AdminToken)()
+ arv.Delete("keep_services", uuid, nil, nil)
+}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index c165120..d52191c 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -77,10 +77,10 @@ type ServiceList struct {
KeepServers []ServerAddress `json:"items"`
}
-var serviceType string
+var supportedServiceType string
func init() {
- flag.StringVar(&serviceType,
+ flag.StringVar(&supportedServiceType,
"service-type",
"disk",
"Supported keepservice type. Default is disk.")
@@ -131,18 +131,19 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
return
}
- // Currently, only "disk" types are supported. Stop if any other service types are found.
- foundSupportedServieType := false
+ // Ignore any services that are of type other than the "supportedServiceType".
+ // If no services of the "supportedServiceType" are found, raise an error.
+ var indexableKeepServers []ServerAddress
for _, server := range sdkResponse.KeepServers {
- if server.ServiceType == serviceType {
- foundSupportedServieType = true
+ if server.ServiceType == supportedServiceType {
+ indexableKeepServers = append(indexableKeepServers, server)
} else {
log.Printf("Ignore unsupported service type: %v", server.ServiceType)
}
}
- if !foundSupportedServieType {
- return results, fmt.Errorf("Found no keepservices with the supported type %v", serviceType)
+ if len(indexableKeepServers) == 0 {
+ return results, fmt.Errorf("Found no keepservices with the supported type %v", supportedServiceType)
}
if params.Logger != nil {
@@ -151,6 +152,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
keepInfo["num_keep_servers_available"] = sdkResponse.ItemsAvailable
keepInfo["num_keep_servers_received"] = len(sdkResponse.KeepServers)
keepInfo["keep_servers"] = sdkResponse.KeepServers
+ keepInfo["indexable_keep_servers"] = indexableKeepServers
})
}
@@ -160,7 +162,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
return results, fmt.Errorf("Did not receive all available keep servers: %+v", sdkResponse)
}
- results.KeepServerIndexToAddress = sdkResponse.KeepServers
+ results.KeepServerIndexToAddress = indexableKeepServers
results.KeepServerAddressToIndex = make(map[ServerAddress]int)
for i, address := range results.KeepServerIndexToAddress {
results.KeepServerAddressToIndex[address] = i
@@ -170,7 +172,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
// Send off all the index requests concurrently
responseChan := make(chan ServerResponse)
- for _, keepServer := range sdkResponse.KeepServers {
+ for _, keepServer := range results.KeepServerIndexToAddress {
// The above keepsServer variable is reused for each iteration, so
// it would be shared across all goroutines. This would result in
// us querying one server n times instead of n different servers
@@ -188,7 +190,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
results.BlockToServers = make(map[blockdigest.DigestWithSize][]BlockServerInfo)
// Read all the responses
- for i := range sdkResponse.KeepServers {
+ for i := range results.KeepServerIndexToAddress {
_ = i // Here to prevent go from complaining.
response := <-responseChan
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list