[ARVADOS] updated: 5790a95b32d3a2268feea07d5ae580e28235bd4c
git at public.curoverse.com
git at public.curoverse.com
Fri Oct 9 08:34:06 EDT 2015
Summary of changes:
tools/keep-rsync/keep-rsync.go | 16 +++++--
tools/keep-rsync/keep-rsync_test.go | 90 +++++++++++++++++++++++++++++++------
2 files changed, 90 insertions(+), 16 deletions(-)
via 5790a95b32d3a2268feea07d5ae580e28235bd4c (commit)
via 84de88800275db7046e3c784ff73d4af1a6f03b1 (commit)
via e531e2bb20a6ad0acaef4c7a19fb7625f94ccc0c (commit)
via dfc31c02280518a4040a7168326b5fbfe742ed28 (commit)
from ac2426ef6ba7c2c722da0f7b0add2c5040529bd3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5790a95b32d3a2268feea07d5ae580e28235bd4c
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 17:39:20 2015 -0400
7167: add tests with prefix during rsync
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index fdce662..88caf90 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -221,11 +221,11 @@ func performKeepRsync() error {
}
// Get list of unique locators from the specified cluster
-func getUniqueLocators(kc *keepclient.KeepClient, prefix string) (map[string]bool, error) {
+func getUniqueLocators(kc *keepclient.KeepClient, indexPrefix string) (map[string]bool, error) {
var indexBytes []byte
for uuid := range kc.LocalRoots() {
- reader, err := kc.GetIndex(uuid, prefix)
+ reader, err := kc.GetIndex(uuid, indexPrefix)
if err != nil {
return nil, err
}
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index c612e53..4561f5d 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"os"
+ "strings"
"testing"
"time"
@@ -43,7 +44,7 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
// The test setup hence tweaks keep-rsync initialization to achieve this.
// First invoke initializeKeepRsync and then invoke StartKeepWithParams
// to create the keep servers to be used as destination.
-func setupRsync(c *C, enforcePermissions bool) {
+func setupRsync(c *C, enforcePermissions bool, overwriteReplications bool) {
// srcConfig
srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
@@ -69,7 +70,12 @@ func setupRsync(c *C, enforcePermissions bool) {
// Create two more keep servers to be used as destination
arvadostest.StartKeepWithParams(true, enforcePermissions)
- // load kcDst; set Want_replicas as 1 since that is how many keep servers are created for dst.
+ // set replications to 1 since those many keep servers were created for dst.
+ if overwriteReplications {
+ replications = 1
+ }
+
+ // load kcDst
kcDst, err = keepclient.MakeKeepClient(&arvDst)
c.Assert(err, Equals, nil)
kcDst.Want_replicas = 1
@@ -105,7 +111,7 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
// Do a Get in dst for the src hash, which should raise block not found error.
// Do a Get in src for the dst hash, which should raise block not found error.
func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
- setupRsync(c, false)
+ setupRsync(c, false, true)
// Put a block in src using kcSrc and Get it
srcData := []byte("test-data1")
@@ -150,7 +156,7 @@ func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
srcKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
- setupRsync(c, false)
+ setupRsync(c, false, true)
localRoots := kcSrc.LocalRoots()
c.Check(localRoots != nil, Equals, true)
@@ -178,7 +184,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
// Do a Get in dst for the src hash, which should raise block not found error.
// Do a Get in src for the dst hash, which should raise block not found error.
func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
- setupRsync(c, true)
+ setupRsync(c, true, true)
// Put a block in src using kcSrc and Get it
srcData := []byte("test-data1")
@@ -228,7 +234,7 @@ func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShou
// Test keep-rsync initialization with default replications count
func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
- setupRsync(c, false)
+ setupRsync(c, false, false)
// Must have got default replications value as 2 from dst discovery document
c.Assert(replications, Equals, 2)
@@ -236,10 +242,10 @@ func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C)
// Test keep-rsync initialization with replications count argument
func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
- // set replications to 3 to mimic passing input argument
+ // First set replications to 3 to mimic passing input argument
replications = 3
- setupRsync(c, false)
+ setupRsync(c, false, false)
// Since replications value is provided, default is not used
c.Assert(replications, Equals, 3)
@@ -248,26 +254,46 @@ func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
// Put some blocks in Src and some more in Dst
// And copy missing blocks from Src to Dst
func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
- testKeepRsync(c, false)
+ testKeepRsync(c, false, "")
}
// Put some blocks in Src and some more in Dst with blob signing enabled.
// And copy missing blocks from Src to Dst
func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
- testKeepRsync(c, true)
+ testKeepRsync(c, true, "")
+}
+
+// Put some blocks in Src and some more in Dst
+// Use prefix while doing rsync
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) {
+ data := []byte("test-data-4")
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ testKeepRsync(c, false, hash[0:3])
+}
+
+// Put some blocks in Src and some more in Dst
+// Use prefix not in src while doing rsync
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) {
+ testKeepRsync(c, false, "999")
}
// Put 5 blocks in src. Put 2 of those blocks in dst
// Hence there are 3 additional blocks in src
// Also, put 2 extra blocks in dst; they are hence only in dst
// Run rsync and verify that those 7 blocks are now available in dst
-func testKeepRsync(c *C, enforcePermissions bool) {
- setupRsync(c, enforcePermissions)
+func testKeepRsync(c *C, enforcePermissions bool, indexPrefix string) {
+ setupRsync(c, enforcePermissions, true)
+
+ prefix = indexPrefix
tomorrow := time.Now().AddDate(0, 0, 1)
// Put a few blocks in src using kcSrc
var srcLocators []string
+ var srcLocatorsMatchingPrefix []string
for i := 0; i < 5; i++ {
data := []byte(fmt.Sprintf("test-data-%d", i))
hash := fmt.Sprintf("%x", md5.Sum(data))
@@ -289,9 +315,12 @@ func testKeepRsync(c *C, enforcePermissions bool) {
c.Check(all, DeepEquals, data)
srcLocators = append(srcLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ if strings.HasPrefix(hash, indexPrefix) {
+ srcLocatorsMatchingPrefix = append(srcLocatorsMatchingPrefix, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
}
- // Put just two of those blocks in dst using kcDst
+ // Put first two of those src blocks in dst using kcDst
var dstLocators []string
for i := 0; i < 2; i++ {
data := []byte(fmt.Sprintf("test-data-%d", i))
@@ -347,12 +376,29 @@ func testKeepRsync(c *C, enforcePermissions bool) {
// Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
dstIndex, err := getUniqueLocators(kcDst, "")
c.Check(err, Equals, nil)
- for _, locator := range srcLocators {
- _, ok := dstIndex[locator]
- c.Assert(ok, Equals, true)
+
+ if prefix == "" {
+ for _, locator := range srcLocators {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
+ } else {
+ for _, locator := range srcLocatorsMatchingPrefix {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
}
+
for _, locator := range extraDstLocators {
_, ok := dstIndex[locator]
c.Assert(ok, Equals, true)
}
+
+ if prefix == "" {
+ // all blocks from src and the two extra blocks
+ c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators))
+ } else {
+ // one matching prefix, 2 that were initially copied into dst along with src, and the extra blocks
+ c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2)
+ }
}
commit 84de88800275db7046e3c784ff73d4af1a6f03b1
Merge: ac2426e e531e2b
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 16:22:45 2015 -0400
Merge branch '7167-keep-rsync-test-setup' into 7167-keep-rsync
Conflicts:
tools/keep-rsync/keep-rsync_test.go
diff --cc tools/keep-rsync/keep-rsync_test.go
index 6c81528,6bc0cb7..c612e53
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@@ -227,114 -226,21 +226,133 @@@ func (s *ServerRequiredSuite) TestRsync
c.Assert(err.Error(), Equals, "Block not found")
}
+ // Test keep-rsync initialization with default replications count
+ func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
+ setupRsync(c, false)
+
+ // Must have got default replications value as 2 from dst discovery document
+ c.Assert(replications, Equals, 2)
+ }
+
+ // Test keep-rsync initialization with replications count argument
+ func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
+ // set replications to 3 to mimic passing input argument
+ replications = 3
+
+ setupRsync(c, false)
+
+ // Since replications value is provided, default is not used
+ c.Assert(replications, Equals, 3)
+ }
++
+// Put some blocks in Src and some more in Dst
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
+ testKeepRsync(c, false)
+}
+
+// Put some blocks in Src and some more in Dst with blob signing enabled.
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
+ testKeepRsync(c, true)
+}
+
+// Put 5 blocks in src. Put 2 of those blocks in dst
+// Hence there are 3 additional blocks in src
+// Also, put 2 extra blocks in dst; they are hence only in dst
+// Run rsync and verify that those 7 blocks are now available in dst
+func testKeepRsync(c *C, enforcePermissions bool) {
+ setupRsync(c, enforcePermissions)
+
+ tomorrow := time.Now().AddDate(0, 0, 1)
+
+ // Put a few blocks in src using kcSrc
+ var srcLocators []string
+ for i := 0; i < 5; i++ {
+ data := []byte(fmt.Sprintf("test-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcSrc.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+11(\+.+)?$`, hash))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcSrc.Get(getLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(11))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ srcLocators = append(srcLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ // Put just two of those blocks in dst using kcDst
+ var dstLocators []string
+ for i := 0; i < 2; i++ {
+ data := []byte(fmt.Sprintf("test-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcDst.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+11(\+.+)?$`, hash))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcDst.Get(getLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(11))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ dstLocators = append(dstLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ // Put two more blocks in dst; they are not in src at all
+ var extraDstLocators []string
+ for i := 0; i < 2; i++ {
+ data := []byte(fmt.Sprintf("other-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcDst.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+12(\+.+)?$`, hash))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcDst.Get(getLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(12))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ extraDstLocators = append(extraDstLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ err := performKeepRsync()
+ c.Check(err, Equals, nil)
+
+ // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
+ dstIndex, err := getUniqueLocators(kcDst, "")
+ c.Check(err, Equals, nil)
+ for _, locator := range srcLocators {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
+ for _, locator := range extraDstLocators {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
+}
commit e531e2bb20a6ad0acaef4c7a19fb7625f94ccc0c
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 16:19:37 2015 -0400
7167: add tests to replications count
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 3c1a402..d569421 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -58,7 +58,7 @@ func main() {
flag.IntVar(
&replications,
"replications",
- 3,
+ 0,
"Number of replications to write to the destination.")
flag.StringVar(
@@ -151,10 +151,14 @@ func initializeKeepRsync() (err error) {
return
}
- // Get default replication value from destination
- value, err := arvDst.Discovery("defaultCollectionReplication")
- if err == nil {
- replications = int(value.(float64))
+ // Get default replications value from destination, if it is not already provided
+ if replications == 0 {
+ value, err := arvDst.Discovery("defaultCollectionReplication")
+ if err == nil {
+ replications = int(value.(float64))
+ } else {
+ replications = 2
+ }
}
// if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 52a4eba..6bc0cb7 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -66,9 +66,6 @@ func setupRsync(c *C, enforcePermissions bool) {
err := initializeKeepRsync()
c.Assert(err, Equals, nil)
- // Must have got replication value as 2 from dst discovery document
- c.Assert(replications, Equals, 2)
-
// Create two more keep servers to be used as destination
arvadostest.StartKeepWithParams(true, enforcePermissions)
@@ -228,3 +225,22 @@ func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShou
_, _, _, err = kcSrc.Get(locatorInDst)
c.Assert(err.Error(), Equals, "Block not found")
}
+
+// Test keep-rsync initialization with default replications count
+func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) {
+ setupRsync(c, false)
+
+ // Must have got default replications value as 2 from dst discovery document
+ c.Assert(replications, Equals, 2)
+}
+
+// Test keep-rsync initialization with replications count argument
+func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) {
+ // set replications to 3 to mimic passing input argument
+ replications = 3
+
+ setupRsync(c, false)
+
+ // Since replications value is provided, default is not used
+ c.Assert(replications, Equals, 3)
+}
commit dfc31c02280518a4040a7168326b5fbfe742ed28
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 16:00:14 2015 -0400
7167: get replications count from destination api discovery doc and use it as default.
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 313c188..3c1a402 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -151,6 +151,12 @@ func initializeKeepRsync() (err error) {
return
}
+ // Get default replication value from destination
+ value, err := arvDst.Discovery("defaultCollectionReplication")
+ if err == nil {
+ replications = int(value.(float64))
+ }
+
// if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
if srcKeepServicesJSON == "" {
kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index c2a8112..52a4eba 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -54,7 +54,6 @@ func setupRsync(c *C, enforcePermissions bool) {
dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
- replications = 1
if enforcePermissions {
blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
}
@@ -67,10 +66,13 @@ func setupRsync(c *C, enforcePermissions bool) {
err := initializeKeepRsync()
c.Assert(err, Equals, nil)
+ // Must have got replication value as 2 from dst discovery document
+ c.Assert(replications, Equals, 2)
+
// Create two more keep servers to be used as destination
arvadostest.StartKeepWithParams(true, enforcePermissions)
- // load kcDst
+ // load kcDst; set Want_replicas as 1 since that is how many keep servers are created for dst.
kcDst, err = keepclient.MakeKeepClient(&arvDst)
c.Assert(err, Equals, nil)
kcDst.Want_replicas = 1
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list