[ARVADOS] updated: ac2426ef6ba7c2c722da0f7b0add2c5040529bd3
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 8 13:13:20 EDT 2015
Summary of changes:
tools/keep-rsync/keep-rsync.go | 11 ++++++--
tools/keep-rsync/keep-rsync_test.go | 55 ++++++++++++++++++++++++++++---------
2 files changed, 51 insertions(+), 15 deletions(-)
via ac2426ef6ba7c2c722da0f7b0add2c5040529bd3 (commit)
from fb9e39f948b75cbae8de76359f589a9f33ebd1ac (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ac2426ef6ba7c2c722da0f7b0add2c5040529bd3
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 13:12:37 2015 -0400
7167: honor blob signing key while getting blocks.
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index f33c816..ae76aa8 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -9,6 +9,7 @@ import (
"log"
"regexp"
"strings"
+ "time"
)
// keep-rsync arguments
@@ -92,7 +93,7 @@ func main() {
// Initialize keep-rsync
err = initializeKeepRsync()
if err != nil {
- log.Fatal("Error configurating keep-rsync: %s", err.Error())
+ log.Fatal("Error configuring keep-rsync: %s", err.Error())
}
// Copy blocks not found in dst from src
@@ -267,7 +268,13 @@ func copyBlocksToDst(toBeCopied []string) {
log.Printf("Getting block: %v", locator)
- reader, _, _, err := kcSrc.Get(locator)
+ getLocator := locator
+ expiresAt := time.Now().AddDate(0, 0, 1)
+ if blobSigningKey != "" {
+ getLocator = keepclient.SignLocator(getLocator, arvSrc.ApiToken, expiresAt, []byte(blobSigningKey))
+ }
+
+ reader, _, _, err := kcSrc.Get(getLocator)
if err != nil {
log.Printf("Error getting block: %q %v", locator, err)
failed = append(failed, locator)
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 5cfd30a..6c81528 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -40,8 +40,8 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
}
// Testing keep-rsync needs two sets of keep services: src and dst.
-// The test setup hence tweaks keep-rsync initialzation to achieve this.
-// First invoke initializeKeepRsync and then invoke StartKeepAdditional
+// The test setup hence tweaks keep-rsync initialization to achieve this.
+// First invoke initializeKeepRsync and then invoke StartKeepWithParams
// to create the keep servers to be used as destination.
func setupRsync(c *C, enforcePermissions bool) {
// srcConfig
@@ -138,11 +138,11 @@ func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
all, err = ioutil.ReadAll(reader)
c.Check(all, DeepEquals, dstData)
- // Get srcLocator using kcDst should fail with NotFound error
+ // Get srcLocator using kcDst should fail with Not Found error
_, _, _, err = kcDst.Get(locatorInSrc)
c.Assert(err.Error(), Equals, "Block not found")
- // Get dstLocator using kcSrc should fail with NotFound error
+ // Get dstLocator using kcSrc should fail with Not Found error
_, _, _, err = kcSrc.Get(locatorInDst)
c.Assert(err.Error(), Equals, "Block not found")
}
@@ -173,7 +173,7 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
c.Check(foundIt, Equals, true)
}
-// Test keep-rsync initialization, with src and dst keep servers with blobSingingKey.
+// Test keep-rsync initialization, with src and dst keep servers with blobSigningKey.
// Do a Put and Get in src, both of which should succeed.
// Do a Put and Get in dst, both of which should succeed.
// Do a Get in dst for the src hash, which should raise block not found error.
@@ -216,23 +216,37 @@ func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShou
all, err = ioutil.ReadAll(reader)
c.Check(all, DeepEquals, dstData)
- // Get srcLocator using kcDst should fail with NotFound error
+ // Get srcLocator using kcDst should fail with Not Found error
signedLocator = keepclient.SignLocator(locatorInSrc, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
_, _, _, err = kcDst.Get(locatorInSrc)
c.Assert(err.Error(), Equals, "Block not found")
- // Get dstLocator using kcSrc should fail with NotFound error
+ // Get dstLocator using kcSrc should fail with Not Found error
signedLocator = keepclient.SignLocator(locatorInDst, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
_, _, _, err = kcSrc.Get(locatorInDst)
c.Assert(err.Error(), Equals, "Block not found")
}
+// Put some blocks in Src and some more in Dst
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
+ testKeepRsync(c, false)
+}
+
+// Put some blocks in Src and some more in Dst with blob signing enabled.
+// And copy missing blocks from Src to Dst
+func (s *ServerRequiredSuite) TestKeepRsync_WithBlobSigning(c *C) {
+ testKeepRsync(c, true)
+}
+
// Put 5 blocks in src. Put 2 of those blocks in dst
// Hence there are 3 additional blocks in src
-// Also, put 2 extra blocks in dts; they are hence only in dst
+// Also, put 2 extra blocks in dst; they are hence only in dst
// Run rsync and verify that those 7 blocks are now available in dst
-func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
- setupRsync(c, false)
+func testKeepRsync(c *C, enforcePermissions bool) {
+ setupRsync(c, enforcePermissions)
+
+ tomorrow := time.Now().AddDate(0, 0, 1)
// Put a few blocks in src using kcSrc
var srcLocators []string
@@ -245,7 +259,12 @@ func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
c.Check(rep, Equals, 2)
c.Check(err, Equals, nil)
- reader, blocklen, _, err := kcSrc.Get(hash)
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcSrc.Get(getLocator)
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(11))
all, err := ioutil.ReadAll(reader)
@@ -265,7 +284,12 @@ func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
c.Check(rep, Equals, 1)
c.Check(err, Equals, nil)
- reader, blocklen, _, err := kcDst.Get(hash)
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcDst.Get(getLocator)
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(11))
all, err := ioutil.ReadAll(reader)
@@ -285,7 +309,12 @@ func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
c.Check(rep, Equals, 1)
c.Check(err, Equals, nil)
- reader, blocklen, _, err := kcDst.Get(hash)
+ getLocator := hash
+ if enforcePermissions {
+ getLocator = keepclient.SignLocator(getLocator, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ }
+
+ reader, blocklen, _, err := kcDst.Get(getLocator)
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(12))
all, err := ioutil.ReadAll(reader)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list