[ARVADOS] updated: fb9e39f948b75cbae8de76359f589a9f33ebd1ac
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 8 11:13:56 EDT 2015
Summary of changes:
sdk/go/arvadostest/run_servers.go | 18 ++++--
sdk/python/tests/run_test_server.py | 18 ++++--
.../arvnodeman/computenode/dispatch/slurm.py | 15 ++++-
.../tests/test_computenode_dispatch_slurm.py | 13 +++-
tools/keep-rsync/keep-rsync.go | 7 +-
tools/keep-rsync/keep-rsync_test.go | 74 ++++++++++++++++++++--
6 files changed, 120 insertions(+), 25 deletions(-)
via fb9e39f948b75cbae8de76359f589a9f33ebd1ac (commit)
via 9cb175a358a7ac410901827d5f7de6a5aabe52bf (commit)
via 0fa9a01d6fb567fa6e2f72cae3596da2c12c35fd (commit)
via 0bae20041e6dd78b086ba1354b8de9aee4e6baed (commit)
via 7c97dd88e541a0245272b8e93a33e4d2fe4e32cd (commit)
via 2a94b125b93a3aba204f55c37ecdc2876d81d642 (commit)
via 9c95a4c2dcd650627d524513e1e18596c8533ac0 (commit)
from 0384dc6b233e42537dc4e16e7de7283a2ef94897 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fb9e39f948b75cbae8de76359f589a9f33ebd1ac
Merge: 0384dc6 9cb175a
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 11:13:41 2015 -0400
Merge branch '7167-keep-rsync-test-setup' into 7167-keep-rsync
Conflicts:
tools/keep-rsync/keep-rsync_test.go
diff --cc tools/keep-rsync/keep-rsync_test.go
index 87cc0f5,c2a8112..5cfd30a
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@@ -167,85 -173,56 +173,139 @@@ func (s *ServerRequiredSuite) TestRsync
c.Check(foundIt, Equals, true)
}
+ // Test keep-rsync initialization, with src and dst keep servers with blobSingingKey.
+ // Do a Put and Get in src, both of which should succeed.
+ // Do a Put and Get in dst, both of which should succeed.
+ // Do a Get in dst for the src hash, which should raise block not found error.
+ // Do a Get in src for the dst hash, which should raise block not found error.
+ func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
+ setupRsync(c, true)
+
+ // Put a block in src using kcSrc and Get it
+ srcData := []byte("test-data1")
+ locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData))
+
+ hash, rep, err := kcSrc.PutB(srcData)
+ c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ tomorrow := time.Now().AddDate(0, 0, 1)
+ signedLocator := keepclient.SignLocator(locatorInSrc, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+
+ reader, blocklen, _, err := kcSrc.Get(signedLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(10))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, srcData)
+
+ // Put a different block in src using kcSrc and Get it
+ dstData := []byte("test-data2")
+ locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData))
+
+ hash, rep, err = kcDst.PutB(dstData)
+ c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ signedLocator = keepclient.SignLocator(locatorInDst, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+
+ reader, blocklen, _, err = kcDst.Get(signedLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(10))
+ all, err = ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, dstData)
+
+ // Get srcLocator using kcDst should fail with NotFound error
+ signedLocator = keepclient.SignLocator(locatorInSrc, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ _, _, _, err = kcDst.Get(locatorInSrc)
+ c.Assert(err.Error(), Equals, "Block not found")
+
+ // Get dstLocator using kcSrc should fail with NotFound error
+ signedLocator = keepclient.SignLocator(locatorInDst, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+ _, _, _, err = kcSrc.Get(locatorInDst)
+ c.Assert(err.Error(), Equals, "Block not found")
+ }
++
+// Put 5 blocks in src. Put 2 of those blocks in dst
+// Hence there are 3 additional blocks in src
+// Also, put 2 extra blocks in dts; they are hence only in dst
+// Run rsync and verify that those 7 blocks are now available in dst
+func (s *ServerRequiredSuite) TestKeepRsync(c *C) {
- setupRsync(c)
++ setupRsync(c, false)
+
+ // Put a few blocks in src using kcSrc
+ var srcLocators []string
+ for i := 0; i < 5; i++ {
+ data := []byte(fmt.Sprintf("test-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcSrc.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+11(\+.+)?$`, hash))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kcSrc.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(11))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ srcLocators = append(srcLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ // Put just two of those blocks in dst using kcDst
+ var dstLocators []string
+ for i := 0; i < 2; i++ {
+ data := []byte(fmt.Sprintf("test-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcDst.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+11(\+.+)?$`, hash))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kcDst.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(11))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ dstLocators = append(dstLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ // Put two more blocks in dst; they are not in src at all
+ var extraDstLocators []string
+ for i := 0; i < 2; i++ {
+ data := []byte(fmt.Sprintf("other-data-%d", i))
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcDst.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+12(\+.+)?$`, hash))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kcDst.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(12))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ extraDstLocators = append(extraDstLocators, fmt.Sprintf("%s+%d", hash, blocklen))
+ }
+
+ err := performKeepRsync()
+ c.Check(err, Equals, nil)
+
+ // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
+ dstIndex, err := getUniqueLocators(kcDst, "")
+ c.Check(err, Equals, nil)
+ for _, locator := range srcLocators {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
+ for _, locator := range extraDstLocators {
+ _, ok := dstIndex[locator]
+ c.Assert(ok, Equals, true)
+ }
+}
commit 9cb175a358a7ac410901827d5f7de6a5aabe52bf
Author: radhika <radhika at curoverse.com>
Date: Thu Oct 8 11:06:57 2015 -0400
7167: set enforce_permissions to true if blob signing key argument is provided.
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 6bfbbac..313c188 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -14,6 +14,7 @@ import (
var (
srcConfig arvadosclient.APIConfig
dstConfig arvadosclient.APIConfig
+ blobSigningKey string
srcKeepServicesJSON string
dstKeepServicesJSON string
replications int
@@ -30,7 +31,7 @@ func main() {
"",
"Source configuration filename with full path that contains "+
"an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
- "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
flag.StringVar(
&dstConfigFile,
@@ -38,7 +39,7 @@ func main() {
"",
"Destination configuration filename with full path that contains "+
"an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
- "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
flag.StringVar(
&srcKeepServicesJSON,
@@ -121,6 +122,8 @@ func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
config.APIHostInsecure = matchTrue.MatchString(kv[1])
case "ARVADOS_EXTERNAL_CLIENT":
config.ExternalClient = matchTrue.MatchString(kv[1])
+ case "ARVADOS_BLOB_SIGNING_KEY":
+ blobSigningKey = kv[1]
}
}
return config, nil
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index a4d26fd..c2a8112 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"testing"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
@@ -42,7 +43,7 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
// The test setup hence tweaks keep-rsync initialzation to achieve this.
// First invoke initializeKeepRsync and then invoke StartKeepAdditional
// to create the keep servers to be used as destination.
-func setupRsync(c *C) {
+func setupRsync(c *C, enforcePermissions bool) {
// srcConfig
srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
@@ -54,17 +55,20 @@ func setupRsync(c *C) {
dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
replications = 1
+ if enforcePermissions {
+ blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
+ }
// Start API and Keep servers
arvadostest.StartAPI()
- arvadostest.StartKeep()
+ arvadostest.StartKeepWithParams(false, enforcePermissions)
// initialize keep-rsync
err := initializeKeepRsync()
c.Assert(err, Equals, nil)
// Create two more keep servers to be used as destination
- arvadostest.StartKeepWithParams(true, false)
+ arvadostest.StartKeepWithParams(true, enforcePermissions)
// load kcDst
kcDst, err = keepclient.MakeKeepClient(&arvDst)
@@ -81,7 +85,8 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
fileContent := "ARVADOS_API_HOST=testhost\n"
fileContent += "ARVADOS_API_TOKEN=testtoken\n"
- fileContent += "ARVADOS_API_HOST_INSECURE=true"
+ fileContent += "ARVADOS_API_HOST_INSECURE=true\n"
+ fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
_, err = file.Write([]byte(fileContent))
@@ -92,6 +97,7 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
c.Assert(config.APIToken, Equals, "testtoken")
c.Assert(config.APIHostInsecure, Equals, true)
c.Assert(config.ExternalClient, Equals, false)
+ c.Assert(blobSigningKey, Equals, "abcdefg")
}
// Test keep-rsync initialization, with src and dst keep servers.
@@ -100,7 +106,7 @@ func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
// Do a Get in dst for the src hash, which should raise block not found error.
// Do a Get in src for the dst hash, which should raise block not found error.
func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
- setupRsync(c)
+ setupRsync(c, false)
// Put a block in src using kcSrc and Get it
srcData := []byte("test-data1")
@@ -145,7 +151,7 @@ func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) {
func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
srcKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
- setupRsync(c)
+ setupRsync(c, false)
localRoots := kcSrc.LocalRoots()
c.Check(localRoots != nil, Equals, true)
@@ -166,3 +172,57 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
}
c.Check(foundIt, Equals, true)
}
+
+// Test keep-rsync initialization, with src and dst keep servers with blobSingingKey.
+// Do a Put and Get in src, both of which should succeed.
+// Do a Put and Get in dst, both of which should succeed.
+// Do a Get in dst for the src hash, which should raise block not found error.
+// Do a Get in src for the dst hash, which should raise block not found error.
+func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) {
+ setupRsync(c, true)
+
+ // Put a block in src using kcSrc and Get it
+ srcData := []byte("test-data1")
+ locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData))
+
+ hash, rep, err := kcSrc.PutB(srcData)
+ c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ tomorrow := time.Now().AddDate(0, 0, 1)
+ signedLocator := keepclient.SignLocator(locatorInSrc, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+
+ reader, blocklen, _, err := kcSrc.Get(signedLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(10))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, srcData)
+
+ // Put a different block in src using kcSrc and Get it
+ dstData := []byte("test-data2")
+ locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData))
+
+ hash, rep, err = kcDst.PutB(dstData)
+ c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst))
+ c.Check(rep, Equals, 1)
+ c.Check(err, Equals, nil)
+
+ signedLocator = keepclient.SignLocator(locatorInDst, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+
+ reader, blocklen, _, err = kcDst.Get(signedLocator)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(10))
+ all, err = ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, dstData)
+
+ // Get srcLocator using kcDst should fail with NotFound error
+ signedLocator = keepclient.SignLocator(locatorInSrc, arvDst.ApiToken, tomorrow, []byte(blobSigningKey))
+ _, _, _, err = kcDst.Get(locatorInSrc)
+ c.Assert(err.Error(), Equals, "Block not found")
+
+ // Get dstLocator using kcSrc should fail with NotFound error
+ signedLocator = keepclient.SignLocator(locatorInDst, arvSrc.ApiToken, tomorrow, []byte(blobSigningKey))
+ _, _, _, err = kcSrc.Get(locatorInDst)
+ c.Assert(err.Error(), Equals, "Block not found")
+}
commit 0fa9a01d6fb567fa6e2f72cae3596da2c12c35fd
Author: radhika <radhika at curoverse.com>
Date: Wed Oct 7 18:24:51 2015 -0400
7167: add --keep-enforce-permissions to run_test_servers.py
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index a175136..ed75b4b 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -98,19 +98,29 @@ func StopAPI() {
exec.Command("python", "run_test_server.py", "stop").Run()
}
+// StartKeep is used to start keep servers
+// with keepExisting = false and enforcePermissions = false
func StartKeep() {
- StartKeepAdditional(false)
+ StartKeepWithParams(false, false)
}
-func StartKeepAdditional(keepExisting bool) {
+// StartKeepWithParams is used to start keep servers while specifying
+// keepExisting and enforcePermissions parameters.
+func StartKeepWithParams(keepExisting bool, enforcePermissions bool) {
cwd, _ := os.Getwd()
defer os.Chdir(cwd)
chdirToPythonTests()
- cmd := exec.Command("python", "run_test_server.py", "start_keep")
+ keepExistingStr := ""
if keepExisting {
- cmd = exec.Command("python", "run_test_server.py", "start_keep", "--keep_existing", "true")
+ keepExistingStr = "true"
}
+ enforcePermissionsStr := ""
+ if enforcePermissions {
+ enforcePermissionsStr = "true"
+ }
+ cmd := exec.Command("python", "run_test_server.py", "start_keep", "--keep-existing", keepExistingStr, "--keep-enforce-permissions", enforcePermissionsStr)
+
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatalf("Setting up stderr pipe: %s", err)
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index fba9bb6..a7b60e7 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -43,7 +43,8 @@ if not os.path.exists(TEST_TMPDIR):
my_api_host = None
_cached_config = {}
-keep_existing = None
+keep_existing = False
+enforce_permissions = False
def find_server_pid(PID_PATH, wait=10):
now = time.time()
@@ -324,8 +325,8 @@ def _start_keep(n, keep_args):
return port
-def run_keep(blob_signing_key=None, enforce_permissions=False):
- if keep_existing is None:
+def run_keep(blob_signing_key=None):
+ if not keep_existing:
stop_keep()
keep_args = {}
@@ -354,7 +355,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
start_index = 0
end_index = 2
- if keep_existing is not None:
+ if keep_existing:
start_index = 2
end_index = 3
for d in range(start_index, end_index):
@@ -605,10 +606,15 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
- parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
+ parser.add_argument('--keep-existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
+ parser.add_argument('--keep-enforce-permissions', type=str, help="Enforce keep permissions")
+
args = parser.parse_args()
- keep_existing = args.keep_existing
+ if args.keep_existing == 'true':
+ keep_existing = True
+ if args.keep_enforce_permissions == 'true':
+ enforce_permissions = True
if args.action not in actions:
print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions), file=sys.stderr)
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 4fdc6a8..a4d26fd 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -64,7 +64,7 @@ func setupRsync(c *C) {
c.Assert(err, Equals, nil)
// Create two more keep servers to be used as destination
- arvadostest.StartKeepAdditional(true)
+ arvadostest.StartKeepWithParams(true, false)
// load kcDst
kcDst, err = keepclient.MakeKeepClient(&arvDst)
commit 0bae20041e6dd78b086ba1354b8de9aee4e6baed
Merge: 96dab84 7c97dd8
Author: radhika <radhika at curoverse.com>
Date: Wed Oct 7 16:50:04 2015 -0400
Merge branch 'master' into 7167-keep-rsync-test-setup
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list