[ARVADOS] updated: f735a9c42905fbc900cd1f3612f35b9438c730f2
Git user
git at public.curoverse.com
Tue Apr 12 10:18:01 EDT 2016
Summary of changes:
build/run-build-packages.sh | 5 +
build/run-tests.sh | 7 +-
sdk/cli/bin/crunch-job | 24 +++--
sdk/cwl/arvados_cwl/__init__.py | 1 +
sdk/cwl/test_with_arvbox.sh | 4 +
sdk/cwl/tests/test_submit.py | 57 +++++++++--
sdk/python/arvados/events.py | 78 ++++++++++++---
sdk/python/tests/test_websockets.py | 109 ++++++++++++++++++++-
services/dockercleaner/setup.py | 2 +-
services/fuse/arvados_fuse/fusedir.py | 5 +
services/fuse/tests/test_cache.py | 45 +++++++++
.../arvnodeman/computenode/dispatch/__init__.py | 15 ++-
.../arvnodeman/computenode/dispatch/slurm.py | 58 ++++++++---
.../arvnodeman/computenode/driver/__init__.py | 51 +++++++---
.../arvnodeman/computenode/driver/azure.py | 7 +-
.../arvnodeman/computenode/driver/ec2.py | 4 +-
.../arvnodeman/computenode/driver/gce.py | 7 +-
services/nodemanager/arvnodeman/daemon.py | 24 ++++-
.../nodemanager/tests/test_computenode_dispatch.py | 4 +-
.../tests/test_computenode_dispatch_slurm.py | 64 ++++++++++++
services/nodemanager/tests/test_daemon.py | 26 ++++-
services/nodemanager/tests/testutil.py | 25 +++++
tools/arvbox/bin/arvbox | 9 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 4 +
tools/arvbox/lib/arvbox/docker/common.sh | 8 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 8 +-
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 11 ++-
.../docker/service/arv-git-httpd/run-service | 12 ++-
.../service/crunch-dispatch-local/run-service | 8 +-
.../lib/arvbox/docker/service/keep-web/run-service | 12 ++-
.../arvbox/docker/service/keepproxy/run-service | 12 ++-
.../lib/arvbox/docker/service/sdk/run-service | 4 +-
tools/keep-block-check/keep-block-check.go | 2 +-
tools/keep-block-check/keep-block-check_test.go | 95 +++++++-----------
34 files changed, 645 insertions(+), 162 deletions(-)
create mode 100644 services/fuse/tests/test_cache.py
via f735a9c42905fbc900cd1f3612f35b9438c730f2 (commit)
via 55b017847fe87e00e7fb9c6bfd0444b83f8ca12c (commit)
via d5a21b4c6198d861a09ebeefec02bf78395f2e4e (commit)
via a8a65836a931811e869a3abba2244cc529ef2dd6 (commit)
via 4ab126d1574b7db2fdb5b0cea253b2df28d7b130 (commit)
via 2d112118532f0e059bc7e72d85ee63083a98156f (commit)
via 791bfa0fabd9b37ff8b52fb934f16b47461731da (commit)
via 92d7a504629d848927d5334b0a801cd3993a585b (commit)
via ba534230bcc23aaea69e5f256693b9e143200b63 (commit)
via 95253d987794df9c27db93b1b9f8be2d32229309 (commit)
via 7ddbf6b763f15bb84d7d897d373e37e85a8c3d6b (commit)
via 7c607284a048225b86b210c8c5397ff8a983d820 (commit)
via 0c848aa00c9af71ea2c7a575d8ee04b420a5eae7 (commit)
via 10f81953efe1884f170a7a2be14e6f1c32012c86 (commit)
via 47c2e05d4d86206e15fdf7d14e6f00637dd81f94 (commit)
via a4f8073c3c8520bf0e12f335e2c829a7f7f43dd1 (commit)
via 68d551b65a837664be72bf08aad55ab76d778d07 (commit)
via c244fbb0d880b45a44300e9ed650a32a954fd7d9 (commit)
via a2dc68267b5ff974460cef796820e7f8b90e04bc (commit)
via 86996de865c2b4d395c2a3977b05f7278924f627 (commit)
via e13c868ed9216b4ad414adc435a9f9ed5afe2b89 (commit)
via ae6ced575905d226963516bb7780c2bae391621c (commit)
via 448ec6cc59fe837eb71c416a82b8a72a959eddc8 (commit)
via fc6f80e4d1157b4b69bad852be1a7bb921fcb91b (commit)
via 241ef75ec8b6cf5dd14ce19fa068462adaeb0386 (commit)
via 2d18e31468b269e526cb6e4918912b60a48540b2 (commit)
via f37349e9436294472bdbdec820c5ec44ebe64430 (commit)
via 9c3a1a10b702ec7ee79bb0aa1beb3c227bec95dd (commit)
via 91b379c4a76b8278354903158a65e5d47babd363 (commit)
via b8ae9ce66c24055f3bf66b0956e15c44c9dd51f0 (commit)
via 8b93280fb2fae252e29263660e9591fcaa2d8722 (commit)
via a2c147e469bc4ccc0e788f2e72b2d94aa81ba368 (commit)
via 7b2cd50f499a1f9c5b21ab6aa641f14701402885 (commit)
via 55778ad90a9b869f514e729b3f85c12383f4e2a7 (commit)
via eea5f75077ca5d3adced3ad3d90f755d0fa5b3d8 (commit)
from 65efe2535e49058ccf8250f93e958ea653f55cd8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f735a9c42905fbc900cd1f3612f35b9438c730f2
Author: radhika <radhika at curoverse.com>
Date: Tue Apr 12 10:17:24 2016 -0400
8724: some more cleanup of tests.
diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index 646d417..ceafa8c 100644
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -198,7 +198,7 @@ func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey string, blo
for _, locator := range blockLocators {
current++
if verbose {
- log.Printf("Checking block %d of %d: %v", current, totalBlocks, locator)
+ log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator)
}
getLocator := locator
if blobSigningKey != "" {
diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go
index 5bd6062..65d3e34 100644
--- a/tools/keep-block-check/keep-block-check_test.go
+++ b/tools/keep-block-check/keep-block-check_test.go
@@ -30,10 +30,10 @@ type ServerRequiredSuite struct{}
type DoMainTestSuite struct{}
var kc *keepclient.KeepClient
-var keepServicesJSON, blobSigningKey string
+var logBuffer bytes.Buffer
+
var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0"
var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
-var allLocators []string
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
@@ -44,12 +44,7 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.ResetEnv()
}
-var logBuffer bytes.Buffer
-
func (s *ServerRequiredSuite) SetUpTest(c *C) {
- blobSigningKey = ""
- keepServicesJSON = ""
-
logOutput := io.MultiWriter(&logBuffer)
log.SetOutput(logOutput)
}
@@ -63,12 +58,7 @@ func (s *ServerRequiredSuite) TearDownTest(c *C) {
func (s *DoMainTestSuite) SetUpSuite(c *C) {
}
-var testArgs = []string{}
-
func (s *DoMainTestSuite) SetUpTest(c *C) {
- blobSigningKey = ""
- keepServicesJSON = ""
-
logOutput := io.MultiWriter(&logBuffer)
log.SetOutput(logOutput)
}
@@ -76,17 +66,13 @@ func (s *DoMainTestSuite) SetUpTest(c *C) {
func (s *DoMainTestSuite) TearDownTest(c *C) {
log.SetOutput(os.Stdout)
log.Printf("%v", logBuffer.String())
- testArgs = []string{}
}
-func setupKeepBlockCheck(c *C, enforcePermissions bool) {
+func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) {
var config apiConfig
config.APIHost = os.Getenv("ARVADOS_API_HOST")
config.APIToken = arvadostest.DataManagerToken
config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
- if enforcePermissions {
- blobSigningKey = arvadostest.BlobSigningKey
- }
// Start Keep servers
arvadostest.StartKeep(2, enforcePermissions)
@@ -98,8 +84,8 @@ func setupKeepBlockCheck(c *C, enforcePermissions bool) {
}
// Setup test data
-func setupTestData(c *C) {
- allLocators = []string{}
+func setupTestData(c *C) []string {
+ allLocators := []string{}
// Put a few blocks
for i := 0; i < 5; i++ {
@@ -107,6 +93,8 @@ func setupTestData(c *C) {
c.Check(err, IsNil)
allLocators = append(allLocators, strings.Split(hash, "+A")[0])
}
+
+ return allLocators
}
func setupConfigFile(c *C, fileName string) string {
@@ -161,42 +149,42 @@ func checkNoErrorsLogged(c *C, prefix, suffix string) {
}
func (s *ServerRequiredSuite) TestBlockCheck(c *C) {
- setupKeepBlockCheck(c, false)
- setupTestData(c)
- err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
+ err := performKeepBlockCheck(kc, "", allLocators, true)
c.Check(err, IsNil)
checkNoErrorsLogged(c, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
- setupKeepBlockCheck(c, true)
- setupTestData(c)
- err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
+ setupKeepBlockCheck(c, true, "")
+ allLocators := setupTestData(c)
+ err := performKeepBlockCheck(kc, arvadostest.BlobSigningKey, allLocators, true)
c.Check(err, IsNil)
checkNoErrorsLogged(c, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
- setupKeepBlockCheck(c, false)
- setupTestData(c)
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
- err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true)
+ err := performKeepBlockCheck(kc, "", allLocators, true)
c.Check(err, NotNil)
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) {
- setupKeepBlockCheck(c, false)
- setupTestData(c)
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
defer os.Remove(locatorFile)
locators, err := getBlockLocators(locatorFile, "aaa")
c.Check(err, IsNil)
- err = performKeepBlockCheck(kc, blobSigningKey, locators, true)
+ err = performKeepBlockCheck(kc, "", locators, true)
c.Check(err, NotNil)
// Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
@@ -204,26 +192,26 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C
}
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) {
- setupKeepBlockCheck(c, false)
- setupTestData(c)
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
allLocators = append(allLocators, TestHash)
allLocators = append(allLocators, TestHash2)
locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
defer os.Remove(locatorFile)
locators, err := getBlockLocators(locatorFile, "999")
c.Check(err, IsNil)
- err = performKeepBlockCheck(kc, blobSigningKey, locators, true)
- c.Check(err, IsNil) // there were no matching locators and hence no errors
+ err = performKeepBlockCheck(kc, "", locators, true)
+ c.Check(err, IsNil) // there were no matching locators in file and hence nothing was checked
}
func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) {
- setupKeepBlockCheck(c, true)
+ setupKeepBlockCheck(c, true, "")
setupTestData(c)
err := performKeepBlockCheck(kc, "badblobsigningkey", []string{TestHash, TestHash2}, false)
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403")
// verbose logging not requested
- c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, false)
+ c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false)
}
var testKeepServicesJSON = `{
@@ -254,17 +242,15 @@ var testKeepServicesJSON = `{
// Setup block-check using keepServicesJSON with fake keepservers.
// Expect error during performKeepBlockCheck due to unreachable keepservers.
func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
- keepServicesJSON = testKeepServicesJSON
- setupKeepBlockCheck(c, false)
- err := performKeepBlockCheck(kc, blobSigningKey, []string{TestHash, TestHash2}, true)
+ setupKeepBlockCheck(c, false, testKeepServicesJSON)
+ err := performKeepBlockCheck(kc, "", []string{TestHash, TestHash2}, true)
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "")
}
// Test keep-block-check initialization with keepServicesJSON
func (s *ServerRequiredSuite) TestKeepBlockCheck_InitializeWithKeepServicesJSON(c *C) {
- keepServicesJSON = testKeepServicesJSON
- setupKeepBlockCheck(c, false)
+ setupKeepBlockCheck(c, false, testKeepServicesJSON)
found := 0
for k := range kc.LocalRoots() {
if k == "zzzzz-bi6l4-123456789012340" || k == "zzzzz-bi6l4-123456789012341" {
@@ -293,16 +279,14 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) {
func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) {
args := []string{"-prefix", "a"}
- testArgs = append(testArgs, args...)
- err := doMain(testArgs)
+ err := doMain(args)
c.Check(err, NotNil)
c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true)
}
func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) {
args := []string{"-config", "no-such-file"}
- testArgs = append(testArgs, args...)
- err := doMain(testArgs)
+ err := doMain(args)
c.Check(err, NotNil)
c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}
@@ -311,14 +295,12 @@ func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) {
config := setupConfigFile(c, "config")
defer os.Remove(config)
- args := []string{"-config", config}
- testArgs = append(testArgs, args...)
-
// Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
- err := doMain(testArgs)
+ args := []string{"-config", config}
+ err := doMain(args)
c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
}
@@ -326,14 +308,11 @@ func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) {
config := setupConfigFile(c, "config")
defer os.Remove(config)
- args := []string{"-config", config, "-block-hash-file", "no-such-file"}
- testArgs = append(testArgs, args...)
-
- // Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
- err := doMain(testArgs)
+ args := []string{"-config", config, "-block-hash-file", "no-such-file"}
+ err := doMain(args)
c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}
@@ -349,11 +328,9 @@ func (s *DoMainTestSuite) Test_doMain(c *C) {
defer os.Remove(locatorFile)
args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
- testArgs = append(testArgs, args...)
-
- err := doMain(testArgs)
+ err := doMain(args)
c.Check(err, NotNil)
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
- c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, true)
+ c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
}
commit 55b017847fe87e00e7fb9c6bfd0444b83f8ca12c
Merge: 65efe25 d5a21b4
Author: radhika <radhika at curoverse.com>
Date: Mon Apr 11 23:37:22 2016 -0400
Merge branch 'master' into 8724-keep-block-check-script
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list