[ARVADOS] updated: e1c0499b7f75c98d2a5450358546e10ffd06faac
git at public.curoverse.com
git at public.curoverse.com
Fri Nov 20 17:13:11 EST 2015
Summary of changes:
services/datamanager/collection/collection.go | 95 ++++----
services/datamanager/datamanager.go | 47 ++--
services/datamanager/datamanager_test.go | 99 +++++---
services/datamanager/keep/keep.go | 125 +++++-----
services/datamanager/keep/keep_test.go | 265 ++++++++++++++++++++-
services/datamanager/summary/file.go | 22 +-
services/datamanager/summary/pull_list.go | 10 +-
.../arvnodeman/computenode/dispatch/__init__.py | 27 +++
services/nodemanager/arvnodeman/daemon.py | 19 +-
services/nodemanager/arvnodeman/jobqueue.py | 12 +-
.../nodemanager/tests/test_computenode_dispatch.py | 25 +-
services/nodemanager/tests/testutil.py | 1 +
12 files changed, 560 insertions(+), 187 deletions(-)
via e1c0499b7f75c98d2a5450358546e10ffd06faac (commit)
via c4023fb9b27ba31926faf530a4e59b9aee0989b3 (commit)
via 1bac47fce49d9bf1515ff525244cb36ea733a2e3 (commit)
via 190433c5fcece3e1675adad98684f5a93f837b2c (commit)
via 3ed87fefbe5839572d7f1c7cfa9938ad85073088 (commit)
via 8b3478bda6764b3f30aef69ec0a93729495296c0 (commit)
via c8be92df805483d492acfc1ce9516daf8f1fb39e (commit)
via 055ac1cc91121d3f27405d4f45455a8de5a0e57e (commit)
via 743341f8a70b126b82ef3293bd9f6c2bc47ec29a (commit)
via d14d34b534e6914dea107878212436dd995c6560 (commit)
via 013a5d60ce55b627f314ed05000e4a7fa6b1cae8 (commit)
via 74e4c93a1f333352914314b3a53172a0164a20e9 (commit)
via 07f18157a5f8b72c20c0050c4617564effe91bd2 (commit)
via d1caf8fe2d5645fc8b611b32e22405e6e0632f20 (commit)
via 1795bd4ba883c85c8277848f4e1f7a4cf9983ec8 (commit)
via d7a0d54602493bfdfe4a69887f0c2627ccc6d461 (commit)
via 7b223f5209150fe125f611a8e218e5fbbf8a73ee (commit)
via c5602bf4ba4724376c2d5405302d240daaae506f (commit)
via c3f49b76d173818386f5c65db46b353e1d334d1e (commit)
via c0b9d41a5570862dcaac06b727d88be4d0ac9dc6 (commit)
via 2255281ed7406e7c10027ed778f53ee28e6869fa (commit)
via da68b65f0264961902d0a26aea42466a69523148 (commit)
via dfb9a5e285e4275c26b6f959a04babd5a8ad836e (commit)
via 192e5a9d12fb9fd16baff2b61af011611b671983 (commit)
via 142953f3b9f4a7da5bd456c3e0c6ee93d0f54055 (commit)
via c79792f08d8faf712f70c67ed1216feabc5d9408 (commit)
via af6d31cba8346ac86bc0027eb0f675144fb43056 (commit)
via 668cf29c1a7f8270ed5ef0bcfbe693618a8155da (commit)
from b10dd532008ee5ea7b994b07bb6f6e458786c0eb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e1c0499b7f75c98d2a5450358546e10ffd06faac
Merge: b10dd53 c4023fb
Author: radhika <radhika at curoverse.com>
Date: Fri Nov 20 17:13:00 2015 -0500
Merge branch 'master' into 7255-manifests-in-datamanager
Conflicts:
services/datamanager/datamanager_test.go
diff --cc services/datamanager/datamanager_test.go
index 983e25c,53fb46b..5ed2b4f
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@@ -52,6 -58,6 +58,8 @@@ func SetupDataManagerTest(t *testing.T
func TearDownDataManagerTest(t *testing.T) {
arvadostest.StopKeep(2)
arvadostest.StopAPI()
++ summary.WriteDataTo = ""
++ collection.HeapProfileFilename = ""
}
func putBlock(t *testing.T, data string) string {
@@@ -531,100 -529,55 +539,129 @@@ func TestRunDatamanagerAsNonAdminUser(
}
}
+ func TestPutAndGetBlocks_NoErrorDuringSingleRun(t *testing.T) {
+ testOldBlocksNotDeletedOnDataManagerError(t, "", "", false, false)
+ }
+
+ func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) {
+ testOldBlocksNotDeletedOnDataManagerError(t, "/badwritetofile", "", true, true)
+ }
+
+ func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) {
+ testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", true, true)
+ }
+
+ /*
+ Create some blocks and backdate some of them.
+ Run datamanager while producing an error condition.
+ Verify that the blocks are hence not deleted.
+ */
+ func testOldBlocksNotDeletedOnDataManagerError(t *testing.T, writeDataTo string, heapProfileFile string, expectError bool, expectOldBlocks bool) {
+ defer TearDownDataManagerTest(t)
+ SetupDataManagerTest(t)
+
+ // Put some blocks and backdate them.
+ var oldUnusedBlockLocators []string
+ oldUnusedBlockData := "this block will have older mtime"
+ for i := 0; i < 5; i++ {
+ oldUnusedBlockLocators = append(oldUnusedBlockLocators, putBlock(t, fmt.Sprintf("%s%d", oldUnusedBlockData, i)))
+ }
+ backdateBlocks(t, oldUnusedBlockLocators)
+
+ // Run data manager
+ summary.WriteDataTo = writeDataTo
+ collection.HeapProfileFilename = heapProfileFile
+
+ err := singlerun(arv)
+ if !expectError {
+ if err != nil {
+ t.Fatalf("Got an error during datamanager singlerun: %v", err)
+ }
+ } else {
+ if err == nil {
+ t.Fatalf("Expected error during datamanager singlerun")
+ }
+ }
+ waitUntilQueuesFinishWork(t)
+
+ // Get block indexes and verify that all backdated blocks are not/deleted as expected
+ if expectOldBlocks {
+ verifyBlocks(t, nil, oldUnusedBlockLocators, 2)
+ } else {
+ verifyBlocks(t, oldUnusedBlockLocators, nil, 2)
+ }
+ }
++
+// Create a collection with multiple streams and blocks
+func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, numBlocks int) (string, []string) {
+ defer switchToken(arvadostest.AdminToken)()
+
+ manifest := ""
+ locators := make(map[string]bool)
+ for s := 0; s < numStreams; s++ {
+ manifest += fmt.Sprintf("./stream%d ", s)
+ for b := 0; b < numBlocks; b++ {
+ locator, _, err := keepClient.PutB([]byte(fmt.Sprintf("%s in stream %d and block %d", data, s, b)))
+ if err != nil {
+ t.Fatalf("Error creating block %d in stream %d: %v", b, s, err)
+ }
+ locators[strings.Split(locator, "+A")[0]] = true
+ manifest += locator + " "
+ }
+ manifest += "0:1:dummyfile.txt\n"
+ }
+
+ collection := make(Dict)
+ err := arv.Create("collections",
+ arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": manifest}},
+ &collection)
+
+ if err != nil {
+ t.Fatalf("Error creating collection %v", err)
+ }
+
+ var locs []string
+ for k, _ := range locators {
+ locs = append(locs, k)
+ }
+
+ return collection["uuid"].(string), locs
+}
+
+/*
+ Create collection with multiple streams and blocks; backdate the blocks and but do not delete the collection.
- Create another collection with multiple streams and blocks; backdate it's blocks and delete the collection.
- After datamanager run: expect blocks from the first collection, but none from the second collection.
++ Also, create stray block and backdate it.
++ After datamanager run: expect blocks from the collection, but not the stray block.
+*/
+func TestPutAndGetCollectionsWithMultipleStreamsAndBlocks(t *testing.T) {
+ defer TearDownDataManagerTest(t)
+ SetupDataManagerTest(t)
+
+ // create collection whose blocks will be backdated
+ collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10)
+ if collectionWithOldBlocks == "" {
+ t.Fatalf("Failed to create collection with 1000 blocks")
+ }
+ if len(oldBlocks) != 1000 {
+ t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks))
+ }
+
- // create another collection, whose blocks will be backdated and the collection will be deleted
- toBeDeletedCollection, toBeDeletedCollectionBlocks := createMultiStreamBlockCollection(t, "new block", 2, 5)
- if toBeDeletedCollection == "" {
- t.Fatalf("Failed to create collection with 10 blocks")
- }
- if len(toBeDeletedCollectionBlocks) != 10 {
- t.Fatalf("Not all blocks are created: expected %v, found %v", 10, len(toBeDeletedCollectionBlocks))
- }
-
+ // create a stray block that will be backdated
+ strayOldBlock := putBlock(t, "this stray block is old")
+
- // create another block that will not be backdated
- strayNewBlock := putBlock(t, "this stray block is new")
-
- expected := []string{}
++ expected := []string{strayOldBlock}
+ expected = append(expected, oldBlocks...)
- expected = append(expected, toBeDeletedCollectionBlocks...)
- expected = append(expected, strayOldBlock)
- expected = append(expected, strayNewBlock)
+ verifyBlocks(t, nil, expected, 2)
+
+ // Backdate old blocks; but the collection still references these blocks
+ backdateBlocks(t, oldBlocks)
+
- // Backdate first block from the newer blocks and delete the collection; the rest are still be reachable
- backdateBlocks(t, toBeDeletedCollectionBlocks)
- deleteCollection(t, toBeDeletedCollection)
-
+ // also backdate the stray old block
+ backdateBlocks(t, []string{strayOldBlock})
+
+ // run datamanager
+ dataManagerSingleRun(t)
+
- expected = []string{strayNewBlock}
- expected = append(expected, oldBlocks...)
-
- notExpected := []string{strayOldBlock}
- notExpected = append(notExpected, toBeDeletedCollectionBlocks...)
-
- verifyBlocks(t, notExpected, expected, 2)
++ // verify that strayOldBlock is not to be found, but the collections blocks are still there
++ verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list