[ARVADOS] updated: e1c0499b7f75c98d2a5450358546e10ffd06faac

git at public.curoverse.com git at public.curoverse.com
Fri Nov 20 17:13:11 EST 2015


Summary of changes:
 services/datamanager/collection/collection.go      |  95 ++++----
 services/datamanager/datamanager.go                |  47 ++--
 services/datamanager/datamanager_test.go           |  99 +++++---
 services/datamanager/keep/keep.go                  | 125 +++++-----
 services/datamanager/keep/keep_test.go             | 265 ++++++++++++++++++++-
 services/datamanager/summary/file.go               |  22 +-
 services/datamanager/summary/pull_list.go          |  10 +-
 .../arvnodeman/computenode/dispatch/__init__.py    |  27 +++
 services/nodemanager/arvnodeman/daemon.py          |  19 +-
 services/nodemanager/arvnodeman/jobqueue.py        |  12 +-
 .../nodemanager/tests/test_computenode_dispatch.py |  25 +-
 services/nodemanager/tests/testutil.py             |   1 +
 12 files changed, 560 insertions(+), 187 deletions(-)

       via  e1c0499b7f75c98d2a5450358546e10ffd06faac (commit)
       via  c4023fb9b27ba31926faf530a4e59b9aee0989b3 (commit)
       via  1bac47fce49d9bf1515ff525244cb36ea733a2e3 (commit)
       via  190433c5fcece3e1675adad98684f5a93f837b2c (commit)
       via  3ed87fefbe5839572d7f1c7cfa9938ad85073088 (commit)
       via  8b3478bda6764b3f30aef69ec0a93729495296c0 (commit)
       via  c8be92df805483d492acfc1ce9516daf8f1fb39e (commit)
       via  055ac1cc91121d3f27405d4f45455a8de5a0e57e (commit)
       via  743341f8a70b126b82ef3293bd9f6c2bc47ec29a (commit)
       via  d14d34b534e6914dea107878212436dd995c6560 (commit)
       via  013a5d60ce55b627f314ed05000e4a7fa6b1cae8 (commit)
       via  74e4c93a1f333352914314b3a53172a0164a20e9 (commit)
       via  07f18157a5f8b72c20c0050c4617564effe91bd2 (commit)
       via  d1caf8fe2d5645fc8b611b32e22405e6e0632f20 (commit)
       via  1795bd4ba883c85c8277848f4e1f7a4cf9983ec8 (commit)
       via  d7a0d54602493bfdfe4a69887f0c2627ccc6d461 (commit)
       via  7b223f5209150fe125f611a8e218e5fbbf8a73ee (commit)
       via  c5602bf4ba4724376c2d5405302d240daaae506f (commit)
       via  c3f49b76d173818386f5c65db46b353e1d334d1e (commit)
       via  c0b9d41a5570862dcaac06b727d88be4d0ac9dc6 (commit)
       via  2255281ed7406e7c10027ed778f53ee28e6869fa (commit)
       via  da68b65f0264961902d0a26aea42466a69523148 (commit)
       via  dfb9a5e285e4275c26b6f959a04babd5a8ad836e (commit)
       via  192e5a9d12fb9fd16baff2b61af011611b671983 (commit)
       via  142953f3b9f4a7da5bd456c3e0c6ee93d0f54055 (commit)
       via  c79792f08d8faf712f70c67ed1216feabc5d9408 (commit)
       via  af6d31cba8346ac86bc0027eb0f675144fb43056 (commit)
       via  668cf29c1a7f8270ed5ef0bcfbe693618a8155da (commit)
      from  b10dd532008ee5ea7b994b07bb6f6e458786c0eb (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit e1c0499b7f75c98d2a5450358546e10ffd06faac
Merge: b10dd53 c4023fb
Author: radhika <radhika at curoverse.com>
Date:   Fri Nov 20 17:13:00 2015 -0500

    Merge branch 'master' into 7255-manifests-in-datamanager
    
    Conflicts:
    	services/datamanager/datamanager_test.go

diff --cc services/datamanager/datamanager_test.go
index 983e25c,53fb46b..5ed2b4f
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@@ -52,6 -58,6 +58,8 @@@ func SetupDataManagerTest(t *testing.T
  func TearDownDataManagerTest(t *testing.T) {
  	arvadostest.StopKeep(2)
  	arvadostest.StopAPI()
++	summary.WriteDataTo = ""
++	collection.HeapProfileFilename = ""
  }
  
  func putBlock(t *testing.T, data string) string {
@@@ -531,100 -529,55 +539,129 @@@ func TestRunDatamanagerAsNonAdminUser(
  	}
  }
  
+ func TestPutAndGetBlocks_NoErrorDuringSingleRun(t *testing.T) {
+ 	testOldBlocksNotDeletedOnDataManagerError(t, "", "", false, false)
+ }
+ 
+ func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) {
+ 	testOldBlocksNotDeletedOnDataManagerError(t, "/badwritetofile", "", true, true)
+ }
+ 
+ func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) {
+ 	testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", true, true)
+ }
+ 
+ /*
+   Create some blocks and backdate some of them.
+   Run datamanager while producing an error condition.
+   Verify that the blocks are hence not deleted.
+ */
+ func testOldBlocksNotDeletedOnDataManagerError(t *testing.T, writeDataTo string, heapProfileFile string, expectError bool, expectOldBlocks bool) {
+ 	defer TearDownDataManagerTest(t)
+ 	SetupDataManagerTest(t)
+ 
+ 	// Put some blocks and backdate them.
+ 	var oldUnusedBlockLocators []string
+ 	oldUnusedBlockData := "this block will have older mtime"
+ 	for i := 0; i < 5; i++ {
+ 		oldUnusedBlockLocators = append(oldUnusedBlockLocators, putBlock(t, fmt.Sprintf("%s%d", oldUnusedBlockData, i)))
+ 	}
+ 	backdateBlocks(t, oldUnusedBlockLocators)
+ 
+ 	// Run data manager
+ 	summary.WriteDataTo = writeDataTo
+ 	collection.HeapProfileFilename = heapProfileFile
+ 
+ 	err := singlerun(arv)
+ 	if !expectError {
+ 		if err != nil {
+ 			t.Fatalf("Got an error during datamanager singlerun: %v", err)
+ 		}
+ 	} else {
+ 		if err == nil {
+ 			t.Fatalf("Expected error during datamanager singlerun")
+ 		}
+ 	}
+ 	waitUntilQueuesFinishWork(t)
+ 
+ 	// Get block indexes and verify that all backdated blocks are not/deleted as expected
+ 	if expectOldBlocks {
+ 		verifyBlocks(t, nil, oldUnusedBlockLocators, 2)
+ 	} else {
+ 		verifyBlocks(t, oldUnusedBlockLocators, nil, 2)
+ 	}
+ }
++
 +// Create a collection with multiple streams and blocks
 +func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, numBlocks int) (string, []string) {
 +	defer switchToken(arvadostest.AdminToken)()
 +
 +	manifest := ""
 +	locators := make(map[string]bool)
 +	for s := 0; s < numStreams; s++ {
 +		manifest += fmt.Sprintf("./stream%d ", s)
 +		for b := 0; b < numBlocks; b++ {
 +			locator, _, err := keepClient.PutB([]byte(fmt.Sprintf("%s in stream %d and block %d", data, s, b)))
 +			if err != nil {
 +				t.Fatalf("Error creating block %d in stream %d: %v", b, s, err)
 +			}
 +			locators[strings.Split(locator, "+A")[0]] = true
 +			manifest += locator + " "
 +		}
 +		manifest += "0:1:dummyfile.txt\n"
 +	}
 +
 +	collection := make(Dict)
 +	err := arv.Create("collections",
 +		arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": manifest}},
 +		&collection)
 +
 +	if err != nil {
 +		t.Fatalf("Error creating collection %v", err)
 +	}
 +
 +	var locs []string
 +	for k, _ := range locators {
 +		locs = append(locs, k)
 +	}
 +
 +	return collection["uuid"].(string), locs
 +}
 +
 +/*
 +  Create collection with multiple streams and blocks; backdate the blocks and but do not delete the collection.
-   Create another collection with multiple streams and blocks; backdate it's blocks and delete the collection.
-   After datamanager run: expect blocks from the first collection, but none from the second collection.
++  Also, create stray block and backdate it.
++  After datamanager run: expect blocks from the collection, but not the stray block.
 +*/
 +func TestPutAndGetCollectionsWithMultipleStreamsAndBlocks(t *testing.T) {
 +	defer TearDownDataManagerTest(t)
 +	SetupDataManagerTest(t)
 +
 +	// create collection whose blocks will be backdated
 +	collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10)
 +	if collectionWithOldBlocks == "" {
 +		t.Fatalf("Failed to create collection with 1000 blocks")
 +	}
 +	if len(oldBlocks) != 1000 {
 +		t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks))
 +	}
 +
- 	// create another collection, whose blocks will be backdated and the collection will be deleted
- 	toBeDeletedCollection, toBeDeletedCollectionBlocks := createMultiStreamBlockCollection(t, "new block", 2, 5)
- 	if toBeDeletedCollection == "" {
- 		t.Fatalf("Failed to create collection with 10 blocks")
- 	}
- 	if len(toBeDeletedCollectionBlocks) != 10 {
- 		t.Fatalf("Not all blocks are created: expected %v, found %v", 10, len(toBeDeletedCollectionBlocks))
- 	}
- 
 +	// create a stray block that will be backdated
 +	strayOldBlock := putBlock(t, "this stray block is old")
 +
- 	// create another block that will not be backdated
- 	strayNewBlock := putBlock(t, "this stray block is new")
- 
- 	expected := []string{}
++	expected := []string{strayOldBlock}
 +	expected = append(expected, oldBlocks...)
- 	expected = append(expected, toBeDeletedCollectionBlocks...)
- 	expected = append(expected, strayOldBlock)
- 	expected = append(expected, strayNewBlock)
 +	verifyBlocks(t, nil, expected, 2)
 +
 +	// Backdate old blocks; but the collection still references these blocks
 +	backdateBlocks(t, oldBlocks)
 +
- 	// Backdate first block from the newer blocks and delete the collection; the rest are still be reachable
- 	backdateBlocks(t, toBeDeletedCollectionBlocks)
- 	deleteCollection(t, toBeDeletedCollection)
- 
 +	// also backdate the stray old block
 +	backdateBlocks(t, []string{strayOldBlock})
 +
 +	// run datamanager
 +	dataManagerSingleRun(t)
 +
- 	expected = []string{strayNewBlock}
- 	expected = append(expected, oldBlocks...)
- 
- 	notExpected := []string{strayOldBlock}
- 	notExpected = append(notExpected, toBeDeletedCollectionBlocks...)
- 
- 	verifyBlocks(t, notExpected, expected, 2)
++	// verify that strayOldBlock is not to be found, but the collections blocks are still there
++	verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
 +}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list