[ARVADOS] updated: 3272913e4a91f5b80b256f9f09e4d4f432d07058
git at public.curoverse.com
git at public.curoverse.com
Fri Nov 20 12:27:05 EST 2015
Summary of changes:
apps/workbench/app/views/application/404.html.erb | 7 +-
.../app/views/application/_report_error.html.erb | 46 ++--
.../test/controllers/projects_controller_test.rb | 2 +-
sdk/cli/bin/crunch-job | 30 ++-
sdk/python/arvados/events.py | 25 +--
sdk/python/arvados/keep.py | 117 +++++++---
sdk/python/tests/test_keep_client.py | 7 +
services/datamanager/datamanager_test.go | 184 +++++-----------
services/fuse/arvados_fuse/__init__.py | 20 +-
services/fuse/bin/arv-mount | 67 ++++++
services/fuse/setup.py | 2 +-
services/fuse/tests/test_mount.py | 4 +
.../arvnodeman/computenode/driver/__init__.py | 2 +
.../arvnodeman/computenode/driver/azure.py | 8 +
.../arvnodeman/computenode/driver/dummy.py | 1 +
.../arvnodeman/computenode/driver/ec2.py | 8 +
services/nodemanager/arvnodeman/config.py | 1 +
services/nodemanager/arvnodeman/daemon.py | 123 +++++++----
services/nodemanager/arvnodeman/jobqueue.py | 5 +-
services/nodemanager/arvnodeman/launcher.py | 8 +-
services/nodemanager/doc/azure.example.cfg | 35 ++-
services/nodemanager/doc/ec2.example.cfg | 28 ++-
services/nodemanager/doc/gce.example.cfg | 30 ++-
services/nodemanager/tests/test_daemon.py | 239 +++++++++++++++++----
services/nodemanager/tests/test_jobqueue.py | 50 ++++-
services/nodemanager/tests/testutil.py | 24 ++-
26 files changed, 737 insertions(+), 336 deletions(-)
via 3272913e4a91f5b80b256f9f09e4d4f432d07058 (commit)
via c4ae463b42ee22180982d78289e6898224e508e4 (commit)
via 76683cdf4d90868944a51e6b433dce3ff8f5495e (commit)
via 94442974428c037df716204296a3579dbc645bbc (commit)
via 23c3347fe9779c3826179285de86e8540d46d749 (commit)
via 41694642af159d156751afc7dc964d46e0fd8a81 (commit)
via 73a127e5492bc2711530b2f5a7c30a5021232d40 (commit)
via d6f163ed99c6ae234c551bb3ecd23c7af8610c8f (commit)
via 1c859533008748afa273aa7c379ef5aec88d4ce6 (commit)
via 155b1aa3772eb2d9421c5bfc678320401eb3b058 (commit)
via 1d6bcbbdbc374aa25b70c85717f987d2675c1205 (commit)
via 04784067f7f39e15161cc165365aa2d0552dc219 (commit)
via dec876f5ea5d88ea9674edc01b86ed2a4fa95078 (commit)
via db55813cc9f24c699db1d3386a56bc0a36debab0 (commit)
via 58692c916bb6dfe2997838ca4147109d9410c86a (commit)
via 99ce6a94ab4b6df30f727c6d07039aa55c2bad32 (commit)
via b2ca3a093f36370719d4a6e89ba46a45111f4f63 (commit)
via 55e78d1096d8a76b77911bcb1561412cda7ffb8a (commit)
via d01d6bdf901b49e8856e199a131fb263f98b370e (commit)
via 4c5e5f9070e4e0fd9b07757c546b3bae48f2b664 (commit)
via c112d8551b75514b9ccbde44ee3e37904051dbb1 (commit)
via f4d556b93687fc915897c9cec97afebd7b86572e (commit)
via f34c10a5b22e3f9941f2cfe60f6f9d7a81319b6f (commit)
via f3b78f5c0690a93d4cc9370519f970ded430c453 (commit)
via ba9ea75ee6d6322f5d2a9da0b1c01c2738c0927d (commit)
via 05c4cc66de22d6c4071af69d902f8864b9f2907d (commit)
via f16505d89d60d3bd5abc04762712f4ddf78e39fe (commit)
via f684cda0d2a332f1d2054e7ac8e9de3cb67e23a9 (commit)
via 1ad098e2521d57ba6d66d0a0d9dfffab76061924 (commit)
via 80d30b6a1662e03d56d33f7b29a211d0b3413e2c (commit)
via 6c8cf507184707e1529ef98ffd98269835a39243 (commit)
from b36c3f351209d55d8512951bb0a10de001b06890 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3272913e4a91f5b80b256f9f09e4d4f432d07058
Author: radhika <radhika at curoverse.com>
Date: Fri Nov 20 12:26:05 2015 -0500
7255: improve createMultiStreamBlockCollection to create collection with multiple blocks per stream.
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 374b901..094cd44 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -531,170 +531,92 @@ func TestRunDatamanagerAsNonAdminUser(t *testing.T) {
}
}
-/*
- Create a collection with multiple streams and blocks using arv-put
- Generated manifest will be for the format:
- ./stream034036412 ae1426cd6bb371ffd4e8eedf5e9f8288+265+A28b017187d22154d8ae2836d5644312196ddede9 at 565f7801 0:53:temp-test-file101128526 53:53:temp-test-file525370191 106:53:temp-test-file767521515 159:53:temp-test-file914425264 212:53:temp-test-file989413461
- ./stream043441762 2b421156d2751447d6fa22fda6742769+265+A4766df9d5a455d76ec3fbd5d1ceea6ab1207967d at 565f7801 0:53:temp-test-file016029341 53:53:temp-test-file546920630 106:53:temp-test-file688432627 159:53:temp-test-file823040996 212:53:temp-test-file843401817
-*/
-func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, numBlocks int) string {
- tempdir, err := ioutil.TempDir(os.TempDir(), "temp-test-dir")
- if err != nil {
- t.Fatalf("Error creating tempdir %s", err)
- }
- defer os.Remove(tempdir)
-
- for i := 0; i < numStreams; i++ {
- stream, err := ioutil.TempDir(tempdir, "stream")
- if err != nil {
- t.Fatalf("Error creating stream tempdir %s", err)
- }
- defer os.Remove(stream)
-
- for j := 0; j < numBlocks; j++ {
- tempfile, err := ioutil.TempFile(stream, "temp-test-file")
- if err != nil {
- t.Fatalf("Error creating tempfile %s", err)
- }
- defer os.Remove(tempfile.Name())
+// Create a collection with multiple streams and blocks
+func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, numBlocks int) (string, []string) {
+ defer switchToken(arvadostest.AdminToken)()
- _, err = tempfile.Write([]byte(fmt.Sprintf("%s%d", data, i)))
+ manifest := ""
+ var locators []string
+ for s := 0; s < numStreams; s++ {
+ manifest += fmt.Sprintf("./stream%d ", s)
+ for b := 0; b < numBlocks; b++ {
+ locator, _, err := keepClient.PutB([]byte(fmt.Sprintf("%s in stream %d and block %d", data, s, b)))
if err != nil {
- t.Fatalf("Error writing to tempfile %v", err)
+ t.Fatalf("Error creating block %d in stream %d: %v", b, s, err)
}
+ locators = append(locators, strings.Split(locator, "+A")[0])
+ manifest += locator + " "
}
+ manifest += "0:1:dummyfile.txt\n"
}
- output, err := exec.Command("arv-put", tempdir).Output()
- if err != nil {
- t.Fatalf("Error running arv-put %s", err)
- }
+ collection := make(Dict)
+ err := arv.Create("collections",
+ arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": manifest}},
+ &collection)
- uuid := string(output[0:27]) // trim terminating char
- return uuid
-}
-
-func geLocatorsFromCollection(t *testing.T, uuid string) []string {
- manifest := getCollection(t, uuid)["manifest_text"].(string)
-
- locators := []string{}
- splits := strings.Split(manifest, " ")
- for _, locator := range splits {
- match := locatorMatcher.FindStringSubmatch(locator)
- if match != nil {
- locators = append(locators, match[1]+"+"+match[2])
- }
+ if err != nil {
+ t.Fatalf("Error creating collection %v", err)
}
- return locators
+ return collection["uuid"].(string), locators
}
/*
- Create collection with multiple streams and blocks; backdate the blocks and delete collection.
- Create another collection with multiple streams and blocks; backdate it's first block and delete the collection
- After datamanager run: expect only the undeleted blocks from second collection, and none of the backdated blocks.
+ Create collection with multiple streams and blocks; backdate the blocks and but do not delete the collection.
+ Create another collection with multiple streams and blocks; backdate it's blocks and delete the collection.
+ After datamanager run: expect blocks from the first collection, but none from the second collection.
*/
func TestPutAndGetCollectionsWithMultipleStreamsAndBlocks(t *testing.T) {
defer TearDownDataManagerTest(t)
SetupDataManagerTest(t)
- // Put some blocks which will be backdated later on
- collectionWithOldBlocks := createMultiStreamBlockCollection(t, "to be deleted collection with old blocks", 5, 5)
- oldBlocks := geLocatorsFromCollection(t, collectionWithOldBlocks)
-
- collectionWithNewerBlocks := createMultiStreamBlockCollection(t, "to be deleted collection with newer and older blocks", 5, 5)
- newerBlocks := geLocatorsFromCollection(t, collectionWithNewerBlocks)
-
- expected := []string{}
- expected = append(expected, oldBlocks...)
- expected = append(expected, newerBlocks...)
- verifyBlocks(t, nil, expected, 2)
-
- // Backdate old blocks and delete the collection
- backdateBlocks(t, oldBlocks)
- deleteCollection(t, collectionWithOldBlocks)
-
- // Backdate first block from the newer blocks and delete the collection; the rest are still be reachable
- backdateBlocks(t, newerBlocks[0:1])
- deleteCollection(t, collectionWithNewerBlocks)
-
- // run datamanager
- dataManagerSingleRun(t)
-
- notExpected := []string{}
- notExpected = append(notExpected, oldBlocks...)
- notExpected = append(notExpected, newerBlocks[0])
-
- verifyBlocks(t, notExpected, newerBlocks[1:], 2)
-}
-
-/*
- Create a collection with multiple blocks in one stream using arv-put
- Generated manifest will be for the format:
- . 83d2e2d0938718a56c9b0c518a4b2930+41+A4b671b8c7525c0af302365b03a44406999e42eec at 565f7809 0:41:temp-test-file053866981
- . cb790454ba6cc9a3ffab377937e06225+41+Ab5460755c3480fb899025b74dc59fadb71402bfc at 565f7809 0:41:temp-test-file213181952
-*/
-func createMultiBlockCollection(t *testing.T, data string, numBlocks int) string {
- tempdir, err := ioutil.TempDir(os.TempDir(), "temp-test-dir")
- defer os.Remove(tempdir)
-
- filenames := []string{}
- for i := 0; i < numBlocks; i++ {
- tempfile, err := ioutil.TempFile(tempdir, "temp-test-file")
- defer os.Remove(tempfile.Name())
-
- _, err = tempfile.Write([]byte(fmt.Sprintf("%s%d", data, i)))
- if err != nil {
- t.Fatalf("Error writing to tempfile %v", err)
- }
-
- filenames = append(filenames, tempfile.Name())
+ // create collection whose blocks will be backdated
+ collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10)
+ if collectionWithOldBlocks == "" {
+ t.Fatalf("Failed to create collection with 1000 blocks")
}
-
- output, err := exec.Command("arv-put", filenames...).Output()
- if err != nil {
- t.Fatalf("Error running arv-put %s", err)
+ if len(oldBlocks) != 1000 {
+ t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks))
}
- uuid := string(output[0:27]) // trim terminating char
- return uuid
-}
-
-/*
- Create collection with multiple blocks with a single stream; backdate the blocks and delete collection.
- Create another collection with multiple blocks; backdate it's first block and delete the collection
- After datamanager run: expect only the undeleted blocks from second collection, and none of the backdated blocks.
-*/
-func TestPutAndGetCollectionsWithMultipleBlocks(t *testing.T) {
- defer TearDownDataManagerTest(t)
- SetupDataManagerTest(t)
+ // create another collection, whose blocks will be backdated and the collection will be deleted
+ toBeDeletedCollection, toBeDeletedCollectionBlocks := createMultiStreamBlockCollection(t, "new block", 2, 5)
+ if toBeDeletedCollection == "" {
+ t.Fatalf("Failed to create collection with 10 blocks")
+ }
- // Put some blocks which will be backdated later on
- collectionWithOldBlocks := createMultiBlockCollection(t, "to be deleted collection with old blocks", 5)
- oldBlocks := geLocatorsFromCollection(t, collectionWithOldBlocks)
+ // create a stray block that will be backdated
+ strayOldBlock := putBlock(t, "this stray block is old")
- collectionWithNewerBlocks := createMultiBlockCollection(t, "to be deleted collection with newer and older blocks", 5)
- newerBlocks := geLocatorsFromCollection(t, collectionWithNewerBlocks)
+ // create another block that will not be backdated
+ strayNewBlock := putBlock(t, "this stray block is new")
expected := []string{}
expected = append(expected, oldBlocks...)
- expected = append(expected, newerBlocks...)
+ expected = append(expected, toBeDeletedCollectionBlocks...)
+ expected = append(expected, strayOldBlock)
+ expected = append(expected, strayNewBlock)
verifyBlocks(t, nil, expected, 2)
- // Backdate old blocks and delete the collection
+ // Backdate old blocks; but the collection still references these blocks
backdateBlocks(t, oldBlocks)
- deleteCollection(t, collectionWithOldBlocks)
// Backdate first block from the newer blocks and delete the collection; the rest are still be reachable
- backdateBlocks(t, newerBlocks[0:1])
- deleteCollection(t, collectionWithNewerBlocks)
+ backdateBlocks(t, toBeDeletedCollectionBlocks)
+ deleteCollection(t, toBeDeletedCollection)
+
+ // also backdate the stray old block
+ backdateBlocks(t, []string{strayOldBlock})
// run datamanager
dataManagerSingleRun(t)
- notExpected := []string{}
- notExpected = append(notExpected, oldBlocks...)
- notExpected = append(notExpected, newerBlocks[0])
+ expected = []string{strayNewBlock}
+ expected = append(expected, oldBlocks...)
+
+ notExpected := []string{strayOldBlock}
+ notExpected = append(notExpected, toBeDeletedCollectionBlocks...)
- verifyBlocks(t, notExpected, newerBlocks[1:], 2)
+ verifyBlocks(t, notExpected, expected, 2)
}
commit c4ae463b42ee22180982d78289e6898224e508e4
Merge: b36c3f3 76683cd
Author: radhika <radhika at curoverse.com>
Date: Fri Nov 20 09:41:38 2015 -0500
Merge branch 'master' into 7255-manifests-in-datamanager
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list