[ARVADOS] created: 81b8e05dcd7397de4e2facd47be899edb9a21c6a
git at public.curoverse.com
git at public.curoverse.com
Tue Aug 25 12:40:19 EDT 2015
at 81b8e05dcd7397de4e2facd47be899edb9a21c6a (commit)
commit 81b8e05dcd7397de4e2facd47be899edb9a21c6a
Merge: 5b5fd04 b5a7183
Author: radhika <radhika at curoverse.com>
Date: Tue Aug 25 12:40:09 2015 -0400
Merge branch 'master' into 6260-test-datamanager
commit 5b5fd04b19193fce6fa9a5d99842f17d83f499df
Author: radhika <radhika at curoverse.com>
Date: Tue Aug 25 12:39:09 2015 -0400
6260: datamanager integration testing - WIP
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 1c5162b..8ef87c2 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -332,6 +332,9 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
f.write(blob_signing_key)
if enforce_permissions:
keep_args['--enforce-permissions'] = 'true'
+ with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
+ keep_args['--data-manager-token-file'] = f.name
+ f.write(os.environ['ARVADOS_API_TOKEN'])
api = arvados.api(
version='v1',
@@ -364,6 +367,8 @@ def _stop_keep(n):
os.unlink("{}/keep{}.volume".format(TEST_TMPDIR, n))
if os.path.exists(os.path.join(TEST_TMPDIR, "keep.blob_signing_key")):
os.remove(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"))
+ if os.path.exists(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file")):
+ os.remove(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"))
def stop_keep():
_stop_keep(0)
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
new file mode 100644
index 0000000..64b71dc
--- /dev/null
+++ b/services/datamanager/datamanager_test.go
@@ -0,0 +1,281 @@
+package main
+
+import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "git.curoverse.com/arvados.git/services/datamanager/keep"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "os/exec"
+ "regexp"
+ "strings"
+ "testing"
+ "time"
+)
+
+var arv arvadosclient.ArvadosClient
+var keepClient *keepclient.KeepClient
+var keepServers []string
+
+func SetupDataManagerTest(t *testing.T) {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+
+ // start api and keep servers
+ arvadostest.ResetEnv()
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+
+ // make arvadosclient
+ var err error
+ arv, err = arvadosclient.MakeArvadosClient()
+ if err != nil {
+ t.Fatal("Error creating arv")
+ }
+
+ // keep client
+ keepClient = &keepclient.KeepClient{
+ Arvados: &arv,
+ Want_replicas: 1,
+ Using_proxy: true,
+ Client: &http.Client{},
+ }
+
+ // discover keep services
+ if err := keepClient.DiscoverKeepServers(); err != nil {
+ t.Fatal("Error discovering keep services")
+ }
+ for _, host := range keepClient.LocalRoots() {
+ keepServers = append(keepServers, host)
+ }
+}
+
+func TearDownDataManagerTest(t *testing.T) {
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
+}
+
+func PutBlock(t *testing.T, data string) string {
+ locator, _, err := keepClient.PutB([]byte(data))
+ if err != nil {
+ t.Fatalf("Error putting test data for %s %s %v", data, locator, err)
+ }
+ if locator == "" {
+ t.Fatalf("No locator found after putting test data")
+ }
+
+ return locator
+}
+
+func GetBlock(t *testing.T, locator string, data string) {
+ reader, blocklen, _, err := keepClient.Get(locator)
+ if err != nil {
+ t.Fatalf("Error getting test data in setup for %s %s %v", data, locator, err)
+ }
+ if reader == nil {
+ t.Fatalf("No reader found after putting test data")
+ }
+ if blocklen != int64(len(data)) {
+ t.Fatalf("blocklen %d did not match data len %d", blocklen, len(data))
+ }
+
+ all, err := ioutil.ReadAll(reader)
+ if string(all) != data {
+ t.Fatalf("Data read %s did not match expected data %s", string(all), data)
+ }
+}
+
+// Create a collection using arv-put
+func CreateCollection(t *testing.T, data string) string {
+ tempfile, err := ioutil.TempFile(os.TempDir(), "temp-test-file")
+ defer os.Remove(tempfile.Name())
+
+ _, err = tempfile.Write([]byte(data))
+ if err != nil {
+ t.Fatalf("Error writing to tempfile %v", err)
+ }
+
+ // arv-put
+ output, err := exec.Command("arv-put", "--use-filename", "test.txt", tempfile.Name()).Output()
+ if err != nil {
+ t.Fatalf("Error running arv-put %s", err)
+ }
+
+ collection_uuid := string(output[0:27]) // trim terminating char
+ return collection_uuid
+}
+
+// Get collection using arv-get
+var locatorMatcher = regexp.MustCompile("^([0-9a-f]{32})([+](.*))?$")
+
+func GetCollection(t *testing.T, collection_uuid string) string {
+ // get collection
+ output, err := exec.Command("arv-get", collection_uuid).Output()
+
+ if err != nil {
+ t.Fatalf("Error during arv-get %s", err)
+ }
+
+ locator := strings.Split(string(output), " ")[1]
+ match := locatorMatcher.FindStringSubmatch(locator)
+ if match == nil {
+ t.Fatalf("No locator found in collection manifest %s", string(output))
+ }
+ return match[1]
+}
+
+type Dict map[string]interface{}
+
+func DeleteCollection(t *testing.T, collection_uuid string) {
+ getback := make(Dict)
+ err := arv.Delete("collections", collection_uuid, nil, &getback)
+ if err != nil {
+ t.Fatalf("Error deleting collection %s", err)
+ }
+ if getback["uuid"] != collection_uuid {
+ t.Fatalf("Delete collection uuid did not match original: $s, result: $s", collection_uuid, getback["uuid"])
+ }
+}
+
+func DataManagerSingleRun(t *testing.T) {
+ err := singlerun()
+ if err != nil {
+ t.Fatalf("Error during singlerun %s", err)
+ }
+}
+
+func MakeRequest(t *testing.T, path string) string {
+ client := http.Client{}
+ req, err := http.NewRequest("GET", path, strings.NewReader("resp"))
+ req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+ req.Header.Add("Content-Type", "application/octet-stream")
+ // resp, err := client.Do(req)
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatalf("Error during %s %s", path, err)
+ }
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("Error reading response from %s %s", path, err)
+ }
+
+ return string(body)
+}
+
+func GetBlockIndexes(t *testing.T) []string {
+ var indexes []string
+
+ for i := 0; i < len(keepServers); i++ {
+ resp := MakeRequest(t, keepServers[i]+"/index")
+ lines := strings.Split(resp, "\n")
+ for _, line := range lines {
+ indexes = append(indexes, strings.Split(line, " ")...)
+ }
+ }
+
+ return indexes
+}
+
+func VerifyBlocks(t *testing.T, not_expected []string, expected []string) {
+ blocks := GetBlockIndexes(t)
+ for _, block := range not_expected {
+ exists := ValueInArray(block, blocks)
+ if exists {
+ t.Fatalf("Found unexpected block in index %s", block)
+ }
+ }
+ for _, block := range expected {
+ exists := ValueInArray(block, blocks)
+ if !exists {
+ t.Fatalf("Did not find expected block in index %s", block)
+ }
+ }
+}
+
+func ValueInArray(value string, list []string) bool {
+ for _, v := range list {
+ if strings.HasPrefix(v, value) {
+ return true
+ }
+ }
+ return false
+}
+
+func TestPutAndGetBlocks(t *testing.T) {
+ log.Print("TestPutAndGetBlocks start")
+ defer TearDownDataManagerTest(t)
+ SetupDataManagerTest(t)
+
+ // Put some blocks and change their mtime to old
+ var oldBlockLocators []string
+ oldBlockData := "this block will have older mtime"
+ for i := 0; i < 2; i++ {
+ oldBlockLocators = append(oldBlockLocators, PutBlock(t, oldBlockData+string(i)))
+ }
+ for i := 0; i < 2; i++ {
+ GetBlock(t, oldBlockLocators[i], oldBlockData+string(i))
+ }
+
+ // Put some more new blocks
+ var newBlockLocators []string
+ newBlockData := "this block is newer"
+ for i := 0; i < 1; i++ {
+ newBlockLocators = append(newBlockLocators, PutBlock(t, newBlockData+string(i)))
+ }
+ for i := 0; i < 1; i++ {
+ GetBlock(t, newBlockLocators[i], newBlockData+string(i))
+ }
+
+ // Create a collection
+ collection_uuid := CreateCollection(t, "some data for collection creation")
+
+ collection_locator := GetCollection(t, collection_uuid)
+
+ /*
+ // Invoking datamanager singlerun or /index several times is resulting in errors
+ // Hence, for now just invoke once at the end of test
+ var expected []string
+ expected = append(expected, oldBlockLocators...)
+ expected = append(expected, newBlockLocators...)
+ expected = append(expected, collection_locator)
+
+ VerifyBlocks(t, nil, expected)
+
+ // Run datamanager in singlerun mode
+ DataManagerSingleRun(t)
+ */
+
+ // Change mtime on old blocks and delete the collection
+ DeleteCollection(t, collection_uuid)
+
+ time.Sleep(1 * time.Second)
+ DataManagerSingleRun(t)
+
+ // Give some time for pull worker and trash worker to finish
+ time.Sleep(10 * time.Second)
+
+ // Get block indexes and verify that the deleted collection block is no longer found
+ var not_expected []string
+ not_expected = append(not_expected, oldBlockLocators...)
+ not_expected = append(not_expected, collection_locator)
+ //VerifyBlocks(t, not_expected, newBlockLocators)
+ VerifyBlocks(t, nil, newBlockLocators)
+}
+
+// Invoking datamanager singlerun several times results in errors.
+// Until that issue is resolved, don't run this test in the meantime.
+func x_TestInvokeDatamanagerSingleRunRepeatedly(t *testing.T) {
+ defer TearDownDataManagerTest(t)
+ SetupDataManagerTest(t)
+
+ for i := 0; i < 10; i++ {
+ err := singlerun()
+ if err != nil {
+ t.Fatalf("Got an error during datamanager singlerun: %v", err)
+ }
+ time.Sleep(1 * time.Second)
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list