[ARVADOS] updated: ba032ff921ffb7be57d9f831e2a3dce8b38e266e

git at public.curoverse.com git at public.curoverse.com
Tue Mar 3 11:05:39 EST 2015


Summary of changes:
 services/keepstore/pull_worker.go      |   5 +-
 services/keepstore/pull_worker_test.go | 298 ++++++++++++++++++++++++---------
 2 files changed, 221 insertions(+), 82 deletions(-)

       via  ba032ff921ffb7be57d9f831e2a3dce8b38e266e (commit)
      from  753d0ef652e1593c0fbd204a3973554794215c2f (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit ba032ff921ffb7be57d9f831e2a3dce8b38e266e
Author: Radhika Chippada <radhika at curoverse.com>
Date:   Tue Mar 3 11:05:11 2015 -0500

    3761: improved tests with delays

diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 6d64677..db2b54d 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -68,8 +68,7 @@ func Pull(pullRequest PullRequest) (err error) {
 	expires_at := time.Now().Add(60 * time.Second)
 	signedLocator := SignLocator(pullRequest.Locator, GenerateRandomApiToken(), expires_at)
 
-	reader, contentLen, _, err := GetContent(signedLocator)
-
+	reader, contentLen, _, err := GetContent(pullRequest.Locator, signedLocator)
 	if err != nil {
 		return
 	}
@@ -92,7 +91,7 @@ func Pull(pullRequest PullRequest) (err error) {
 }
 
 // Fetch the content for the given locator using keepclient.
-var GetContent = func(signedLocator string) (reader io.ReadCloser, contentLength int64, url string, err error) {
+var GetContent = func(locator string, signedLocator string) (reader io.ReadCloser, contentLength int64, url string, err error) {
 	reader, blocklen, url, err := keepClient.Get(signedLocator)
 	return reader, blocklen, url, err
 }
diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go
index 8e6241f..de9ec41 100644
--- a/services/keepstore/pull_worker_test.go
+++ b/services/keepstore/pull_worker_test.go
@@ -3,40 +3,66 @@ package main
 import (
 	"bytes"
 	"errors"
+	. "gopkg.in/check.v1"
 	"io"
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 )
 
-func TestPullWorker(t *testing.T) {
-	defer teardown()
+var testPullLists map[string]string
+var processedPullLists map[string]string
+
+type PullWorkerTestSuite struct{}
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+// Gocheck boilerplate
+var _ = Suite(&PullWorkerTestSuite{})
 
+func (s *PullWorkerTestSuite) SetUpSuite(c *C) {
 	// Since keepstore does not come into picture in tests,
 	// we need to explicitly start the goroutine in tests.
 	go RunPullWorker(pullq.NextItem)
 
-	data_manager_token = "DATA MANAGER TOKEN"
+	// When a new pull request arrives, the old one will be overwritten.
+	// This behavior is simulated with delay tests below.
+	testPullLists = make(map[string]string)
+	processedPullLists = make(map[string]string)
+}
+
+func (s *PullWorkerTestSuite) TearDownSuite(c *C) {
+	// give the channel enough time to read and process all pull list entries
+	time.Sleep(1000 * time.Millisecond)
+
+	expectWorkerChannelEmpty(c, pullq.NextItem)
+
+	c.Assert(len(processedPullLists), Not(Equals), len(testPullLists))
+}
 
-	first_pull_list := []byte(`[
+var first_pull_list = []byte(`[
 		{
-			"locator":"locator1_to_verify_first_pull_list",
+			"locator":"locator1",
 			"servers":[
 				"server_1",
 				"server_2"
 		 	]
 		},
     {
-			"locator":"locator2_to_verify_first_pull_list",
+			"locator":"locator2",
 			"servers":[
 				"server_3"
 		 	]
 		}
 	]`)
 
-	second_pull_list := []byte(`[
+var second_pull_list = []byte(`[
 		{
-			"locator":"locator_to_verify_second_pull_list",
+			"locator":"locator3",
 			"servers":[
 				"server_1",
         "server_2"
@@ -44,85 +70,191 @@ func TestPullWorker(t *testing.T) {
 		}
 	]`)
 
-	type PullWorkerTestData struct {
-		name          string
-		req           RequestTester
-		response_code int
-		response_body string
-		read_content  string
-		read_error    bool
-		put_error     bool
+type PullWorkerTestData struct {
+	name          string
+	req           RequestTester
+	response_code int
+	response_body string
+	read_content  string
+	read_error    bool
+	put_error     bool
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_pull_list_with_two_locators(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_pull_list_with_two_locators",
+		RequestTester{"/pull", data_manager_token, "PUT", first_pull_list},
+		http.StatusOK,
+		"Received 2 pull requests\n",
+		"hello",
+		false,
+		false,
 	}
-	var testcases = []PullWorkerTestData{
-		{
-			"Pull request 1 from the data manager in worker",
-			RequestTester{"/pull", data_manager_token, "PUT", first_pull_list},
-			http.StatusOK,
-			"Received 2 pull requests\n",
-			"hello",
-			false,
-			false,
-		},
-		{
-			"Pull request 2 from the data manager in worker",
-			RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
-			http.StatusOK,
-			"Received 1 pull requests\n",
-			"hola",
-			false,
-			false,
-		},
-		{
-			"Pull request with error on get",
-			RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
-			http.StatusOK,
-			"Received 1 pull requests\n",
-			"unused",
-			true,
-			false,
-		},
-		{
-			"Pull request with error on put",
-			RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
-			http.StatusOK,
-			"Received 1 pull requests\n",
-			"unused",
-			false,
-			true,
-		},
+
+	performTest(testData, c)
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_pull_list_with_one_locator(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_pull_list_with_one_locator",
+		RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
+		http.StatusOK,
+		"Received 1 pull requests\n",
+		"hola",
+		false,
+		false,
 	}
 
-	for _, testData := range testcases {
-		// Override GetContent to mock keepclient functionality
-		GetContent = func(signedLocator string) (reader io.ReadCloser, contentLength int64, url string, err error) {
-			if testData.read_error {
-				return nil, 0, "", errors.New("Error getting data")
-			} else {
-				cb := &ClosingBuffer{bytes.NewBufferString("Hi!")}
-				var rc io.ReadCloser
-				rc = cb
-				return rc, 3, "", nil
-			}
-		}
+	performTest(testData, c)
+}
 
-		// Override PutContent to mock PutBlock functionality
-		PutContent = func(content []byte, locator string) (err error) {
-			if testData.put_error {
-				return errors.New("Error putting data")
-			} else {
-				return nil
-			}
-		}
+// When a new pull request arrives, the old one will be overwritten.
+// Simulate this behavior by inducing delay in GetContent for the delay test(s).
+// To ensure this delay test is not the last one executed and
+// hence we cannot verify this behavior, let's run the delay test twice.
+func (s *PullWorkerTestSuite) TestPullWorker_pull_list_with_one_locator_with_delay_1(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_pull_list_with_one_locator_with_delay_1",
+		RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
+		http.StatusOK,
+		"Received 1 pull requests\n",
+		"hola",
+		false,
+		false,
+	}
+
+	performTest(testData, c)
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_pull_list_with_one_locator_with_delay_2(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_pull_list_with_one_locator_with_delay_2",
+		RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
+		http.StatusOK,
+		"Received 1 pull requests\n",
+		"hola",
+		false,
+		false,
+	}
+
+	performTest(testData, c)
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_one_locator(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_error_on_get_one_locator",
+		RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
+		http.StatusOK,
+		"Received 1 pull requests\n",
+		"unused",
+		true,
+		false,
+	}
+
+	performTest(testData, c)
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_error_on_get_two_locators(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_error_on_get_two_locators",
+		RequestTester{"/pull", data_manager_token, "PUT", first_pull_list},
+		http.StatusOK,
+		"Received 2 pull requests\n",
+		"unused",
+		true,
+		false,
+	}
+
+	performTest(testData, c)
+}
+
+func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_one_locator(c *C) {
+	defer teardown()
+
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_error_on_put_one_locator",
+		RequestTester{"/pull", data_manager_token, "PUT", second_pull_list},
+		http.StatusOK,
+		"Received 1 pull requests\n",
+		"unused",
+		false,
+		true,
+	}
 
-		response := IssueRequest(&testData.req)
-		ExpectStatusCode(t, testData.name, testData.response_code, response)
-		ExpectBody(t, testData.name, testData.response_body, response)
+	performTest(testData, c)
+}
 
-		// give the channel a second to read and process all pull list entries
-		time.Sleep(1000 * time.Millisecond)
+func (s *PullWorkerTestSuite) TestPullWorker_error_on_put_two_locators(c *C) {
+	defer teardown()
 
-		expectChannelEmpty(t, pullq.NextItem)
+	data_manager_token = "DATA MANAGER TOKEN"
+	testData := PullWorkerTestData{
+		"TestPullWorker_error_on_put_two_locators",
+		RequestTester{"/pull", data_manager_token, "PUT", first_pull_list},
+		http.StatusOK,
+		"Received 2 pull requests\n",
+		"unused",
+		false,
+		true,
 	}
+
+	performTest(testData, c)
+}
+
+func performTest(testData PullWorkerTestData, c *C) {
+	testPullLists[testData.name] = testData.response_body
+
+	// We need to make sure the tests have a slight delay so that we can verify the pull list channel overwrites.
+	time.Sleep(25 * time.Millisecond)
+
+	// Override GetContent to mock keepclient functionality
+	GetContent = func(locator string, signedLocator string) (reader io.ReadCloser, contentLength int64, url string, err error) {
+		if strings.HasPrefix(testData.name, "TestPullWorker_pull_list_with_one_locator_with_delay") {
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		processedPullLists[testData.name] = testData.response_body
+		if testData.read_error {
+			return nil, 0, "", errors.New("Error getting data")
+		} else {
+			cb := &ClosingBuffer{bytes.NewBufferString("Hi!")}
+			var rc io.ReadCloser
+			rc = cb
+			return rc, 3, "", nil
+		}
+	}
+
+	// Override PutContent to mock PutBlock functionality
+	PutContent = func(content []byte, locator string) (err error) {
+		if testData.put_error {
+			return errors.New("Error putting data")
+		} else {
+			return nil
+		}
+	}
+
+	response := IssueRequest(&testData.req)
+	c.Assert(testData.response_code, Equals, response.Code)
+	c.Assert(testData.response_body, Equals, response.Body.String())
 }
 
 type ClosingBuffer struct {
@@ -132,3 +264,11 @@ type ClosingBuffer struct {
 func (cb *ClosingBuffer) Close() (err error) {
 	return
 }
+
+func expectWorkerChannelEmpty(c *C, workerChannel <-chan interface{}) {
+	select {
+	case item := <-workerChannel:
+		c.Fatalf("Received value (%v) from channel that was expected to be empty", item)
+	default:
+	}
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list