[ARVADOS] updated: 11353b25e92155e86a2f1a5d7f9d7462fb8aa8a0
git at public.curoverse.com
git at public.curoverse.com
Wed Nov 25 23:06:56 EST 2015
Summary of changes:
sdk/go/arvadostest/stub.go | 52 +++++------------
sdk/go/manifest/manifest.go | 15 +++--
sdk/go/manifest/manifest_test.go | 41 +++++++-------
services/datamanager/collection/collection_test.go | 66 +++++++++++-----------
services/datamanager/keep/keep_test.go | 20 +++----
5 files changed, 85 insertions(+), 109 deletions(-)
via 11353b25e92155e86a2f1a5d7f9d7462fb8aa8a0 (commit)
via 8fbaec671a04acd587c20015ddf88b1b0209b418 (commit)
from 405386652b50a97b9da8aa442aa5800f6d08f068 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 11353b25e92155e86a2f1a5d7f9d7462fb8aa8a0
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 25 22:50:19 2015 -0500
7253: consume all blocks from BlockIterWithDuplicates, not just the first in test.
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 9ce3d82..49faa01 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -199,11 +199,10 @@ func parseManifestStream(s string) (m ManifestStream) {
return
}
- fileTokens := m.FileTokens
- for j := range m.FileTokens {
- _, _, _, err := parseFileToken(fileTokens[j])
+ for _, ft := range m.FileTokens {
+ _, _, _, err := parseFileToken(ft)
if err != nil {
- m.Err = fmt.Errorf("Invalid file token: %s", fileTokens[j])
+ m.Err = fmt.Errorf("Invalid file token: %s", ft)
break
}
}
@@ -258,12 +257,12 @@ type ManifestBlockLocator struct {
func (m *Manifest) BlockIterWithDuplicates() <-chan ManifestBlockLocator {
blockChannel := make(chan ManifestBlockLocator)
go func(streamChannel <-chan ManifestStream) {
- for m := range streamChannel {
- if m.Err != nil {
- blockChannel <- ManifestBlockLocator{Locator: blockdigest.BlockLocator{}, Err: m.Err}
+ for ms := range streamChannel {
+ if ms.Err != nil {
+ blockChannel <- ManifestBlockLocator{Locator: blockdigest.BlockLocator{}, Err: ms.Err}
continue
}
- for _, block := range m.Blocks {
+ for _, block := range ms.Blocks {
b, err := blockdigest.ParseBlockLocator(block)
if err == nil {
blockChannel <- ManifestBlockLocator{b, nil}
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index d8227ed..65ea7f1 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -199,37 +199,36 @@ func TestFileSegmentIterByName(t *testing.T) {
func TestBlockIterWithBadManifest(t *testing.T) {
testCases := [][]string{
- {"notavalidstreamname acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: notavalidstreamname"},
- {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", ""},
+ {"badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: badstream"},
+ {"/badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: /badstream"},
{". acbd18db4cc2f85cedef654fccc4a4d8+3 file1.txt", "Invalid file token: file1.txt"},
- {". acbd18db4cc2f85cedef654fccc4a4d+3 0:1:file1.txt", "Invalid file token: acbd18db4cc2f85cedef654fccc4a4d.*"},
+ {". acbd18db4cc2f85cedef654fccc4a4+3 0:1:file1.txt", "Invalid file token: acbd18db4cc2f85cedef654fccc4a4.*"},
{". acbd18db4cc2f85cedef654fccc4a4d8 0:1:file1.txt", "Invalid file token: acbd18db4cc2f85cedef654fccc4a4d8"},
{". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt 1:2:file3.txt", "Invalid file token: file2.txt"},
- {"/badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt 1:2:file3.txt", "Invalid stream name: /badstream"},
- {"./goodstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt 1:2:file2.txt", ""},
+ {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt. bcde18db4cc2f85cedef654fccc4a4d8+3 1:2:file3.txt", "Invalid file token: bcde18db4cc2f85cedef654fccc4a4d8.*"},
+ {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt\n. acbd18db4cc2f85cedef654fccc4a4d8+3 ::file2.txt\n", "Invalid file token: ::file2.txt"},
+ {". acbd18db4cc2f85cedef654fccc4a4d8+3 bcde18db4cc2f85cedef654fccc4a4d8+3\n", "Invalid file token: bcde18db4cc2f85cedef654fccc4a4d8.*"},
}
+
for _, testCase := range testCases {
manifest := Manifest{string(testCase[0])}
blockChannel := manifest.BlockIterWithDuplicates()
- block := <-blockChannel
-
- if testCase[1] != "" { // expecting error
- if block.Err == nil {
- t.Errorf("Expected error")
- }
-
- matched, err := regexp.MatchString(testCase[1], block.Err.Error())
- if err != nil {
- t.Errorf("Got error verifying returned block locator error: %v", err)
- }
- if !matched {
- t.Errorf("Expected error not found. Expected: %v; Found = %v", testCase[1], block.Err.Error())
- }
- } else {
+ var err error
+ for block := range blockChannel {
if block.Err != nil {
- t.Errorf("Got error: %v", block.Err)
+ err = block.Err
}
}
+
+ // completed reading from blockChannel; now check for errors
+ if err == nil {
+ t.Errorf("Expected error")
+ }
+
+ matched, _ := regexp.MatchString(testCase[1], err.Error())
+ if !matched {
+ t.Errorf("Expected error not found. Expected: %v; Found: %v", testCase[1], err.Error())
+ }
}
}
commit 8fbaec671a04acd587c20015ddf88b1b0209b418
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 25 18:07:30 2015 -0500
7253: update arvadostest.stub to offer ServerStub and remove the redundant KeepServerStub; update tests accordingly.
diff --git a/sdk/go/arvadostest/stub.go b/sdk/go/arvadostest/stub.go
index 3a40b3c..15a0254 100644
--- a/sdk/go/arvadostest/stub.go
+++ b/sdk/go/arvadostest/stub.go
@@ -4,54 +4,30 @@ import (
"net/http"
)
-// StatusAndBody struct with response status and body
-type StatusAndBody struct {
- ResponseStatus int
- ResponseBody string
+// StubResponse struct with response status and body
+type StubResponse struct {
+ Status int
+ Body string
}
-// APIStub with Data map of path and StatusAndBody
-// Ex: /arvados/v1/keep_services = arvadostest.StatusAndBody{200, string(`{}`)}
-type APIStub struct {
- Data map[string]StatusAndBody
+// ServerStub with response map of path and StubResponse
+// Ex: /arvados/v1/keep_services = arvadostest.StubResponse{200, string(`{}`)}
+type ServerStub struct {
+ Responses map[string]StubResponse
}
-func (stub *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (stub *ServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/redirect-loop" {
http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
return
}
- pathResponse := stub.Data[req.URL.Path]
- if pathResponse.ResponseStatus == -1 {
+ pathResponse := stub.Responses[req.URL.Path]
+ if pathResponse.Status == -1 {
http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
- } else if pathResponse.ResponseBody != "" {
- resp.WriteHeader(pathResponse.ResponseStatus)
- resp.Write([]byte(pathResponse.ResponseBody))
- } else {
- resp.WriteHeader(500)
- resp.Write([]byte(``))
- }
-}
-
-// KeepServerStub with Data map of path and StatusAndBody
-// Ex: /status.json = arvadostest.StatusAndBody{200, string(`{}`)}
-type KeepServerStub struct {
- Data map[string]StatusAndBody
-}
-
-func (stub *KeepServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- if req.URL.Path == "/redirect-loop" {
- http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
- return
- }
-
- pathResponse := stub.Data[req.URL.Path]
- if pathResponse.ResponseStatus == -1 {
- http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
- } else if pathResponse.ResponseBody != "" {
- resp.WriteHeader(pathResponse.ResponseStatus)
- resp.Write([]byte(pathResponse.ResponseBody))
+ } else if pathResponse.Body != "" {
+ resp.WriteHeader(pathResponse.Status)
+ resp.Write([]byte(pathResponse.Body))
} else {
resp.WriteHeader(500)
resp.Write([]byte(``))
diff --git a/services/datamanager/collection/collection_test.go b/services/datamanager/collection/collection_test.go
index 6bb7ae5..71a4785 100644
--- a/services/datamanager/collection/collection_test.go
+++ b/services/datamanager/collection/collection_test.go
@@ -128,56 +128,58 @@ func (s *MySuite) TestSummarizeOverlapping(checker *C) {
type APITestData struct {
// path and response map
- data map[string]arvadostest.StatusAndBody
+ responses map[string]arvadostest.StubResponse
// expected error, if any
expectedError string
}
func (s *MySuite) TestGetCollectionsAndSummarize_DiscoveryError(c *C) {
- testData := APITestData{}
- testData.expectedError = "arvados API server error: 500.*"
- testGetCollectionsAndSummarize(c, testData)
+ testGetCollectionsAndSummarize(c,
+ APITestData{
+ responses: make(map[string]arvadostest.StubResponse),
+ expectedError: "arvados API server error: 500.*",
+ })
}
func (s *MySuite) TestGetCollectionsAndSummarize_ApiErrorGetCollections(c *C) {
- dataMap := make(map[string]arvadostest.StatusAndBody)
- dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
- dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{-1, ``}
-
- testData := APITestData{}
- testData.data = dataMap
- testData.expectedError = "arvados API server error: 302.*"
-
- testGetCollectionsAndSummarize(c, testData)
+ respMap := make(map[string]arvadostest.StubResponse)
+ respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
+ respMap["/arvados/v1/collections"] = arvadostest.StubResponse{-1, ``}
+
+ testGetCollectionsAndSummarize(c,
+ APITestData{
+ responses: respMap,
+ expectedError: "arvados API server error: 302.*",
+ })
}
func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadStreamName(c *C) {
- dataMap := make(map[string]arvadostest.StatusAndBody)
- dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
- dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"badstreamname"}]}`}
-
- testData := APITestData{}
- testData.data = dataMap
- testData.expectedError = "Invalid stream name: badstreamname"
-
- testGetCollectionsAndSummarize(c, testData)
+ respMap := make(map[string]arvadostest.StubResponse)
+ respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
+ respMap["/arvados/v1/collections"] = arvadostest.StubResponse{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"badstreamname"}]}`}
+
+ testGetCollectionsAndSummarize(c,
+ APITestData{
+ responses: respMap,
+ expectedError: "Invalid stream name: badstreamname",
+ })
}
func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadFileToken(c *C) {
- dataMap := make(map[string]arvadostest.StatusAndBody)
- dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
- dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"./goodstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt"}]}`}
-
- testData := APITestData{}
- testData.data = dataMap
- testData.expectedError = "Invalid file token: file2.txt"
-
- testGetCollectionsAndSummarize(c, testData)
+ respMap := make(map[string]arvadostest.StubResponse)
+ respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
+ respMap["/arvados/v1/collections"] = arvadostest.StubResponse{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"./goodstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt"}]}`}
+
+ testGetCollectionsAndSummarize(c,
+ APITestData{
+ responses: respMap,
+ expectedError: "Invalid file token: file2.txt",
+ })
}
func testGetCollectionsAndSummarize(c *C, testData APITestData) {
- apiStub := arvadostest.APIStub{testData.data}
+ apiStub := arvadostest.ServerStub{testData.responses}
api := httptest.NewServer(&apiStub)
defer api.Close()
diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 454661a..0691e64 100644
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -125,9 +125,9 @@ func testGetKeepServersFromAPI(c *C, testData APITestData, expectedError string)
}
ksJSON, _ := json.Marshal(keepServers)
- apiStubData := make(map[string]arvadostest.StatusAndBody)
- apiStubData["/arvados/v1/keep_services"] = arvadostest.StatusAndBody{testData.statusCode, string(ksJSON)}
- apiStub := arvadostest.APIStub{apiStubData}
+ apiStubResponses := make(map[string]arvadostest.StubResponse)
+ apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{testData.statusCode, string(ksJSON)}
+ apiStub := arvadostest.ServerStub{apiStubResponses}
api := httptest.NewServer(&apiStub)
defer api.Close()
@@ -202,10 +202,10 @@ func (s *KeepSuite) TestGetKeepServers_ReadServerResponseWithTwoBlocks(c *C) {
}
func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
- ksStubData := make(map[string]arvadostest.StatusAndBody)
- ksStubData["/status.json"] = arvadostest.StatusAndBody{testData.statusStatusCode, string(`{}`)}
- ksStubData["/index"] = arvadostest.StatusAndBody{testData.indexStatusCode, testData.indexResponseBody}
- ksStub := arvadostest.KeepServerStub{ksStubData}
+ ksStubResponses := make(map[string]arvadostest.StubResponse)
+ ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)}
+ ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody}
+ ksStub := arvadostest.ServerStub{ksStubResponses}
ks := httptest.NewServer(&ksStub)
defer ks.Close()
@@ -226,9 +226,9 @@ func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
}},
}
ksJSON, _ := json.Marshal(servers_list)
- apiStubData := make(map[string]arvadostest.StatusAndBody)
- apiStubData["/arvados/v1/keep_services"] = arvadostest.StatusAndBody{200, string(ksJSON)}
- apiStub := arvadostest.APIStub{apiStubData}
+ apiStubResponses := make(map[string]arvadostest.StubResponse)
+ apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)}
+ apiStub := arvadostest.ServerStub{apiStubResponses}
api := httptest.NewServer(&apiStub)
defer api.Close()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list