[ARVADOS] updated: c5602bf4ba4724376c2d5405302d240daaae506f
git at public.curoverse.com
git at public.curoverse.com
Sat Nov 14 19:06:36 EST 2015
Summary of changes:
doc/install/install-api-server.html.textile.liquid | 13 +-
.../install-arv-git-httpd.html.textile.liquid | 3 +
.../install-workbench-app.html.textile.liquid | 6 +
sdk/cli/bin/crunch-job | 12 +-
sdk/python/arvados/keep.py | 112 ++++++-------
sdk/python/tests/arvados_testutil.py | 5 +-
sdk/python/tests/manifest_examples.py | 4 +-
sdk/python/tests/test_arv_ls.py | 6 +-
sdk/python/tests/test_arvfile.py | 5 +-
sdk/python/tests/test_collections.py | 17 +-
sdk/python/tests/test_keep_client.py | 36 +++-
services/datamanager/keep/keep_test.go | 184 +++++++++++++++------
12 files changed, 262 insertions(+), 141 deletions(-)
via c5602bf4ba4724376c2d5405302d240daaae506f (commit)
via c3f49b76d173818386f5c65db46b353e1d334d1e (commit)
via be2d7e55af1036699282d06f629805b6508b6ceb (commit)
via e704e66bc35a0f0990620313ae9e2a630fa6821b (commit)
via c33d036cfef0b0784d16593e66e3f4fce018f783 (commit)
via 7563fb986662a066f0aa3a9c4c1dd35159fb69cc (commit)
via f4c3100bad26dff3c99ff4bb9fa19b0d9f7995c7 (commit)
via 31e10ca041ac1cfd296b24d4878264f6af50d64b (commit)
via 6de60c7db7a98405ef7ae4ac5eb20498f095416c (commit)
via a1dc811844d2dc76bea5ebfdc2f571a12cb41b49 (commit)
via d3171a9d75eac20b68cee6729dddc28cd7e6c612 (commit)
via dc061e4b3c5be2d8687316c6d192fd053b4041e0 (commit)
via 0f0a0cd26446403fb987121af58bec550f3cf681 (commit)
from c0b9d41a5570862dcaac06b727d88be4d0ac9dc6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c5602bf4ba4724376c2d5405302d240daaae506f
Author: radhika <radhika at curoverse.com>
Date: Sat Nov 14 19:04:56 2015 -0500
7490: added several error condition check tests for datamanager/keep package; increased code coverage from 14.6% to 72%
diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 246a8e5..38bb1b8 100644
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -10,6 +10,7 @@ import (
"testing"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
. "gopkg.in/check.v1"
@@ -90,37 +91,93 @@ func (s *KeepSuite) TestSendTrashListUnreachable(c *C) {
sendTrashListError(c, httptest.NewUnstartedServer(&TestHandler{}))
}
-type APIStub struct {
+type StatusAndBody struct {
respStatus int
responseBody string
}
-func (h *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- resp.WriteHeader(h.respStatus)
- resp.Write([]byte(h.responseBody))
+type APIStub struct {
+ data map[string]StatusAndBody
+}
+
+func (stub *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if req.URL.Path == "/redirect-loop" {
+ http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
+ return
+ }
+
+ pathResponse := stub.data[req.URL.Path]
+ if pathResponse.responseBody != "" {
+ if pathResponse.respStatus == -1 {
+ http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
+ } else {
+ resp.WriteHeader(pathResponse.respStatus)
+ resp.Write([]byte(pathResponse.responseBody))
+ }
+ } else {
+ resp.WriteHeader(500)
+ resp.Write([]byte(``))
+ }
}
type KeepServerStub struct {
+ data map[string]StatusAndBody
+}
+
+func (stub *KeepServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ if req.URL.Path == "/redirect-loop" {
+ http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
+ return
+ }
+
+ pathResponse := stub.data[req.URL.Path]
+ if pathResponse.responseBody != "" {
+ if pathResponse.respStatus == -1 {
+ http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
+ } else {
+ resp.WriteHeader(pathResponse.respStatus)
+ resp.Write([]byte(pathResponse.responseBody))
+ }
+ } else {
+ resp.WriteHeader(500)
+ resp.Write([]byte(``))
+ }
}
-func (ts *KeepServerStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- http.Error(resp, "oops", 500)
+type APITestData struct {
+ numServers int
+ serverType string
+ statusCode int
}
func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) {
+ testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200})
+}
+
+func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) {
+ testGetKeepServersFromAPI(c, APITestData{2, "disk", 200})
+}
+
+func (s *KeepSuite) TestGetKeepServers_ServerError(c *C) {
+ testGetKeepServersFromAPI(c, APITestData{-1, "disk", -1})
+}
+
+func testGetKeepServersFromAPI(c *C, testData APITestData) {
keepServers := ServiceList{
- ItemsAvailable: 1,
+ ItemsAvailable: testData.numServers,
KeepServers: []ServerAddress{{
SSL: false,
Host: "example.com",
Port: 12345,
UUID: "abcdefg",
- ServiceType: "nondisk",
+ ServiceType: testData.serverType,
}},
}
ksJSON, _ := json.Marshal(keepServers)
- apiStub := APIStub{200, string(ksJSON)}
+ apiData := make(map[string]StatusAndBody)
+ apiData["/arvados/v1/keep_services"] = StatusAndBody{testData.statusCode, string(ksJSON)}
+ apiStub := APIStub{apiData}
api := httptest.NewServer(&apiStub)
defer api.Close()
@@ -144,51 +201,59 @@ func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) {
}
_, err := GetKeepServersAndSummarize(params)
- c.Assert(err, ErrorMatches, ".*Unsupported service type.*")
+ if testData.numServers > 1 {
+ c.Assert(err, ErrorMatches, ".*Did not receive all available keep servers.*")
+ } else if testData.serverType != "disk" {
+ c.Assert(err, ErrorMatches, ".*Unsupported service type.*")
+ }
}
-func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) {
- keepServers := ServiceList{
- ItemsAvailable: 2,
- KeepServers: []ServerAddress{{
- SSL: false,
- Host: "example.com",
- Port: 12345,
- UUID: "abcdefg",
- ServiceType: "disk",
- }},
- }
+type KeepServerTestData struct {
+ // handle /status.json
+ statusStatusCode int
- ksJSON, _ := json.Marshal(keepServers)
- apiStub := APIStub{200, string(ksJSON)}
+ // handle /index
+ indexStatusCode int
+ indexResponseBody string
- api := httptest.NewServer(&apiStub)
- defer api.Close()
+ // Is error expected?
+ expectedError bool
+}
- arv := arvadosclient.ArvadosClient{
- Scheme: "http",
- ApiServer: api.URL[7:],
- ApiToken: "abc123",
- Client: &http.Client{Transport: &http.Transport{}},
- }
+func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{500, 200, "ok", true})
+}
- kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
- kc.SetServiceRoots(map[string]string{"xxxx": "http://example.com:23456"},
- map[string]string{"xxxx": "http://example.com:23456"},
- map[string]string{})
+func (s *KeepSuite) TestGetKeepServers_GettingIndex(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, -1, "notok", true})
+}
- params := GetKeepServersParams{
- Client: arv,
- Logger: nil,
- Limit: 10,
- }
+func (s *KeepSuite) TestGetKeepServers_ErrorReadServerResponse(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 500, "notok", true})
+}
- _, err := GetKeepServersAndSummarize(params)
- c.Assert(err, ErrorMatches, ".*Did not receive all available keep servers.*")
+func (s *KeepSuite) TestGetKeepServers_ReadServerResponseTuncatedAtLineOne(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "notterminatedwithnewline", true})
}
-func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
- ksStub := KeepServerStub{}
+func (s *KeepSuite) TestGetKeepServers_InvalidBlockLocatorPattern(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "testing\n", true})
+}
+
+func (s *KeepSuite) TestGetKeepServers_ReadServerResponseEmpty(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "\n", false})
+}
+
+func (s *KeepSuite) TestGetKeepServers_ReadServerResponseWithTwoBlocks(c *C) {
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200,
+ "51752ba076e461ec9ec1d27400a08548+20 1447526361\na048cc05c02ba1ee43ad071274b9e547+52 1447526362\n\n", false})
+}
+
+func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
+ ksData := make(map[string]StatusAndBody)
+ ksData["/status.json"] = StatusAndBody{testData.statusStatusCode, string(`{}`)}
+ ksData["/index"] = StatusAndBody{testData.indexStatusCode, testData.indexResponseBody}
+ ksStub := KeepServerStub{ksData}
ks := httptest.NewServer(&ksStub)
defer ks.Close()
@@ -209,7 +274,9 @@ func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
}},
}
ksJSON, _ := json.Marshal(servers_list)
- apiStub := APIStub{200, string(ksJSON)}
+ apiData := make(map[string]StatusAndBody)
+ apiData["/arvados/v1/keep_services"] = StatusAndBody{200, string(ksJSON)}
+ apiStub := APIStub{apiData}
api := httptest.NewServer(&apiStub)
defer api.Close()
@@ -232,7 +299,28 @@ func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
Limit: 10,
}
- // This fails during GetServerStatus
- _, err = GetKeepServersAndSummarize(params)
- c.Assert(err, ErrorMatches, ".*Error during GetServerContents; no host info found.*")
+ // GetKeepServersAndSummarize
+ results, err := GetKeepServersAndSummarize(params)
+
+ if testData.expectedError == false {
+ c.Assert(err, IsNil)
+ c.Assert(results, NotNil)
+
+ blockToServers := results.BlockToServers
+
+ blockLocators := strings.Split(testData.indexResponseBody, "\n")
+ for _, loc := range blockLocators {
+ locator := strings.Split(loc, " ")[0]
+ if locator != "" {
+ blockLocator, err := blockdigest.ParseBlockLocator(locator)
+ c.Assert(err, IsNil)
+
+ blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)}
+ blockServerInfo := blockToServers[blockDigestWithSize]
+ c.Assert(blockServerInfo[0].Mtime, NotNil)
+ }
+ }
+ } else {
+ c.Assert(err, ErrorMatches, ".*Error during GetServerContents; no host info found.*")
+ }
}
commit c3f49b76d173818386f5c65db46b353e1d334d1e
Merge: c0b9d41 be2d7e5
Author: radhika <radhika at curoverse.com>
Date: Fri Nov 13 13:05:09 2015 -0500
Merge branch 'master' into 7490-datamanager-dont-die-return-error
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list