[ARVADOS] updated: d1caf8fe2d5645fc8b611b32e22405e6e0632f20
git at public.curoverse.com
git at public.curoverse.com
Thu Nov 19 12:44:49 EST 2015
Summary of changes:
apps/workbench/app/views/application/404.html.erb | 7 +-
.../app/views/application/_report_error.html.erb | 46 ++--
.../test/controllers/projects_controller_test.rb | 2 +-
sdk/cli/bin/crunch-job | 33 ++-
sdk/python/arvados/events.py | 25 +--
sdk/python/arvados/keep.py | 167 +++++++++-----
sdk/python/tests/keepstub.py | 69 +++++-
sdk/python/tests/test_keep_client.py | 146 ++++++++++---
services/datamanager/collection/collection.go | 6 +-
services/datamanager/datamanager.go | 4 +-
services/datamanager/datamanager_test.go | 2 +-
services/datamanager/keep/keep.go | 32 ++-
services/datamanager/keep/keep_test.go | 33 ++-
services/fuse/arvados_fuse/__init__.py | 20 +-
services/fuse/bin/arv-mount | 67 ++++++
services/fuse/setup.py | 2 +-
services/fuse/tests/test_mount.py | 4 +
services/nodemanager/arvnodeman/config.py | 1 +
services/nodemanager/arvnodeman/daemon.py | 124 +++++++----
services/nodemanager/arvnodeman/jobqueue.py | 5 +-
services/nodemanager/arvnodeman/launcher.py | 8 +-
services/nodemanager/doc/azure.example.cfg | 35 ++-
services/nodemanager/doc/ec2.example.cfg | 28 ++-
services/nodemanager/doc/gce.example.cfg | 30 ++-
services/nodemanager/tests/test_daemon.py | 239 +++++++++++++++++----
services/nodemanager/tests/test_jobqueue.py | 50 ++++-
services/nodemanager/tests/testutil.py | 24 ++-
27 files changed, 915 insertions(+), 294 deletions(-)
via d1caf8fe2d5645fc8b611b32e22405e6e0632f20 (commit)
via 1795bd4ba883c85c8277848f4e1f7a4cf9983ec8 (commit)
via 73a127e5492bc2711530b2f5a7c30a5021232d40 (commit)
via d6f163ed99c6ae234c551bb3ecd23c7af8610c8f (commit)
via 1c859533008748afa273aa7c379ef5aec88d4ce6 (commit)
via 155b1aa3772eb2d9421c5bfc678320401eb3b058 (commit)
via 1d6bcbbdbc374aa25b70c85717f987d2675c1205 (commit)
via 04784067f7f39e15161cc165365aa2d0552dc219 (commit)
via dec876f5ea5d88ea9674edc01b86ed2a4fa95078 (commit)
via db55813cc9f24c699db1d3386a56bc0a36debab0 (commit)
via 58692c916bb6dfe2997838ca4147109d9410c86a (commit)
via 99ce6a94ab4b6df30f727c6d07039aa55c2bad32 (commit)
via b2ca3a093f36370719d4a6e89ba46a45111f4f63 (commit)
via 55e78d1096d8a76b77911bcb1561412cda7ffb8a (commit)
via d01d6bdf901b49e8856e199a131fb263f98b370e (commit)
via 4c5e5f9070e4e0fd9b07757c546b3bae48f2b664 (commit)
via d7a0d54602493bfdfe4a69887f0c2627ccc6d461 (commit)
via c112d8551b75514b9ccbde44ee3e37904051dbb1 (commit)
via f4d556b93687fc915897c9cec97afebd7b86572e (commit)
via f34c10a5b22e3f9941f2cfe60f6f9d7a81319b6f (commit)
via 84260dab5182907cae91849acd652c138c2d5095 (commit)
via e9c78ef7855e7ae263fe461e069c89ff7fc0b798 (commit)
via f3b78f5c0690a93d4cc9370519f970ded430c453 (commit)
via 714c555bda26a6a27fad7caef382d1d6705ad215 (commit)
via ba9ea75ee6d6322f5d2a9da0b1c01c2738c0927d (commit)
via 05c4cc66de22d6c4071af69d902f8864b9f2907d (commit)
via f16505d89d60d3bd5abc04762712f4ddf78e39fe (commit)
via f684cda0d2a332f1d2054e7ac8e9de3cb67e23a9 (commit)
via 1ad098e2521d57ba6d66d0a0d9dfffab76061924 (commit)
via 80d30b6a1662e03d56d33f7b29a211d0b3413e2c (commit)
via 6c8cf507184707e1529ef98ffd98269835a39243 (commit)
from 7b223f5209150fe125f611a8e218e5fbbf8a73ee (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d1caf8fe2d5645fc8b611b32e22405e6e0632f20
Author: radhika <radhika at curoverse.com>
Date: Thu Nov 19 12:44:15 2015 -0500
7490: Add Err to collection.ReadCollections and keep.ServerResponse so that the error can be propagated to clients accessing these through a channel read.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 463c847..f0f9433 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -10,7 +10,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/sdk/go/util"
- "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
"os"
"runtime/pprof"
@@ -42,6 +41,7 @@ type ReadCollections struct {
CollectionUUIDToIndex map[string]int
CollectionIndexToUUID []string
BlockToCollectionIndices map[blockdigest.DigestWithSize][]int
+ Err error
}
// GetCollectionsParams params
@@ -99,8 +99,7 @@ func WriteHeapProfile() error {
func GetCollectionsAndSummarize(arvLogger *logger.Logger, params GetCollectionsParams) (results ReadCollections) {
results, err := GetCollections(params)
if err != nil {
- loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error during GetCollections with params %v: %v", params, err))
- results = ReadCollections{}
+ results.Err = err
return
}
@@ -125,6 +124,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
if ¶ms.Client == nil {
err = fmt.Errorf("params.Client passed to GetCollections() should " +
"contain a valid ArvadosClient, but instead it is nil.")
+ return
}
fieldsWanted := []string{"manifest_text",
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index e7d22b8..f3be93c 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -107,8 +107,8 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
dataFetcher(arvLogger, &readCollections, &keepServerInfo)
- if len(readCollections.UUIDToCollection) == 0 {
- return nil // no collections read so no more work to do?
+ if readCollections.Err != nil {
+ return readCollections.Err
}
_, err = summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 6e5a4df..4d82b49 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -538,7 +538,7 @@ func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) {
}
func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) {
- testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", false, true)
+ testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", true, true)
}
/*
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 20244fa..5d3d2ed 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -11,7 +11,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/logger"
- "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"io"
"io/ioutil"
"log"
@@ -51,6 +50,7 @@ type ServerContents struct {
type ServerResponse struct {
Address ServerAddress
Contents ServerContents
+ Err error
}
// ReadServers struct
@@ -93,6 +93,9 @@ func (s ServerAddress) URL() string {
// GetKeepServersAndSummarize gets keep servers from api
func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServers, err error) {
results, err = GetKeepServers(params)
+ if err != nil {
+ return
+ }
log.Printf("Returned %d keep disks", len(results.ServerToContents))
results.Summarize(params.Logger)
@@ -172,9 +175,9 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error
_ = i // Here to prevent go from complaining.
response := <-responseChan
- // There might have been an error during GetServerContents; so check if the response is empty
- if response.Address.Host == "" {
- return results, fmt.Errorf("Error during GetServerContents; no host info found")
+ // Check if there were any errors during GetServerContents
+ if response.Err != nil {
+ return results, response.Err
}
log.Printf("Received channel response from %v containing %d files",
@@ -199,31 +202,26 @@ func GetServerContents(arvLogger *logger.Logger,
err := GetServerStatus(arvLogger, keepServer, arv)
if err != nil {
- loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error during GetServerStatus: %v", err))
- return ServerResponse{}
+ response.Err = err
+ return
}
req, err := CreateIndexRequest(arvLogger, keepServer, arv)
if err != nil {
- loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error building CreateIndexRequest: %v", err))
- return ServerResponse{}
+ response.Err = err
+ return
}
resp, err := arv.Client.Do(req)
if err != nil {
- loggerutil.LogErrorMessage(arvLogger,
- fmt.Sprintf("Error fetching %s: %v. Response was %+v",
- req.URL.String(),
- err,
- resp))
- return ServerResponse{}
+ response.Err = err
+ return
}
response, err = ReadServerResponse(arvLogger, keepServer, resp)
if err != nil {
- loggerutil.LogErrorMessage(arvLogger,
- fmt.Sprintf("Error during ReadServerResponse %v", err))
- return ServerResponse{}
+ response.Err = err
+ return
}
return
diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 38bb1b8..b5091f2 100644
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -2,6 +2,7 @@ package keep
import (
"encoding/json"
+ "fmt"
"net/http"
"net/http/httptest"
"net/url"
@@ -216,37 +217,42 @@ type KeepServerTestData struct {
indexStatusCode int
indexResponseBody string
- // Is error expected?
- expectedError bool
+ // expected error, if any
+ expectedError string
}
func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{500, 200, "ok", true})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{500, 200, "ok",
+ "500 Internal Server Error"})
}
func (s *KeepSuite) TestGetKeepServers_GettingIndex(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{200, -1, "notok", true})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, -1, "notok",
+ "redirect-loop"})
}
func (s *KeepSuite) TestGetKeepServers_ErrorReadServerResponse(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{200, 500, "notok", true})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 500, "notok",
+ "500 Internal Server Error"})
}
func (s *KeepSuite) TestGetKeepServers_ReadServerResponseTuncatedAtLineOne(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "notterminatedwithnewline", true})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200,
+ "notterminatedwithnewline", "truncated at line 1"})
}
func (s *KeepSuite) TestGetKeepServers_InvalidBlockLocatorPattern(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "testing\n", true})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "testing\n",
+ "Error parsing BlockInfo from index line"})
}
func (s *KeepSuite) TestGetKeepServers_ReadServerResponseEmpty(c *C) {
- testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "\n", false})
+ testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "\n", ""})
}
func (s *KeepSuite) TestGetKeepServers_ReadServerResponseWithTwoBlocks(c *C) {
testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200,
- "51752ba076e461ec9ec1d27400a08548+20 1447526361\na048cc05c02ba1ee43ad071274b9e547+52 1447526362\n\n", false})
+ "51752ba076e461ec9ec1d27400a08548+20 1447526361\na048cc05c02ba1ee43ad071274b9e547+52 1447526362\n\n", ""})
}
func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
@@ -302,7 +308,7 @@ func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
// GetKeepServersAndSummarize
results, err := GetKeepServersAndSummarize(params)
- if testData.expectedError == false {
+ if testData.expectedError == "" {
c.Assert(err, IsNil)
c.Assert(results, NotNil)
@@ -321,6 +327,11 @@ func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
}
}
} else {
- c.Assert(err, ErrorMatches, ".*Error during GetServerContents; no host info found.*")
+ if testData.expectedError == "Error parsing BlockInfo from index line" {
+ // In this case ErrorMatches does not work because the error message contains regexp match characters
+ strings.Contains(err.Error(), testData.expectedError)
+ } else {
+ c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*", testData.expectedError))
+ }
}
}
commit 1795bd4ba883c85c8277848f4e1f7a4cf9983ec8
Merge: d7a0d54 73a127e
Author: radhika <radhika at curoverse.com>
Date: Thu Nov 19 11:12:22 2015 -0500
Merge branch 'master' into 7490-datamanager-dont-die-return-error
commit d7a0d54602493bfdfe4a69887f0c2627ccc6d461
Merge: 7b223f5 84260da
Author: radhika <radhika at curoverse.com>
Date: Wed Nov 18 08:38:38 2015 -0500
Merge branch 'master' into 7490-datamanager-dont-die-return-error
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list