[ARVADOS] updated: 5e27876fa4d3faf3b973282bfb4f152c02345bdc
git at public.curoverse.com
git at public.curoverse.com
Tue Jul 14 13:39:47 EDT 2015
Summary of changes:
sdk/go/arvadosclient/arvadosclient.go | 2 +-
sdk/go/blockdigest/blockdigest.go | 64 ++++-
sdk/go/blockdigest/blockdigest_test.go | 93 ++++++-
sdk/go/blockdigest/testing.go | 16 ++
sdk/go/logger/logger.go | 7 +-
sdk/go/logger/main/testlogger.go | 29 ---
sdk/go/logger/util.go | 20 ++
sdk/go/manifest/manifest.go | 47 +---
sdk/go/manifest/manifest_test.go | 76 +-----
services/datamanager/collection/collection.go | 109 +++++---
services/datamanager/collection/collection_test.go | 123 +++++++++
services/datamanager/collection/testing.go | 60 +++++
services/datamanager/datamanager.go | 95 +++++--
services/datamanager/keep/keep.go | 90 +++----
services/datamanager/loggerutil/loggerutil.go | 13 +-
services/datamanager/summary/canonical_string.go | 27 ++
services/datamanager/summary/file.go | 120 +++++++++
services/datamanager/summary/pull_list.go | 194 ++++++++++++++
services/datamanager/summary/pull_list_test.go | 279 +++++++++++++++++++++
services/datamanager/summary/summary.go | 267 ++++++++++++++++++++
services/datamanager/summary/summary_test.go | 220 ++++++++++++++++
services/fuse/arvados_fuse/__init__.py | 3 +-
services/fuse/tests/test_mount.py | 15 ++
23 files changed, 1715 insertions(+), 254 deletions(-)
create mode 100644 sdk/go/blockdigest/testing.go
delete mode 100644 sdk/go/logger/main/testlogger.go
create mode 100644 sdk/go/logger/util.go
create mode 100644 services/datamanager/collection/collection_test.go
create mode 100644 services/datamanager/collection/testing.go
create mode 100644 services/datamanager/summary/canonical_string.go
create mode 100644 services/datamanager/summary/file.go
create mode 100644 services/datamanager/summary/pull_list.go
create mode 100644 services/datamanager/summary/pull_list_test.go
create mode 100644 services/datamanager/summary/summary.go
create mode 100644 services/datamanager/summary/summary_test.go
via 5e27876fa4d3faf3b973282bfb4f152c02345bdc (commit)
via f78434fcb802949eaae131adf625950ad9981ede (commit)
via 0e57453d2b637a3d105d4e3d67031f3915f9d302 (commit)
via 9007e6691362f389e2fc282a63233562bffb8b05 (commit)
via 0dd85fcd31ef5f251dcf143fef4118d6ea56f700 (commit)
via 61c3f86eb779ab8e723e43354eddafe219bc27d9 (commit)
via 3812c3e89d4048a91db28b4b9276f0c45dba7201 (commit)
via f7cf309e7d7003b5bf9407af81e9a2b1374cde8e (commit)
via 6a59d473c1574eec4db1f83b5d1a963b4f976e5a (commit)
via 982f7db5b67194ddb3b3dd1fae594784f58b35b8 (commit)
via c63ff55687f32dfdff01b9827b411b3757d48ee7 (commit)
via 47d1bdc0960af5bfc8f2793c352f60483539c389 (commit)
via d7635fbe751b2d00dd722a038723577f344406e1 (commit)
via 5824ee2e5198dd46a7813fe2adbd380a114f9ac4 (commit)
via 999c05af58ef1bb6f2b6804301f6ef8d02544a2b (commit)
via b2329e2ce27a24a2d964743a87091413b0e5858e (commit)
via 6eb3d1fb8fe71623fa63da46c250184cf2e4fbb8 (commit)
via b9bf39b18d1e161b6b971270d15c1024ece86243 (commit)
via 5a662d84f00c0c2693c18d333bab9d0fdda7e28e (commit)
via aed4d13a2fd3a97679c512eab2b4a9e5e24df112 (commit)
via 8a0e311e4f8c402c551d61f6290e5bdca149a619 (commit)
via ef6773029872db261f1b3bf3fb2ca86f47dfdcfe (commit)
via 53f785c298338645b6880f22f26b0c36a7cfab4d (commit)
via 0191604164c265a7b972a5802680477f20451d9d (commit)
via 8e1a832244c109351a20a7181da0f65c73e63987 (commit)
via 290ddb6fb0776106cbd68a68f5753452437f357f (commit)
via b1233abce0628266a1805e52d9f9fc651c2a5c59 (commit)
via f09da35ed284ebc1ed1e941f3e3cb63b06a35d51 (commit)
via a56406d730f2a07dd442b9e99ef9dab7b7d81895 (commit)
via a5a4f79e91aa8bba1794394646808f6d4c444661 (commit)
via d3f9fad0cc83a9af47589894998a781db9c60989 (commit)
via 6b5b6890158830b26161b3879a0d1eeaa122659f (commit)
via 248d2ebb4ab7ea3d9060838bcfbfe3b9330da5ee (commit)
from bb8a8e8ab7dc201dccf8f2f3cf243e63ba8d14cd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5e27876fa4d3faf3b973282bfb4f152c02345bdc
Merge: f78434f 9007e66
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jul 14 13:40:24 2015 -0400
Merge branch '6484-fuse-fsync-erofs' closes #6484
commit f78434fcb802949eaae131adf625950ad9981ede
Merge: bb8a8e8 0e57453
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jul 14 13:40:07 2015 -0400
Merge branch 'github-3408-production-datamanager' refs #3408
commit 0e57453d2b637a3d105d4e3d67031f3915f9d302
Merge: 0dd85fc bb8a8e8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jul 14 13:39:36 2015 -0400
Merge branch 'master' into github-3408-production-datamanager
Conflicts:
services/datamanager/keep/keep.go
diff --cc services/datamanager/keep/keep.go
index c666337,912fcf5..c2346cd
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@@ -10,7 -10,9 +10,8 @@@ import
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
"git.curoverse.com/arvados.git/sdk/go/logger"
- "git.curoverse.com/arvados.git/sdk/go/manifest"
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
+ "io"
"io/ioutil"
"log"
"net/http"
@@@ -327,12 -328,36 +328,36 @@@ func ReadServerResponse(arvLogger *logg
response.Address = keepServer
response.Contents.BlockDigestToInfo =
- make(map[blockdigest.BlockDigest]BlockInfo)
+ make(map[blockdigest.DigestWithSize]BlockInfo)
- scanner := bufio.NewScanner(resp.Body)
+ reader := bufio.NewReader(resp.Body)
numLines, numDuplicates, numSizeDisagreements := 0, 0, 0
- for scanner.Scan() {
+ for {
numLines++
- blockInfo, err := parseBlockInfoFromIndexLine(scanner.Text())
+ line, err := reader.ReadString('\n')
+ if err == io.EOF {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Index from %s truncated at line %d",
+ keepServer.String(), numLines))
+ } else if err != nil {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Error reading index response from %s at line %d: %v",
+ keepServer.String(), numLines, err))
+ }
+ if line == "\n" {
+ if _, err := reader.Peek(1); err == nil {
+ extra, _ := reader.ReadString('\n')
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Index from %s had trailing data at line %d after EOF marker: %s",
+ keepServer.String(), numLines+1, extra))
+ } else if err != io.EOF {
+ loggerutil.FatalWithMessage(arvLogger,
+ fmt.Sprintf("Index from %s had read error after EOF marker at line %d: %v",
+ keepServer.String(), numLines, err))
+ }
+ numLines--
+ break
+ }
+ blockInfo, err := parseBlockInfoFromIndexLine(line)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf("Error parsing BlockInfo from index line "+
@@@ -352,31 -402,25 +377,25 @@@
response.Contents.BlockDigestToInfo[blockInfo.Digest] = blockInfo
}
}
- if err := scanner.Err(); err != nil {
- loggerutil.FatalWithMessage(arvLogger,
- fmt.Sprintf("Received error scanning index response from %s: %v",
- keepServer.String(),
- err))
- } else {
- log.Printf("%s index contained %d lines with %d duplicates with "+
- "%d size disagreements",
- keepServer.String(),
- numLines,
- numDuplicates,
- numSizeDisagreements)
-
- if arvLogger != nil {
- now := time.Now()
- arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
- keepInfo := logger.GetOrCreateMap(p, "keep_info")
- serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
-
- serverInfo["processing_finished_at"] = now
- serverInfo["lines_received"] = numLines
- serverInfo["duplicates_seen"] = numDuplicates
- serverInfo["size_disagreements_seen"] = numSizeDisagreements
- })
- }
+
+ log.Printf("%s index contained %d lines with %d duplicates with "+
+ "%d size disagreements",
+ keepServer.String(),
+ numLines,
+ numDuplicates,
+ numSizeDisagreements)
+
+ if arvLogger != nil {
+ now := time.Now()
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
- keepInfo := p["keep_info"].(map[string]interface{})
++ keepInfo := logger.GetOrCreateMap(p, "keep_info")
+ serverInfo := keepInfo[keepServer.Uuid].(map[string]interface{})
+
+ serverInfo["processing_finished_at"] = now
+ serverInfo["lines_received"] = numLines
+ serverInfo["duplicates_seen"] = numDuplicates
+ serverInfo["size_disagreements_seen"] = numSizeDisagreements
+ })
}
resp.Body.Close()
return
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list