[ARVADOS] updated: 982f7db5b67194ddb3b3dd1fae594784f58b35b8
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 9 13:54:57 EDT 2015
Summary of changes:
services/datamanager/collection/collection.go | 23 +++++++++++-----------
services/datamanager/collection/collection_test.go | 18 ++++++++---------
services/datamanager/datamanager.go | 4 ++--
services/datamanager/summary/pull_list.go | 4 ++--
services/datamanager/summary/summary.go | 4 ++--
5 files changed, 27 insertions(+), 26 deletions(-)
via 982f7db5b67194ddb3b3dd1fae594784f58b35b8 (commit)
from c63ff55687f32dfdff01b9827b411b3757d48ee7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 982f7db5b67194ddb3b3dd1fae594784f58b35b8
Author: mishaz <misha at curoverse.com>
Date: Tue Jun 9 17:51:31 2015 +0000
Renamed BlockToReplication BlockToDesiredReplication.
Added protocol field to servers in pull list.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index ed6df9d..93fad0e 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -39,13 +39,13 @@ type Collection struct {
}
type ReadCollections struct {
- ReadAllCollections bool
- UuidToCollection map[string]Collection
- OwnerToCollectionSize map[string]int
- BlockToReplication map[blockdigest.DigestWithSize]int
- CollectionUuidToIndex map[string]int
- CollectionIndexToUuid []string
- BlockToCollectionIndices map[blockdigest.DigestWithSize][]int
+ ReadAllCollections bool
+ UuidToCollection map[string]Collection
+ OwnerToCollectionSize map[string]int
+ BlockToDesiredReplication map[blockdigest.DigestWithSize]int
+ CollectionUuidToIndex map[string]int
+ CollectionIndexToUuid []string
+ BlockToCollectionIndices map[blockdigest.DigestWithSize][]int
}
type GetCollectionsParams struct {
@@ -283,7 +283,7 @@ func ProcessCollections(arvLogger *logger.Logger,
func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) {
readCollections.OwnerToCollectionSize = make(map[string]int)
- readCollections.BlockToReplication = make(map[blockdigest.DigestWithSize]int)
+ readCollections.BlockToDesiredReplication = make(map[blockdigest.DigestWithSize]int)
numCollections := len(readCollections.UuidToCollection)
readCollections.CollectionUuidToIndex = make(map[string]int, numCollections)
readCollections.CollectionIndexToUuid = make([]string, 0, numCollections)
@@ -303,9 +303,10 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) {
readCollections.BlockToCollectionIndices[locator] =
append(readCollections.BlockToCollectionIndices[locator],
collectionIndex)
- storedReplication := readCollections.BlockToReplication[locator]
+ storedReplication := readCollections.BlockToDesiredReplication[locator]
if coll.ReplicationLevel > storedReplication {
- readCollections.BlockToReplication[locator] = coll.ReplicationLevel
+ readCollections.BlockToDesiredReplication[locator] =
+ coll.ReplicationLevel
}
}
}
@@ -319,7 +320,7 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) {
collectionInfo["owner_to_collection_size"] =
readCollections.OwnerToCollectionSize
collectionInfo["distinct_blocks_named"] =
- len(readCollections.BlockToReplication)
+ len(readCollections.BlockToDesiredReplication)
})
}
diff --git a/services/datamanager/collection/collection_test.go b/services/datamanager/collection/collection_test.go
index 4af5d4c..1669bb7 100644
--- a/services/datamanager/collection/collection_test.go
+++ b/services/datamanager/collection/collection_test.go
@@ -20,9 +20,9 @@ var _ = Suite(&MySuite{})
// indeterminate, we replace BlockToCollectionIndices with
// BlockToCollectionUuids.
type ExpectedSummary struct {
- OwnerToCollectionSize map[string]int
- BlockToReplication map[blockdigest.DigestWithSize]int
- BlockToCollectionUuids map[blockdigest.DigestWithSize][]string
+ OwnerToCollectionSize map[string]int
+ BlockToDesiredReplication map[blockdigest.DigestWithSize]int
+ BlockToCollectionUuids map[blockdigest.DigestWithSize][]string
}
func CompareSummarizedReadCollections(c *C,
@@ -32,8 +32,8 @@ func CompareSummarizedReadCollections(c *C,
c.Assert(summarized.OwnerToCollectionSize, DeepEquals,
expected.OwnerToCollectionSize)
- c.Assert(summarized.BlockToReplication, DeepEquals,
- expected.BlockToReplication)
+ c.Assert(summarized.BlockToDesiredReplication, DeepEquals,
+ expected.BlockToDesiredReplication)
summarizedBlockToCollectionUuids :=
make(map[blockdigest.DigestWithSize]map[string]struct{})
@@ -73,9 +73,9 @@ func (s *MySuite) TestSummarizeSimple(checker *C) {
blockDigest2 := blockdigest.MakeTestDigestWithSize(2)
expected := ExpectedSummary{
- OwnerToCollectionSize: map[string]int{c.OwnerUuid: c.TotalSize},
- BlockToReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
- BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.Uuid}, blockDigest2: []string{c.Uuid}},
+ OwnerToCollectionSize: map[string]int{c.OwnerUuid: c.TotalSize},
+ BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
+ BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.Uuid}, blockDigest2: []string{c.Uuid}},
}
CompareSummarizedReadCollections(checker, rc, expected)
@@ -107,7 +107,7 @@ func (s *MySuite) TestSummarizeOverlapping(checker *C) {
c0.OwnerUuid: c0.TotalSize,
c1.OwnerUuid: c1.TotalSize,
},
- BlockToReplication: map[blockdigest.DigestWithSize]int{
+ BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{
blockDigest1: 5,
blockDigest2: 8,
blockDigest3: 8,
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 3e84308..9161266 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -102,7 +102,7 @@ func singlerun() {
log.Printf("Blocks In Collections: %d, "+
"\nBlocks In Keep: %d.",
- len(readCollections.BlockToReplication),
+ len(readCollections.BlockToDesiredReplication),
len(keepServerInfo.BlockToServers))
log.Println(replicationCounts.PrettyPrint())
@@ -121,7 +121,7 @@ func singlerun() {
pullServers := summary.ComputePullServers(kc,
&keepServerInfo,
- readCollections.BlockToReplication,
+ readCollections.BlockToDesiredReplication,
replicationSummary.UnderReplicatedBlocks)
pullLists := summary.BuildPullLists(pullServers)
diff --git a/services/datamanager/summary/pull_list.go b/services/datamanager/summary/pull_list.go
index e9bd5d1..542a669 100644
--- a/services/datamanager/summary/pull_list.go
+++ b/services/datamanager/summary/pull_list.go
@@ -17,7 +17,6 @@ import (
type Locator blockdigest.DigestWithSize
func (l Locator) MarshalJSON() ([]byte, error) {
- //return []byte("\"" + l.Digest.String() + "\""), nil
return []byte("\"" + blockdigest.DigestWithSize(l).String() + "\""), nil
}
@@ -127,7 +126,8 @@ func CreatePullServers(cs CanonicalString,
server := cs.Get(RemoveProtocolPrefix(host))
_, hasBlock := serverHasBlock[server]
if hasBlock {
- ps.From = append(ps.From, server)
+ // The from field should include the protocol.
+ ps.From = append(ps.From, cs.Get(host))
} else if len(ps.To) < maxToFields {
_, writable := writableServers[server]
if writable {
diff --git a/services/datamanager/summary/summary.go b/services/datamanager/summary/summary.go
index efb6061..edd760b 100644
--- a/services/datamanager/summary/summary.go
+++ b/services/datamanager/summary/summary.go
@@ -196,7 +196,7 @@ func BucketReplication(readCollections collection.ReadCollections,
keepServerInfo keep.ReadServers) (rlbsm ReplicationLevelBlockSetMap) {
rlbsm = make(ReplicationLevelBlockSetMap)
- for block, requestedReplication := range readCollections.BlockToReplication {
+ for block, requestedReplication := range readCollections.BlockToDesiredReplication {
rlbsm.Insert(
ReplicationLevels{
Requested: requestedReplication,
@@ -205,7 +205,7 @@ func BucketReplication(readCollections collection.ReadCollections,
}
for block, servers := range keepServerInfo.BlockToServers {
- if 0 == readCollections.BlockToReplication[block] {
+ if 0 == readCollections.BlockToDesiredReplication[block] {
rlbsm.Insert(
ReplicationLevels{Requested: 0, Actual: len(servers)},
block)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list