[ARVADOS] updated: 0ec4051e60f34786bf7cf78f5b07f50796c68235
git at public.curoverse.com
git at public.curoverse.com
Thu Nov 26 17:38:28 EST 2015
Summary of changes:
services/datamanager/collection/collection.go | 3 +++
services/datamanager/datamanager.go | 27 +++++++++++++++++++++------
services/datamanager/datamanager_test.go | 20 +++++++++++++++++---
services/datamanager/keep/keep.go | 19 +++++++++++++++++++
services/datamanager/loggerutil/loggerutil.go | 1 +
services/datamanager/summary/file.go | 13 ++++++-------
6 files changed, 67 insertions(+), 16 deletions(-)
via 0ec4051e60f34786bf7cf78f5b07f50796c68235 (commit)
from 76e2a1fc805e80f6882117848d99782b569b281c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0ec4051e60f34786bf7cf78f5b07f50796c68235
Author: radhika <radhika at curoverse.com>
Date: Thu Nov 26 17:24:41 2015 -0500
7748: add dry-run command line argument.
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 33970d8..4bc5240 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -357,6 +357,9 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) {
readCollections.OwnerToCollectionSize
collectionInfo["distinct_blocks_named"] =
len(readCollections.BlockToDesiredReplication)
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 3c53b4a..bba36fc 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -15,6 +15,7 @@ import (
"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"git.curoverse.com/arvados.git/services/datamanager/summary"
"log"
+ "os"
"time"
)
@@ -22,6 +23,7 @@ var (
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
+ dryRun bool
)
func init() {
@@ -36,11 +38,16 @@ func init() {
flag.IntVar(&minutesBetweenRuns,
"minutes-between-runs",
0,
- "How many minutes we wait betwen data manager runs. 0 means run once and exit.")
+ "How many minutes we wait between data manager runs. 0 means run once and exit.")
+ flag.BoolVar(&dryRun,
+ "dry-run",
+ false,
+ "Perform a dry run. Log how many blocks would be deleted/moved, but do not issue any changes to keepstore.")
}
func main() {
flag.Parse()
+
if minutesBetweenRuns == 0 {
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
@@ -132,11 +139,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
rlbss.Count)
}
- kc, err := keepclient.MakeKeepClient(&arv)
- if err != nil {
- return fmt.Errorf("Error setting up keep client %v", err.Error())
- }
-
// Log that we're finished. We force the recording, since go will
// not wait for the write timer before exiting.
if arvLogger != nil {
@@ -147,9 +149,22 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
p["summary_info"] = summaryInfo
p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
+ p["run_info"].(map[string]interface{})["args"] = os.Args
})
}
+ // If dry-run, do not issue any changes to keepstore
+ if dryRun {
+ log.Printf("Datamanager dry-run. Returning without issuing any keepstore updates.")
+ return nil
+ }
+
+ // Not dry-run; issue changes to keepstore
+ kc, err := keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ return fmt.Errorf("Error setting up keep client %v", err.Error())
+ }
+
pullServers := summary.ComputePullServers(kc,
&keepServerInfo,
readCollections.BlockToDesiredReplication,
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 28faf98..f432c07 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -614,7 +614,7 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
}
var locs []string
- for k, _ := range locators {
+ for k := range locators {
locs = append(locs, k)
}
@@ -625,6 +625,14 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
// Also, create stray block and backdate it.
// After datamanager run: expect blocks from the collection, but not the stray block.
func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
+ testManifestWithMultipleStreamsAndBlocks(t, false)
+}
+
+func TestManifestWithMultipleStreamsAndBlocks_DryRun(t *testing.T) {
+ testManifestWithMultipleStreamsAndBlocks(t, true)
+}
+
+func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
defer TearDownDataManagerTest(t)
SetupDataManagerTest(t)
@@ -651,8 +659,14 @@ func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
backdateBlocks(t, []string{strayOldBlock})
// run datamanager
+ dryRun = isDryRun
dataManagerSingleRun(t)
- // verify that strayOldBlock is not to be found, but the collections blocks are still there
- verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
+ if dryRun {
+ // verify that all blocks, including strayOldBlock, are still to be found
+ verifyBlocks(t, nil, expected, 2)
+ } else {
+ // verify that strayOldBlock is not to be found, but the collections blocks are still there
+ verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
+ }
}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 7da74b1..0271ff4 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -15,6 +15,7 @@ import (
"io/ioutil"
"log"
"net/http"
+ "os"
"strconv"
"strings"
"time"
@@ -245,6 +246,9 @@ func GetServerStatus(arvLogger *logger.Logger,
serverInfo["port"] = keepServer.Port
keepInfo[keepServer.UUID] = serverInfo
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
@@ -272,6 +276,9 @@ func GetServerStatus(arvLogger *logger.Logger,
serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
serverInfo["status_response_processed_at"] = now
serverInfo["status"] = keepStatus
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
@@ -291,6 +298,9 @@ func CreateIndexRequest(arvLogger *logger.Logger,
keepInfo := logger.GetOrCreateMap(p, "keep_info")
serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
serverInfo["index_request_sent_at"] = now
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
@@ -319,6 +329,9 @@ func ReadServerResponse(arvLogger *logger.Logger,
keepInfo := logger.GetOrCreateMap(p, "keep_info")
serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
serverInfo["index_response_received_at"] = now
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
@@ -386,6 +399,9 @@ func ReadServerResponse(arvLogger *logger.Logger,
serverInfo["lines_received"] = numLines
serverInfo["duplicates_seen"] = numDuplicates
serverInfo["size_disagreements_seen"] = numSizeDisagreements
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
resp.Body.Close()
@@ -435,6 +451,9 @@ func (readServers *ReadServers) Summarize(arvLogger *logger.Logger) {
arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
keepInfo := logger.GetOrCreateMap(p, "keep_info")
keepInfo["distinct_blocks_stored"] = len(readServers.BlockToServers)
+
+ runInfo := logger.GetOrCreateMap(p, "run_info")
+ runInfo["args"] = os.Args
})
}
}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index 8c655cd..7c45839 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -45,6 +45,7 @@ func FatalWithMessage(arvLogger *logger.Logger, message string) {
p["FATAL"] = message
runInfo := logger.GetOrCreateMap(p, "run_info")
runInfo["finished_at"] = time.Now()
+ runInfo["args"] = os.Args
})
}
diff --git a/services/datamanager/summary/file.go b/services/datamanager/summary/file.go
index 63c8ade..9f1eab6 100644
--- a/services/datamanager/summary/file.go
+++ b/services/datamanager/summary/file.go
@@ -9,7 +9,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/logger"
"git.curoverse.com/arvados.git/services/datamanager/collection"
"git.curoverse.com/arvados.git/services/datamanager/keep"
- "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
"log"
"os"
)
@@ -89,13 +88,13 @@ func ReadData(arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
keepServerInfo *keep.ReadServers) {
if readDataFrom == "" {
- loggerutil.FatalWithMessage(arvLogger,
- "ReadData() called with empty filename.")
+ readCollections.Err = fmt.Errorf("ReadData() called with empty filename.")
+ return
} else {
summaryFile, err := os.Open(readDataFrom)
if err != nil {
- loggerutil.FatalWithMessage(arvLogger,
- fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
+ readCollections.Err = err
+ return
}
defer summaryFile.Close()
@@ -103,8 +102,8 @@ func ReadData(arvLogger *logger.Logger,
data := serializedData{}
err = dec.Decode(&data)
if err != nil {
- loggerutil.FatalWithMessage(arvLogger,
- fmt.Sprintf("Failed to read summary data: %v", err))
+ readCollections.Err = err
+ return
}
// re-summarize data, so that we can update our summarizing
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list