[ARVADOS] updated: 0ec4051e60f34786bf7cf78f5b07f50796c68235

git at public.curoverse.com git at public.curoverse.com
Thu Nov 26 17:38:28 EST 2015


Summary of changes:
 services/datamanager/collection/collection.go |  3 +++
 services/datamanager/datamanager.go           | 27 +++++++++++++++++++++------
 services/datamanager/datamanager_test.go      | 20 +++++++++++++++++---
 services/datamanager/keep/keep.go             | 19 +++++++++++++++++++
 services/datamanager/loggerutil/loggerutil.go |  1 +
 services/datamanager/summary/file.go          | 13 ++++++-------
 6 files changed, 67 insertions(+), 16 deletions(-)

       via  0ec4051e60f34786bf7cf78f5b07f50796c68235 (commit)
      from  76e2a1fc805e80f6882117848d99782b569b281c (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 0ec4051e60f34786bf7cf78f5b07f50796c68235
Author: radhika <radhika at curoverse.com>
Date:   Thu Nov 26 17:24:41 2015 -0500

    7748: add dry-run command line argument.

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 33970d8..4bc5240 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -357,6 +357,9 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) {
 				readCollections.OwnerToCollectionSize
 			collectionInfo["distinct_blocks_named"] =
 				len(readCollections.BlockToDesiredReplication)
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 3c53b4a..bba36fc 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -15,6 +15,7 @@ import (
 	"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
 	"git.curoverse.com/arvados.git/services/datamanager/summary"
 	"log"
+	"os"
 	"time"
 )
 
@@ -22,6 +23,7 @@ var (
 	logEventTypePrefix  string
 	logFrequencySeconds int
 	minutesBetweenRuns  int
+	dryRun              bool
 )
 
 func init() {
@@ -36,11 +38,16 @@ func init() {
 	flag.IntVar(&minutesBetweenRuns,
 		"minutes-between-runs",
 		0,
-		"How many minutes we wait betwen data manager runs. 0 means run once and exit.")
+		"How many minutes we wait between data manager runs. 0 means run once and exit.")
+	flag.BoolVar(&dryRun,
+		"dry-run",
+		false,
+		"Perform a dry run. Log how many blocks would be deleted/moved, but do not issue any changes to keepstore.")
 }
 
 func main() {
 	flag.Parse()
+
 	if minutesBetweenRuns == 0 {
 		arv, err := arvadosclient.MakeArvadosClient()
 		if err != nil {
@@ -132,11 +139,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
 			rlbss.Count)
 	}
 
-	kc, err := keepclient.MakeKeepClient(&arv)
-	if err != nil {
-		return fmt.Errorf("Error setting up keep client %v", err.Error())
-	}
-
 	// Log that we're finished. We force the recording, since go will
 	// not wait for the write timer before exiting.
 	if arvLogger != nil {
@@ -147,9 +149,22 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
 			p["summary_info"] = summaryInfo
 
 			p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
+			p["run_info"].(map[string]interface{})["args"] = os.Args
 		})
 	}
 
+	// If dry-run, do not issue any changes to keepstore
+	if dryRun {
+		log.Printf("Datamanager dry-run. Returning without issuing any keepstore updates.")
+		return nil
+	}
+
+	// Not dry-run; issue changes to keepstore
+	kc, err := keepclient.MakeKeepClient(&arv)
+	if err != nil {
+		return fmt.Errorf("Error setting up keep client %v", err.Error())
+	}
+
 	pullServers := summary.ComputePullServers(kc,
 		&keepServerInfo,
 		readCollections.BlockToDesiredReplication,
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 28faf98..f432c07 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -614,7 +614,7 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
 	}
 
 	var locs []string
-	for k, _ := range locators {
+	for k := range locators {
 		locs = append(locs, k)
 	}
 
@@ -625,6 +625,14 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
 // Also, create stray block and backdate it.
 // After datamanager run: expect blocks from the collection, but not the stray block.
 func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
+	testManifestWithMultipleStreamsAndBlocks(t, false)
+}
+
+func TestManifestWithMultipleStreamsAndBlocks_DryRun(t *testing.T) {
+	testManifestWithMultipleStreamsAndBlocks(t, true)
+}
+
+func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
 	defer TearDownDataManagerTest(t)
 	SetupDataManagerTest(t)
 
@@ -651,8 +659,14 @@ func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
 	backdateBlocks(t, []string{strayOldBlock})
 
 	// run datamanager
+	dryRun = isDryRun
 	dataManagerSingleRun(t)
 
-	// verify that strayOldBlock is not to be found, but the collections blocks are still there
-	verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
+	if dryRun {
+		// verify that all blocks, including strayOldBlock, are still to be found
+		verifyBlocks(t, nil, expected, 2)
+	} else {
+		// verify that strayOldBlock is not to be found, but the collections blocks are still there
+		verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
+	}
 }
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 7da74b1..0271ff4 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -15,6 +15,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -245,6 +246,9 @@ func GetServerStatus(arvLogger *logger.Logger,
 			serverInfo["port"] = keepServer.Port
 
 			keepInfo[keepServer.UUID] = serverInfo
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 
@@ -272,6 +276,9 @@ func GetServerStatus(arvLogger *logger.Logger,
 			serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
 			serverInfo["status_response_processed_at"] = now
 			serverInfo["status"] = keepStatus
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 
@@ -291,6 +298,9 @@ func CreateIndexRequest(arvLogger *logger.Logger,
 			keepInfo := logger.GetOrCreateMap(p, "keep_info")
 			serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
 			serverInfo["index_request_sent_at"] = now
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 
@@ -319,6 +329,9 @@ func ReadServerResponse(arvLogger *logger.Logger,
 			keepInfo := logger.GetOrCreateMap(p, "keep_info")
 			serverInfo := keepInfo[keepServer.UUID].(map[string]interface{})
 			serverInfo["index_response_received_at"] = now
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 
@@ -386,6 +399,9 @@ func ReadServerResponse(arvLogger *logger.Logger,
 			serverInfo["lines_received"] = numLines
 			serverInfo["duplicates_seen"] = numDuplicates
 			serverInfo["size_disagreements_seen"] = numSizeDisagreements
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 	resp.Body.Close()
@@ -435,6 +451,9 @@ func (readServers *ReadServers) Summarize(arvLogger *logger.Logger) {
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			keepInfo := logger.GetOrCreateMap(p, "keep_info")
 			keepInfo["distinct_blocks_stored"] = len(readServers.BlockToServers)
+
+			runInfo := logger.GetOrCreateMap(p, "run_info")
+			runInfo["args"] = os.Args
 		})
 	}
 }
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index 8c655cd..7c45839 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -45,6 +45,7 @@ func FatalWithMessage(arvLogger *logger.Logger, message string) {
 			p["FATAL"] = message
 			runInfo := logger.GetOrCreateMap(p, "run_info")
 			runInfo["finished_at"] = time.Now()
+			runInfo["args"] = os.Args
 		})
 	}
 
diff --git a/services/datamanager/summary/file.go b/services/datamanager/summary/file.go
index 63c8ade..9f1eab6 100644
--- a/services/datamanager/summary/file.go
+++ b/services/datamanager/summary/file.go
@@ -9,7 +9,6 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/logger"
 	"git.curoverse.com/arvados.git/services/datamanager/collection"
 	"git.curoverse.com/arvados.git/services/datamanager/keep"
-	"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
 	"log"
 	"os"
 )
@@ -89,13 +88,13 @@ func ReadData(arvLogger *logger.Logger,
 	readCollections *collection.ReadCollections,
 	keepServerInfo *keep.ReadServers) {
 	if readDataFrom == "" {
-		loggerutil.FatalWithMessage(arvLogger,
-			"ReadData() called with empty filename.")
+		readCollections.Err = fmt.Errorf("ReadData() called with empty filename.")
+		return
 	} else {
 		summaryFile, err := os.Open(readDataFrom)
 		if err != nil {
-			loggerutil.FatalWithMessage(arvLogger,
-				fmt.Sprintf("Failed to open %s: %v", readDataFrom, err))
+			readCollections.Err = err
+			return
 		}
 		defer summaryFile.Close()
 
@@ -103,8 +102,8 @@ func ReadData(arvLogger *logger.Logger,
 		data := serializedData{}
 		err = dec.Decode(&data)
 		if err != nil {
-			loggerutil.FatalWithMessage(arvLogger,
-				fmt.Sprintf("Failed to read summary data: %v", err))
+			readCollections.Err = err
+			return
 		}
 
 		// re-summarize data, so that we can update our summarizing

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list