[ARVADOS] updated: 8d6532d6f5db2927f353ebe1cd75dcd1e189873a

git at public.curoverse.com git at public.curoverse.com
Mon Feb 9 21:44:24 EST 2015


Summary of changes:
 sdk/go/logger/logger.go                       | 181 +++++++++++++-------------
 services/datamanager/collection/collection.go |   9 --
 services/datamanager/datamanager.go           |  46 ++-----
 services/datamanager/keep/keep.go             |  25 ++--
 services/datamanager/loggerutil/loggerutil.go |  38 +++++-
 5 files changed, 147 insertions(+), 152 deletions(-)

       via  8d6532d6f5db2927f353ebe1cd75dcd1e189873a (commit)
       via  40e42c383646edade9b4723dfb8001cb4c873ea5 (commit)
       via  bdf093d22ebbdaaadcd822fe32a5fe150fe649f6 (commit)
      from  c37ed03a4d05ca49820628d2b43d7bf140668f47 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8d6532d6f5db2927f353ebe1cd75dcd1e189873a
Author: mishaz <misha at curoverse.com>
Date:   Tue Feb 10 02:39:39 2015 +0000

    Added different event types for started, partially complete and final log entries.

diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index e6e1ed6..e7bc8d1 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -1,9 +1,8 @@
 // Logger periodically writes a log to the Arvados SDK.
 //
 // This package is useful for maintaining a log object that is updated
-// over time. Every time the object is updated, it will be written to
-// the log. Writes will be throttled to no more frequent than
-// WriteInterval.
+// over time. This log object will be periodically written to the log,
+// as specified by WriteInterval in the Params.
 //
 // This package is safe for concurrent use as long as:
 // The maps passed to a LogMutator are not accessed outside of the
@@ -26,10 +25,16 @@ import (
 	"time"
 )
 
+const (
+	startSuffix   = "-start"
+	partialSuffix = "-partial"
+	finalSuffix   = "-final"
+)
+
 type LoggerParams struct {
-	Client        arvadosclient.ArvadosClient // The client we use to write log entries
-	EventType     string                      // The event type to assign to the log entry.
-	WriteInterval time.Duration               // Wait at least this long between log writes
+	Client          arvadosclient.ArvadosClient // The client we use to write log entries
+	EventTypePrefix string                      // The prefix we use for the event type in the log entry
+	WriteInterval   time.Duration               // Wait at least this long between log writes
 }
 
 // A LogMutator is a function which modifies the log entry.
@@ -57,6 +62,7 @@ type Logger struct {
 	modified    bool            // Has this data been modified since the last write?
 	workToDo    chan LogMutator // Work to do in the worker thread.
 	writeTicker *time.Ticker    // On each tick we write the log data to arvados, if it has been modified.
+	hasWritten  bool            // Whether we've written at all yet.
 
 	writeHooks []LogMutator // Mutators we call before each write.
 }
@@ -67,8 +73,8 @@ func NewLogger(params LoggerParams) *Logger {
 	if &params.Client == nil {
 		log.Fatal("Nil arvados client in LoggerParams passed in to NewLogger()")
 	}
-	if params.EventType == "" {
-		log.Fatal("Empty event type in LoggerParams passed in to NewLogger()")
+	if params.EventTypePrefix == "" {
+		log.Fatal("Empty event type prefix in LoggerParams passed in to NewLogger()")
 	}
 
 	l := &Logger{data: make(map[string]interface{}),
@@ -119,11 +125,9 @@ func (l *Logger) FinalUpdate(mutator LogMutator) {
 	// Apply the final update
 	l.workToDo <- mutator
 
-	// Perform the write and signal that we can return.
+	// Perform the final write and signal that we can return.
 	l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
-		// TODO(misha): Add a boolean arg to write() to indicate that it's
-		// final so that we can set the appropriate event type.
-		l.write()
+		l.write(true)
 		done <- true
 	}
 
@@ -146,7 +150,7 @@ func (l *Logger) work() {
 		select {
 		case <-l.writeTicker.C:
 			if l.modified {
-				l.write()
+				l.write(false)
 				l.modified = false
 			}
 		case mutator := <-l.workToDo:
@@ -157,16 +161,22 @@ func (l *Logger) work() {
 }
 
 // Actually writes the log entry.
-func (l *Logger) write() {
+func (l *Logger) write(isFinal bool) {
 
 	// Run all our hooks
 	for _, hook := range l.writeHooks {
 		hook(l.properties, l.entry)
 	}
 
-	// Update the event type in case it was modified or is missing.
-	// TODO(misha): Fix this to write different event types.
-	l.entry["event_type"] = l.params.EventType
+	// Update the event type.
+	if isFinal {
+		l.entry["event_type"] = l.params.EventTypePrefix + finalSuffix
+	} else if l.hasWritten {
+		l.entry["event_type"] = l.params.EventTypePrefix + partialSuffix
+	} else {
+		l.entry["event_type"] = l.params.EventTypePrefix + startSuffix
+	}
+	l.hasWritten = true
 
 	// Write the log entry.
 	// This is a network write and will take a while, which is bad
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index bd68db1..2c5c36d 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -15,15 +15,15 @@ import (
 )
 
 var (
-	logEventType        string
+	logEventTypePrefix        string
 	logFrequencySeconds int
 )
 
 func init() {
-	flag.StringVar(&logEventType,
-		"log-event-type",
-		"experimental-data-manager-report",
-		"event_type to use in our arvados log entries. Set to empty to turn off logging")
+	flag.StringVar(&logEventTypePrefix,
+		"log-event-type-prefix",
+		"experimental-data-manager",
+		"Prefix to use in the event_type of our arvados log entries. Set to empty to turn off logging")
 	flag.IntVar(&logFrequencySeconds,
 		"log-frequency-seconds",
 		20,
@@ -45,9 +45,9 @@ func main() {
 	}
 
 	var arvLogger *logger.Logger
-	if logEventType != "" {
+	if logEventTypePrefix != "" {
 		arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
-			EventType:     logEventType,
+			EventTypePrefix:     logEventTypePrefix,
 			WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
 	}
 

commit 40e42c383646edade9b4723dfb8001cb4c873ea5
Author: mishaz <misha at curoverse.com>
Date:   Tue Feb 10 02:11:34 2015 +0000

    Moved some logging code from datamananager to loggerutil.

diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index f63f462..bd68db1 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -9,9 +9,8 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/util"
 	"git.curoverse.com/arvados.git/services/datamanager/collection"
 	"git.curoverse.com/arvados.git/services/datamanager/keep"
+	"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
 	"log"
-	"os"
-	"runtime"
 	"time"
 )
 
@@ -52,23 +51,9 @@ func main() {
 			WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
 	}
 
+	loggerutil.LogRunInfo(arvLogger)
 	if arvLogger != nil {
-		now := time.Now()
-		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
-			runInfo := make(map[string]interface{})
-			runInfo["time_started"] = now
-			runInfo["args"] = os.Args
-			hostname, err := os.Hostname()
-			if err != nil {
-				runInfo["hostname_error"] = err.Error()
-			} else {
-				runInfo["hostname"] = hostname
-			}
-			runInfo["pid"] = os.Getpid()
-			p["run_info"] = runInfo
-		})
-
-		arvLogger.AddWriteHook(LogMemoryAlloc)
+		arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
 	}
 
 	collectionChannel := make(chan collection.ReadCollections)
@@ -96,11 +81,3 @@ func main() {
 		})
 	}
 }
-
-// TODO(misha): Consider moving this to loggerutil
-func LogMemoryAlloc(properties map[string]interface{}, entry map[string]interface{}) {
-	runInfo := properties["run_info"].(map[string]interface{})
-	var memStats runtime.MemStats
-	runtime.ReadMemStats(&memStats)
-	runInfo["alloc_bytes_in_use"] = memStats.Alloc
-}
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index c19a7ab..1514922 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -5,9 +5,42 @@ package loggerutil
 import (
 	"git.curoverse.com/arvados.git/sdk/go/logger"
 	"log"
+	"os"
+	"runtime"
 	"time"
 )
 
+// Useful to call at the begining of execution to log info about the
+// current run.
+func LogRunInfo(arvLogger *logger.Logger) {
+	if arvLogger != nil {
+		now := time.Now()
+		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+			runInfo := make(map[string]interface{})
+			runInfo["time_started"] = now
+			runInfo["args"] = os.Args
+			hostname, err := os.Hostname()
+			if err != nil {
+				runInfo["hostname_error"] = err.Error()
+			} else {
+				runInfo["hostname"] = hostname
+			}
+			runInfo["pid"] = os.Getpid()
+			p["run_info"] = runInfo
+		})
+	}
+}
+
+// A LogMutator that records the current memory usage. This is most useful as a logger write hook.
+//
+// Assumes we already have a map named "run_info" in properties. LogRunInfo() can create such a map for you if you call it.
+func LogMemoryAlloc(p map[string]interface{}, e map[string]interface{}) {
+	runInfo := p["run_info"].(map[string]interface{})
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	runInfo["alloc_bytes_in_use"] = memStats.Alloc
+}
+
 func FatalWithMessage(arvLogger *logger.Logger, message string) {
 	if arvLogger != nil {
 		arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {

commit bdf093d22ebbdaaadcd822fe32a5fe150fe649f6
Author: mishaz <misha at curoverse.com>
Date:   Tue Feb 10 01:55:37 2015 +0000

    Updated logger to do all work in a dedicated goroutine, so we don't need to worry about locking. Small changes to calling code.

diff --git a/sdk/go/logger/logger.go b/sdk/go/logger/logger.go
index a53ab3c..e6e1ed6 100644
--- a/sdk/go/logger/logger.go
+++ b/sdk/go/logger/logger.go
@@ -2,8 +2,8 @@
 //
 // This package is useful for maintaining a log object that is updated
 // over time. Every time the object is updated, it will be written to
-// the log. Writes will be throttled to no more than one every
-// WriteFrequencySeconds
+// the log. Writes will be throttled to no more frequent than
+// WriteInterval.
 //
 // This package is safe for concurrent use as long as:
 // The maps passed to a LogMutator are not accessed outside of the
@@ -23,14 +23,13 @@ package logger
 import (
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"log"
-	"sync"
 	"time"
 )
 
 type LoggerParams struct {
-	Client               arvadosclient.ArvadosClient // The client we use to write log entries
-	EventType            string                      // The event type to assign to the log entry.
-	MinimumWriteInterval time.Duration               // Wait at least this long between log writes
+	Client        arvadosclient.ArvadosClient // The client we use to write log entries
+	EventType     string                      // The event type to assign to the log entry.
+	WriteInterval time.Duration               // Wait at least this long between log writes
 }
 
 // A LogMutator is a function which modifies the log entry.
@@ -52,13 +51,12 @@ type Logger struct {
 	entry      map[string]interface{} // Convenience shortcut into data
 	properties map[string]interface{} // Convenience shortcut into data
 
-	lock   sync.Locker  // Synchronizes access to this struct
 	params LoggerParams // Parameters we were given
 
-	// Variables used to determine when and if we write to the log.
-	nextWriteAllowed time.Time // The next time we can write, respecting MinimumWriteInterval
-	modified         bool      // Has this data been modified since the last write?
-	writeScheduled   bool      // Is a write been scheduled for the future?
+	// Variables to coordinate updating and writing.
+	modified    bool            // Has this data been modified since the last write?
+	workToDo    chan LogMutator // Work to do in the worker thread.
+	writeTicker *time.Ticker    // On each tick we write the log data to arvados, if it has been modified.
 
 	writeHooks []LogMutator // Mutators we call before each write.
 }
@@ -74,91 +72,88 @@ func NewLogger(params LoggerParams) *Logger {
 	}
 
 	l := &Logger{data: make(map[string]interface{}),
-		lock:   &sync.Mutex{},
 		params: params}
 	l.entry = make(map[string]interface{})
 	l.data["log"] = l.entry
 	l.properties = make(map[string]interface{})
 	l.entry["properties"] = l.properties
-	return l
-}
 
-// Updates the log data and then writes it to the api server. If the
-// log has been recently written then the write will be postponed to
-// respect MinimumWriteInterval and this function will return before
-// the write occurs.
-func (l *Logger) Update(mutator LogMutator) {
-	l.lock.Lock()
+	l.workToDo = make(chan LogMutator, 10)
+	l.writeTicker = time.NewTicker(params.WriteInterval)
 
-	mutator(l.properties, l.entry)
-	l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
+	// Start the worker goroutine.
+	go l.work()
 
-	l.considerWriting()
-
-	l.lock.Unlock()
+	return l
 }
 
-// Similar to Update(), but forces a write without respecting the
-// MinimumWriteInterval. This is useful if you know that you're about
-// to quit (e.g. if you discovered a fatal error, or you're finished),
-// since go will not wait for timers (including the pending write
-// timer) to go off before exiting.
-func (l *Logger) ForceUpdate(mutator LogMutator) {
-	l.lock.Lock()
+// Exported functions will be called from other goroutines, therefore
+// all they are allowed to do is enqueue work to be done in the worker
+// goroutine.
 
-	mutator(l.properties, l.entry)
-	l.modified = true // We assume the mutator modified the log, even though we don't know for sure.
-
-	l.write()
-	l.lock.Unlock()
-}
-
-// Adds a hook which will be called every time this logger writes an entry.
-func (l *Logger) AddWriteHook(hook LogMutator) {
-	l.lock.Lock()
-	l.writeHooks = append(l.writeHooks, hook)
-	// TODO(misha): Consider setting modified and attempting a write.
-	l.lock.Unlock()
+// Enqueues an update. This will happen in another goroutine after
+// this method returns.
+func (l *Logger) Update(mutator LogMutator) {
+	l.workToDo <- mutator
 }
 
-// This function is called on a timer when we have something to write,
-// but need to schedule the write for the future to respect
-// MinimumWriteInterval.
-func (l *Logger) acquireLockConsiderWriting() {
-	l.lock.Lock()
+// Similar to Update(), but writes the log entry as soon as possible
+// (ignoring MinimumWriteInterval) and blocks until the entry has been
+// written. This is useful if you know that you're about to quit
+// (e.g. if you discovered a fatal error, or you're finished), since
+// go will not wait for timers (including the pending write timer) to
+// go off before exiting.
+func (l *Logger) FinalUpdate(mutator LogMutator) {
+	// Block on this channel until everything finishes
+	done := make(chan bool)
+
+	// TODO(misha): Consider not accepting any future updates somehow,
+	// since they won't get written if they come in after this.
+
+	// Stop the periodic write ticker. We'll perform the final write
+	// before returning from this function.
+	l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+		l.writeTicker.Stop()
+	}
 
-	// We are the scheduled write, so there are no longer future writes
-	// scheduled.
-	l.writeScheduled = false
+	// Apply the final update
+	l.workToDo <- mutator
 
-	l.considerWriting()
+	// Perform the write and signal that we can return.
+	l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+		// TODO(misha): Add a boolean arg to write() to indicate that it's
+		// final so that we can set the appropriate event type.
+		l.write()
+		done <- true
+	}
 
-	l.lock.Unlock()
+	// Wait until we've performed the write.
+	<-done
 }
 
-// The above methods each acquire the lock and release it.
-// =======================================================
-// The below methods all assume we're holding a lock.
-
-// Check whether we have anything to write. If we do, then either
-// write it now or later, based on what we're allowed.
-func (l *Logger) considerWriting() {
-	if !l.modified {
-		// Nothing to write
-	} else if l.writeAllowedNow() {
-		l.write()
-	} else if l.writeScheduled {
-		// A future write is already scheduled, we don't need to do anything.
-	} else {
-		writeAfter := l.nextWriteAllowed.Sub(time.Now())
-		time.AfterFunc(writeAfter, l.acquireLockConsiderWriting)
-		l.writeScheduled = true
+// Adds a hook which will be called every time this logger writes an entry.
+func (l *Logger) AddWriteHook(hook LogMutator) {
+	// We do the work in a LogMutator so that it happens in the worker
+	// goroutine.
+	l.workToDo <- func(p map[string]interface{}, e map[string]interface{}) {
+		l.writeHooks = append(l.writeHooks, hook)
 	}
 }
 
-// Whether writing now would respect MinimumWriteInterval
-func (l *Logger) writeAllowedNow() bool {
-	return l.nextWriteAllowed.Before(time.Now())
+// The worker loop
+func (l *Logger) work() {
+	for {
+		select {
+		case <-l.writeTicker.C:
+			if l.modified {
+				l.write()
+				l.modified = false
+			}
+		case mutator := <-l.workToDo:
+			mutator(l.properties, l.entry)
+			l.modified = true
+		}
+	}
 }
 
 // Actually writes the log entry.
@@ -170,24 +165,20 @@ func (l *Logger) write() {
 	}
 
 	// Update the event type in case it was modified or is missing.
+	// TODO(misha): Fix this to write different event types.
 	l.entry["event_type"] = l.params.EventType
 
 	// Write the log entry.
 	// This is a network write and will take a while, which is bad
-	// because we're holding a lock and all other goroutines will back
-	// up behind it.
+	// because we're blocking all the other work on this goroutine.
 	//
 	// TODO(misha): Consider rewriting this so that we can encode l.data
-	// into a string, release the lock, write the string, and then
-	// acquire the lock again to note that we succeeded in writing. This
-	// will be tricky and will require support in the client.
+	// into a string, and then perform the actual write in another
+	// routine. This will be tricky and will require support in the
+	// client.
 	err := l.params.Client.Create("logs", l.data, nil)
 	if err != nil {
 		log.Printf("Attempted to log: %v", l.data)
 		log.Fatalf("Received error writing log: %v", err)
 	}
-
-	// Update stats.
-	l.nextWriteAllowed = time.Now().Add(l.params.MinimumWriteInterval)
-	l.modified = false
 }
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 424db83..9a7a838 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -65,15 +65,6 @@ func init() {
 		"File to write the heap profiles to. Leave blank to skip profiling.")
 }
 
-// // Methods to implement util.SdkListResponse Interface
-// func (s SdkCollectionList) NumItemsAvailable() (numAvailable int, err error) {
-// 	return s.ItemsAvailable, nil
-// }
-
-// func (s SdkCollectionList) NumItemsContained() (numContained int, err error) {
-// 	return len(s.Items), nil
-// }
-
 // Write the heap profile to a file for later review.
 // Since a file is expected to only contain a single heap profile this
 // function overwrites the previously written profile, so it is safe
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 398c877..f63f462 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -48,14 +48,15 @@ func main() {
 	var arvLogger *logger.Logger
 	if logEventType != "" {
 		arvLogger = logger.NewLogger(logger.LoggerParams{Client: arv,
-			EventType:            logEventType,
-			MinimumWriteInterval: time.Second * time.Duration(logFrequencySeconds)})
+			EventType:     logEventType,
+			WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
 	}
 
 	if arvLogger != nil {
+		now := time.Now()
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			runInfo := make(map[string]interface{})
-			runInfo["time_started"] = time.Now()
+			runInfo["time_started"] = now
 			runInfo["args"] = os.Args
 			hostname, err := os.Hostname()
 			if err != nil {
@@ -90,7 +91,7 @@ func main() {
 	// Log that we're finished. We force the recording, since go will
 	// not wait for the timer before exiting.
 	if arvLogger != nil {
-		arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+		arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
 			p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
 		})
 	}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 20a5931..dcd6c49 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -69,15 +69,6 @@ type KeepServiceList struct {
 	KeepServers    []ServerAddress `json:"items"`
 }
 
-// Methods to implement util.SdkListResponse Interface
-func (k KeepServiceList) NumItemsAvailable() (numAvailable int, err error) {
-	return k.ItemsAvailable, nil
-}
-
-func (k KeepServiceList) NumItemsContained() (numContained int, err error) {
-	return len(k.KeepServers), nil
-}
-
 var (
 	// Don't access the token directly, use getDataManagerToken() to
 	// make sure it's been read.
@@ -244,10 +235,11 @@ func GetServerStatus(arvLogger *logger.Logger,
 		keepServer.Port)
 
 	if arvLogger != nil {
+		now := time.Now()
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			keepInfo := p["keep_info"].(map[string]interface{})
 			serverInfo := make(map[string]interface{})
-			serverInfo["time_status_request_sent"] = time.Now()
+			serverInfo["time_status_request_sent"] = now
 
 			keepInfo[keepServer.String()] = serverInfo
 		})
@@ -274,10 +266,11 @@ func GetServerStatus(arvLogger *logger.Logger,
 	}
 
 	if arvLogger != nil {
+		now := time.Now()
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			keepInfo := p["keep_info"].(map[string]interface{})
 			serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-			serverInfo["time_status_response_processed"] = time.Now()
+			serverInfo["time_status_response_processed"] = now
 			serverInfo["status"] = keepStatus
 		})
 	}
@@ -289,10 +282,11 @@ func CreateIndexRequest(arvLogger *logger.Logger,
 	log.Println("About to fetch keep server contents from " + url)
 
 	if arvLogger != nil {
+		now := time.Now()
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			keepInfo := p["keep_info"].(map[string]interface{})
 			serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-			serverInfo["time_index_request_sent"] = time.Now()
+			serverInfo["time_index_request_sent"] = now
 		})
 	}
 
@@ -319,11 +313,11 @@ func ReadServerResponse(arvLogger *logger.Logger,
 	}
 
 	if arvLogger != nil {
+		now := time.Now()
 		arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 			keepInfo := p["keep_info"].(map[string]interface{})
 			serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
-
-			serverInfo["time_index_response_received"] = time.Now()
+			serverInfo["time_index_response_received"] = now
 		})
 	}
 
@@ -393,11 +387,12 @@ func ReadServerResponse(arvLogger *logger.Logger,
 			numSizeDisagreements)
 
 		if arvLogger != nil {
+			now := time.Now()
 			arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
 				keepInfo := p["keep_info"].(map[string]interface{})
 				serverInfo := keepInfo[keepServer.String()].(map[string]interface{})
 
-				serverInfo["time_processing_finished"] = time.Now()
+				serverInfo["time_processing_finished"] = now
 				serverInfo["lines_received"] = numLines
 				serverInfo["duplicates_seen"] = numDuplicates
 				serverInfo["size_disagreements_seen"] = numSizeDisagreements
diff --git a/services/datamanager/loggerutil/loggerutil.go b/services/datamanager/loggerutil/loggerutil.go
index fa876d4..c19a7ab 100644
--- a/services/datamanager/loggerutil/loggerutil.go
+++ b/services/datamanager/loggerutil/loggerutil.go
@@ -8,12 +8,9 @@ import (
 	"time"
 )
 
-// Assumes you haven't already called arvLogger.Edit()!
-// If you have called arvLogger.Edit() this method will hang waiting
-// for the lock you're already holding.
 func FatalWithMessage(arvLogger *logger.Logger, message string) {
 	if arvLogger != nil {
-		arvLogger.ForceUpdate(func(p map[string]interface{}, e map[string]interface{}) {
+		arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
 			p["FATAL"] = message
 			p["run_info"].(map[string]interface{})["time_finished"] = time.Now()
 		})

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list