[ARVADOS] created: 1.1.0-22-g6ee6e65

[Git user]

        at  6ee6e654bc873db10037c735a63697d295ec40cb (commit)


commit 6ee6e654bc873db10037c735a63697d295ec40cb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Oct 12 21:13:17 2017 -0400

    12447: Reduce memory footprint of crunch-run.
    
    * BlockCache reads into a fixed-capacity buffer, to avoid reallocations (this
      cuts resident memory usage by 1/2 and makes memory footprint extremely
      stable)
    * Limit the crunch-run BlockCache to 2 blocks instead of 4 (cuts footprint by
      1/2 again)
    * Clear the crunch-run BlockCache after loading the Docker image (probably cuts
      post-image-load footprint by 1/2 again).
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/go/keepclient/block_cache.go b/sdk/go/keepclient/block_cache.go
index 88617ea..18f7de9 100644
--- a/sdk/go/keepclient/block_cache.go
+++ b/sdk/go/keepclient/block_cache.go
@@ -5,7 +5,8 @@
 package keepclient
 
 import (
-	"io/ioutil"
+	"bytes"
+	"io"
 	"sort"
 	"sync"
 	"time"
@@ -65,16 +66,16 @@ func (c *BlockCache) Get(kc *KeepClient, locator string) ([]byte, error) {
 		c.cache[cacheKey] = b
 		go func() {
 			rdr, _, _, err := kc.Get(locator)
-			var data []byte
+			data := bytes.NewBuffer(make([]byte, 0, BLOCKSIZE))
 			if err == nil {
-				data, err = ioutil.ReadAll(rdr)
+				_, err = io.Copy(data, rdr)
 				err2 := rdr.Close()
 				if err == nil {
 					err = err2
 				}
 			}
 			c.mtx.Lock()
-			b.data, b.err = data, err
+			b.data, b.err = data.Bytes(), err
 			c.mtx.Unlock()
 			close(b.fetched)
 			go c.Sweep()
@@ -96,6 +97,12 @@ func (c *BlockCache) setup() {
 	c.cache = make(map[string]*cacheBlock)
 }
 
+func (c *BlockCache) Clear() {
+	c.mtx.Lock()
+	c.setup()
+	c.mtx.Unlock()
+}
+
 type timeSlice []time.Time
 
 func (ts timeSlice) Len() int { return len(ts) }
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 8af1210..6d4b06b 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -11,6 +11,11 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"git.curoverse.com/arvados.git/lib/crunchstat"
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"git.curoverse.com/arvados.git/sdk/go/manifest"
 	"io"
 	"io/ioutil"
 	"log"
@@ -19,18 +24,14 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"runtime"
+	"runtime/pprof"
 	"sort"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
-	"git.curoverse.com/arvados.git/lib/crunchstat"
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
-	"git.curoverse.com/arvados.git/sdk/go/manifest"
-
 	dockertypes "github.com/docker/docker/api/types"
 	dockercontainer "github.com/docker/docker/api/types/container"
 	dockernetwork "github.com/docker/docker/api/types/network"
@@ -264,6 +265,8 @@ func (runner *ContainerRunner) LoadImage() (err error) {
 
 	runner.ContainerConfig.Image = imageID
 
+	keepclient.DefaultBlockCache.Clear()
+
 	return nil
 }
 
@@ -1429,6 +1432,7 @@ func main() {
 	networkMode := flag.String("container-network-mode", "default",
 		`Set networking mode for container.  Corresponds to Docker network mode (--net).
     	`)
+	memprofile := flag.String("memprofile", "", "write memory profile to `file`")
 	flag.Parse()
 
 	containerId := flag.Arg(0)
@@ -1472,9 +1476,21 @@ func main() {
 		cr.expectCgroupParent = p
 	}
 
-	err = cr.Run()
-	if err != nil {
-		log.Fatalf("%s: %v", containerId, err)
+	runerr := cr.Run()
+
+	if *memprofile != "" {
+		f, err := os.Create(*memprofile)
+		if err != nil {
+			log.Fatal("could not create memory profile: ", err)
+		}
+		runtime.GC() // get up-to-date statistics
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			log.Fatal("could not write memory profile: ", err)
+		}
+		f.Close()
 	}
 
+	if runerr != nil {
+		log.Fatalf("%s: %v", containerId, runerr)
+	}
 }

-----------------------------------------------------------------------


hooks/post-receive
--