[ARVADOS] updated: 1.1.3-216-g304a807
Git user
git at public.curoverse.com
Thu Mar 15 12:50:18 EDT 2018
Summary of changes:
services/crunch-run/crunchrun_test.go | 42 ++++++++++++++++++++++++++++-------
services/crunch-run/logging_test.go | 9 ++++++--
services/crunch-run/upload_test.go | 27 ++++++++++++++++------
3 files changed, 61 insertions(+), 17 deletions(-)
via 304a807e24363f3c38949b2da10c500f338d08b0 (commit)
from bc27c169f78471a21f6f5c35a879e4b57afa0e22 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 304a807e24363f3c38949b2da10c500f338d08b0
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Mar 15 12:45:52 2018 -0400
11583: 13100: Fix memory leak in test suite.
Something (an unterminated goroutine?) holds a reference to the keep
client, which (in the test suite) holds a reference to the last block
it wrote. Releasing the reference to the block reduces test suite
memory use by about 2.6 GB.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index dd977c3..48da8ed 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -364,6 +364,10 @@ func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error
func (*KeepTestClient) ClearBlockCache() {
}
+func (client *KeepTestClient) Close() {
+ client.Content = nil
+}
+
type FileWrapper struct {
io.ReadCloser
len int64
@@ -408,6 +412,7 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
func (s *TestSuite) TestLoadImage(c *C) {
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(&ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
_, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
@@ -514,7 +519,9 @@ func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename
func (s *TestSuite) TestLoadImageArvError(c *C) {
// (1) Arvados error
- cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Container.ContainerImage = hwPDH
err := cr.LoadImage()
@@ -585,7 +592,9 @@ func (s *TestSuite) TestRunContainer(c *C) {
t.logWriter.Write(dockerLog(1, "Hello world\n"))
t.logWriter.Close()
}
- cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(&ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
var logs TestLogs
cr.NewLogWriter = logs.NewTestLoggingWriter
@@ -610,6 +619,7 @@ func (s *TestSuite) TestRunContainer(c *C) {
func (s *TestSuite) TestCommitLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
@@ -630,6 +640,7 @@ func (s *TestSuite) TestCommitLogs(c *C) {
func (s *TestSuite) TestUpdateContainerRunning(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
err := cr.UpdateContainerRunning()
@@ -641,6 +652,7 @@ func (s *TestSuite) TestUpdateContainerRunning(c *C) {
func (s *TestSuite) TestUpdateContainerComplete(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.LogsPDH = new(string)
@@ -661,6 +673,7 @@ func (s *TestSuite) TestUpdateContainerComplete(c *C) {
func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.cCancelled = true
cr.finalState = "Cancelled"
@@ -695,7 +708,9 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
api = &ArvTestClient{Container: rec}
s.docker.api = api
- cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr = NewContainerRunner(api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.statInterval = 100 * time.Millisecond
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
@@ -971,7 +986,9 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api := &ArvTestClient{Container: rec}
- cr := NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
cr.MkArvClient = func(token string) (IArvadosClient, error) {
return &ArvTestClient{}, nil
@@ -1041,6 +1058,7 @@ func stubCert(temp string) string {
func (s *TestSuite) TestSetupMounts(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
@@ -1449,7 +1467,9 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
- cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr = NewContainerRunner(api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
cr.MkArvClient = func(token string) (IArvadosClient, error) {
@@ -1888,7 +1908,9 @@ func (s *TestSuite) TestStderrMount(c *C) {
}
func (s *TestSuite) TestNumberRoundTrip(c *C) {
- cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(&ArvTestClient{callraw: true}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.fetchContainerRecord()
jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content)
@@ -1898,7 +1920,9 @@ func (s *TestSuite) TestNumberRoundTrip(c *C) {
}
func (s *TestSuite) TestEvalSymlinks(c *C) {
- cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(&ArvTestClient{callraw: true}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
realTemp, err := ioutil.TempDir("", "crunchrun_test-")
c.Assert(err, IsNil)
@@ -1928,7 +1952,9 @@ func (s *TestSuite) TestEvalSymlinks(c *C) {
}
func (s *TestSuite) TestEvalSymlinkDir(c *C) {
- cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cr := NewContainerRunner(&ArvTestClient{callraw: true}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
realTemp, err := ioutil.TempDir("", "crunchrun_test-")
c.Assert(err, IsNil)
diff --git a/services/crunch-run/logging_test.go b/services/crunch-run/logging_test.go
index 87593be..abac2bb 100644
--- a/services/crunch-run/logging_test.go
+++ b/services/crunch-run/logging_test.go
@@ -6,11 +6,12 @@ package main
import (
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- . "gopkg.in/check.v1"
"strings"
"testing"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ . "gopkg.in/check.v1"
)
type LoggingTestSuite struct{}
@@ -34,6 +35,7 @@ var _ = Suite(&LoggingTestSuite{})
func (s *LoggingTestSuite) TestWriteLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
@@ -61,6 +63,7 @@ func (s *LoggingTestSuite) TestWriteLogsLarge(c *C) {
}
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
cr.CrunchLog.Immediate = nil
@@ -82,6 +85,7 @@ func (s *LoggingTestSuite) TestWriteLogsLarge(c *C) {
func (s *LoggingTestSuite) TestWriteMultipleLogs(c *C) {
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
ts := &TestTimestamper{}
cr.CrunchLog.Timestamper = ts.Timestamp
@@ -135,6 +139,7 @@ func testWriteLogsWithRateLimit(c *C, throttleParam string, throttleValue int, t
api := &ArvTestClient{}
kc := &KeepTestClient{}
+ defer kc.Close()
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
diff --git a/services/crunch-run/upload_test.go b/services/crunch-run/upload_test.go
index 86dab41..24333c3 100644
--- a/services/crunch-run/upload_test.go
+++ b/services/crunch-run/upload_test.go
@@ -5,13 +5,14 @@
package main
import (
- . "gopkg.in/check.v1"
"io/ioutil"
"log"
"os"
"path/filepath"
"sync"
"syscall"
+
+ . "gopkg.in/check.v1"
)
type UploadTestSuite struct{}
@@ -46,7 +47,9 @@ func (s *TestSuite) TestSimpleUpload(c *C) {
ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
c.Check(str, Equals, ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt\n")
@@ -67,7 +70,9 @@ func (s *TestSuite) TestUploadThreeFiles(c *C) {
c.Assert(err, IsNil)
}
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
@@ -85,7 +90,9 @@ func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
ioutil.WriteFile(tmpdir+"/subdir/file2.txt", []byte("bar"), 0600)
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
@@ -119,7 +126,9 @@ func (s *TestSuite) TestSimpleUploadLarge(c *C) {
ioutil.WriteFile(tmpdir+"/"+"file2.txt", []byte("bar"), 0600)
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
@@ -136,7 +145,9 @@ func (s *TestSuite) TestUploadEmptySubdir(c *C) {
ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte("foo"), 0600)
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
@@ -152,7 +163,9 @@ func (s *TestSuite) TestUploadEmptyFile(c *C) {
ioutil.WriteFile(tmpdir+"/"+"file1.txt", []byte(""), 0600)
- cw := CollectionWriter{0, &KeepTestClient{}, nil, nil, sync.Mutex{}}
+ kc := &KeepTestClient{}
+ defer kc.Close()
+ cw := CollectionWriter{0, kc, nil, nil, sync.Mutex{}}
str, err := writeTree(&cw, tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list