[ARVADOS] updated: 1.1.2-123-ge6fa5ac
Git user
git at public.curoverse.com
Tue Jan 23 15:15:50 EST 2018
Summary of changes:
services/crunch-run/crunchrun.go | 50 +++++-------
services/crunch-run/crunchrun_test.go | 139 +++++++++++++++++-----------------
2 files changed, 89 insertions(+), 100 deletions(-)
via e6fa5ac95cff84a7b00dd8ab32349d88473400da (commit)
via e8de8d362df44459ecbdff44ed27a12b7652762c (commit)
via 03ee5e57fab4ec34b5830e8a93d329a084adf42c (commit)
from 8758f3ddc81b9ef9aaab111eb331e452c4ec7de5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e6fa5ac95cff84a7b00dd8ab32349d88473400da
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Jan 23 15:08:08 2018 -0500
12891: Don't wait for WaitContainer to receive loggingDone.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 72df182..7eefb1a 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -582,7 +582,6 @@ func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) {
}
}
- runner.loggingDone <- true
close(runner.loggingDone)
return
}
commit e8de8d362df44459ecbdff44ed27a12b7652762c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Jan 23 15:05:22 2018 -0500
12891: Fix some test races.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 8e6381c..22989bb 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -42,11 +42,17 @@ func TestCrunchExec(t *testing.T) {
TestingT(t)
}
-type TestSuite struct{}
-
// Gocheck boilerplate
var _ = Suite(&TestSuite{})
+type TestSuite struct {
+ docker *TestDockerClient
+}
+
+func (s *TestSuite) SetUpTest(c *C) {
+ s.docker = NewTestDockerClient()
+}
+
type ArvTestClient struct {
Total int64
Calls int
@@ -88,18 +94,18 @@ type TestDockerClient struct {
logReader io.ReadCloser
logWriter io.WriteCloser
fn func(t *TestDockerClient)
- finish int
+ exitCode int
stop chan bool
cwd string
env []string
api *ArvTestClient
realTemp string
+ calledWait bool
}
-func NewTestDockerClient(exitCode int) *TestDockerClient {
+func NewTestDockerClient() *TestDockerClient {
t := &TestDockerClient{}
t.logReader, t.logWriter = io.Pipe()
- t.finish = exitCode
t.stop = make(chan bool, 1)
t.cwd = "/"
return t
@@ -131,16 +137,16 @@ func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockerco
}
func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
- if t.finish == 3 {
+ if t.exitCode == 3 {
return errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
}
- if t.finish == 4 {
+ if t.exitCode == 4 {
return errors.New(`panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`)
}
- if t.finish == 5 {
+ if t.exitCode == 5 {
return errors.New(`Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`)
}
- if t.finish == 6 {
+ if t.exitCode == 6 {
return errors.New(`Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`)
}
@@ -158,19 +164,18 @@ func (t *TestDockerClient) ContainerRemove(ctx context.Context, container string
}
func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error) {
- body := make(chan dockercontainer.ContainerWaitOKBody)
+ t.calledWait = true
+ body := make(chan dockercontainer.ContainerWaitOKBody, 1)
err := make(chan error)
go func() {
t.fn(t)
- body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)}
- close(body)
- close(err)
+ body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.exitCode)}
}()
return body, err
}
func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
- if t.finish == 2 {
+ if t.exitCode == 2 {
return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
}
@@ -182,7 +187,7 @@ func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string
}
func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
- if t.finish == 2 {
+ if t.exitCode == 2 {
return dockertypes.ImageLoadResponse{}, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
}
_, err := io.Copy(ioutil.Discard, input)
@@ -396,8 +401,7 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
func (s *TestSuite) TestLoadImage(c *C) {
kc := &KeepTestClient{}
- docker := NewTestDockerClient(0)
- cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr := NewContainerRunner(&ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
_, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
@@ -512,8 +516,7 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
func (s *TestSuite) TestLoadImageKeepError(c *C) {
// (2) Keep error
- docker := NewTestDockerClient(0)
- cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Container.ContainerImage = hwPDH
err := cr.LoadImage()
@@ -531,8 +534,7 @@ func (s *TestSuite) TestLoadImageCollectionError(c *C) {
func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
// (4) Collection doesn't contain image
- docker := NewTestDockerClient(0)
- cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Container.ContainerImage = hwPDH
err := cr.LoadImage()
@@ -572,12 +574,11 @@ func dockerLog(fd byte, msg string) []byte {
}
func (s *TestSuite) TestRunContainer(c *C) {
- docker := NewTestDockerClient(0)
- docker.fn = func(t *TestDockerClient) {
+ s.docker.fn = func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, "Hello world\n"))
t.logWriter.Close()
}
- cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
var logs TestLogs
cr.NewLogWriter = logs.NewTestLoggingWriter
@@ -667,18 +668,18 @@ func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
// Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
// dress rehearsal of the Run() function, starting from a JSON container record.
-func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
+func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
rec := arvados.Container{}
err := json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient(exitCode)
- docker.fn = fn
- docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.exitCode = exitCode
+ s.docker.fn = fn
+ s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
- docker.api = api
- cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ s.docker.api = api
+ cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.statInterval = 100 * time.Millisecond
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
@@ -687,7 +688,7 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f
c.Assert(err, IsNil)
defer os.RemoveAll(realTemp)
- docker.realTemp = realTemp
+ s.docker.realTemp = realTemp
tempcount := 0
cr.MkTempDir = func(_ string, prefix string) (string, error) {
@@ -730,7 +731,7 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f
}
func (s *TestSuite) TestFullRunHello(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -751,7 +752,7 @@ func (s *TestSuite) TestFullRunHello(c *C) {
}
func (s *TestSuite) TestCrunchstat(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["sleep", "1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -784,7 +785,7 @@ func (s *TestSuite) TestCrunchstat(c *C) {
func (s *TestSuite) TestNodeInfoLog(c *C) {
os.Setenv("SLURMD_NODENAME", "compute2")
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["sleep", "1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -818,7 +819,7 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
}
func (s *TestSuite) TestContainerRecordLog(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["sleep", "1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -841,7 +842,7 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
}
func (s *TestSuite) TestFullRunStderr(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -866,7 +867,7 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
}
func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["pwd"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -887,7 +888,7 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
}
func (s *TestSuite) TestFullRunSetCwd(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["pwd"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -909,7 +910,7 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
func (s *TestSuite) TestStopOnSignal(c *C) {
s.testStopContainer(c, func(cr *ContainerRunner) {
go func() {
- for cr.ContainerID == "" {
+ for !s.docker.calledWait {
time.Sleep(time.Millisecond)
}
cr.SigChan <- syscall.SIGINT
@@ -943,16 +944,15 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
err := json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient(0)
- docker.fn = func(t *TestDockerClient) {
+ s.docker.fn = func(t *TestDockerClient) {
<-t.stop
t.logWriter.Write(dockerLog(1, "foo\n"))
t.logWriter.Close()
}
- docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api := &ArvTestClient{Container: rec}
- cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr := NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
setup(cr)
@@ -978,7 +978,7 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
}
func (s *TestSuite) TestFullRunSetEnv(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1359,7 +1359,7 @@ func (s *TestSuite) TestStdout(c *C) {
"runtime_constraints": {}
}`
- api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1370,17 +1370,16 @@ func (s *TestSuite) TestStdout(c *C) {
}
// Used by the TestStdoutWithWrongPath*()
-func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
+func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
rec := arvados.Container{}
err = json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient(0)
- docker.fn = fn
- docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.fn = fn
+ s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
- cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
@@ -1389,7 +1388,7 @@ func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (ap
}
func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
- _, _, err := StdoutErrorRunHelper(c, `{
+ _, _, err := s.stdoutErrorRunHelper(c, `{
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
"output_path": "/tmp"
}`, func(t *TestDockerClient) {})
@@ -1399,7 +1398,7 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
}
func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
- _, _, err := StdoutErrorRunHelper(c, `{
+ _, _, err := s.stdoutErrorRunHelper(c, `{
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
"output_path": "/tmp"
}`, func(t *TestDockerClient) {})
@@ -1409,7 +1408,7 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
- _, _, err := StdoutErrorRunHelper(c, `{
+ _, _, err := s.stdoutErrorRunHelper(c, `{
"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
"output_path": "/tmp"
}`, func(t *TestDockerClient) {})
@@ -1421,7 +1420,7 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
func (s *TestSuite) TestFullRunWithAPI(c *C) {
os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
defer os.Unsetenv("ARVADOS_API_HOST")
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1444,7 +1443,7 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
func (s *TestSuite) TestFullRunSetOutput(c *C) {
os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
defer os.Unsetenv("ARVADOS_API_HOST")
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1484,7 +1483,7 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1519,7 +1518,7 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
}
- api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, runner, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1571,7 +1570,7 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
"b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1612,7 +1611,7 @@ func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz")
os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2")
os.Symlink("/keep/foo2/subdir1", t.realTemp+"/2/baz3")
@@ -1654,7 +1653,7 @@ func (s *TestSuite) TestOutputError(c *C) {
extraMounts := []string{}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
os.Symlink("/etc/hosts", t.realTemp+"/2/baz")
t.logWriter.Close()
})
@@ -1678,7 +1677,7 @@ func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
extraMounts := []string{}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
rf, _ := os.Create(t.realTemp + "/2/realfile")
rf.Write([]byte("foo"))
rf.Close()
@@ -1735,7 +1734,7 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
"b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1770,7 +1769,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
"runtime_constraints": {}
}`
- api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+ api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
})
@@ -1790,7 +1789,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
}
func (s *TestSuite) TestStderrMount(c *C) {
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello;exit 1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -1887,7 +1886,7 @@ exec echo killme
ech := tf.Name()
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -1912,7 +1911,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
ech := ""
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -1935,7 +1934,7 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) {
ech := ""
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -1957,7 +1956,7 @@ func (s *TestSuite) TestBadCommand1(c *C) {
ech := ""
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -1979,7 +1978,7 @@ func (s *TestSuite) TestBadCommand2(c *C) {
ech := ""
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -2001,7 +2000,7 @@ func (s *TestSuite) TestBadCommand3(c *C) {
ech := ""
brokenNodeHook = &ech
- api, _, _ := FullRunHelper(c, `{
+ api, _, _ := s.fullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
commit 03ee5e57fab4ec34b5830e8a93d329a084adf42c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Jan 23 14:39:26 2018 -0500
12891: Simplify "arv-mount exits before container ends" detection.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index a249273..72df182 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -133,7 +133,6 @@ type ContainerRunner struct {
setCgroupParent string
cStateLock sync.Mutex
- cStarted bool // StartContainer() succeeded
cCancelled bool // StopContainer() invoked
enableNetwork string // one of "default" or "always"
@@ -161,7 +160,7 @@ func (runner *ContainerRunner) setupSignals() {
func (runner *ContainerRunner) stop() {
runner.cStateLock.Lock()
defer runner.cStateLock.Unlock()
- if !runner.cStarted {
+ if runner.ContainerID == "" {
return
}
runner.cCancelled = true
@@ -920,46 +919,38 @@ func (runner *ContainerRunner) StartContainer() error {
}
return fmt.Errorf("could not start container: %v%s", err, advice)
}
- runner.cStarted = true
return nil
}
// WaitFinish waits for the container to terminate, capture the exit code, and
// close the stdout/stderr logging.
-func (runner *ContainerRunner) WaitFinish() (err error) {
+func (runner *ContainerRunner) WaitFinish() error {
runner.CrunchLog.Print("Waiting for container to finish")
waitOk, waitErr := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID, dockercontainer.WaitConditionNotRunning)
+ arvMountExit := runner.ArvMountExit
+ for {
+ select {
+ case waitBody := <-waitOk:
+ runner.CrunchLog.Printf("Container exited with code: %v", waitBody.StatusCode)
+ code := int(waitBody.StatusCode)
+ runner.ExitCode = &code
+
+ // wait for stdout/stderr to complete
+ <-runner.loggingDone
+ return nil
- go func() {
- <-runner.ArvMountExit
- if runner.cStarted {
+ case err := <-waitErr:
+ return fmt.Errorf("container wait: %v", err)
+
+ case <-arvMountExit:
runner.CrunchLog.Printf("arv-mount exited while container is still running. Stopping container.")
runner.stop()
+ // arvMountExit will always be ready now that
+ // it's closed, but that doesn't interest us.
+ arvMountExit = nil
}
- }()
-
- var waitBody dockercontainer.ContainerWaitOKBody
- select {
- case waitBody = <-waitOk:
- case err = <-waitErr:
}
-
- // Container isn't running any more
- runner.cStarted = false
-
- if err != nil {
- return fmt.Errorf("container wait: %v", err)
- }
-
- runner.CrunchLog.Printf("Container exited with code: %v", waitBody.StatusCode)
- code := int(waitBody.StatusCode)
- runner.ExitCode = &code
-
- // wait for stdout/stderr to complete
- <-runner.loggingDone
-
- return nil
}
var ErrNotInOutputDir = fmt.Errorf("Must point to path within the output directory")
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index a17dff3..8e6381c 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -909,7 +909,7 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
func (s *TestSuite) TestStopOnSignal(c *C) {
s.testStopContainer(c, func(cr *ContainerRunner) {
go func() {
- for !cr.cStarted {
+ for cr.ContainerID == "" {
time.Sleep(time.Millisecond)
}
cr.SigChan <- syscall.SIGINT
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list