[ARVADOS] updated: a56f0f66cdff32b44b222eed178f64e6483e518e
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 16 13:28:32 EST 2016
Summary of changes:
services/crunch-run/crunchrun.go | 26 ++++++++++++++++++--------
services/crunch-run/crunchrun_test.go | 8 ++++----
services/crunch-run/logging.go | 4 ++++
services/crunch-run/logging_test.go | 1 +
services/crunch-run/upload_test.go | 11 +++++++++--
5 files changed, 36 insertions(+), 14 deletions(-)
via a56f0f66cdff32b44b222eed178f64e6483e518e (commit)
from 090e546b14e13798d15ac61728da05a0d9fb1317 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a56f0f66cdff32b44b222eed178f64e6483e518e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Feb 16 13:28:20 2016 -0500
8015: Fix tests again for endianness of Docker logs, added --read-write to arv-mount command line.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 2dbe986..64f0d77 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -243,7 +243,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
pdhOnly := true
tmpcount := 0
- arvMountCmd := []string{"--foreground", "--allow-other"}
+ arvMountCmd := []string{"--foreground", "--allow-other", "--read-write"}
collectionPaths := []string{}
runner.Binds = nil
@@ -289,7 +289,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if staterr != nil {
return fmt.Errorf("While Stat on temp dir: %v", staterr)
}
- err = os.Chmod(runner.HostOutputDir, st.Mode()|os.ModeSetgid)
+ err = os.Chmod(runner.HostOutputDir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
return fmt.Errorf("While Chmod temp dir: %v", err)
}
@@ -338,7 +338,7 @@ func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) {
_, readerr := io.ReadAtLeast(containerReader, header, 8)
if readerr == nil {
- readsize := int64(header[4]) | (int64(header[5]) << 8) | (int64(header[6]) << 16) | (int64(header[7]) << 24)
+ readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
if header[0] == 1 {
// stdout
_, readerr = io.CopyN(runner.Stdout, containerReader, readsize)
@@ -600,6 +600,13 @@ func (runner *ContainerRunner) NewArvLogWriter(name string) io.WriteCloser {
func (runner *ContainerRunner) Run() (err error) {
runner.CrunchLog.Printf("Executing container '%s'", runner.ContainerRecord.UUID)
+ hostname, hosterr := os.Hostname()
+ if hosterr != nil {
+ runner.CrunchLog.Printf("Error getting hostname '%v'", hosterr)
+ } else {
+ runner.CrunchLog.Printf("Executing on host '%s'", runner.ContainerRecord.UUID, hostname)
+ }
+
var runerr, waiterr error
defer func() {
@@ -706,36 +713,39 @@ func NewContainerRunner(api IArvadosClient,
cr.LogCollection = &CollectionWriter{kc, nil, sync.Mutex{}}
cr.ContainerRecord.UUID = containerUUID
cr.CrunchLog = NewThrottledLogger(cr.NewLogWriter("crunch-run"))
+ cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
return cr
}
func main() {
flag.Parse()
+ containerId := flag.Arg(0)
+
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Fatalf("%s: %v", flag.Arg(0), err)
+ log.Fatalf("%s: %v", containerId, err)
}
api.Retries = 8
var kc *keepclient.KeepClient
kc, err = keepclient.MakeKeepClient(&api)
if err != nil {
- log.Fatalf("%s: %v", flag.Arg(0), err)
+ log.Fatalf("%s: %v", containerId, err)
}
kc.Retries = 4
var docker *dockerclient.DockerClient
docker, err = dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
if err != nil {
- log.Fatalf("%s: %v", flag.Arg(0), err)
+ log.Fatalf("%s: %v", containerId, err)
}
- cr := NewContainerRunner(api, kc, docker, flag.Arg(0))
+ cr := NewContainerRunner(api, kc, docker, containerId)
err = cr.Run()
if err != nil {
- log.Fatalf("%s: %v", flag.Arg(0), err)
+ log.Fatalf("%s: %v", containerId, err)
}
}
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 6569df1..bf75e7f 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -355,7 +355,7 @@ func dockerLog(fd byte, msg string) []byte {
by := []byte(msg)
header := make([]byte, 8+len(by))
header[0] = fd
- header[4] = byte(len(by))
+ header[7] = byte(len(by))
copy(header[8:], by)
return header
}
@@ -681,7 +681,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
err := cr.SetupMounts()
c.Check(err, IsNil)
- c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir2:/tmp"})
cr.CleanupDirs()
}
@@ -696,7 +696,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
err := cr.SetupMounts()
c.Check(err, IsNil)
- c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir1/tmp0:/keeptmp"})
cr.CleanupDirs()
}
@@ -713,7 +713,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
err := cr.SetupMounts()
c.Check(err, IsNil)
- c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
+ c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "/tmp/mktmpdir1"})
var ss sort.StringSlice = cr.Binds
ss.Sort()
c.Check(cr.Binds, DeepEquals, []string{"/tmp/mktmpdir1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
diff --git a/services/crunch-run/logging.go b/services/crunch-run/logging.go
index 09b328a..20928db 100644
--- a/services/crunch-run/logging.go
+++ b/services/crunch-run/logging.go
@@ -35,6 +35,7 @@ type ThrottledLogger struct {
stop bool
flusherDone chan bool
Timestamper
+ Immediate *log.Logger
}
// RFC3339Fixed is a fixed-width version of RFC3339 with microsecond precision,
@@ -59,6 +60,9 @@ func (tl *ThrottledLogger) Write(p []byte) (n int, err error) {
sc := bufio.NewScanner(bytes.NewBuffer(p))
for sc.Scan() {
_, err = fmt.Fprintf(tl.buf, "%s %s\n", now, sc.Text())
+ if tl.Immediate != nil {
+ tl.Immediate.Printf("%s %s\n", now, sc.Text())
+ }
}
return len(p), err
}
diff --git a/services/crunch-run/logging_test.go b/services/crunch-run/logging_test.go
index 9b07d32..bce324d 100644
--- a/services/crunch-run/logging_test.go
+++ b/services/crunch-run/logging_test.go
@@ -50,6 +50,7 @@ func (s *LoggingTestSuite) TestWriteLogsLarge(c *C) {
kc := &KeepTestClient{}
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
+ cr.CrunchLog.Immediate = nil
for i := 0; i < 2000000; i += 1 {
cr.CrunchLog.Printf("Hello %d", i)
diff --git a/services/crunch-run/upload_test.go b/services/crunch-run/upload_test.go
index d66d9de..b4b1efd 100644
--- a/services/crunch-run/upload_test.go
+++ b/services/crunch-run/upload_test.go
@@ -58,9 +58,16 @@ func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
str, err := cw.WriteTree(tmpdir, log.New(os.Stdout, "", 0))
c.Check(err, IsNil)
- c.Check(str, Equals, `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
+
+ // streams can get added in either order because of scheduling
+ // of goroutines.
+ if str != `. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
-`)
+` && str != `./subdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:file2.txt
+. acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:file1.txt
+` {
+ c.Error("Did not get expected manifest text")
+ }
}
func (s *TestSuite) TestSimpleUploadLarge(c *C) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list