[ARVADOS] created: 1230d8a106c5c62edcbb9fcf6d1b94585e5596b2

[Git user]

        at  1230d8a106c5c62edcbb9fcf6d1b94585e5596b2 (commit)


commit 1230d8a106c5c62edcbb9fcf6d1b94585e5596b2
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Wed Mar 15 19:21:10 2017 -0300

    10218: Logging node information (cpu, mem, disk) by storing command outputs on the log collection. Added relevant test.

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 3b3cdf1..88c93e5 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -97,6 +97,7 @@ type ContainerRunner struct {
 	ArvMountExit   chan error
 	finalState     string
 
+	infoLogger   io.WriteCloser
 	statLogger   io.WriteCloser
 	statReporter *crunchstat.Reporter
 	statInterval time.Duration
@@ -504,6 +505,51 @@ func (runner *ContainerRunner) StartCrunchstat() {
 	runner.statReporter.Start()
 }
 
+type infoCommand struct {
+	label   string
+	command string
+	args    []string
+}
+
+func newInfoCommand(label string, command string) infoCommand {
+	cmd := strings.Split(command, " ")
+	return infoCommand{
+		label:   label,
+		command: cmd[0],
+		args:    cmd[1:],
+	}
+}
+
+// Gather node information and store it on the log for debugging
+// purposes.
+func (runner *ContainerRunner) LogNodeInfo() (err error) {
+	w := runner.NewLogWriter("node-info")
+	logger := log.New(w, "node-info", 0)
+
+	commands := []infoCommand{
+		newInfoCommand("Host Information", "uname -a"),
+		newInfoCommand("CPU Information", "cat /proc/cpuinfo"),
+		newInfoCommand("Memory Information", "cat /proc/meminfo"),
+		newInfoCommand("Disk Space", "df -m"),
+	}
+
+	var out []byte
+	for _, command := range commands {
+		out, err = exec.Command(command.command, command.args...).Output()
+		if err != nil {
+			return fmt.Errorf("While running command '%s': %v",
+				command.command, err)
+		}
+		logger.Printf("%s:\n%s\n", command.label, out)
+	}
+
+	err = w.Close()
+	if err != nil {
+		return fmt.Errorf("While closing node-info logs: %v", err)
+	}
+	return nil
+}
+
 // AttachLogs connects the docker container stdout and stderr logs to the
 // Arvados logger which logs to Keep and the API server logs table.
 func (runner *ContainerRunner) AttachStreams() (err error) {
@@ -990,6 +1036,12 @@ func (runner *ContainerRunner) Run() (err error) {
 		return
 	}
 
+	// Gather and record node information
+	err = runner.LogNodeInfo()
+	if err != nil {
+		return
+	}
+
 	runner.StartCrunchstat()
 
 	if runner.IsCancelled() {
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 36ddd36..1286659 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -650,6 +650,32 @@ func (s *TestSuite) TestCrunchstat(c *C) {
 	c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
 }
 
+func (s *TestSuite) TestNodeInfo(c *C) {
+	api, _, _ := FullRunHelper(c, `{
+		"command": ["sleep", "1"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": ".",
+		"environment": {},
+		"mounts": {"/tmp": {"kind": "tmp"} },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`, nil, func(t *TestDockerClient) {
+		time.Sleep(time.Second)
+		t.logWriter.Close()
+		t.finish <- dockerclient.WaitResult{}
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+
+	c.Assert(api.Logs["node-info"], NotNil)
+	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
+	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
+	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
+	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
+}
+
 func (s *TestSuite) TestFullRunStderr(c *C) {
 	api, _, _ := FullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],

-----------------------------------------------------------------------


hooks/post-receive
--