[ARVADOS] updated: c744c6cecd4f049ce5a2f6e922ca9f74b03fa3a3
git at public.curoverse.com
git at public.curoverse.com
Mon Oct 20 12:34:56 EDT 2014
Summary of changes:
services/crunchstat/crunchstat.go | 142 +++++++++++++++------------------
services/crunchstat/crunchstat_test.go | 33 +++++---
2 files changed, 88 insertions(+), 87 deletions(-)
via c744c6cecd4f049ce5a2f6e922ca9f74b03fa3a3 (commit)
via 06d0338d638ce74b51ca3cfe10523e5515269fdf (commit)
via ae670519e4075b75346d6152abedf87d41e2cea7 (commit)
via 85c6c9ee95ce15f6a12b987fe611e917f043e589 (commit)
from db0472215ce261406503d7336006bcd64c9feacd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c744c6cecd4f049ce5a2f6e922ca9f74b03fa3a3
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 20 12:18:48 2014 -0400
3826: Fix confusing reuse of variable name.
diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go
index 91027d7..728f1a4 100644
--- a/services/crunchstat/crunchstat.go
+++ b/services/crunchstat/crunchstat.go
@@ -242,12 +242,12 @@ func DoNetworkStats(cgroup Cgroup, lastSample map[string]IoSample) {
nextSample.txBytes = tx
nextSample.rxBytes = rx
var delta string
- if lastSample, ok := lastSample[ifName]; ok {
- interval := nextSample.sampleTime.Sub(lastSample.sampleTime).Seconds()
+ if prev, ok := lastSample[ifName]; ok {
+ interval := nextSample.sampleTime.Sub(prev.sampleTime).Seconds()
delta = fmt.Sprintf(" -- interval %.4f seconds %d tx %d rx",
interval,
- tx-lastSample.txBytes,
- rx-lastSample.rxBytes)
+ tx-prev.txBytes,
+ rx-prev.rxBytes)
}
LogPrintf("net:%s %d tx %d rx%s", ifName, tx, rx, delta)
lastSample[ifName] = nextSample
commit 06d0338d638ce74b51ca3cfe10523e5515269fdf
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 20 12:18:13 2014 -0400
3826: Use strconv instead of scanner + sscanf.
diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go
index 50dea18..91027d7 100644
--- a/services/crunchstat/crunchstat.go
+++ b/services/crunchstat/crunchstat.go
@@ -12,6 +12,7 @@ import (
"os"
"os/exec"
"os/signal"
+ "strconv"
"strings"
"syscall"
"time"
@@ -217,33 +218,25 @@ func DoNetworkStats(cgroup Cgroup, lastSample map[string]IoSample) {
}
scanner := bufio.NewScanner(stats)
-Iface:
for scanner.Scan() {
var ifName string
var rx, tx int64
- words := bufio.NewScanner(strings.NewReader(scanner.Text()))
- words.Split(bufio.ScanWords)
- wordIndex := 0
- for words.Scan() {
- word := words.Text()
- switch wordIndex {
- case 0:
- ifName = strings.TrimRight(word, ":")
- case 1:
- if _, err := fmt.Sscanf(word, "%d", &rx); err != nil {
- continue Iface
- }
- case 9:
- if _, err := fmt.Sscanf(word, "%d", &tx); err != nil {
- continue Iface
- }
- }
- wordIndex++
+ words := strings.Fields(scanner.Text())
+ if len(words) != 17 {
+ // Skip lines with wrong format
+ continue
}
- if ifName == "lo" || ifName == "" || wordIndex != 17 {
+ ifName = strings.TrimRight(words[0], ":")
+ if ifName == "lo" || ifName == "" {
// Skip loopback interface and lines with wrong format
continue
}
+ if tx, err = strconv.ParseInt(words[9], 10, 64); err != nil {
+ continue
+ }
+ if rx, err = strconv.ParseInt(words[1], 10, 64); err != nil {
+ continue
+ }
nextSample := IoSample{}
nextSample.sampleTime = sampleTime
nextSample.txBytes = tx
commit ae670519e4075b75346d6152abedf87d41e2cea7
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 20 12:10:12 2014 -0400
3826: Use global logChan. Use ioutil.ReadFile.
diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go
index e5f58a7..50dea18 100644
--- a/services/crunchstat/crunchstat.go
+++ b/services/crunchstat/crunchstat.go
@@ -47,22 +47,18 @@ func CopyChanToPipe(in <-chan string, out io.Writer) {
}
}
-func OpenAndReadAll(filename string, log_chan chan<- string) ([]byte, error) {
- in, err := os.Open(filename)
- if err != nil {
- if log_chan != nil {
- log_chan <- fmt.Sprintf("crunchstat: open %s: %s", filename, err)
- }
- return nil, err
+var logChan chan string
+func LogPrintf(format string, args ...interface{}) {
+ if logChan == nil {
+ return
}
- defer in.Close()
- return ReadAllOrWarn(in, log_chan)
+ logChan <- fmt.Sprintf("crunchstat: " + format, args...)
}
-func ReadAllOrWarn(in *os.File, log_chan chan<- string) ([]byte, error) {
+func ReadAllOrWarn(in *os.File) ([]byte, error) {
content, err := ioutil.ReadAll(in)
- if err != nil && log_chan != nil {
- log_chan <- fmt.Sprintf("crunchstat: read %s: %s", in.Name(), err)
+ if err != nil {
+ LogPrintf("read %s: %s", in.Name(), err)
}
return content, err
}
@@ -104,16 +100,16 @@ func OpenStatFile(cgroup Cgroup, statgroup string, stat string) (*os.File, error
// [b] after all contained processes have exited.
reportedStatFile[stat] = path
if path == "" {
- stderr <- fmt.Sprintf("crunchstat: did not find stats file: stat %s, statgroup %s, cid %s, parent %s, root %s", stat, statgroup, cgroup.cid, cgroup.parent, cgroup.root)
+ LogPrintf("did not find stats file: stat %s, statgroup %s, cid %s, parent %s, root %s", stat, statgroup, cgroup.cid, cgroup.parent, cgroup.root)
} else {
- stderr <- fmt.Sprintf("crunchstat: reading stats from %s", path)
+ LogPrintf("reading stats from %s", path)
}
}
return file, err
}
-func GetContainerNetStats(stderr chan<- string, cgroup Cgroup) (io.Reader, error) {
- procsFile, err := OpenStatFile(stderr, cgroup, "cpuacct", "cgroup.procs")
+func GetContainerNetStats(cgroup Cgroup) (io.Reader, error) {
+ procsFile, err := OpenStatFile(cgroup, "cpuacct", "cgroup.procs")
if err != nil {
return nil, err
}
@@ -122,8 +118,9 @@ func GetContainerNetStats(stderr chan<- string, cgroup Cgroup) (io.Reader, error
for reader.Scan() {
taskPid := reader.Text()
statsFilename := fmt.Sprintf("/proc/%s/net/dev", taskPid)
- stats, err := OpenAndReadAll(statsFilename, stderr)
+ stats, err := ioutil.ReadFile(statsFilename)
if err != nil {
+ LogPrintf("read %s: %s", statsFilename, err)
continue
}
return strings.NewReader(string(stats)), nil
@@ -137,8 +134,8 @@ type IoSample struct {
rxBytes int64
}
-func DoBlkIoStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoSample) {
- c, err := OpenStatFile(stderr, cgroup, "blkio", "blkio.io_service_bytes")
+func DoBlkIoStats(cgroup Cgroup, lastSample map[string]IoSample) {
+ c, err := OpenStatFile(cgroup, "blkio", "blkio.io_service_bytes")
if err != nil {
return
}
@@ -176,7 +173,7 @@ func DoBlkIoStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoS
sample.txBytes-prev.txBytes,
sample.rxBytes-prev.rxBytes)
}
- stderr <- fmt.Sprintf("crunchstat: blkio:%s %d write %d read%s", dev, sample.txBytes, sample.rxBytes, delta)
+ LogPrintf("blkio:%s %d write %d read%s", dev, sample.txBytes, sample.rxBytes, delta)
lastSample[dev] = sample
}
}
@@ -186,8 +183,8 @@ type MemSample struct {
memStat map[string]int64
}
-func DoMemoryStats(stderr chan<- string, cgroup Cgroup) {
- c, err := OpenStatFile(stderr, cgroup, "memory", "memory.stat")
+func DoMemoryStats(cgroup Cgroup) {
+ c, err := OpenStatFile(cgroup, "memory", "memory.stat")
if err != nil {
return
}
@@ -209,12 +206,12 @@ func DoMemoryStats(stderr chan<- string, cgroup Cgroup) {
outstat.WriteString(fmt.Sprintf(" %d %s", val, key))
}
}
- stderr <- fmt.Sprintf("crunchstat: mem%s", outstat.String())
+ LogPrintf("mem%s", outstat.String())
}
-func DoNetworkStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoSample) {
+func DoNetworkStats(cgroup Cgroup, lastSample map[string]IoSample) {
sampleTime := time.Now()
- stats, err := GetContainerNetStats(stderr, cgroup)
+ stats, err := GetContainerNetStats(cgroup)
if err != nil {
return
}
@@ -259,8 +256,7 @@ Iface:
tx-lastSample.txBytes,
rx-lastSample.rxBytes)
}
- stderr <- fmt.Sprintf("crunchstat: net:%s %d tx %d rx%s",
- ifName, tx, rx, delta)
+ LogPrintf("net:%s %d tx %d rx%s", ifName, tx, rx, delta)
lastSample[ifName] = nextSample
}
}
@@ -275,13 +271,13 @@ type CpuSample struct {
// Return the number of CPUs available in the container. Return 0 if
// we can't figure out the real number of CPUs.
-func GetCpuCount(stderr chan<- string, cgroup Cgroup) int64 {
- cpusetFile, err := OpenStatFile(stderr, cgroup, "cpuset", "cpuset.cpus")
+func GetCpuCount(cgroup Cgroup) int64 {
+ cpusetFile, err := OpenStatFile(cgroup, "cpuset", "cpuset.cpus")
if err != nil {
return 0
}
defer cpusetFile.Close()
- b, err := ReadAllOrWarn(cpusetFile, stderr)
+ b, err := ReadAllOrWarn(cpusetFile)
sp := strings.Split(string(b), ",")
cpus := int64(0)
for _, v := range sp {
@@ -296,18 +292,18 @@ func GetCpuCount(stderr chan<- string, cgroup Cgroup) int64 {
return cpus
}
-func DoCpuStats(stderr chan<- string, cgroup Cgroup, lastSample *CpuSample) {
- statFile, err := OpenStatFile(stderr, cgroup, "cpuacct", "cpuacct.stat")
+func DoCpuStats(cgroup Cgroup, lastSample *CpuSample) {
+ statFile, err := OpenStatFile(cgroup, "cpuacct", "cpuacct.stat")
if err != nil {
return
}
defer statFile.Close()
- b, err := ReadAllOrWarn(statFile, stderr)
+ b, err := ReadAllOrWarn(statFile)
if err != nil {
return
}
- nextSample := CpuSample{true, time.Now(), 0, 0, GetCpuCount(stderr, cgroup)}
+ nextSample := CpuSample{true, time.Now(), 0, 0, GetCpuCount(cgroup)}
var userTicks, sysTicks int64
fmt.Sscanf(string(b), "user %d\nsystem %d", &userTicks, &sysTicks)
user_hz := float64(C.sysconf(C._SC_CLK_TCK))
@@ -321,12 +317,12 @@ func DoCpuStats(stderr chan<- string, cgroup Cgroup, lastSample *CpuSample) {
nextSample.user-lastSample.user,
nextSample.sys-lastSample.sys)
}
- stderr <- fmt.Sprintf("crunchstat: cpu %.4f user %.4f sys %d cpus%s",
+ LogPrintf("cpu %.4f user %.4f sys %d cpus%s",
nextSample.user, nextSample.sys, nextSample.cpus, delta)
*lastSample = nextSample
}
-func PollCgroupStats(cgroup Cgroup, stderr chan string, poll int64, stop_poll_chan <-chan bool) {
+func PollCgroupStats(cgroup Cgroup, poll int64, stop_poll_chan <-chan bool) {
var lastNetSample = map[string]IoSample{}
var lastDiskSample = map[string]IoSample{}
var lastCpuSample = CpuSample{}
@@ -347,10 +343,10 @@ func PollCgroupStats(cgroup Cgroup, stderr chan string, poll int64, stop_poll_ch
case <-poll_chan:
// Emit stats, then select again.
}
- DoMemoryStats(stderr, cgroup)
- DoCpuStats(stderr, cgroup, &lastCpuSample)
- DoBlkIoStats(stderr, cgroup, lastDiskSample)
- DoNetworkStats(stderr, cgroup, lastNetSample)
+ DoMemoryStats(cgroup)
+ DoCpuStats(cgroup, &lastCpuSample)
+ DoBlkIoStats(cgroup, lastDiskSample)
+ DoNetworkStats(cgroup, lastNetSample)
}
}
@@ -376,12 +372,12 @@ func run(logger *log.Logger) error {
logger.Fatal("Must provide -cgroup-root")
}
- stderr_chan := make(chan string, 1)
- defer close(stderr_chan)
+ logChan = make(chan string, 1)
+ defer close(logChan)
finish_chan := make(chan bool)
defer close(finish_chan)
- go CopyChanToPipe(stderr_chan, os.Stderr)
+ go CopyChanToPipe(logChan, os.Stderr)
var cmd *exec.Cmd
@@ -413,7 +409,7 @@ func run(logger *log.Logger) error {
if err != nil {
logger.Fatal(err)
}
- go CopyPipeToChan(stderr_pipe, stderr_chan, finish_chan)
+ go CopyPipeToChan(stderr_pipe, logChan, finish_chan)
// Run subprocess
if err := cmd.Start(); err != nil {
@@ -432,7 +428,7 @@ func run(logger *log.Logger) error {
ok := false
var i time.Duration
for i = 0; i < time.Duration(wait)*time.Second; i += (100 * time.Millisecond) {
- cid, err := OpenAndReadAll(cgroup_cidfile, nil)
+ cid, err := ioutil.ReadFile(cgroup_cidfile)
if err == nil && len(cid) > 0 {
ok = true
container_id = string(cid)
@@ -447,7 +443,7 @@ func run(logger *log.Logger) error {
stop_poll_chan := make(chan bool, 1)
cgroup := Cgroup{cgroup_root, cgroup_parent, container_id}
- go PollCgroupStats(cgroup, stderr_chan, poll, stop_poll_chan)
+ go PollCgroupStats(cgroup, poll, stop_poll_chan)
// When the child exits, tell the polling goroutine to stop.
defer func() { stop_poll_chan <- true }()
diff --git a/services/crunchstat/crunchstat_test.go b/services/crunchstat/crunchstat_test.go
index 00cd4d8..48988a1 100644
--- a/services/crunchstat/crunchstat_test.go
+++ b/services/crunchstat/crunchstat_test.go
@@ -1,31 +1,42 @@
package main
import (
+ "os"
"regexp"
"testing"
)
-func TestOpenAndReadAllFail(t *testing.T) {
- log_chan := make(chan string)
+func TestReadAllOrWarnFail(t *testing.T) {
+ logChan = make(chan string)
go func() {
- defer close(log_chan)
- if x, err := OpenAndReadAll("/nonexistent/file", log_chan); err == nil {
+ defer close(logChan)
+ // The special file /proc/self/mem can be opened for
+ // reading, but reading from byte 0 returns an error.
+ f, err := os.Open("/proc/self/mem")
+ if err != nil {
+ t.Fatalf("Opening /proc/self/mem: %s", err)
+ }
+ if x, err := ReadAllOrWarn(f); err == nil {
t.Fatalf("Expected error, got %v", x)
}
}()
- if _, ok := <-log_chan; !ok {
+ if _, ok := <-logChan; !ok {
t.Fatalf("Expected error message about nonexistent file")
}
- if msg, ok := <-log_chan; ok {
+ if msg, ok := <-logChan; ok {
t.Fatalf("Expected channel to close, got %s", msg)
}
}
-func TestOpenAndReadAllSuccess(t *testing.T) {
- log_chan := make(chan string)
+func TestReadAllOrWarnSuccess(t *testing.T) {
+ logChan = make(chan string)
go func() {
- defer close(log_chan)
- data, err := OpenAndReadAll("./crunchstat_test.go", log_chan)
+ defer close(logChan)
+ f, err := os.Open("./crunchstat_test.go")
+ if err != nil {
+ t.Fatalf("Opening ./crunchstat_test.go: %s", err)
+ }
+ data, err := ReadAllOrWarn(f)
if err != nil {
t.Fatalf("got error %s", err)
}
@@ -33,7 +44,7 @@ func TestOpenAndReadAllSuccess(t *testing.T) {
t.Fatalf("data failed regexp: %s", err)
}
}()
- if msg, ok := <-log_chan; ok {
+ if msg, ok := <-logChan; ok {
t.Fatalf("Expected channel to close, got %s", msg)
}
}
commit 85c6c9ee95ce15f6a12b987fe611e917f043e589
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 20 11:57:27 2014 -0400
3826: Clean up slice initialization.
diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go
index 1531ec3..e5f58a7 100644
--- a/services/crunchstat/crunchstat.go
+++ b/services/crunchstat/crunchstat.go
@@ -77,12 +77,13 @@ var reportedStatFile = map[string]string{}
// container, and read /proc/PID/cgroup to determine the appropriate
// cgroup root for the given statgroup. (This will avoid falling back
// to host-level stats during container setup and teardown.)
-func OpenStatFile(stderr chan<- string, cgroup Cgroup, statgroup string, stat string) (*os.File, error) {
- var paths = []string{}
- paths = append(paths, fmt.Sprintf("%s/%s/%s/%s/%s", cgroup.root, statgroup, cgroup.parent, cgroup.cid, stat))
- paths = append(paths, fmt.Sprintf("%s/%s/%s/%s", cgroup.root, cgroup.parent, cgroup.cid, stat))
- paths = append(paths, fmt.Sprintf("%s/%s/%s", cgroup.root, statgroup, stat))
- paths = append(paths, fmt.Sprintf("%s/%s", cgroup.root, stat))
+func OpenStatFile(cgroup Cgroup, statgroup string, stat string) (*os.File, error) {
+ var paths = []string{
+ fmt.Sprintf("%s/%s/%s/%s/%s", cgroup.root, statgroup, cgroup.parent, cgroup.cid, stat),
+ fmt.Sprintf("%s/%s/%s/%s", cgroup.root, cgroup.parent, cgroup.cid, stat),
+ fmt.Sprintf("%s/%s/%s", cgroup.root, statgroup, stat),
+ fmt.Sprintf("%s/%s", cgroup.root, stat),
+ }
var path string
var file *os.File
var err error
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list