[ARVADOS] updated: fc390927833d14b6c439db8ea72d3d52b60a5e6d
Git user
git at public.curoverse.com
Fri Dec 16 16:02:31 EST 2016
Summary of changes:
.../controllers/container_requests_controller.rb | 28 ++
.../controllers/pipeline_instances_controller.rb | 2 +-
.../application/_extra_tab_line_buttons.html.erb | 0
.../views/application/_title_and_buttons.html.erb | 3 +
.../_extra_tab_line_buttons.html.erb | 10 +
apps/workbench/config/application.default.yml | 5 +
apps/workbench/config/routes.rb | 1 +
.../container_requests_controller_test.rb | 27 ++
.../test/diagnostics/container_request_test.rb | 49 ++
apps/workbench/test/diagnostics/pipeline_test.rb | 45 +-
apps/workbench/test/diagnostics_test_helper.rb | 43 ++
sdk/cli/test/test_arv-keep-put.rb | 8 +-
sdk/cwl/tests/test_container.py | 12 +-
sdk/go/arvadosclient/arvadosclient.go | 43 +-
sdk/go/crunchrunner/crunchrunner.go | 20 -
sdk/go/keepclient/keepclient.go | 3 +-
sdk/python/arvados/arvfile.py | 8 +
sdk/python/arvados/collection.py | 13 +-
sdk/python/arvados/commands/put.py | 492 ++++++++++++++-------
sdk/python/arvados/util.py | 2 +
sdk/python/tests/test_arv_put.py | 155 ++++++-
sdk/python/tests/test_arvfile.py | 21 +
sdk/python/tests/test_collections.py | 2 +
services/api/app/models/arvados_model.rb | 7 +-
services/api/app/models/job.rb | 4 +
.../20161213172944_full_text_search_indexes.rb | 32 ++
services/api/db/structure.sql | 18 +-
services/crunch-run/crunchrun.go | 19 +
services/crunch-run/crunchrun_test.go | 34 +-
29 files changed, 827 insertions(+), 279 deletions(-)
copy tools/crunchstat-summary/tests/__init__.py => apps/workbench/app/views/application/_extra_tab_line_buttons.html.erb (100%)
create mode 100644 apps/workbench/app/views/container_requests/_extra_tab_line_buttons.html.erb
create mode 100644 apps/workbench/test/diagnostics/container_request_test.rb
create mode 100644 services/api/db/migrate/20161213172944_full_text_search_indexes.rb
discards dd16592d7def0c70d2d95547975425695af8a184 (commit)
via fc390927833d14b6c439db8ea72d3d52b60a5e6d (commit)
via ffd4738242c61fa5acd423f927339f836dfb0ffb (commit)
via a619c2ce3c6d5bc00b9898dd084ac0e2efc2ee76 (commit)
via 005cb811d937fe52cee11d76484252ed7167f9a8 (commit)
via 6a370a002d008dffaf9f47b7db3da47b40e57254 (commit)
via 28db15c830a8f129283f43682727b470862572d8 (commit)
via f987de07f96595851187ed2a4e0212fa0e10e999 (commit)
via 08e03489e053779f7bbc168677deeebd86d6249c (commit)
via cc1dbda8f559ab43f326c77595d4af87e8ca7a33 (commit)
via 738ecb0cd43e2ae728b04534697198d11d6bd85f (commit)
via 83a4e42fb15030a9108e76dcf01a1a7d6c14cda5 (commit)
via e54cce12bbda235dcaebf3b7565804cd8c296100 (commit)
via f2e2487ddba3944d3acd46fae3424a87fc624be9 (commit)
via dcca309415bc0683f57f38ad65c30b539d80e191 (commit)
via 409d130ddcc4b76def5fa8d61d2584725c61152a (commit)
via 55e9587e140e1621dcd374afcf30ee0aec2c6d24 (commit)
via aa94204c3e88f5386d14cb90e3e13a5e9caaf251 (commit)
via ae9c556899557f64acaa54b339a62c9f5c5966c9 (commit)
via d13cc16ebbe287fe81d06c5faa1d15fe588bcf16 (commit)
via ccc1201e103e168b04801b135b662577d7ea475b (commit)
via 190e6d0b21800dcaec1c5d0b2346d2c2bf43eeea (commit)
via b5af6347690934f0a0fba10b79b6d8dcf2c84243 (commit)
via 9f2fccd1c01823a762044c8a73e6fa0f7ed9086b (commit)
via 5646f9476220ef151d1811acb4eff88ebe8ef530 (commit)
via c86fec0e51f30b2771c9ae95152f76b438db6542 (commit)
via 2e104941dbf1e4bf92e0632cadeb946be0595d67 (commit)
via 8be35986ae62ce39691c4c6d491601012255b1ba (commit)
via 1c96d4eb740dc4285d9f30ba0ce73499f7b7d59e (commit)
via a62e41101a75e01f9b0dd7124eef81714443d8a1 (commit)
via a175a62538e4db86296e49e5412be41e026e73b6 (commit)
via 8303a85b6c3fed92710a590fa5653557b73439cf (commit)
via 90de410ff977f3669d33f532adf0cb71ac8ac261 (commit)
via 2225be2cb92613495450528ba24a9ca14f232748 (commit)
via 50128b53da4003912635b03fb27b5be2c5beaca1 (commit)
via 0d2d039368f65490e9479bd7500a1f8bdf30849c (commit)
via fd7db907627aac75dae62430b6f2fa948719a3af (commit)
via 53d8d85b53e445ffb8c16eed86145adee2ff9e37 (commit)
via 261b9d42a367704e4815f3d59565467b98853787 (commit)
via 5413abe433a089f11f7ea595073a6a20ffa371de (commit)
via bfabb2a8fd5fc3914035a7e2be6bd29d14fc7850 (commit)
via ec5ec8c10756217482930b0ad763ff174b7843fd (commit)
via 17b7582b6116a4891a27769ee06f9a604bbabdd7 (commit)
via a3d45597b1f4ea9443d35370ebfa7925fdd5ce90 (commit)
via 3885bc2e043ce123929acceaaa8dbbf20dbb8c12 (commit)
via 3dec7045b24acdd53dc054bddcfd4c7f77739f00 (commit)
via 845d6e6521b284a111534447919a6bc594573ee1 (commit)
via 391a1d8378b4bc6b17b71904b3d6494160b51627 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (dd16592d7def0c70d2d95547975425695af8a184)
\
N -- N -- N (fc390927833d14b6c439db8ea72d3d52b60a5e6d)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fc390927833d14b6c439db8ea72d3d52b60a5e6d
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Dec 16 16:02:24 2016 -0500
10585: Add crunchstat -signal-on-dead-ppid option.
diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go
index cae95fd..510df3b 100644
--- a/services/crunchstat/crunchstat.go
+++ b/services/crunchstat/crunchstat.go
@@ -16,6 +16,9 @@ import (
const MaxLogLine = 1 << 14 // Child stderr lines >16KiB will be split
+var signalOnDeadPPID int
+var ppidCheckInterval = time.Second
+
func main() {
reporter := crunchstat.Reporter{
Logger: log.New(os.Stderr, "crunchstat: ", 0),
@@ -24,6 +27,8 @@ func main() {
flag.StringVar(&reporter.CgroupRoot, "cgroup-root", "", "Root of cgroup tree")
flag.StringVar(&reporter.CgroupParent, "cgroup-parent", "", "Name of container parent under cgroup")
flag.StringVar(&reporter.CIDFile, "cgroup-cid", "", "Path to container id file")
+ flag.IntVar(&signalOnDeadPPID, "signal-on-dead-ppid", 15, "Signal to send child if crunchstat's parent process disappears")
+ flag.DurationVar(&ppidCheckInterval, "ppid-check-interval", ppidCheckInterval, "Time between checks for parent process disappearance")
pollMsec := flag.Int64("poll", 1000, "Reporting interval, in milliseconds")
flag.Parse()
@@ -77,6 +82,11 @@ func runCommand(argv []string, logger *log.Logger) error {
signal.Notify(sigChan, syscall.SIGTERM)
signal.Notify(sigChan, syscall.SIGINT)
+ // Kill our child proc if our parent process disappears
+ if signalOnDeadPPID != 0 {
+ go sendSignalOnDeadPPID(signalOnDeadPPID, os.Getppid(), cmd, logger)
+ }
+
// Funnel stderr through our channel
stderr_pipe, err := cmd.StderrPipe()
if err != nil {
@@ -97,6 +107,26 @@ func runCommand(argv []string, logger *log.Logger) error {
return cmd.Wait()
}
+func sendSignalOnDeadPPID(signum, ppidOrig int, cmd *exec.Cmd, logger *log.Logger) {
+ for _ = range time.NewTicker(ppidCheckInterval).C {
+ ppid := os.Getppid()
+ if ppid == ppidOrig {
+ continue
+ }
+ if cmd.Process == nil {
+ // Child process isn't running yet
+ continue
+ }
+ logger.Printf("notice: crunchstat ppid changed from %d to %d -- killing child pid %d with signal %d", ppidOrig, ppid, cmd.Process.Pid, signum)
+ err := cmd.Process.Signal(syscall.Signal(signum))
+ if err != nil {
+ logger.Printf("error: sending signal: %d", err)
+ continue
+ }
+ break
+ }
+}
+
func copyPipeToChildLog(in io.ReadCloser, logger *log.Logger) {
reader := bufio.NewReaderSize(in, MaxLogLine)
var prefix string
diff --git a/services/crunchstat/crunchstat_test.go b/services/crunchstat/crunchstat_test.go
index fe3b56d..759b3aa 100644
--- a/services/crunchstat/crunchstat_test.go
+++ b/services/crunchstat/crunchstat_test.go
@@ -3,9 +3,15 @@ package main
import (
"bufio"
"bytes"
+ "fmt"
"io"
+ "io/ioutil"
"log"
"math/rand"
+ "os"
+ "os/exec"
+ "sync"
+ "syscall"
"testing"
"time"
)
@@ -82,3 +88,147 @@ func bufLogger() (*log.Logger, *bufio.Reader) {
logger := log.New(w, "", 0)
return logger, bufio.NewReader(r)
}
+
+func TestSignalOnDeadPPID(t *testing.T) {
+ if !testDeadParent(t, 0) {
+ t.Fatal("child should still be alive after parent dies")
+ }
+ if testDeadParent(t, 15) {
+ t.Fatal("child should have been killed when parent died")
+ }
+}
+
+// testDeadParent returns true if crunchstat's child proc is still
+// alive after its parent dies.
+func testDeadParent(t *testing.T, signum int) bool {
+ var err error
+ var bin, childlockfile, parentlockfile *os.File
+ for _, f := range []**os.File{&bin, &childlockfile, &parentlockfile} {
+ *f, err = ioutil.TempFile("", "crunchstat_")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer (*f).Close()
+ defer os.Remove((*f).Name())
+ }
+
+ bin.Close()
+ err = exec.Command("go", "build", "-o", bin.Name()).Run()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = syscall.Flock(int(parentlockfile.Fd()), syscall.LOCK_EX)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cmd := exec.Command("bash", "-c", `
+set -e
+"$BINFILE" -cgroup-root=/none -ppid-check-interval=10ms -signal-on-dead-ppid="$SIGNUM" bash -c '
+ set -e
+ unlock() {
+ flock --unlock "$CHILDLOCKFD"
+ kill %1
+ }
+ trap unlock TERM
+ flock --exclusive "$CHILDLOCKFD"
+ echo -n "$$" > "$CHILDLOCKFILE"
+ flock --unlock "$PARENTLOCKFD"
+ sleep 20 </dev/null >/dev/null 2>/dev/null &
+ wait %1
+ unlock
+' &
+
+# wait for inner bash to start, to ensure $BINFILE has seen this bash proc as its initial PPID
+flock --exclusive "$PARENTLOCKFILE" true
+`)
+ cmd.Env = append(os.Environ(),
+ "SIGNUM="+fmt.Sprintf("%d", signum),
+ "PARENTLOCKFD=3",
+ "PARENTLOCKFILE="+parentlockfile.Name(),
+ "CHILDLOCKFD=4",
+ "CHILDLOCKFILE="+childlockfile.Name(),
+ "BINFILE="+bin.Name())
+ cmd.ExtraFiles = []*os.File{parentlockfile, childlockfile}
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd.Start()
+ defer cmd.Wait()
+
+ var wg sync.WaitGroup
+ wg.Add(2)
+ defer wg.Wait()
+ for _, rdr := range []io.ReadCloser{stderr, stdout} {
+ go func(rdr io.ReadCloser) {
+ defer wg.Done()
+ buf := make([]byte, 1024)
+ for {
+ n, err := rdr.Read(buf)
+ if n > 0 {
+ t.Logf("%s", buf[:n])
+ }
+ if err != nil {
+ return
+ }
+ }
+ }(rdr)
+ }
+
+ // Wait until inner bash process releases parentlockfile
+ // (which means it has locked childlockfile and written its
+ // PID)
+ err = exec.Command("flock", "--exclusive", parentlockfile.Name(), "true").Run()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ childDone := make(chan bool)
+ go func() {
+ // Notify the main thread when the inner bash process
+ // releases its lock on childlockfile (which means
+ // either its sleep process ended or it received a
+ // TERM signal).
+ t0 := time.Now()
+ err = exec.Command("flock", "--exclusive", childlockfile.Name(), "true").Run()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("child done after %s", time.Since(t0))
+ close(childDone)
+ }()
+
+ select {
+ case <-time.After(500 * time.Millisecond):
+ // Inner bash process is still alive after the timeout
+ // period. Kill it now, so our stdout and stderr pipes
+ // can finish and we don't leave a mess of child procs
+ // behind.
+ buf, err := ioutil.ReadFile(childlockfile.Name())
+ if err != nil {
+ t.Fatal(err)
+ }
+ var childPID int
+ _, err = fmt.Sscanf(string(buf), "%d", &childPID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ child, err := os.FindProcess(childPID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ child.Signal(syscall.Signal(15))
+ return true
+
+ case <-childDone:
+ // Inner bash process ended soon after its grandparent
+ // ended.
+ return false
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list