[ARVADOS] updated: a31038ebef04b7c8473222d90586ff8ae0a4a904

Git user git at public.curoverse.com
Sat Jan 28 20:15:35 EST 2017


Summary of changes:
 services/boot/arvados_go.go              | 56 +++++++++++++++++++++
 services/boot/booter.go                  |  2 +-
 services/boot/config.go                  | 85 ++++++++++++++------------------
 services/boot/consul.go                  |  6 +--
 services/boot/controller.go              | 18 ++++++-
 services/boot/main.go                    | 41 +++++++++++++++
 services/boot/os_package.go              |  5 +-
 services/boot/runit.go                   | 55 +++++++++++++++++++++
 services/boot/supervisor.go              | 47 ++++++++++++++++++
 services/boot/testimage_runit/Dockerfile |  2 +-
 services/boot/{server.go => webgui.go}   | 42 +++-------------
 11 files changed, 268 insertions(+), 91 deletions(-)
 create mode 100644 services/boot/arvados_go.go
 create mode 100644 services/boot/main.go
 create mode 100644 services/boot/runit.go
 rename services/boot/{server.go => webgui.go} (69%)

  discards  fa3bb426d9a1b70ecb92cf814596437531216807 (commit)
       via  a31038ebef04b7c8473222d90586ff8ae0a4a904 (commit)
       via  0dda0005f658935c3f7f08dba9633be2b55ba31e (commit)
       via  42ae3ec76c4b3c9d87af09e9dff5b2b9838f6954 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (fa3bb426d9a1b70ecb92cf814596437531216807)
            \
             N -- N -- N (a31038ebef04b7c8473222d90586ff8ae0a4a904)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit a31038ebef04b7c8473222d90586ff8ae0a4a904
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Jan 28 20:14:44 2017 -0500

    install+start arvados services (wip)

diff --git a/services/boot/arvados_go.go b/services/boot/arvados_go.go
new file mode 100644
index 0000000..b95f7c0
--- /dev/null
+++ b/services/boot/arvados_go.go
@@ -0,0 +1,56 @@
+package main
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+)
+
+var (
+	dispatchLocal = &arvadosGoBooter{name: "crunch-dispatch-local"}
+	dispatchSLURM = &arvadosGoBooter{name: "crunch-dispatch-slurm"}
+	gitHTTP       = &arvadosGoBooter{name: "arvados-git-httpd"}
+	keepbalance   = &arvadosGoBooter{name: "keep-balance"}
+	keepproxy     = &arvadosGoBooter{name: "keepproxy"}
+	keepstore     = &arvadosGoBooter{name: "keepstore"}
+	websocket     = &arvadosGoBooter{name: "arvados-ws"}
+)
+
+type arvadosGoBooter struct {
+	name string
+}
+
+func availablePort() int {
+	return rand.Intn(10000) + 20000
+}
+
+func (agb *arvadosGoBooter) Boot(ctx context.Context) error {
+	cfg := cfg(ctx)
+	cmd := path.Join(cfg.UsrDir, "bin", agb.name)
+	if _, err := os.Stat(cmd); err != nil {
+		if found, err := filepath.Glob(path.Join(cfg.UsrDir, "pkg", agb.name+"_*.deb")); err == nil && len(found) > 0 {
+			cmd := exec.Command("dpkg", "-i", found[0])
+			cmd.Stdout = os.Stderr
+			cmd.Stderr = os.Stderr
+			osPackageMutex.Lock()
+			err = cmd.Run()
+			osPackageMutex.Unlock()
+			if err != nil {
+				log.Print(err)
+			}
+		}
+	}
+	return Series{
+		&osPackage{
+			Debian: agb.name,
+		},
+		&supervisedService{
+			cmd:  path.Join(cfg.UsrDir, "bin", "consul-template"),
+			args: []string{"blah"},
+		},
+	}.Boot(ctx)
+}
diff --git a/services/boot/booter.go b/services/boot/booter.go
index cb672ab..3ca800d 100644
--- a/services/boot/booter.go
+++ b/services/boot/booter.go
@@ -75,7 +75,7 @@ func NewMultipleError(errs []error) error {
 		return errors[0]
 	}
 	return &MultipleError{
-		error:  fmt.Errorf("%d errors", len(errors)),
+		error:  fmt.Errorf("%d errors %q", len(errors), errors),
 		errors: errors,
 	}
 }
diff --git a/services/boot/consul.go b/services/boot/consul.go
index 8761b39..a536d1a 100644
--- a/services/boot/consul.go
+++ b/services/boot/consul.go
@@ -20,6 +20,9 @@ func (cb *consulBooter) Boot(ctx context.Context) error {
 	cb.Lock()
 	defer cb.Unlock()
 
+	if cb.check(ctx) == nil {
+		return nil
+	}
 	cfg := cfg(ctx)
 	bin := cfg.UsrDir + "/bin/consul"
 	err := (&download{
@@ -31,9 +34,6 @@ func (cb *consulBooter) Boot(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	if cb.check(ctx) == nil {
-		return nil
-	}
 	dataDir := cfg.DataDir + "/consul"
 	if err := os.MkdirAll(dataDir, 0700); err != nil {
 		return err
diff --git a/services/boot/controller.go b/services/boot/controller.go
index 526a627..cc996bf 100644
--- a/services/boot/controller.go
+++ b/services/boot/controller.go
@@ -2,18 +2,34 @@ package main
 
 import (
 	"context"
+	"path"
 )
 
 type controller struct{}
 
 func (c *controller) Boot(ctx context.Context) error {
+	cfg := cfg(ctx)
 	return Series{
 		Concurrent{
-			cfg(ctx),
+			cfg,
 			installCerts,
 		},
 		Concurrent{
 			consul,
+			&download{
+				URL:  "https://releases.hashicorp.com/consul-template/0.18.0/consul-template_0.18.0_linux_amd64.zip",
+				Dest: path.Join(cfg.UsrDir, "bin", "consul-template"),
+				Mode: 0755,
+			},
+		},
+		Concurrent{
+			dispatchLocal,
+			dispatchSLURM,
+			gitHTTP,
+			keepbalance,
+			keepproxy,
+			keepstore,
+			websocket,
 		},
 	}.Boot(ctx)
 }
diff --git a/services/boot/os_package.go b/services/boot/os_package.go
index fc82687..db953a1 100644
--- a/services/boot/os_package.go
+++ b/services/boot/os_package.go
@@ -13,6 +13,9 @@ var (
 	installCerts = &osPackage{
 		Debian: "ca-certificates",
 	}
+	installNginx = &osPackage{
+		Debian: "nginx",
+	}
 	installRunit = &osPackage{
 		Debian: "runit",
 	}
diff --git a/services/boot/supervisor.go b/services/boot/supervisor.go
index d095c23..2fcb336 100644
--- a/services/boot/supervisor.go
+++ b/services/boot/supervisor.go
@@ -2,7 +2,10 @@ package main
 
 import (
 	"context"
+	"log"
 	"os"
+
+	"github.com/hashicorp/consul/api"
 )
 
 type supervisor interface {
@@ -25,3 +28,47 @@ func newSupervisor(ctx context.Context, name, cmd string, args ...string) superv
 	}
 }
 
+type supervisedService struct {
+	name string
+	cmd  string
+	args []string
+}
+
+func (s *supervisedService) Boot(ctx context.Context) error {
+	bin := s.cmd
+	if bin == "" {
+		bin = s.name
+	}
+	if _, err := os.Stat(bin); err != nil {
+		return err
+	}
+	sup := newSupervisor(ctx, s.name, bin)
+	if ok, err := sup.Running(ctx); err != nil {
+		return err
+	} else if !ok {
+		if err := sup.Start(ctx); err != nil {
+			return err
+		}
+	}
+	if err := consul.Boot(ctx); err != nil {
+		return err
+	}
+	consul, err := api.NewClient(consulCfg)
+	if err != nil {
+		return err
+	}
+	agent := consul.Agent()
+	svcs, err := agent.Services()
+	if err != nil {
+		return err
+	}
+	if svc, ok := svcs[s.name]; ok {
+		log.Printf("%s is registered: %#v", s.name, svc)
+		return nil
+	}
+	return agent.ServiceRegister(&api.AgentServiceRegistration{
+		ID:   s.name,
+		Name: s.name,
+		Port: availablePort(),
+	})
+}
diff --git a/services/boot/testimage_runit/Dockerfile b/services/boot/testimage_runit/Dockerfile
index 9e363fc..b65a158 100644
--- a/services/boot/testimage_runit/Dockerfile
+++ b/services/boot/testimage_runit/Dockerfile
@@ -5,6 +5,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends ru
 RUN mkdir /etc/sv/arvados-boot && ln -s /usr/bin/arvados-boot /etc/sv/arvados-boot/run
 
 # preload (but don't install) packages arvados-boot might decide to install
-RUN DEBIAN_FRONTEND=noninteractive apt-get -dy install ca-certificates
+RUN DEBIAN_FRONTEND=noninteractive apt-get -dy install --no-install-recommends ca-certificates nginx
 
 CMD ["sh", "-c", "runsvdir /etc/sv"]

commit 0dda0005f658935c3f7f08dba9633be2b55ba31e
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Jan 28 18:43:36 2017 -0500

    split webgui,main from server.go

diff --git a/services/boot/config.go b/services/boot/config.go
index 2bbdb6e..2eeac01 100644
--- a/services/boot/config.go
+++ b/services/boot/config.go
@@ -15,27 +15,27 @@ type Config struct {
 	// in production. System functions only when a majority are
 	// alive.
 	ControlHosts []string
+	Ports        portsConfig
+	WebGUI       webguiConfig
+	DataDir      string
+	UsrDir       string
+	RunitSvDir   string
+}
 
-	ConsulPorts struct {
-		DNS     int
-		HTTP    int
-		HTTPS   int
-		RPC     int
-		SerfLAN int `json:"Serf_LAN"`
-		SerfWAN int `json:"Serf_WAN"`
-		Server  int
-	}
-
-	WebGUI struct {
-		// addr:port to serve web-based setup/monitoring
-		// application
-		Listen string
-	}
-
-	DataDir string
-	UsrDir  string
+type portsConfig struct {
+	ConsulDNS     int
+	ConsulHTTP    int
+	ConsulHTTPS   int
+	ConsulRPC     int
+	ConsulSerfLAN int `json:"Serf_LAN"`
+	ConsulSerfWAN int `json:"Serf_WAN"`
+	ConsulServer  int
+}
 
-	RunitSvDir string
+type webguiConfig struct {
+	// addr:port to serve web-based setup/monitoring
+	// application
+	Listen string
 }
 
 func (c *Config) Boot(ctx context.Context) error {
@@ -52,34 +52,23 @@ func (c *Config) Boot(ctx context.Context) error {
 	return nil
 }
 
-func (c *Config) SetDefaults() {
-	if len(c.ControlHosts) == 0 {
-		c.ControlHosts = []string{"127.0.0.1"}
-	}
-	defaultPort := []int{18600, 18500, -1, 18400, 18301, 18302, 18300}
-	for i, port := range []*int{
-		&c.ConsulPorts.DNS,
-		&c.ConsulPorts.HTTP,
-		&c.ConsulPorts.HTTPS,
-		&c.ConsulPorts.RPC,
-		&c.ConsulPorts.SerfLAN,
-		&c.ConsulPorts.SerfWAN,
-		&c.ConsulPorts.Server,
-	} {
-		if *port == 0 {
-			*port = defaultPort[i]
-		}
-	}
-	if c.DataDir == "" {
-		c.DataDir = "/var/lib/arvados"
-	}
-	if c.UsrDir == "" {
-		c.UsrDir = "/usr/local/arvados"
-	}
-	if c.RunitSvDir == "" {
-		c.RunitSvDir = "/etc/sv"
-	}
-	if c.WebGUI.Listen == "" {
-		c.WebGUI.Listen = "localhost:18000"
+func DefaultConfig() *Config {
+	return &Config{
+		ControlHosts: []string{"127.0.0.1"},
+		Ports: portsConfig{
+			ConsulDNS:     18600,
+			ConsulHTTP:    18500,
+			ConsulHTTPS:   -1,
+			ConsulRPC:     18400,
+			ConsulSerfLAN: 18301,
+			ConsulSerfWAN: 18302,
+			ConsulServer:  18300,
+		},
+		DataDir:    "/var/lib/arvados",
+		UsrDir:     "/usr/local/arvados",
+		RunitSvDir: "/etc/sv",
+		WebGUI: webguiConfig{
+			Listen: "127.0.0.1:18000",
+		},
 	}
 }
diff --git a/services/boot/main.go b/services/boot/main.go
index ebc2b99..86846b3 100644
--- a/services/boot/main.go
+++ b/services/boot/main.go
@@ -2,12 +2,9 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"flag"
 	"log"
-	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"git.curoverse.com/arvados.git/sdk/go/config"
@@ -19,22 +16,18 @@ func main() {
 	cfgPath := flag.String("config", defaultCfgPath, "`path` to config file")
 	flag.Parse()
 
-	var cfg Config
-	if err := config.LoadFile(&cfg, *cfgPath); os.IsNotExist(err) && *cfgPath == defaultCfgPath {
+	cfg := DefaultConfig()
+	if err := config.LoadFile(cfg, *cfgPath); os.IsNotExist(err) && *cfgPath == defaultCfgPath {
 		log.Printf("WARNING: No config file specified or found, starting fresh!")
 	} else if err != nil {
 		log.Fatal(err)
 	}
-	cfg.SetDefaults()
-	go func() {
-		log.Printf("starting server at %s", cfg.WebGUI.Listen)
-		log.Fatal(http.ListenAndServe(cfg.WebGUI.Listen, stack(cfg.logger, cfg.apiOrAssets)))
-	}()
+	go runWebGUI(cfg)
 	go func() {
 		var ctl Booter = &controller{}
 		ticker := time.NewTicker(5 * time.Second)
 		for {
-			err := ctl.Boot(withCfg(context.Background(), &cfg))
+			err := ctl.Boot(withCfg(context.Background(), cfg))
 			if err != nil {
 				log.Printf("controller boot failed: %v", err)
 			} else {
@@ -46,73 +39,3 @@ func main() {
 	<-(chan struct{})(nil)
 }
 
-type middleware func(http.Handler) http.Handler
-
-var notFound = http.NotFoundHandler()
-
-// returns a handler that implements a stack of middlewares.
-func stack(m ...middleware) http.Handler {
-	if len(m) == 0 {
-		return notFound
-	}
-	return m[0](stack(m[1:]...))
-}
-
-// logs each request.
-func (cfg *Config) logger(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		t := time.Now()
-		next.ServeHTTP(w, r)
-		log.Printf("%.6f %q %q %q", time.Since(t).Seconds(), r.RemoteAddr, r.Method, r.URL.Path)
-	})
-}
-
-// dispatches /api/ to the API stack, everything else to the static
-// assets stack.
-func (cfg *Config) apiOrAssets(next http.Handler) http.Handler {
-	mux := http.NewServeMux()
-	mux.Handle("/api/", stack(cfg.apiHeaders, cfg.apiRoutes))
-	mux.Handle("/", http.FileServer(assetFS()))
-	return mux
-}
-
-// adds response headers suitable for API responses
-func (cfg *Config) apiHeaders(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/json")
-		next.ServeHTTP(w, r)
-	})
-}
-
-// dispatches API routes
-func (cfg *Config) apiRoutes(http.Handler) http.Handler {
-	mux := http.NewServeMux()
-	mux.HandleFunc("/api/ping", func(w http.ResponseWriter, r *http.Request) {
-		json.NewEncoder(w).Encode(map[string]interface{}{"time": time.Now().UTC()})
-	})
-	mux.HandleFunc("/api/status/controller", func(w http.ResponseWriter, r *http.Request) {
-		timeout := time.Minute
-		if v, err := strconv.ParseInt(r.FormValue("timeout"), 10, 64); err == nil {
-			timeout = time.Duration(v) * time.Second
-		}
-		if v, err := strconv.ParseInt(r.FormValue("newerThan"), 10, 64); err == nil {
-			log.Println(v, timeout)
-			// TODO: wait
-			// TaskState.Wait(version(v), timeout, r.Context())
-		}
-		// TODO:
-		// rep, v := report(ctlTasks)
-		json.NewEncoder(w).Encode(map[string]interface{}{
-			// "Version": v,
-			// "Tasks":   rep,
-			// TODO:
-			"Version": 1,
-			"Tasks":   []int{},
-		})
-	})
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusNotFound)
-		json.NewEncoder(w).Encode(map[string]string{"error": "not found"})
-	})
-	return mux
-}
diff --git a/services/boot/main.go b/services/boot/webgui.go
similarity index 70%
copy from services/boot/main.go
copy to services/boot/webgui.go
index ebc2b99..18daf73 100644
--- a/services/boot/main.go
+++ b/services/boot/webgui.go
@@ -1,49 +1,19 @@
 package main
 
 import (
-	"context"
 	"encoding/json"
-	"flag"
 	"log"
 	"net/http"
-	"os"
 	"strconv"
 	"time"
-
-	"git.curoverse.com/arvados.git/sdk/go/config"
 )
 
-const defaultCfgPath = "/etc/arvados/boot/boot.yml"
-
-func main() {
-	cfgPath := flag.String("config", defaultCfgPath, "`path` to config file")
-	flag.Parse()
-
-	var cfg Config
-	if err := config.LoadFile(&cfg, *cfgPath); os.IsNotExist(err) && *cfgPath == defaultCfgPath {
-		log.Printf("WARNING: No config file specified or found, starting fresh!")
-	} else if err != nil {
-		log.Fatal(err)
+func runWebGUI(cfg *Config) {
+	if cfg.WebGUI.Listen == "" {
+		return
 	}
-	cfg.SetDefaults()
-	go func() {
-		log.Printf("starting server at %s", cfg.WebGUI.Listen)
-		log.Fatal(http.ListenAndServe(cfg.WebGUI.Listen, stack(cfg.logger, cfg.apiOrAssets)))
-	}()
-	go func() {
-		var ctl Booter = &controller{}
-		ticker := time.NewTicker(5 * time.Second)
-		for {
-			err := ctl.Boot(withCfg(context.Background(), &cfg))
-			if err != nil {
-				log.Printf("controller boot failed: %v", err)
-			} else {
-				log.Printf("controller boot OK")
-			}
-			<-ticker.C
-		}
-	}()
-	<-(chan struct{})(nil)
+	log.Printf("starting webgui at %s", cfg.WebGUI.Listen)
+	log.Fatal(http.ListenAndServe(cfg.WebGUI.Listen, stack(cfg.logger, cfg.apiOrAssets)))
 }
 
 type middleware func(http.Handler) http.Handler

commit 42ae3ec76c4b3c9d87af09e9dff5b2b9838f6954
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Jan 28 17:14:18 2017 -0500

    add runit support

diff --git a/services/boot/config.go b/services/boot/config.go
index 84cb853..2bbdb6e 100644
--- a/services/boot/config.go
+++ b/services/boot/config.go
@@ -1,5 +1,11 @@
 package main
 
+import (
+	"context"
+	"fmt"
+	"os"
+)
+
 type Config struct {
 	// 5 alphanumeric chars. Must be either xx*, yy*, zz*, or
 	// globally unique.
@@ -26,8 +32,24 @@ type Config struct {
 		Listen string
 	}
 
-	UsrDir  string
 	DataDir string
+	UsrDir  string
+
+	RunitSvDir string
+}
+
+func (c *Config) Boot(ctx context.Context) error {
+	for _, path := range []string{c.DataDir, c.UsrDir, c.UsrDir + "/bin"} {
+		if fi, err := os.Stat(path); err != nil {
+			err = os.MkdirAll(path, 0755)
+			if err != nil {
+				return err
+			}
+		} else if !fi.IsDir() {
+			return fmt.Errorf("%s: is not a directory", path)
+		}
+	}
+	return nil
 }
 
 func (c *Config) SetDefaults() {
@@ -52,7 +74,10 @@ func (c *Config) SetDefaults() {
 		c.DataDir = "/var/lib/arvados"
 	}
 	if c.UsrDir == "" {
-		c.DataDir = "/usr/local/arvados"
+		c.UsrDir = "/usr/local/arvados"
+	}
+	if c.RunitSvDir == "" {
+		c.RunitSvDir = "/etc/sv"
 	}
 	if c.WebGUI.Listen == "" {
 		c.WebGUI.Listen = "localhost:18000"
diff --git a/services/boot/consul.go b/services/boot/consul.go
index 095ea89..8761b39 100644
--- a/services/boot/consul.go
+++ b/services/boot/consul.go
@@ -34,20 +34,24 @@ func (cb *consulBooter) Boot(ctx context.Context) error {
 	if cb.check(ctx) == nil {
 		return nil
 	}
+	dataDir := cfg.DataDir + "/consul"
+	if err := os.MkdirAll(dataDir, 0700); err != nil {
+		return err
+	}
 	args := []string{
 		"agent",
 		"-server",
 		"-advertise=127.0.0.1",
-		"-data-dir", cfg.DataDir + "/consul",
+		"-data-dir", dataDir,
 		"-bootstrap-expect", fmt.Sprintf("%d", len(cfg.ControlHosts))}
-	supervisor := newSupervisor("consul", bin, args...)
-	running, err := supervisor.Running()
+	supervisor := newSupervisor(ctx, "consul", bin, args...)
+	running, err := supervisor.Running(ctx)
 	if err != nil {
 		return err
 	}
 	if !running {
 		defer feedbackf(ctx, "starting consul service")()
-		err = supervisor.Start()
+		err = supervisor.Start(ctx)
 		if err != nil {
 			return fmt.Errorf("starting consul: %s", err)
 		}
diff --git a/services/boot/controller.go b/services/boot/controller.go
index 396e37a..526a627 100644
--- a/services/boot/controller.go
+++ b/services/boot/controller.go
@@ -7,7 +7,13 @@ import (
 type controller struct{}
 
 func (c *controller) Boot(ctx context.Context) error {
-	return Concurrent{
-		consul,
+	return Series{
+		Concurrent{
+			cfg(ctx),
+			installCerts,
+		},
+		Concurrent{
+			consul,
+		},
 	}.Boot(ctx)
 }
diff --git a/services/boot/server.go b/services/boot/main.go
similarity index 95%
rename from services/boot/server.go
rename to services/boot/main.go
index 2c477e4..ebc2b99 100644
--- a/services/boot/server.go
+++ b/services/boot/main.go
@@ -35,7 +35,11 @@ func main() {
 		ticker := time.NewTicker(5 * time.Second)
 		for {
 			err := ctl.Boot(withCfg(context.Background(), &cfg))
-			log.Printf("ctl.Boot: %v", err)
+			if err != nil {
+				log.Printf("controller boot failed: %v", err)
+			} else {
+				log.Printf("controller boot OK")
+			}
 			<-ticker.C
 		}
 	}()
@@ -103,7 +107,7 @@ func (cfg *Config) apiRoutes(http.Handler) http.Handler {
 			// "Tasks":   rep,
 			// TODO:
 			"Version": 1,
-			"Tasks": []int{},
+			"Tasks":   []int{},
 		})
 	})
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
diff --git a/services/boot/os_package.go b/services/boot/os_package.go
new file mode 100644
index 0000000..fc82687
--- /dev/null
+++ b/services/boot/os_package.go
@@ -0,0 +1,70 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+var (
+	installCerts = &osPackage{
+		Debian: "ca-certificates",
+	}
+	installRunit = &osPackage{
+		Debian: "runit",
+	}
+)
+
+type osPackage struct {
+	Debian string
+	RedHat string
+}
+
+var (
+	osPackageMutex     sync.Mutex
+	osPackageDidUpdate bool
+)
+
+func (pkg *osPackage) Boot(ctx context.Context) error {
+	osPackageMutex.Lock()
+	defer osPackageMutex.Unlock()
+
+	if _, err := os.Stat("/var/lib/dpkg/info/" + pkg.Debian + ".list"); err == nil {
+		return nil
+	}
+	if !osPackageDidUpdate {
+		d, err := os.Open("/var/lib/apt/lists")
+		if err != nil {
+			return err
+		}
+		defer d.Close()
+		if files, err := d.Readdir(4); len(files) < 4 || err != nil {
+			err = pkg.aptGet("update")
+			if err != nil {
+				return err
+			}
+			osPackageDidUpdate = true
+		}
+	}
+	return pkg.aptGet("install", "-y", "--no-install-recommends", pkg.Debian)
+}
+
+func (*osPackage) aptGet(args ...string) error {
+	cmd := exec.Command("apt-get", args...)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	for _, kv := range os.Environ() {
+		if !strings.HasPrefix(kv, "DEBIAN_FRONTEND=") {
+			cmd.Env = append(cmd.Env, kv)
+		}
+	}
+	cmd.Env = append(cmd.Env, "DEBIAN_FRONTEND=noninteractive")
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("%s: %s", cmd.Args, err)
+	}
+	return nil
+}
diff --git a/services/boot/package.json b/services/boot/package.json
index ec2d95a..340c69f 100644
--- a/services/boot/package.json
+++ b/services/boot/package.json
@@ -14,6 +14,7 @@
   "scripts": {
     "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/boot",
     "dev-as-root": "WEBPACK_FLAGS=-d go generate && go get ./... && sudo $GOPATH/bin/boot",
+    "dev-docker": "WEBPACK_FLAGS=-d go generate && go get ./... && docker build --tag=arvados-boot-test-runit testimage_runit && docker run -it --rm --volume=${GOPATH}/bin/boot:/usr/bin/arvados-boot:ro arvados-boot-test-runit",
     "test": "./node_modules/.bin/tap 'js/**/*_test.js'",
     "build": "go generate && go get ./...",
     "start": "npm run build && $GOPATH/bin/boot",
diff --git a/services/boot/runit.go b/services/boot/runit.go
new file mode 100644
index 0000000..e5bf520
--- /dev/null
+++ b/services/boot/runit.go
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+type runitService struct {
+	name string
+	cmd  string
+	args []string
+}
+
+func (r *runitService) Start(ctx context.Context) error {
+	if err := installRunit.Boot(ctx); err != nil {
+		return err
+	}
+	svdir := r.svdir(ctx)
+	if err := os.MkdirAll(svdir, 0755); err != nil {
+		return err
+	}
+	tmp, err := ioutil.TempFile(svdir, "run~")
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(tmp, "#!/bin/sh\n\nexec %q", r.cmd)
+	for _, arg := range r.args {
+		fmt.Fprintf(tmp, " %q", arg)
+	}
+	fmt.Fprintf(tmp, " 2>&1\n")
+	tmp.Close()
+	if err := os.Chmod(tmp.Name(), 0755); err != nil {
+		os.Remove(tmp.Name())
+		return err
+	}
+	if err := os.Rename(tmp.Name(), path.Join(svdir, "run")); err != nil {
+		os.Remove(tmp.Name())
+		return err
+	}
+	return nil
+}
+
+func (r *runitService) Running(ctx context.Context) (bool, error) {
+	if err := installRunit.Boot(ctx); err != nil {
+		return false, err
+	}
+	return runStatusCmd("sv", "stat", r.svdir(ctx))
+}
+
+func (r *runitService) svdir(ctx context.Context) string {
+	return path.Join(cfg(ctx).RunitSvDir, r.name)
+}
diff --git a/services/boot/supervisor.go b/services/boot/supervisor.go
new file mode 100644
index 0000000..d095c23
--- /dev/null
+++ b/services/boot/supervisor.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+	"context"
+	"os"
+)
+
+type supervisor interface {
+	Running(ctx context.Context) (bool, error)
+	Start(ctx context.Context) error
+}
+
+func newSupervisor(ctx context.Context, name, cmd string, args ...string) supervisor {
+	if _, err := os.Stat("/run/systemd/system"); err == nil {
+		return &systemdUnit{
+			name: name,
+			cmd:  cmd,
+			args: args,
+		}
+	}
+	return &runitService{
+		name: name,
+		cmd:  cmd,
+		args: args,
+	}
+}
+
diff --git a/services/boot/systemd.go b/services/boot/systemd.go
index 3015433..4cde7a9 100644
--- a/services/boot/systemd.go
+++ b/services/boot/systemd.go
@@ -1,31 +1,19 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 )
 
-type supervisor interface {
-	Running() (bool, error)
-	Start() error
-}
-
-func newSupervisor(name, cmd string, args ...string) supervisor {
-	return &systemdUnit{
-		name: name,
-		cmd:  cmd,
-		args: args,
-	}
-}
-
 type systemdUnit struct {
 	name string
 	cmd  string
 	args []string
 }
 
-func (u *systemdUnit) Start() error {
+func (u *systemdUnit) Start(ctx context.Context) error {
 	cmd := exec.Command("systemd-run", append([]string{"--unit=arvados-" + u.name, u.cmd}, u.args...)...)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
@@ -36,8 +24,12 @@ func (u *systemdUnit) Start() error {
 	return err
 }
 
-func (u *systemdUnit) Running() (bool, error) {
-	cmd := exec.Command("systemctl", "status", "arvados-"+u.name)
+func (u *systemdUnit) Running(ctx context.Context) (bool, error) {
+	return runStatusCmd("systemctl", "status", "arvados-"+u.name)
+}
+
+func runStatusCmd(prog string, args ...string) (bool, error) {
+	cmd := exec.Command(prog, args...)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	err := cmd.Run()
diff --git a/services/boot/testimage_runit/Dockerfile b/services/boot/testimage_runit/Dockerfile
new file mode 100644
index 0000000..9e363fc
--- /dev/null
+++ b/services/boot/testimage_runit/Dockerfile
@@ -0,0 +1,10 @@
+FROM debian:8
+RUN apt-get update
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends runit
+RUN mkdir /etc/sv/arvados-boot && ln -s /usr/bin/arvados-boot /etc/sv/arvados-boot/run
+
+# preload (but don't install) packages arvados-boot might decide to install
+RUN DEBIAN_FRONTEND=noninteractive apt-get -dy install ca-certificates
+
+CMD ["sh", "-c", "runsvdir /etc/sv"]

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list