[ARVADOS] created: fa3bb426d9a1b70ecb92cf814596437531216807
Git user
git at public.curoverse.com
Sat Jan 28 17:18:04 EST 2017
at fa3bb426d9a1b70ecb92cf814596437531216807 (commit)
commit fa3bb426d9a1b70ecb92cf814596437531216807
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Jan 28 17:14:18 2017 -0500
add runit support
diff --git a/services/boot/config.go b/services/boot/config.go
index 84cb853..2bbdb6e 100644
--- a/services/boot/config.go
+++ b/services/boot/config.go
@@ -1,5 +1,11 @@
package main
+import (
+ "context"
+ "fmt"
+ "os"
+)
+
type Config struct {
// 5 alphanumeric chars. Must be either xx*, yy*, zz*, or
// globally unique.
@@ -26,8 +32,24 @@ type Config struct {
Listen string
}
- UsrDir string
DataDir string
+ UsrDir string
+
+ RunitSvDir string
+}
+
+func (c *Config) Boot(ctx context.Context) error {
+ for _, path := range []string{c.DataDir, c.UsrDir, c.UsrDir + "/bin"} {
+ if fi, err := os.Stat(path); err != nil {
+ err = os.MkdirAll(path, 0755)
+ if err != nil {
+ return err
+ }
+ } else if !fi.IsDir() {
+ return fmt.Errorf("%s: is not a directory", path)
+ }
+ }
+ return nil
}
func (c *Config) SetDefaults() {
@@ -52,7 +74,10 @@ func (c *Config) SetDefaults() {
c.DataDir = "/var/lib/arvados"
}
if c.UsrDir == "" {
- c.DataDir = "/usr/local/arvados"
+ c.UsrDir = "/usr/local/arvados"
+ }
+ if c.RunitSvDir == "" {
+ c.RunitSvDir = "/etc/sv"
}
if c.WebGUI.Listen == "" {
c.WebGUI.Listen = "localhost:18000"
diff --git a/services/boot/consul.go b/services/boot/consul.go
index 095ea89..8761b39 100644
--- a/services/boot/consul.go
+++ b/services/boot/consul.go
@@ -34,20 +34,24 @@ func (cb *consulBooter) Boot(ctx context.Context) error {
if cb.check(ctx) == nil {
return nil
}
+ dataDir := cfg.DataDir + "/consul"
+ if err := os.MkdirAll(dataDir, 0700); err != nil {
+ return err
+ }
args := []string{
"agent",
"-server",
"-advertise=127.0.0.1",
- "-data-dir", cfg.DataDir + "/consul",
+ "-data-dir", dataDir,
"-bootstrap-expect", fmt.Sprintf("%d", len(cfg.ControlHosts))}
- supervisor := newSupervisor("consul", bin, args...)
- running, err := supervisor.Running()
+ supervisor := newSupervisor(ctx, "consul", bin, args...)
+ running, err := supervisor.Running(ctx)
if err != nil {
return err
}
if !running {
defer feedbackf(ctx, "starting consul service")()
- err = supervisor.Start()
+ err = supervisor.Start(ctx)
if err != nil {
return fmt.Errorf("starting consul: %s", err)
}
diff --git a/services/boot/controller.go b/services/boot/controller.go
index 396e37a..526a627 100644
--- a/services/boot/controller.go
+++ b/services/boot/controller.go
@@ -7,7 +7,13 @@ import (
type controller struct{}
func (c *controller) Boot(ctx context.Context) error {
- return Concurrent{
- consul,
+ return Series{
+ Concurrent{
+ cfg(ctx),
+ installCerts,
+ },
+ Concurrent{
+ consul,
+ },
}.Boot(ctx)
}
diff --git a/services/boot/os_package.go b/services/boot/os_package.go
new file mode 100644
index 0000000..45c8bb7
--- /dev/null
+++ b/services/boot/os_package.go
@@ -0,0 +1,70 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "strings"
+ "sync"
+)
+
+var (
+ installCerts = &osPackage{
+ Debian: "ca-certificates",
+ }
+ installRunit = &osPackage{
+ Debian: "runit",
+ }
+)
+
+type osPackage struct {
+ Debian string
+ RedHat string
+}
+
+var (
+ osPackageMutex sync.Mutex
+ osPackageDidUpdate bool
+)
+
+func (pkg *osPackage) Boot(ctx context.Context) error {
+ osPackageMutex.Lock()
+ defer osPackageMutex.Unlock()
+
+ if _, err := os.Stat("/var/lib/dpkg/info/" + pkg.Debian + ".list"); err == nil {
+ return nil
+ }
+ if !osPackageDidUpdate {
+ d, err := os.Open("/var/lib/apt/lists")
+ if err != nil {
+ return err
+ }
+ defer d.Close()
+ if files, err := d.Readdir(4); len(files) < 4 || err != nil {
+ err = pkg.aptGet("update")
+ if err != nil {
+ return err
+ }
+ osPackageDidUpdate = true
+ }
+ }
+ return pkg.aptGet("install", "-y", "--no-install-recommends", pkg.Debian)
+}
+
+func (*osPackage) aptGet(args ...string) error {
+ cmd := exec.Command("apt-get", args...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ for _, kv := range os.Environ() {
+ if !strings.HasPrefix(kv, "DEBIAN_FRONTEND=") {
+ cmd.Env = append(cmd.Env, kv)
+ }
+ }
+ cmd.Env = append(cmd.Env, "DEBIAN_FRONTEND=noninteractive")
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("%s: %s", cmd.Args, err)
+ }
+ return nil
+}
diff --git a/services/boot/package.json b/services/boot/package.json
index ec2d95a..340c69f 100644
--- a/services/boot/package.json
+++ b/services/boot/package.json
@@ -14,6 +14,7 @@
"scripts": {
"dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/boot",
"dev-as-root": "WEBPACK_FLAGS=-d go generate && go get ./... && sudo $GOPATH/bin/boot",
+ "dev-docker": "WEBPACK_FLAGS=-d go generate && go get ./... && docker build --tag=arvados-boot-test-runit testimage_runit && docker run -it --rm --volume=${GOPATH}/bin/boot:/usr/bin/arvados-boot:ro arvados-boot-test-runit",
"test": "./node_modules/.bin/tap 'js/**/*_test.js'",
"build": "go generate && go get ./...",
"start": "npm run build && $GOPATH/bin/boot",
diff --git a/services/boot/server.go b/services/boot/server.go
index 2c477e4..8531f96 100644
--- a/services/boot/server.go
+++ b/services/boot/server.go
@@ -35,7 +35,11 @@ func main() {
ticker := time.NewTicker(5 * time.Second)
for {
err := ctl.Boot(withCfg(context.Background(), &cfg))
- log.Printf("ctl.Boot: %v", err)
+ if err != nil {
+ log.Printf("controller boot failed: %v", err)
+ } else {
+ log.Printf("controller boot OK")
+ }
<-ticker.C
}
}()
diff --git a/services/boot/supervisor.go b/services/boot/supervisor.go
new file mode 100644
index 0000000..d095c23
--- /dev/null
+++ b/services/boot/supervisor.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+ "context"
+ "os"
+)
+
+type supervisor interface {
+ Running(ctx context.Context) (bool, error)
+ Start(ctx context.Context) error
+}
+
+func newSupervisor(ctx context.Context, name, cmd string, args ...string) supervisor {
+ if _, err := os.Stat("/run/systemd/system"); err == nil {
+ return &systemdUnit{
+ name: name,
+ cmd: cmd,
+ args: args,
+ }
+ }
+ return &runitService{
+ name: name,
+ cmd: cmd,
+ args: args,
+ }
+}
+
diff --git a/services/boot/systemd.go b/services/boot/systemd.go
index 3015433..4cde7a9 100644
--- a/services/boot/systemd.go
+++ b/services/boot/systemd.go
@@ -1,31 +1,19 @@
package main
import (
+ "context"
"fmt"
"os"
"os/exec"
)
-type supervisor interface {
- Running() (bool, error)
- Start() error
-}
-
-func newSupervisor(name, cmd string, args ...string) supervisor {
- return &systemdUnit{
- name: name,
- cmd: cmd,
- args: args,
- }
-}
-
type systemdUnit struct {
name string
cmd string
args []string
}
-func (u *systemdUnit) Start() error {
+func (u *systemdUnit) Start(ctx context.Context) error {
cmd := exec.Command("systemd-run", append([]string{"--unit=arvados-" + u.name, u.cmd}, u.args...)...)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
@@ -36,8 +24,12 @@ func (u *systemdUnit) Start() error {
return err
}
-func (u *systemdUnit) Running() (bool, error) {
- cmd := exec.Command("systemctl", "status", "arvados-"+u.name)
+func (u *systemdUnit) Running(ctx context.Context) (bool, error) {
+ return runStatusCmd("systemctl", "status", "arvados-"+u.name)
+}
+
+func runStatusCmd(prog string, args ...string) (bool, error) {
+ cmd := exec.Command(prog, args...)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
err := cmd.Run()
diff --git a/services/boot/testimage_runit/Dockerfile b/services/boot/testimage_runit/Dockerfile
new file mode 100644
index 0000000..9e363fc
--- /dev/null
+++ b/services/boot/testimage_runit/Dockerfile
@@ -0,0 +1,10 @@
+FROM debian:8
+RUN apt-get update
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends runit
+RUN mkdir /etc/sv/arvados-boot && ln -s /usr/bin/arvados-boot /etc/sv/arvados-boot/run
+
+# preload (but don't install) packages arvados-boot might decide to install
+RUN DEBIAN_FRONTEND=noninteractive apt-get -dy install ca-certificates
+
+CMD ["sh", "-c", "runsvdir /etc/sv"]
commit aede26e6edbd6c2456d4bd46db2ed32740b46808
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Jan 28 01:56:01 2017 -0500
refactor as procedural
diff --git a/services/boot/.gitignore b/services/boot/.gitignore
index d08895d..8c8b8e3 100644
--- a/services/boot/.gitignore
+++ b/services/boot/.gitignore
@@ -2,3 +2,4 @@
bindata.tmp
node_modules
bindata_assetfs.go
+npm-debug.log
diff --git a/services/boot/booter.go b/services/boot/booter.go
new file mode 100644
index 0000000..cb672ab
--- /dev/null
+++ b/services/boot/booter.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "sync"
+)
+
+// A Booter ensures some piece of the system ("target") is correctly
+// installed, configured, running, or working.
+type Booter interface {
+ // Inspect, repair, and report the current state of the target.
+ Boot(context.Context) error
+}
+
+var cfgKey = &struct{}{}
+
+func cfg(ctx context.Context) *Config {
+ return ctx.Value(cfgKey).(*Config)
+}
+
+func withCfg(ctx context.Context, cfg *Config) context.Context {
+ return context.WithValue(ctx, cfgKey, cfg)
+}
+
+type Series []Booter
+
+func (sb Series) Boot(ctx context.Context) error {
+ for _, b := range sb {
+ err := b.Boot(ctx)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+type Concurrent []Booter
+
+func (cb Concurrent) Boot(ctx context.Context) error {
+ errs := make([]error, len(cb))
+ var wg sync.WaitGroup
+ wg.Add(len(cb))
+ for i, b := range cb {
+ i, b := i, b
+ go func() {
+ defer wg.Done()
+ errs[i] = b.Boot(ctx)
+ }()
+ }
+ wg.Wait()
+ return NewMultipleError(errs)
+}
+
+type MultipleError struct {
+ error
+ errors []error
+}
+
+func NewMultipleError(errs []error) error {
+ var errors []error
+ for _, err := range errs {
+ switch err := err.(type) {
+ case *MultipleError:
+ errors = append(errors, err.errors...)
+ case nil:
+ default:
+ errors = append(errors, err)
+ }
+ }
+ if len(errors) == 0 {
+ return nil
+ }
+ if len(errors) == 1 {
+ return errors[0]
+ }
+ return &MultipleError{
+ error: fmt.Errorf("%d errors", len(errors)),
+ errors: errors,
+ }
+}
diff --git a/services/boot/config.go b/services/boot/config.go
index 0eb1c30..84cb853 100644
--- a/services/boot/config.go
+++ b/services/boot/config.go
@@ -10,8 +10,21 @@ type Config struct {
// alive.
ControlHosts []string
- // addr:port to serve web-based setup/monitoring application
- WebListen string
+ ConsulPorts struct {
+ DNS int
+ HTTP int
+ HTTPS int
+ RPC int
+ SerfLAN int `json:"Serf_LAN"`
+ SerfWAN int `json:"Serf_WAN"`
+ Server int
+ }
+
+ WebGUI struct {
+ // addr:port to serve web-based setup/monitoring
+ // application
+ Listen string
+ }
UsrDir string
DataDir string
@@ -21,13 +34,27 @@ func (c *Config) SetDefaults() {
if len(c.ControlHosts) == 0 {
c.ControlHosts = []string{"127.0.0.1"}
}
+ defaultPort := []int{18600, 18500, -1, 18400, 18301, 18302, 18300}
+ for i, port := range []*int{
+ &c.ConsulPorts.DNS,
+ &c.ConsulPorts.HTTP,
+ &c.ConsulPorts.HTTPS,
+ &c.ConsulPorts.RPC,
+ &c.ConsulPorts.SerfLAN,
+ &c.ConsulPorts.SerfWAN,
+ &c.ConsulPorts.Server,
+ } {
+ if *port == 0 {
+ *port = defaultPort[i]
+ }
+ }
if c.DataDir == "" {
c.DataDir = "/var/lib/arvados"
}
if c.UsrDir == "" {
- c.DataDir = "/usr/local"
+ c.DataDir = "/usr/local/arvados"
}
- if c.WebListen == "" {
- c.WebListen = "localhost:8000"
+ if c.WebGUI.Listen == "" {
+ c.WebGUI.Listen = "localhost:18000"
}
}
diff --git a/services/boot/consul.go b/services/boot/consul.go
new file mode 100644
index 0000000..095ea89
--- /dev/null
+++ b/services/boot/consul.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "sync"
+
+ "github.com/hashicorp/consul/api"
+)
+
+var consul = &consulBooter{}
+
+type consulBooter struct {
+ sync.Mutex
+}
+
+func (cb *consulBooter) Boot(ctx context.Context) error {
+ cb.Lock()
+ defer cb.Unlock()
+
+ cfg := cfg(ctx)
+ bin := cfg.UsrDir + "/bin/consul"
+ err := (&download{
+ URL: "https://releases.hashicorp.com/consul/0.7.2/consul_0.7.2_linux_amd64.zip",
+ Dest: bin,
+ Size: 29079005,
+ Mode: 0755,
+ }).Boot(ctx)
+ if err != nil {
+ return err
+ }
+ if cb.check(ctx) == nil {
+ return nil
+ }
+ args := []string{
+ "agent",
+ "-server",
+ "-advertise=127.0.0.1",
+ "-data-dir", cfg.DataDir + "/consul",
+ "-bootstrap-expect", fmt.Sprintf("%d", len(cfg.ControlHosts))}
+ supervisor := newSupervisor("consul", bin, args...)
+ running, err := supervisor.Running()
+ if err != nil {
+ return err
+ }
+ if !running {
+ defer feedbackf(ctx, "starting consul service")()
+ err = supervisor.Start()
+ if err != nil {
+ return fmt.Errorf("starting consul: %s", err)
+ }
+ if len(cfg.ControlHosts) > 1 {
+ cmd := exec.Command(bin, append([]string{"join"}, cfg.ControlHosts...)...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("consul join: %s", err)
+ }
+ }
+ }
+ return cb.check(ctx)
+}
+
+var consulCfg = api.DefaultConfig()
+
+func (cb *consulBooter) check(ctx context.Context) error {
+ cfg := cfg(ctx)
+ consulCfg.Datacenter = cfg.SiteID
+ consul, err := api.NewClient(consulCfg)
+ if err != nil {
+ return err
+ }
+ _, err = consul.Catalog().Datacenters()
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/services/boot/consul_task.go b/services/boot/consul_task.go
deleted file mode 100644
index 4de5512..0000000
--- a/services/boot/consul_task.go
+++ /dev/null
@@ -1,81 +0,0 @@
-package main
-
-import (
- "fmt"
- "os"
- "os/exec"
- "time"
-
- "github.com/hashicorp/consul/api"
-)
-
-type consulService struct {
- supervisor
-}
-
-func (cs *consulService) Init(cfg *Config) {
- args := []string{
- "agent",
- "-server",
- "-advertise=127.0.0.1",
- "-data-dir", cfg.DataDir + "/consul",
- "-bootstrap-expect", fmt.Sprintf("%d", len(cfg.ControlHosts))}
- cs.supervisor = newSupervisor("consul", "/usr/local/bin/consul", args...)
-}
-
-func (cs *consulService) Children() []task {
- return nil
-}
-
-func (cs *consulService) ShortName() string {
- return "consul running"
-}
-
-func (cs *consulService) String() string {
- return "Ensure consul daemon is supervised & running"
-}
-
-func (cs *consulService) Check() error {
- consul, err := api.NewClient(api.DefaultConfig())
- if err != nil {
- return err
- }
- _, err = consul.Catalog().Datacenters()
- if err != nil {
- return err
- }
- return nil
-}
-
-func (cs *consulService) CanFix() bool {
- return true
-}
-
-func (cs *consulService) Fix() error {
- err := cs.supervisor.Start()
- if err != nil {
- return err
- }
-
- if len(cfg.ControlHosts) > 1 {
- cmd := exec.Command("/usr/local/bin/consul", append([]string{"join"}, cfg.ControlHosts...)...)
- cmd.Stdout = os.Stderr
- cmd.Stderr = os.Stderr
- err := cmd.Run()
- if err != nil {
- return err
- }
- }
-
- timeout := time.After(10 * time.Second)
- ticker := time.NewTicker(50 * time.Millisecond)
- defer ticker.Stop()
- for cs.Check() != nil {
- select {
- case <-ticker.C:
- case <-timeout:
- return cs.Check()
- }
- }
- return nil
-}
diff --git a/services/boot/controller.go b/services/boot/controller.go
new file mode 100644
index 0000000..396e37a
--- /dev/null
+++ b/services/boot/controller.go
@@ -0,0 +1,13 @@
+package main
+
+import (
+ "context"
+)
+
+type controller struct{}
+
+func (c *controller) Boot(ctx context.Context) error {
+ return Concurrent{
+ consul,
+ }.Boot(ctx)
+}
diff --git a/services/boot/ctl_tasks.go b/services/boot/ctl_tasks.go
deleted file mode 100644
index 8737316..0000000
--- a/services/boot/ctl_tasks.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package main
-
-// tasks to run on a controller node
-var ctlTasks = []task{
- &download{
- URL: "https://releases.hashicorp.com/consul/0.7.2/consul_0.7.2_linux_amd64.zip",
- Dest: "/usr/local/bin/consul",
- Size: 29079005,
- Mode: 0755,
- },
- &consulService{},
-}
diff --git a/services/boot/download_task.go b/services/boot/download.go
similarity index 71%
rename from services/boot/download_task.go
rename to services/boot/download.go
index da70678..e047aab 100644
--- a/services/boot/download_task.go
+++ b/services/boot/download.go
@@ -2,6 +2,7 @@ package main
import (
"archive/zip"
+ "context"
"fmt"
"io"
"io/ioutil"
@@ -20,39 +21,22 @@ type download struct {
Hash string
}
-func (d *download) Init(cfg *Config) {}
-
-func (d *download) Children() []task {
- return nil
-}
-
-func (d *download) ShortName() string {
- return d.Dest
-}
-
-func (d *download) String() string {
- return fmt.Sprintf("Download %q from %q", d.Dest, d.URL)
-}
-
-func (d *download) Check() error {
+func (d *download) Boot(ctx context.Context) error {
fi, err := os.Stat(d.Dest)
- if err != nil {
+ if os.IsNotExist(err) {
+ // fall through to fix
+ } else if err != nil {
return err
+ } else if d.Size > 0 && fi.Size() != d.Size {
+ err = fmt.Errorf("Size mismatch: %q is %d bytes, expected %d", d.Dest, fi.Size(), d.Size)
+ } else if d.Mode > 0 && fi.Mode() != d.Mode {
+ err = fmt.Errorf("Mode mismatch: %q is %s, expected %s", d.Dest, fi.Mode(), d.Mode)
+ } else {
+ return nil
}
- if d.Size > 0 && fi.Size() != d.Size {
- return fmt.Errorf("Size mismatch: %q is %d bytes, expected %d", d.Dest, fi.Size(), d.Size)
- }
- if d.Mode > 0 && fi.Mode() != d.Mode {
- return fmt.Errorf("Mode mismatch: %q is %s, expected %s", d.Dest, fi.Mode(), d.Mode)
- }
- return nil
-}
-func (d *download) CanFix() bool {
- return true
-}
+ defer feedbackf(ctx, "downloading %s", d.URL)()
-func (d *download) Fix() error {
out, err := ioutil.TempFile(path.Dir(d.Dest), path.Base(d.Dest))
if err != nil {
return err
diff --git a/services/boot/feedback.go b/services/boot/feedback.go
new file mode 100644
index 0000000..6cc509f
--- /dev/null
+++ b/services/boot/feedback.go
@@ -0,0 +1,15 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+)
+
+func feedbackf(ctx context.Context, f string, args ...interface{}) func() {
+ msg := fmt.Sprintf(f, args...)
+ log.Print("start: ", msg)
+ return func() {
+ log.Print(" done: ", msg)
+ }
+}
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
index b4a8744..9ed5c04 100644
--- a/services/boot/js/index.js
+++ b/services/boot/js/index.js
@@ -50,7 +50,7 @@ var Home = {
m('table.table', {style: {width: '350px'}},
m('tbody', {style: {opacity: ctl().Outdated ? .5 : 1}}, ctl().Tasks.map(function(task) {
return m('tr', [
- m('td', task.ShortName),
+ m('td', task.Name),
m('td',
m('span.badge',
{class: task.State == 'OK' ? 'badge-success' : 'badge-danger'},
diff --git a/services/boot/server.go b/services/boot/server.go
index f9aad57..2c477e4 100644
--- a/services/boot/server.go
+++ b/services/boot/server.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"encoding/json"
"flag"
"log"
@@ -14,12 +15,11 @@ import (
const defaultCfgPath = "/etc/arvados/boot/boot.yml"
-var cfg Config
-
func main() {
cfgPath := flag.String("config", defaultCfgPath, "`path` to config file")
flag.Parse()
+ var cfg Config
if err := config.LoadFile(&cfg, *cfgPath); os.IsNotExist(err) && *cfgPath == defaultCfgPath {
log.Printf("WARNING: No config file specified or found, starting fresh!")
} else if err != nil {
@@ -27,13 +27,15 @@ func main() {
}
cfg.SetDefaults()
go func() {
- log.Printf("starting server at %s", cfg.WebListen)
- log.Fatal(http.ListenAndServe(cfg.WebListen, stack(logger, apiOrAssets)))
+ log.Printf("starting server at %s", cfg.WebGUI.Listen)
+ log.Fatal(http.ListenAndServe(cfg.WebGUI.Listen, stack(cfg.logger, cfg.apiOrAssets)))
}()
go func() {
+ var ctl Booter = &controller{}
ticker := time.NewTicker(5 * time.Second)
for {
- runTasks(&cfg, ctlTasks)
+ err := ctl.Boot(withCfg(context.Background(), &cfg))
+ log.Printf("ctl.Boot: %v", err)
<-ticker.C
}
}()
@@ -53,7 +55,7 @@ func stack(m ...middleware) http.Handler {
}
// logs each request.
-func logger(next http.Handler) http.Handler {
+func (cfg *Config) logger(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t := time.Now()
next.ServeHTTP(w, r)
@@ -63,15 +65,15 @@ func logger(next http.Handler) http.Handler {
// dispatches /api/ to the API stack, everything else to the static
// assets stack.
-func apiOrAssets(next http.Handler) http.Handler {
+func (cfg *Config) apiOrAssets(next http.Handler) http.Handler {
mux := http.NewServeMux()
- mux.Handle("/api/", stack(apiHeaders, apiRoutes))
+ mux.Handle("/api/", stack(cfg.apiHeaders, cfg.apiRoutes))
mux.Handle("/", http.FileServer(assetFS()))
return mux
}
// adds response headers suitable for API responses
-func apiHeaders(next http.Handler) http.Handler {
+func (cfg *Config) apiHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
next.ServeHTTP(w, r)
@@ -79,23 +81,29 @@ func apiHeaders(next http.Handler) http.Handler {
}
// dispatches API routes
-func apiRoutes(http.Handler) http.Handler {
+func (cfg *Config) apiRoutes(http.Handler) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/api/ping", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{"time": time.Now().UTC()})
})
- mux.HandleFunc("/api/tasks/ctl", func(w http.ResponseWriter, r *http.Request) {
+ mux.HandleFunc("/api/status/controller", func(w http.ResponseWriter, r *http.Request) {
timeout := time.Minute
if v, err := strconv.ParseInt(r.FormValue("timeout"), 10, 64); err == nil {
timeout = time.Duration(v) * time.Second
}
if v, err := strconv.ParseInt(r.FormValue("newerThan"), 10, 64); err == nil {
- TaskState.Wait(version(v), timeout, r.Context())
+ log.Println(v, timeout)
+ // TODO: wait
+ // TaskState.Wait(version(v), timeout, r.Context())
}
- rep, v := report(ctlTasks)
+ // TODO:
+ // rep, v := report(ctlTasks)
json.NewEncoder(w).Encode(map[string]interface{}{
- "Version": v,
- "Tasks": rep,
+ // "Version": v,
+ // "Tasks": rep,
+ // TODO:
+ "Version": 1,
+ "Tasks": []int{},
})
})
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
diff --git a/services/boot/shell.go b/services/boot/shell.go
new file mode 100644
index 0000000..8dcfbe2
--- /dev/null
+++ b/services/boot/shell.go
@@ -0,0 +1,17 @@
+package main
+
+import (
+ "bytes"
+ "os/exec"
+ "strings"
+)
+
+func BashScript(script string) ([]byte, []byte, error) {
+ cmd := exec.Command("bash", "-e", "-x")
+ cmd.Stdin = strings.NewReader(script)
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ err := cmd.Run()
+ return stdout.Bytes(), stderr.Bytes(), err
+}
diff --git a/services/boot/systemd.go b/services/boot/systemd.go
index e88ecc3..3015433 100644
--- a/services/boot/systemd.go
+++ b/services/boot/systemd.go
@@ -7,26 +7,26 @@ import (
)
type supervisor interface {
- Check() (bool, error)
+ Running() (bool, error)
Start() error
}
func newSupervisor(name, cmd string, args ...string) supervisor {
return &systemdUnit{
name: name,
- cmd: cmd,
+ cmd: cmd,
args: args,
}
}
type systemdUnit struct {
name string
- cmd string
+ cmd string
args []string
}
func (u *systemdUnit) Start() error {
- cmd := exec.Command("systemd-run", append([]string{"--unit=arvados-"+u.name, u.cmd}, u.args...)...)
+ cmd := exec.Command("systemd-run", append([]string{"--unit=arvados-" + u.name, u.cmd}, u.args...)...)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
err := cmd.Run()
@@ -36,7 +36,7 @@ func (u *systemdUnit) Start() error {
return err
}
-func (u *systemdUnit) Check() (bool, error) {
+func (u *systemdUnit) Running() (bool, error) {
cmd := exec.Command("systemctl", "status", "arvados-"+u.name)
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
diff --git a/services/boot/task.go b/services/boot/task.go
deleted file mode 100644
index 75c49bd..0000000
--- a/services/boot/task.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package main
-
-import (
- "context"
- "log"
- "sync"
- "time"
-)
-
-type taskState string
-
-const (
- StateUnchecked taskState = "Unchecked"
- StateChecking = "Checking"
- StateFixing = "Fixing"
- StateFailed = "Failed"
- StateOK = "OK"
-)
-
-type version int64
-
-type taskStateMap struct {
- s map[task]taskState
- cond *sync.Cond
- version version
-}
-
-var TaskState = taskStateMap{
- s: make(map[task]taskState),
- cond: sync.NewCond(&sync.Mutex{}),
-}
-
-func (m *taskStateMap) Set(t task, s taskState) {
- m.cond.L.Lock()
- defer m.cond.L.Unlock()
- if old, ok := m.s[t]; ok && old == s {
- return
- }
- m.s[t] = s
- m.version++
- m.cond.Broadcast()
-}
-
-func (m *taskStateMap) Version() version {
- m.cond.L.Lock()
- defer m.cond.L.Unlock()
- return m.version
-}
-
-func (m *taskStateMap) Get(t task) taskState {
- m.cond.L.Lock()
- defer m.cond.L.Unlock()
- if s, ok := m.s[t]; ok {
- return s
- } else {
- return StateUnchecked
- }
-}
-
-type repEnt struct {
- ShortName string
- Description string
- State taskState
- Children []repEnt
-}
-
-func (m *taskStateMap) Wait(v version, t time.Duration, ctx context.Context) bool {
- ready := make(chan struct{})
- var done bool
- go func() {
- m.cond.L.Lock()
- defer m.cond.L.Unlock()
- for v == m.version && !done {
- m.cond.Wait()
- }
- close(ready)
- }()
- select {
- case <-ready:
- return true
- case <-ctx.Done():
- case <-time.After(t):
- }
- done = true
- m.cond.Broadcast()
- return false
-}
-
-func report(tasks []task) ([]repEnt, version) {
- v := TaskState.Version()
- if len(tasks) == 0 {
- return nil, v
- }
- var rep []repEnt
- for _, t := range tasks {
- crep, _ := report(t.Children())
- rep = append(rep, repEnt{
- ShortName: t.ShortName(),
- Description: t.String(),
- State: TaskState.Get(t),
- Children: crep,
- })
- }
- return rep, v
-}
-
-func runTasks(cfg *Config, tasks []task) {
- for _, t := range tasks {
- t.Init(cfg)
- }
- for _, t := range tasks {
- if TaskState.Get(t) == taskState("") {
- TaskState.Set(t, StateChecking)
- }
- err := t.Check()
- if err == nil {
- log.Printf("%s: OK", t)
- TaskState.Set(t, StateOK)
- continue
- }
- log.Printf("%s: %s", t, err)
- if !t.CanFix() {
- log.Printf("%s: can't fix")
- TaskState.Set(t, StateFailed)
- continue
- }
- TaskState.Set(t, StateFixing)
- if err = t.Fix(); err != nil {
- log.Printf("%s: can't fix: %s", t, err)
- TaskState.Set(t, StateFailed)
- continue
- }
- if err = t.Check(); err != nil {
- log.Printf("%s: fixed, but still broken?!: %s", t, err)
- TaskState.Set(t, StateFailed)
- continue
- }
- log.Printf("%s: OK", t)
- TaskState.Set(t, StateOK)
- }
-}
-
-type task interface {
- Init(*Config)
- ShortName() string
- String() string
- Check() error
- CanFix() bool
- Fix() error
- Children() []task
-}
commit cdc28f63adaa8b64c44fdce6025bf1799e90be58
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jan 26 01:57:33 2017 -0500
fade out when stale
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
index d1d483e..b4a8744 100644
--- a/services/boot/js/index.js
+++ b/services/boot/js/index.js
@@ -6,39 +6,36 @@ require('./example.js')
var m = require('mithril')
var Stream = require('mithril/stream')
-var ctl = Stream({Tasks: [], Version: 0})
+const refreshInterval = 5
+
+var ctl = Stream({Tasks: [], Version: 0, Outdated: true})
-refresh.next = null
refresh.xhr = null
function refresh() {
- const timeout = 60
if (refresh.xhr !== null) {
refresh.xhr.abort()
refresh.xhr = null
+ ctl().Outdated = true
+ m.redraw()
}
- if (refresh.next !== null)
- window.clearTimeout(refresh.next)
- refresh.next = window.setTimeout(refresh, timeout*1000)
- var version = ctl().Version
m.request({
method: 'GET',
- url: '/api/tasks/ctl?timeout='+timeout+'&newerThan='+version,
+ url: '/api/tasks/ctl?timeout='+refreshInterval+'&newerThan='+ctl().version,
config: function(xhr) { refresh.xhr = xhr },
})
- .then(ctl)
- .then(function() {
- if (ctl().Version != version) {
+ .then(function(data) {
+ var isNew = data.Version != ctl().Version
+ ctl(data)
+ refresh.xhr = null
+ if (isNew)
// Got a new version -- assume the server is obeying
// newerThan, and start listening for the next version
// right away.
refresh()
- } else {
- if (refresh.next !== null)
- window.clearTimeout(refresh.next)
- refresh.next = window.setTimeout(refresh, 5000)
- }
})
}
+window.setInterval(refresh, refreshInterval*1000)
+refresh()
var Home = {
view: function(vnode) {
@@ -51,7 +48,7 @@ var Home = {
m('a.nav-link[href=/]', {config: m.route}, 'health', m('span.sr-only', '(current)')))))),
m('.x-spacer', {height: '1em'}),
m('table.table', {style: {width: '350px'}},
- m('tbody', ctl().Tasks.map(function(task) {
+ m('tbody', {style: {opacity: ctl().Outdated ? .5 : 1}}, ctl().Tasks.map(function(task) {
return m('tr', [
m('td', task.ShortName),
m('td',
@@ -67,5 +64,3 @@ var Home = {
m.route(document.getElementById('app'), '/', {
'/': Home,
})
-
-refresh()
commit cf2e2be269c5f8c575f18425dd32db2a16af8023
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Jan 25 01:00:31 2017 -0500
add consul task
diff --git a/services/boot/config.go b/services/boot/config.go
new file mode 100644
index 0000000..0eb1c30
--- /dev/null
+++ b/services/boot/config.go
@@ -0,0 +1,33 @@
+package main
+
+type Config struct {
+ // 5 alphanumeric chars. Must be either xx*, yy*, zz*, or
+ // globally unique.
+ SiteID string
+
+ // Hostnames or IP addresses of control hosts. Use at least 3
+ // in production. System functions only when a majority are
+ // alive.
+ ControlHosts []string
+
+ // addr:port to serve web-based setup/monitoring application
+ WebListen string
+
+ UsrDir string
+ DataDir string
+}
+
+func (c *Config) SetDefaults() {
+ if len(c.ControlHosts) == 0 {
+ c.ControlHosts = []string{"127.0.0.1"}
+ }
+ if c.DataDir == "" {
+ c.DataDir = "/var/lib/arvados"
+ }
+ if c.UsrDir == "" {
+ c.DataDir = "/usr/local"
+ }
+ if c.WebListen == "" {
+ c.WebListen = "localhost:8000"
+ }
+}
diff --git a/services/boot/consul_task.go b/services/boot/consul_task.go
new file mode 100644
index 0000000..4de5512
--- /dev/null
+++ b/services/boot/consul_task.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "time"
+
+ "github.com/hashicorp/consul/api"
+)
+
+type consulService struct {
+ supervisor
+}
+
+func (cs *consulService) Init(cfg *Config) {
+ args := []string{
+ "agent",
+ "-server",
+ "-advertise=127.0.0.1",
+ "-data-dir", cfg.DataDir + "/consul",
+ "-bootstrap-expect", fmt.Sprintf("%d", len(cfg.ControlHosts))}
+ cs.supervisor = newSupervisor("consul", "/usr/local/bin/consul", args...)
+}
+
+func (cs *consulService) Children() []task {
+ return nil
+}
+
+func (cs *consulService) ShortName() string {
+ return "consul running"
+}
+
+func (cs *consulService) String() string {
+ return "Ensure consul daemon is supervised & running"
+}
+
+func (cs *consulService) Check() error {
+ consul, err := api.NewClient(api.DefaultConfig())
+ if err != nil {
+ return err
+ }
+ _, err = consul.Catalog().Datacenters()
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (cs *consulService) CanFix() bool {
+ return true
+}
+
+func (cs *consulService) Fix() error {
+ err := cs.supervisor.Start()
+ if err != nil {
+ return err
+ }
+
+ if len(cfg.ControlHosts) > 1 {
+ cmd := exec.Command("/usr/local/bin/consul", append([]string{"join"}, cfg.ControlHosts...)...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ return err
+ }
+ }
+
+ timeout := time.After(10 * time.Second)
+ ticker := time.NewTicker(50 * time.Millisecond)
+ defer ticker.Stop()
+ for cs.Check() != nil {
+ select {
+ case <-ticker.C:
+ case <-timeout:
+ return cs.Check()
+ }
+ }
+ return nil
+}
diff --git a/services/boot/ctl_tasks.go b/services/boot/ctl_tasks.go
index 87ce42f..8737316 100644
--- a/services/boot/ctl_tasks.go
+++ b/services/boot/ctl_tasks.go
@@ -8,4 +8,5 @@ var ctlTasks = []task{
Size: 29079005,
Mode: 0755,
},
+ &consulService{},
}
diff --git a/services/boot/download_task.go b/services/boot/download_task.go
index 508402d..da70678 100644
--- a/services/boot/download_task.go
+++ b/services/boot/download_task.go
@@ -20,6 +20,8 @@ type download struct {
Hash string
}
+func (d *download) Init(cfg *Config) {}
+
func (d *download) Children() []task {
return nil
}
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
index 16b1e38..d1d483e 100644
--- a/services/boot/js/index.js
+++ b/services/boot/js/index.js
@@ -6,49 +6,66 @@ require('./example.js')
var m = require('mithril')
var Stream = require('mithril/stream')
-var checklist = [
- {
- name: 'arvados-boot web gui',
- api: null,
- lastCheck: (new Date()).valueOf(),
- error: Stream(null),
- response: Stream('ok'),
- },
- {
- name: 'arvados-boot web backend',
- api: '/api/ping',
- },
- {
- name: 'arvados-boot fail canary',
- api: '/api/error',
- },
- {
- name: 'arvados control node',
- api: '/api/tasks/ctl',
- },
-]
+var ctl = Stream({Tasks: [], Version: 0})
-checklist.map(function(check) {
- if (!check.api) return
- if (!check.response) check.response = Stream()
- if (!check.error) check.error = Stream()
- m.request({method: 'GET', url: check.api}).then(check.response).catch(check.error)
-})
+refresh.next = null
+refresh.xhr = null
+function refresh() {
+ const timeout = 60
+ if (refresh.xhr !== null) {
+ refresh.xhr.abort()
+ refresh.xhr = null
+ }
+ if (refresh.next !== null)
+ window.clearTimeout(refresh.next)
+ refresh.next = window.setTimeout(refresh, timeout*1000)
+ var version = ctl().Version
+ m.request({
+ method: 'GET',
+ url: '/api/tasks/ctl?timeout='+timeout+'&newerThan='+version,
+ config: function(xhr) { refresh.xhr = xhr },
+ })
+ .then(ctl)
+ .then(function() {
+ if (ctl().Version != version) {
+ // Got a new version -- assume the server is obeying
+ // newerThan, and start listening for the next version
+ // right away.
+ refresh()
+ } else {
+ if (refresh.next !== null)
+ window.clearTimeout(refresh.next)
+ refresh.next = window.setTimeout(refresh, 5000)
+ }
+ })
+}
var Home = {
view: function(vnode) {
- return m('.panel', checklist.map(function(check) {
- return m('div.alert',
- {class: (!check.response() || check.error()) ? 'alert-danger' : 'alert-success'},
- [
- check.name,
- ': ',
- JSON.stringify(check.response()),
- ])
- }))
+ return [
+ m('nav.navbar.navbar-toggleable-md.navbar-inverse.bg-primary',
+ m('a.navbar-brand[href=#]', 'arvados-boot'),
+ m('.collapse.navbar-collapse',
+ m('ul.navbar-nav',
+ m('li.nav-item.active',
+ m('a.nav-link[href=/]', {config: m.route}, 'health', m('span.sr-only', '(current)')))))),
+ m('.x-spacer', {height: '1em'}),
+ m('table.table', {style: {width: '350px'}},
+ m('tbody', ctl().Tasks.map(function(task) {
+ return m('tr', [
+ m('td', task.ShortName),
+ m('td',
+ m('span.badge',
+ {class: task.State == 'OK' ? 'badge-success' : 'badge-danger'},
+ task.State)),
+ ])
+ }))),
+ ]
}
}
m.route(document.getElementById('app'), '/', {
'/': Home,
})
+
+refresh()
diff --git a/services/boot/package.json b/services/boot/package.json
index 0a1640d..ec2d95a 100644
--- a/services/boot/package.json
+++ b/services/boot/package.json
@@ -12,10 +12,12 @@
"webpack": ""
},
"scripts": {
- "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/boot -listen :${PORT:-8000}",
+ "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/boot",
+ "dev-as-root": "WEBPACK_FLAGS=-d go generate && go get ./... && sudo $GOPATH/bin/boot",
"test": "./node_modules/.bin/tap 'js/**/*_test.js'",
"build": "go generate && go get ./...",
- "start": "npm run build && $GOPATH/bin/boot -listen :${PORT:-8000}",
+ "start": "npm run build && $GOPATH/bin/boot",
+ "start-as-root": "npm run build && sudo $GOPATH/bin/boot",
"webpack": "webpack $WEBPACK_FLAGS"
}
}
diff --git a/services/boot/server.go b/services/boot/server.go
index 5edd3c1..f9aad57 100644
--- a/services/boot/server.go
+++ b/services/boot/server.go
@@ -5,17 +5,38 @@ import (
"flag"
"log"
"net/http"
+ "os"
+ "strconv"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/config"
)
+const defaultCfgPath = "/etc/arvados/boot/boot.yml"
+
+var cfg Config
+
func main() {
- listen := flag.String("listen", ":80", "addr:port or :port to listen on")
+ cfgPath := flag.String("config", defaultCfgPath, "`path` to config file")
flag.Parse()
+
+ if err := config.LoadFile(&cfg, *cfgPath); os.IsNotExist(err) && *cfgPath == defaultCfgPath {
+ log.Printf("WARNING: No config file specified or found, starting fresh!")
+ } else if err != nil {
+ log.Fatal(err)
+ }
+ cfg.SetDefaults()
+ go func() {
+ log.Printf("starting server at %s", cfg.WebListen)
+ log.Fatal(http.ListenAndServe(cfg.WebListen, stack(logger, apiOrAssets)))
+ }()
go func() {
- log.Printf("starting server at %s", *listen)
- log.Fatal(http.ListenAndServe(*listen, stack(logger, apiOrAssets)))
+ ticker := time.NewTicker(5 * time.Second)
+ for {
+ runTasks(&cfg, ctlTasks)
+ <-ticker.C
+ }
}()
- go runTasks(ctlTasks)
<-(chan struct{})(nil)
}
@@ -64,6 +85,13 @@ func apiRoutes(http.Handler) http.Handler {
json.NewEncoder(w).Encode(map[string]interface{}{"time": time.Now().UTC()})
})
mux.HandleFunc("/api/tasks/ctl", func(w http.ResponseWriter, r *http.Request) {
+ timeout := time.Minute
+ if v, err := strconv.ParseInt(r.FormValue("timeout"), 10, 64); err == nil {
+ timeout = time.Duration(v) * time.Second
+ }
+ if v, err := strconv.ParseInt(r.FormValue("newerThan"), 10, 64); err == nil {
+ TaskState.Wait(version(v), timeout, r.Context())
+ }
rep, v := report(ctlTasks)
json.NewEncoder(w).Encode(map[string]interface{}{
"Version": v,
diff --git a/services/boot/systemd.go b/services/boot/systemd.go
new file mode 100644
index 0000000..e88ecc3
--- /dev/null
+++ b/services/boot/systemd.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+)
+
+type supervisor interface {
+ Check() (bool, error)
+ Start() error
+}
+
+func newSupervisor(name, cmd string, args ...string) supervisor {
+ return &systemdUnit{
+ name: name,
+ cmd: cmd,
+ args: args,
+ }
+}
+
+type systemdUnit struct {
+ name string
+ cmd string
+ args []string
+}
+
+func (u *systemdUnit) Start() error {
+ cmd := exec.Command("systemd-run", append([]string{"--unit=arvados-"+u.name, u.cmd}, u.args...)...)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("systemd-run: %s", err)
+ }
+ return err
+}
+
+func (u *systemdUnit) Check() (bool, error) {
+ cmd := exec.Command("systemctl", "status", "arvados-"+u.name)
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ switch err.(type) {
+ case *exec.ExitError:
+ return false, nil
+ case nil:
+ return true, nil
+ default:
+ return false, err
+ }
+}
diff --git a/services/boot/task.go b/services/boot/task.go
index 1cc71a7..75c49bd 100644
--- a/services/boot/task.go
+++ b/services/boot/task.go
@@ -1,8 +1,10 @@
package main
import (
+ "context"
"log"
"sync"
+ "time"
)
type taskState string
@@ -19,33 +21,35 @@ type version int64
type taskStateMap struct {
s map[task]taskState
- lock sync.Mutex
+ cond *sync.Cond
version version
}
var TaskState = taskStateMap{
- s: make(map[task]taskState),
+ s: make(map[task]taskState),
+ cond: sync.NewCond(&sync.Mutex{}),
}
func (m *taskStateMap) Set(t task, s taskState) {
- m.lock.Lock()
- defer m.lock.Unlock()
+ m.cond.L.Lock()
+ defer m.cond.L.Unlock()
if old, ok := m.s[t]; ok && old == s {
return
}
m.s[t] = s
m.version++
+ m.cond.Broadcast()
}
func (m *taskStateMap) Version() version {
- m.lock.Lock()
- defer m.lock.Unlock()
+ m.cond.L.Lock()
+ defer m.cond.L.Unlock()
return m.version
}
func (m *taskStateMap) Get(t task) taskState {
- m.lock.Lock()
- defer m.lock.Unlock()
+ m.cond.L.Lock()
+ defer m.cond.L.Unlock()
if s, ok := m.s[t]; ok {
return s
} else {
@@ -60,6 +64,28 @@ type repEnt struct {
Children []repEnt
}
+func (m *taskStateMap) Wait(v version, t time.Duration, ctx context.Context) bool {
+ ready := make(chan struct{})
+ var done bool
+ go func() {
+ m.cond.L.Lock()
+ defer m.cond.L.Unlock()
+ for v == m.version && !done {
+ m.cond.Wait()
+ }
+ close(ready)
+ }()
+ select {
+ case <-ready:
+ return true
+ case <-ctx.Done():
+ case <-time.After(t):
+ }
+ done = true
+ m.cond.Broadcast()
+ return false
+}
+
func report(tasks []task) ([]repEnt, version) {
v := TaskState.Version()
if len(tasks) == 0 {
@@ -78,9 +104,14 @@ func report(tasks []task) ([]repEnt, version) {
return rep, v
}
-func runTasks(tasks []task) {
+func runTasks(cfg *Config, tasks []task) {
for _, t := range tasks {
- TaskState.Set(t, StateChecking)
+ t.Init(cfg)
+ }
+ for _, t := range tasks {
+ if TaskState.Get(t) == taskState("") {
+ TaskState.Set(t, StateChecking)
+ }
err := t.Check()
if err == nil {
log.Printf("%s: OK", t)
@@ -93,6 +124,7 @@ func runTasks(tasks []task) {
TaskState.Set(t, StateFailed)
continue
}
+ TaskState.Set(t, StateFixing)
if err = t.Fix(); err != nil {
log.Printf("%s: can't fix: %s", t, err)
TaskState.Set(t, StateFailed)
@@ -109,6 +141,7 @@ func runTasks(tasks []task) {
}
type task interface {
+ Init(*Config)
ShortName() string
String() string
Check() error
commit e95856da1e600b7beef1526c3554b90675569c29
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Jan 23 00:56:43 2017 -0500
report state of all tasks
diff --git a/services/boot/download_task.go b/services/boot/download_task.go
index 3d1bc3d..508402d 100644
--- a/services/boot/download_task.go
+++ b/services/boot/download_task.go
@@ -20,6 +20,14 @@ type download struct {
Hash string
}
+func (d *download) Children() []task {
+ return nil
+}
+
+func (d *download) ShortName() string {
+ return d.Dest
+}
+
func (d *download) String() string {
return fmt.Sprintf("Download %q from %q", d.Dest, d.URL)
}
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
index 9e8452f..16b1e38 100644
--- a/services/boot/js/index.js
+++ b/services/boot/js/index.js
@@ -22,6 +22,10 @@ var checklist = [
name: 'arvados-boot fail canary',
api: '/api/error',
},
+ {
+ name: 'arvados control node',
+ api: '/api/tasks/ctl',
+ },
]
checklist.map(function(check) {
diff --git a/services/boot/server.go b/services/boot/server.go
index 3a9825c..5edd3c1 100644
--- a/services/boot/server.go
+++ b/services/boot/server.go
@@ -63,6 +63,13 @@ func apiRoutes(http.Handler) http.Handler {
mux.HandleFunc("/api/ping", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{"time": time.Now().UTC()})
})
+ mux.HandleFunc("/api/tasks/ctl", func(w http.ResponseWriter, r *http.Request) {
+ rep, v := report(ctlTasks)
+ json.NewEncoder(w).Encode(map[string]interface{}{
+ "Version": v,
+ "Tasks": rep,
+ })
+ })
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
json.NewEncoder(w).Encode(map[string]string{"error": "not found"})
diff --git a/services/boot/task.go b/services/boot/task.go
index 2f227a9..1cc71a7 100644
--- a/services/boot/task.go
+++ b/services/boot/task.go
@@ -2,36 +2,117 @@ package main
import (
"log"
+ "sync"
)
+type taskState string
+
+const (
+ StateUnchecked taskState = "Unchecked"
+ StateChecking = "Checking"
+ StateFixing = "Fixing"
+ StateFailed = "Failed"
+ StateOK = "OK"
+)
+
+type version int64
+
+type taskStateMap struct {
+ s map[task]taskState
+ lock sync.Mutex
+ version version
+}
+
+var TaskState = taskStateMap{
+ s: make(map[task]taskState),
+}
+
+func (m *taskStateMap) Set(t task, s taskState) {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+ if old, ok := m.s[t]; ok && old == s {
+ return
+ }
+ m.s[t] = s
+ m.version++
+}
+
+func (m *taskStateMap) Version() version {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+ return m.version
+}
+
+func (m *taskStateMap) Get(t task) taskState {
+ m.lock.Lock()
+ defer m.lock.Unlock()
+ if s, ok := m.s[t]; ok {
+ return s
+ } else {
+ return StateUnchecked
+ }
+}
+
+type repEnt struct {
+ ShortName string
+ Description string
+ State taskState
+ Children []repEnt
+}
+
+func report(tasks []task) ([]repEnt, version) {
+ v := TaskState.Version()
+ if len(tasks) == 0 {
+ return nil, v
+ }
+ var rep []repEnt
+ for _, t := range tasks {
+ crep, _ := report(t.Children())
+ rep = append(rep, repEnt{
+ ShortName: t.ShortName(),
+ Description: t.String(),
+ State: TaskState.Get(t),
+ Children: crep,
+ })
+ }
+ return rep, v
+}
+
func runTasks(tasks []task) {
for _, t := range tasks {
+ TaskState.Set(t, StateChecking)
err := t.Check()
if err == nil {
log.Printf("%s: OK", t)
+ TaskState.Set(t, StateOK)
continue
}
log.Printf("%s: %s", t, err)
if !t.CanFix() {
log.Printf("%s: can't fix")
+ TaskState.Set(t, StateFailed)
continue
}
if err = t.Fix(); err != nil {
log.Printf("%s: can't fix: %s", t, err)
+ TaskState.Set(t, StateFailed)
continue
}
if err = t.Check(); err != nil {
log.Printf("%s: fixed, but still broken?!: %s", t, err)
+ TaskState.Set(t, StateFailed)
continue
}
log.Printf("%s: OK", t)
+ TaskState.Set(t, StateOK)
}
}
type task interface {
+ ShortName() string
String() string
Check() error
CanFix() bool
Fix() error
+ Children() []task
}
-
commit 5e8fb0fa926686606706be32cc958131eab0ff7d
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Jan 22 20:42:55 2017 -0500
add "download consul binary" task
diff --git a/services/boot/ctl_tasks.go b/services/boot/ctl_tasks.go
new file mode 100644
index 0000000..87ce42f
--- /dev/null
+++ b/services/boot/ctl_tasks.go
@@ -0,0 +1,11 @@
+package main
+
+// tasks to run on a controller node
+var ctlTasks = []task{
+ &download{
+ URL: "https://releases.hashicorp.com/consul/0.7.2/consul_0.7.2_linux_amd64.zip",
+ Dest: "/usr/local/bin/consul",
+ Size: 29079005,
+ Mode: 0755,
+ },
+}
diff --git a/services/boot/download_task.go b/services/boot/download_task.go
new file mode 100644
index 0000000..3d1bc3d
--- /dev/null
+++ b/services/boot/download_task.go
@@ -0,0 +1,120 @@
+package main
+
+import (
+ "archive/zip"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+)
+
+type download struct {
+ URL string
+ Dest string
+ Size int64
+ Mode os.FileMode
+ Hash string
+}
+
+func (d *download) String() string {
+ return fmt.Sprintf("Download %q from %q", d.Dest, d.URL)
+}
+
+func (d *download) Check() error {
+ fi, err := os.Stat(d.Dest)
+ if err != nil {
+ return err
+ }
+ if d.Size > 0 && fi.Size() != d.Size {
+ return fmt.Errorf("Size mismatch: %q is %d bytes, expected %d", d.Dest, fi.Size(), d.Size)
+ }
+ if d.Mode > 0 && fi.Mode() != d.Mode {
+ return fmt.Errorf("Mode mismatch: %q is %s, expected %s", d.Dest, fi.Mode(), d.Mode)
+ }
+ return nil
+}
+
+func (d *download) CanFix() bool {
+ return true
+}
+
+func (d *download) Fix() error {
+ out, err := ioutil.TempFile(path.Dir(d.Dest), path.Base(d.Dest))
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if out != nil {
+ os.Remove(out.Name())
+ out.Close()
+ }
+ }()
+
+ resp, err := http.Get(d.URL)
+ if err != nil {
+ return err
+ }
+ n, err := io.Copy(out, resp.Body)
+ resp.Body.Close()
+ if err != nil {
+ return err
+ }
+
+ if strings.HasSuffix(d.URL, ".zip") && !strings.HasSuffix(d.Dest, ".zip") {
+ r, err := zip.NewReader(out, n)
+ if err != nil {
+ return err
+ }
+ defer os.Remove(out.Name())
+ out = nil
+
+ found := false
+ for _, f := range r.File {
+ if !strings.HasSuffix(d.Dest, "/"+f.Name) {
+ continue
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer rc.Close()
+
+ out, err = ioutil.TempFile(path.Dir(d.Dest), path.Base(d.Dest))
+ if err != nil {
+ return err
+ }
+
+ n, err = io.Copy(out, rc)
+ if err != nil {
+ return err
+ }
+ found = true
+ break
+ }
+ if !found {
+ return fmt.Errorf("File not found in archive")
+ }
+ }
+
+ if d.Size > 0 && d.Size != n {
+ return fmt.Errorf("Size mismatch: got %d bytes, expected %d", n, d.Size)
+ } else if d.Size == 0 {
+ log.Printf("%s: size was %d", d, n)
+ }
+ if err = out.Close(); err != nil {
+ return err
+ }
+ if err = os.Chmod(out.Name(), d.Mode); err != nil {
+ return err
+ }
+ err = os.Rename(out.Name(), d.Dest)
+ if err == nil {
+ // skip deferred os.Remove(out.Name())
+ out = nil
+ }
+ return err
+}
diff --git a/services/boot/server.go b/services/boot/server.go
index 61adb07..3a9825c 100644
--- a/services/boot/server.go
+++ b/services/boot/server.go
@@ -11,8 +11,12 @@ import (
func main() {
listen := flag.String("listen", ":80", "addr:port or :port to listen on")
flag.Parse()
- log.Printf("starting server at %s", *listen)
- log.Fatal(http.ListenAndServe(*listen, stack(logger, apiOrAssets)))
+ go func() {
+ log.Printf("starting server at %s", *listen)
+ log.Fatal(http.ListenAndServe(*listen, stack(logger, apiOrAssets)))
+ }()
+ go runTasks(ctlTasks)
+ <-(chan struct{})(nil)
}
type middleware func(http.Handler) http.Handler
diff --git a/services/boot/task.go b/services/boot/task.go
new file mode 100644
index 0000000..2f227a9
--- /dev/null
+++ b/services/boot/task.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+ "log"
+)
+
+func runTasks(tasks []task) {
+ for _, t := range tasks {
+ err := t.Check()
+ if err == nil {
+ log.Printf("%s: OK", t)
+ continue
+ }
+ log.Printf("%s: %s", t, err)
+ if !t.CanFix() {
+ log.Printf("%s: can't fix")
+ continue
+ }
+ if err = t.Fix(); err != nil {
+ log.Printf("%s: can't fix: %s", t, err)
+ continue
+ }
+ if err = t.Check(); err != nil {
+ log.Printf("%s: fixed, but still broken?!: %s", t, err)
+ continue
+ }
+ log.Printf("%s: OK", t)
+ }
+}
+
+type task interface {
+ String() string
+ Check() error
+ CanFix() bool
+ Fix() error
+}
+
commit 9202497767fc6b06444032c43748d14e17351a12
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 20 23:27:01 2017 -0500
add to arvados build/test scripts
diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile
index 4fcd640..8035ef4 100644
--- a/build/package-build-dockerfiles/centos7/Dockerfile
+++ b/build/package-build-dockerfiles/centos7/Dockerfile
@@ -4,9 +4,12 @@ MAINTAINER Brett Smith <brett at curoverse.com>
# Install build dependencies provided in base distribution
RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
+# Node.js
+RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - && yum install -y nodejs
+
# Install golang binary
ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+RUN ln -s /usr/local/go/bin/go* /usr/local/bin/
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
diff --git a/build/package-build-dockerfiles/debian8/Dockerfile b/build/package-build-dockerfiles/debian8/Dockerfile
index 977cd24..f014306 100644
--- a/build/package-build-dockerfiles/debian8/Dockerfile
+++ b/build/package-build-dockerfiles/debian8/Dockerfile
@@ -2,7 +2,10 @@ FROM debian:jessie
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip && apt-get clean
+
+# Node.js
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash && apt-get install -q -y nodejs && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
@@ -14,7 +17,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
# Install golang binary
ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+RUN ln -s /usr/local/go/bin/go* /usr/local/bin/
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
diff --git a/build/package-build-dockerfiles/ubuntu1204/Dockerfile b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
index b0dd906..390b4e2 100644
--- a/build/package-build-dockerfiles/ubuntu1204/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1204/Dockerfile
@@ -2,7 +2,10 @@ FROM ubuntu:precise
MAINTAINER Ward Vandewege <ward at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip build-essential unzip && apt-get clean
+
+# Node.js
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash && apt-get install -q -y nodejs && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
@@ -14,7 +17,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
# Install golang binary
ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+RUN ln -s /usr/local/go/bin/go* /usr/local/bin/
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1204"]
diff --git a/build/package-build-dockerfiles/ubuntu1404/Dockerfile b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
index 91c5e5b..2b1fb1f 100644
--- a/build/package-build-dockerfiles/ubuntu1404/Dockerfile
+++ b/build/package-build-dockerfiles/ubuntu1404/Dockerfile
@@ -2,7 +2,10 @@ FROM ubuntu:trusty
MAINTAINER Brett Smith <brett at curoverse.com>
# Install dependencies and set up system.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip && apt-get clean
+
+# Node.js
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash && apt-get install -q -y nodejs && apt-get clean
# Install RVM
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
@@ -14,7 +17,7 @@ RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
# Install golang binary
ADD generated/go1.7.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+RUN ln -s /usr/local/go/bin/go* /usr/local/bin/
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
diff --git a/build/run-build-packages-all-targets.sh b/build/run-build-packages-all-targets.sh
index a4dd9a6..2d16147 100755
--- a/build/run-build-packages-all-targets.sh
+++ b/build/run-build-packages-all-targets.sh
@@ -38,7 +38,7 @@ fi
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,only-build: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -66,6 +66,9 @@ while [ $# -gt 0 ]; do
--test-packages)
TEST_PACKAGES="--test-packages"
;;
+ --only-build)
+ ONLY_BUILD="$1 $2"; shift
+ ;;
--only-test)
ONLY_TEST="$1 $2"; shift
;;
@@ -84,7 +87,7 @@ cd $(dirname $0)
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST $ONLY_BUILD ; then
true
else
FINAL_EXITCODE=$?
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 6a1ec9c..69cae05 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -128,6 +128,7 @@ popd
if test -z "$packages" ; then
packages="arvados-api-server
+ arvados-boot
arvados-docker-cleaner
arvados-git-httpd
arvados-node-manager
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7840b3c..88336fd 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -337,6 +337,8 @@ package_go_binary sdk/go/crunchrunner crunchrunner \
"Crunchrunner executes a command inside a container and uploads the output"
package_go_binary services/arv-git-httpd arvados-git-httpd \
"Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/boot arvados-boot \
+ "Coordinate Arvados system services"
package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
"Dispatch Crunch containers on the local system"
package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
diff --git a/build/run-library.sh b/build/run-library.sh
index a13470b..8a3fb39 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -112,6 +112,8 @@ package_go_binary() {
fi
fi
+ go generate || return 1
+
cd $WORKSPACE/packages/$TARGET
test_package_presence $prog $version go
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4b6c813..249783e 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -69,6 +69,7 @@ apps/workbench_profile
doc
services/api
services/arv-git-httpd
+services/boot
services/crunchstat
services/dockercleaner
services/fuse
@@ -543,22 +544,24 @@ do_test_once() {
then
covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ gopkgpath="git.curoverse.com/arvados.git/$1"
# We do "go get -t" here to catch compilation errors
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
# compilation errors.
- go get -t "git.curoverse.com/arvados.git/$1" || return 1
- cd "$WORKSPACE/$1" || return 1
+ cd "$GOPATH/src/${gopkgpath}" || return 1
+ go get -t . || return 1
+ go generate || return 1
gofmt -e -d . | egrep . && result=1
if [[ -n "${testargs[$1]}" ]]
then
# "go test -check.vv giturl" doesn't work, but this
# does:
- cd "$WORKSPACE/$1" && go test ${short:+-short} ${testargs[$1]}
+ go test ${short:+-short} ${testargs[$1]} .
else
# The above form gets verbose even when testargs is
# empty, so use this form in such cases:
- go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+ go test ${short:+-short} ${coverflags[@]} .
fi
result=${result:-$?}
if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]]
@@ -590,6 +593,9 @@ do_test_once() {
else
"test_$1"
fi
+ if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then
+ cd "${WORKSPACE}/${1}" && npm test || result=1
+ fi
result=${result:-$?}
checkexit $result "$1 tests"
title "End of $1 tests (`timer`)"
@@ -607,9 +613,15 @@ do_install() {
do_install_once() {
title "Running $1 install"
timer_reset
+ cd "${WORKSPACE}/${1}" || return 1
+ if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then
+ npm install || return 1
+ fi
if [[ "$2" == "go" ]]
then
- go get -t "git.curoverse.com/arvados.git/$1"
+ go get -d -t "git.curoverse.com/arvados.git/$1" \
+ && go generate \
+ && go get "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
# $3 can name a path directory for us to use, including trailing
@@ -624,8 +636,7 @@ do_install_once() {
# install" ensures that the dependencies are met, the second "pip
# install" ensures that we've actually installed the local package
# we just built.
- cd "$WORKSPACE/$1" \
- && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
&& cd "$WORKSPACE" \
&& "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
&& "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
@@ -786,6 +797,7 @@ gostuff=(
services/crunch-dispatch-slurm
services/crunch-run
services/ws
+ services/boot
tools/keep-block-check
tools/keep-exercise
tools/keep-rsync
diff --git a/services/boot/generate.go b/services/boot/generate.go
index d9353e1..88e80e6 100644
--- a/services/boot/generate.go
+++ b/services/boot/generate.go
@@ -4,6 +4,7 @@
//go:generate sh -c "if [ -e bindata.tmp ]; then rm -r bindata.tmp; fi && mkdir bindata.tmp"
//go:generate sh -c "npm run webpack ${WEBPACK_FLAGS:-p}"
//go:generate sh -c "cp -rpL static/* bindata.tmp/"
-//go:generate go-bindata-assetfs -nometadata bindata.tmp/...
+//go:generate sh -c "PATH=${GOPATH}/bin:${PATH} go-bindata-assetfs -nometadata bindata.tmp/..."
+//go:generate gofmt -w bindata_assetfs.go
package main
commit c34b698a734537a9819289e3da76f8a97a32a7d9
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 20 17:03:24 2017 -0500
remove bindata, update name, add new bootstrap + mithril
diff --git a/services/boot/generate.go b/services/boot/generate.go
index b3ce61b..d9353e1 100644
--- a/services/boot/generate.go
+++ b/services/boot/generate.go
@@ -1,7 +1,7 @@
//go:generate sh -c "which go-bindata 2>&1 >/dev/null || go get github.com/jteeuwen/go-bindata/..."
//go:generate sh -c "which go-bindata-assetfs 2>&1 >/dev/null || go get github.com/elazarl/go-bindata-assetfs/..."
//go:generate sh -c "[ -d node_modules ] || npm install"
-//go:generate sh -c "rm -r bindata.tmp && mkdir bindata.tmp"
+//go:generate sh -c "if [ -e bindata.tmp ]; then rm -r bindata.tmp; fi && mkdir bindata.tmp"
//go:generate sh -c "npm run webpack ${WEBPACK_FLAGS:-p}"
//go:generate sh -c "cp -rpL static/* bindata.tmp/"
//go:generate go-bindata-assetfs -nometadata bindata.tmp/...
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
index 5915b92..9e8452f 100644
--- a/services/boot/js/index.js
+++ b/services/boot/js/index.js
@@ -1,2 +1,50 @@
// application entry point
+window.jQuery = require('jquery')
+window.Tether = require('tether')
+require('bootstrap')
require('./example.js')
+var m = require('mithril')
+var Stream = require('mithril/stream')
+
+var checklist = [
+ {
+ name: 'arvados-boot web gui',
+ api: null,
+ lastCheck: (new Date()).valueOf(),
+ error: Stream(null),
+ response: Stream('ok'),
+ },
+ {
+ name: 'arvados-boot web backend',
+ api: '/api/ping',
+ },
+ {
+ name: 'arvados-boot fail canary',
+ api: '/api/error',
+ },
+]
+
+checklist.map(function(check) {
+ if (!check.api) return
+ if (!check.response) check.response = Stream()
+ if (!check.error) check.error = Stream()
+ m.request({method: 'GET', url: check.api}).then(check.response).catch(check.error)
+})
+
+var Home = {
+ view: function(vnode) {
+ return m('.panel', checklist.map(function(check) {
+ return m('div.alert',
+ {class: (!check.response() || check.error()) ? 'alert-danger' : 'alert-success'},
+ [
+ check.name,
+ ': ',
+ JSON.stringify(check.response()),
+ ])
+ }))
+ }
+}
+
+m.route(document.getElementById('app'), '/', {
+ '/': Home,
+})
diff --git a/services/boot/package.json b/services/boot/package.json
index e9f6101..0a1640d 100644
--- a/services/boot/package.json
+++ b/services/boot/package.json
@@ -1,20 +1,21 @@
{
- "name": "gowebapp",
+ "name": "boot",
"version": "0.1.0",
"license": "Apache-2.0",
"dependencies": {},
"devDependencies": {
- "bootstrap": "^3.3.7",
+ "bootstrap": "^4.0.0-alpha.6",
"check-dependencies": "",
+ "mithril": "^1.0.0-rc.8",
"tap": "^9.0.3",
"tape": "^4.6.3",
"webpack": ""
},
"scripts": {
- "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/gowebapp -listen :${PORT:-8000}",
+ "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/boot -listen :${PORT:-8000}",
"test": "./node_modules/.bin/tap 'js/**/*_test.js'",
"build": "go generate && go get ./...",
- "start": "npm run build && $GOPATH/bin/gowebapp -listen :${PORT:-8000}",
+ "start": "npm run build && $GOPATH/bin/boot -listen :${PORT:-8000}",
"webpack": "webpack $WEBPACK_FLAGS"
}
}
diff --git a/services/boot/static/css/bootstrap-theme.min.css b/services/boot/static/css/bootstrap-theme.min.css
deleted file mode 120000
index 827ff2f..0000000
--- a/services/boot/static/css/bootstrap-theme.min.css
+++ /dev/null
@@ -1 +0,0 @@
-../../node_modules/bootstrap/dist/css/bootstrap-theme.min.css
\ No newline at end of file
diff --git a/services/boot/static/css/bootstrap-theme.min.css.map b/services/boot/static/css/bootstrap-theme.min.css.map
deleted file mode 120000
index 9b7dbcb..0000000
--- a/services/boot/static/css/bootstrap-theme.min.css.map
+++ /dev/null
@@ -1 +0,0 @@
-../../node_modules/bootstrap/dist/css/bootstrap-theme.min.css.map
\ No newline at end of file
diff --git a/services/boot/static/fonts b/services/boot/static/fonts
deleted file mode 120000
index 0e6e04a..0000000
--- a/services/boot/static/fonts
+++ /dev/null
@@ -1 +0,0 @@
-../node_modules/bootstrap/dist/fonts
\ No newline at end of file
diff --git a/services/boot/static/index.html b/services/boot/static/index.html
index 5387e21..7bc78fe 100644
--- a/services/boot/static/index.html
+++ b/services/boot/static/index.html
@@ -1,14 +1,12 @@
-<!doctype html>
-<html>
+<!DOCTYPE html>
+<html lang="en">
<head>
+ <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <link rel="stylesheet" href="css/bootstrap.min.css" />
- <link rel="stylesheet" href="css/bootstrap-theme.min.css" />
+ <link rel="stylesheet" href="css/bootstrap.min.css" integrity="sha384-rwoIResjU2yc3z8GV/NPeZWAv56rSmLldC3R/AZzGRnGxQQKnKkoFVhFQhNUwEyJ" crossorigin="anonymous">
</head>
<body>
- <div class="container-fluid">
- <a class="btn btn-default" href="./">btn-default</a>
- </div>
+ <div id="app" class="container-fluid"></div>
+ <script src="js.js"></script>
</body>
- <script src="js.js"></script>
</html>
commit 92a84c2d51b69feaa0a6c1a3862e55cce1ddb839
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 20 13:20:24 2017 -0500
update readme
diff --git a/services/boot/README.md b/services/boot/README.md
index 789a608..2122378 100644
--- a/services/boot/README.md
+++ b/services/boot/README.md
@@ -1,6 +1,6 @@
-# gowebapp
+# arvados-boot
-A basic skeleton web application server. Just add HTML, client-side JavaScript code, and server-side APIs.
+Coordinates Arvados system services.
Strategy:
* In development, use npm to install JavaScript libraries.
@@ -53,21 +53,6 @@ module.exports = {
...
```
-## generate before commit
-
-To make your project `go get`able, run `go generate` before committing. This updates `bindata_assetfs.go`. Consider doing this in `.git/hooks/pre-commit` in case you forget.
-
-If you don't need `go get` to work, and you prefer to keep generated files out of your source tree, you can:
-
-```sh
-git rm bindata_assetfs.go
-echo bindata_assetfs.go >>.gitignore
-git add .gitignore
-git commit -m 'remove generated data'
-```
-
-In this case, your build pipeline must run `go generate` before `go build`.
-
## run dev-mode server
This runs webpack, updates bindata_assetfs.go with the new filesystem, builds a new Go binary, and runs it:
@@ -113,8 +98,3 @@ The server binary will be installed to `$GOPATH/bin/`.
```sh
npm start
```
-
-## TODO
-
-* live dev mode with fsnotify and `webpack --watch -d`
-* etags
commit 0ee4aa669dc0dfb520412da5673ef8195a251f64
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 20 13:16:22 2017 -0500
add gowebapp as conductor
diff --git a/services/boot/.gitignore b/services/boot/.gitignore
new file mode 100644
index 0000000..d08895d
--- /dev/null
+++ b/services/boot/.gitignore
@@ -0,0 +1,4 @@
+*~
+bindata.tmp
+node_modules
+bindata_assetfs.go
diff --git a/services/boot/README.md b/services/boot/README.md
new file mode 100644
index 0000000..789a608
--- /dev/null
+++ b/services/boot/README.md
@@ -0,0 +1,120 @@
+# gowebapp
+
+A basic skeleton web application server. Just add HTML, client-side JavaScript code, and server-side APIs.
+
+Strategy:
+* In development, use npm to install JavaScript libraries.
+* At build time, use webpack on nodejs to compile JavaScript assets.
+* Deploy with a single Go binary -- no nodejs, no asset files.
+
+## dev/build dependencies
+
+Go:
+
+```
+curl https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz \
+ | sudo tar -C /usr/local -xzf - \
+ && (cd /usr/local/bin && sudo ln -s ../go/bin/* .)
+```
+
+nodejs:
+
+```sh
+curl -sL https://deb.nodesource.com/setup_6.x | sudo bash -
+sudo apt-get install nodejs
+```
+
+## add/edit static files
+
+Everything in the `static` directory will be served at `/`.
+
+```sh
+echo foo > static/foo.txt
+# http://webapp/foo.txt
+```
+
+## add/edit javascript files
+
+A webpack will be built using the entry point `js/index.js`, and served at `/js.js`.
+
+```sh
+echo 'function foo() { console.log("foo") }' > js/foo.js
+echo 'require("./foo"); foo()' > js/index.js
+```
+
+The default entry point and published location can be changed by editing `webpack.config.js`. For example, to build separate packs from `js/` and `js-admin/` source directories and serve them at `/user.js` and `/admin.js`:
+
+```javascript
+module.exports = {
+ entry: {
+ admin: './js-admin',
+ user: './js'
+ },
+ ...
+```
+
+## generate before commit
+
+To make your project `go get`able, run `go generate` before committing. This updates `bindata_assetfs.go`. Consider doing this in `.git/hooks/pre-commit` in case you forget.
+
+If you don't need `go get` to work, and you prefer to keep generated files out of your source tree, you can:
+
+```sh
+git rm bindata_assetfs.go
+echo bindata_assetfs.go >>.gitignore
+git add .gitignore
+git commit -m 'remove generated data'
+```
+
+In this case, your build pipeline must run `go generate` before `go build`.
+
+## run dev-mode server
+
+This runs webpack, updates bindata_assetfs.go with the new filesystem, builds a new Go binary, and runs it:
+
+```sh
+npm run dev
+```
+
+To use a port other than the default 8000:
+
+```sh
+PORT=8888 npm run dev
+```
+
+In dev mode, source maps are served, and JS is not minified.
+
+After changing any source code (including static content), `^C` and run `npm run dev` again.
+
+## run tests
+
+Use nodejs to run JavaScript unit tests in `js/**/*_test.js` (see `js/example_test.js`).
+
+```sh
+npm test
+```
+
+Run Go tests the usual way.
+
+```sh
+go test ./...
+```
+
+## build production-mode server
+
+```sh
+npm build
+```
+
+The server binary will be installed to `$GOPATH/bin/`.
+
+## build & run production-mode server
+
+```sh
+npm start
+```
+
+## TODO
+
+* live dev mode with fsnotify and `webpack --watch -d`
+* etags
diff --git a/services/boot/generate.go b/services/boot/generate.go
new file mode 100644
index 0000000..b3ce61b
--- /dev/null
+++ b/services/boot/generate.go
@@ -0,0 +1,9 @@
+//go:generate sh -c "which go-bindata 2>&1 >/dev/null || go get github.com/jteeuwen/go-bindata/..."
+//go:generate sh -c "which go-bindata-assetfs 2>&1 >/dev/null || go get github.com/elazarl/go-bindata-assetfs/..."
+//go:generate sh -c "[ -d node_modules ] || npm install"
+//go:generate sh -c "rm -r bindata.tmp && mkdir bindata.tmp"
+//go:generate sh -c "npm run webpack ${WEBPACK_FLAGS:-p}"
+//go:generate sh -c "cp -rpL static/* bindata.tmp/"
+//go:generate go-bindata-assetfs -nometadata bindata.tmp/...
+
+package main
diff --git a/services/boot/js/example.js b/services/boot/js/example.js
new file mode 100644
index 0000000..c6c1a3c
--- /dev/null
+++ b/services/boot/js/example.js
@@ -0,0 +1,4 @@
+// example
+module.exports = function example() {
+ return 42
+}
diff --git a/services/boot/js/example_test.js b/services/boot/js/example_test.js
new file mode 100644
index 0000000..2137cb7
--- /dev/null
+++ b/services/boot/js/example_test.js
@@ -0,0 +1,6 @@
+test = require('tape')
+example = require('./example')
+test('example is 42', function(t) {
+ t.equal(42, example())
+ t.end()
+})
diff --git a/services/boot/js/index.js b/services/boot/js/index.js
new file mode 100644
index 0000000..5915b92
--- /dev/null
+++ b/services/boot/js/index.js
@@ -0,0 +1,2 @@
+// application entry point
+require('./example.js')
diff --git a/services/boot/package.json b/services/boot/package.json
new file mode 100644
index 0000000..e9f6101
--- /dev/null
+++ b/services/boot/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "gowebapp",
+ "version": "0.1.0",
+ "license": "Apache-2.0",
+ "dependencies": {},
+ "devDependencies": {
+ "bootstrap": "^3.3.7",
+ "check-dependencies": "",
+ "tap": "^9.0.3",
+ "tape": "^4.6.3",
+ "webpack": ""
+ },
+ "scripts": {
+ "dev": "WEBPACK_FLAGS=-d go generate && go get ./... && $GOPATH/bin/gowebapp -listen :${PORT:-8000}",
+ "test": "./node_modules/.bin/tap 'js/**/*_test.js'",
+ "build": "go generate && go get ./...",
+ "start": "npm run build && $GOPATH/bin/gowebapp -listen :${PORT:-8000}",
+ "webpack": "webpack $WEBPACK_FLAGS"
+ }
+}
diff --git a/services/boot/server.go b/services/boot/server.go
new file mode 100644
index 0000000..61adb07
--- /dev/null
+++ b/services/boot/server.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "log"
+ "net/http"
+ "time"
+)
+
+func main() {
+ listen := flag.String("listen", ":80", "addr:port or :port to listen on")
+ flag.Parse()
+ log.Printf("starting server at %s", *listen)
+ log.Fatal(http.ListenAndServe(*listen, stack(logger, apiOrAssets)))
+}
+
+type middleware func(http.Handler) http.Handler
+
+var notFound = http.NotFoundHandler()
+
+// returns a handler that implements a stack of middlewares.
+func stack(m ...middleware) http.Handler {
+ if len(m) == 0 {
+ return notFound
+ }
+ return m[0](stack(m[1:]...))
+}
+
+// logs each request.
+func logger(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ t := time.Now()
+ next.ServeHTTP(w, r)
+ log.Printf("%.6f %q %q %q", time.Since(t).Seconds(), r.RemoteAddr, r.Method, r.URL.Path)
+ })
+}
+
+// dispatches /api/ to the API stack, everything else to the static
+// assets stack.
+func apiOrAssets(next http.Handler) http.Handler {
+ mux := http.NewServeMux()
+ mux.Handle("/api/", stack(apiHeaders, apiRoutes))
+ mux.Handle("/", http.FileServer(assetFS()))
+ return mux
+}
+
+// adds response headers suitable for API responses
+func apiHeaders(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ next.ServeHTTP(w, r)
+ })
+}
+
+// dispatches API routes
+func apiRoutes(http.Handler) http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/ping", func(w http.ResponseWriter, r *http.Request) {
+ json.NewEncoder(w).Encode(map[string]interface{}{"time": time.Now().UTC()})
+ })
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ json.NewEncoder(w).Encode(map[string]string{"error": "not found"})
+ })
+ return mux
+}
diff --git a/services/boot/static/css/bootstrap-theme.min.css b/services/boot/static/css/bootstrap-theme.min.css
new file mode 120000
index 0000000..827ff2f
--- /dev/null
+++ b/services/boot/static/css/bootstrap-theme.min.css
@@ -0,0 +1 @@
+../../node_modules/bootstrap/dist/css/bootstrap-theme.min.css
\ No newline at end of file
diff --git a/services/boot/static/css/bootstrap-theme.min.css.map b/services/boot/static/css/bootstrap-theme.min.css.map
new file mode 120000
index 0000000..9b7dbcb
--- /dev/null
+++ b/services/boot/static/css/bootstrap-theme.min.css.map
@@ -0,0 +1 @@
+../../node_modules/bootstrap/dist/css/bootstrap-theme.min.css.map
\ No newline at end of file
diff --git a/services/boot/static/css/bootstrap.min.css b/services/boot/static/css/bootstrap.min.css
new file mode 120000
index 0000000..93c3bac
--- /dev/null
+++ b/services/boot/static/css/bootstrap.min.css
@@ -0,0 +1 @@
+../../node_modules/bootstrap/dist/css/bootstrap.min.css
\ No newline at end of file
diff --git a/services/boot/static/css/bootstrap.min.css.map b/services/boot/static/css/bootstrap.min.css.map
new file mode 120000
index 0000000..7a98d20
--- /dev/null
+++ b/services/boot/static/css/bootstrap.min.css.map
@@ -0,0 +1 @@
+../../node_modules/bootstrap/dist/css/bootstrap.min.css.map
\ No newline at end of file
diff --git a/services/boot/static/fonts b/services/boot/static/fonts
new file mode 120000
index 0000000..0e6e04a
--- /dev/null
+++ b/services/boot/static/fonts
@@ -0,0 +1 @@
+../node_modules/bootstrap/dist/fonts
\ No newline at end of file
diff --git a/services/boot/static/index.html b/services/boot/static/index.html
new file mode 100644
index 0000000..5387e21
--- /dev/null
+++ b/services/boot/static/index.html
@@ -0,0 +1,14 @@
+<!doctype html>
+<html>
+ <head>
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel="stylesheet" href="css/bootstrap.min.css" />
+ <link rel="stylesheet" href="css/bootstrap-theme.min.css" />
+ </head>
+ <body>
+ <div class="container-fluid">
+ <a class="btn btn-default" href="./">btn-default</a>
+ </div>
+ </body>
+ <script src="js.js"></script>
+</html>
diff --git a/services/boot/webpack.config.js b/services/boot/webpack.config.js
new file mode 100644
index 0000000..836afea
--- /dev/null
+++ b/services/boot/webpack.config.js
@@ -0,0 +1,9 @@
+module.exports = {
+ entry: {
+ js: './js',
+ },
+ output: {
+ directory: 'bindata.tmp',
+ filename: 'bindata.tmp/[name].js',
+ },
+};
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list