[ARVADOS] created: 2.1.0-1915-g343cedddf
Git user
git at public.arvados.org
Thu Feb 17 20:10:39 UTC 2022
at 343cedddf4364c861b268da6e0a0082016ceb79b (commit)
commit 343cedddf4364c861b268da6e0a0082016ceb79b
Author: Tom Clegg <tom at curii.com>
Date: Thu Feb 17 10:25:11 2022 -0500
18700: Don't start workbench2 in integration test clusters.
Default to -no-workbench2=true until wb2 test setup script can be
updated.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index e13848db7..9c3047a7b 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -70,6 +70,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
+ flags.BoolVar(&super.NoWorkbench2, "no-workbench2", true, "do not run workbench2")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 8746183e6..0905fd44a 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -44,6 +44,7 @@ type Supervisor struct {
ControllerAddr string // e.g., 127.0.0.1:8000
Workbench2Source string // e.g., /home/username/src/arvados-workbench2
NoWorkbench1 bool
+ NoWorkbench2 bool
OwnTemporaryDatabase bool
Stderr io.Writer
@@ -251,7 +252,6 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
- runWorkbench2{svc: super.cluster.Services.Workbench2},
seedDatabase{},
}
if !super.NoWorkbench1 {
@@ -260,6 +260,11 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
)
}
+ if !super.NoWorkbench2 {
+ tasks = append(tasks,
+ runWorkbench2{svc: super.cluster.Services.Workbench2},
+ )
+ }
if super.ClusterType != "test" {
tasks = append(tasks,
runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
@@ -704,7 +709,8 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"}
}
}
- if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 {
+ if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 ||
+ super.NoWorkbench2 && svc == &cluster.Services.Workbench2 {
// When workbench1 is disabled, it gets an
// ExternalURL (so we have a valid listening
// port to write in our Nginx config) but no
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 9f5d12598..6ffbbd272 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -135,6 +135,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
tc.Super.NoWorkbench1 = true
+ tc.Super.NoWorkbench2 = true
tc.Start()
s.testClusters[id] = tc
}
diff --git a/tools/sync-groups/federation_test.go b/tools/sync-groups/federation_test.go
index 1bbdaa3fa..d5fed3e29 100644
--- a/tools/sync-groups/federation_test.go
+++ b/tools/sync-groups/federation_test.go
@@ -115,6 +115,7 @@ func (s *FederationSuite) SetUpSuite(c *check.C) {
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
tc.Super.NoWorkbench1 = true
+ tc.Super.NoWorkbench2 = true
tc.Start()
s.testClusters[id] = tc
}
commit f833920de81bf5bd439adab8b7ffa1ca7658a612
Author: Tom Clegg <tom at curii.com>
Date: Thu Feb 17 10:17:35 2022 -0500
18700: Add workbench2 to arvados-boot.
Includes related changes:
* Health aggregator obeys TLS.Insecure=true
* Use host part of -controller-address for autofilled ExternalURLs
* Use /var/lib/acme/live/{domain}/privkey if readable
* Obey -listen-host for ExternalURLs too
* Always run own postgresql instance on localhost
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index 1437f4b77..8268ea9ba 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -53,6 +53,11 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
return err
}
defer os.RemoveAll(tmpdir)
+ if abs, err := filepath.Abs(tmpdir); err != nil {
+ return fmt.Errorf("error getting absolute path of tmpdir %s: %w", tmpdir, err)
+ } else {
+ tmpdir = abs
+ }
selfbin, err := os.Readlink("/proc/self/exe")
if err != nil {
@@ -87,7 +92,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
@@ -120,7 +125,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index ca63929e9..23a78d608 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -64,12 +64,12 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
// Remove unneeded files. This is much faster than "fpm
// --exclude X" because fpm copies everything into a staging
// area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go /var/lib/arvados/arvados-workbench2")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("rm -rf [...]: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
format := "deb" // TODO: rpm
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index c8b945bea..c5465ee56 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -5,6 +5,10 @@
package main
import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
"os"
"git.arvados.org/arvados.git/lib/boot"
@@ -40,6 +44,7 @@ var (
"init": install.InitCommand,
"keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
+ "workbench2": wb2command{},
"ws": ws.Command,
})
)
@@ -47,3 +52,31 @@ var (
func main() {
os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
+
+type wb2command struct{}
+
+func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ if len(args) != 3 {
+ fmt.Fprintf(stderr, "usage: %s api-host listen-addr app-dir\n", prog)
+ return 1
+ }
+ configJSON, err := json.Marshal(map[string]string{"API_HOST": args[0]})
+ if err != nil {
+ fmt.Fprintf(stderr, "json.Marshal: %s\n", err)
+ return 1
+ }
+ mux := http.NewServeMux()
+ mux.Handle("/", http.FileServer(http.Dir(args[2])))
+ mux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+ w.Write(configJSON)
+ })
+ mux.HandleFunc("/_health/ping", func(w http.ResponseWriter, _ *http.Request) {
+ io.WriteString(w, `{"health":"OK"}`)
+ })
+ err = http.ListenAndServe(args[1], mux)
+ if err != nil {
+ fmt.Fprintln(stderr, err.Error())
+ return 1
+ }
+ return 0
+}
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 96241d24b..e13848db7 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -68,6 +68,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
+ flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 5826e5c01..44bcbc395 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -40,6 +40,15 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
"TMPDIR": super.wwwtempdir,
}
+ ctrlHost, _, err := net.SplitHostPort(super.cluster.Services.Controller.ExternalURL.Host)
+ if err != nil {
+ return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
+ }
+ if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
+ f.Close()
+ vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
+ vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
+ }
for _, cmpt := range []struct {
varname string
svc arvados.Service
@@ -51,6 +60,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
{"GIT", super.cluster.Services.GitHTTP},
{"HEALTH", super.cluster.Services.Health},
{"WORKBENCH1", super.cluster.Services.Workbench1},
+ {"WORKBENCH2", super.cluster.Services.Workbench2},
{"WS", super.cluster.Services.Websocket},
} {
var host, port string
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 2c89ccdb0..8746183e6 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -42,6 +42,7 @@ type Supervisor struct {
ClusterType string // e.g., production
ListenHost string // e.g., localhost
ControllerAddr string // e.g., 127.0.0.1:8000
+ Workbench2Source string // e.g., /home/username/src/arvados-workbench2
NoWorkbench1 bool
OwnTemporaryDatabase bool
Stderr io.Writer
@@ -250,6 +251,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ runWorkbench2{svc: super.cluster.Services.Workbench2},
seedDatabase{},
}
if !super.NoWorkbench1 {
@@ -482,6 +484,7 @@ type runOptions struct {
output io.Writer // attach stdout
env []string // add/replace environment variables
user string // run as specified user
+ stdin io.Reader
}
// RunProgram runs prog with args, using dir as working directory. If ctx is
@@ -525,6 +528,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
}
cmd := exec.Command(super.lookPath(prog), args...)
+ cmd.Stdin = opts.stdin
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
@@ -628,32 +632,42 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
return err
}
usedPort := map[string]bool{}
- nextPort := func(host string) string {
+ nextPort := func(host string) (string, error) {
for {
port, err := availablePort(host)
if err != nil {
- panic(err)
+ port, err = availablePort(super.ListenHost)
+ }
+ if err != nil {
+ return "", err
}
if usedPort[port] {
continue
}
usedPort[port] = true
- return port
+ return port, nil
}
}
if cluster.Services.Controller.ExternalURL.Host == "" {
h, p, err := net.SplitHostPort(super.ControllerAddr)
if err != nil {
- return err
+ return fmt.Errorf("SplitHostPort(ControllerAddr): %w", err)
}
if h == "" {
h = super.ListenHost
}
if p == "0" {
- p = nextPort(h)
+ p, err = nextPort(h)
+ if err != nil {
+ return err
+ }
}
cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
+ defaultExtHost, _, err := net.SplitHostPort(cluster.Services.Controller.ExternalURL.Host)
+ if err != nil {
+ return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
+ }
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
&cluster.Services.DispatchCloud,
@@ -666,21 +680,28 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
&cluster.Services.WebDAVDownload,
&cluster.Services.Websocket,
&cluster.Services.Workbench1,
+ &cluster.Services.Workbench2,
} {
if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
continue
}
if svc.ExternalURL.Host == "" {
+ port, err := nextPort(defaultExtHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(defaultExtHost, port)
if svc == &cluster.Services.Controller ||
svc == &cluster.Services.GitHTTP ||
svc == &cluster.Services.Health ||
svc == &cluster.Services.Keepproxy ||
svc == &cluster.Services.WebDAV ||
svc == &cluster.Services.WebDAVDownload ||
- svc == &cluster.Services.Workbench1 {
- svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
+ svc == &cluster.Services.Workbench1 ||
+ svc == &cluster.Services.Workbench2 {
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: host, Path: "/"}
} else if svc == &cluster.Services.Websocket {
- svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"}
}
}
if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 {
@@ -692,8 +713,13 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
continue
}
if len(svc.InternalURLs) == 0 {
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
- {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {},
+ {Scheme: "http", Host: host, Path: "/"}: {},
}
}
}
@@ -721,7 +747,12 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
if super.ClusterType == "test" {
// Add a second keepstore process.
- cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: host, Path: "/"}] = arvados.ServiceInstance{}
// Create a directory-backed volume for each keepstore
// process.
@@ -755,10 +786,14 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
}
if super.OwnTemporaryDatabase {
+ port, err := nextPort("localhost")
+ if err != nil {
+ return err
+ }
cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
"client_encoding": "utf8",
- "host": super.ListenHost,
- "port": nextPort(super.ListenHost),
+ "host": "localhost",
+ "port": port,
"dbname": "arvados_test",
"user": "arvados",
"password": "insecure_arvados_test",
diff --git a/lib/boot/workbench2.go b/lib/boot/workbench2.go
new file mode 100644
index 000000000..5a319ebfe
--- /dev/null
+++ b/lib/boot/workbench2.go
@@ -0,0 +1,73 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "io/ioutil"
+ "net"
+ "os"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+type runWorkbench2 struct {
+ svc arvados.Service
+}
+
+func (runner runWorkbench2) String() string {
+ return "runWorkbench2"
+}
+
+func (runner runWorkbench2) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ host, port, err := internalPort(runner.svc)
+ if err != nil {
+ return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ if super.ClusterType == "production" {
+ err = super.RunProgram(ctx, "/var/lib/arvados/workbench2", runOptions{
+ user: "www-data",
+ }, "arvados-server", "workbench2", super.cluster.Services.Controller.ExternalURL.Host, net.JoinHostPort(host, port), ".")
+ } else if super.Workbench2Source == "" {
+ super.logger.Info("skipping Workbench2: Workbench2Source==\"\" and not in production mode")
+ return
+ } else {
+ stdinr, stdinw := io.Pipe()
+ defer stdinw.Close()
+ go func() {
+ <-ctx.Done()
+ stdinw.Close()
+ }()
+ if err = os.Mkdir(super.Workbench2Source+"/public/_health", 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+ fail(err)
+ return
+ }
+ if err = ioutil.WriteFile(super.Workbench2Source+"/public/_health/ping", []byte(`{"health":"OK"}`), 0666); err != nil {
+ fail(err)
+ return
+ }
+ err = super.RunProgram(ctx, super.Workbench2Source, runOptions{
+ env: []string{
+ "CI=true",
+ "HTTPS=false",
+ "PORT=" + port,
+ "REACT_APP_ARVADOS_API_HOST=" + super.cluster.Services.Controller.ExternalURL.Host,
+ },
+ // If we don't connect stdin, "yarn start" just exits.
+ stdin: stdinr,
+ }, "yarn", "start")
+ fail(errors.New("`yarn start` exited"))
+ }
+ fail(err)
+ }()
+ return nil
+}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 1d3cc0927..d5d440744 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -28,8 +28,12 @@ import (
var Command cmd.Handler = &installCommand{}
-const devtestDatabasePassword = "insecure_arvados_test"
-const goversion = "1.17.1"
+const goversion = "1.17.7"
+
+const (
+ devtestDatabasePassword = "insecure_arvados_test"
+ workbench2version = "5e805cf2209d3afe42699e4658d8a12e50bcd5a4"
+)
type installCommand struct {
ClusterType string
@@ -247,7 +251,7 @@ make install
cd /tmp
rm -rf /var/lib/arvados/go/
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
-ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
+ln -sfv /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -263,7 +267,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
-ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
+ln -sfv /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -277,21 +281,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
err = inst.runBash(`
GD=v`+geckoversion+`
wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
-ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
-`, stdout, stderr)
- if err != nil {
- return 1
- }
- }
-
- nodejsversion := "v12.22.2"
- if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
- logger.Print("nodejs " + nodejsversion + " already installed")
- } else {
- err = inst.runBash(`
-NJS=`+nodejsversion+`
-wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
-ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+ln -sfv /var/lib/arvados/bin/geckodriver /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -308,7 +298,7 @@ zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
unzip -o -d /var/lib/arvados ${zip}
-ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
+ln -sfv /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
rm ${zip}
`, stdout, stderr)
if err != nil {
@@ -458,7 +448,75 @@ make -C ./builddir install
}
}
+ if !prod {
+ nodejsversion := "v12.22.2"
+ if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
+ logger.Print("nodejs " + nodejsversion + " already installed")
+ } else {
+ err = inst.runBash(`
+NJS=`+nodejsversion+`
+wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if haveyarnversion, err := exec.Command("/usr/local/bin/yarn", "--version").CombinedOutput(); err == nil && len(haveyarnversion) > 0 {
+ logger.Print("yarn " + strings.TrimSpace(string(haveyarnversion)) + " already installed")
+ } else {
+ err = inst.runBash(`
+npm install -g yarn
+ln -sfv /var/lib/arvados/node-`+nodejsversion+`-linux-x64/bin/{yarn,yarnpkg} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if havewb2version, err := exec.Command("git", "--git-dir=/var/lib/arvados/arvados-workbench2/.git", "log", "-n1", "--format=%H").CombinedOutput(); err == nil && string(havewb2version) == workbench2version+"\n" {
+ logger.Print("workbench2 repo is already at " + workbench2version)
+ } else {
+ err = inst.runBash(`
+V=`+workbench2version+`
+cd /var/lib/arvados
+if [[ ! -e arvados-workbench2 ]]; then
+ git clone https://git.arvados.org/arvados-workbench2.git
+ cd arvados-workbench2
+ git checkout $V
+else
+ cd arvados-workbench2
+ if ! git checkout $V; then
+ git fetch
+ git checkout $V
+ fi
+fi
+rm -rf build
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if err = inst.runBash(`
+cd /var/lib/arvados/arvados-workbench2
+yarn install --non-interactive
+`, stdout, stderr); err != nil {
+ return 1
+ }
+ }
+
if prod || pkg {
+ // Install workbench2 app to /var/lib/arvados/workbench2/
+ if err = inst.runBash(`
+cd /var/lib/arvados/arvados-workbench2
+yarn build
+rsync -a --delete-after build/ /var/lib/arvados/workbench2/
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
for dstdir, srcdir := range map[string]string{
"railsapi": "services/api",
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index a666ef8ec..296ef65dd 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -6,8 +6,8 @@ package health
import (
"context"
+ "crypto/tls"
"encoding/json"
- "errors"
"fmt"
"net/http"
"net/url"
@@ -35,7 +35,13 @@ type Aggregator struct {
}
func (agg *Aggregator) setup() {
- agg.httpClient = http.DefaultClient
+ agg.httpClient = &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: agg.Cluster.TLS.Insecure,
+ },
+ },
+ }
if agg.timeout == 0 {
// this is always the case, except in the test suite
agg.timeout = defaultTimeout
@@ -176,19 +182,14 @@ func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) {
func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
t0 := time.Now()
-
- var err error
defer func() {
result.ResponseTime = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
- if err != nil {
- result.Health, result.Error = "ERROR", err.Error()
- } else {
- result.Health = "OK"
- }
}()
+ result.Health = "ERROR"
req, err := http.NewRequest("GET", target.String(), nil)
if err != nil {
+ result.Error = err.Error()
return
}
req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
@@ -201,22 +202,26 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
req = req.WithContext(ctx)
resp, err := agg.httpClient.Do(req)
if err != nil {
+ result.Error = err.Error()
return
}
result.HTTPStatusCode = resp.StatusCode
result.HTTPStatusText = resp.Status
err = json.NewDecoder(resp.Body).Decode(&result.Response)
if err != nil {
- err = fmt.Errorf("cannot decode response: %s", err)
+ result.Error = fmt.Sprintf("cannot decode response: %s", err)
} else if resp.StatusCode != http.StatusOK {
- err = fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
+ result.Error = fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status)
} else if h, _ := result.Response["health"].(string); h != "OK" {
if e, ok := result.Response["error"].(string); ok && e != "" {
- err = errors.New(e)
+ result.Error = e
+ return
} else {
- err = fmt.Errorf("health=%q in ping response", h)
+ result.Error = fmt.Sprintf("health=%q in ping response", h)
+ return
}
}
+ result.Health = "OK"
return
}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 35b780071..44f8469b0 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -147,7 +147,7 @@ http {
}
server {
listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
- server_name workbench1 workbench.*;
+ server_name workbench1 workbench1.* workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -158,4 +158,17 @@ http {
proxy_redirect off;
}
}
+ server {
+ listen {{LISTENHOST}}:{{WORKBENCH2SSLPORT}} ssl;
+ server_name workbench2 workbench2.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://{{LISTENHOST}}:{{WORKBENCH2PORT}};
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index f91783250..9c45a3205 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -634,6 +634,8 @@ def run_nginx():
nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")
+ nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2")
+ nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2")
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
@@ -667,6 +669,8 @@ def setup_config():
websocket_external_port = find_available_port()
workbench1_port = find_available_port()
workbench1_external_port = find_available_port()
+ workbench2_port = find_available_port()
+ workbench2_external_port = find_available_port()
git_httpd_port = find_available_port()
git_httpd_external_port = find_available_port()
health_httpd_port = find_available_port()
@@ -720,6 +724,12 @@ def setup_config():
"http://%s:%s"%(localhost, workbench1_port): {},
},
},
+ "Workbench2": {
+ "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port),
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, workbench2_port): {},
+ },
+ },
"GitHTTP": {
"ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
"InternalURLs": {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list