[arvados] updated: 2.1.0-2650-gbd5f21b6a
git repository hosting
git at public.arvados.org
Thu Jun 30 20:03:07 UTC 2022
Summary of changes:
cmd/arvados-package/fpm.go | 7 ++-
cmd/arvados-package/install.go | 4 +-
cmd/arvados-server/cmd.go | 18 ++++++-
lib/boot/cmd.go | 4 +-
lib/boot/nginx.go | 3 +-
lib/boot/service.go | 10 +++-
lib/boot/supervisor.go | 48 ++++++++---------
lib/install/deps.go | 37 +++++++------
lib/install/init.go | 101 ++++++++++++++++++++++++++++--------
sdk/python/tests/nginx.conf | 16 +++---
sdk/python/tests/run_test_server.py | 1 +
11 files changed, 168 insertions(+), 81 deletions(-)
via bd5f21b6a567248d05126d78bab091555764a052 (commit)
via de4f7a1ed61ed17a961732668a2c5957130d4740 (commit)
via 1dadff7a6d839e3a4e1ebf4d2c91e4144a036f35 (commit)
via 46f0c0faf4f032697d59c2d663018ae67d1059d4 (commit)
via 51fd368253a0b61c58f62bea829446109bf6af3a (commit)
via 005a2e1472484182eec807fce9fa1137bd607c5e (commit)
via c6d8eb619ad9dee27793c068f4e178f2c932906b (commit)
via 2361a96130524ee57b0e1d652d560e8e31993b04 (commit)
via e6d97ad638edb8aef6b558744d5de4c0ae6e5199 (commit)
via 6b4c21f5c898fdccc66bab6913e8fd2874dfc15e (commit)
via 8e69a90b47486ea39d89b9209e327a6bb475debb (commit)
via e47293395f0908a708680df773780ce418b48de9 (commit)
via 8b3dd08ed1d43121bd92452c7cb0504098eb5887 (commit)
via 49ce9a4b07da6ae0da25e19a34e91b4664d8b2a5 (commit)
via f1b67a590f30b5f8824b14f7b49520106a13bb29 (commit)
from 5722e7f91d3ab4df898dec0d301c0653ac7995b3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit bd5f21b6a567248d05126d78bab091555764a052
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 16:02:26 2022 -0400
16552: Add arvados-server init -admin-email option.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 73483d3b3..b2de067ff 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -36,6 +36,7 @@ type initCommand struct {
PostgreSQLPassword string
Login string
TLS string
+ AdminEmail string
Start bool
LoginPAM bool
@@ -70,6 +71,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''")
+ flags.StringVar(&initcmd.AdminEmail, "admin-email", "", "give admin privileges to user with given `email`")
flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none")
flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
@@ -87,6 +89,9 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
initcmd.LoginGoogleClientSecret = fields[2]
} else if initcmd.Login == "test" {
initcmd.LoginTest = true
+ if initcmd.AdminEmail == "" {
+ initcmd.AdminEmail = "admin at example.com"
+ }
} else if initcmd.Login == "pam" {
initcmd.LoginPAM = true
} else if initcmd.Login == "" {
@@ -235,7 +240,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
Enable: true
Users:
admin:
- Email: admin at example.com
+ Email: {{printf "%q" .AdminEmail}}
Password: admin
{{else if .LoginGoogle}}
Login:
@@ -244,10 +249,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
ClientID: {{printf "%q" .LoginGoogleClientID}}
ClientSecret: {{printf "%q" .LoginGoogleClientSecret}}
{{end}}
- {{if .LoginTest}}
Users:
- AutoAdminUserWithEmail: admin at example.com
- {{end}}
+ AutoAdminUserWithEmail: {{printf "%q" .AdminEmail}}
`)
if err != nil {
return 1
commit de4f7a1ed61ed17a961732668a2c5957130d4740
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 15:11:32 2022 -0400
16552: Fix warnings re wrong process signalling service readiness.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/service.go b/lib/boot/service.go
index b27a74622..506407f4e 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -47,7 +47,15 @@ func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, runOptions{env: []string{"ARVADOS_SERVICE_INTERNAL_URL=" + u.String()}}, binfile, runner.name, "-config", super.configfile))
+ fail(super.RunProgram(ctx, super.tempdir, runOptions{
+ env: []string{
+ "ARVADOS_SERVICE_INTERNAL_URL=" + u.String(),
+ // Child process should not
+ // try to tell systemd that we
+ // are ready.
+ "NOTIFY_SOCKET=",
+ },
+ }, binfile, runner.name, "-config", super.configfile))
}()
}
return nil
commit 1dadff7a6d839e3a4e1ebf4d2c91e4144a036f35
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 15:04:11 2022 -0400
16552: Fix cwd warning from psql.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index b7ca0701f..73483d3b3 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -339,6 +339,7 @@ func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.Postgre
`CREATE EXTENSION IF NOT EXISTS pg_trgm`,
} {
cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+ cmd.Dir = "/"
cmd.Stdout = stderr
cmd.Stderr = stderr
err := cmd.Run()
commit 46f0c0faf4f032697d59c2d663018ae67d1059d4
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 14:56:18 2022 -0400
16552: change default db name to just arvados.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index f65eae54c..b7ca0701f 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -201,7 +201,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
PostgreSQL:
Connection:
- dbname: arvados_production
+ dbname: arvados
host: localhost
user: arvados
password: {{printf "%q" .PostgreSQLPassword}}
commit 51fd368253a0b61c58f62bea829446109bf6af3a
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 14:53:18 2022 -0400
16552: arvados-server init -login "google $clientid $clientsecret"
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 46be478cb..f65eae54c 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -18,6 +18,7 @@ import (
"os/user"
"regexp"
"strconv"
+ "strings"
"text/template"
"git.arvados.org/arvados.git/lib/cmd"
@@ -36,6 +37,12 @@ type initCommand struct {
Login string
TLS string
Start bool
+
+ LoginPAM bool
+ LoginTest bool
+ LoginGoogle bool
+ LoginGoogleClientID string
+ LoginGoogleClientSecret string
}
func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -62,7 +69,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
- flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''")
+ flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, 'google {client-id} {client-secret}', or ''")
flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none")
flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
@@ -74,6 +81,21 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
+ if fields := strings.Fields(initcmd.Login); len(fields) == 3 && fields[0] == "google" {
+ initcmd.LoginGoogle = true
+ initcmd.LoginGoogleClientID = fields[1]
+ initcmd.LoginGoogleClientSecret = fields[2]
+ } else if initcmd.Login == "test" {
+ initcmd.LoginTest = true
+ } else if initcmd.Login == "pam" {
+ initcmd.LoginPAM = true
+ } else if initcmd.Login == "" {
+ // none; login will show an error page
+ } else {
+ err = fmt.Errorf("invalid argument to -login: %q: should be 'test', 'pam', 'google {client-id} {client-secret}', or empty")
+ return 1
+ }
+
confdir := "/etc/arvados"
conffile := confdir + "/config.yml"
if _, err = os.Stat(conffile); err == nil {
@@ -203,26 +225,29 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
Replication: 2
Workbench:
SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+ {{if .LoginPAM}}
Login:
- {{if eq .Login "pam"}}
PAM:
Enable: true
- {{else if eq .Login "test"}}
+ {{else if .LoginTest}}
+ Login:
Test:
Enable: true
Users:
admin:
Email: admin at example.com
Password: admin
- {{else}}
- {}
- {{end}}
+ {{else if .LoginGoogle}}
+ Login:
+ Google:
+ Enable: true
+ ClientID: {{printf "%q" .LoginGoogleClientID}}
+ ClientSecret: {{printf "%q" .LoginGoogleClientSecret}}
+ {{end}}
+ {{if .LoginTest}}
Users:
- {{if eq .Login "test"}}
AutoAdminUserWithEmail: admin at example.com
- {{else}}
- {}
- {{end}}
+ {{end}}
`)
if err != nil {
return 1
commit 005a2e1472484182eec807fce9fa1137bd607c5e
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 11:44:52 2022 -0400
16552: Add arvados-cli gem to arvados-server-easy package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index 4c9a00619..9a20ec76f 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -104,6 +104,8 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"/lib/systemd/system/arvados.service",
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
+ "/usr/bin/arv",
+ "/usr/bin/arv-tag",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
diff --git a/lib/install/deps.go b/lib/install/deps.go
index caecdd017..017771b63 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -578,19 +578,6 @@ yarn install
}
}
- // Symlink user-facing Go programs /usr/bin/x ->
- // /var/lib/arvados/bin/x
- for _, prog := range []string{"arvados-client", "arvados-server"} {
- err = os.Remove("/usr/bin/" + prog)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return 1
- }
- err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog)
- if err != nil {
- return 1
- }
- }
-
// Copy assets from source tree to /var/lib/arvados/share
cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
cmd.Stdout = stdout
@@ -676,6 +663,14 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/
return 1
}
+ // Install arvados-cli gem (binaries go in
+ // /var/lib/arvados/bin)
+ if err = inst.runBash(`
+/var/lib/arvados/bin/gem install --conservative --no-document arvados-cli
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
err = os.WriteFile("/lib/systemd/system/arvados.service", arvadosServiceFile, 0777)
if err != nil {
return 1
@@ -692,6 +687,19 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/
if err != nil {
return 1
}
+
+ // Symlink user-facing programs /usr/bin/x ->
+ // /var/lib/arvados/bin/x
+ for _, prog := range []string{"arvados-client", "arvados-server", "arv", "arv-tag"} {
+ err = os.Remove("/usr/bin/" + prog)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return 1
+ }
+ err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog)
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit c6d8eb619ad9dee27793c068f4e178f2c932906b
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 11:25:53 2022 -0400
16552: Improve "init" feedback when config already exists.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 2257fc57c..46be478cb 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -74,6 +74,13 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
+ confdir := "/etc/arvados"
+ conffile := confdir + "/config.yml"
+ if _, err = os.Stat(conffile); err == nil {
+ err = fmt.Errorf("config file %s already exists; delete it first if you really want to start over", conffile)
+ return 1
+ }
+
wwwuser, err := user.Lookup("www-data")
if err != nil {
err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
@@ -92,15 +99,19 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
}
fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
- err = os.Mkdir("/etc/arvados", 0750)
+ err = os.Mkdir(confdir, 0750)
if err != nil && !os.IsExist(err) {
- err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ err = fmt.Errorf("mkdir %s: %w", confdir, err)
+ return 1
+ }
+ err = os.Chown(confdir, 0, wwwgid)
+ if err != nil {
+ err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err)
return 1
}
- err = os.Chown("/etc/arvados", 0, wwwgid)
- f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ f, err := os.OpenFile(conffile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
if err != nil {
- err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("open %s: %w", conffile, err)
return 1
}
tmpl, err := template.New("config").Parse(`Clusters:
@@ -218,21 +229,21 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
}
err = tmpl.Execute(f, initcmd)
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ err = fmt.Errorf("%s: tmpl.Execute: %w", conffile, err)
return 1
}
err = f.Close()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ err = fmt.Errorf("%s: close: %w", conffile, err)
return 1
}
- fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+ fmt.Fprintln(stderr, "created", conffile)
ldr := config.NewLoader(nil, logger)
ldr.SkipLegacy = true
cfg, err := ldr.Load()
if err != nil {
- err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ err = fmt.Errorf("%s: %w", conffile, err)
return 1
}
cluster, err := cfg.GetCluster("")
@@ -251,7 +262,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- err = fmt.Errorf("rake db:setup: %w", err)
+ err = fmt.Errorf("rake db:setup failed: %w", err)
return 1
}
fmt.Fprintln(stderr, "initialized database")
commit 2361a96130524ee57b0e1d652d560e8e31993b04
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 11:22:53 2022 -0400
16552: Start systemd service after successful init.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 72688dc0a..9273ac9c7 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -124,7 +124,7 @@ eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-
SUDO_FORCE_REMOVE=yes apt-get autoremove -y
/etc/init.d/postgresql start
-arvados-server init -cluster-id x1234 -domain=$domain -login=test $initargs
+arvados-server init -cluster-id x1234 -domain=$domain -login=test -start=false $initargs
exec arvados-server boot -listen-host=0.0.0.0 $bootargs
`)
cmd.Stdout = stdout
diff --git a/lib/install/init.go b/lib/install/init.go
index 6954a60d8..2257fc57c 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -35,6 +35,7 @@ type initCommand struct {
PostgreSQLPassword string
Login string
TLS string
+ Start bool
}
func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -63,6 +64,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''")
flags.StringVar(&initcmd.TLS, "tls", "none", "tls certificate `source`: acme, auto, insecure, or none")
+ flags.BoolVar(&initcmd.Start, "start", true, "start systemd service after creating config")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
return code
} else if *versionFlag {
@@ -254,6 +256,19 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
}
fmt.Fprintln(stderr, "initialized database")
+ if initcmd.Start {
+ fmt.Fprintln(stderr, "starting systemd service")
+ cmd := exec.CommandContext(ctx, "systemctl", "start", "--no-block", "arvados")
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ }
+
return 0
}
commit e6d97ad638edb8aef6b558744d5de4c0ae6e5199
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 11:11:49 2022 -0400
16552: Drop crunch-dispatch-local from arvados-server-easy package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 44215a773..caecdd017 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -563,7 +563,6 @@ yarn install
for _, srcdir := range []string{
"cmd/arvados-client",
"cmd/arvados-server",
- "services/crunch-dispatch-local",
"services/crunch-dispatch-slurm",
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
commit 6b4c21f5c898fdccc66bab6913e8fd2874dfc15e
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 11:11:44 2022 -0400
16552: Use fpm features to manage systemd unit.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index d81abab58..4c9a00619 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -97,7 +97,10 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"--verbose",
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
- "/etc/systemd/system/multi-user.target.wants/arvados.service",
+ "--deb-systemd", "/etc/systemd/system/multi-user.target.wants/arvados.service",
+ "--deb-systemd-enable",
+ "--no-deb-systemd-auto-start",
+ "--no-deb-systemd-restart-after-upgrade",
"/lib/systemd/system/arvados.service",
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
commit 8e69a90b47486ea39d89b9209e327a6bb475debb
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 02:22:20 2022 -0400
16552: Add workbench2 catch-all index.html rule.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index 3a1fcd4c6..d9c41ca58 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -11,6 +11,9 @@ import (
"io"
"net/http"
"os"
+ "path"
+ "path/filepath"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/cloud/cloudtest"
@@ -80,8 +83,21 @@ func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout
fmt.Fprintf(stderr, "json.Marshal: %s\n", err)
return 1
}
+ servefs := http.FileServer(http.Dir(args[2]))
mux := http.NewServeMux()
- mux.Handle("/", http.FileServer(http.Dir(args[2])))
+ mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ for _, ent := range strings.Split(req.URL.Path, "/") {
+ if ent == ".." {
+ http.Error(w, "invalid URL path", http.StatusBadRequest)
+ return
+ }
+ }
+ fnm := filepath.Join(args[2], filepath.FromSlash(path.Clean("/"+req.URL.Path)))
+ if _, err := os.Stat(fnm); os.IsNotExist(err) {
+ req.URL.Path = "/"
+ }
+ servefs.ServeHTTP(w, req)
+ }))
mux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
w.Write(configJSON)
})
commit e47293395f0908a708680df773780ce418b48de9
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 01:36:06 2022 -0400
16552: Update workbench2 version, boot it by default.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index f923d5ef8..72688dc0a 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -125,7 +125,7 @@ SUDO_FORCE_REMOVE=yes apt-get autoremove -y
/etc/init.d/postgresql start
arvados-server init -cluster-id x1234 -domain=$domain -login=test $initargs
-exec arvados-server boot -listen-host=0.0.0.0 -no-workbench2=false $bootargs
+exec arvados-server boot -listen-host=0.0.0.0 $bootargs
`)
cmd.Stdout = stdout
cmd.Stderr = stderr
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index e407ec5b5..e22ae7ad7 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -70,7 +70,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
- flags.BoolVar(&super.NoWorkbench2, "no-workbench2", true, "do not run workbench2")
+ flags.BoolVar(&super.NoWorkbench2, "no-workbench2", false, "do not run workbench2")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
diff --git a/lib/install/deps.go b/lib/install/deps.go
index a7c32c505..44215a773 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -40,7 +40,7 @@ const (
gradleversion = "5.3.1"
nodejsversion = "v12.22.11"
devtestDatabasePassword = "insecure_arvados_test"
- workbench2version = "5e020488f67b5bc919796e0dc8b0b9f3b3ff23b0"
+ workbench2version = "2454ac35292a79594c32a80430740317ed5005cf"
)
//go:embed arvados.service
commit 8b3dd08ed1d43121bd92452c7cb0504098eb5887
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 01:08:40 2022 -0400
16552: Nginx listens on all interfaces, not just loopback.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 15af548e9..e407ec5b5 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -66,7 +66,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ConfigPath, "config", "/etc/arvados/config.yml", "arvados config file `path`")
flags.StringVar(&super.SourcePath, "source", ".", "arvados source tree `directory`")
flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
- flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for external services, and internal services whose InternalURLs are not configured")
+ flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for internal services whose InternalURLs are not configured")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 48d3bba47..8a29823a1 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -33,7 +33,8 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
return err
}
vars := map[string]string{
- "LISTENHOST": super.ListenHost,
+ "LISTENHOST": "0.0.0.0",
+ "UPSTREAMHOST": super.ListenHost,
"SSLCERT": filepath.Join(super.tempdir, "server.crt"),
"SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 9ed92cb1a..ddc17953d 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -247,13 +247,9 @@ func (super *Supervisor) runCluster() error {
}
if super.ListenHost == "" {
- if urlhost := super.cluster.Services.Controller.ExternalURL.Host; urlhost != "" {
- if h, _, _ := net.SplitHostPort(urlhost); h != "" {
- super.ListenHost = h
- } else {
- super.ListenHost = urlhost
- }
- } else {
+ u := url.URL(super.cluster.Services.Controller.ExternalURL)
+ super.ListenHost = u.Hostname()
+ if super.ListenHost == "" {
super.ListenHost = "0.0.0.0"
}
}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 543390004..1c624e324 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -16,7 +16,7 @@ http {
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ server {{UPSTREAMHOST}}:{{CONTROLLERPORT}};
}
server {
listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
@@ -37,7 +37,7 @@ http {
}
}
upstream arv-git-http {
- server {{LISTENHOST}}:{{GITPORT}};
+ server {{UPSTREAMHOST}}:{{GITPORT}};
}
server {
listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
@@ -53,7 +53,7 @@ http {
}
}
upstream keepproxy {
- server {{LISTENHOST}}:{{KEEPPROXYPORT}};
+ server {{UPSTREAMHOST}}:{{KEEPPROXYPORT}};
}
server {
listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
@@ -73,7 +73,7 @@ http {
}
}
upstream keep-web {
- server {{LISTENHOST}}:{{KEEPWEBPORT}};
+ server {{UPSTREAMHOST}}:{{KEEPWEBPORT}};
}
server {
listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
@@ -93,7 +93,7 @@ http {
}
}
upstream health {
- server {{LISTENHOST}}:{{HEALTHPORT}};
+ server {{UPSTREAMHOST}}:{{HEALTHPORT}};
}
server {
listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
@@ -129,7 +129,7 @@ http {
}
}
upstream ws {
- server {{LISTENHOST}}:{{WSPORT}};
+ server {{UPSTREAMHOST}}:{{WSPORT}};
}
server {
listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
@@ -147,7 +147,7 @@ http {
}
}
upstream workbench1 {
- server {{LISTENHOST}}:{{WORKBENCH1PORT}};
+ server {{UPSTREAMHOST}}:{{WORKBENCH1PORT}};
}
server {
listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
@@ -163,7 +163,7 @@ http {
}
}
upstream workbench2 {
- server {{LISTENHOST}}:{{WORKBENCH2PORT}};
+ server {{UPSTREAMHOST}}:{{WORKBENCH2PORT}};
}
server {
listen {{LISTENHOST}}:{{WORKBENCH2SSLPORT}} ssl;
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 2c01b35ae..28cb0953f 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -635,6 +635,7 @@ def run_nginx():
return
stop_nginx()
nginxconf = {}
+ nginxconf['UPSTREAMHOST'] = 'localhost'
nginxconf['LISTENHOST'] = 'localhost'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))
commit 49ce9a4b07da6ae0da25e19a34e91b4664d8b2a5
Author: Tom Clegg <tom at curii.com>
Date: Thu Jun 30 00:10:44 2022 -0400
16552: Fix signal handling.
Signal handlers shouldn't stop when Start returns.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 52cd0b161..9ed92cb1a 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -113,28 +113,24 @@ func (super *Supervisor) Start(ctx context.Context) {
super.done = make(chan struct{})
sigch := make(chan os.Signal)
- signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
- defer signal.Stop(sigch)
+ signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
go func() {
- for sig := range sigch {
- super.logger.WithField("signal", sig).Info("caught signal")
- if super.err == nil {
- super.err = fmt.Errorf("caught signal %s", sig)
- }
- super.cancel()
- }
- }()
-
- hupch := make(chan os.Signal)
- signal.Notify(hupch, syscall.SIGHUP)
- defer signal.Stop(hupch)
- go func() {
- for sig := range hupch {
- super.logger.WithField("signal", sig).Info("caught signal")
- if super.err == nil {
- super.err = errNeedConfigReload
+ defer signal.Stop(sigch)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case sig := <-sigch:
+ super.logger.WithField("signal", sig).Info("caught signal")
+ if super.err == nil {
+ if sig == syscall.SIGHUP {
+ super.err = errNeedConfigReload
+ } else {
+ super.err = fmt.Errorf("caught signal %s", sig)
+ }
+ }
+ super.cancel()
}
- super.cancel()
}
}()
commit f1b67a590f30b5f8824b14f7b49520106a13bb29
Author: Tom Clegg <tom at curii.com>
Date: Wed Jun 29 23:59:37 2022 -0400
16552: boot: wait for child proc cleanup when shutting down.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 8eb375b87..52cd0b161 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -471,6 +471,7 @@ func (super *Supervisor) WaitReady() bool {
super.logger.Infof("waiting for %s to be ready", id)
if !super2.WaitReady() {
super.logger.Infof("%s startup failed", id)
+ super.Stop()
return false
}
super.logger.Infof("%s is ready", id)
@@ -484,6 +485,7 @@ func (super *Supervisor) WaitReady() bool {
select {
case <-ticker.C:
case <-super.ctx.Done():
+ super.Stop()
return false
}
if super.healthChecker == nil {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list