[ARVADOS] created: 2.1.0-2018-g8257b9e90
Git user
git at public.arvados.org
Sun Mar 13 16:58:55 UTC 2022
at 8257b9e9049a2592c9858941775a11b5a98ec1f7 (commit)
commit 8257b9e9049a2592c9858941775a11b5a98ec1f7
Author: Tom Clegg <tom at curii.com>
Date: Sun Mar 13 12:50:33 2022 -0400
18700: Fix nginx temp dir permissions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index f212254d2..e67bc1d90 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -12,7 +12,6 @@ import (
"net/url"
"os"
"os/exec"
- "os/user"
"path/filepath"
"regexp"
@@ -117,28 +116,18 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
}
}
- args := []string{
- "-g", "error_log stderr info;",
- "-g", "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + ";",
- "-c", conffile,
- }
- // Nginx ignores "user www-data;" when running as a non-root
- // user... except that it causes it to ignore our other -g
- // options. So we still have to decide for ourselves whether
- // it's needed.
- if u, err := user.Current(); err != nil {
- return fmt.Errorf("user.Current(): %w", err)
- } else if u.Uid == "0" {
- args = append([]string{"-g", "user www-data;"}, args...)
- }
+ configs := "error_log stderr info; "
+ configs += "pid " + filepath.Join(super.wwwtempdir, "nginx.pid") + "; "
+ configs += "user www-data; "
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, ".", runOptions{}, nginx, args...))
+ fail(super.RunProgram(ctx, ".", runOptions{}, nginx, "-g", configs, "-c", conffile))
}()
// Choose one of the ports where Nginx should listen, and wait
- // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+ // here until we can connect. If ExternalURL is https://foo
+ // (with no port) then we connect to "foo:https"
testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
if testurl.Port() == "" {
testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 00e981afc..323f67234 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -60,8 +60,8 @@ type Supervisor struct {
waitShutdown sync.WaitGroup
bindir string
- tempdir string
- wwwtempdir string
+ tempdir string // in production mode, this is accessible only to root
+ wwwtempdir string // in production mode, this is accessible only to www-data
configfile string
environ []string // for child processes
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 414bb4820..6090a51a7 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -220,7 +220,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
os.Mkdir("/var/lib/arvados", 0755)
os.Mkdir("/var/lib/arvados/tmp", 0700)
if prod || pkg {
- os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
u, er := user.Lookup("www-data")
if er != nil {
err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
@@ -228,6 +227,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
}
uid, _ := strconv.Atoi(u.Uid)
gid, _ := strconv.Atoi(u.Gid)
+ os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
if err != nil {
return 1
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index bfb1226f7..25a6f2096 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -3,7 +3,6 @@
# SPDX-License-Identifier: Apache-2.0
daemon off;
-error_log "{{ERRORLOG}}" info; # Yes, must be specified here _and_ cmdline
events {
}
http {
@@ -11,11 +10,11 @@ http {
'[$time_local] "$http_x_request_id" $server_name $status $body_bytes_sent $request_time $request_method "$scheme://$http_host$request_uri" $remote_addr:$remote_port '
'"$http_referer" "$http_user_agent"';
access_log "{{ACCESSLOG}}" customlog;
- client_body_temp_path "{{TMPDIR}}/nginx";
- proxy_temp_path "{{TMPDIR}}/nginx";
- fastcgi_temp_path "{{TMPDIR}}/nginx";
- uwsgi_temp_path "{{TMPDIR}}/nginx";
- scgi_temp_path "{{TMPDIR}}/nginx";
+ client_body_temp_path "{{TMPDIR}}";
+ proxy_temp_path "{{TMPDIR}}";
+ fastcgi_temp_path "{{TMPDIR}}";
+ uwsgi_temp_path "{{TMPDIR}}";
+ scgi_temp_path "{{TMPDIR}}";
upstream controller {
server {{LISTENHOST}}:{{CONTROLLERPORT}};
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 9f27510f4..6514c2af4 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -641,7 +641,7 @@ def run_nginx():
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
nginxconf['ERRORLOG'] = _logfilename('nginx_error')
- nginxconf['TMPDIR'] = TEST_TMPDIR
+ nginxconf['TMPDIR'] = TEST_TMPDIR + '/nginx'
conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
@@ -656,8 +656,7 @@ def run_nginx():
nginx = subprocess.Popen(
['nginx',
- '-g', 'error_log stderr info;',
- '-g', 'pid '+_pidfile('nginx')+';',
+ '-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
_wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
commit 02d727c7ad1c60a9781ae707cdb9f78d403ec494
Author: Tom Clegg <tom at curii.com>
Date: Fri Mar 11 12:17:56 2022 -0500
18700: Configure test login.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 0e05f96d6..324f3b4d2 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -118,7 +118,7 @@ eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-
SUDO_FORCE_REMOVE=yes apt-get autoremove -y
/etc/init.d/postgresql start
-arvados-server init -cluster-id x1234 -domain=$domain
+arvados-server init -cluster-id x1234 -domain=$domain -login=test
exec arvados-server boot -listen-host=0.0.0.0 $bootargs
`)
cmd.Stdout = stdout
diff --git a/lib/install/init.go b/lib/install/init.go
index adfa64b3b..98f9fffd0 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -33,6 +33,7 @@ type initCommand struct {
ClusterID string
Domain string
PostgreSQLPassword string
+ Login string
}
func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -59,6 +60,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ flags.StringVar(&initcmd.Login, "login", "", "login `backend`: test, pam, or ''")
if ok, code := cmd.ParseFlags(flags, prog, args, "", stderr); !ok {
return code
} else if *versionFlag {
@@ -169,6 +171,26 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
Replication: 2
Workbench:
SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
+ Login:
+ {{if eq .Login "pam"}}
+ PAM:
+ Enable: true
+ {{else if eq .Login "test"}}
+ Test:
+ Enable: true
+ Users:
+ admin:
+ Email: admin at example.com
+ Password: admin
+ {{else}}
+ {}
+ {{end}}
+ Users:
+ {{if eq .Login "test"}}
+ AutoAdminUserWithEmail: admin at example.com
+ {{else}}
+ {}
+ {{end}}
`)
if err != nil {
return 1
commit affd65b8a7df47c142a182eddff51949eb9d22b8
Author: Tom Clegg <tom at curii.com>
Date: Fri Mar 11 11:20:37 2022 -0500
18700: Remove bogus workbench2 InternalURLs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index f79fae782..adfa64b3b 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -143,8 +143,6 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
"http://0.0.0.0:8001/": {}
ExternalURL: {{printf "%q" ( print "https://" .Domain ":4441/" ) }}
Workbench2:
- InternalURLs:
- "http://0.0.0.0:8002/": {}
ExternalURL: {{printf "%q" ( print "https://" .Domain ":4442/" ) }}
Health:
InternalURLs:
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index 296ef65dd..23d7e8d43 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -133,7 +133,14 @@ func (agg *Aggregator) ClusterHealth() ClusterHealthResponse {
}
mtx.Unlock()
+ checkURLs := map[arvados.URL]bool{}
for addr := range svc.InternalURLs {
+ checkURLs[addr] = true
+ }
+ if len(checkURLs) == 0 && svc.ExternalURL.Host != "" {
+ checkURLs[svc.ExternalURL] = true
+ }
+ for addr := range checkURLs {
wg.Add(1)
go func(svcName arvados.ServiceName, addr arvados.URL) {
defer wg.Done()
commit 105c4f587fa4fedbdc03154da46869803ceedd5f
Author: Tom Clegg <tom at curii.com>
Date: Fri Mar 11 09:38:30 2022 -0500
18700: Add 'arvados-package testinstall -live=vhost' option.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/cmd.go b/cmd/arvados-package/cmd.go
index a64d44074..6e3de08e9 100644
--- a/cmd/arvados-package/cmd.go
+++ b/cmd/arvados-package/cmd.go
@@ -65,6 +65,7 @@ type opts struct {
TargetOS string
Maintainer string
Vendor string
+ Live string
}
func parseFlags(prog string, args []string, stderr io.Writer) (_ opts, ok bool, exitCode int) {
@@ -82,6 +83,7 @@ func parseFlags(prog string, args []string, stderr io.Writer) (_ opts, ok bool,
flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
flags.StringVar(&opts.Maintainer, "package-maintainer", opts.Maintainer, "maintainer to be listed in package metadata")
flags.StringVar(&opts.Vendor, "package-vendor", opts.Vendor, "vendor to be listed in package metadata")
+ flags.StringVar(&opts.Live, "live", opts.Live, "run controller at https://`example.com`, use host's /var/lib/acme/live certificates, wait for ^C before shutting down")
flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
flags.Usage = func() {
fmt.Fprint(flags.Output(), `Usage: arvados-package <subcommand> [options]
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index b0a87ba3b..0e05f96d6 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -85,9 +85,21 @@ rm /etc/apt/sources.list.d/arvados-local.list
versionsuffix = "=" + opts.PackageVersion
}
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
- "--tmpfs", "/tmp:exec,mode=01777",
- "-v", absPackageDir+":/pkg:ro",
- "--env", "DEBIAN_FRONTEND=noninteractive",
+ "--tmpfs=/tmp:exec,mode=01777",
+ "--volume="+absPackageDir+":/pkg:ro",
+ "--env=DEBIAN_FRONTEND=noninteractive")
+ if opts.Live != "" {
+ cmd.Args = append(cmd.Args,
+ "--env=domain="+opts.Live,
+ "--env=bootargs=",
+ "--publish=:4430-4450:4430-4450",
+ "--volume=/var/lib/acme:/var/lib/acme:ro")
+ } else {
+ cmd.Args = append(cmd.Args,
+ "--env=domain=localhost",
+ "--env=bootargs=-shutdown")
+ }
+ cmd.Args = append(cmd.Args,
depsImageName,
"bash", "-c", `
set -e -o pipefail
@@ -106,8 +118,8 @@ eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-
SUDO_FORCE_REMOVE=yes apt-get autoremove -y
/etc/init.d/postgresql start
-arvados-server init -cluster-id x1234
-exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+arvados-server init -cluster-id x1234 -domain=$domain
+exec arvados-server boot -listen-host=0.0.0.0 $bootargs
`)
cmd.Stdout = stdout
cmd.Stderr = stderr
diff --git a/lib/install/init.go b/lib/install/init.go
index 1d063506b..f79fae782 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -103,49 +103,49 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
Controller:
InternalURLs:
"http://0.0.0.0:8003/": {}
- ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4430/" ) }}
RailsAPI:
InternalURLs:
"http://0.0.0.0:8004/": {}
Websocket:
InternalURLs:
"http://0.0.0.0:8005/": {}
- ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "wss://." .Domain ":4436/" ) }}
Keepbalance:
InternalURLs:
"http://0.0.0.0:9005/": {}
GitHTTP:
InternalURLs:
"http://0.0.0.0:9001/": {}
- ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4437/" ) }}
DispatchCloud:
InternalURLs:
"http://0.0.0.0:9006/": {}
Keepproxy:
InternalURLs:
"http://0.0.0.0:25108/": {}
- ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4438/" ) }}
WebDAV:
InternalURLs:
"http://0.0.0.0:9002/": {}
- ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4439/" ) }}
WebDAVDownload:
InternalURLs:
"http://0.0.0.0:8004/": {}
- ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4439/" ) }}
Keepstore:
InternalURLs:
"http://0.0.0.0:25107/": {}
Composer:
- ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4440/composer" ) }}
Workbench1:
InternalURLs:
"http://0.0.0.0:8001/": {}
- ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
- #Workbench2:
- # InternalURLs:
- # "http://0.0.0.0:8002/": {}
- # ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4441/" ) }}
+ Workbench2:
+ InternalURLs:
+ "http://0.0.0.0:8002/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain ":4442/" ) }}
Health:
InternalURLs:
"http://0.0.0.0:9007/": {}
commit 3c4835a24aa6af9c7060c8b6d760a6eb32227774
Author: Tom Clegg <tom at curii.com>
Date: Fri Mar 11 09:37:38 2022 -0500
18700: Fix wb2 section in nginx.conf template.
Currently only for production mode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index b90c9b537..f212254d2 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -33,12 +33,13 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
return err
}
vars := map[string]string{
- "LISTENHOST": super.ListenHost,
- "SSLCERT": filepath.Join(super.tempdir, "server.crt"),
- "SSLKEY": filepath.Join(super.tempdir, "server.key"),
- "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
- "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
- "TMPDIR": super.wwwtempdir,
+ "LISTENHOST": super.ListenHost,
+ "SSLCERT": filepath.Join(super.tempdir, "server.crt"),
+ "SSLKEY": filepath.Join(super.tempdir, "server.key"),
+ "ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
+ "ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
+ "TMPDIR": super.wwwtempdir,
+ "ARVADOS_API_HOST": super.cluster.Services.Controller.ExternalURL.Host,
}
u := url.URL(super.cluster.Services.Controller.ExternalURL)
ctrlHost := u.Hostname()
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 44f8469b0..bfb1226f7 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -163,12 +163,13 @@ http {
server_name workbench2 workbench2.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://{{LISTENHOST}}:{{WORKBENCH2PORT}};
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
+ location /config.json {
+ return 200 '{ "API_HOST": "{{ARVADOS_API_HOST}}" }';
+ }
+ location / {
+ root /var/lib/arvados/workbench2;
+ index index.html;
+ try_files $uri $uri/ /index.html;
}
}
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 9c45a3205..9f27510f4 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -620,6 +620,7 @@ def run_nginx():
nginxconf = {}
nginxconf['LISTENHOST'] = 'localhost'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
+ nginxconf['ARVADOS_API_HOST'] = "0.0.0.0:" + str(external_port_from_config("Controller"))
nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
nginxconf['KEEPWEBDLSSLPORT'] = external_port_from_config("WebDAVDownload")
commit 405afdbe244c1b162781bcb393154870f4b7c56a
Author: Tom Clegg <tom at curii.com>
Date: Thu Mar 10 10:41:21 2022 -0500
18700: Move more dependency version numbers to consts.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index d31de66b1..414bb4820 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -31,6 +31,12 @@ var Command cmd.Handler = &installCommand{}
const goversion = "1.17.7"
const (
+ rubyversion = "2.7.5"
+ singularityversion = "3.7.4"
+ pjsversion = "1.9.8"
+ geckoversion = "0.24.0"
+ gradleversion = "5.3.1"
+ nodejsversion = "v12.22.2"
devtestDatabasePassword = "insecure_arvados_test"
workbench2version = "cfa81dfc3041cb459c8a0918a2732dfcf3a11d40"
)
@@ -227,7 +233,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
}
- rubyversion := "2.7.2"
rubymajorversion := rubyversion[:strings.LastIndex(rubyversion, ".")]
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
@@ -264,7 +269,6 @@ ln -sfv /var/lib/arvados/go/bin/* /usr/local/bin/
}
if !prod && !pkg {
- pjsversion := "1.9.8"
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
} else {
@@ -278,7 +282,6 @@ ln -sfv /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
}
}
- geckoversion := "0.24.0"
if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
logger.Print("geckodriver " + geckoversion + " already installed")
} else {
@@ -292,7 +295,6 @@ ln -sfv /var/lib/arvados/bin/geckodriver /usr/local/bin/
}
}
- gradleversion := "5.3.1"
if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
logger.Print("gradle " + gradleversion + " already installed")
} else {
@@ -310,7 +312,6 @@ rm ${zip}
}
}
- singularityversion := "3.7.4"
if havesingularityversion, err := exec.Command("/var/lib/arvados/bin/singularity", "--version").CombinedOutput(); err == nil && strings.Contains(string(havesingularityversion), singularityversion) {
logger.Print("singularity " + singularityversion + " already installed")
} else if dev || test {
@@ -453,7 +454,6 @@ make -C ./builddir install
}
if !prod {
- nodejsversion := "v12.22.2"
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
commit b106d92f8032f8ded98860f5c2da381ae04ec5a7
Author: Tom Clegg <tom at curii.com>
Date: Thu Mar 10 10:35:09 2022 -0500
18700: iceweasel->firefox for debian>10, drop from non-test env.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index d4b79ace2..d31de66b1 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -143,7 +143,6 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
"default-jdk-headless",
"default-jre-headless",
"gettext",
- "iceweasel",
"libattr1-dev",
"libcrypt-ssleay-perl",
"libfuse-dev",
@@ -181,10 +180,15 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
"wget",
"xvfb",
)
+ if test {
+ if osv.Debian && osv.Major <= 10 {
+ pkgs = append(pkgs, "iceweasel")
+ } else {
+ pkgs = append(pkgs, "firefox")
+ }
+ }
if dev || test {
- pkgs = append(pkgs,
- "squashfs-tools", // for singularity
- )
+ pkgs = append(pkgs, "squashfs-tools") // for singularity
}
switch {
case osv.Debian && osv.Major >= 10:
commit 5d4a115fc4a6d0d4c85bc85bbc4d257490d38d67
Author: Tom Clegg <tom at curii.com>
Date: Thu Mar 10 10:22:52 2022 -0500
18700: Update yarn build cmd, add VERSION, BUILD_NUMBER, GIT_COMMIT.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 69db3e4ef..d4b79ace2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -501,7 +501,7 @@ rm -rf build
if err = inst.runBash(`
cd /var/lib/arvados/arvados-workbench2
-yarn install --non-interactive
+yarn install
`, stdout, stderr); err != nil {
return 1
}
@@ -511,7 +511,7 @@ yarn install --non-interactive
// Install workbench2 app to /var/lib/arvados/workbench2/
if err = inst.runBash(`
cd /var/lib/arvados/arvados-workbench2
-yarn build
+VERSION="`+inst.PackageVersion+`" BUILD_NUMBER=1 GIT_COMMIT="`+workbench2version+`" yarn build
rsync -a --delete-after build/ /var/lib/arvados/workbench2/
`, stdout, stderr); err != nil {
return 1
commit 927a3b3bacc0fd9d87600ef176588fe74d33ee25
Author: Tom Clegg <tom at curii.com>
Date: Wed Mar 9 13:50:40 2022 -0500
18700: Use wb2 yarn3 branch.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ab099306b..69db3e4ef 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -32,7 +32,7 @@ const goversion = "1.17.7"
const (
devtestDatabasePassword = "insecure_arvados_test"
- workbench2version = "5e805cf2209d3afe42699e4658d8a12e50bcd5a4"
+ workbench2version = "cfa81dfc3041cb459c8a0918a2732dfcf3a11d40"
)
type installCommand struct {
commit 1cfb44788d81c316aff0bd6ff4f0e26eb3c70c90
Author: Tom Clegg <tom at curii.com>
Date: Wed Mar 9 02:50:12 2022 -0500
18700: Exclude nodejs from package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index 23a78d608..aec645a68 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -64,7 +64,7 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
// Remove unneeded files. This is much faster than "fpm
// --exclude X" because fpm copies everything into a staging
// area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go /var/lib/arvados/arvados-workbench2")
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go /var/lib/arvados/arvados-workbench2 /var/lib/arvados/node-*")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
commit a80d66bb3a2bea4b7339e5dd65315f8af5ffe99b
Author: Tom Clegg <tom at curii.com>
Date: Wed Mar 9 02:41:19 2022 -0500
18700: Accept relative package dir path.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/install.go b/cmd/arvados-package/install.go
index 5fdb7a875..b0a87ba3b 100644
--- a/cmd/arvados-package/install.go
+++ b/cmd/arvados-package/install.go
@@ -21,6 +21,10 @@ import (
func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
depsImageName := "arvados-package-deps-" + opts.TargetOS
depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+ absPackageDir, err := filepath.Abs(opts.PackageDir)
+ if err != nil {
+ return fmt.Errorf("error resolving PackageDir %q: %w", opts.PackageDir, err)
+ }
_, prog := filepath.Split(os.Args[0])
tmpdir, err := ioutil.TempDir("", prog+".")
@@ -40,7 +44,7 @@ func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr
cmd := exec.CommandContext(ctx, "docker", "run",
"--name", depsCtrName,
"--tmpfs", "/tmp:exec,mode=01777",
- "-v", opts.PackageDir+":/pkg:ro",
+ "-v", absPackageDir+":/pkg:ro",
"--env", "DEBIAN_FRONTEND=noninteractive",
opts.TargetOS,
"bash", "-c", `
@@ -64,7 +68,7 @@ rm /etc/apt/sources.list.d/arvados-local.list
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
@@ -72,7 +76,7 @@ rm /etc/apt/sources.list.d/arvados-local.list
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker commit: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
}
@@ -82,7 +86,7 @@ rm /etc/apt/sources.list.d/arvados-local.list
}
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
"--tmpfs", "/tmp:exec,mode=01777",
- "-v", opts.PackageDir+":/pkg:ro",
+ "-v", absPackageDir+":/pkg:ro",
"--env", "DEBIAN_FRONTEND=noninteractive",
depsImageName,
"bash", "-c", `
@@ -109,7 +113,7 @@ exec arvados-server boot -listen-host 0.0.0.0 -shutdown
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
return nil
}
commit 0203f7c34ebeedf9611b11de5d9309d5a125d3a5
Author: Tom Clegg <tom at curii.com>
Date: Wed Mar 9 02:39:26 2022 -0500
18700: Fix URL handling when controller external url is :443.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 44bcbc395..b90c9b537 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -40,10 +40,8 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
"TMPDIR": super.wwwtempdir,
}
- ctrlHost, _, err := net.SplitHostPort(super.cluster.Services.Controller.ExternalURL.Host)
- if err != nil {
- return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
- }
+ u := url.URL(super.cluster.Services.Controller.ExternalURL)
+ ctrlHost := u.Hostname()
if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
f.Close()
vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 0905fd44a..00e981afc 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -669,10 +669,8 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
- defaultExtHost, _, err := net.SplitHostPort(cluster.Services.Controller.ExternalURL.Host)
- if err != nil {
- return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
- }
+ u := url.URL(cluster.Services.Controller.ExternalURL)
+ defaultExtHost := u.Hostname()
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
&cluster.Services.DispatchCloud,
commit adbbe252483b5442707c1e2e03f5347f913e1125
Author: Tom Clegg <tom at curii.com>
Date: Thu Feb 17 10:25:11 2022 -0500
18700: Don't start workbench2 in integration test clusters.
Default to -no-workbench2=true until wb2 test setup script can be
updated.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index e13848db7..9c3047a7b 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -70,6 +70,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
+ flags.BoolVar(&super.NoWorkbench2, "no-workbench2", true, "do not run workbench2")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 8746183e6..0905fd44a 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -44,6 +44,7 @@ type Supervisor struct {
ControllerAddr string // e.g., 127.0.0.1:8000
Workbench2Source string // e.g., /home/username/src/arvados-workbench2
NoWorkbench1 bool
+ NoWorkbench2 bool
OwnTemporaryDatabase bool
Stderr io.Writer
@@ -251,7 +252,6 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
- runWorkbench2{svc: super.cluster.Services.Workbench2},
seedDatabase{},
}
if !super.NoWorkbench1 {
@@ -260,6 +260,11 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
)
}
+ if !super.NoWorkbench2 {
+ tasks = append(tasks,
+ runWorkbench2{svc: super.cluster.Services.Workbench2},
+ )
+ }
if super.ClusterType != "test" {
tasks = append(tasks,
runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
@@ -704,7 +709,8 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"}
}
}
- if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 {
+ if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 ||
+ super.NoWorkbench2 && svc == &cluster.Services.Workbench2 {
// When workbench1 is disabled, it gets an
// ExternalURL (so we have a valid listening
// port to write in our Nginx config) but no
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 9f5d12598..6ffbbd272 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -135,6 +135,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
tc.Super.NoWorkbench1 = true
+ tc.Super.NoWorkbench2 = true
tc.Start()
s.testClusters[id] = tc
}
diff --git a/tools/sync-groups/federation_test.go b/tools/sync-groups/federation_test.go
index 1bbdaa3fa..d5fed3e29 100644
--- a/tools/sync-groups/federation_test.go
+++ b/tools/sync-groups/federation_test.go
@@ -115,6 +115,7 @@ func (s *FederationSuite) SetUpSuite(c *check.C) {
filepath.Join(cwd, "..", ".."),
id, cfg, "127.0.0."+id[3:], c.Log)
tc.Super.NoWorkbench1 = true
+ tc.Super.NoWorkbench2 = true
tc.Start()
s.testClusters[id] = tc
}
commit 060dc61639b8a5ac4458bc21d1120b3ad508b0a3
Author: Tom Clegg <tom at curii.com>
Date: Thu Feb 17 10:17:35 2022 -0500
18700: Add workbench2 to arvados-boot.
Includes related changes:
* Health aggregator obeys TLS.Insecure=true
* Use host part of -controller-address for autofilled ExternalURLs
* Use /var/lib/acme/live/{domain}/privkey if readable
* Obey -listen-host for ExternalURLs too
* Always run own postgresql instance on localhost
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/build.go b/cmd/arvados-package/build.go
index 1437f4b77..8268ea9ba 100644
--- a/cmd/arvados-package/build.go
+++ b/cmd/arvados-package/build.go
@@ -53,6 +53,11 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
return err
}
defer os.RemoveAll(tmpdir)
+ if abs, err := filepath.Abs(tmpdir); err != nil {
+ return fmt.Errorf("error getting absolute path of tmpdir %s: %w", tmpdir, err)
+ } else {
+ tmpdir = abs
+ }
selfbin, err := os.Readlink("/proc/self/exe")
if err != nil {
@@ -87,7 +92,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
@@ -120,7 +125,7 @@ func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Wr
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("docker run: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index ca63929e9..23a78d608 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -64,12 +64,12 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
// Remove unneeded files. This is much faster than "fpm
// --exclude X" because fpm copies everything into a staging
// area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go /var/lib/arvados/arvados-workbench2")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("rm -rf [...]: %w", err)
+ return fmt.Errorf("%v: %w", cmd.Args, err)
}
format := "deb" // TODO: rpm
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index c8b945bea..c5465ee56 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -5,6 +5,10 @@
package main
import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
"os"
"git.arvados.org/arvados.git/lib/boot"
@@ -40,6 +44,7 @@ var (
"init": install.InitCommand,
"keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
+ "workbench2": wb2command{},
"ws": ws.Command,
})
)
@@ -47,3 +52,31 @@ var (
func main() {
os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
+
+type wb2command struct{}
+
+func (wb2command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ if len(args) != 3 {
+ fmt.Fprintf(stderr, "usage: %s api-host listen-addr app-dir\n", prog)
+ return 1
+ }
+ configJSON, err := json.Marshal(map[string]string{"API_HOST": args[0]})
+ if err != nil {
+ fmt.Fprintf(stderr, "json.Marshal: %s\n", err)
+ return 1
+ }
+ mux := http.NewServeMux()
+ mux.Handle("/", http.FileServer(http.Dir(args[2])))
+ mux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) {
+ w.Write(configJSON)
+ })
+ mux.HandleFunc("/_health/ping", func(w http.ResponseWriter, _ *http.Request) {
+ io.WriteString(w, `{"health":"OK"}`)
+ })
+ err = http.ListenAndServe(args[1], mux)
+ if err != nil {
+ fmt.Fprintln(stderr, err.Error())
+ return 1
+ }
+ return 0
+}
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 96241d24b..e13848db7 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -68,6 +68,7 @@ func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, std
flags.StringVar(&super.ClusterType, "type", "production", "cluster `type`: development, test, or production")
flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
+ flags.StringVar(&super.Workbench2Source, "workbench2-source", "../arvados-workbench2", "path to arvados-workbench2 source tree")
flags.BoolVar(&super.NoWorkbench1, "no-workbench1", false, "do not run workbench1")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 5826e5c01..44bcbc395 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -40,6 +40,15 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
"TMPDIR": super.wwwtempdir,
}
+ ctrlHost, _, err := net.SplitHostPort(super.cluster.Services.Controller.ExternalURL.Host)
+ if err != nil {
+ return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
+ }
+ if f, err := os.Open("/var/lib/acme/live/" + ctrlHost + "/privkey"); err == nil {
+ f.Close()
+ vars["SSLCERT"] = "/var/lib/acme/live/" + ctrlHost + "/cert"
+ vars["SSLKEY"] = "/var/lib/acme/live/" + ctrlHost + "/privkey"
+ }
for _, cmpt := range []struct {
varname string
svc arvados.Service
@@ -51,6 +60,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
{"GIT", super.cluster.Services.GitHTTP},
{"HEALTH", super.cluster.Services.Health},
{"WORKBENCH1", super.cluster.Services.Workbench1},
+ {"WORKBENCH2", super.cluster.Services.Workbench2},
{"WS", super.cluster.Services.Websocket},
} {
var host, port string
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 2c89ccdb0..8746183e6 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -42,6 +42,7 @@ type Supervisor struct {
ClusterType string // e.g., production
ListenHost string // e.g., localhost
ControllerAddr string // e.g., 127.0.0.1:8000
+ Workbench2Source string // e.g., /home/username/src/arvados-workbench2
NoWorkbench1 bool
OwnTemporaryDatabase bool
Stderr io.Writer
@@ -250,6 +251,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api"}}},
+ runWorkbench2{svc: super.cluster.Services.Workbench2},
seedDatabase{},
}
if !super.NoWorkbench1 {
@@ -482,6 +484,7 @@ type runOptions struct {
output io.Writer // attach stdout
env []string // add/replace environment variables
user string // run as specified user
+ stdin io.Reader
}
// RunProgram runs prog with args, using dir as working directory. If ctx is
@@ -525,6 +528,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, opts runOpt
}
cmd := exec.Command(super.lookPath(prog), args...)
+ cmd.Stdin = opts.stdin
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
@@ -628,32 +632,42 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
return err
}
usedPort := map[string]bool{}
- nextPort := func(host string) string {
+ nextPort := func(host string) (string, error) {
for {
port, err := availablePort(host)
if err != nil {
- panic(err)
+ port, err = availablePort(super.ListenHost)
+ }
+ if err != nil {
+ return "", err
}
if usedPort[port] {
continue
}
usedPort[port] = true
- return port
+ return port, nil
}
}
if cluster.Services.Controller.ExternalURL.Host == "" {
h, p, err := net.SplitHostPort(super.ControllerAddr)
if err != nil {
- return err
+ return fmt.Errorf("SplitHostPort(ControllerAddr): %w", err)
}
if h == "" {
h = super.ListenHost
}
if p == "0" {
- p = nextPort(h)
+ p, err = nextPort(h)
+ if err != nil {
+ return err
+ }
}
cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
}
+ defaultExtHost, _, err := net.SplitHostPort(cluster.Services.Controller.ExternalURL.Host)
+ if err != nil {
+ return fmt.Errorf("SplitHostPort(Controller.ExternalURL.Host): %w", err)
+ }
for _, svc := range []*arvados.Service{
&cluster.Services.Controller,
&cluster.Services.DispatchCloud,
@@ -666,21 +680,28 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
&cluster.Services.WebDAVDownload,
&cluster.Services.Websocket,
&cluster.Services.Workbench1,
+ &cluster.Services.Workbench2,
} {
if svc == &cluster.Services.DispatchCloud && super.ClusterType == "test" {
continue
}
if svc.ExternalURL.Host == "" {
+ port, err := nextPort(defaultExtHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(defaultExtHost, port)
if svc == &cluster.Services.Controller ||
svc == &cluster.Services.GitHTTP ||
svc == &cluster.Services.Health ||
svc == &cluster.Services.Keepproxy ||
svc == &cluster.Services.WebDAV ||
svc == &cluster.Services.WebDAVDownload ||
- svc == &cluster.Services.Workbench1 {
- svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
+ svc == &cluster.Services.Workbench1 ||
+ svc == &cluster.Services.Workbench2 {
+ svc.ExternalURL = arvados.URL{Scheme: "https", Host: host, Path: "/"}
} else if svc == &cluster.Services.Websocket {
- svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
+ svc.ExternalURL = arvados.URL{Scheme: "wss", Host: host, Path: "/websocket"}
}
}
if super.NoWorkbench1 && svc == &cluster.Services.Workbench1 {
@@ -692,8 +713,13 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
continue
}
if len(svc.InternalURLs) == 0 {
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
- {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {},
+ {Scheme: "http", Host: host, Path: "/"}: {},
}
}
}
@@ -721,7 +747,12 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
if super.ClusterType == "test" {
// Add a second keepstore process.
- cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
+ port, err := nextPort(super.ListenHost)
+ if err != nil {
+ return err
+ }
+ host := net.JoinHostPort(super.ListenHost, port)
+ cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: host, Path: "/"}] = arvados.ServiceInstance{}
// Create a directory-backed volume for each keepstore
// process.
@@ -755,10 +786,14 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
}
}
if super.OwnTemporaryDatabase {
+ port, err := nextPort("localhost")
+ if err != nil {
+ return err
+ }
cluster.PostgreSQL.Connection = arvados.PostgreSQLConnection{
"client_encoding": "utf8",
- "host": super.ListenHost,
- "port": nextPort(super.ListenHost),
+ "host": "localhost",
+ "port": port,
"dbname": "arvados_test",
"user": "arvados",
"password": "insecure_arvados_test",
diff --git a/lib/boot/workbench2.go b/lib/boot/workbench2.go
new file mode 100644
index 000000000..5a319ebfe
--- /dev/null
+++ b/lib/boot/workbench2.go
@@ -0,0 +1,73 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "io/ioutil"
+ "net"
+ "os"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+type runWorkbench2 struct {
+ svc arvados.Service
+}
+
+func (runner runWorkbench2) String() string {
+ return "runWorkbench2"
+}
+
+func (runner runWorkbench2) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ host, port, err := internalPort(runner.svc)
+ if err != nil {
+ return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
+ }
+ super.waitShutdown.Add(1)
+ go func() {
+ defer super.waitShutdown.Done()
+ if super.ClusterType == "production" {
+ err = super.RunProgram(ctx, "/var/lib/arvados/workbench2", runOptions{
+ user: "www-data",
+ }, "arvados-server", "workbench2", super.cluster.Services.Controller.ExternalURL.Host, net.JoinHostPort(host, port), ".")
+ } else if super.Workbench2Source == "" {
+ super.logger.Info("skipping Workbench2: Workbench2Source==\"\" and not in production mode")
+ return
+ } else {
+ stdinr, stdinw := io.Pipe()
+ defer stdinw.Close()
+ go func() {
+ <-ctx.Done()
+ stdinw.Close()
+ }()
+ if err = os.Mkdir(super.Workbench2Source+"/public/_health", 0777); err != nil && !errors.Is(err, fs.ErrExist) {
+ fail(err)
+ return
+ }
+ if err = ioutil.WriteFile(super.Workbench2Source+"/public/_health/ping", []byte(`{"health":"OK"}`), 0666); err != nil {
+ fail(err)
+ return
+ }
+ err = super.RunProgram(ctx, super.Workbench2Source, runOptions{
+ env: []string{
+ "CI=true",
+ "HTTPS=false",
+ "PORT=" + port,
+ "REACT_APP_ARVADOS_API_HOST=" + super.cluster.Services.Controller.ExternalURL.Host,
+ },
+ // If we don't connect stdin, "yarn start" just exits.
+ stdin: stdinr,
+ }, "yarn", "start")
+ fail(errors.New("`yarn start` exited"))
+ }
+ fail(err)
+ }()
+ return nil
+}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 483ce9c93..ab099306b 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -28,8 +28,12 @@ import (
var Command cmd.Handler = &installCommand{}
-const devtestDatabasePassword = "insecure_arvados_test"
-const goversion = "1.17.1"
+const goversion = "1.17.7"
+
+const (
+ devtestDatabasePassword = "insecure_arvados_test"
+ workbench2version = "5e805cf2209d3afe42699e4658d8a12e50bcd5a4"
+)
type installCommand struct {
ClusterType string
@@ -247,7 +251,7 @@ make install
cd /tmp
rm -rf /var/lib/arvados/go/
wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
-ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
+ln -sfv /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -263,7 +267,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
wget --progress=dot:giga -O- https://cache.arvados.org/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
-ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
+ln -sfv /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -277,21 +281,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
err = inst.runBash(`
GD=v`+geckoversion+`
wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
-ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
-`, stdout, stderr)
- if err != nil {
- return 1
- }
- }
-
- nodejsversion := "v12.22.2"
- if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
- logger.Print("nodejs " + nodejsversion + " already installed")
- } else {
- err = inst.runBash(`
-NJS=`+nodejsversion+`
-wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
-ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+ln -sfv /var/lib/arvados/bin/geckodriver /usr/local/bin/
`, stdout, stderr)
if err != nil {
return 1
@@ -308,7 +298,7 @@ zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
unzip -o -d /var/lib/arvados ${zip}
-ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
+ln -sfv /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
rm ${zip}
`, stdout, stderr)
if err != nil {
@@ -458,7 +448,75 @@ make -C ./builddir install
}
}
+ if !prod {
+ nodejsversion := "v12.22.2"
+ if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
+ logger.Print("nodejs " + nodejsversion + " already installed")
+ } else {
+ err = inst.runBash(`
+NJS=`+nodejsversion+`
+wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+ln -sfv /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if haveyarnversion, err := exec.Command("/usr/local/bin/yarn", "--version").CombinedOutput(); err == nil && len(haveyarnversion) > 0 {
+ logger.Print("yarn " + strings.TrimSpace(string(haveyarnversion)) + " already installed")
+ } else {
+ err = inst.runBash(`
+npm install -g yarn
+ln -sfv /var/lib/arvados/node-`+nodejsversion+`-linux-x64/bin/{yarn,yarnpkg} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if havewb2version, err := exec.Command("git", "--git-dir=/var/lib/arvados/arvados-workbench2/.git", "log", "-n1", "--format=%H").CombinedOutput(); err == nil && string(havewb2version) == workbench2version+"\n" {
+ logger.Print("workbench2 repo is already at " + workbench2version)
+ } else {
+ err = inst.runBash(`
+V=`+workbench2version+`
+cd /var/lib/arvados
+if [[ ! -e arvados-workbench2 ]]; then
+ git clone https://git.arvados.org/arvados-workbench2.git
+ cd arvados-workbench2
+ git checkout $V
+else
+ cd arvados-workbench2
+ if ! git checkout $V; then
+ git fetch
+ git checkout $V
+ fi
+fi
+rm -rf build
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if err = inst.runBash(`
+cd /var/lib/arvados/arvados-workbench2
+yarn install --non-interactive
+`, stdout, stderr); err != nil {
+ return 1
+ }
+ }
+
if prod || pkg {
+ // Install workbench2 app to /var/lib/arvados/workbench2/
+ if err = inst.runBash(`
+cd /var/lib/arvados/arvados-workbench2
+yarn build
+rsync -a --delete-after build/ /var/lib/arvados/workbench2/
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
for dstdir, srcdir := range map[string]string{
"railsapi": "services/api",
diff --git a/sdk/go/health/aggregator.go b/sdk/go/health/aggregator.go
index a666ef8ec..296ef65dd 100644
--- a/sdk/go/health/aggregator.go
+++ b/sdk/go/health/aggregator.go
@@ -6,8 +6,8 @@ package health
import (
"context"
+ "crypto/tls"
"encoding/json"
- "errors"
"fmt"
"net/http"
"net/url"
@@ -35,7 +35,13 @@ type Aggregator struct {
}
func (agg *Aggregator) setup() {
- agg.httpClient = http.DefaultClient
+ agg.httpClient = &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: agg.Cluster.TLS.Insecure,
+ },
+ },
+ }
if agg.timeout == 0 {
// this is always the case, except in the test suite
agg.timeout = defaultTimeout
@@ -176,19 +182,14 @@ func (agg *Aggregator) pingURL(svcURL arvados.URL) (*url.URL, error) {
func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
t0 := time.Now()
-
- var err error
defer func() {
result.ResponseTime = json.Number(fmt.Sprintf("%.6f", time.Since(t0).Seconds()))
- if err != nil {
- result.Health, result.Error = "ERROR", err.Error()
- } else {
- result.Health = "OK"
- }
}()
+ result.Health = "ERROR"
req, err := http.NewRequest("GET", target.String(), nil)
if err != nil {
+ result.Error = err.Error()
return
}
req.Header.Set("Authorization", "Bearer "+agg.Cluster.ManagementToken)
@@ -201,22 +202,26 @@ func (agg *Aggregator) ping(target *url.URL) (result CheckResult) {
req = req.WithContext(ctx)
resp, err := agg.httpClient.Do(req)
if err != nil {
+ result.Error = err.Error()
return
}
result.HTTPStatusCode = resp.StatusCode
result.HTTPStatusText = resp.Status
err = json.NewDecoder(resp.Body).Decode(&result.Response)
if err != nil {
- err = fmt.Errorf("cannot decode response: %s", err)
+ result.Error = fmt.Sprintf("cannot decode response: %s", err)
} else if resp.StatusCode != http.StatusOK {
- err = fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
+ result.Error = fmt.Sprintf("HTTP %d %s", resp.StatusCode, resp.Status)
} else if h, _ := result.Response["health"].(string); h != "OK" {
if e, ok := result.Response["error"].(string); ok && e != "" {
- err = errors.New(e)
+ result.Error = e
+ return
} else {
- err = fmt.Errorf("health=%q in ping response", h)
+ result.Error = fmt.Sprintf("health=%q in ping response", h)
+ return
}
}
+ result.Health = "OK"
return
}
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 35b780071..44f8469b0 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -147,7 +147,7 @@ http {
}
server {
listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
- server_name workbench1 workbench.*;
+ server_name workbench1 workbench1.* workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -158,4 +158,17 @@ http {
proxy_redirect off;
}
}
+ server {
+ listen {{LISTENHOST}}:{{WORKBENCH2SSLPORT}} ssl;
+ server_name workbench2 workbench2.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://{{LISTENHOST}}:{{WORKBENCH2PORT}};
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index f91783250..9c45a3205 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -634,6 +634,8 @@ def run_nginx():
nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
nginxconf['WORKBENCH1SSLPORT'] = external_port_from_config("Workbench1")
+ nginxconf['WORKBENCH2PORT'] = internal_port_from_config("Workbench2")
+ nginxconf['WORKBENCH2SSLPORT'] = external_port_from_config("Workbench2")
nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
nginxconf['ACCESSLOG'] = _logfilename('nginx_access')
@@ -667,6 +669,8 @@ def setup_config():
websocket_external_port = find_available_port()
workbench1_port = find_available_port()
workbench1_external_port = find_available_port()
+ workbench2_port = find_available_port()
+ workbench2_external_port = find_available_port()
git_httpd_port = find_available_port()
git_httpd_external_port = find_available_port()
health_httpd_port = find_available_port()
@@ -720,6 +724,12 @@ def setup_config():
"http://%s:%s"%(localhost, workbench1_port): {},
},
},
+ "Workbench2": {
+ "ExternalURL": "https://%s:%s/" % (localhost, workbench2_external_port),
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, workbench2_port): {},
+ },
+ },
"GitHTTP": {
"ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
"InternalURLs": {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list