[ARVADOS] created: 1.3.0-2751-g426103b2d
Git user
git at public.arvados.org
Fri Aug 21 17:55:47 UTC 2020
at 426103b2d2f071ab0d57b3f9aaea58a3f8455c4f (commit)
commit 426103b2d2f071ab0d57b3f9aaea58a3f8455c4f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Aug 17 09:14:52 2020 -0400
16306: Add --os argument to package build scripts.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 7eca33e79..39a8a56c8 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -19,6 +19,11 @@ set -e -o pipefail
declare -A opts=()
while [[ $# -gt 0 ]]; do
case "$1" in
+ --os)
+ shift
+ opts[os]="$1"
+ shift
+ ;;
--shell)
shift
opts[shell]=1
@@ -54,7 +59,7 @@ for inject in "$@"; do
esac
done
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
installimage=arvados-installpackage-${osbase}
command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
if [[ "${opts[shell]}" ]]; then
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 63b5b5c2d..78b08d7dc 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -25,6 +25,10 @@ while [[ $# -gt 0 ]]; do
--force-installimage)
opts[force-installimage]=1
;;
+ --os)
+ opts[os]="$1"
+ shift
+ ;;
*)
echo >&2 "invalid argument '$arg'"
exit 2
@@ -43,7 +47,7 @@ cleanup() {
trap cleanup ERR EXIT
version=$(git describe --tag --dirty)
-osbase=debian:10
+osbase=${opts[os]:-debian:10}
mkdir -p /tmp/pkg
commit bb1054c0e6a9bed14e593004f380111b2e948047
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Aug 17 09:13:45 2020 -0400
16306: Update gem command line option spelling.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index aee955ea2..00713550d 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -62,7 +62,7 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
}
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 481300b45..03464aaf7 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -56,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 7b22f45fd..b39515cb1 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -431,7 +431,7 @@ rm ${zip}
{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
commit f2da5f54256377e604c76b74805381b2acf38f50
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Aug 7 15:00:22 2020 -0400
16306: Add "ping" health check to keep-balance.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 65bd8d4cf..8b4ee84c7 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -9,13 +9,13 @@ import (
"flag"
"fmt"
"io"
- "net/http"
"os"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@@ -83,7 +83,6 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
}
srv := &Server{
- Handler: http.NotFoundHandler(),
Cluster: cluster,
ArvClient: ac,
RunOptions: options,
@@ -91,6 +90,11 @@ func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.W
Logger: options.Logger,
Dumper: options.Dumper,
}
+ srv.Handler = &health.Handler{
+ Token: cluster.ManagementToken,
+ Prefix: "/_health/",
+ Routes: health.Routes{"ping": srv.CheckHealth},
+ }
go srv.run()
return srv
commit f81d411d2becc23fe640366620d9568149d89bd6
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Aug 7 10:51:39 2020 -0400
16306: Set Workbench1 secret key.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/init.go b/lib/install/init.go
index ca8d4da92..8debf59be 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -178,6 +178,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
DriverParameters:
Root: /var/lib/arvados/keep
Replication: 2
+ Workbench:
+ SecretKeyBase: {{printf "%q" ( .RandomHex 50 )}}
`)
if err != nil {
return 1
commit 24987125b33655306c048ab64f0ffdfb2f6a6c5b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 15:18:40 2020 -0400
16306: Fix nginx port probe.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index a7c09a722..b11d9fd49 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"path/filepath"
@@ -107,5 +108,11 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
"-g", "pid "+filepath.Join(super.tempdir, "nginx.pid")+";",
"-c", conffile))
}()
- return waitForConnect(ctx, super.cluster.Services.Controller.ExternalURL.Host)
+ // Choose one of the ports where Nginx should listen, and wait
+ // here until we can connect. If ExternalURL is https://foo (with no port) then we connect to "foo:https"
+ testurl := url.URL(super.cluster.Services.Controller.ExternalURL)
+ if testurl.Port() == "" {
+ testurl.Host = net.JoinHostPort(testurl.Host, testurl.Scheme)
+ }
+ return waitForConnect(ctx, testurl.Host)
}
commit 1c2f9259deeedaad103b80fda7f440b8e60caa3f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 14:27:51 2020 -0400
16306: Include wwwtmp dir in package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index c5596d104..7b22f45fd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -178,7 +178,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
os.Mkdir("/var/lib/arvados", 0755)
os.Mkdir("/var/lib/arvados/tmp", 0700)
- if prod {
+ if prod || pkg {
os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
u, er := user.Lookup("www-data")
if er != nil {
commit 75d050ab135619fcefecbfa32aaad4dab33e7588
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:30:08 2020 -0400
16306: Fixup nginx in arvados-boot production mode.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index 0b9874295..7eca33e79 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -40,11 +40,18 @@ tmpdir=$(mktemp -d)
version=$(git describe --tag --dirty)
declare -a volargs=()
-for srcdir in "$@"; do
- echo >&2 "building $srcdir..."
- (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
- cmd="$(basename "$srcdir")"
- volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+for inject in "$@"; do
+ case "$inject" in
+ nginx.conf)
+ volargs+=(-v "$(pwd)/sdk/python/tests/$inject:/var/lib/arvados/share/$inject:ro")
+ ;;
+ *)
+ echo >&2 "building $inject..."
+ (cd $inject && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+ cmd="$(basename "$inject")"
+ volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+ ;;
+ esac
done
osbase=debian:10
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index c1da7d18d..a7c09a722 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -69,7 +69,13 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
}
vars[cmpt.varname+"SSLPORT"] = port
}
- tmpl, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf"))
+ var conftemplate string
+ if super.ClusterType == "production" {
+ conftemplate = "/var/lib/arvados/share/nginx.conf"
+ } else {
+ conftemplate = filepath.Join(super.SourcePath, "sdk", "python", "tests", "nginx.conf")
+ }
+ tmpl, err := ioutil.ReadFile(conftemplate)
if err != nil {
return err
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 5f92a6569..51f5c8bfd 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -252,7 +252,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
}
if super.ClusterType != "test" {
tasks = append(tasks,
- runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
+ runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud},
runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3f19aa1a8..c5596d104 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -486,6 +486,15 @@ rm ${zip}
return 1
}
}
+
+ // Copy assets from source tree to /var/lib/arvados/share
+ cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf"))
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
}
return 0
diff --git a/lib/install/init.go b/lib/install/init.go
index 6d4f19757..ca8d4da92 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -170,6 +170,8 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
user: arvados
password: {{printf "%q" .PostgreSQLPassword}}
SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ TLS:
+ Insecure: true
Volumes:
{{.ClusterID}}-nyw5e-000000000000000:
Driver: Directory
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 85b4f5b37..cdca68dd6 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -16,12 +16,28 @@ http {
fastcgi_temp_path "{{TMPDIR}}";
uwsgi_temp_path "{{TMPDIR}}";
scgi_temp_path "{{TMPDIR}}";
+ upstream controller {
+ server {{LISTENHOST}}:{{CONTROLLERPORT}};
+ }
+ server {
+ listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl;
+ server_name controller ~.*;
+ ssl_certificate "{{SSLCERT}}";
+ ssl_certificate_key "{{SSLKEY}}";
+ location / {
+ proxy_pass http://controller;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream arv-git-http {
server {{LISTENHOST}}:{{GITPORT}};
}
server {
- listen {{LISTENHOST}}:{{GITSSLPORT}} ssl default_server;
- server_name arv-git-http;
+ listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
+ server_name arv-git-http git.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -36,8 +52,8 @@ http {
server {{LISTENHOST}}:{{KEEPPROXYPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl default_server;
- server_name keepproxy;
+ listen {{LISTENHOST}}:{{KEEPPROXYSSLPORT}} ssl;
+ server_name keepproxy keep.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -55,8 +71,8 @@ http {
server {{LISTENHOST}}:{{KEEPWEBPORT}};
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl default_server;
- server_name keep-web;
+ listen {{LISTENHOST}}:{{KEEPWEBSSLPORT}} ssl;
+ server_name keep-web collections.* ~\.collections\.;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -75,8 +91,8 @@ http {
server {{LISTENHOST}}:{{HEALTHPORT}};
}
server {
- listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
- server_name health;
+ listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl;
+ server_name health health.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -91,8 +107,8 @@ http {
}
}
server {
- listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
- server_name keep-web-dl ~.*;
+ listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl;
+ server_name keep-web-dl download.* ~.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -111,8 +127,8 @@ http {
server {{LISTENHOST}}:{{WSPORT}};
}
server {
- listen {{LISTENHOST}}:{{WSSSLPORT}} ssl default_server;
- server_name websocket;
+ listen {{LISTENHOST}}:{{WSSSLPORT}} ssl;
+ server_name websocket ws.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -129,8 +145,8 @@ http {
server {{LISTENHOST}}:{{WORKBENCH1PORT}};
}
server {
- listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl default_server;
- server_name workbench1;
+ listen {{LISTENHOST}}:{{WORKBENCH1SSLPORT}} ssl;
+ server_name workbench1 workbench.*;
ssl_certificate "{{SSLCERT}}";
ssl_certificate_key "{{SSLKEY}}";
location / {
@@ -141,20 +157,4 @@ http {
proxy_redirect off;
}
}
- upstream controller {
- server {{LISTENHOST}}:{{CONTROLLERPORT}};
- }
- server {
- listen {{LISTENHOST}}:{{CONTROLLERSSLPORT}} ssl default_server;
- server_name controller;
- ssl_certificate "{{SSLCERT}}";
- ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://controller;
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
}
commit 8891660b68c10dd9d019f1caf8b7bc0b1eb493d8
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:28:11 2020 -0400
16306: Option to start shell in docker-boot.sh.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
index e8703e41f..0b9874295 100755
--- a/cmd/arvados-dev/docker-boot.sh
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -16,6 +16,19 @@
set -e -o pipefail
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --shell)
+ shift
+ opts[shell]=1
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
cleanup() {
if [[ -n "${tmpdir}" ]]; then
rm -rf "${tmpdir}"
@@ -36,7 +49,11 @@ done
osbase=debian:10
installimage=arvados-installpackage-${osbase}
+command="/var/lib/arvados/bin/arvados-server boot -listen-host 0.0.0.0"
+if [[ "${opts[shell]}" ]]; then
+ command="bash -login"
+fi
docker run -it --rm \
"${volargs[@]}" \
"${installimage}" \
- bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
+ bash -c "/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && $command"
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
index 3c6e177cf..63b5b5c2d 100755
--- a/cmd/arvados-dev/docker-build-install.sh
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -27,7 +27,8 @@ while [[ $# -gt 0 ]]; do
;;
*)
echo >&2 "invalid argument '$arg'"
- exit 1
+ exit 2
+ ;;
esac
done
commit 398ddf58f05416e0ce62ac8faef018a44bf8cc4c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:26:57 2020 -0400
16306: Add hostname as subjectAltName.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index f0797c2ac..8f6339e63 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io/ioutil"
"net"
+ "os"
"path/filepath"
)
@@ -27,9 +28,14 @@ func (createCertificates) String() string {
func (createCertificates) Run(ctx context.Context, fail func(error), super *Supervisor) error {
var san string
if net.ParseIP(super.ListenHost) != nil {
- san = fmt.Sprintf("IP:%s", super.ListenHost)
+ san += fmt.Sprintf(",IP:%s", super.ListenHost)
} else {
- san = fmt.Sprintf("DNS:%s", super.ListenHost)
+ san += fmt.Sprintf(",DNS:%s", super.ListenHost)
+ }
+ if hostname, err := os.Hostname(); err != nil {
+ return fmt.Errorf("hostname: %w", err)
+ } else {
+ san += ",DNS:" + hostname
}
// Generate root key
@@ -52,7 +58,7 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
if err != nil {
return err
}
- err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain,%s\n", san))...), 0644)
+ err = ioutil.WriteFile(filepath.Join(super.tempdir, "server.cfg"), append(defaultconf, []byte(fmt.Sprintf("\n[SAN]\nsubjectAltName=DNS:localhost,DNS:localhost.localdomain%s\n", san))...), 0644)
if err != nil {
return err
}
commit 7ac1ea4e9bc1b399368a95649647a55c8093d649
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 30 11:25:35 2020 -0400
16306: Show logprefix like 'passenger at railsapi'
for commands like 'sudo ... bundle exec passenger ...'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 2d7697233..5f92a6569 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -483,17 +483,34 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "setuidgid" && len(args) >= 3 {
- logprefix = args[2]
- }
- logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
- if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
- logprefix = args[1]
- } else if logprefix == "arvados-server" && len(args) > 1 {
- logprefix = args[0]
- }
- if !strings.HasPrefix(dir, "/") {
- logprefix = dir + ": " + logprefix
+ {
+ if logprefix == "setuidgid" && len(args) >= 3 {
+ logprefix = args[2]
+ }
+ innerargs := args
+ if logprefix == "sudo" {
+ for i := 0; i < len(args); i++ {
+ if args[i] == "-u" {
+ i++
+ } else if args[i] == "-E" || strings.Contains(args[i], "=") {
+ } else {
+ logprefix = args[i]
+ innerargs = args[i+1:]
+ break
+ }
+ }
+ }
+ logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/")
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
+ if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" {
+ _, dirbase := filepath.Split(dir)
+ logprefix = innerargs[1] + "@" + dirbase
+ } else if logprefix == "arvados-server" && len(args) > 1 {
+ logprefix = args[0]
+ }
+ if !strings.HasPrefix(dir, "/") {
+ logprefix = dir + ": " + logprefix
+ }
}
cmd := exec.Command(super.lookPath(prog), args...)
commit 60801a7c485c50e3c2f518f9b3ad6f86c8783fca
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Jul 23 17:31:39 2020 -0400
16306: Packaging dev cycle, type=production support in lib/boot.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
index 4b395c4ba..aee955ea2 100644
--- a/cmd/arvados-dev/buildpackage.go
+++ b/cmd/arvados-dev/buildpackage.go
@@ -85,12 +85,14 @@ func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin
cmd.Args = append(cmd.Args, "--depends", pkg)
}
cmd.Args = append(cmd.Args,
+ "--deb-use-file-permissions",
+ "--rpm-use-file-permissions",
"--exclude", "/var/lib/arvados/go",
"/var/lib/arvados",
"/var/www/.gem",
"/var/www/.passenger",
)
- fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
cmd.Dir = bldr.OutputDir
cmd.Stdout = stdout
cmd.Stderr = stderr
diff --git a/cmd/arvados-dev/docker-boot.sh b/cmd/arvados-dev/docker-boot.sh
new file mode 100755
index 000000000..e8703e41f
--- /dev/null
+++ b/cmd/arvados-dev/docker-boot.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Bring up a docker container with some locally-built commands (e.g.,
+# cmd/arvados-server) replacing the ones that came with
+# arvados-server-easy when the arvados-installpackage-* image was
+# built.
+#
+# Assumes docker-build-install.sh has already succeeded.
+#
+# Example:
+#
+# docker-boot.sh cmd/arvados-server services/keep-balance
+
+set -e -o pipefail
+
+cleanup() {
+ if [[ -n "${tmpdir}" ]]; then
+ rm -rf "${tmpdir}"
+ fi
+}
+trap cleanup ERR EXIT
+
+tmpdir=$(mktemp -d)
+version=$(git describe --tag --dirty)
+
+declare -a volargs=()
+for srcdir in "$@"; do
+ echo >&2 "building $srcdir..."
+ (cd $srcdir && GOBIN=$tmpdir go install -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}")
+ cmd="$(basename "$srcdir")"
+ volargs+=(-v "$tmpdir/$cmd:/var/lib/arvados/bin/$cmd:ro")
+done
+
+osbase=debian:10
+installimage=arvados-installpackage-${osbase}
+docker run -it --rm \
+ "${volargs[@]}" \
+ "${installimage}" \
+ bash -c '/etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/docker-build-install.sh b/cmd/arvados-dev/docker-build-install.sh
new file mode 100755
index 000000000..3c6e177cf
--- /dev/null
+++ b/cmd/arvados-dev/docker-build-install.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Build an arvados-server-easy package, then install and run it on a
+# base OS image.
+#
+# Examples:
+#
+# docker-build-install.sh --force-buildimage --force-installimage # always build fresh docker images
+#
+# docker-build-install.sh # reuse cached docker images if possible
+
+set -e -o pipefail
+
+declare -A opts=()
+while [[ $# -gt 0 ]]; do
+ arg="$1"
+ shift
+ case "$arg" in
+ --force-buildimage)
+ opts[force-buildimage]=1
+ ;;
+ --force-installimage)
+ opts[force-installimage]=1
+ ;;
+ *)
+ echo >&2 "invalid argument '$arg'"
+ exit 1
+ esac
+done
+
+cleanup() {
+ if [[ -n "${buildctr}" ]]; then
+ docker rm "${buildctr}" || true
+ fi
+ if [[ -n "${installctr}" ]]; then
+ docker rm "${installctr}" || true
+ fi
+}
+trap cleanup ERR EXIT
+
+version=$(git describe --tag --dirty)
+osbase=debian:10
+
+mkdir -p /tmp/pkg
+
+buildimage=arvados-buildpackage-${osbase}
+if [[ "${opts[force-buildimage]}" || -z "$(docker images --format {{.Repository}} "${buildimage}")" ]]; then
+ (
+ echo >&2 building arvados-server...
+ cd cmd/arvados-server
+ go install
+ )
+ echo >&2 building ${buildimage}...
+ buildctr=${buildimage/:/-}
+ docker rm "${buildctr}" || true
+ docker run \
+ --name "${buildctr}" \
+ -v /tmp/pkg:/pkg \
+ -v "${GOPATH:-${HOME}/go}"/bin/arvados-server:/arvados-server:ro \
+ -v "$(pwd)":/arvados:ro \
+ "${osbase}" \
+ /arvados-server install \
+ -type package \
+ -source /arvados \
+ -package-version "${version}"
+ docker commit "${buildctr}" "${buildimage}"
+ docker rm "${buildctr}"
+ buildctr=
+fi
+
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+rm -v -f "${pkgfile}"
+
+(
+ echo >&2 building arvados-dev...
+ cd cmd/arvados-dev
+ go install
+)
+echo >&2 building ${pkgfile}...
+docker run --rm \
+ -v /tmp/pkg:/pkg \
+ -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+ -v "$(pwd)":/arvados:ro \
+ "${buildimage}" \
+ /arvados-dev buildpackage \
+ -source /arvados \
+ -package-version "${version}" \
+ -output-directory /pkg
+
+ls -l ${pkgfile}
+(
+ echo >&2 dpkg-scanpackages...
+ cd /tmp/pkg
+ dpkg-scanpackages . | gzip > Packages.gz
+)
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+installimage="arvados-installpackage-${osbase}"
+if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+ echo >&2 building ${installimage}...
+ installctr=${installimage/:/-}
+ docker rm "${installctr}" || true
+ docker run -it \
+ --name "${installctr}" \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ "${osbase}" \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy postgresql'
+ docker commit "${installctr}" "${installimage}"
+ docker rm "${installctr}"
+ installctr=
+fi
+
+echo >&2 installing ${pkgfile} in ${installimage}, then starting arvados...
+docker run -it --rm \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ "${installimage}" \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install --reinstall -y arvados-server-easy postgresql && /etc/init.d/postgresql start && /var/lib/arvados/bin/arvados-server init -cluster-id x1234 && /var/lib/arvados/bin/arvados-server boot'
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
deleted file mode 100755
index 072dfcf9c..000000000
--- a/cmd/arvados-dev/example.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-set -e -o pipefail
-
-version="${PACKAGE_VERSION:-0.9.99}"
-
-# mkdir -p /tmp/pkg
-# (
-# cd cmd/arvados-dev
-# go install
-# )
-# docker run --rm \
-# -v /tmp/pkg:/pkg \
-# -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
-# -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
-# /arvados-dev buildpackage \
-# -source /arvados \
-# -package-version "${version}" \
-# -output-directory /pkg
-pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
-# ls -l ${pkgfile}
-# (
-# cd /tmp/pkg
-# dpkg-scanpackages . | gzip > Packages.gz
-# )
-sourcesfile=/tmp/sources.conf.d-arvados
-echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
-docker run -it --rm \
- -v /tmp/pkg:/pkg:ro \
- -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
- ${INSTALLIMAGE:-debian:10} \
- bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index ff99de75c..d0aa9da94 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -34,6 +34,7 @@ var (
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "init": install.InitCommand,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
})
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 0f105d6b6..c1da7d18d 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -53,7 +53,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
} {
port, err := internalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s internal port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s internal port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
@@ -62,7 +62,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
port, err = externalPort(cmpt.svc)
if err != nil {
- return fmt.Errorf("%s external port: %s (%v)", cmpt.varname, err, cmpt.svc)
+ return fmt.Errorf("%s external port: %w (%v)", cmpt.varname, err, cmpt.svc)
}
if ok, err := addrIsLocal(net.JoinHostPort(super.ListenHost, port)); !ok || err != nil {
return fmt.Errorf("urlIsLocal() failed for host %q port %q: %v", super.ListenHost, port, err)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index f18300c4c..481300b45 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -37,6 +37,10 @@ func (runner installPassenger) String() string {
}
func (runner installPassenger) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ if super.ClusterType == "production" {
+ // passenger has already been installed via package
+ return nil
+ }
err := super.wait(ctx, runner.depends...)
if err != nil {
return err
@@ -52,7 +56,7 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "--conservative", "--no-rdoc", "--no-ri", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
@@ -83,9 +87,10 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
}
type runPassenger struct {
- src string
- svc arvados.Service
- depends []supervisedTask
+ src string // path to app in source tree
+ varlibdir string // path to app (relative to /var/lib/arvados) in OS package
+ svc arvados.Service
+ depends []supervisedTask
}
func (runner runPassenger) String() string {
@@ -101,6 +106,12 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
if err != nil {
return fmt.Errorf("bug: no internalPort for %q: %v (%#v)", runner, err, runner.svc)
}
+ var appdir string
+ if super.ClusterType == "production" {
+ appdir = "/var/lib/arvados/" + runner.varlibdir
+ } else {
+ appdir = runner.src
+ }
loglevel := "4"
if lvl, ok := map[string]string{
"debug": "5",
@@ -116,13 +127,30 @@ func (runner runPassenger) Run(ctx context.Context, fail func(error), super *Sup
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- err = super.RunProgram(ctx, runner.src, nil, railsEnv, "bundle", "exec",
+ cmdline := []string{
+ "bundle", "exec",
"passenger", "start",
"-p", port,
- "--log-file", "/dev/stderr",
"--log-level", loglevel,
"--no-friendly-error-pages",
- "--pid-file", filepath.Join(super.tempdir, "passenger."+strings.Replace(runner.src, "/", "_", -1)+".pid"))
+ "--disable-anonymous-telemetry",
+ "--disable-security-update-check",
+ "--no-compile-runtime",
+ "--no-install-runtime",
+ "--pid-file", filepath.Join(super.wwwtempdir, "passenger."+strings.Replace(appdir, "/", "_", -1)+".pid"),
+ }
+ if super.ClusterType == "production" {
+ cmdline = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle"}, cmdline[1:]...)
+ } else {
+ // This would be desirable in the production
+ // case too, but it fails with sudo because
+ // /dev/stderr is a symlink to a pty owned by
+ // root: "nginx: [emerg] open() "/dev/stderr"
+ // failed (13: Permission denied)"
+ cmdline = append(cmdline, "--log-file", "/dev/stderr")
+ }
+ env := append([]string{"TMPDIR=" + super.wwwtempdir}, railsEnv...)
+ err = super.RunProgram(ctx, appdir, nil, env, cmdline[0], cmdline[1:]...)
fail(err)
}()
return nil
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 34ccf04a8..199a93a9d 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -36,6 +36,10 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
+
iamroot := false
if u, err := user.Current(); err != nil {
return fmt.Errorf("user.Current(): %s", err)
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
index d1cf2a870..1f07601a0 100644
--- a/lib/boot/seed.go
+++ b/lib/boot/seed.go
@@ -20,6 +20,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor
if err != nil {
return err
}
+ if super.ClusterType == "production" {
+ return nil
+ }
err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup")
if err != nil {
return err
diff --git a/lib/boot/service.go b/lib/boot/service.go
index 5afacfe71..77fdc98be 100644
--- a/lib/boot/service.go
+++ b/lib/boot/service.go
@@ -30,8 +30,8 @@ func (runner runServiceCommand) String() string {
}
func (runner runServiceCommand) Run(ctx context.Context, fail func(error), super *Supervisor) error {
- binfile := filepath.Join(super.tempdir, "bin", "arvados-server")
- err := super.RunProgram(ctx, super.tempdir, nil, nil, binfile, "-version")
+ binfile := filepath.Join(super.bindir, "arvados-server")
+ err := super.RunProgram(ctx, super.bindir, nil, nil, binfile, "-version")
if err != nil {
return err
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index e38a4775e..2d7697233 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -14,6 +14,7 @@ import (
"io"
"io/ioutil"
"net"
+ "net/url"
"os"
"os/exec"
"os/signal"
@@ -54,7 +55,9 @@ type Supervisor struct {
tasksReady map[string]chan bool
waitShutdown sync.WaitGroup
+ bindir string
tempdir string
+ wwwtempdir string
configfile string
environ []string // for child processes
}
@@ -131,13 +134,26 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
return err
}
- super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
- if err != nil {
- return err
- }
- defer os.RemoveAll(super.tempdir)
- if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil {
- return err
+ // Choose bin and temp dirs: /var/lib/arvados/... in
+ // production, transient tempdir otherwise.
+ if super.ClusterType == "production" {
+ // These dirs have already been created by
+ // "arvados-server install" (or by extracting a
+ // package).
+ super.tempdir = "/var/lib/arvados/tmp"
+ super.wwwtempdir = "/var/lib/arvados/wwwtmp"
+ super.bindir = "/var/lib/arvados/bin"
+ } else {
+ super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(super.tempdir)
+ super.wwwtempdir = super.tempdir
+ super.bindir = filepath.Join(super.tempdir, "bin")
+ if err := os.Mkdir(super.bindir, 0755); err != nil {
+ return err
+ }
}
// Fill in any missing config keys, and write the resulting
@@ -166,7 +182,10 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
+ super.prependEnv("PATH", "/var/lib/arvados/bin:")
+ if super.ClusterType != "production" {
+ super.prependEnv("PATH", super.tempdir+"/bin:")
+ }
super.cluster, err = cfg.GetCluster("")
if err != nil {
@@ -182,7 +201,9 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
"PID": os.Getpid(),
})
- if super.SourceVersion == "" {
+ if super.SourceVersion == "" && super.ClusterType == "production" {
+ // don't need SourceVersion
+ } else if super.SourceVersion == "" {
// Find current source tree version.
var buf bytes.Buffer
err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat")
@@ -224,15 +245,15 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}},
installPassenger{src: "services/api"},
- runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}},
installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup
- runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
+ runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}},
seedDatabase{},
}
if super.ClusterType != "test" {
tasks = append(tasks,
runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller},
- runGoProgram{src: "services/keep-balance"},
+ runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance},
)
}
super.tasksReady = map[string]chan bool{}
@@ -382,9 +403,11 @@ func dedupEnv(in []string) []string {
func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) {
_, basename := filepath.Split(srcpath)
- bindir := filepath.Join(super.tempdir, "bin")
- binfile := filepath.Join(bindir, basename)
- err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
+ binfile := filepath.Join(super.bindir, basename)
+ if super.ClusterType == "production" {
+ return binfile, nil
+ }
+ err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion)
return binfile, err
}
@@ -401,10 +424,19 @@ func (super *Supervisor) setupRubyEnv() error {
"GEM_PATH=",
})
gem := "gem"
- if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" {
gem = "/var/lib/arvados/bin/gem"
}
cmd := exec.Command(gem, "env", "gempath")
+ if super.ClusterType == "production" {
+ cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...)
+ path, err := exec.LookPath("sudo")
+ if err != nil {
+ return fmt.Errorf("LookPath(\"sudo\"): %w", err)
+ }
+ cmd.Path = path
+ }
+ cmd.Stderr = super.Stderr
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
@@ -694,11 +726,10 @@ func internalPort(svc arvados.Service) (string, error) {
return "", errors.New("internalPort() doesn't work with multiple InternalURLs")
}
for u := range svc.InternalURLs {
- if _, p, err := net.SplitHostPort(u.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(u)
+ if p := u.Port(); p != "" {
return p, nil
- } else if u.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "ws" {
return "443", nil
} else {
return "80", nil
@@ -708,11 +739,10 @@ func internalPort(svc arvados.Service) (string, error) {
}
func externalPort(svc arvados.Service) (string, error) {
- if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil {
- return "", err
- } else if p != "" {
+ u := url.URL(svc.ExternalURL)
+ if p := u.Port(); p != "" {
return p, nil
- } else if svc.ExternalURL.Scheme == "https" {
+ } else if u.Scheme == "https" || u.Scheme == "wss" {
return "443", nil
} else {
return "80", nil
diff --git a/lib/install/deps.go b/lib/install/deps.go
index f9b962fdd..3f19aa1a8 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
"io"
"os"
"os/exec"
+ "os/user"
"path/filepath"
"strconv"
"strings"
@@ -176,12 +177,26 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
}
os.Mkdir("/var/lib/arvados", 0755)
+ os.Mkdir("/var/lib/arvados/tmp", 0700)
+ if prod {
+ os.Mkdir("/var/lib/arvados/wwwtmp", 0700)
+ u, er := user.Lookup("www-data")
+ if er != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er)
+ return 1
+ }
+ uid, _ := strconv.Atoi(u.Uid)
+ gid, _ := strconv.Atoi(u.Gid)
+ err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid)
+ if err != nil {
+ return 1
+ }
+ }
rubyversion := "2.5.7"
if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
logger.Print("ruby " + rubyversion + " already installed")
} else {
err = runBash(`
-mkdir -p /var/lib/arvados/tmp
tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
@@ -189,7 +204,9 @@ cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
make -j8
make install
-/var/lib/arvados/bin/gem install bundler
+/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc
+# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc
rm -r ${tmp}
`, stdout, stderr)
if err != nil {
@@ -262,7 +279,6 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
} else {
err = runBash(`
G=`+gradleversion+`
-mkdir -p /var/lib/arvados/tmp
zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
trap "rm ${zip}" ERR
wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
@@ -414,8 +430,7 @@ rm ${zip}
for _, cmdline := range [][]string{
{"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
{"touch", "log/production.log"},
- // {"chown", "-R", "root:root", "."},
- {"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
@@ -426,6 +441,7 @@ rm ${zip}
cmd.Dir = "/var/lib/arvados/" + dstdir
cmd.Stdout = stdout
cmd.Stderr = stderr
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
err = cmd.Run()
if err != nil {
return 1
@@ -569,6 +585,7 @@ func prodpkgs(osv osversion) []string {
"make",
"nginx",
"python",
+ "sudo",
}
if osv.Debian || osv.Ubuntu {
if osv.Debian && osv.Major == 8 {
diff --git a/lib/install/init.go b/lib/install/init.go
new file mode 100644
index 000000000..6d4f19757
--- /dev/null
+++ b/lib/install/init.go
@@ -0,0 +1,265 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/user"
+ "regexp"
+ "strconv"
+ "text/template"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
+)
+
+var InitCommand cmd.Handler = &initCommand{}
+
+type initCommand struct {
+ ClusterID string
+ Domain string
+ PostgreSQLPassword string
+}
+
+func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ err = fmt.Errorf("Hostname(): %w", err)
+ return 1
+ }
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ flags.StringVar(&initcmd.ClusterID, "cluster-id", "", "cluster `id`, like x1234 for a dev cluster")
+ flags.StringVar(&initcmd.Domain, "domain", hostname, "cluster public DNS `name`, like x1234.arvadosapi.com")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
+ } else if !regexp.MustCompile(`^[a-z][a-z0-9]{4}`).MatchString(initcmd.ClusterID) {
+ err = fmt.Errorf("cluster ID %q is invalid; must be an ASCII letter followed by 4 alphanumerics (try -help)", initcmd.ClusterID)
+ return 1
+ }
+
+ wwwuser, err := user.Lookup("www-data")
+ if err != nil {
+ err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
+ return 1
+ }
+ wwwgid, err := strconv.Atoi(wwwuser.Gid)
+ if err != nil {
+ return 1
+ }
+ initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+
+ err = os.Mkdir("/var/lib/arvados/keep", 0600)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+
+ err = os.Mkdir("/etc/arvados", 0750)
+ if err != nil && !os.IsExist(err) {
+ err = fmt.Errorf("mkdir /etc/arvados: %w", err)
+ return 1
+ }
+ err = os.Chown("/etc/arvados", 0, wwwgid)
+ f, err := os.OpenFile("/etc/arvados/config.yml", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ if err != nil {
+ err = fmt.Errorf("open /etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ tmpl, err := template.New("config").Parse(`Clusters:
+ {{.ClusterID}}:
+ Services:
+ Controller:
+ InternalURLs:
+ "http://0.0.0.0:8003/": {}
+ ExternalURL: {{printf "%q" ( print "https://" .Domain "/" ) }}
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ Websocket:
+ InternalURLs:
+ "http://0.0.0.0:8005/": {}
+ ExternalURL: {{printf "%q" ( print "wss://ws." .Domain "/" ) }}
+ Keepbalance:
+ InternalURLs:
+ "http://0.0.0.0:9005/": {}
+ GitHTTP:
+ InternalURLs:
+ "http://0.0.0.0:9001/": {}
+ ExternalURL: {{printf "%q" ( print "https://git." .Domain "/" ) }}
+ DispatchCloud:
+ InternalURLs:
+ "http://0.0.0.0:9006/": {}
+ Keepproxy:
+ InternalURLs:
+ "http://0.0.0.0:25108/": {}
+ ExternalURL: {{printf "%q" ( print "https://keep." .Domain "/" ) }}
+ WebDAV:
+ InternalURLs:
+ "http://0.0.0.0:9002/": {}
+ ExternalURL: {{printf "%q" ( print "https://*.collections." .Domain "/" ) }}
+ WebDAVDownload:
+ InternalURLs:
+ "http://0.0.0.0:8004/": {}
+ ExternalURL: {{printf "%q" ( print "https://download." .Domain "/" ) }}
+ Keepstore:
+ InternalURLs:
+ "http://0.0.0.0:25107/": {}
+ Composer:
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/composer" ) }}
+ Workbench1:
+ InternalURLs:
+ "http://0.0.0.0:8001/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench." .Domain "/" ) }}
+ Workbench2:
+ InternalURLs:
+ "http://0.0.0.0:8002/": {}
+ ExternalURL: {{printf "%q" ( print "https://workbench2." .Domain "/" ) }}
+ Health:
+ InternalURLs:
+ "http://0.0.0.0:9007/": {}
+ API:
+ RailsSessionSecretToken: {{printf "%q" ( .RandomHex 50 )}}
+ Collections:
+ BlobSigningKey: {{printf "%q" ( .RandomHex 50 )}}
+ Containers:
+ DispatchPrivateKey: {{printf "%q" .GenerateSSHPrivateKey}}
+ ManagementToken: {{printf "%q" ( .RandomHex 50 )}}
+ PostgreSQL:
+ Connection:
+ dbname: arvados_production
+ host: localhost
+ user: arvados
+ password: {{printf "%q" .PostgreSQLPassword}}
+ SystemRootToken: {{printf "%q" ( .RandomHex 50 )}}
+ Volumes:
+ {{.ClusterID}}-nyw5e-000000000000000:
+ Driver: Directory
+ DriverParameters:
+ Root: /var/lib/arvados/keep
+ Replication: 2
+`)
+ if err != nil {
+ return 1
+ }
+ err = tmpl.Execute(f, initcmd)
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: tmpl.Execute: %w", err)
+ return 1
+ }
+ err = f.Close()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: close: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "created /etc/arvados/config.yml")
+
+ ldr := config.NewLoader(nil, logger)
+ ldr.SkipLegacy = true
+ cfg, err := ldr.Load()
+ if err != nil {
+ err = fmt.Errorf("/etc/arvados/config.yml: %w", err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+
+ err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
+ if err != nil {
+ return 1
+ }
+
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ cmd.Dir = "/var/lib/arvados/railsapi"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("rake db:setup: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "initialized database")
+
+ return 0
+}
+
+func (initcmd *initCommand) GenerateSSHPrivateKey() (string, error) {
+ privkey, err := rsa.GenerateKey(rand.Reader, 4096)
+ if err != nil {
+ return "", err
+ }
+ err = privkey.Validate()
+ if err != nil {
+ return "", err
+ }
+ return string(pem.EncodeToMemory(&pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: x509.MarshalPKCS1PrivateKey(privkey),
+ })), nil
+}
+
+func (initcmd *initCommand) RandomHex(chars int) string {
+ b := make([]byte, chars/2)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", b)
+}
+
+func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
+ for _, sql := range []string{
+ `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
+ `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+ } {
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
+ }
+ }
+ return nil
+}
commit ee7d60f02525760a2480157011659e7b2210f6b8
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Jul 24 10:43:29 2020 -0400
16306: Sync bundler version to services/api/Gemfile.lock.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index ff6ead0fa..b3eacfe31 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -555,7 +555,7 @@ setup_ruby_environment() {
(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- versions=(1.11.0 1.17.3 2.0.2)
+ versions=(1.16.6 1.17.3 2.0.2)
for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
diff --git a/lib/boot/passenger.go b/lib/boot/passenger.go
index 6a2c4b61f..f18300c4c 100644
--- a/lib/boot/passenger.go
+++ b/lib/boot/passenger.go
@@ -50,9 +50,9 @@ func (runner installPassenger) Run(ctx context.Context, fail func(error), super
if err != nil {
return err
}
- for _, version := range []string{"1.11.0", "1.17.3", "2.0.2"} {
+ for _, version := range []string{"1.16.6", "1.17.3", "2.0.2"} {
if !strings.Contains(buf.String(), "("+version+")") {
- err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2")
+ err = super.RunProgram(ctx, runner.src, nil, nil, "gem", "install", "--user", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2")
if err != nil {
return err
}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index d28823f4a..f9b962fdd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -416,7 +416,7 @@ rm ${zip}
{"touch", "log/production.log"},
// {"chown", "-R", "root:root", "."},
{"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
commit 12441486d7fb3b51d2fea9d9b1da0d8e4f3ef3e0
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Jul 1 09:42:48 2020 -0400
16306: Add "install -type package" and "buildpackage".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-dev/buildpackage.go b/cmd/arvados-dev/buildpackage.go
new file mode 100644
index 000000000..4b395c4ba
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage.go
@@ -0,0 +1,98 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os/exec"
+
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+type buildPackage struct{}
+
+func (bld buildPackage) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ err := (&builder{
+ PackageVersion: "0.0.0",
+ logger: logger,
+ }).run(context.Background(), prog, args, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type builder struct {
+ PackageVersion string
+ SourcePath string
+ OutputDir string
+ SkipInstall bool
+ logger logrus.FieldLogger
+}
+
+func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
+ flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
+ flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
+ flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
+ err := flags.Parse(args)
+ if err != nil {
+ return err
+ }
+ if len(flags.Args()) > 0 {
+ return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if !bldr.SkipInstall {
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", bldr.PackageVersion,
+ "-source", bldr.SourcePath,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ }
+ }
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
+ format := "deb" // TODO: rpm
+
+ cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+ "--name", "arvados-server-easy",
+ "--version", bldr.PackageVersion,
+ "--input-type", "dir",
+ "--output-type", format)
+ deps, err := install.ProductionDependencies()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range deps {
+ cmd.Args = append(cmd.Args, "--depends", pkg)
+ }
+ cmd.Args = append(cmd.Args,
+ "--exclude", "/var/lib/arvados/go",
+ "/var/lib/arvados",
+ "/var/www/.gem",
+ "/var/www/.passenger",
+ )
+ fmt.Fprintf(stderr, "%s...\n", cmd.Args)
+ cmd.Dir = bldr.OutputDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ return cmd.Run()
+}
diff --git a/cmd/arvados-dev/buildpackage_test.go b/cmd/arvados-dev/buildpackage_test.go
new file mode 100644
index 000000000..61bdeae82
--- /dev/null
+++ b/cmd/arvados-dev/buildpackage_test.go
@@ -0,0 +1,77 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "flag"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+var buildimage string
+
+func init() {
+ os.Args = append(os.Args, "-test.timeout=30m") // kludge
+
+ // This enables a hack to speed up repeated tests: hit "docker
+ // commit --pause {containername} checkpointtag" after the
+ // test container has downloaded/compiled some stuff, then run
+ // tests with "-test.buildimage=checkpointtag" next time to
+ // retry/resume/update from that point.
+ flag.StringVar(&buildimage, "test.buildimage", "debian:10", "docker image to use when running buildpackage")
+}
+
+type BuildpackageSuite struct{}
+
+var _ = check.Suite(&BuildpackageSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *BuildpackageSuite) TestBuildAndInstall(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+ tmpdir := c.MkDir()
+ defer os.RemoveAll(tmpdir)
+
+ err := os.Mkdir(tmpdir+"/pkg", 0755)
+ c.Assert(err, check.IsNil)
+ err = os.Mkdir(tmpdir+"/bin", 0755)
+ c.Assert(err, check.IsNil)
+
+ cmd := exec.Command("go", "install")
+ cmd.Env = append(append([]string(nil), os.Environ()...), "GOPATH="+tmpdir)
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ srctree, err := filepath.Abs("../..")
+ c.Assert(err, check.IsNil)
+
+ cmd = exec.Command("docker", "run", "--rm",
+ "-v", tmpdir+"/pkg:/pkg",
+ "-v", tmpdir+"/bin/arvados-dev:/arvados-dev:ro",
+ "-v", srctree+":/usr/local/src/arvados:ro",
+ buildimage,
+ "/arvados-dev", "buildpackage",
+ "-package-version", "0.9.99",
+ "-source", "/usr/local/src/arvados",
+ "-output-directory", "/pkg")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ fi, err := os.Stat(tmpdir + "/pkg/arvados-server_0.9.99_amd64.deb")
+ c.Assert(err, check.IsNil)
+ c.Logf("%#v", fi)
+}
diff --git a/cmd/arvados-dev/cmd.go b/cmd/arvados-dev/cmd.go
new file mode 100644
index 000000000..e0d5144e5
--- /dev/null
+++ b/cmd/arvados-dev/cmd.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "os"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+)
+
+var (
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "version": cmd.Version,
+ "-version": cmd.Version,
+ "--version": cmd.Version,
+
+ "buildpackage": buildPackage{},
+ })
+)
+
+func main() {
+ os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
+}
diff --git a/cmd/arvados-dev/example.sh b/cmd/arvados-dev/example.sh
new file mode 100755
index 000000000..072dfcf9c
--- /dev/null
+++ b/cmd/arvados-dev/example.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e -o pipefail
+
+version="${PACKAGE_VERSION:-0.9.99}"
+
+# mkdir -p /tmp/pkg
+# (
+# cd cmd/arvados-dev
+# go install
+# )
+# docker run --rm \
+# -v /tmp/pkg:/pkg \
+# -v "${GOPATH:-${HOME}/go}"/bin/arvados-dev:/arvados-dev:ro \
+# -v "$(pwd)":/arvados:ro "${BUILDIMAGE:-debian:10}" \
+# /arvados-dev buildpackage \
+# -source /arvados \
+# -package-version "${version}" \
+# -output-directory /pkg
+pkgfile=/tmp/pkg/arvados-server-easy_${version}_amd64.deb
+# ls -l ${pkgfile}
+# (
+# cd /tmp/pkg
+# dpkg-scanpackages . | gzip > Packages.gz
+# )
+sourcesfile=/tmp/sources.conf.d-arvados
+echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+docker run -it --rm \
+ -v /tmp/pkg:/pkg:ro \
+ -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+ ${INSTALLIMAGE:-debian:10} \
+ bash -c 'apt update && DEBIAN_FRONTEND=noninteractive apt install -y arvados-server-easy && bash -login'
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ba57c20c3..d28823f4a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -14,6 +14,7 @@ import (
"io"
"os"
"os/exec"
+ "path/filepath"
"strconv"
"strings"
"syscall"
@@ -24,13 +25,17 @@ import (
"github.com/lib/pq"
)
-var Command cmd.Handler = installCommand{}
+var Command cmd.Handler = &installCommand{}
const devtestDatabasePassword = "insecure_arvados_test"
-type installCommand struct{}
+type installCommand struct {
+ ClusterType string
+ SourcePath string
+ PackageVersion string
+}
-func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := ctxlog.New(stderr, "text", "info")
ctx := ctxlog.Context(context.Background(), logger)
ctx, cancel := context.WithCancel(ctx)
@@ -46,7 +51,9 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.SetOutput(stderr)
versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
- clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package")
+ flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)")
+ flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -55,18 +62,23 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
return 2
} else if *versionFlag {
return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ } else if len(flags.Args()) > 0 {
+ err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ return 2
}
- var dev, test, prod bool
- switch *clusterType {
+ var dev, test, prod, pkg bool
+ switch inst.ClusterType {
case "development":
dev = true
case "test":
test = true
case "production":
prod = true
+ case "package":
+ pkg = true
default:
- err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType)
+ err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType)
return 2
}
@@ -96,53 +108,47 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
}
}
- if dev || test {
- debs := []string{
- "bison",
+ pkgs := prodpkgs(osv)
+
+ if pkg {
+ pkgs = append(pkgs,
+ "dpkg-dev",
+ "rsync",
+ )
+ }
+
+ if dev || test || pkg {
+ pkgs = append(pkgs,
"bsdmainutils",
"build-essential",
- "ca-certificates",
"cadaver",
- "curl",
"cython",
"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"default-jdk-headless",
"default-jre-headless",
- "fuse",
"gettext",
- "git",
- "gitolite3",
- "graphviz",
- "haveged",
"iceweasel",
"libattr1-dev",
"libcrypt-ssleay-perl",
- "libcrypt-ssleay-perl",
- "libcurl3-gnutls",
- "libcurl4-openssl-dev",
"libfuse-dev",
"libgnutls28-dev",
"libjson-perl",
- "libjson-perl",
"libpam-dev",
"libpcre3-dev",
- "libpq-dev",
"libpython2.7-dev",
"libreadline-dev",
"libssl-dev",
"libwww-perl",
"libxml2-dev",
- "libxslt1.1",
+ "libxslt1-dev",
"linkchecker",
"lsof",
"net-tools",
- "nginx",
"pandoc",
"perl-modules",
"pkg-config",
"postgresql",
"postgresql-contrib",
- "python",
"python3-dev",
"python-epydoc",
"r-base",
@@ -151,16 +157,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"virtualenv",
"wget",
"xvfb",
- "zlib1g-dev",
- }
+ )
switch {
case osv.Debian && osv.Major >= 10:
- debs = append(debs, "libcurl4")
+ pkgs = append(pkgs, "libcurl4")
default:
- debs = append(debs, "libcurl3")
+ pkgs = append(pkgs, "libcurl3")
}
cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
- cmd.Args = append(cmd.Args, debs...)
+ cmd.Args = append(cmd.Args, pkgs...)
cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
cmd.Stdout = stdout
cmd.Stderr = stderr
@@ -182,7 +187,7 @@ trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
-make -j4
+make -j8
make install
/var/lib/arvados/bin/gem install bundler
rm -r ${tmp}
@@ -206,7 +211,9 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
return 1
}
}
+ }
+ if !prod && !pkg {
pjsversion := "1.9.8"
if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
logger.Print("phantomjs " + pjsversion + " already installed")
@@ -389,12 +396,89 @@ rm ${zip}
}
}
+ if pkg {
+ // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
+ for dstdir, srcdir := range map[string]string{
+ "railsapi": "services/api",
+ "workbench1": "apps/workbench",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("rsync", "-a", "--no-owner", "--delete-after", "--exclude", "/tmp", "--exclude", "/log", "--exclude", "/vendor", "./", "/var/lib/arvados/"+dstdir+"/")
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ for _, cmdline := range [][]string{
+ {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.passenger"},
+ {"touch", "log/production.log"},
+ // {"chown", "-R", "root:root", "."},
+ {"chown", "-R", "www-data:www-data", "/var/www/.gem", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--no-rdoc", "--no-ri", "--conservative", "bundler:1.11", "bundler:1.17.3", "bundler:2.0.2"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"},
+ } {
+ cmd = exec.Command(cmdline[0], cmdline[1:]...)
+ cmd.Env = append([]string{}, os.Environ()...)
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install")
+ cmd.Dir = "/var/lib/arvados/" + dstdir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil && !strings.Contains(err.Error(), "exit status 2") {
+ // Exit code 2 indicates there were warnings (like
+ // "other passenger installations have been detected",
+ // which we can't expect to avoid) but no errors.
+ // Other non-zero exit codes (1, 9) indicate errors.
+ return 1
+ }
+ }
+
+ // Install Go programs to /var/lib/arvados/bin/
+ for _, srcdir := range []string{
+ "cmd/arvados-client",
+ "cmd/arvados-server",
+ "services/arv-git-httpd",
+ "services/crunch-dispatch-local",
+ "services/crunch-dispatch-slurm",
+ "services/health",
+ "services/keep-balance",
+ "services/keep-web",
+ "services/keepproxy",
+ "services/keepstore",
+ "services/ws",
+ } {
+ fmt.Fprintf(stderr, "building %s...\n", srcdir)
+ cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion)
+ cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...)
+ cmd.Dir = filepath.Join(inst.SourcePath, srcdir)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+ }
+
return 0
}
type osversion struct {
Debian bool
Ubuntu bool
+ Centos bool
Major int
}
@@ -432,6 +516,8 @@ func identifyOS() (osversion, error) {
osv.Ubuntu = true
case "debian":
osv.Debian = true
+ case "centos":
+ osv.Centos = true
default:
return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
}
@@ -466,3 +552,56 @@ func runBash(script string, stdout, stderr io.Writer) error {
cmd.Stderr = stderr
return cmd.Run()
}
+
+func prodpkgs(osv osversion) []string {
+ pkgs := []string{
+ "automake",
+ "bison",
+ "ca-certificates",
+ "curl",
+ "fuse",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "libcurl3-gnutls",
+ "libxslt1.1",
+ "make",
+ "nginx",
+ "python",
+ }
+ if osv.Debian || osv.Ubuntu {
+ if osv.Debian && osv.Major == 8 {
+ pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl
+ } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 {
+ pkgs = append(pkgs, "python3-distutils") // sdk/cwl
+ }
+ return append(pkgs,
+ "g++",
+ "libcurl4-openssl-dev", // services/api
+ "libpq-dev",
+ "libpython2.7", // services/fuse
+ "mime-support", // keep-web
+ "zlib1g-dev", // services/api
+ )
+ } else if osv.Centos {
+ return append(pkgs,
+ "fuse-libs", // services/fuse
+ "gcc",
+ "gcc-c++",
+ "libcurl-devel", // services/api
+ "mailcap", // keep-web
+ "postgresql-devel", // services/api
+ )
+ } else {
+ panic("os version not supported")
+ }
+}
+
+func ProductionDependencies() ([]string, error) {
+ osv, err := identifyOS()
+ if err != nil {
+ return nil, err
+ }
+ return prodpkgs(osv), nil
+}
commit b4d6c92791088a0b6a39a71b2b8b6ddc4b13af41
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Jun 2 11:14:10 2020 -0400
16306: Add automatic install page.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/_config.yml b/doc/_config.yml
index 7d7101f41..e38f0fa8f 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -190,6 +190,8 @@ navbar:
- install/arvados-on-kubernetes.html.textile.liquid
- install/arvados-on-kubernetes-minikube.html.textile.liquid
- install/arvados-on-kubernetes-GKE.html.textile.liquid
+ - Automatic installation:
+ - install/automatic.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- install/packages.html.textile.liquid
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
new file mode 100644
index 000000000..049bf6891
--- /dev/null
+++ b/doc/install/automatic.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: installguide
+title: Automatic single-node install
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This method sets up a new Arvados cluster using a single host/VM. It is the easiest way to get a new production cluster up and running.
+
+A single-node installation supports all Arvados functionality at small scale. Substantial workloads will require additional nodes and configuration steps.
+
+h2. Prerequisites
+
+You will need:
+* a server host running Debian 10 (buster).
+* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a Google account (use it in place of <code>example at gmail.com.example</code> in the instructions below).
+
+h2. Initialize the cluster
+
+<pre>
+# apt-get install arvados-server
+# arvados-server init -type production -cluster-id x9999 -controller-address x9999.example.com -admin-email example at gmail.com.example
+</pre>
+
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+
+h2. Enable login
+
+Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+
+After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+
+<pre>
+# systemctl restart arvados-server
+</pre>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 1a41980e2..81d7b2159 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -23,6 +23,7 @@ table(table table-bordered table-condensed).
||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
+|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes|
|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
|"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list