[ARVADOS] updated: 1.3.0-2362-gf75883d78
Git user
git at public.arvados.org
Tue Mar 17 13:24:23 UTC 2020
Summary of changes:
build/run-tests.sh | 5 +++++
lib/boot/cmd.go | 19 ++++++++++++++++++-
lib/boot/postgresql.go | 13 +++++++++----
lib/boot/supervisor.go | 2 +-
lib/install/deps.go | 31 ++++++++++++++++++++-----------
lib/install/deps_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 97 insertions(+), 17 deletions(-)
create mode 100644 lib/install/deps_test.go
via f75883d7826d92894638e4eff6d397196dd1123f (commit)
via 5c4e2f9158f496f07c0b9838b2310da892730bce (commit)
via af3fe384bb15c4fc5b915cdee5cd1f2a87b6301f (commit)
via 812d1e0ef6023add65ab5e4229278e4c56c13fa4 (commit)
via e6e89288f8d6fe79d7e982ded9c1347221021e2e (commit)
via 4fd6857ff316d1e3638f63c90403168940adb32f (commit)
from 5e31613d2a0647647b710a0558e11408bc157406 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f75883d7826d92894638e4eff6d397196dd1123f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 17:00:21 2020 -0400
16053: Add debian:10 install-and-boot test.
Run with "go test -check.vv -tags docker".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps_test.go b/lib/install/deps_test.go
new file mode 100644
index 000000000..d3e63986f
--- /dev/null
+++ b/lib/install/deps_test.go
@@ -0,0 +1,44 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// Skip this slow test unless invoked as "go test -tags docker":
+// +build docker
+
+package install
+
+import (
+ "os"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (*Suite) TestInstallDeps(c *check.C) {
+ tmp := c.MkDir()
+ script := `
+set -x
+export GOPATH=${GOPATH:-${HOME}/go}
+tmp="` + tmp + `"
+sourcepath="$(realpath ../..)"
+(cd ${sourcepath} && go build -o ${tmp} ./cmd/arvados-server)
+docker run -i --rm --workdir /arvados \
+ -v ${tmp}/arvados-server:/arvados-server:ro \
+ -v ${sourcepath}:/arvados:ro \
+ -v /arvados/services/api/.bundle \
+ -v /arvados/apps/workbench/.bundle \
+ --env http_proxy \
+ --env https_proxy \
+ debian:10 \
+ bash -c "/arvados-server install -type test && /arvados-server boot -type test -config doc/examples/config/zzzzz.yml -own-temporary-database -shutdown -timeout 9m"
+`
+ c.Check(runBash(script, os.Stdout, os.Stderr), check.IsNil)
+}
commit 5c4e2f9158f496f07c0b9838b2310da892730bce
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:22:21 2020 -0400
16053: Install en_US.UTF-8 locale.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3b904c458..30110ddcd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -233,6 +233,16 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
return 1
}
}
+
+ wantlocale := "en_US.UTF-8"
+ if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
+ logger.Print("locale " + wantlocale + " already installed")
+ } else {
+ err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit af3fe384bb15c4fc5b915cdee5cd1f2a87b6301f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:22:09 2020 -0400
16053: Less verbose wget progress bars.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 6f2a2756a..3b904c458 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -162,17 +162,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
logger.Print("ruby " + rubyversion + " already installed")
} else {
err = runBash(`
-mkdir -p /var/lib/arvados/src
-cd /var/lib/arvados/src
-wget -c https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz
-tar xzf ruby-`+rubyversion+`.tar.gz
-cd ruby-`+rubyversion+`
+mkdir -p /var/lib/arvados/tmp
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
+cd /var/lib/arvados/tmp/ruby-`+rubyversion+`
./configure --disable-install-doc --prefix /var/lib/arvados
make -j4
make install
/var/lib/arvados/bin/gem install bundler
-cd ..
-rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
+cd /var/lib/arvados/tmp
+rm -r ruby-`+rubyversion+`
`, stdout, stderr)
if err != nil {
return 1
@@ -186,7 +184,7 @@ rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
} else {
err = runBash(`
cd /tmp
-wget -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
+wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -200,7 +198,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
} else {
err = runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
-wget -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
+wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -214,7 +212,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
} else {
err = runBash(`
GD=v`+geckoversion+`
-wget -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
+wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -228,7 +226,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
} else {
err = runBash(`
NJS=`+nodejsversion+`
-wget -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
`, stdout, stderr)
if err != nil {
commit 812d1e0ef6023add65ab5e4229278e4c56c13fa4
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:21:28 2020 -0400
16053: Prefer dependencies in /var/lib/arvados/bin if they exist.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7328fad45..f5c184e49 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -590,6 +590,11 @@ setup_virtualenv() {
}
initialize() {
+ # If dependencies like ruby, go, etc. are installed in
+ # /var/lib/arvados -- presumably by "arvados-server install" --
+ # then we want to use those versions, instead of whatever happens
+ # to be installed in /usr.
+ PATH="/var/lib/arvados/bin:${PATH}"
sanity_checks
echo "WORKSPACE=$WORKSPACE"
commit e6e89288f8d6fe79d7e982ded9c1347221021e2e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:20:51 2020 -0400
16053: Use setuidgid instead of sudo to drop privileges.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index a08180e0c..df90f36af 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -81,8 +81,13 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
- args = append([]string{"-u", "postgres", prog}, args...)
- prog = "sudo"
+ // We can't use "sudo -u" here because it creates an
+ // intermediate process that interferes with our
+ // ability to reliably kill postgres. The setuidgid
+ // program just calls exec without forking, so it
+ // doesn't have this problem.
+ args = append([]string{"postgres", prog}, args...)
+ prog = "setuidgid"
}
err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
if err != nil {
@@ -112,8 +117,8 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
"-p", super.cluster.PostgreSQL.Connection["port"],
}
if iamroot {
- args = append([]string{"-u", "postgres", prog}, args...)
- prog = "sudo"
+ args = append([]string{"postgres", prog}, args...)
+ prog = "setuidgid"
}
fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
}()
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index e75de3244..8ef7e6ac1 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -411,7 +411,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "sudo" && len(args) >= 3 && args[0] == "-u" {
+ if logprefix == "setuidgid" && len(args) >= 3 {
logprefix = args[2]
}
logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 50eab6aef..6f2a2756a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -93,6 +93,7 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"cadaver",
"curl",
"cython",
+ "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"fuse",
"gettext",
"git",
commit 4fd6857ff316d1e3638f63c90403168940adb32f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:04:31 2020 -0400
16053: Add boot flags: -timeout and -shutdown.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 1abc93722..5147e3ac3 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -6,9 +6,11 @@ package boot
import (
"context"
+ "errors"
"flag"
"fmt"
"io"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
@@ -56,6 +58,8 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
+ timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
+ shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -77,14 +81,27 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
super.Start(ctx, cfg)
defer super.Stop()
+
+ var timer *time.Timer
+ if *timeout > 0 {
+ timer = time.AfterFunc(*timeout, super.Stop)
+ }
+
url, ok := super.WaitReady()
- if !ok {
+ if timer != nil && !timer.Stop() {
+ err = errors.New("boot timed out")
+ return 1
+ } else if !ok {
+ err = errors.New("boot failed")
return 1
}
// Write controller URL to stdout. Nothing else goes to
// stdout, so this provides an easy way for a calling script
// to discover the controller URL when everything is ready.
fmt.Fprintln(stdout, url)
+ if *shutdown {
+ super.Stop()
+ }
// Wait for signal/crash + orderly shutdown
<-super.done
return 0
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list