[ARVADOS] created: 1.3.0-2380-g9e48b6795
Git user
git at public.arvados.org
Tue Mar 24 19:41:33 UTC 2020
at 9e48b679541d2651e2519c7566e975049c00660c (commit)
commit 9e48b679541d2651e2519c7566e975049c00660c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 24 15:40:59 2020 -0400
16053: Add ca-certificates to deps.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 167763e9f..5496be81a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -95,6 +95,7 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"bison",
"bsdmainutils",
"build-essential",
+ "ca-certificates",
"cadaver",
"curl",
"cython",
commit fd69d825f50dfa1a6618ba77512bc06cf53e8e00
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 24 12:12:30 2020 -0400
16053: Add example script for installing from scratch.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/example_from_scratch.sh b/lib/install/example_from_scratch.sh
new file mode 100644
index 000000000..03d9b7f63
--- /dev/null
+++ b/lib/install/example_from_scratch.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+# Starting with a base debian buster system, like "docker run -it
+# debian:10"...
+
+apt update
+apt upgrade
+apt install --no-install-recommends build-essential ca-certificates git golang
+git clone https://git.arvados.org/arvados.git
+cd arvados
+[[ -e lib/install ]] || git checkout origin/16053-install-deps
+cd cmd/arvados-server
+go run ./cmd/arvados-server install -type test
+pg_isready || pg_ctlcluster 11 main start # only needed if there's no init process (as in docker)
+build/run-tests.sh
commit 5b416cd1674bf397b28e10f6338cf648c704ece6
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 24 12:03:36 2020 -0400
16053: Add scripts for running tests in docker.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/arvadostest_docker_build.sh b/lib/install/arvadostest_docker_build.sh
new file mode 100755
index 000000000..e0defa888
--- /dev/null
+++ b/lib/install/arvadostest_docker_build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+SRC=$(realpath $(dirname ${BASH_SOURCE[0]})/../..)
+
+ctrname=arvadostest
+ctrbase=${ctrname}
+if [[ "${1}" != "--update" ]] || ! docker images --format={{.Repository}} | grep -x ${ctrbase}; then
+ ctrbase=debian:10
+fi
+
+if docker ps -a --format={{.Names}} | grep -x ${ctrname}; then
+ echo >&2 "container name already in use -- another builder running?"
+ exit 1
+fi
+
+(cd ${SRC}/cmd/arvados-server && go install)
+trap "docker rm --volumes ${ctrname}" ERR
+docker run -it --name ${ctrname} \
+ -v ${GOPATH:-${HOME}/go}/bin/arvados-server:/bin/arvados-server:ro \
+ -v ${SRC}:/src/arvados:ro \
+ -v /tmp \
+ --env http_proxy \
+ --env https_proxy \
+ ${ctrbase} \
+ bash -c "
+set -ex -o pipefail
+arvados-server install -type test
+pg_ctlcluster 11 main start
+cp -a /src/arvados /tmp/
+cd /tmp/arvados
+rm -rf tmp config.yml database.yml services/api/config/database.yml
+mkdir tmp
+build/run-tests.sh WORKSPACE=\$PWD --temp /tmp/arvados/tmp --only x"
+docker commit ${ctrname} ${ctrname}
+trap - ERR
+docker rm --volumes ${ctrname}
diff --git a/lib/install/arvadostest_docker_run.sh b/lib/install/arvadostest_docker_run.sh
new file mode 100755
index 000000000..ca5365592
--- /dev/null
+++ b/lib/install/arvadostest_docker_run.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Example:
+#
+# ./arvadostest_docker_build.sh # build the base image ("arvadostest")
+# ./arvadostest_docker_build.sh --update # update the base image with current version of `arvados-server install`
+# ./arvadostest_docker_run.sh --interactive # start a container using the previously built base image, copy this source tree into it, and invoke run-tests.sh with the given args
+
+set -ex -o pipefail
+
+declare -a qargs
+for arg in "$@"; do
+ qargs+=("${arg at Q}")
+done
+
+SRC=$(realpath $(dirname ${BASH_SOURCE[0]})/../..)
+
+docker run --rm -it \
+ --privileged \
+ -v /dev/fuse:/dev/fuse \
+ -v ${SRC}:/src/arvados:ro \
+ -v /tmp \
+ --env http_proxy \
+ --env https_proxy \
+ arvadostest \
+ bash -c "
+set -ex -o pipefail
+pg_ctlcluster 11 main start
+cp -a /src/arvados /tmp/
+cd /tmp/arvados
+rm -rf tmp config.yml database.yml services/api/config/database.yml
+mkdir tmp
+go run ./cmd/arvados-server install -type test
+build/run-tests.sh WORKSPACE=\$PWD --temp /tmp/arvados/tmp ${qargs[@]}"
commit c6ccbde68f2ddeddc4ec4977865cc3b87081d781
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 17:40:29 2020 -0400
16053: Fix check for installed bundler versions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d403296c0..113404338 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -552,9 +552,10 @@ setup_ruby_environment() {
(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- for v in 1.11 1.17.3 2.0.2; do
+ versions=(1.11.0 1.17.3 2.0.2)
+ for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
- gem install --user bundler:1.11 bundler:1.17.3 bundler:2.0.2
+ gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
break
fi
done
commit f2c0f32cad805e0516cab0d0ca6d2c71dc08bd72
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 16:30:55 2020 -0400
16053: Fix test for already-installed en_US.UTF-8 locale.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index b08dd33bf..167763e9f 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -261,8 +261,11 @@ rm ${zip}
}
}
+ // The entry in /etc/locale.gen is "en_US.UTF-8"; once
+ // it's installed, locale -a reports it as
+ // "en_US.utf8".
wantlocale := "en_US.UTF-8"
- if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
+ if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
logger.Print("locale " + wantlocale + " already installed")
} else {
err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
commit 736ac33134b2925e4e5c3545f92a7fe6981c66b0
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 15:21:44 2020 -0400
15063: Fix tests to accommodate curl speed check granularity.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 68158d760..27e3cf633 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -832,7 +832,9 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
kc = self.keepClient()
loc = kc.put(self.DATA, copies=1, num_retries=0)
self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
- self.server.setdelays(response=self.TIMEOUT_TIME)
+ # Note the actual delay must be 1s longer than the low speed
+ # limit interval in order for curl to detect it reliably.
+ self.server.setdelays(response=self.TIMEOUT_TIME+1)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepReadError):
kc.get(loc, num_retries=0)
@@ -846,7 +848,9 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
kc = self.keepClient()
loc = kc.put(self.DATA, copies=1, num_retries=0)
self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
- self.server.setdelays(mid_write=self.TIMEOUT_TIME, mid_read=self.TIMEOUT_TIME)
+ # Note the actual delay must be 1s longer than the low speed
+ # limit interval in order for curl to detect it reliably.
+ self.server.setdelays(mid_write=self.TIMEOUT_TIME+1, mid_read=self.TIMEOUT_TIME+1)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepReadError) as e:
kc.get(loc, num_retries=0)
commit ea65b065f0651703fc58939ec382988d2f523592
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 14:19:10 2020 -0400
16053: Fix keepstore tests that fail when running as root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index a60aa416a..7777363b9 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -13,7 +13,6 @@ import (
"io"
"io/ioutil"
"os"
- "strings"
"sync"
"syscall"
"time"
@@ -168,11 +167,10 @@ func (s *UnixVolumeSuite) TestPutBadVolume(c *check.C) {
v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
- os.Chmod(v.Root, 000)
- err := v.Put(context.Background(), TestHash, TestBlock)
- if err == nil {
- c.Error("Write should have failed")
- }
+ err := os.RemoveAll(v.Root)
+ c.Assert(err, check.IsNil)
+ err = v.Put(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.IsNil)
}
func (s *UnixVolumeSuite) TestUnixVolumeReadonly(c *check.C) {
@@ -330,11 +328,14 @@ func (s *UnixVolumeSuite) TestUnixVolumeCompare(c *check.C) {
c.Errorf("Got err %q, expected %q", err, DiskHashError)
}
- p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
- os.Chmod(p, 000)
- err = v.Compare(context.Background(), TestHash, TestBlock)
- if err == nil || strings.Index(err.Error(), "permission denied") < 0 {
- c.Errorf("Got err %q, expected %q", err, "permission denied")
+ if os.Getuid() == 0 {
+ c.Log("skipping 'permission denied' check when running as root")
+ } else {
+ p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
+ err = os.Chmod(p, 000)
+ c.Assert(err, check.IsNil)
+ err = v.Compare(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.ErrorMatches, ".*permission denied.*")
}
}
commit 42c73e8144a7aa6fe0cd209b20897c03c2253d31
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 14:02:57 2020 -0400
16053: Upgrade mockito for JDK 11 compatibility.
Fixes:
17:19:59.486 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest STARTED
17:19:59.573 [DEBUG] [TestEventLogger]
17:19:59.573 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest > classMethod STARTED
17:19:59.593 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest > classMethod FAILED
17:19:59.594 [DEBUG] [TestEventLogger] org.mockito.exceptions.base.MockitoException:
17:19:59.594 [DEBUG] [TestEventLogger] Mockito cannot mock this class: class org.arvados.client.api.client.CollectionsApiClient.
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] If you're not sure why you're getting this error, please report to the mailing list.
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] Java : 11
17:19:59.595 [DEBUG] [TestEventLogger] JVM vendor name : Debian
17:19:59.595 [DEBUG] [TestEventLogger] JVM vendor version : 11.0.6+10-post-Debian-1deb10u1
17:19:59.595 [DEBUG] [TestEventLogger] JVM name : OpenJDK 64-Bit Server VM
17:19:59.595 [DEBUG] [TestEventLogger] JVM version : 11.0.6+10-post-Debian-1deb10u1
17:19:59.595 [DEBUG] [TestEventLogger] JVM info : mixed mode, sharing
17:19:59.595 [DEBUG] [TestEventLogger] OS name : Linux
17:19:59.595 [DEBUG] [TestEventLogger] OS version : 4.9.0-12-amd64
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger] You are seeing this disclaimer because Mockito is configured to create inlined mocks.
17:19:59.595 [DEBUG] [TestEventLogger] You can learn about inline mocks and their limitations under item #39 of the Mockito class javadoc.
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger] Underlying exception : org.mockito.exceptions.base.MockitoException: Could not modify all classes [class java.lang.Object, class org.arvados.client.api.client.BaseApiClient, class org.arvados.client.api.client.CollectionsApiClient, class org.arvados.client.api.client.BaseStandardApiClient]
17:19:59.595 [DEBUG] [TestEventLogger] at org.mockito.internal.runners.DefaultInternalRunner$1.withBefores(DefaultInternalRunner.java:38)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/java-v2/build.gradle b/sdk/java-v2/build.gradle
index 89f8a0cbb..5b09db948 100644
--- a/sdk/java-v2/build.gradle
+++ b/sdk/java-v2/build.gradle
@@ -21,7 +21,7 @@ dependencies {
api 'com.typesafe:config:1.3.2'
testImplementation 'junit:junit:4.12'
- testImplementation 'org.mockito:mockito-core:2.12.0'
+ testImplementation 'org.mockito:mockito-core:3.3.3'
testImplementation 'org.assertj:assertj-core:3.8.0'
testImplementation 'com.squareup.okhttp3:mockwebserver:3.9.1'
}
commit 1a46c2b0e24c0086e644fb9fe69b344883a34233
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 12:42:14 2020 -0400
16053: Pass along test args to gradle in java-v2 test suite.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 3cb559720..d403296c0 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1065,7 +1065,7 @@ test_sdk/cli() {
}
test_sdk/java-v2() {
- cd "$WORKSPACE/sdk/java-v2" && gradle test
+ cd "$WORKSPACE/sdk/java-v2" && gradle test ${testargs[sdk/java-v2]}
}
test_services/login-sync() {
commit ff2844616f4da0055039d39c88486e0e042c1f3e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 12:39:37 2020 -0400
16053: Install java runtime and jdk.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ff3adbb0a..b08dd33bf 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -99,6 +99,8 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"curl",
"cython",
"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
+ "default-jdk-headless",
+ "default-jre-headless",
"fuse",
"gettext",
"git",
commit 63842ff25274318349f511a507effe27f47d9fcc
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 20 17:04:38 2020 -0400
16053: Install gradle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index e0c3da5d2..ff3adbb0a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -168,14 +168,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
} else {
err = runBash(`
mkdir -p /var/lib/arvados/tmp
+tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
+trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd /var/lib/arvados/tmp/ruby-`+rubyversion+`
+cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
make -j4
make install
/var/lib/arvados/bin/gem install bundler
-cd /var/lib/arvados/tmp
-rm -r ruby-`+rubyversion+`
+rm -r ${tmp}
`, stdout, stderr)
if err != nil {
return 1
@@ -239,6 +240,25 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
}
}
+ gradleversion := "5.3.1"
+ if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
+ logger.Print("gradle " + gradleversion + " already installed")
+ } else {
+ err = runBash(`
+G=`+gradleversion+`
+mkdir -p /var/lib/arvados/tmp
+zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
+trap "rm ${zip}" ERR
+wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
+unzip -o -d /var/lib/arvados ${zip}
+ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
+rm ${zip}
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
wantlocale := "en_US.UTF-8"
if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
logger.Print("locale " + wantlocale + " already installed")
commit 1ea2920b1b525939868be927abe7a1512a690ef7
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 16:29:50 2020 -0400
16053: Fix warnings when installer cwd is not readable by postgres.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ffe0a8598..e0c3da5d2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -303,10 +303,13 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
}
withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
- if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff)
+ cmd.Dir = "/"
+ if err := cmd.Run(); err == nil {
logger.Print("arvados role exists; superuser privileges added, password updated")
} else {
cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+ cmd.Dir = "/"
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
commit 1baa00713cf519da9b65408a2b00ab6bf1f9784d
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 14:57:26 2020 -0400
16053: Fix permissions for test db user.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4146042bd..ffe0a8598 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -302,7 +302,7 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
// might never have been run.
}
- withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+ withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
logger.Print("arvados role exists; superuser privileges added, password updated")
} else {
commit b0766c77a7bfce908fdf9e02cd12a0cf35c0ee4c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:35:23 2020 -0400
16053: Run initdb, start postgresql, and add arvados user if needed.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 30110ddcd..4146042bd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -15,13 +15,18 @@ import (
"os/exec"
"strconv"
"strings"
+ "syscall"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
)
var Command cmd.Handler = installCommand{}
+const devtestDatabasePassword = "insecure_arvados_test"
+
type installCommand struct{}
func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -243,6 +248,72 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
return 1
}
}
+
+ var pgc struct {
+ Version string
+ Cluster string
+ Port int
+ Status string
+ Owner string
+ DataDirectory string
+ LogFile string
+ }
+ if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ err = fmt.Errorf("pg_lsclusters: %s", err2)
+ return 1
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
+ } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
+ err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
+ return 1
+ } else if pgc.Status == "online" {
+ logger.Infof("postgresql cluster %s-%s is online", pgc.Version, pgc.Cluster)
+ } else {
+ logger.Infof("postgresql cluster %s-%s is %s; trying to start", pgc.Version, pgc.Cluster, pgc.Status)
+ cmd := exec.Command("pg_ctlcluster", "--foreground", pgc.Version, pgc.Cluster, "start")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Start()
+ if err != nil {
+ return 1
+ }
+ defer func() {
+ cmd.Process.Signal(syscall.SIGTERM)
+ logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+ cmd.Wait()
+ }()
+ for deadline := time.Now().Add(10 * time.Second); ; {
+ output, err2 := exec.Command("pg_isready").CombinedOutput()
+ if err2 == nil {
+ break
+ } else if time.Now().After(deadline) {
+ err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+ return 1
+ } else {
+ time.Sleep(time.Second)
+ }
+ }
+ }
+
+ if os.Getpid() == 1 {
+ // We are the init process (presumably in a
+ // docker container) so although postgresql is
+ // installed, it's not running, and initdb
+ // might never have been run.
+ }
+
+ withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+ if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+ logger.Print("arvados role exists; superuser privileges added, password updated")
+ } else {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit e2afee2d51fcc81e1388a36a35dc9dd5b088e53b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:57 2020 -0400
16053: Install all three needed versions of bundler.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4e8254b72..3cb559720 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -551,8 +551,13 @@ setup_ruby_environment() {
bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
(
export HOME=$GEMHOME
- ("$bundle" version | grep -q 2.0.2) \
- || gem install --user bundler -v 2.0.2
+ bundlers="$(gem list --details bundler)"
+ for v in 1.11 1.17.3 2.0.2; do
+ if ! echo "$bundlers" | fgrep -q "($v)"; then
+ gem install --user bundler:1.11 bundler:1.17.3 bundler:2.0.2
+ break
+ fi
+ done
"$bundle" version | tee /dev/stderr | grep -q 'version 2'
) || fatal 'install bundler'
fi
commit aeeb1f6f7f11cbee30603284c594aab89c7fc610
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:35 2020 -0400
16053: Make CONFIGSRC optional again.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index f5c184e49..4e8254b72 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -35,7 +35,7 @@ Options:
--short Skip (or scale down) some slow tests.
--interactive Set up, then prompt for test/install steps to perform.
WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests.
services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
Restrict apiserver tests to the given file
sdk/python_test="--test-suite tests.test_keep_locator"
@@ -197,10 +197,8 @@ sanity_checks() {
[[ -n "${skip[sanity]}" ]] && return 0
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
- [[ -n "$CONFIGSRC" ]] \
- || fatal "CONFIGSRC environment not set (see: $0 --help)"
- [[ -s "$CONFIGSRC/config.yml" ]] \
- || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+ [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
+ || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
echo Checking dependencies:
echo "locale: ${LANG}"
[[ "$(locale charmap)" = "UTF-8" ]] \
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 262b9d2a2..f5ee9c383 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -661,11 +661,21 @@ def setup_config():
keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
- dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
-
- print("Getting config from %s" % dbconf, file=sys.stderr)
-
- pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ configsrc = os.environ.get("CONFIGSRC", None)
+ if configsrc:
+ clusterconf = os.path.join(configsrc, "config.yml")
+ print("Getting config from %s" % clusterconf, file=sys.stderr)
+ pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ else:
+ # assume "arvados-server install -type test" has set up the
+ # conventional db credentials
+ pgconnection = {
+ "client_encoding": "utf8",
+ "host": "localhost",
+ "dbname": "arvados_test",
+ "user": "arvados",
+ "password": "insecure_arvados_test",
+ }
localhost = "127.0.0.1"
services = {
commit 0344940255604764cb37cf9ea248b538cce04ff4
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 17 09:37:15 2020 -0400
16053: Remove unneeded GOPATH mangling.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps_test.go b/lib/install/deps_test.go
index c2344d878..a19bcb05f 100644
--- a/lib/install/deps_test.go
+++ b/lib/install/deps_test.go
@@ -29,7 +29,6 @@ func (*Suite) TestInstallDeps(c *check.C) {
tmp := c.MkDir()
script := `
set -x
-export GOPATH=${GOPATH:-${HOME}/go}
tmp="` + tmp + `"
sourcepath="$(realpath ../..)"
(cd ${sourcepath} && go build -o ${tmp} ./cmd/arvados-server)
commit 7d79f1914845742c68d99da5dea9c17bf92b24b9
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 17 09:28:15 2020 -0400
16053: Note test timeout trap.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps_test.go b/lib/install/deps_test.go
index d3e63986f..c2344d878 100644
--- a/lib/install/deps_test.go
+++ b/lib/install/deps_test.go
@@ -2,7 +2,10 @@
//
// SPDX-License-Identifier: AGPL-3.0
-// Skip this slow test unless invoked as "go test -tags docker":
+// Skip this slow test unless invoked as "go test -tags docker".
+// Depending on host/network speed, Go's default 10m test timeout
+// might be too short; recommend "go test -timeout 20m -tags docker".
+//
// +build docker
package install
commit f75883d7826d92894638e4eff6d397196dd1123f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 17:00:21 2020 -0400
16053: Add debian:10 install-and-boot test.
Run with "go test -check.vv -tags docker".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps_test.go b/lib/install/deps_test.go
new file mode 100644
index 000000000..d3e63986f
--- /dev/null
+++ b/lib/install/deps_test.go
@@ -0,0 +1,44 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// Skip this slow test unless invoked as "go test -tags docker":
+// +build docker
+
+package install
+
+import (
+ "os"
+ "testing"
+
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (*Suite) TestInstallDeps(c *check.C) {
+ tmp := c.MkDir()
+ script := `
+set -x
+export GOPATH=${GOPATH:-${HOME}/go}
+tmp="` + tmp + `"
+sourcepath="$(realpath ../..)"
+(cd ${sourcepath} && go build -o ${tmp} ./cmd/arvados-server)
+docker run -i --rm --workdir /arvados \
+ -v ${tmp}/arvados-server:/arvados-server:ro \
+ -v ${sourcepath}:/arvados:ro \
+ -v /arvados/services/api/.bundle \
+ -v /arvados/apps/workbench/.bundle \
+ --env http_proxy \
+ --env https_proxy \
+ debian:10 \
+ bash -c "/arvados-server install -type test && /arvados-server boot -type test -config doc/examples/config/zzzzz.yml -own-temporary-database -shutdown -timeout 9m"
+`
+ c.Check(runBash(script, os.Stdout, os.Stderr), check.IsNil)
+}
commit 5c4e2f9158f496f07c0b9838b2310da892730bce
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:22:21 2020 -0400
16053: Install en_US.UTF-8 locale.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 3b904c458..30110ddcd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -233,6 +233,16 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
return 1
}
}
+
+ wantlocale := "en_US.UTF-8"
+ if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
+ logger.Print("locale " + wantlocale + " already installed")
+ } else {
+ err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit af3fe384bb15c4fc5b915cdee5cd1f2a87b6301f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:22:09 2020 -0400
16053: Less verbose wget progress bars.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 6f2a2756a..3b904c458 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -162,17 +162,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
logger.Print("ruby " + rubyversion + " already installed")
} else {
err = runBash(`
-mkdir -p /var/lib/arvados/src
-cd /var/lib/arvados/src
-wget -c https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz
-tar xzf ruby-`+rubyversion+`.tar.gz
-cd ruby-`+rubyversion+`
+mkdir -p /var/lib/arvados/tmp
+wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
+cd /var/lib/arvados/tmp/ruby-`+rubyversion+`
./configure --disable-install-doc --prefix /var/lib/arvados
make -j4
make install
/var/lib/arvados/bin/gem install bundler
-cd ..
-rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
+cd /var/lib/arvados/tmp
+rm -r ruby-`+rubyversion+`
`, stdout, stderr)
if err != nil {
return 1
@@ -186,7 +184,7 @@ rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
} else {
err = runBash(`
cd /tmp
-wget -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
+wget --progress=dot:giga -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -200,7 +198,7 @@ ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
} else {
err = runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
-wget -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
+wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -214,7 +212,7 @@ ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
} else {
err = runBash(`
GD=v`+geckoversion+`
-wget -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
+wget --progress=dot:giga -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
`, stdout, stderr)
if err != nil {
@@ -228,7 +226,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
} else {
err = runBash(`
NJS=`+nodejsversion+`
-wget -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+wget --progress=dot:giga -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
`, stdout, stderr)
if err != nil {
commit 812d1e0ef6023add65ab5e4229278e4c56c13fa4
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:21:28 2020 -0400
16053: Prefer dependencies in /var/lib/arvados/bin if they exist.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7328fad45..f5c184e49 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -590,6 +590,11 @@ setup_virtualenv() {
}
initialize() {
+ # If dependencies like ruby, go, etc. are installed in
+ # /var/lib/arvados -- presumably by "arvados-server install" --
+ # then we want to use those versions, instead of whatever happens
+ # to be installed in /usr.
+ PATH="/var/lib/arvados/bin:${PATH}"
sanity_checks
echo "WORKSPACE=$WORKSPACE"
commit e6e89288f8d6fe79d7e982ded9c1347221021e2e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:20:51 2020 -0400
16053: Use setuidgid instead of sudo to drop privileges.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index a08180e0c..df90f36af 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -81,8 +81,13 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
- args = append([]string{"-u", "postgres", prog}, args...)
- prog = "sudo"
+ // We can't use "sudo -u" here because it creates an
+ // intermediate process that interferes with our
+ // ability to reliably kill postgres. The setuidgid
+ // program just calls exec without forking, so it
+ // doesn't have this problem.
+ args = append([]string{"postgres", prog}, args...)
+ prog = "setuidgid"
}
err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
if err != nil {
@@ -112,8 +117,8 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
"-p", super.cluster.PostgreSQL.Connection["port"],
}
if iamroot {
- args = append([]string{"-u", "postgres", prog}, args...)
- prog = "sudo"
+ args = append([]string{"postgres", prog}, args...)
+ prog = "setuidgid"
}
fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
}()
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index e75de3244..8ef7e6ac1 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -411,7 +411,7 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
logprefix := prog
- if logprefix == "sudo" && len(args) >= 3 && args[0] == "-u" {
+ if logprefix == "setuidgid" && len(args) >= 3 {
logprefix = args[2]
}
logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 50eab6aef..6f2a2756a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -93,6 +93,7 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"cadaver",
"curl",
"cython",
+ "daemontools", // lib/boot uses setuidgid to drop privileges when running as root
"fuse",
"gettext",
"git",
commit 4fd6857ff316d1e3638f63c90403168940adb32f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 16 16:04:31 2020 -0400
16053: Add boot flags: -timeout and -shutdown.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 1abc93722..5147e3ac3 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -6,9 +6,11 @@ package boot
import (
"context"
+ "errors"
"flag"
"fmt"
"io"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
@@ -56,6 +58,8 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
flags.StringVar(&super.ListenHost, "listen-host", "localhost", "host name or interface address for service listeners")
flags.StringVar(&super.ControllerAddr, "controller-address", ":0", "desired controller address, `host:port` or `:port`")
flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
+ timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
+ shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
@@ -77,14 +81,27 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
super.Start(ctx, cfg)
defer super.Stop()
+
+ var timer *time.Timer
+ if *timeout > 0 {
+ timer = time.AfterFunc(*timeout, super.Stop)
+ }
+
url, ok := super.WaitReady()
- if !ok {
+ if timer != nil && !timer.Stop() {
+ err = errors.New("boot timed out")
+ return 1
+ } else if !ok {
+ err = errors.New("boot failed")
return 1
}
// Write controller URL to stdout. Nothing else goes to
// stdout, so this provides an easy way for a calling script
// to discover the controller URL when everything is ready.
fmt.Fprintln(stdout, url)
+ if *shutdown {
+ super.Stop()
+ }
// Wait for signal/crash + orderly shutdown
<-super.done
return 0
commit 5e31613d2a0647647b710a0558e11408bc157406
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:16:56 2020 -0400
16053: Add "install" command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index a9d927d87..328599826 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -14,6 +14,7 @@ import (
"git.arvados.org/arvados.git/lib/controller"
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
+ "git.arvados.org/arvados.git/lib/install"
)
var (
@@ -30,6 +31,7 @@ var (
"controller": controller.Command,
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
+ "install": install.Command,
})
)
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index d230dcc3e..e75de3244 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -126,7 +126,7 @@ func (super *Supervisor) run(cfg *arvados.Config) error {
super.setEnv("ARVADOS_CONFIG", super.configfile)
super.setEnv("RAILS_ENV", super.ClusterType)
super.setEnv("TMPDIR", super.tempdir)
- super.prependEnv("PATH", filepath.Join(super.tempdir, "bin")+":")
+ super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:")
super.cluster, err = cfg.GetCluster("")
if err != nil {
@@ -360,7 +360,11 @@ func (super *Supervisor) setupRubyEnv() error {
"GEM_HOME=",
"GEM_PATH=",
})
- cmd := exec.Command("gem", "env", "gempath")
+ gem := "gem"
+ if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil {
+ gem = "/var/lib/arvados/bin/gem"
+ }
+ cmd := exec.Command(gem, "env", "gempath")
cmd.Env = super.environ
buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:...
if err != nil || len(buf) == 0 {
diff --git a/lib/install/deps.go b/lib/install/deps.go
new file mode 100644
index 000000000..50eab6aef
--- /dev/null
+++ b/lib/install/deps.go
@@ -0,0 +1,302 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package install
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+var Command cmd.Handler = installCommand{}
+
+type installCommand struct{}
+
+func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ var err error
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Info("exiting")
+ }
+ }()
+
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0")
+ clusterType := flags.String("type", "production", "cluster `type`: development, test, or production")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ } else if *versionFlag {
+ return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+ }
+
+ var dev, test, prod bool
+ switch *clusterType {
+ case "development":
+ dev = true
+ case "test":
+ test = true
+ case "production":
+ prod = true
+ default:
+ err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
+ return 2
+ }
+
+ osv, err := identifyOS()
+ if err != nil {
+ return 1
+ }
+
+ listdir, err := os.Open("/var/lib/apt/lists")
+ if err != nil {
+ logger.Warnf("error while checking whether to run apt-get update: %s", err)
+ } else if names, _ := listdir.Readdirnames(1); len(names) == 0 {
+ // Special case for a base docker image where the
+ // package cache has been deleted and all "apt-get
+ // install" commands will fail unless we fetch repos.
+ cmd := exec.CommandContext(ctx, "apt-get", "update")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ if dev || test {
+ debs := []string{
+ "bison",
+ "bsdmainutils",
+ "build-essential",
+ "cadaver",
+ "curl",
+ "cython",
+ "fuse",
+ "gettext",
+ "git",
+ "gitolite3",
+ "graphviz",
+ "haveged",
+ "iceweasel",
+ "libattr1-dev",
+ "libcrypt-ssleay-perl",
+ "libcrypt-ssleay-perl",
+ "libcurl3-gnutls",
+ "libcurl4-openssl-dev",
+ "libfuse-dev",
+ "libgnutls28-dev",
+ "libjson-perl",
+ "libjson-perl",
+ "libpam-dev",
+ "libpcre3-dev",
+ "libpq-dev",
+ "libpython2.7-dev",
+ "libreadline-dev",
+ "libssl-dev",
+ "libwww-perl",
+ "libxml2-dev",
+ "libxslt1.1",
+ "linkchecker",
+ "lsof",
+ "net-tools",
+ "nginx",
+ "pandoc",
+ "perl-modules",
+ "pkg-config",
+ "postgresql",
+ "postgresql-contrib",
+ "python",
+ "python3-dev",
+ "python-epydoc",
+ "r-base",
+ "r-cran-testthat",
+ "sudo",
+ "virtualenv",
+ "wget",
+ "xvfb",
+ "zlib1g-dev",
+ }
+ switch {
+ case osv.Debian && osv.Major >= 10:
+ debs = append(debs, "libcurl4")
+ default:
+ debs = append(debs, "libcurl3")
+ }
+ cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends")
+ cmd.Args = append(cmd.Args, debs...)
+ cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
+
+ os.Mkdir("/var/lib/arvados", 0755)
+ rubyversion := "2.5.7"
+ if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) {
+ logger.Print("ruby " + rubyversion + " already installed")
+ } else {
+ err = runBash(`
+mkdir -p /var/lib/arvados/src
+cd /var/lib/arvados/src
+wget -c https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz
+tar xzf ruby-`+rubyversion+`.tar.gz
+cd ruby-`+rubyversion+`
+./configure --disable-install-doc --prefix /var/lib/arvados
+make -j4
+make install
+/var/lib/arvados/bin/gem install bundler
+cd ..
+rm -r ruby-`+rubyversion+` ruby-`+rubyversion+`.tar.gz
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ if !prod {
+ goversion := "1.14"
+ if havegoversion, err := exec.Command("/usr/local/bin/go", "version").CombinedOutput(); err == nil && bytes.HasPrefix(havegoversion, []byte("go version go"+goversion+" ")) {
+ logger.Print("go " + goversion + " already installed")
+ } else {
+ err = runBash(`
+cd /tmp
+wget -O- https://storage.googleapis.com/golang/go`+goversion+`.linux-amd64.tar.gz | tar -C /var/lib/arvados -xzf -
+ln -sf /var/lib/arvados/go/bin/* /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ pjsversion := "1.9.8"
+ if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" {
+ logger.Print("phantomjs " + pjsversion + " already installed")
+ } else {
+ err = runBash(`
+PJS=phantomjs-`+pjsversion+`-linux-x86_64
+wget -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
+ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ geckoversion := "0.24.0"
+ if havegeckoversion, err := exec.Command("/usr/local/bin/geckodriver", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegeckoversion), " "+geckoversion+" ") {
+ logger.Print("geckodriver " + geckoversion + " already installed")
+ } else {
+ err = runBash(`
+GD=v`+geckoversion+`
+wget -O- https://github.com/mozilla/geckodriver/releases/download/$GD/geckodriver-$GD-linux64.tar.gz | tar -C /var/lib/arvados/bin -xzf - geckodriver
+ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
+ nodejsversion := "v8.15.1"
+ if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
+ logger.Print("nodejs " + nodejsversion + " already installed")
+ } else {
+ err = runBash(`
+NJS=`+nodejsversion+`
+wget -O- https://nodejs.org/dist/${NJS}/node-${NJS}-linux-x64.tar.xz | sudo tar -C /var/lib/arvados -xJf -
+ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+ }
+
+ return 0
+}
+
+type osversion struct {
+ Debian bool
+ Ubuntu bool
+ Major int
+}
+
+func identifyOS() (osversion, error) {
+ var osv osversion
+ f, err := os.Open("/etc/os-release")
+ if err != nil {
+ return osv, err
+ }
+ defer f.Close()
+
+ kv := map[string]string{}
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+ toks := strings.SplitN(line, "=", 2)
+ if len(toks) != 2 {
+ return osv, fmt.Errorf("invalid line in /etc/os-release: %q", line)
+ }
+ k := toks[0]
+ v := strings.Trim(toks[1], `"`)
+ if v == toks[1] {
+ v = strings.Trim(v, `'`)
+ }
+ kv[k] = v
+ }
+ if err = scanner.Err(); err != nil {
+ return osv, err
+ }
+ switch kv["ID"] {
+ case "ubuntu":
+ osv.Ubuntu = true
+ case "debian":
+ osv.Debian = true
+ default:
+ return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"])
+ }
+ vstr := kv["VERSION_ID"]
+ if i := strings.Index(vstr, "."); i > 0 {
+ vstr = vstr[:i]
+ }
+ osv.Major, err = strconv.Atoi(vstr)
+ if err != nil {
+ return osv, fmt.Errorf("incomprehensible VERSION_ID in /etc/os/release: %q", kv["VERSION_ID"])
+ }
+ return osv, nil
+}
+
+func runBash(script string, stdout, stderr io.Writer) error {
+ cmd := exec.Command("bash", "-")
+ cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ return cmd.Run()
+}
commit af4d0a2c71b4b633996fe33a9f63295244ea978f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:16:36 2020 -0400
16053: Run postgresql as "postgres" user if supervisor is root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index df9890415..a08180e0c 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -11,7 +11,9 @@ import (
"fmt"
"os"
"os/exec"
+ "os/user"
"path/filepath"
+ "strconv"
"strings"
"time"
@@ -34,6 +36,13 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
return err
}
+ iamroot := false
+ if u, err := user.Current(); err != nil {
+ return fmt.Errorf("user.Current(): %s", err)
+ } else if u.Uid == "0" {
+ iamroot = true
+ }
+
buf := bytes.NewBuffer(nil)
err = super.RunProgram(ctx, super.tempdir, buf, nil, "pg_config", "--bindir")
if err != nil {
@@ -42,11 +51,40 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
bindir := strings.TrimSpace(buf.String())
datadir := filepath.Join(super.tempdir, "pgdata")
- err = os.Mkdir(datadir, 0755)
+ err = os.Mkdir(datadir, 0700)
if err != nil {
return err
}
- err = super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "initdb"), "-D", datadir)
+ prog, args := filepath.Join(bindir, "initdb"), []string{"-D", datadir, "-E", "utf8"}
+ if iamroot {
+ postgresUser, err := user.Lookup("postgres")
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
+ }
+ postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
+ }
+ postgresGid, err := strconv.Atoi(postgresUser.Gid)
+ if err != nil {
+ return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric gid?: %q", postgresUser.Gid)
+ }
+ err = os.Chown(super.tempdir, 0, postgresGid)
+ if err != nil {
+ return err
+ }
+ err = os.Chmod(super.tempdir, 0710)
+ if err != nil {
+ return err
+ }
+ err = os.Chown(datadir, postgresUid, 0)
+ if err != nil {
+ return err
+ }
+ args = append([]string{"-u", "postgres", prog}, args...)
+ prog = "sudo"
+ }
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...)
if err != nil {
return err
}
@@ -55,18 +93,29 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
+ if iamroot {
+ err = super.RunProgram(ctx, super.tempdir, nil, nil, "chown", "postgres", datadir+"/server.crt", datadir+"/server.key")
+ if err != nil {
+ return err
+ }
+ }
port := super.cluster.PostgreSQL.Connection["port"]
super.waitShutdown.Add(1)
go func() {
defer super.waitShutdown.Done()
- fail(super.RunProgram(ctx, super.tempdir, nil, nil, filepath.Join(bindir, "postgres"),
+ prog, args := filepath.Join(bindir, "postgres"), []string{
"-l", // enable ssl
"-D", datadir, // data dir
"-k", datadir, // socket dir
"-p", super.cluster.PostgreSQL.Connection["port"],
- ))
+ }
+ if iamroot {
+ args = append([]string{"-u", "postgres", prog}, args...)
+ prog = "sudo"
+ }
+ fail(super.RunProgram(ctx, super.tempdir, nil, nil, prog, args...))
}()
for {
@@ -78,11 +127,15 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
}
time.Sleep(time.Second / 2)
}
- db, err := sql.Open("postgres", arvados.PostgreSQLConnection{
+ pgconn := arvados.PostgreSQLConnection{
"host": datadir,
"port": port,
"dbname": "postgres",
- }.String())
+ }
+ if iamroot {
+ pgconn["user"] = "postgres"
+ }
+ db, err := sql.Open("postgres", pgconn.String())
if err != nil {
return fmt.Errorf("db open failed: %s", err)
}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 4a4ab4d98..d230dcc3e 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -406,7 +406,11 @@ func (super *Supervisor) RunProgram(ctx context.Context, dir string, output io.W
cmdline := fmt.Sprintf("%s", append([]string{prog}, args...))
super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing")
- logprefix := strings.TrimPrefix(prog, super.tempdir+"/bin/")
+ logprefix := prog
+ if logprefix == "sudo" && len(args) >= 3 && args[0] == "-u" {
+ logprefix = args[2]
+ }
+ logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/")
if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" {
logprefix = args[1]
} else if logprefix == "arvados-server" && len(args) > 1 {
commit 919428dfcbb1f7d13a6082e4ffcb094059a773de
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 13 21:13:37 2020 -0400
16053: 15954: Use auto-generated certificates in nginx config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index 6b2d6777f..ecbb7a9d3 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -26,15 +26,18 @@ func (runNginx) String() string {
}
func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, createCertificates{})
+ if err != nil {
+ return err
+ }
vars := map[string]string{
"LISTENHOST": super.ListenHost,
- "SSLCERT": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.pem"), // TODO: root ca
- "SSLKEY": filepath.Join(super.SourcePath, "services", "api", "tmp", "self-signed.key"), // TODO: root ca
+ "SSLCERT": filepath.Join(super.tempdir, "server.crt"),
+ "SSLKEY": filepath.Join(super.tempdir, "server.key"),
"ACCESSLOG": filepath.Join(super.tempdir, "nginx_access.log"),
"ERRORLOG": filepath.Join(super.tempdir, "nginx_error.log"),
"TMPDIR": super.tempdir,
}
- var err error
for _, cmpt := range []struct {
varname string
svc arvados.Service
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list