[ARVADOS] updated: 1.3.0-2379-gaa3b45062
Git user
git at public.arvados.org
Tue Mar 24 17:45:19 UTC 2020
Summary of changes:
build/run-tests.sh | 7 +++---
lib/install/arvadostest_docker_build.sh | 38 +++++++++++++++++++++++++++++++++
lib/install/arvadostest_docker_run.sh | 32 +++++++++++++++++++++++++++
lib/install/deps.go | 38 ++++++++++++++++++++++++++++-----
lib/install/example_from_scratch.sh | 17 +++++++++++++++
sdk/java-v2/build.gradle | 2 +-
sdk/python/tests/test_keep_client.py | 8 +++++--
services/keepstore/unix_volume_test.go | 23 ++++++++++----------
8 files changed, 143 insertions(+), 22 deletions(-)
create mode 100755 lib/install/arvadostest_docker_build.sh
create mode 100755 lib/install/arvadostest_docker_run.sh
create mode 100644 lib/install/example_from_scratch.sh
via aa3b45062e8be2c9d7c14a436bc91a205a78fb9c (commit)
via 54fc5e527bcbcc8d0d7f75865624a0cf3f8632c7 (commit)
via c6ccbde68f2ddeddc4ec4977865cc3b87081d781 (commit)
via f2c0f32cad805e0516cab0d0ca6d2c71dc08bd72 (commit)
via 736ac33134b2925e4e5c3545f92a7fe6981c66b0 (commit)
via ea65b065f0651703fc58939ec382988d2f523592 (commit)
via 42c73e8144a7aa6fe0cd209b20897c03c2253d31 (commit)
via 1a46c2b0e24c0086e644fb9fe69b344883a34233 (commit)
via ff2844616f4da0055039d39c88486e0e042c1f3e (commit)
via 63842ff25274318349f511a507effe27f47d9fcc (commit)
via 1ea2920b1b525939868be927abe7a1512a690ef7 (commit)
from 1baa00713cf519da9b65408a2b00ab6bf1f9784d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit aa3b45062e8be2c9d7c14a436bc91a205a78fb9c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 24 12:12:30 2020 -0400
16053: Add example script for installing from scratch.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/example_from_scratch.sh b/lib/install/example_from_scratch.sh
new file mode 100644
index 000000000..03d9b7f63
--- /dev/null
+++ b/lib/install/example_from_scratch.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+# Starting with a base debian buster system, like "docker run -it
+# debian:10"...
+
+apt update
+apt upgrade
+apt install --no-install-recommends build-essential ca-certificates git golang
+git clone https://git.arvados.org/arvados.git
+cd arvados
+[[ -e lib/install ]] || git checkout origin/16053-install-deps
+cd cmd/arvados-server
+go run ./cmd/arvados-server install -type test
+pg_isready || pg_ctlcluster 11 main start # only needed if there's no init process (as in docker)
+build/run-tests.sh
commit 54fc5e527bcbcc8d0d7f75865624a0cf3f8632c7
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Mar 24 12:03:36 2020 -0400
16053: Add scripts for running tests in docker.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/arvadostest_docker_build.sh b/lib/install/arvadostest_docker_build.sh
new file mode 100755
index 000000000..57ca99b26
--- /dev/null
+++ b/lib/install/arvadostest_docker_build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+SRC=$(realpath $(dirname ${BASH_SOURCE[0]})/../..)
+
+ctrname=arvadostest
+ctrbase=${ctrname}
+if [[ "${1}" != "--update" ]] || ! docker images --format={{.Repository}} | grep -x ${ctrbase}; then
+ ctrbase=debian:10
+fi
+
+if docker ps -a --format={{.Names}} | grep -x ${ctrname}; then
+ echo >&2 "container name already in use -- another builder running?"
+ exit 1
+fi
+
+(cd ${SRC}/cmd/arvados-server && go install)
+trap "docker rm --volumes ${ctrname}" ERR
+docker run -it --name ${ctrname} \
+ -v ${GOPATH:-${HOME}/go}/bin/arvados-server:/bin/arvados-server:ro \
+ -v ${SRC}:/src/arvados:ro \
+ -v /tmp \
+ --env http_proxy \
+ --env https_proxy \
+ ${ctrbase} \
+ bash -c "
+set -ex -o pipefail
+arvados-server install -type test
+pg_ctlcluster 11 main start
+cp -a /src/arvados /tmp/
+cd /tmp/arvados
+rm -rf tmp services/api/config/database.yml
+mkdir tmp
+build/run-tests.sh WORKSPACE=\$PWD --temp /tmp/arvados/tmp --only x"
+docker commit ${ctrname} ${ctrname}
+trap - ERR
+docker rm --volumes ${ctrname}
diff --git a/lib/install/arvadostest_docker_run.sh b/lib/install/arvadostest_docker_run.sh
new file mode 100755
index 000000000..a6d7d57e2
--- /dev/null
+++ b/lib/install/arvadostest_docker_run.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Example:
+#
+# ./arvadostest_docker_build.sh # build the base image ("arvadostest")
+# ./arvadostest_docker_build.sh --update # update the base image with current version of `arvados-server install`
+# ./arvadostest_docker_run.sh --interactive # start a container using the previously built base image, copy this source tree into it, and invoke run-tests.sh with the given args
+
+set -ex -o pipefail
+
+declare -a qargs
+for arg in "$@"; do
+ qargs+=("${arg at Q}")
+done
+
+SRC=$(realpath $(dirname ${BASH_SOURCE[0]})/../..)
+
+docker run --rm -it \
+ -v ${SRC}:/src/arvados:ro \
+ -v /tmp \
+ --env http_proxy \
+ --env https_proxy \
+ arvadostest \
+ bash -c "
+set -ex -o pipefail
+pg_ctlcluster 11 main start
+cp -a /src/arvados /tmp/
+cd /tmp/arvados
+rm -rf tmp services/api/config/database.yml
+mkdir tmp
+go run ./cmd/arvados-server install -type test
+build/run-tests.sh WORKSPACE=\$PWD --temp /tmp/arvados/tmp ${qargs[@]}"
commit c6ccbde68f2ddeddc4ec4977865cc3b87081d781
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 17:40:29 2020 -0400
16053: Fix check for installed bundler versions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index d403296c0..113404338 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -552,9 +552,10 @@ setup_ruby_environment() {
(
export HOME=$GEMHOME
bundlers="$(gem list --details bundler)"
- for v in 1.11 1.17.3 2.0.2; do
+ versions=(1.11.0 1.17.3 2.0.2)
+ for v in ${versions[@]}; do
if ! echo "$bundlers" | fgrep -q "($v)"; then
- gem install --user bundler:1.11 bundler:1.17.3 bundler:2.0.2
+ gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
break
fi
done
commit f2c0f32cad805e0516cab0d0ca6d2c71dc08bd72
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 16:30:55 2020 -0400
16053: Fix test for already-installed en_US.UTF-8 locale.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index b08dd33bf..167763e9f 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -261,8 +261,11 @@ rm ${zip}
}
}
+ // The entry in /etc/locale.gen is "en_US.UTF-8"; once
+ // it's installed, locale -a reports it as
+ // "en_US.utf8".
wantlocale := "en_US.UTF-8"
- if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
+ if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(strings.Replace(wantlocale+"\n", "UTF-", "utf", 1))) {
logger.Print("locale " + wantlocale + " already installed")
} else {
err = runBash(`sed -i 's/^# *\(`+wantlocale+`\)/\1/' /etc/locale.gen && locale-gen`, stdout, stderr)
commit 736ac33134b2925e4e5c3545f92a7fe6981c66b0
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 15:21:44 2020 -0400
15063: Fix tests to accommodate curl speed check granularity.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 68158d760..27e3cf633 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -832,7 +832,9 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
kc = self.keepClient()
loc = kc.put(self.DATA, copies=1, num_retries=0)
self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
- self.server.setdelays(response=self.TIMEOUT_TIME)
+ # Note the actual delay must be 1s longer than the low speed
+ # limit interval in order for curl to detect it reliably.
+ self.server.setdelays(response=self.TIMEOUT_TIME+1)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepReadError):
kc.get(loc, num_retries=0)
@@ -846,7 +848,9 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
kc = self.keepClient()
loc = kc.put(self.DATA, copies=1, num_retries=0)
self.server.setbandwidth(self.BANDWIDTH_LOW_LIM)
- self.server.setdelays(mid_write=self.TIMEOUT_TIME, mid_read=self.TIMEOUT_TIME)
+ # Note the actual delay must be 1s longer than the low speed
+ # limit interval in order for curl to detect it reliably.
+ self.server.setdelays(mid_write=self.TIMEOUT_TIME+1, mid_read=self.TIMEOUT_TIME+1)
with self.assertTakesGreater(self.TIMEOUT_TIME):
with self.assertRaises(arvados.errors.KeepReadError) as e:
kc.get(loc, num_retries=0)
commit ea65b065f0651703fc58939ec382988d2f523592
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 14:19:10 2020 -0400
16053: Fix keepstore tests that fail when running as root.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index a60aa416a..7777363b9 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -13,7 +13,6 @@ import (
"io"
"io/ioutil"
"os"
- "strings"
"sync"
"syscall"
"time"
@@ -168,11 +167,10 @@ func (s *UnixVolumeSuite) TestPutBadVolume(c *check.C) {
v := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
defer v.Teardown()
- os.Chmod(v.Root, 000)
- err := v.Put(context.Background(), TestHash, TestBlock)
- if err == nil {
- c.Error("Write should have failed")
- }
+ err := os.RemoveAll(v.Root)
+ c.Assert(err, check.IsNil)
+ err = v.Put(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.IsNil)
}
func (s *UnixVolumeSuite) TestUnixVolumeReadonly(c *check.C) {
@@ -330,11 +328,14 @@ func (s *UnixVolumeSuite) TestUnixVolumeCompare(c *check.C) {
c.Errorf("Got err %q, expected %q", err, DiskHashError)
}
- p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
- os.Chmod(p, 000)
- err = v.Compare(context.Background(), TestHash, TestBlock)
- if err == nil || strings.Index(err.Error(), "permission denied") < 0 {
- c.Errorf("Got err %q, expected %q", err, "permission denied")
+ if os.Getuid() == 0 {
+ c.Log("skipping 'permission denied' check when running as root")
+ } else {
+ p := fmt.Sprintf("%s/%s/%s", v.Root, TestHash[:3], TestHash)
+ err = os.Chmod(p, 000)
+ c.Assert(err, check.IsNil)
+ err = v.Compare(context.Background(), TestHash, TestBlock)
+ c.Check(err, check.ErrorMatches, ".*permission denied.*")
}
}
commit 42c73e8144a7aa6fe0cd209b20897c03c2253d31
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 14:02:57 2020 -0400
16053: Upgrade mockito for JDK 11 compatibility.
Fixes:
17:19:59.486 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest STARTED
17:19:59.573 [DEBUG] [TestEventLogger]
17:19:59.573 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest > classMethod STARTED
17:19:59.593 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] org.arvados.client.logic.keep.FileDownloaderTest > classMethod FAILED
17:19:59.594 [DEBUG] [TestEventLogger] org.mockito.exceptions.base.MockitoException:
17:19:59.594 [DEBUG] [TestEventLogger] Mockito cannot mock this class: class org.arvados.client.api.client.CollectionsApiClient.
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] If you're not sure why you're getting this error, please report to the mailing list.
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger]
17:19:59.594 [DEBUG] [TestEventLogger] Java : 11
17:19:59.595 [DEBUG] [TestEventLogger] JVM vendor name : Debian
17:19:59.595 [DEBUG] [TestEventLogger] JVM vendor version : 11.0.6+10-post-Debian-1deb10u1
17:19:59.595 [DEBUG] [TestEventLogger] JVM name : OpenJDK 64-Bit Server VM
17:19:59.595 [DEBUG] [TestEventLogger] JVM version : 11.0.6+10-post-Debian-1deb10u1
17:19:59.595 [DEBUG] [TestEventLogger] JVM info : mixed mode, sharing
17:19:59.595 [DEBUG] [TestEventLogger] OS name : Linux
17:19:59.595 [DEBUG] [TestEventLogger] OS version : 4.9.0-12-amd64
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger] You are seeing this disclaimer because Mockito is configured to create inlined mocks.
17:19:59.595 [DEBUG] [TestEventLogger] You can learn about inline mocks and their limitations under item #39 of the Mockito class javadoc.
17:19:59.595 [DEBUG] [TestEventLogger]
17:19:59.595 [DEBUG] [TestEventLogger] Underlying exception : org.mockito.exceptions.base.MockitoException: Could not modify all classes [class java.lang.Object, class org.arvados.client.api.client.BaseApiClient, class org.arvados.client.api.client.CollectionsApiClient, class org.arvados.client.api.client.BaseStandardApiClient]
17:19:59.595 [DEBUG] [TestEventLogger] at org.mockito.internal.runners.DefaultInternalRunner$1.withBefores(DefaultInternalRunner.java:38)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/java-v2/build.gradle b/sdk/java-v2/build.gradle
index 89f8a0cbb..5b09db948 100644
--- a/sdk/java-v2/build.gradle
+++ b/sdk/java-v2/build.gradle
@@ -21,7 +21,7 @@ dependencies {
api 'com.typesafe:config:1.3.2'
testImplementation 'junit:junit:4.12'
- testImplementation 'org.mockito:mockito-core:2.12.0'
+ testImplementation 'org.mockito:mockito-core:3.3.3'
testImplementation 'org.assertj:assertj-core:3.8.0'
testImplementation 'com.squareup.okhttp3:mockwebserver:3.9.1'
}
commit 1a46c2b0e24c0086e644fb9fe69b344883a34233
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 12:42:14 2020 -0400
16053: Pass along test args to gradle in java-v2 test suite.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 3cb559720..d403296c0 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1065,7 +1065,7 @@ test_sdk/cli() {
}
test_sdk/java-v2() {
- cd "$WORKSPACE/sdk/java-v2" && gradle test
+ cd "$WORKSPACE/sdk/java-v2" && gradle test ${testargs[sdk/java-v2]}
}
test_services/login-sync() {
commit ff2844616f4da0055039d39c88486e0e042c1f3e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Mar 23 12:39:37 2020 -0400
16053: Install java runtime and jdk.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ff3adbb0a..b08dd33bf 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -99,6 +99,8 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
"curl",
"cython",
"daemontools", // lib/boot uses setuidgid to drop privileges when running as root
+ "default-jdk-headless",
+ "default-jre-headless",
"fuse",
"gettext",
"git",
commit 63842ff25274318349f511a507effe27f47d9fcc
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Mar 20 17:04:38 2020 -0400
16053: Install gradle.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index e0c3da5d2..ff3adbb0a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -168,14 +168,15 @@ func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, st
} else {
err = runBash(`
mkdir -p /var/lib/arvados/tmp
+tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+`
+trap "rm -r ${tmp}" ERR
wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf -
-cd /var/lib/arvados/tmp/ruby-`+rubyversion+`
+cd ${tmp}
./configure --disable-install-doc --prefix /var/lib/arvados
make -j4
make install
/var/lib/arvados/bin/gem install bundler
-cd /var/lib/arvados/tmp
-rm -r ruby-`+rubyversion+`
+rm -r ${tmp}
`, stdout, stderr)
if err != nil {
return 1
@@ -239,6 +240,25 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
}
}
+ gradleversion := "5.3.1"
+ if havegradleversion, err := exec.Command("/usr/local/bin/gradle", "--version").CombinedOutput(); err == nil && strings.Contains(string(havegradleversion), "Gradle "+gradleversion+"\n") {
+ logger.Print("gradle " + gradleversion + " already installed")
+ } else {
+ err = runBash(`
+G=`+gradleversion+`
+mkdir -p /var/lib/arvados/tmp
+zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip
+trap "rm ${zip}" ERR
+wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip
+unzip -o -d /var/lib/arvados ${zip}
+ln -sf /var/lib/arvados/gradle-${G}/bin/gradle /usr/local/bin/
+rm ${zip}
+`, stdout, stderr)
+ if err != nil {
+ return 1
+ }
+ }
+
wantlocale := "en_US.UTF-8"
if havelocales, err := exec.Command("locale", "-a").CombinedOutput(); err == nil && bytes.Contains(havelocales, []byte(wantlocale+"\n")) {
logger.Print("locale " + wantlocale + " already installed")
commit 1ea2920b1b525939868be927abe7a1512a690ef7
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 16:29:50 2020 -0400
16053: Fix warnings when installer cwd is not readable by postgres.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index ffe0a8598..e0c3da5d2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -303,10 +303,13 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
}
withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
- if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff)
+ cmd.Dir = "/"
+ if err := cmd.Run(); err == nil {
logger.Print("arvados role exists; superuser privileges added, password updated")
} else {
cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+ cmd.Dir = "/"
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list