[ARVADOS] updated: 1.3.0-2367-g68b7a9e73
Git user
git at public.arvados.org
Thu Mar 19 17:44:51 UTC 2020
Summary of changes:
build/run-tests.sh | 17 +++++----
lib/install/deps.go | 71 +++++++++++++++++++++++++++++++++++++
sdk/python/tests/run_test_server.py | 12 ++++---
3 files changed, 88 insertions(+), 12 deletions(-)
via 68b7a9e7357ffd21eccce0e8c8b3023148db24fe (commit)
via 74f0ef26d5edb4dde48bc13c91ff041fde971bef (commit)
via c65a90102ffb05b6a2dd557ccedb6a158b426597 (commit)
from 0344940255604764cb37cf9ea248b538cce04ff4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 68b7a9e7357ffd21eccce0e8c8b3023148db24fe
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:35:23 2020 -0400
16053: Run initdb, start postgresql, and add arvados user if needed.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 30110ddcd..4146042bd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -15,13 +15,18 @@ import (
"os/exec"
"strconv"
"strings"
+ "syscall"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
)
var Command cmd.Handler = installCommand{}
+const devtestDatabasePassword = "insecure_arvados_test"
+
type installCommand struct{}
func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -243,6 +248,72 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
return 1
}
}
+
+ var pgc struct {
+ Version string
+ Cluster string
+ Port int
+ Status string
+ Owner string
+ DataDirectory string
+ LogFile string
+ }
+ if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ err = fmt.Errorf("pg_lsclusters: %s", err2)
+ return 1
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
+ } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
+ err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
+ return 1
+ } else if pgc.Status == "online" {
+ logger.Infof("postgresql cluster %s-%s is online", pgc.Version, pgc.Cluster)
+ } else {
+ logger.Infof("postgresql cluster %s-%s is %s; trying to start", pgc.Version, pgc.Cluster, pgc.Status)
+ cmd := exec.Command("pg_ctlcluster", "--foreground", pgc.Version, pgc.Cluster, "start")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Start()
+ if err != nil {
+ return 1
+ }
+ defer func() {
+ cmd.Process.Signal(syscall.SIGTERM)
+ logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+ cmd.Wait()
+ }()
+ for deadline := time.Now().Add(10 * time.Second); ; {
+ output, err2 := exec.Command("pg_isready").CombinedOutput()
+ if err2 == nil {
+ break
+ } else if time.Now().After(deadline) {
+ err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+ return 1
+ } else {
+ time.Sleep(time.Second)
+ }
+ }
+ }
+
+ if os.Getpid() == 1 {
+ // We are the init process (presumably in a
+ // docker container) so although postgresql is
+ // installed, it's not running, and initdb
+ // might never have been run.
+ }
+
+ withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+ if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+ logger.Print("arvados role exists; superuser privileges added, password updated")
+ } else {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit 74f0ef26d5edb4dde48bc13c91ff041fde971bef
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:57 2020 -0400
16053: Install all three needed versions of bundler.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4e8254b72..3cb559720 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -551,8 +551,13 @@ setup_ruby_environment() {
bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
(
export HOME=$GEMHOME
- ("$bundle" version | grep -q 2.0.2) \
- || gem install --user bundler -v 2.0.2
+ bundlers="$(gem list --details bundler)"
+ for v in 1.11 1.17.3 2.0.2; do
+ if ! echo "$bundlers" | fgrep -q "($v)"; then
+ gem install --user bundler:1.11 bundler:1.17.3 bundler:2.0.2
+ break
+ fi
+ done
"$bundle" version | tee /dev/stderr | grep -q 'version 2'
) || fatal 'install bundler'
fi
commit c65a90102ffb05b6a2dd557ccedb6a158b426597
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:35 2020 -0400
16053: Make CONFIGSRC optional again.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index f5c184e49..4e8254b72 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -35,7 +35,7 @@ Options:
--short Skip (or scale down) some slow tests.
--interactive Set up, then prompt for test/install steps to perform.
WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests.
services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
Restrict apiserver tests to the given file
sdk/python_test="--test-suite tests.test_keep_locator"
@@ -197,10 +197,8 @@ sanity_checks() {
[[ -n "${skip[sanity]}" ]] && return 0
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
- [[ -n "$CONFIGSRC" ]] \
- || fatal "CONFIGSRC environment not set (see: $0 --help)"
- [[ -s "$CONFIGSRC/config.yml" ]] \
- || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+ [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
+ || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
echo Checking dependencies:
echo "locale: ${LANG}"
[[ "$(locale charmap)" = "UTF-8" ]] \
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 262b9d2a2..fd7fda540 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -661,11 +661,13 @@ def setup_config():
keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
- dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
-
- print("Getting config from %s" % dbconf, file=sys.stderr)
-
- pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ configsrc = os.environ.get("CONFIGSRC", None)
+ if configsrc:
+ clusterconf = os.path.join(configsrc, "config.yml")
+ else:
+ clusterconf = os.path.join(ARVADOS_DIR, "doc", "examples", "config", "zzzzz.yml")
+ print("Getting config from %s" % clusterconf, file=sys.stderr)
+ pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
localhost = "127.0.0.1"
services = {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list