[ARVADOS] updated: 1.3.0-2368-g1baa00713
Git user
git at public.arvados.org
Thu Mar 19 18:58:08 UTC 2020
Summary of changes:
lib/install/deps.go | 2 +-
sdk/python/tests/run_test_server.py | 14 +++++++++++---
2 files changed, 12 insertions(+), 4 deletions(-)
discards 68b7a9e7357ffd21eccce0e8c8b3023148db24fe (commit)
discards 74f0ef26d5edb4dde48bc13c91ff041fde971bef (commit)
discards c65a90102ffb05b6a2dd557ccedb6a158b426597 (commit)
via 1baa00713cf519da9b65408a2b00ab6bf1f9784d (commit)
via b0766c77a7bfce908fdf9e02cd12a0cf35c0ee4c (commit)
via e2afee2d51fcc81e1388a36a35dc9dd5b088e53b (commit)
via aeeb1f6f7f11cbee30603284c594aab89c7fc610 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (68b7a9e7357ffd21eccce0e8c8b3023148db24fe)
\
N -- N -- N (1baa00713cf519da9b65408a2b00ab6bf1f9784d)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1baa00713cf519da9b65408a2b00ab6bf1f9784d
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 14:57:26 2020 -0400
16053: Fix permissions for test db user.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4146042bd..ffe0a8598 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -302,7 +302,7 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
// might never have been run.
}
- withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+ withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
logger.Print("arvados role exists; superuser privileges added, password updated")
} else {
commit b0766c77a7bfce908fdf9e02cd12a0cf35c0ee4c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:35:23 2020 -0400
16053: Run initdb, start postgresql, and add arvados user if needed.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 30110ddcd..4146042bd 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -15,13 +15,18 @@ import (
"os/exec"
"strconv"
"strings"
+ "syscall"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/lib/pq"
)
var Command cmd.Handler = installCommand{}
+const devtestDatabasePassword = "insecure_arvados_test"
+
type installCommand struct{}
func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
@@ -243,6 +248,72 @@ ln -sf /var/lib/arvados/node-${NJS}-linux-x64/bin/{node,npm} /usr/local/bin/
return 1
}
}
+
+ var pgc struct {
+ Version string
+ Cluster string
+ Port int
+ Status string
+ Owner string
+ DataDirectory string
+ LogFile string
+ }
+ if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ err = fmt.Errorf("pg_lsclusters: %s", err2)
+ return 1
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
+ } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
+ err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
+ return 1
+ } else if pgc.Status == "online" {
+ logger.Infof("postgresql cluster %s-%s is online", pgc.Version, pgc.Cluster)
+ } else {
+ logger.Infof("postgresql cluster %s-%s is %s; trying to start", pgc.Version, pgc.Cluster, pgc.Status)
+ cmd := exec.Command("pg_ctlcluster", "--foreground", pgc.Version, pgc.Cluster, "start")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Start()
+ if err != nil {
+ return 1
+ }
+ defer func() {
+ cmd.Process.Signal(syscall.SIGTERM)
+ logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+ cmd.Wait()
+ }()
+ for deadline := time.Now().Add(10 * time.Second); ; {
+ output, err2 := exec.Command("pg_isready").CombinedOutput()
+ if err2 == nil {
+ break
+ } else if time.Now().After(deadline) {
+ err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+ return 1
+ } else {
+ time.Sleep(time.Second)
+ }
+ }
+ }
+
+ if os.Getpid() == 1 {
+ // We are the init process (presumably in a
+ // docker container) so although postgresql is
+ // installed, it's not running, and initdb
+ // might never have been run.
+ }
+
+ withstuff := "WITH SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
+ if err := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff).Run(); err == nil {
+ logger.Print("arvados role exists; superuser privileges added, password updated")
+ } else {
+ cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE ROLE arvados "+withstuff)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return 1
+ }
+ }
}
return 0
commit e2afee2d51fcc81e1388a36a35dc9dd5b088e53b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:57 2020 -0400
16053: Install all three needed versions of bundler.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4e8254b72..3cb559720 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -551,8 +551,13 @@ setup_ruby_environment() {
bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
(
export HOME=$GEMHOME
- ("$bundle" version | grep -q 2.0.2) \
- || gem install --user bundler -v 2.0.2
+ bundlers="$(gem list --details bundler)"
+ for v in 1.11 1.17.3 2.0.2; do
+ if ! echo "$bundlers" | fgrep -q "($v)"; then
+ gem install --user bundler:1.11 bundler:1.17.3 bundler:2.0.2
+ break
+ fi
+ done
"$bundle" version | tee /dev/stderr | grep -q 'version 2'
) || fatal 'install bundler'
fi
commit aeeb1f6f7f11cbee30603284c594aab89c7fc610
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Mar 19 13:34:35 2020 -0400
16053: Make CONFIGSRC optional again.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index f5c184e49..4e8254b72 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -35,7 +35,7 @@ Options:
--short Skip (or scale down) some slow tests.
--interactive Set up, then prompt for test/install steps to perform.
WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests.
services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
Restrict apiserver tests to the given file
sdk/python_test="--test-suite tests.test_keep_locator"
@@ -197,10 +197,8 @@ sanity_checks() {
[[ -n "${skip[sanity]}" ]] && return 0
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
- [[ -n "$CONFIGSRC" ]] \
- || fatal "CONFIGSRC environment not set (see: $0 --help)"
- [[ -s "$CONFIGSRC/config.yml" ]] \
- || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+ [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
+ || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
echo Checking dependencies:
echo "locale: ${LANG}"
[[ "$(locale charmap)" = "UTF-8" ]] \
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 262b9d2a2..f5ee9c383 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -661,11 +661,21 @@ def setup_config():
keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
- dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
-
- print("Getting config from %s" % dbconf, file=sys.stderr)
-
- pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ configsrc = os.environ.get("CONFIGSRC", None)
+ if configsrc:
+ clusterconf = os.path.join(configsrc, "config.yml")
+ print("Getting config from %s" % clusterconf, file=sys.stderr)
+ pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ else:
+ # assume "arvados-server install -type test" has set up the
+ # conventional db credentials
+ pgconnection = {
+ "client_encoding": "utf8",
+ "host": "localhost",
+ "dbname": "arvados_test",
+ "user": "arvados",
+ "password": "insecure_arvados_test",
+ }
localhost = "127.0.0.1"
services = {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list