[ARVADOS] updated: 2.1.0-1062-g13b4d2193
Git user
git at public.arvados.org
Fri Jul 16 14:41:54 UTC 2021
Summary of changes:
lib/crunchrun/crunchrun.go | 36 ++++++++++++++++++++++++++++++------
lib/crunchrun/crunchrun_test.go | 4 ----
lib/crunchrun/docker.go | 15 +++------------
lib/crunchrun/executor.go | 6 +++---
lib/crunchrun/singularity.go | 2 --
5 files changed, 36 insertions(+), 27 deletions(-)
discards eec5086af5c2d1c1f17bbc525cc68d394c9680f4 (commit)
discards ec5fd6d8072cc038648b254a5500868dfdf6a17b (commit)
discards f2ee5bac37391ce9fe084306da332becd7620ca7 (commit)
discards 753d479b0b5960674bf8e5a27ee98f68b3cd06ce (commit)
discards 3bd5ef5231bcc2d378c9cf756168390dbd06bb32 (commit)
discards 237f9a7c77495bc8c71404f495f5cc6879110e9e (commit)
discards afd47207817a4d2a0ac76acd8175ed9a49fc1b6d (commit)
discards a473534ef25367c250cc5a0bc7c57cd1e11b8c57 (commit)
discards a2fb4946954504e7e0eaa0ebc6a0af14029ec797 (commit)
discards e3b8b3683f0ac0ce91f9e2684dc593e5f4bea7ab (commit)
via 13b4d219384a81141846588b20f07792d64cb489 (commit)
via ffa821a7f872d211f534914bbd21f0fbd9064668 (commit)
via d2f732034404f96ebd50999313cbe9a6727d3f95 (commit)
via 879bde382ebf26aa593869cfff22cc7e85be6bb0 (commit)
via 570af13f6dfd7f725abf49335b7abdb46aae5048 (commit)
via 8b5c403446d137d7fc55df278ccb02ef84e87e0a (commit)
via bf01d196dd3ec9ed51a56b39aff9a52cd726de48 (commit)
via 6771c21313fa4ba22cdd3007c4c5a03b8de8227c (commit)
via c727a33fd753e97137d4ec092a73d592253b917c (commit)
via 4879256386a5be9566f31f2c266b682993029e14 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (eec5086af5c2d1c1f17bbc525cc68d394c9680f4)
\
N -- N -- N (13b4d219384a81141846588b20f07792d64cb489)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 13b4d219384a81141846588b20f07792d64cb489
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 15 13:50:22 2021 -0400
17816: Set the current working directory in the singularity container
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 4bec8c3eb..2a294cf03 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -74,7 +74,7 @@ func (e *singularityExecutor) Create(spec containerSpec) error {
}
func (e *singularityExecutor) Start() error {
- args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+ args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
if !e.spec.EnableNetwork {
args = append(args, "--net", "--network=none")
}
commit ffa821a7f872d211f534914bbd21f0fbd9064668
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 15 15:58:45 2021 -0400
17816: crunch-dispatch-local uses cluster config.
Added upgrade note.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 13f093394..3c283c354 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -35,10 +35,14 @@ TODO: extract this information based on git commit messages and generate changel
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2021-06-03)
+h2(#main). development main (as of 2021-07-15)
"Upgrading from 2.2.0":#v2_2_0
+h3. crunch-dispatch-local now requires config.yml
+
+The @crunch-dispatch-local@ dispatcher now reads the API host and token from the system wide @/etc/arvados/config.yml@ . It will fail to start that file is not found or not readable.
+
h2(#v2_2_0). v2.2.0 (2021-06-03)
"Upgrading from 2.1.0":#v2_1_0
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 4a45f1054..c202e683f 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -86,6 +86,26 @@ func doMain() error {
runningCmds = make(map[string]*exec.Cmd)
+ var client arvados.Client
+ client.APIHost = cluster.Services.Controller.ExternalURL.Host
+ client.AuthToken = cluster.SystemRootToken
+ client.Insecure = cluster.TLS.Insecure
+
+ if client.APIHost != "" || client.AuthToken != "" {
+ // Copy real configs into env vars so [a]
+ // MakeArvadosClient() uses them, and [b] they get
+ // propagated to crunch-run via SLURM.
+ os.Setenv("ARVADOS_API_HOST", client.APIHost)
+ os.Setenv("ARVADOS_API_TOKEN", client.AuthToken)
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+ if client.Insecure {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+ }
+ os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+ } else {
+ logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
+ }
+
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
logger.Errorf("error making Arvados client: %v", err)
commit d2f732034404f96ebd50999313cbe9a6727d3f95
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:47:45 2021 -0400
17816: Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 5f51134df..d976bf081 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -81,8 +81,10 @@ func (s *TestSuite) TestIntegration(c *C) {
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
cancel()
}
@@ -184,8 +186,10 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
cancel()
}
commit 879bde382ebf26aa593869cfff22cc7e85be6bb0
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:39:02 2021 -0400
17816: Make singularity-in-arvbox work
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
deleted file mode 120000
index a388c8b67..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
+++ /dev/null
@@ -1 +0,0 @@
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
new file mode 100755
index 000000000..821afdce5
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
+install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
+ln -sf arvados-server /usr/local/bin/crunch-run
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+cat > /usr/local/bin/crunch-run.sh <<EOF
+#!/bin/sh
+exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
+EOF
+chmod +x /usr/local/bin/crunch-run.sh
+
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
+
+exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
deleted file mode 100755
index 821afdce5..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
-install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
-ln -sf arvados-server /usr/local/bin/crunch-run
-
-if test "$1" = "--only-deps" ; then
- exit
-fi
-
-cat > /usr/local/bin/crunch-run.sh <<EOF
-#!/bin/sh
-exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
-EOF
-chmod +x /usr/local/bin/crunch-run.sh
-
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index b29dafed7..f49e9ea26 100755
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -41,7 +41,7 @@ for sdk_app in arv arv-get cwl-runner arv-mount ; do
fi
done
-if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
+if ! (ps ax | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
waiting="$waiting crunch-dispatch"
fi
commit 570af13f6dfd7f725abf49335b7abdb46aae5048
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:37:19 2021 -0400
17816: Add --runtime-engine to crunch-dispatch-local and crunch-dispatch-slurm
refs #17816
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 2922817b5..4a45f1054 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -17,6 +17,7 @@ import (
"syscall"
"time"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/dispatch"
@@ -74,6 +75,13 @@ func doMain() error {
return nil
}
+ loader := config.NewLoader(nil, logger)
+ cfg, err := loader.Load()
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return fmt.Errorf("config error: %s", err)
+ }
+
logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
@@ -90,7 +98,7 @@ func doMain() error {
dispatcher := dispatch.Dispatcher{
Logger: logger,
Arv: arv,
- RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx}).run,
+ RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx, cluster}).run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
@@ -128,6 +136,7 @@ type LocalRun struct {
startCmd func(container arvados.Container, cmd *exec.Cmd) error
concurrencyLimit chan bool
ctx context.Context
+ cluster *arvados.Cluster
}
// Run a container.
@@ -169,7 +178,7 @@ func (lr *LocalRun) run(dispatcher *dispatch.Dispatcher,
waitGroup.Add(1)
defer waitGroup.Done()
- cmd := exec.Command(*crunchRunCommand, uuid)
+ cmd := exec.Command(*crunchRunCommand, "--runtime-engine="+lr.cluster.Containers.RuntimeEngine, uuid)
cmd.Stdin = nil
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index a5899ce8a..2f2f013c7 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -255,6 +255,7 @@ func (disp *Dispatcher) submit(container arvados.Container, crunchRunCommand []s
// append() here avoids modifying crunchRunCommand's
// underlying array, which is shared with other goroutines.
crArgs := append([]string(nil), crunchRunCommand...)
+ crArgs = append(crArgs, "--runtime-engine="+disp.cluster.Containers.RuntimeEngine)
crArgs = append(crArgs, container.UUID)
crScript := strings.NewReader(execScript(crArgs))
commit 8b5c403446d137d7fc55df278ccb02ef84e87e0a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:50:07 2021 -0400
Don't pull arvbox checkouts after clone
We already have latest and tags/detached HEADS can't be pulled anyway
(and they fail, which is the real problem).
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index ba3f2cd07..fd464974f 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -250,17 +250,14 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
git -C "$COMPOSER_ROOT" checkout arvados-fork
- git -C "$COMPOSER_ROOT" pull
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index c285d53ca..92099614a 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -10,13 +10,10 @@ ARG workbench2_version=main
RUN cd /usr/src && \
git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
- git -C arvados pull && \
git clone --no-checkout https://github.com/arvados/composer.git && \
git -C composer checkout ${composer_version} && \
- git -C composer pull && \
git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
git -C workbench2 checkout ${workbench2_version} && \
- git -C workbench2 pull && \
chown -R 1000:1000 /usr/src
# avoid rebuilding arvados-server, it's already been built as part of the base image
commit bf01d196dd3ec9ed51a56b39aff9a52cd726de48
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:12:08 2021 -0400
arvbox docs include a docker image tag
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 8b3994edd..a8235ee70 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Quick start
<pre>
$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
$ chmod +x arvbox
-$ ./arvbox start localdemo
+$ ./arvbox start localdemo latest
$ ./arvbox adduser demouser demo at example.com
</pre>
commit 6771c21313fa4ba22cdd3007c4c5a03b8de8227c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 13:57:37 2021 -0400
Add ARVADOS_BRANCH and WORKBENCH2_BRANCH to arvbox
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 3c77ade8d..8b3994edd 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -14,8 +14,8 @@ Arvbox is a Docker-based self-contained development, demonstration and testing e
h2. Quick start
<pre>
-$ git clone https://github.com/arvados/arvados.git
-$ cd arvados/tools/arvbox/bin
+$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
+$ chmod +x arvbox
$ ./arvbox start localdemo
$ ./arvbox adduser demouser demo at example.com
</pre>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 9bf6fe1b2..ba3f2cd07 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -52,6 +52,14 @@ if test -z "$WORKBENCH2_ROOT" ; then
WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
fi
+if test -z "$ARVADOS_BRANCH" ; then
+ ARVADOS_BRANCH=main
+fi
+
+if test -z "$WORKBENCH2_BRANCH" ; then
+ WORKBENCH2_BRANCH=main
+fi
+
PG_DATA="$ARVBOX_DATA/postgres"
VAR_DATA="$ARVBOX_DATA/var"
PASSENGER="$ARVBOX_DATA/passenger"
@@ -61,7 +69,7 @@ NPMCACHE="$ARVBOX_DATA/npm"
GOSTUFF="$ARVBOX_DATA/gopath"
RLIBS="$ARVBOX_DATA/Rlibs"
ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox"
-GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0"
+GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.7.0"
getip() {
docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
@@ -241,6 +249,8 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
@@ -249,6 +259,8 @@ run() {
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
@@ -390,9 +402,24 @@ build() {
BUILDTYPE=dev
fi
- docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$LOCAL_ARVADOS_ROOT"
+ if test "$ARVADOS_BRANCH" = "main" ; then
+ ARVADOS_BRANCH=$GITHEAD
+ fi
+
+ docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ --build-arg=workdir=/tools/arvbox/lib/arvbox/docker \
+ -t arvados/arvbox-base:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.base" \
+ "$LOCAL_ARVADOS_ROOT"
docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
- docker build $NO_CACHE -t arvados/arvbox-$BUILDTYPE:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" "$ARVBOX_DOCKER"
+ docker build $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ -t arvados/arvbox-$BUILDTYPE:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" \
+ "$ARVBOX_DOCKER"
docker tag $FORCE arvados/arvbox-$BUILDTYPE:$GITHEAD arvados/arvbox-$BUILDTYPE:latest
}
@@ -623,7 +650,7 @@ sv stop keepproxy
cd /usr/src/arvados/services/api
export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
export RAILS_ENV=development
-flock $GEM_HOME/gems.lock bundle exec rake db:drop
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:drop
rm $ARVADOS_CONTAINER_PATH/api_database_setup
rm $ARVADOS_CONTAINER_PATH/superuser_token
sv start api
commit c727a33fd753e97137d4ec092a73d592253b917c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 11:19:58 2021 -0400
Fix "wrong bundler" error starting API server in arvbox
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 96f3666cd..9bf6fe1b2 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -566,7 +566,7 @@ case "$subcmd" in
else
echo "Usage: $0 $subcmd <start|stop|restart> <service>"
echo "Available services:"
- exec docker execa $ARVBOX_CONTAINER ls /etc/service
+ exec docker exec $ARVBOX_CONTAINER ls /etc/service
fi
;;
diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index 4ad2aed0c..b1b6d37c0 100755
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -56,16 +56,16 @@ EOF
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- flock $GEM_HOME/gems.lock bundle exec rake db:setup
+ flock $GEM_HOME/gems.lock bin/bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEM_HOME/gems.lock bin/bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-flock $GEM_HOME/gems.lock bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:migrate
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index eb53e1904..d8f368049 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -61,27 +61,17 @@ fi
run_bundler() {
if test -f Gemfile.lock ; then
- # The 'gem install bundler line below' is cf.
- # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html,
- # until we get bundler 2.7.10/3.0.0 or higher
- flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
frozen=--frozen
else
frozen=""
fi
- # if ! test -x $GEM_HOME/bin/bundler ; then
- # bundleversion=2.0.2
- # bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true)
- # if test -n "$bundlergem" ; then
- # flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem
- # else
- # flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
- # fi
- # fi
- # Make sure to put the gem binaries in the right place
- flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin
- if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
- flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
+ BUNDLER=bundler
+ if test -x $PWD/bin/bundler ; then
+ # If present, use the one associated with rails workbench or API
+ BUNDLER=$PWD/bin/bundler
+ fi
+ if ! flock $GEM_HOME/gems.lock $BUNDLER install --verbose --local --no-deployment $frozen "$@" ; then
+ flock $GEM_HOME/gems.lock $BUNDLER install --verbose --no-deployment $frozen "$@"
fi
}
diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service
index d2691e7ed..c949bffa7 100755
--- a/tools/arvbox/lib/arvbox/docker/service/api/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service
@@ -17,8 +17,8 @@ else
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
@@ -33,4 +33,4 @@ fi
touch $ARVADOS_CONTAINER_PATH/api.ready
-exec bundle exec passenger start --port=${services[api]}
+exec bin/bundle exec passenger start --port=${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run b/tools/arvbox/lib/arvbox/docker/service/workbench/run
index b8a28fa76..9b588fc4f 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run
@@ -23,7 +23,7 @@ fi
if test "$1" != "--only-deps" ; then
openssl verify -CAfile $root_cert $server_cert
- exec bundle exec passenger start --port=${services[workbench]} \
+ exec bin/bundle exec passenger start --port=${services[workbench]} \
--ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \
--ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \
--user arvbox
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 32efea51b..e6f0ad4a4 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -23,8 +23,8 @@ else
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
if test "$1" = "--only-deps" ; then
@@ -34,7 +34,7 @@ cat >config/application.yml <<EOF
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
rm config/application.yml
exit
fi
@@ -43,5 +43,5 @@ set -u
secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
-RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
-flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bin/bundle exec rake assets:precompile
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list