[ARVADOS] updated: 2.1.0-1063-g696f75b0c

Git user git at public.arvados.org
Fri Jul 16 15:00:24 UTC 2021


Summary of changes:
 lib/crunchrun/crunchrun.go      | 35 ++++++++++++++++++++++++++++++-----
 lib/crunchrun/crunchrun_test.go |  4 ----
 lib/crunchrun/docker.go         | 15 +++------------
 lib/crunchrun/executor.go       |  6 +++---
 4 files changed, 36 insertions(+), 24 deletions(-)

  discards  d08083912c64b429e4ec06b9a42edd001c1e52a6 (commit)
  discards  ec5fd6d8072cc038648b254a5500868dfdf6a17b (commit)
  discards  f2ee5bac37391ce9fe084306da332becd7620ca7 (commit)
  discards  753d479b0b5960674bf8e5a27ee98f68b3cd06ce (commit)
  discards  3bd5ef5231bcc2d378c9cf756168390dbd06bb32 (commit)
  discards  237f9a7c77495bc8c71404f495f5cc6879110e9e (commit)
  discards  afd47207817a4d2a0ac76acd8175ed9a49fc1b6d (commit)
  discards  a473534ef25367c250cc5a0bc7c57cd1e11b8c57 (commit)
  discards  a2fb4946954504e7e0eaa0ebc6a0af14029ec797 (commit)
  discards  e3b8b3683f0ac0ce91f9e2684dc593e5f4bea7ab (commit)
       via  696f75b0c857a01b31205411cdef7a20fe7b93fe (commit)
       via  13b4d219384a81141846588b20f07792d64cb489 (commit)
       via  ffa821a7f872d211f534914bbd21f0fbd9064668 (commit)
       via  d2f732034404f96ebd50999313cbe9a6727d3f95 (commit)
       via  879bde382ebf26aa593869cfff22cc7e85be6bb0 (commit)
       via  570af13f6dfd7f725abf49335b7abdb46aae5048 (commit)
       via  8b5c403446d137d7fc55df278ccb02ef84e87e0a (commit)
       via  bf01d196dd3ec9ed51a56b39aff9a52cd726de48 (commit)
       via  6771c21313fa4ba22cdd3007c4c5a03b8de8227c (commit)
       via  c727a33fd753e97137d4ec092a73d592253b917c (commit)
       via  4879256386a5be9566f31f2c266b682993029e14 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (d08083912c64b429e4ec06b9a42edd001c1e52a6)
            \
             N -- N -- N (696f75b0c857a01b31205411cdef7a20fe7b93fe)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 696f75b0c857a01b31205411cdef7a20fe7b93fe
Author: Tom Clegg <tom at curii.com>
Date:   Fri Jul 16 10:44:57 2021 -0400

    17816: Add test.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index 4b6a4b1b2..5934c57b6 100644
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -141,6 +141,13 @@ func (s *executorSuite) TestExecEnableNetwork(c *C) {
 	}
 }
 
+func (s *executorSuite) TestExecWorkingDir(c *C) {
+	s.spec.WorkingDir = "/tmp"
+	s.spec.Command = []string{"sh", "-c", "pwd"}
+	s.checkRun(c, 0)
+	c.Check(s.stdout.String(), Equals, "/tmp\n")
+}
+
 func (s *executorSuite) TestExecStdoutStderr(c *C) {
 	s.spec.Command = []string{"sh", "-c", "echo foo; echo -n bar >&2; echo baz; echo waz >&2"}
 	s.checkRun(c, 0)

commit 13b4d219384a81141846588b20f07792d64cb489
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Thu Jul 15 13:50:22 2021 -0400

    17816: Set the current working directory in the singularity container
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 4bec8c3eb..2a294cf03 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -74,7 +74,7 @@ func (e *singularityExecutor) Create(spec containerSpec) error {
 }
 
 func (e *singularityExecutor) Start() error {
-	args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+	args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
 	if !e.spec.EnableNetwork {
 		args = append(args, "--net", "--network=none")
 	}

commit ffa821a7f872d211f534914bbd21f0fbd9064668
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Thu Jul 15 15:58:45 2021 -0400

    17816: crunch-dispatch-local uses cluster config.
    
    Added upgrade note.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 13f093394..3c283c354 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -35,10 +35,14 @@ TODO: extract this information based on git commit messages and generate changel
 <div class="releasenotes">
 </notextile>
 
-h2(#main). development main (as of 2021-06-03)
+h2(#main). development main (as of 2021-07-15)
 
 "Upgrading from 2.2.0":#v2_2_0
 
+h3. crunch-dispatch-local now requires config.yml
+
+The @crunch-dispatch-local@ dispatcher now reads the API host and token from the system wide @/etc/arvados/config.yml@ .  It will fail to start that file is not found or not readable.
+
 h2(#v2_2_0). v2.2.0 (2021-06-03)
 
 "Upgrading from 2.1.0":#v2_1_0
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 4a45f1054..c202e683f 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -86,6 +86,26 @@ func doMain() error {
 
 	runningCmds = make(map[string]*exec.Cmd)
 
+	var client arvados.Client
+	client.APIHost = cluster.Services.Controller.ExternalURL.Host
+	client.AuthToken = cluster.SystemRootToken
+	client.Insecure = cluster.TLS.Insecure
+
+	if client.APIHost != "" || client.AuthToken != "" {
+		// Copy real configs into env vars so [a]
+		// MakeArvadosClient() uses them, and [b] they get
+		// propagated to crunch-run via SLURM.
+		os.Setenv("ARVADOS_API_HOST", client.APIHost)
+		os.Setenv("ARVADOS_API_TOKEN", client.AuthToken)
+		os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+		if client.Insecure {
+			os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+		}
+		os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+	} else {
+		logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
+	}
+
 	arv, err := arvadosclient.MakeArvadosClient()
 	if err != nil {
 		logger.Errorf("error making Arvados client: %v", err)

commit d2f732034404f96ebd50999313cbe9a6727d3f95
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 16:47:45 2021 -0400

    17816: Fix tests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 5f51134df..d976bf081 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -81,8 +81,10 @@ func (s *TestSuite) TestIntegration(c *C) {
 		return cmd.Start()
 	}
 
+	cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
 	dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
-		(&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+		(&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
 		cancel()
 	}
 
@@ -184,8 +186,10 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon
 		return cmd.Start()
 	}
 
+	cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
 	dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
-		(&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+		(&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
 		cancel()
 	}
 

commit 879bde382ebf26aa593869cfff22cc7e85be6bb0
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 16:39:02 2021 -0400

    17816: Make singularity-in-arvbox work
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
deleted file mode 120000
index a388c8b67..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
+++ /dev/null
@@ -1 +0,0 @@
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
new file mode 100755
index 000000000..821afdce5
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
+install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
+ln -sf arvados-server /usr/local/bin/crunch-run
+
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
+cat > /usr/local/bin/crunch-run.sh <<EOF
+#!/bin/sh
+exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
+EOF
+chmod +x /usr/local/bin/crunch-run.sh
+
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
+
+exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
deleted file mode 100755
index 821afdce5..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
-install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
-ln -sf arvados-server /usr/local/bin/crunch-run
-
-if test "$1" = "--only-deps" ; then
-    exit
-fi
-
-cat > /usr/local/bin/crunch-run.sh <<EOF
-#!/bin/sh
-exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
-EOF
-chmod +x /usr/local/bin/crunch-run.sh
-
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index b29dafed7..f49e9ea26 100755
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -41,7 +41,7 @@ for sdk_app in arv arv-get cwl-runner arv-mount ; do
     fi
 done
 
-if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
+if ! (ps ax | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
     waiting="$waiting crunch-dispatch"
 fi
 

commit 570af13f6dfd7f725abf49335b7abdb46aae5048
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 16:37:19 2021 -0400

    17816: Add --runtime-engine to crunch-dispatch-local and crunch-dispatch-slurm
    
    refs #17816
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 2922817b5..4a45f1054 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -17,6 +17,7 @@ import (
 	"syscall"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/dispatch"
@@ -74,6 +75,13 @@ func doMain() error {
 		return nil
 	}
 
+	loader := config.NewLoader(nil, logger)
+	cfg, err := loader.Load()
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return fmt.Errorf("config error: %s", err)
+	}
+
 	logger.Printf("crunch-dispatch-local %s started", version)
 
 	runningCmds = make(map[string]*exec.Cmd)
@@ -90,7 +98,7 @@ func doMain() error {
 	dispatcher := dispatch.Dispatcher{
 		Logger:       logger,
 		Arv:          arv,
-		RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx}).run,
+		RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx, cluster}).run,
 		PollPeriod:   time.Duration(*pollInterval) * time.Second,
 	}
 
@@ -128,6 +136,7 @@ type LocalRun struct {
 	startCmd         func(container arvados.Container, cmd *exec.Cmd) error
 	concurrencyLimit chan bool
 	ctx              context.Context
+	cluster          *arvados.Cluster
 }
 
 // Run a container.
@@ -169,7 +178,7 @@ func (lr *LocalRun) run(dispatcher *dispatch.Dispatcher,
 		waitGroup.Add(1)
 		defer waitGroup.Done()
 
-		cmd := exec.Command(*crunchRunCommand, uuid)
+		cmd := exec.Command(*crunchRunCommand, "--runtime-engine="+lr.cluster.Containers.RuntimeEngine, uuid)
 		cmd.Stdin = nil
 		cmd.Stderr = os.Stderr
 		cmd.Stdout = os.Stderr
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index a5899ce8a..2f2f013c7 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -255,6 +255,7 @@ func (disp *Dispatcher) submit(container arvados.Container, crunchRunCommand []s
 	// append() here avoids modifying crunchRunCommand's
 	// underlying array, which is shared with other goroutines.
 	crArgs := append([]string(nil), crunchRunCommand...)
+	crArgs = append(crArgs, "--runtime-engine="+disp.cluster.Containers.RuntimeEngine)
 	crArgs = append(crArgs, container.UUID)
 	crScript := strings.NewReader(execScript(crArgs))
 

commit 8b5c403446d137d7fc55df278ccb02ef84e87e0a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 14:50:07 2021 -0400

    Don't pull arvbox checkouts after clone
    
    We already have latest and tags/detached HEADS can't be pulled anyway
    (and they fail, which is the real problem).
    
    no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index ba3f2cd07..fd464974f 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -250,17 +250,14 @@ run() {
         if ! test -d "$ARVADOS_ROOT" ; then
             git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
 	    git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
-	    git -C "$ARVADOS_ROOT" pull
         fi
         if ! test -d "$COMPOSER_ROOT" ; then
             git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
             git -C "$COMPOSER_ROOT" checkout arvados-fork
-            git -C "$COMPOSER_ROOT" pull
         fi
         if ! test -d "$WORKBENCH2_ROOT" ; then
             git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
 	    git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
-	    git -C "$ARVADOS_ROOT" pull
         fi
 
         if [[ "$CONFIG" = test ]] ; then
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index c285d53ca..92099614a 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -10,13 +10,10 @@ ARG workbench2_version=main
 RUN cd /usr/src && \
     git clone --no-checkout https://git.arvados.org/arvados.git && \
     git -C arvados checkout ${arvados_version} && \
-    git -C arvados pull && \
     git clone --no-checkout https://github.com/arvados/composer.git && \
     git -C composer checkout ${composer_version} && \
-    git -C composer pull && \
     git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
     git -C workbench2 checkout ${workbench2_version} && \
-    git -C workbench2 pull && \
     chown -R 1000:1000 /usr/src
 
 # avoid rebuilding arvados-server, it's already been built as part of the base image

commit bf01d196dd3ec9ed51a56b39aff9a52cd726de48
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 14:12:08 2021 -0400

    arvbox docs include a docker image tag
    
    no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 8b3994edd..a8235ee70 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Quick start
 <pre>
 $ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
 $ chmod +x arvbox
-$ ./arvbox start localdemo
+$ ./arvbox start localdemo latest
 $ ./arvbox adduser demouser demo at example.com
 </pre>
 

commit 6771c21313fa4ba22cdd3007c4c5a03b8de8227c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 13:57:37 2021 -0400

    Add ARVADOS_BRANCH and WORKBENCH2_BRANCH to arvbox
    
    no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 3c77ade8d..8b3994edd 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -14,8 +14,8 @@ Arvbox is a Docker-based self-contained development, demonstration and testing e
 h2. Quick start
 
 <pre>
-$ git clone https://github.com/arvados/arvados.git
-$ cd arvados/tools/arvbox/bin
+$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
+$ chmod +x arvbox
 $ ./arvbox start localdemo
 $ ./arvbox adduser demouser demo at example.com
 </pre>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 9bf6fe1b2..ba3f2cd07 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -52,6 +52,14 @@ if test -z "$WORKBENCH2_ROOT" ; then
     WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
 fi
 
+if test -z "$ARVADOS_BRANCH" ; then
+    ARVADOS_BRANCH=main
+fi
+
+if test -z "$WORKBENCH2_BRANCH" ; then
+    WORKBENCH2_BRANCH=main
+fi
+
 PG_DATA="$ARVBOX_DATA/postgres"
 VAR_DATA="$ARVBOX_DATA/var"
 PASSENGER="$ARVBOX_DATA/passenger"
@@ -61,7 +69,7 @@ NPMCACHE="$ARVBOX_DATA/npm"
 GOSTUFF="$ARVBOX_DATA/gopath"
 RLIBS="$ARVBOX_DATA/Rlibs"
 ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox"
-GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0"
+GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.7.0"
 
 getip() {
     docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
@@ -241,6 +249,8 @@ run() {
 
         if ! test -d "$ARVADOS_ROOT" ; then
             git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
+	    git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
+	    git -C "$ARVADOS_ROOT" pull
         fi
         if ! test -d "$COMPOSER_ROOT" ; then
             git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
@@ -249,6 +259,8 @@ run() {
         fi
         if ! test -d "$WORKBENCH2_ROOT" ; then
             git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
+	    git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
+	    git -C "$ARVADOS_ROOT" pull
         fi
 
         if [[ "$CONFIG" = test ]] ; then
@@ -390,9 +402,24 @@ build() {
         BUILDTYPE=dev
     fi
 
-    docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$LOCAL_ARVADOS_ROOT"
+    if test "$ARVADOS_BRANCH" = "main" ; then
+	ARVADOS_BRANCH=$GITHEAD
+    fi
+
+    docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE \
+	   --build-arg=arvados_version=$ARVADOS_BRANCH \
+	   --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+	   --build-arg=workdir=/tools/arvbox/lib/arvbox/docker \
+	   -t arvados/arvbox-base:$GITHEAD \
+	   -f "$ARVBOX_DOCKER/Dockerfile.base" \
+	   "$LOCAL_ARVADOS_ROOT"
     docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
-    docker build $NO_CACHE -t arvados/arvbox-$BUILDTYPE:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" "$ARVBOX_DOCKER"
+    docker build $NO_CACHE \
+	   --build-arg=arvados_version=$ARVADOS_BRANCH \
+	   --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+	   -t arvados/arvbox-$BUILDTYPE:$GITHEAD \
+	   -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" \
+	   "$ARVBOX_DOCKER"
     docker tag $FORCE arvados/arvbox-$BUILDTYPE:$GITHEAD arvados/arvbox-$BUILDTYPE:latest
 }
 
@@ -623,7 +650,7 @@ sv stop keepproxy
 cd /usr/src/arvados/services/api
 export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
 export RAILS_ENV=development
-flock $GEM_HOME/gems.lock bundle exec rake db:drop
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:drop
 rm $ARVADOS_CONTAINER_PATH/api_database_setup
 rm $ARVADOS_CONTAINER_PATH/superuser_token
 sv start api

commit c727a33fd753e97137d4ec092a73d592253b917c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed Jul 14 11:19:58 2021 -0400

    Fix "wrong bundler" error starting API server in arvbox
    
    no issue #
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 96f3666cd..9bf6fe1b2 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -566,7 +566,7 @@ case "$subcmd" in
         else
             echo "Usage: $0 $subcmd <start|stop|restart> <service>"
             echo "Available services:"
-            exec docker execa $ARVBOX_CONTAINER ls /etc/service
+            exec docker exec $ARVBOX_CONTAINER ls /etc/service
         fi
         ;;
 
diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index 4ad2aed0c..b1b6d37c0 100755
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -56,16 +56,16 @@ EOF
 fi
 
 if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
-   flock $GEM_HOME/gems.lock bundle exec rake db:setup
+   flock $GEM_HOME/gems.lock bin/bundle exec rake db:setup
    touch $ARVADOS_CONTAINER_PATH/api_database_setup
 fi
 
 if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
-    superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
+    superuser_tok=$(flock $GEM_HOME/gems.lock bin/bundle exec ./script/create_superuser_token.rb)
     echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
 fi
 
 rm -rf tmp
 mkdir -p tmp/cache
 
-flock $GEM_HOME/gems.lock bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:migrate
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index eb53e1904..d8f368049 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -61,27 +61,17 @@ fi
 
 run_bundler() {
     if test -f Gemfile.lock ; then
-        # The 'gem install bundler line below' is cf.
-        # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html,
-        # until we get bundler 2.7.10/3.0.0 or higher
-        flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
         frozen=--frozen
     else
         frozen=""
     fi
-    # if ! test -x $GEM_HOME/bin/bundler ; then
-    # 	bundleversion=2.0.2
-    #     bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true)
-    #     if test -n "$bundlergem" ; then
-    #         flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem
-    #     else
-    #         flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
-    #     fi
-    # fi
-    # Make sure to put the gem binaries in the right place
-    flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin
-    if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
-        flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
+    BUNDLER=bundler
+    if test -x $PWD/bin/bundler ; then
+	# If present, use the one associated with rails workbench or API
+	BUNDLER=$PWD/bin/bundler
+    fi
+    if ! flock $GEM_HOME/gems.lock $BUNDLER install --verbose --local --no-deployment $frozen "$@" ; then
+        flock $GEM_HOME/gems.lock $BUNDLER install --verbose --no-deployment $frozen "$@"
     fi
 }
 
diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service
index d2691e7ed..c949bffa7 100755
--- a/tools/arvbox/lib/arvbox/docker/service/api/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service
@@ -17,8 +17,8 @@ else
 fi
 
 run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
 
 if test "$1" = "--only-deps" ; then
     exit
@@ -33,4 +33,4 @@ fi
 
 touch $ARVADOS_CONTAINER_PATH/api.ready
 
-exec bundle exec passenger start --port=${services[api]}
+exec bin/bundle exec passenger start --port=${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run b/tools/arvbox/lib/arvbox/docker/service/workbench/run
index b8a28fa76..9b588fc4f 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run
@@ -23,7 +23,7 @@ fi
 
 if test "$1" != "--only-deps" ; then
     openssl verify -CAfile $root_cert $server_cert
-    exec bundle exec passenger start --port=${services[workbench]} \
+    exec bin/bundle exec passenger start --port=${services[workbench]} \
 	 --ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \
 	 --ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \
          --user arvbox
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 32efea51b..e6f0ad4a4 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -23,8 +23,8 @@ else
 fi
 
 run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
 mkdir -p /usr/src/arvados/apps/workbench/tmp
 
 if test "$1" = "--only-deps" ; then
@@ -34,7 +34,7 @@ cat >config/application.yml <<EOF
 $RAILS_ENV:
   keep_web_url: https://example.com/c=%{uuid_or_pdh}
 EOF
-   RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+   RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
    rm config/application.yml
    exit
 fi
@@ -43,5 +43,5 @@ set -u
 
 secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
 
-RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
-flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bin/bundle exec rake assets:precompile

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list