[ARVADOS] updated: 2.1.0-1077-gb13e32f7d
Git user
git at public.arvados.org
Mon Jul 19 17:11:13 UTC 2021
Summary of changes:
apps/workbench/Gemfile.lock | 2 +-
doc/_config.yml | 1 +
doc/admin/upgrading.html.textile.liquid | 6 +-
doc/install/arvbox.html.textile.liquid | 6 +-
lib/boot/supervisor.go | 5 ++
lib/config/export.go | 2 +-
lib/crunchrun/crunchrun.go | 37 +++++++--
lib/crunchrun/crunchrun_test.go | 16 ++--
lib/crunchrun/docker.go | 15 +---
lib/crunchrun/executor.go | 6 +-
lib/crunchrun/executor_test.go | 7 ++
lib/crunchrun/singularity.go | 11 ++-
lib/install/deps.go | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/setup.py | 5 +-
.../tests/17879-ignore-sbg-fields-job.yml} | 3 +-
sdk/cwl/tests/17879-ignore-sbg-fields.cwl | 37 +++++++++
sdk/cwl/tests/arvados-tests.yml | 6 ++
sdk/cwl/tests/test_submit.py | 32 ++++----
sdk/go/arvados/api.go | 17 ++++
sdk/go/arvados/fs_backend.go | 7 +-
sdk/go/arvados/fs_collection.go | 28 +++++--
sdk/go/arvados/fs_collection_test.go | 53 +++++++++----
sdk/go/arvados/fs_site_test.go | 36 ++++++---
sdk/go/keepclient/keepclient.go | 39 ++++------
sdk/go/keepclient/keepclient_test.go | 43 +++++++----
sdk/go/keepclient/support.go | 90 ++++++++++++++++------
services/api/Gemfile.lock | 2 +-
.../crunch-dispatch-local/crunch-dispatch-local.go | 33 +++++++-
.../crunch-dispatch-local_test.go | 8 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 1 +
tools/arvbox/bin/arvbox | 36 +++++++--
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 5 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 6 +-
tools/arvbox/lib/arvbox/docker/common.sh | 24 ++----
.../lib/arvbox/docker/service/api/run-service | 6 +-
.../docker/service/crunch-dispatch-local/run | 31 +++++++-
.../service/crunch-dispatch-local/run-service | 30 --------
.../lib/arvbox/docker/service/ready/run-service | 2 +-
.../arvbox/lib/arvbox/docker/service/workbench/run | 2 +-
.../arvbox/docker/service/workbench/run-service | 10 +--
.../arvbox/docker/service/workbench2/run-service | 6 +-
42 files changed, 482 insertions(+), 234 deletions(-)
copy sdk/{cli/test/binstub_docker_noop/docker.io => cwl/tests/17879-ignore-sbg-fields-job.yml} (84%)
mode change 100755 => 100644
create mode 100644 sdk/cwl/tests/17879-ignore-sbg-fields.cwl
mode change 120000 => 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
delete mode 100755 tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
via b13e32f7df598fcf027a9f126f180b9d66be8c17 (commit)
via 9c5741a57e928fbb5797b22788b6e80e59bb5bca (commit)
via 905ae3113859bb82aa60b3d6caf588b5a3a98166 (commit)
via d5aff80365f6bad43bbbc3910655c98b9dc2d7d1 (commit)
via 62bba5da70842e05d3c37e1bafa7a46ebbc5b70b (commit)
via 6457564cecc16e62997ce766eb55e98f402475c7 (commit)
via 7bcfcbd4d6e488deefbe6718f608201bf9867843 (commit)
via 9907362f5f37381b9f72e33f91de0375fcb138eb (commit)
via b5d0cf45ba5977ac7b9962612415469467cde1f6 (commit)
via b398ebefa465580c4f91a71b2054a659552ef736 (commit)
via 4906a939c72d72dd7f944b1b0a6b661820e12f93 (commit)
via 9d7c09d3607dc7d08f2168841a6e6a6f003888b6 (commit)
via 9e9e53befe3db1dd557fe721822604aad8261f52 (commit)
via aaf8966948653e6a4a7945dcb0e9c2c2db530e75 (commit)
via 919c2100e2a19508365efa63ea30e6e89b9c2697 (commit)
via 70ca572900248d24d7c1c0852f7bcbfa45f66e58 (commit)
via 9bd7e42605084c0e2dcb492830d0e16951d7b4d0 (commit)
via 7bc7e20348a3edf92081e54dae4204750e11857b (commit)
via 9fc8c0ec7002d7ed16ac729b62be880bbc83594e (commit)
via e2cda71fa792d17024bc46cac0e93fe230db3e1b (commit)
via 42ee44aeabf810ad627f617e1f6391d767269d3f (commit)
via 29aa7d6c9207b62e06b3d94bcd6051c8946bf44a (commit)
via ebba63532a010e8928f5c872fc06b864832ea631 (commit)
via e10f8abe94d72d969d2d68a007f48b305a749b69 (commit)
via 986dc65bf81cbfd986cb8b87a0f87a1fe890d459 (commit)
via 5cffc002d44396c86a84d4455ca276bf28d51416 (commit)
via 84a8f9c7e7280518baec023d9b71ae1d48faf8de (commit)
via f54f9174873c2bf04243cbd7c93bb00c66390586 (commit)
via 60ed39d991cfaa5ea454bde581c9f7b4e7f4d6fd (commit)
via fe51b07f84306ba3df7945dc0fbea9aedacb2adf (commit)
from 6a62a1628b5775facb14cfcdb0cb8b6d830367c1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b13e32f7df598fcf027a9f126f180b9d66be8c17
Author: Ward Vandewege <ward at curii.com>
Date: Mon Jul 19 10:25:56 2021 -0400
Fix arvbox demo image build.
refs #17816
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 92099614a..cb0dc2d65 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -47,7 +47,7 @@ RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/crunch-dispatch-local/run-service --only-deps
+RUN /var/lib/arvbox/service/crunch-dispatch-local/run --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
commit 9c5741a57e928fbb5797b22788b6e80e59bb5bca
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Jul 14 22:22:37 2021 -0300
17573: Adds test storage classes to arvados-server test mode.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 0f497a443..4e009f45a 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -741,6 +741,11 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
AccessViaHosts: map[arvados.URL]arvados.VolumeAccess{
url: {},
},
+ StorageClasses: map[string]bool{
+ "default": true,
+ "foo": true,
+ "bar": true,
+ },
}
}
}
commit 905ae3113859bb82aa60b3d6caf588b5a3a98166
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Jul 8 16:31:27 2021 -0300
17573: Exposes storage classes for every configured volume.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/export.go b/lib/config/export.go
index 8753b52f2..bb939321c 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -232,7 +232,7 @@ var whitelist = map[string]bool{
"Volumes.*.ReadOnly": true,
"Volumes.*.Replication": true,
"Volumes.*.StorageClasses": true,
- "Volumes.*.StorageClasses.*": false,
+ "Volumes.*.StorageClasses.*": true,
"Workbench": true,
"Workbench.ActivationContactLink": false,
"Workbench.APIClientConnectTimeout": true,
commit d5aff80365f6bad43bbbc3910655c98b9dc2d7d1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jul 16 11:00:57 2021 -0400
17816: Sort bind mounts on singularity command line
Need to mount parent directories before we can mount things inside
them.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 2a294cf03..bcaff3bcc 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -8,6 +8,7 @@ import (
"io/ioutil"
"os"
"os/exec"
+ "sort"
"syscall"
"golang.org/x/net/context"
@@ -82,7 +83,13 @@ func (e *singularityExecutor) Start() error {
false: "rw",
true: "ro",
}
- for path, mount := range e.spec.BindMounts {
+ var binds []string
+ for path, _ := range e.spec.BindMounts {
+ binds = append(binds, path)
+ }
+ sort.Strings(binds)
+ for _, path := range binds {
+ mount := e.spec.BindMounts[path]
args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
}
args = append(args, e.imageFilename)
commit 62bba5da70842e05d3c37e1bafa7a46ebbc5b70b
Author: Tom Clegg <tom at curii.com>
Date: Fri Jul 16 10:44:57 2021 -0400
17816: Add test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index 4b6a4b1b2..5934c57b6 100644
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -141,6 +141,13 @@ func (s *executorSuite) TestExecEnableNetwork(c *C) {
}
}
+func (s *executorSuite) TestExecWorkingDir(c *C) {
+ s.spec.WorkingDir = "/tmp"
+ s.spec.Command = []string{"sh", "-c", "pwd"}
+ s.checkRun(c, 0)
+ c.Check(s.stdout.String(), Equals, "/tmp\n")
+}
+
func (s *executorSuite) TestExecStdoutStderr(c *C) {
s.spec.Command = []string{"sh", "-c", "echo foo; echo -n bar >&2; echo baz; echo waz >&2"}
s.checkRun(c, 0)
commit 6457564cecc16e62997ce766eb55e98f402475c7
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 15 13:50:22 2021 -0400
17816: Set the current working directory in the singularity container
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 4bec8c3eb..2a294cf03 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -74,7 +74,7 @@ func (e *singularityExecutor) Create(spec containerSpec) error {
}
func (e *singularityExecutor) Start() error {
- args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv"}
+ args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
if !e.spec.EnableNetwork {
args = append(args, "--net", "--network=none")
}
commit 7bcfcbd4d6e488deefbe6718f608201bf9867843
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 15 15:58:45 2021 -0400
17816: crunch-dispatch-local uses cluster config.
Added upgrade note.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 13f093394..3c283c354 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -35,10 +35,14 @@ TODO: extract this information based on git commit messages and generate changel
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2021-06-03)
+h2(#main). development main (as of 2021-07-15)
"Upgrading from 2.2.0":#v2_2_0
+h3. crunch-dispatch-local now requires config.yml
+
+The @crunch-dispatch-local@ dispatcher now reads the API host and token from the system wide @/etc/arvados/config.yml@ . It will fail to start that file is not found or not readable.
+
h2(#v2_2_0). v2.2.0 (2021-06-03)
"Upgrading from 2.1.0":#v2_1_0
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 4a45f1054..c202e683f 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -86,6 +86,26 @@ func doMain() error {
runningCmds = make(map[string]*exec.Cmd)
+ var client arvados.Client
+ client.APIHost = cluster.Services.Controller.ExternalURL.Host
+ client.AuthToken = cluster.SystemRootToken
+ client.Insecure = cluster.TLS.Insecure
+
+ if client.APIHost != "" || client.AuthToken != "" {
+ // Copy real configs into env vars so [a]
+ // MakeArvadosClient() uses them, and [b] they get
+ // propagated to crunch-run via SLURM.
+ os.Setenv("ARVADOS_API_HOST", client.APIHost)
+ os.Setenv("ARVADOS_API_TOKEN", client.AuthToken)
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+ if client.Insecure {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+ }
+ os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
+ } else {
+ logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
+ }
+
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
logger.Errorf("error making Arvados client: %v", err)
commit 9907362f5f37381b9f72e33f91de0375fcb138eb
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:47:45 2021 -0400
17816: Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 5f51134df..d976bf081 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -81,8 +81,10 @@ func (s *TestSuite) TestIntegration(c *C) {
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
cancel()
}
@@ -184,8 +186,10 @@ func testWithServerStub(c *C, apiStubResponses map[string]arvadostest.StubRespon
return cmd.Start()
}
+ cl := arvados.Cluster{Containers: arvados.ContainersConfig{RuntimeEngine: "docker"}}
+
dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ (&LocalRun{startCmd, make(chan bool, 8), ctx, &cl}).run(d, c, s)
cancel()
}
commit b5d0cf45ba5977ac7b9962612415469467cde1f6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:39:02 2021 -0400
17816: Make singularity-in-arvbox work
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
deleted file mode 120000
index a388c8b67..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
+++ /dev/null
@@ -1 +0,0 @@
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
new file mode 100755
index 000000000..821afdce5
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
+install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
+ln -sf arvados-server /usr/local/bin/crunch-run
+
+if test "$1" = "--only-deps" ; then
+ exit
+fi
+
+cat > /usr/local/bin/crunch-run.sh <<EOF
+#!/bin/sh
+exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
+EOF
+chmod +x /usr/local/bin/crunch-run.sh
+
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
+export ARVADOS_API_HOST_INSECURE=1
+export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
+
+exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
deleted file mode 100755
index 821afdce5..000000000
--- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/crunch-dispatch-local"
-install $GOPATH/bin/crunch-dispatch-local /usr/local/bin
-ln -sf arvados-server /usr/local/bin/crunch-run
-
-if test "$1" = "--only-deps" ; then
- exit
-fi
-
-cat > /usr/local/bin/crunch-run.sh <<EOF
-#!/bin/sh
-exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
-EOF
-chmod +x /usr/local/bin/crunch-run.sh
-
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1
diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index b29dafed7..f49e9ea26 100755
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -41,7 +41,7 @@ for sdk_app in arv arv-get cwl-runner arv-mount ; do
fi
done
-if ! (ps x | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
+if ! (ps ax | grep -v grep | grep "crunch-dispatch") > /dev/null ; then
waiting="$waiting crunch-dispatch"
fi
commit b398ebefa465580c4f91a71b2054a659552ef736
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 16:37:19 2021 -0400
17816: Add --runtime-engine to crunch-dispatch-local and crunch-dispatch-slurm
refs #17816
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 2922817b5..4a45f1054 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -17,6 +17,7 @@ import (
"syscall"
"time"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/dispatch"
@@ -74,6 +75,13 @@ func doMain() error {
return nil
}
+ loader := config.NewLoader(nil, logger)
+ cfg, err := loader.Load()
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return fmt.Errorf("config error: %s", err)
+ }
+
logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
@@ -90,7 +98,7 @@ func doMain() error {
dispatcher := dispatch.Dispatcher{
Logger: logger,
Arv: arv,
- RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx}).run,
+ RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx, cluster}).run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
@@ -128,6 +136,7 @@ type LocalRun struct {
startCmd func(container arvados.Container, cmd *exec.Cmd) error
concurrencyLimit chan bool
ctx context.Context
+ cluster *arvados.Cluster
}
// Run a container.
@@ -169,7 +178,7 @@ func (lr *LocalRun) run(dispatcher *dispatch.Dispatcher,
waitGroup.Add(1)
defer waitGroup.Done()
- cmd := exec.Command(*crunchRunCommand, uuid)
+ cmd := exec.Command(*crunchRunCommand, "--runtime-engine="+lr.cluster.Containers.RuntimeEngine, uuid)
cmd.Stdin = nil
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index a5899ce8a..2f2f013c7 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -255,6 +255,7 @@ func (disp *Dispatcher) submit(container arvados.Container, crunchRunCommand []s
// append() here avoids modifying crunchRunCommand's
// underlying array, which is shared with other goroutines.
crArgs := append([]string(nil), crunchRunCommand...)
+ crArgs = append(crArgs, "--runtime-engine="+disp.cluster.Containers.RuntimeEngine)
crArgs = append(crArgs, container.UUID)
crScript := strings.NewReader(execScript(crArgs))
commit 4906a939c72d72dd7f944b1b0a6b661820e12f93
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:50:07 2021 -0400
Don't pull arvbox checkouts after clone
We already have latest and tags/detached HEADS can't be pulled anyway
(and they fail, which is the real problem).
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 5bc74e6aa..fd464974f 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -250,7 +250,6 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
@@ -259,7 +258,6 @@ run() {
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
commit 9d7c09d3607dc7d08f2168841a6e6a6f003888b6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:12:08 2021 -0400
arvbox docs include a docker image tag
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 8b3994edd..a8235ee70 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Quick start
<pre>
$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
$ chmod +x arvbox
-$ ./arvbox start localdemo
+$ ./arvbox start localdemo latest
$ ./arvbox adduser demouser demo at example.com
</pre>
commit 9e9e53befe3db1dd557fe721822604aad8261f52
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 13:57:37 2021 -0400
Add ARVADOS_BRANCH and WORKBENCH2_BRANCH to arvbox
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index a8235ee70..8b3994edd 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Quick start
<pre>
$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
$ chmod +x arvbox
-$ ./arvbox start localdemo latest
+$ ./arvbox start localdemo
$ ./arvbox adduser demouser demo at example.com
</pre>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index fd464974f..5bc74e6aa 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -250,6 +250,7 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
@@ -258,6 +259,7 @@ run() {
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
commit aaf8966948653e6a4a7945dcb0e9c2c2db530e75
Author: Tom Clegg <tom at curii.com>
Date: Fri Jul 16 10:24:42 2021 -0400
17816: Close stdin/stdout/stderr from main instead of executor.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 23fbc430b..412f1bbfb 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -77,7 +77,10 @@ type PsProcess interface {
// ContainerRunner is the main stateful struct used for a single execution of a
// container.
type ContainerRunner struct {
- executor containerExecutor
+ executor containerExecutor
+ executorStdin io.Closer
+ executorStdout io.Closer
+ executorStderr io.Closer
// Dispatcher client is initialized with the Dispatcher token.
// This is a privileged token used to manage container status
@@ -106,8 +109,6 @@ type ContainerRunner struct {
ExitCode *int
NewLogWriter NewLogWriter
CrunchLog *ThrottledLogger
- Stdout io.WriteCloser
- Stderr io.WriteCloser
logUUID string
logMtx sync.Mutex
LogCollection arvados.CollectionFileSystem
@@ -877,7 +878,7 @@ func (runner *ContainerRunner) getStdoutFile(mntPath string) (*os.File, error) {
// CreateContainer creates the docker container.
func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[string]bindmount) error {
- var stdin io.ReadCloser
+ var stdin io.ReadCloser = ioutil.NopCloser(bytes.NewReader(nil))
if mnt, ok := runner.Container.Mounts["stdin"]; ok {
switch mnt.Kind {
case "collection":
@@ -954,6 +955,9 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st
if !runner.enableMemoryLimit {
ram = 0
}
+ runner.executorStdin = stdin
+ runner.executorStdout = stdout
+ runner.executorStderr = stderr
return runner.executor.Create(containerSpec{
Image: imageID,
VCPUs: runner.Container.RuntimeConstraints.VCPUs,
@@ -1018,6 +1022,27 @@ func (runner *ContainerRunner) WaitFinish() error {
}
runner.ExitCode = &exitcode
+ var returnErr error
+ if err = runner.executorStdin.Close(); err != nil {
+ err = fmt.Errorf("error closing container stdin: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ returnErr = err
+ }
+ if err = runner.executorStdout.Close(); err != nil {
+ err = fmt.Errorf("error closing container stdout: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ if returnErr == nil {
+ returnErr = err
+ }
+ }
+ if err = runner.executorStderr.Close(); err != nil {
+ err = fmt.Errorf("error closing container stderr: %s", err)
+ runner.CrunchLog.Printf("%s", err)
+ if returnErr == nil {
+ returnErr = err
+ }
+ }
+
if runner.statReporter != nil {
runner.statReporter.Stop()
err = runner.statLogger.Close()
@@ -1025,7 +1050,7 @@ func (runner *ContainerRunner) WaitFinish() error {
runner.CrunchLog.Printf("error closing crunchstat logs: %v", err)
}
}
- return nil
+ return returnErr
}
func (runner *ContainerRunner) updateLogs() {
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 42a2cf3ad..bb7ffdf03 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -120,8 +120,6 @@ func (e *stubExecutor) CgroupID() string { return "cgroupid" }
func (e *stubExecutor) Stop() error { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
func (e *stubExecutor) Close() { e.closed = true }
func (e *stubExecutor) Wait(context.Context) (int, error) {
- defer e.created.Stdout.Close()
- defer e.created.Stderr.Close()
return <-e.exit, e.waitErr
}
@@ -524,8 +522,6 @@ func dockerLog(fd byte, msg string) []byte {
func (s *TestSuite) TestRunContainer(c *C) {
s.executor.runFunc = func() {
fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
- s.executor.created.Stdout.Close()
- s.executor.created.Stderr.Close()
s.executor.exit <- 0
}
diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go
index a39b754b3..861f8c8c1 100644
--- a/lib/crunchrun/docker.go
+++ b/lib/crunchrun/docker.go
@@ -186,7 +186,7 @@ func (e *dockerExecutor) Wait(ctx context.Context) (int, error) {
}
}
-func (e *dockerExecutor) startIO(stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
+func (e *dockerExecutor) startIO(stdin io.Reader, stdout, stderr io.Writer) error {
resp, err := e.dockerclient.ContainerAttach(context.TODO(), e.containerID, dockertypes.ContainerAttachOptions{
Stream: true,
Stdin: stdin != nil,
@@ -213,8 +213,7 @@ func (e *dockerExecutor) startIO(stdin io.ReadCloser, stdout, stderr io.WriteClo
return nil
}
-func (e *dockerExecutor) handleStdin(stdin io.ReadCloser, conn io.Writer, closeConn func() error) error {
- defer stdin.Close()
+func (e *dockerExecutor) handleStdin(stdin io.Reader, conn io.Writer, closeConn func() error) error {
defer closeConn()
_, err := io.Copy(conn, stdin)
if err != nil {
@@ -225,7 +224,7 @@ func (e *dockerExecutor) handleStdin(stdin io.ReadCloser, conn io.Writer, closeC
// Handle docker log protocol; see
// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
-func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.WriteCloser, reader io.Reader) error {
+func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.Writer, reader io.Reader) error {
header := make([]byte, 8)
var err error
for err == nil {
@@ -247,14 +246,6 @@ func (e *dockerExecutor) handleStdoutStderr(stdout, stderr io.WriteCloser, reade
if err != nil {
return fmt.Errorf("error copying stdout/stderr from docker: %v", err)
}
- err = stdout.Close()
- if err != nil {
- return fmt.Errorf("error writing stdout: close: %v", err)
- }
- err = stderr.Close()
- if err != nil {
- return fmt.Errorf("error writing stderr: close: %v", err)
- }
return nil
}
diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go
index c773febe9..f4feaa06c 100644
--- a/lib/crunchrun/executor.go
+++ b/lib/crunchrun/executor.go
@@ -25,9 +25,9 @@ type containerSpec struct {
EnableNetwork bool
NetworkMode string // docker network mode, normally "default"
CgroupParent string
- Stdin io.ReadCloser
- Stdout io.WriteCloser
- Stderr io.WriteCloser
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
}
// containerExecutor is an interface to a container runtime
commit 919c2100e2a19508365efa63ea30e6e89b9c2697
Author: Tom Clegg <tom at curii.com>
Date: Thu Jul 15 10:13:39 2021 -0400
17394: Check by_id dir obeys storage classes.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index dc432114a..3c7c146f6 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -16,18 +16,19 @@ const (
// Importing arvadostest would be an import cycle, so these
// fixtures are duplicated here [until fs moves to a separate
// package].
- fixtureActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
- fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
- fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
- fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
- fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
- fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
- fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
- fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
- fixtureFooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
- fixtureNonexistentCollection = "zzzzz-4zz18-totallynotexist"
- fixtureBlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
- fixtureBlobSigningTTL = 336 * time.Hour
+ fixtureActiveToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
+ fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
+ fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
+ fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
+ fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
+ fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
+ fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
+ fixtureFooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ fixtureNonexistentCollection = "zzzzz-4zz18-totallynotexist"
+ fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
+ fixtureBlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
+ fixtureBlobSigningTTL = 336 * time.Hour
)
var _ = check.Suite(&SiteFSSuite{})
@@ -77,6 +78,17 @@ func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
c.Check(len(fis), check.Equals, 0)
}
+func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
+ f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte("nope"))
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ err = s.fs.Sync()
+ c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
+}
+
func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
f, err := s.fs.Open("/by_id")
c.Assert(err, check.IsNil)
commit 70ca572900248d24d7c1c0852f7bcbfa45f66e58
Author: Tom Clegg <tom at curii.com>
Date: Thu Jul 15 09:44:17 2021 -0400
17394: Update callback name.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 74757bf7c..c032b0716 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -32,7 +32,7 @@ var _ = check.Suite(&CollectionFSSuite{})
type keepClientStub struct {
blocks map[string][]byte
refreshable map[string]bool
- onPut func(bufcopy []byte) // called from PutB, before acquiring lock
+ onWrite func(bufcopy []byte) // called from WriteBlock, before acquiring lock
authToken string // client's auth token (used for signing locators)
sigkey string // blob signing key
sigttl time.Duration // blob signing ttl
@@ -58,8 +58,8 @@ func (kcs *keepClientStub) BlockWrite(_ context.Context, opts BlockWriteOptions)
locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
buf := make([]byte, len(opts.Data))
copy(buf, opts.Data)
- if kcs.onPut != nil {
- kcs.onPut(buf)
+ if kcs.onWrite != nil {
+ kcs.onWrite(buf)
}
for _, sc := range opts.StorageClasses {
if sc != "default" {
@@ -1086,7 +1086,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocksWritingLongFile(c *check.C) {
proceed := make(chan struct{})
var started, concurrent int32
blk2done := false
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
atomic.AddInt32(&concurrent, 1)
switch atomic.AddInt32(&started, 1) {
case 1:
@@ -1152,7 +1152,7 @@ func (s *CollectionFSSuite) TestFlushAll(c *check.C) {
fs, err := (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
// discard flushed data -- otherwise the stub will use
// unlimited memory
time.Sleep(time.Millisecond)
@@ -1196,7 +1196,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
c.Assert(err, check.IsNil)
var flushed int64
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
atomic.AddInt64(&flushed, int64(len(p)))
}
@@ -1264,7 +1264,7 @@ func (s *CollectionFSSuite) TestMaxUnflushed(c *check.C) {
time.AfterFunc(10*time.Second, func() { close(timeout) })
var putCount, concurrency int64
var unflushed int64
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
defer atomic.AddInt64(&unflushed, -int64(len(p)))
cur := atomic.AddInt64(&concurrency, 1)
defer atomic.AddInt64(&concurrency, -1)
@@ -1327,7 +1327,7 @@ func (s *CollectionFSSuite) TestFlushStress(c *check.C) {
})
wrote := 0
- s.kc.onPut = func(p []byte) {
+ s.kc.onWrite = func(p []byte) {
s.kc.Lock()
s.kc.blocks = map[string][]byte{}
wrote++
@@ -1358,7 +1358,7 @@ func (s *CollectionFSSuite) TestFlushStress(c *check.C) {
}
func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
- s.kc.onPut = func([]byte) {
+ s.kc.onWrite = func([]byte) {
s.kc.Lock()
s.kc.blocks = map[string][]byte{}
s.kc.Unlock()
commit 9bd7e42605084c0e2dcb492830d0e16951d7b4d0
Author: Tom Clegg <tom at curii.com>
Date: Thu Jul 15 09:42:21 2021 -0400
17394: Use BlockWrite interface in crunch-run.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 3c9c38161..23fbc430b 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -55,7 +55,7 @@ var ErrCancelled = errors.New("Cancelled")
// IKeepClient is the minimal Keep API methods used by crunch-run.
type IKeepClient interface {
- PutB(buf []byte) (string, int, error)
+ BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error)
ReadAt(locator string, p []byte, off int) (int, error)
ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
LocalLocator(locator string) (string, error)
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 4b1bf8425..42a2cf3ad 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -307,9 +307,11 @@ func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
return locator, nil
}
-func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
- client.Content = buf
- return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
+func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ client.Content = opts.Data
+ return arvados.BlockWriteResponse{
+ Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
+ }, nil
}
func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
@@ -455,8 +457,8 @@ func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvad
return nil, errors.New("KeepError")
}
-func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
- return "", 0, errors.New("KeepError")
+func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ return arvados.BlockWriteResponse{}, errors.New("KeepError")
}
func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
commit 7bc7e20348a3edf92081e54dae4204750e11857b
Author: Tom Clegg <tom at curii.com>
Date: Thu Jul 15 09:27:25 2021 -0400
17394: Test BlockWrite storage classes precedence.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 724c66e13..c52e07b8f 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -6,6 +6,7 @@ package keepclient
import (
"bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
@@ -20,6 +21,7 @@ import (
"testing"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
@@ -244,19 +246,25 @@ func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) {
func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
nServers := 5
for _, trial := range []struct {
- replicas int
- classes []string
- minRequests int
- maxRequests int
- success bool
+ replicas int
+ clientClasses []string
+ putClasses []string // putClasses takes precedence over clientClasses
+ minRequests int
+ maxRequests int
+ success bool
}{
- {1, []string{"class1"}, 1, 1, true},
- {2, []string{"class1"}, 1, 2, true},
- {3, []string{"class1"}, 2, 3, true},
- {1, []string{"class1", "class2"}, 1, 1, true},
- {nServers*2 + 1, []string{"class1"}, nServers, nServers, false},
- {1, []string{"class404"}, nServers, nServers, false},
- {1, []string{"class1", "class404"}, nServers, nServers, false},
+ {1, []string{"class1"}, nil, 1, 1, true},
+ {2, []string{"class1"}, nil, 1, 2, true},
+ {3, []string{"class1"}, nil, 2, 3, true},
+ {1, []string{"class1", "class2"}, nil, 1, 1, true},
+ {3, nil, []string{"class1"}, 2, 3, true},
+ {1, nil, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class404"}, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class1"}, []string{"class404", "class2"}, nServers, nServers, false},
+ {nServers*2 + 1, []string{"class1"}, nil, nServers, nServers, false},
+ {1, []string{"class404"}, nil, nServers, nServers, false},
+ {1, []string{"class1", "class404"}, nil, nServers, nServers, false},
+ {1, nil, []string{"class1", "class404"}, nServers, nServers, false},
} {
c.Logf("%+v", trial)
st := &StubPutHandler{
@@ -272,7 +280,7 @@ func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = trial.replicas
- kc.StorageClasses = trial.classes
+ kc.StorageClasses = trial.clientClasses
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
@@ -283,7 +291,10 @@ func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
}
kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
- _, _, err := kc.PutB([]byte("foo"))
+ _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: []byte("foo"),
+ StorageClasses: trial.putClasses,
+ })
if trial.success {
c.Check(err, check.IsNil)
} else {
commit 9fc8c0ec7002d7ed16ac729b62be880bbc83594e
Author: Tom Clegg <tom at curii.com>
Date: Wed Jul 14 16:52:12 2021 -0400
17394: Propagate storage classes when writing data via collectionfs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 4e0348c08..a57f2a683 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -8,6 +8,7 @@ import (
"bufio"
"context"
"encoding/json"
+ "io"
"net"
"github.com/sirupsen/logrus"
@@ -205,6 +206,22 @@ type LogoutOptions struct {
ReturnTo string `json:"return_to"` // Redirect to this URL after logging out
}
+type BlockWriteOptions struct {
+ Hash string
+ Data []byte
+ Reader io.Reader
+ DataSize int // Must be set if Data is nil.
+ RequestID string
+ StorageClasses []string
+ Replicas int
+ Attempts int
+}
+
+type BlockWriteResponse struct {
+ Locator string
+ Replicas int
+}
+
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs_backend.go
index c8308aea5..32365a531 100644
--- a/sdk/go/arvados/fs_backend.go
+++ b/sdk/go/arvados/fs_backend.go
@@ -4,7 +4,10 @@
package arvados
-import "io"
+import (
+ "context"
+ "io"
+)
type fsBackend interface {
keepClient
@@ -20,7 +23,7 @@ type keepBackend struct {
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
- PutB(p []byte) (string, int, error)
+ BlockWrite(context.Context, BlockWriteOptions) (BlockWriteResponse, error)
LocalLocator(locator string) (string, error)
}
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index b743ab368..4d9db421f 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -42,7 +42,9 @@ type CollectionFileSystem interface {
type collectionFileSystem struct {
fileSystem
- uuid string
+ uuid string
+ replicas int
+ storageClasses []string
}
// FileSystem returns a CollectionFileSystem for the collection.
@@ -52,12 +54,16 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
modTime = time.Now()
}
fs := &collectionFileSystem{
- uuid: c.UUID,
+ uuid: c.UUID,
+ storageClasses: c.StorageClassesDesired,
fileSystem: fileSystem{
fsBackend: keepBackend{apiClient: client, keepClient: kc},
thr: newThrottle(concurrentWriters),
},
}
+ if r := c.ReplicationDesired; r != nil {
+ fs.replicas = *r
+ }
root := &dirnode{
fs: fs,
treenode: treenode{
@@ -321,7 +327,7 @@ func (fn *filenode) seek(startPtr filenodePtr) (ptr filenodePtr) {
// filenode implements inode.
type filenode struct {
parent inode
- fs FileSystem
+ fs *collectionFileSystem
fileinfo fileinfo
segments []segment
// number of times `segments` has changed in a
@@ -610,7 +616,11 @@ func (fn *filenode) pruneMemSegments() {
fn.fs.throttle().Acquire()
go func() {
defer close(done)
- locator, _, err := fn.FS().PutB(buf)
+ resp, err := fn.FS().BlockWrite(context.Background(), BlockWriteOptions{
+ Data: buf,
+ Replicas: fn.fs.replicas,
+ StorageClasses: fn.fs.storageClasses,
+ })
fn.fs.throttle().Release()
fn.Lock()
defer fn.Unlock()
@@ -631,7 +641,7 @@ func (fn *filenode) pruneMemSegments() {
fn.memsize -= int64(len(buf))
fn.segments[idx] = storedSegment{
kc: fn.FS(),
- locator: locator,
+ locator: resp.Locator,
size: len(buf),
offset: 0,
length: len(buf),
@@ -748,7 +758,11 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
go func() {
defer close(done)
defer close(errs)
- locator, _, err := dn.fs.PutB(block)
+ resp, err := dn.fs.BlockWrite(context.Background(), BlockWriteOptions{
+ Data: block,
+ Replicas: dn.fs.replicas,
+ StorageClasses: dn.fs.storageClasses,
+ })
dn.fs.throttle().Release()
if err != nil {
errs <- err
@@ -780,7 +794,7 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
data := ref.fn.segments[ref.idx].(*memSegment).buf
ref.fn.segments[ref.idx] = storedSegment{
kc: dn.fs,
- locator: locator,
+ locator: resp.Locator,
size: blocksize,
offset: offsets[idx],
length: len(data),
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 05c8ea61a..74757bf7c 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -6,6 +6,7 @@ package arvados
import (
"bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
@@ -50,17 +51,25 @@ func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error
return copy(p, buf[off:]), nil
}
-func (kcs *keepClientStub) PutB(p []byte) (string, int, error) {
- locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(p), len(p)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
- buf := make([]byte, len(p))
- copy(buf, p)
+func (kcs *keepClientStub) BlockWrite(_ context.Context, opts BlockWriteOptions) (BlockWriteResponse, error) {
+ if opts.Data == nil {
+ panic("oops, stub is not made for this")
+ }
+ locator := SignLocator(fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)), kcs.authToken, time.Now().Add(kcs.sigttl), kcs.sigttl, []byte(kcs.sigkey))
+ buf := make([]byte, len(opts.Data))
+ copy(buf, opts.Data)
if kcs.onPut != nil {
kcs.onPut(buf)
}
+ for _, sc := range opts.StorageClasses {
+ if sc != "default" {
+ return BlockWriteResponse{}, fmt.Errorf("stub does not write storage class %q", sc)
+ }
+ }
kcs.Lock()
defer kcs.Unlock()
kcs.blocks[locator[:32]] = buf
- return locator, 1, nil
+ return BlockWriteResponse{Locator: locator, Replicas: 1}, nil
}
var reRemoteSignature = regexp.MustCompile(`\+[AR][^+]*`)
@@ -112,6 +121,22 @@ func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
c.Check(ok, check.Equals, true)
}
+func (s *CollectionFSSuite) TestUnattainableStorageClasses(c *check.C) {
+ fs, err := (&Collection{
+ StorageClassesDesired: []string{"unobtainium"},
+ }).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ f, err := fs.OpenFile("/foo", os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte("food"))
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ _, err = fs.MarshalManifest(".")
+ c.Assert(err, check.ErrorMatches, `.*stub does not write storage class \"unobtainium\"`)
+}
+
func (s *CollectionFSSuite) TestColonInFilename(c *check.C) {
fs, err := (&Collection{
ManifestText: "./foo:foo 3858f62230ac3c915f300c664312c63f+3 0:3:bar:bar\n",
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 2b560cff5..2cd6bb4d4 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -8,6 +8,7 @@ package keepclient
import (
"bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
@@ -21,8 +22,8 @@ import (
"sync"
"time"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "git.arvados.org/arvados.git/sdk/go/asyncbuf"
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
@@ -153,23 +154,12 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
// Returns an InsufficientReplicasError if 0 <= replicas <
// kc.Wants_replicas.
func (kc *KeepClient) PutHR(hash string, r io.Reader, dataBytes int64) (string, int, error) {
- // Buffer for reads from 'r'
- var bufsize int
- if dataBytes > 0 {
- if dataBytes > BLOCKSIZE {
- return "", 0, ErrOversizeBlock
- }
- bufsize = int(dataBytes)
- } else {
- bufsize = BLOCKSIZE
- }
-
- buf := asyncbuf.NewBuffer(make([]byte, 0, bufsize))
- go func() {
- _, err := io.Copy(buf, HashCheckingReader{r, md5.New(), hash})
- buf.CloseWithError(err)
- }()
- return kc.putReplicas(hash, buf.NewReader, dataBytes)
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Hash: hash,
+ Reader: r,
+ DataSize: int(dataBytes),
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutHB writes a block to Keep. The hash of the bytes is given in
@@ -177,16 +167,21 @@ func (kc *KeepClient) PutHR(hash string, r io.Reader, dataBytes int64) (string,
//
// Return values are the same as for PutHR.
func (kc *KeepClient) PutHB(hash string, buf []byte) (string, int, error) {
- newReader := func() io.Reader { return bytes.NewBuffer(buf) }
- return kc.putReplicas(hash, newReader, int64(len(buf)))
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Hash: hash,
+ Data: buf,
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutB writes a block to Keep. It computes the hash itself.
//
// Return values are the same as for PutHR.
func (kc *KeepClient) PutB(buffer []byte) (string, int, error) {
- hash := fmt.Sprintf("%x", md5.Sum(buffer))
- return kc.PutHB(hash, buffer)
+ resp, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: buffer,
+ })
+ return resp.Locator, resp.Replicas, err
}
// PutR writes a block to Keep. It first reads all data from r into a buffer
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index f59d16fd3..724c66e13 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -173,7 +173,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
@@ -229,7 +229,7 @@ func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) {
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 7b2e47ff8..a8c82aac0 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -5,6 +5,8 @@
package keepclient
import (
+ "bytes"
+ "context"
"crypto/md5"
"errors"
"fmt"
@@ -16,7 +18,9 @@ import (
"strconv"
"strings"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/asyncbuf"
)
// DebugPrintf emits debug messages. The easiest way to enable
@@ -58,7 +62,7 @@ type uploadStatus struct {
}
func (kc *KeepClient) uploadToKeepServer(host string, hash string, classesTodo []string, body io.Reader,
- uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
+ uploadStatusChan chan<- uploadStatus, expectedLength int, reqid string) {
var req *http.Request
var err error
@@ -69,7 +73,7 @@ func (kc *KeepClient) uploadToKeepServer(host string, hash string, classesTodo [
return
}
- req.ContentLength = expectedLength
+ req.ContentLength = int64(expectedLength)
if expectedLength > 0 {
req.Body = ioutil.NopCloser(body)
} else {
@@ -123,15 +127,57 @@ func (kc *KeepClient) uploadToKeepServer(host string, hash string, classesTodo [
}
}
-func (kc *KeepClient) putReplicas(
- hash string,
- getReader func() io.Reader,
- expectedLength int64) (locator string, replicas int, err error) {
-
- reqid := kc.getRequestID()
+func (kc *KeepClient) BlockWrite(ctx context.Context, req arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
+ var resp arvados.BlockWriteResponse
+ var getReader func() io.Reader
+ if req.Data == nil && req.Reader == nil {
+ return resp, errors.New("invalid BlockWriteOptions: Data and Reader are both nil")
+ }
+ if req.DataSize < 0 {
+ return resp, fmt.Errorf("invalid BlockWriteOptions: negative DataSize %d", req.DataSize)
+ }
+ if req.DataSize > BLOCKSIZE || len(req.Data) > BLOCKSIZE {
+ return resp, ErrOversizeBlock
+ }
+ if req.Data != nil {
+ if req.DataSize > len(req.Data) {
+ return resp, errors.New("invalid BlockWriteOptions: DataSize > len(Data)")
+ }
+ if req.DataSize == 0 {
+ req.DataSize = len(req.Data)
+ }
+ getReader = func() io.Reader { return bytes.NewReader(req.Data[:req.DataSize]) }
+ } else {
+ buf := asyncbuf.NewBuffer(make([]byte, 0, req.DataSize))
+ go func() {
+ _, err := io.Copy(buf, HashCheckingReader{req.Reader, md5.New(), req.Hash})
+ buf.CloseWithError(err)
+ }()
+ getReader = buf.NewReader
+ }
+ if req.Hash == "" {
+ m := md5.New()
+ _, err := io.Copy(m, getReader())
+ if err != nil {
+ return resp, err
+ }
+ req.Hash = fmt.Sprintf("%x", m.Sum(nil))
+ }
+ if req.StorageClasses == nil {
+ req.StorageClasses = kc.StorageClasses
+ }
+ if req.Replicas == 0 {
+ req.Replicas = kc.Want_replicas
+ }
+ if req.RequestID == "" {
+ req.RequestID = kc.getRequestID()
+ }
+ if req.Attempts == 0 {
+ req.Attempts = 1 + kc.Retries
+ }
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), req.Hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
@@ -153,20 +199,18 @@ func (kc *KeepClient) putReplicas(
}()
}()
- replicasWanted := kc.Want_replicas
replicasTodo := map[string]int{}
- for _, c := range kc.StorageClasses {
- replicasTodo[c] = replicasWanted
+ for _, c := range req.StorageClasses {
+ replicasTodo[c] = req.Replicas
}
- replicasDone := 0
replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
- replicasPerThread = replicasWanted
+ replicasPerThread = req.Replicas
}
- retriesRemaining := 1 + kc.Retries
+ retriesRemaining := req.Attempts
var retryServers []string
lastError := make(map[string]string)
@@ -190,7 +234,7 @@ func (kc *KeepClient) putReplicas(
}
}
if !trackingClasses {
- maxConcurrency = replicasWanted - replicasDone
+ maxConcurrency = req.Replicas - resp.Replicas
}
if maxConcurrency < 1 {
// If there are no non-zero entries in
@@ -200,8 +244,8 @@ func (kc *KeepClient) putReplicas(
for active*replicasPerThread < maxConcurrency {
// Start some upload requests
if nextServer < len(sv) {
- DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go kc.uploadToKeepServer(sv[nextServer], hash, classesTodo, getReader(), uploadStatusChan, expectedLength, reqid)
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", req.RequestID, req.Hash, sv[nextServer])
+ go kc.uploadToKeepServer(sv[nextServer], req.Hash, classesTodo, getReader(), uploadStatusChan, req.DataSize, req.RequestID)
nextServer++
active++
} else {
@@ -211,13 +255,13 @@ func (kc *KeepClient) putReplicas(
msg += resp + "; "
}
msg = msg[:len(msg)-2]
- return locator, replicasDone, InsufficientReplicasError(errors.New(msg))
+ return resp, InsufficientReplicasError(errors.New(msg))
}
break
}
}
- DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v", reqid, replicasTodo, active)
+ DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v", req.RequestID, replicasTodo, active)
if active < 1 {
break
}
@@ -228,7 +272,7 @@ func (kc *KeepClient) putReplicas(
if status.statusCode == http.StatusOK {
delete(lastError, status.url)
- replicasDone += status.replicasStored
+ resp.Replicas += status.replicasStored
if len(status.classesStored) == 0 {
// Server doesn't report
// storage classes. Give up
@@ -244,7 +288,7 @@ func (kc *KeepClient) putReplicas(
delete(replicasTodo, className)
}
}
- locator = status.response
+ resp.Locator = status.response
} else {
msg := fmt.Sprintf("[%d] %s", status.statusCode, status.response)
if len(msg) > 100 {
@@ -264,7 +308,7 @@ func (kc *KeepClient) putReplicas(
sv = retryServers
}
- return locator, replicasDone, nil
+ return resp, nil
}
func parseStorageClassesConfirmedHeader(hdr string) (map[string]int, error) {
commit e2cda71fa792d17024bc46cac0e93fe230db3e1b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:50:07 2021 -0400
Don't pull arvbox checkouts after clone
We already have latest and tags/detached HEADS can't be pulled anyway
(and they fail, which is the real problem).
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index ba3f2cd07..fd464974f 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -250,17 +250,14 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
git -C "$COMPOSER_ROOT" checkout arvados-fork
- git -C "$COMPOSER_ROOT" pull
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
- git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index c285d53ca..92099614a 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -10,13 +10,10 @@ ARG workbench2_version=main
RUN cd /usr/src && \
git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
- git -C arvados pull && \
git clone --no-checkout https://github.com/arvados/composer.git && \
git -C composer checkout ${composer_version} && \
- git -C composer pull && \
git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
git -C workbench2 checkout ${workbench2_version} && \
- git -C workbench2 pull && \
chown -R 1000:1000 /usr/src
# avoid rebuilding arvados-server, it's already been built as part of the base image
commit 42ee44aeabf810ad627f617e1f6391d767269d3f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 14:12:08 2021 -0400
arvbox docs include a docker image tag
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 8b3994edd..a8235ee70 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -16,7 +16,7 @@ h2. Quick start
<pre>
$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
$ chmod +x arvbox
-$ ./arvbox start localdemo
+$ ./arvbox start localdemo latest
$ ./arvbox adduser demouser demo at example.com
</pre>
commit 29aa7d6c9207b62e06b3d94bcd6051c8946bf44a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 13:57:37 2021 -0400
Add ARVADOS_BRANCH and WORKBENCH2_BRANCH to arvbox
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 3c77ade8d..8b3994edd 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -14,8 +14,8 @@ Arvbox is a Docker-based self-contained development, demonstration and testing e
h2. Quick start
<pre>
-$ git clone https://github.com/arvados/arvados.git
-$ cd arvados/tools/arvbox/bin
+$ curl -O https://git.arvados.org/arvados.git/blob_plain/refs/heads/main:/tools/arvbox/bin/arvbox
+$ chmod +x arvbox
$ ./arvbox start localdemo
$ ./arvbox adduser demouser demo at example.com
</pre>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 9bf6fe1b2..ba3f2cd07 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -52,6 +52,14 @@ if test -z "$WORKBENCH2_ROOT" ; then
WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
fi
+if test -z "$ARVADOS_BRANCH" ; then
+ ARVADOS_BRANCH=main
+fi
+
+if test -z "$WORKBENCH2_BRANCH" ; then
+ WORKBENCH2_BRANCH=main
+fi
+
PG_DATA="$ARVBOX_DATA/postgres"
VAR_DATA="$ARVBOX_DATA/var"
PASSENGER="$ARVBOX_DATA/passenger"
@@ -61,7 +69,7 @@ NPMCACHE="$ARVBOX_DATA/npm"
GOSTUFF="$ARVBOX_DATA/gopath"
RLIBS="$ARVBOX_DATA/Rlibs"
ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox"
-GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0"
+GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.7.0"
getip() {
docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER
@@ -241,6 +249,8 @@ run() {
if ! test -d "$ARVADOS_ROOT" ; then
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
@@ -249,6 +259,8 @@ run() {
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
+ git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
+ git -C "$ARVADOS_ROOT" pull
fi
if [[ "$CONFIG" = test ]] ; then
@@ -390,9 +402,24 @@ build() {
BUILDTYPE=dev
fi
- docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$LOCAL_ARVADOS_ROOT"
+ if test "$ARVADOS_BRANCH" = "main" ; then
+ ARVADOS_BRANCH=$GITHEAD
+ fi
+
+ docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ --build-arg=workdir=/tools/arvbox/lib/arvbox/docker \
+ -t arvados/arvbox-base:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.base" \
+ "$LOCAL_ARVADOS_ROOT"
docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest
- docker build $NO_CACHE -t arvados/arvbox-$BUILDTYPE:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" "$ARVBOX_DOCKER"
+ docker build $NO_CACHE \
+ --build-arg=arvados_version=$ARVADOS_BRANCH \
+ --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
+ -t arvados/arvbox-$BUILDTYPE:$GITHEAD \
+ -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" \
+ "$ARVBOX_DOCKER"
docker tag $FORCE arvados/arvbox-$BUILDTYPE:$GITHEAD arvados/arvbox-$BUILDTYPE:latest
}
@@ -623,7 +650,7 @@ sv stop keepproxy
cd /usr/src/arvados/services/api
export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
export RAILS_ENV=development
-flock $GEM_HOME/gems.lock bundle exec rake db:drop
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:drop
rm $ARVADOS_CONTAINER_PATH/api_database_setup
rm $ARVADOS_CONTAINER_PATH/superuser_token
sv start api
commit ebba63532a010e8928f5c872fc06b864832ea631
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jul 14 11:19:58 2021 -0400
Fix "wrong bundler" error starting API server in arvbox
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 96f3666cd..9bf6fe1b2 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -566,7 +566,7 @@ case "$subcmd" in
else
echo "Usage: $0 $subcmd <start|stop|restart> <service>"
echo "Available services:"
- exec docker execa $ARVBOX_CONTAINER ls /etc/service
+ exec docker exec $ARVBOX_CONTAINER ls /etc/service
fi
;;
diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index 4ad2aed0c..b1b6d37c0 100755
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -56,16 +56,16 @@ EOF
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- flock $GEM_HOME/gems.lock bundle exec rake db:setup
+ flock $GEM_HOME/gems.lock bin/bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEM_HOME/gems.lock bin/bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-flock $GEM_HOME/gems.lock bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bin/bundle exec rake db:migrate
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index eb53e1904..d8f368049 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -61,27 +61,17 @@ fi
run_bundler() {
if test -f Gemfile.lock ; then
- # The 'gem install bundler line below' is cf.
- # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html,
- # until we get bundler 2.7.10/3.0.0 or higher
- flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')"
frozen=--frozen
else
frozen=""
fi
- # if ! test -x $GEM_HOME/bin/bundler ; then
- # bundleversion=2.0.2
- # bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true)
- # if test -n "$bundlergem" ; then
- # flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem
- # else
- # flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion}
- # fi
- # fi
- # Make sure to put the gem binaries in the right place
- flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin
- if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then
- flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@"
+ BUNDLER=bundler
+ if test -x $PWD/bin/bundler ; then
+ # If present, use the one associated with rails workbench or API
+ BUNDLER=$PWD/bin/bundler
+ fi
+ if ! flock $GEM_HOME/gems.lock $BUNDLER install --verbose --local --no-deployment $frozen "$@" ; then
+ flock $GEM_HOME/gems.lock $BUNDLER install --verbose --no-deployment $frozen "$@"
fi
}
diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service
index d2691e7ed..c949bffa7 100755
--- a/tools/arvbox/lib/arvbox/docker/service/api/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service
@@ -17,8 +17,8 @@ else
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
@@ -33,4 +33,4 @@ fi
touch $ARVADOS_CONTAINER_PATH/api.ready
-exec bundle exec passenger start --port=${services[api]}
+exec bin/bundle exec passenger start --port=${services[api]}
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run b/tools/arvbox/lib/arvbox/docker/service/workbench/run
index b8a28fa76..9b588fc4f 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run
@@ -23,7 +23,7 @@ fi
if test "$1" != "--only-deps" ; then
openssl verify -CAfile $root_cert $server_cert
- exec bundle exec passenger start --port=${services[workbench]} \
+ exec bin/bundle exec passenger start --port=${services[workbench]} \
--ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \
--ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \
--user arvbox
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 32efea51b..e6f0ad4a4 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -23,8 +23,8 @@ else
fi
run_bundler --without=development
-flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
-flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bin/bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
if test "$1" = "--only-deps" ; then
@@ -34,7 +34,7 @@ cat >config/application.yml <<EOF
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
rm config/application.yml
exit
fi
@@ -43,5 +43,5 @@ set -u
secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
-RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
-flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bin/bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bin/bundle exec rake assets:precompile
commit e10f8abe94d72d969d2d68a007f48b305a749b69
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jul 13 17:11:57 2021 -0400
arvbox uses react-scripts, refs #17782
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
index f956eecc6..fb3eaaeee 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
@@ -53,9 +53,11 @@ else
arv api_client create --api-client "$apiclient"
fi
-export HTTPS=false
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
# right process.
export VERSION=$(./version-at-commit.sh)
-exec node node_modules/react-scripts-ts/scripts/start.js
+export BROWSER=none
+export CI=true
+node --version
+exec node node_modules/react-scripts/scripts/start.js
commit 986dc65bf81cbfd986cb8b87a0f87a1fe890d459
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Jul 13 16:14:49 2021 -0300
17913: Upgrades 'addressable' gem to 2.8.0
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index 178e5cdfe..ab9256a38 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -57,7 +57,7 @@ GEM
i18n (>= 0.7, < 2)
minitest (~> 5.1)
tzinfo (~> 1.1)
- addressable (2.7.0)
+ addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
angularjs-rails (1.3.15)
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index ddecd4a18..6e149d45a 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -53,7 +53,7 @@ GEM
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
- addressable (2.7.0)
+ addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
arel (9.0.0)
commit 5cffc002d44396c86a84d4455ca276bf28d51416
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Jul 7 15:50:38 2021 -0300
17782: Updates nodejs to v12, as required by workbench2.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index a1f5d72be..77a64df89 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -290,7 +290,7 @@ ln -sf /var/lib/arvados/bin/geckodriver /usr/local/bin/
}
}
- nodejsversion := "v10.23.1"
+ nodejsversion := "v12.22.2"
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
commit 84a8f9c7e7280518baec023d9b71ae1d48faf8de
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jul 9 10:54:30 2021 -0400
Pin networkx to avoid new version that pulls in scipy/numpy
refs #17879
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 1a701f258..34fe0c15d 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -43,7 +43,8 @@ setup(name='arvados-cwl-runner',
'schema-salad==7.1.20210611090601',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
- 'ciso8601 >= 2.0.0'
+ 'ciso8601 >= 2.0.0',
+ 'networkx < 2.6'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
commit f54f9174873c2bf04243cbd7c93bb00c66390586
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 8 17:02:36 2021 -0400
Add missing link to "restricting-upload-download" to doc refs #17464
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index c2ca74ab1..39fe22fde 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -189,6 +189,7 @@ navbar:
- Data Management:
- admin/collection-versioning.html.textile.liquid
- admin/collection-managed-properties.html.textile.liquid
+ - admin/restricting-upload-download.html.textile.liquid
- admin/keep-balance.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
commit 60ed39d991cfaa5ea454bde581c9f7b4e7f4d6fd
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 8 15:20:51 2021 -0400
17879: Add test that acr doesn't fail on sbg fields
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/17879-ignore-sbg-fields-job.yml b/sdk/cwl/tests/17879-ignore-sbg-fields-job.yml
new file mode 100644
index 000000000..26e9d123a
--- /dev/null
+++ b/sdk/cwl/tests/17879-ignore-sbg-fields-job.yml
@@ -0,0 +1,5 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+sampleName: woble
diff --git a/sdk/cwl/tests/17879-ignore-sbg-fields.cwl b/sdk/cwl/tests/17879-ignore-sbg-fields.cwl
new file mode 100644
index 000000000..131fd0fd5
--- /dev/null
+++ b/sdk/cwl/tests/17879-ignore-sbg-fields.cwl
@@ -0,0 +1,37 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+$namespaces:
+ sbg: https://www.sevenbridges.com/
+class: "Workflow"
+cwlVersion: v1.1
+label: "check that sbg x/y fields are correctly ignored"
+inputs:
+ - id: sampleName
+ type: string
+ label: Sample name
+ 'sbg:x': -22
+ 'sbg:y': 33.4296875
+outputs:
+ - id: outstr
+ type: string
+ outputSource: step1/outstr
+steps:
+ step1:
+ in:
+ sampleName: sampleName
+ out: [outstr]
+ run:
+ class: CommandLineTool
+ inputs:
+ sampleName: string
+ stdout: out.txt
+ outputs:
+ outstr:
+ type: string
+ outputBinding:
+ glob: out.txt
+ loadContents: true
+ outputEval: $(self[0].contents)
+ arguments: [echo, "-n", "foo", $(inputs.sampleName), "bar"]
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 717636cd3..ae22d65f4 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -433,3 +433,9 @@
"val": "keep:f225e6259bdd63bc7240599648dde9f1+97/hg19.fa"
tool: 17858-pack-visit-crash.cwl
doc: "Test issue 17858 - keep ref default inputs on ExpressionTool"
+
+- job: 17879-ignore-sbg-fields-job.yml
+ output:
+ "outstr": "foo woble bar"
+ tool: 17879-ignore-sbg-fields.cwl
+ doc: "Test issue 17879 - ignores sbg fields"
commit fe51b07f84306ba3df7945dc0fbea9aedacb2adf
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jul 8 14:36:00 2021 -0400
17879: Default --thread-count=0, upgrade schema-salad
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 4bfe27278..04db611fb 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -201,7 +201,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help=argparse.SUPPRESS)
parser.add_argument("--thread-count", type=int,
- default=1, help="Number of threads to use for job submit and output collection.")
+ default=0, help="Number of threads to use for job submit and output collection.")
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index eb3d77892..1a701f258 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -40,7 +40,7 @@ setup(name='arvados-cwl-runner',
# build.
install_requires=[
'cwltool==3.0.20210319143721',
- 'schema-salad==7.1.20210316164414',
+ 'schema-salad==7.1.20210611090601',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index c448f218b..12daf6b67 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -303,7 +303,7 @@ def stubs(func):
'state': 'Committed',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
@@ -414,7 +414,7 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--disable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -438,7 +438,7 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
@@ -470,7 +470,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=stop',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -493,7 +493,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
"--output-name="+output_name, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -515,7 +515,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", "--debug",
"--storage-classes=foo", '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -578,7 +578,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
"--intermediate-output-ttl=3600",
@@ -601,7 +601,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
"--trash-intermediate",
@@ -625,7 +625,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
"--output-tags="+output_tags, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -703,7 +703,7 @@ class TestSubmit(unittest.TestCase):
'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
@@ -798,7 +798,7 @@ class TestSubmit(unittest.TestCase):
'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
@@ -862,7 +862,7 @@ class TestSubmit(unittest.TestCase):
expect_container["owner_uuid"] = project_uuid
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- "--eval-timeout=20", "--thread-count=4",
+ "--eval-timeout=20", "--thread-count=0",
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
'--project-uuid='+project_uuid,
@@ -884,7 +884,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=60.0', '--thread-count=4',
+ '--eval-timeout=60.0', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -905,7 +905,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=500",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -997,7 +997,7 @@ class TestSubmit(unittest.TestCase):
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=4',
+ '--eval-timeout=20', '--thread-count=0',
'--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -1063,7 +1063,7 @@ class TestSubmit(unittest.TestCase):
"--disable-validate",
"--disable-color",
"--eval-timeout=20",
- '--thread-count=4',
+ '--thread-count=0',
"--enable-reuse",
"--collection-cache-size=256",
'--debug',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list