[ARVADOS] updated: 2.1.0-801-g8fd331405
Git user
git at public.arvados.org
Tue May 18 20:39:12 UTC 2021
Summary of changes:
lib/config/export.go | 1 +
lib/crunchrun/crunchrun.go | 19 ++--
lib/crunchrun/integration_test.go | 212 ++++++++++++++++++++++++++++++++++++++
lib/crunchrun/singularity.go | 10 +-
4 files changed, 235 insertions(+), 7 deletions(-)
create mode 100644 lib/crunchrun/integration_test.go
via 8fd331405028ebdbb97de58560057564aa530105 (commit)
via 59b0581ca03cae918865f00a7c25fff3e72e96cc (commit)
from 83d6cc94a72c1be8a371976ca6198abfb8bfc5a9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8fd331405028ebdbb97de58560057564aa530105
Author: Tom Clegg <tom at curii.com>
Date: Tue May 18 16:39:00 2021 -0400
17296: Add crunch-run integration test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 4bdaca4d6..77cfcf68b 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -148,9 +148,10 @@ type ContainerRunner struct {
cStateLock sync.Mutex
cCancelled bool // StopContainer() invoked
- enableNetwork string // one of "default" or "always"
- networkMode string // "none", "host", or "" -- passed through to executor
- arvMountLog *ThrottledLogger
+ enableMemoryLimit bool
+ enableNetwork string // one of "default" or "always"
+ networkMode string // "none", "host", or "" -- passed through to executor
+ arvMountLog *ThrottledLogger
containerWatchdogInterval time.Duration
@@ -291,7 +292,7 @@ func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) (
}
runner.arvMountLog = NewThrottledLogger(w)
c.Stdout = runner.arvMountLog
- c.Stderr = runner.arvMountLog
+ c.Stderr = io.MultiWriter(runner.arvMountLog, os.Stderr)
runner.CrunchLog.Printf("Running %v", c.Args)
@@ -386,6 +387,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
if err != nil {
return nil, fmt.Errorf("could not get container token: %s", err)
}
+ runner.CrunchLog.Printf("container token %q", token)
pdhOnly := true
tmpcount := 0
@@ -946,11 +948,14 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st
// both "" and "." mean default
workdir = ""
}
-
+ ram := runner.Container.RuntimeConstraints.RAM
+ if !runner.enableMemoryLimit {
+ ram = 0
+ }
return runner.executor.Create(containerSpec{
Image: imageID,
VCPUs: runner.Container.RuntimeConstraints.VCPUs,
- RAM: runner.Container.RuntimeConstraints.RAM,
+ RAM: ram,
WorkingDir: workdir,
Env: env,
BindMounts: bindmounts,
@@ -1586,6 +1591,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
sleep := flags.Duration("sleep", 0, "Delay before starting (testing use only)")
kill := flags.Int("kill", -1, "Send signal to an existing crunch-run process for given UUID")
list := flags.Bool("list", false, "List UUIDs of existing crunch-run processes")
+ enableMemoryLimit := flags.Bool("enable-memory-limit", true, "tell container runtime to limit container's memory usage")
enableNetwork := flags.String("container-enable-networking", "default", "enable networking \"always\" (for all containers) or \"default\" (for containers that request it)")
networkMode := flags.String("container-network-mode", "default", `Docker network mode for container (use any argument valid for docker --net)`)
memprofile := flags.String("memprofile", "", "write memory profile to `file` after running container")
@@ -1718,6 +1724,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
cr.statInterval = *statInterval
cr.cgroupRoot = *cgroupRoot
cr.expectCgroupParent = *cgroupParent
+ cr.enableMemoryLimit = *enableMemoryLimit
cr.enableNetwork = *enableNetwork
cr.networkMode = *networkMode
if *cgroupParentSubsystem != "" {
diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
new file mode 100644
index 000000000..04a15bcea
--- /dev/null
+++ b/lib/crunchrun/integration_test.go
@@ -0,0 +1,212 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "strings"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ . "gopkg.in/check.v1"
+)
+
+var _ = Suite(&integrationSuite{})
+
+type integrationSuite struct {
+ engine string
+ image arvados.Collection
+ input arvados.Collection
+ stdin bytes.Buffer
+ stdout bytes.Buffer
+ stderr bytes.Buffer
+ cr arvados.ContainerRequest
+ client *arvados.Client
+ ac *arvadosclient.ArvadosClient
+ kc *keepclient.KeepClient
+}
+
+func (s *integrationSuite) SetUpSuite(c *C) {
+ arvadostest.StartKeep(2, true)
+
+ out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput()
+ c.Log(string(out))
+ c.Assert(err, IsNil)
+ out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output()
+ imageUUID := strings.TrimSpace(string(out))
+ c.Logf("image uuid %s", imageUUID)
+ c.Assert(err, IsNil)
+ err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Logf("image pdh %s", s.image.PortableDataHash)
+
+ s.client = arvados.NewClientFromEnv()
+ s.ac, err = arvadosclient.New(s.client)
+ c.Assert(err, IsNil)
+ s.kc = keepclient.New(s.ac)
+ fs, err := s.input.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ f, err := fs.OpenFile("inputfile", os.O_CREATE|os.O_WRONLY, 0755)
+ c.Assert(err, IsNil)
+ _, err = f.Write([]byte("inputdata"))
+ c.Assert(err, IsNil)
+ err = f.Close()
+ c.Assert(err, IsNil)
+ s.input.ManifestText, err = fs.MarshalManifest(".")
+ c.Assert(err, IsNil)
+ err = s.client.RequestAndDecode(&s.input, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "ensure_unique_name": true,
+ "collection": map[string]interface{}{
+ "manifest_text": s.input.ManifestText,
+ },
+ })
+ c.Assert(err, IsNil)
+ c.Logf("input pdh %s", s.input.PortableDataHash)
+}
+
+func (s *integrationSuite) TearDownSuite(c *C) {
+ err := s.client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
+ c.Check(err, IsNil)
+}
+
+func (s *integrationSuite) SetUpTest(c *C) {
+ s.engine = "docker"
+ s.stdin = bytes.Buffer{}
+ s.stdout = bytes.Buffer{}
+ s.stderr = bytes.Buffer{}
+ s.cr = arvados.ContainerRequest{
+ Priority: 1,
+ State: "Committed",
+ OutputPath: "/mnt/out",
+ ContainerImage: s.image.PortableDataHash,
+ Mounts: map[string]arvados.Mount{
+ "/mnt/json": {
+ Kind: "json",
+ Content: []interface{}{
+ "foo",
+ map[string]string{"foo": "bar"},
+ nil,
+ },
+ },
+ "/mnt/in": {
+ Kind: "collection",
+ PortableDataHash: s.input.PortableDataHash,
+ },
+ "/mnt/out": {
+ Kind: "tmp",
+ Capacity: 1000,
+ },
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ RAM: 128000000,
+ VCPUs: 1,
+ API: true,
+ },
+ }
+}
+
+func (s *integrationSuite) setup(c *C) {
+ err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+ "priority": s.cr.Priority,
+ "state": s.cr.State,
+ "command": s.cr.Command,
+ "output_path": s.cr.OutputPath,
+ "container_image": s.cr.ContainerImage,
+ "mounts": s.cr.Mounts,
+ "runtime_constraints": s.cr.RuntimeConstraints,
+ "use_existing": false,
+ }})
+ c.Assert(err, IsNil)
+ c.Assert(s.cr.ContainerUUID, Not(Equals), "")
+ err = s.client.RequestAndDecode(nil, "POST", "arvados/v1/containers/"+s.cr.ContainerUUID+"/lock", nil, nil)
+ c.Assert(err, IsNil)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) {
+ s.engine = "docker"
+ s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) {
+ s.engine = "singularity"
+ s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) testRunTrivialContainer(c *C) {
+ if err := exec.Command("which", s.engine).Run(); err != nil {
+ c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
+ }
+ s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
+ s.setup(c)
+ code := command{}.RunCommand("crunch-run", []string{
+ "-runtime-engine=" + s.engine,
+ "-enable-memory-limit=false",
+ s.cr.ContainerUUID,
+ }, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
+ c.Check(code, Equals, 0)
+ err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Logf("Finished container request: %#v", s.cr)
+
+ var log arvados.Collection
+ err = s.client.RequestAndDecode(&log, "GET", "arvados/v1/collections/"+s.cr.LogUUID, nil, nil)
+ c.Assert(err, IsNil)
+ fs, err := log.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ if d, err := fs.Open("/"); c.Check(err, IsNil) {
+ fis, err := d.Readdir(-1)
+ c.Assert(err, IsNil)
+ for _, fi := range fis {
+ if fi.IsDir() {
+ continue
+ }
+ f, err := fs.Open(fi.Name())
+ c.Assert(err, IsNil)
+ buf, err := ioutil.ReadAll(f)
+ c.Assert(err, IsNil)
+ c.Logf("\n===== %s =====\n%s", fi.Name(), buf)
+ }
+ }
+
+ var output arvados.Collection
+ err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
+ c.Assert(err, IsNil)
+ fs, err = output.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, "inputdata")
+ }
+ if f, err := fs.Open("json"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
+ }
+ if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
+ c.Check(fi.IsDir(), Equals, true)
+ }
+ if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
+ defer d.Close()
+ fis, err := d.Readdir(-1)
+ c.Assert(err, IsNil)
+ // crunch-run still saves a ".keep" file to preserve
+ // empty dirs even though that shouldn't be
+ // necessary. Ideally we would do:
+ // c.Check(fis, HasLen, 0)
+ for _, fi := range fis {
+ c.Check(fi.Name(), Equals, ".keep")
+ }
+ }
+}
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index d783baab9..4bec8c3eb 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -40,8 +40,16 @@ func (e *singularityExecutor) ImageLoaded(string) bool {
// containerImage into a sif file for later use.
func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
e.logf("building singularity image")
+ // "singularity build" does not accept a
+ // docker-archive://... filename containing a ":" character,
+ // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
+ // symlink that doesn't have ":" chars.
+ err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+ if err != nil {
+ return err
+ }
e.imageFilename = e.tmpdir + "/image.sif"
- build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+imageTarballPath)
+ build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
e.logf("%v", build.Args)
out, err := build.CombinedOutput()
// INFO: Starting build...
commit 59b0581ca03cae918865f00a7c25fff3e72e96cc
Author: Tom Clegg <tom at curii.com>
Date: Tue May 18 06:34:15 2021 -0400
17296: Add RUntimeConfig to exported config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/export.go b/lib/config/export.go
index 5c0e9f270..9163421be 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -122,6 +122,7 @@ var whitelist = map[string]bool{
"Containers.MaxRetryAttempts": true,
"Containers.MinRetryPeriod": true,
"Containers.ReserveExtraRAM": true,
+ "Containers.RuntimeEngine": true,
"Containers.ShellAccess": true,
"Containers.ShellAccess.Admin": true,
"Containers.ShellAccess.User": true,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list