[ARVADOS] updated: 2.1.0-624-ge587495bf
Git user
git at public.arvados.org
Thu Apr 29 20:02:21 UTC 2021
Summary of changes:
lib/crunchrun/container_exec.go | 49 ++++-----
lib/crunchrun/container_exec_test.go | 170 ++++++++++++++++++++++++++++++
lib/crunchrun/fixtures/hello.tar | Bin 0 -> 11264 bytes
lib/crunchrun/singularity.go | 196 +++++++++++++++++++++++++++++++++++
4 files changed, 391 insertions(+), 24 deletions(-)
create mode 100644 lib/crunchrun/container_exec_test.go
create mode 100644 lib/crunchrun/fixtures/hello.tar
create mode 100644 lib/crunchrun/singularity.go
via e587495bf03b354e68ae64abe7969d1cb7c66c83 (commit)
from 43b39915bae3f3c24ab31cfbc7aefdce88f84dcb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e587495bf03b354e68ae64abe7969d1cb7c66c83
Author: Nico Cesar <nico at nicocesar.com>
Date: Thu Apr 29 16:01:24 2021 -0400
container_exec.go and test with busybox_uclibc.tar download
Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>
diff --git a/lib/crunchrun/container_exec.go b/lib/crunchrun/container_exec.go
index 809446d73..b65e068a4 100644
--- a/lib/crunchrun/container_exec.go
+++ b/lib/crunchrun/container_exec.go
@@ -55,48 +55,55 @@ type ContainerState struct {
}
type ContainerConfig struct {
- Env []string // List of environment variable to set in the container "VALUE=KEY" format
- Cmd []string // Command to run when starting the container
- Mounts []volumeMounts // List of Mounts used for the container.
-
+ Env []string // List of environment variable to set in the container "VALUE=KEY" format
+ Cmd []string // Command to run when starting the container
+ Mounts []volumeMounts // List of Mounts used for the container.
+ EnableNetworking bool // This will allow the container to reach the outside world
}
// ContainerExecuter interface will implement all the methods needed for Docker,
// Singularity, and others to load images, run containers, get thier outputs,
// etc.
type ContainerExecuter interface {
+ // CheckImageIsLoaded checks if imageID is already in the local environment,
+ // either something we can reference later in Docker API or similar, or a
+ // filepath containing the image to be used
CheckImageIsLoaded(imageID ImageID) bool
- LoadImage(containerImage io.Reader) error
+
+ // LoadImage translates a io.Reader that has a tarball in docker format
+ // (Usually created with 'docker save'), and load it to the local
+ // ContainerExecuter.
+ //
+ // Returns an ImageID to be referenced later, could be an identifier or a
+ // filepath or something else
+ LoadImage(containerImage io.Reader) (ImageID, error)
+
+ // ImageRemove removes the image loaded using LoadImage()
ImageRemove(imageID ImageID) error
ContainerState() (ContainerState, error)
- // CreateContainer will prepare anything in the creation on the container
+ // CreateContainer prepares anything in the creation on the container
// env is an array of string "KEY=VALUE" that represents the environment variables
// containerConfig has all the parameters needed to start the container
- // in the past we also had
- // - volumes: Now this is ExecOptions.mounts
- // - hostConfig: Now this is ExecOptions.enableNetwork and others
- // this are called when StartContainer() is executed.
- // this will also do any prep work needed to Std*Pipe() to work
- CreateContainer(env []string, execOptions ExecOptions, containerConfig ContainerConfig) error
-
- // StartContainer will start the container little questions asked
+ CreateContainer(containerConfig ContainerConfig) error
+
+ // StartContainer starts the container
StartContainer() error
// Kill the container and optionally remove the underlying image returns an
- // error if it didn't work (including timeout)
+ // error including timeout errors
Kill() error
- // this is similar how https://golang.org/pkg/os/exec/#Cmd does it.
+ // This is how https://golang.org/pkg/os/exec/#Cmd does it.
StdinPipe() (io.WriteCloser, error)
StdoutPipe() (io.ReadCloser, error)
StderrPipe() (io.ReadCloser, error)
// Wait for the container to finish
- Wait()
+ Wait() error
- // Returns the exit coui
+ // Returns the exit code of the last executed container
ExitCode() (int, error)
}
@@ -113,12 +120,6 @@ type ContainerExec struct {
// minimize the boilerplate for creating new ones.
}
-// ExecOptions are the options used in StartContainer()
-type ExecOptions struct {
- enableNetworking bool
- mounts []volumeMounts
-}
-
type volumeMounts struct {
hostPath string
containerPath string
diff --git a/lib/crunchrun/container_exec_test.go b/lib/crunchrun/container_exec_test.go
new file mode 100644
index 000000000..904b1f8c2
--- /dev/null
+++ b/lib/crunchrun/container_exec_test.go
@@ -0,0 +1,170 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&containerExecSuite{})
+
+type containerExecSuite struct {
+}
+
+func downloadFile(filepath string, url string) (err error) {
+ // Create the file
+ out, err := os.Create(filepath)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ // Get the data
+ resp, err := http.Get(url)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ // Check server response
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("bad status: %s", resp.Status)
+ }
+
+ // Writer the body to file
+ _, err = io.Copy(out, resp.Body)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *containerExecSuite) SetUpTest(c *check.C) {
+ // Download to ./fixtures/busybox_uclibc.tar from
+ // http://cache.arvados.org/busybox_uclibc.tar if it's not there.
+
+ _, err := os.Stat("./fixtures/busybox_uclibc.tar")
+ if err != nil {
+ err := downloadFile("./fixtures/busybox_uclibc.tar", "http://cache.arvados.org/busybox_uclibc.tar")
+ c.Check(err, check.IsNil)
+ }
+}
+
+func (s *containerExecSuite) TearDownTest(c *check.C) {
+}
+
+func (s *containerExecSuite) TestContainerExecLoadImage(c *check.C) {
+ ce, err := NewSingularityClient()
+ c.Check(err, check.IsNil)
+
+ c.Check(ce.CheckImageIsLoaded("InvalidImageID"), check.Equals, false)
+ f, err := os.Open("./fixtures/hello.tar")
+ c.Check(err, check.IsNil)
+ defer f.Close()
+ imageID, err := ce.LoadImage(f)
+ c.Check(err, check.IsNil)
+ loaded := ce.CheckImageIsLoaded(imageID)
+ c.Check(loaded, check.Equals, true)
+ err = ce.ImageRemove(imageID)
+ c.Check(err, check.IsNil)
+ loaded = ce.CheckImageIsLoaded(imageID)
+ c.Check(loaded, check.Equals, false)
+}
+
+func (s *containerExecSuite) TestContainerExecRunContainer(c *check.C) {
+ // tempfiles c.MkDir() NOTE...
+ // This file will be a random file with known content
+ file, err := ioutil.TempFile("/tmp", "arvados_test")
+ c.Check(err, check.IsNil)
+ defer os.Remove(file.Name())
+ io.Copy(file, bytes.NewBufferString("known content from a file in the host"))
+ err = file.Close()
+ c.Check(err, check.IsNil)
+ localKnownFile := file.Name()
+
+ c.Check(err, check.IsNil)
+ for _, trial := range []struct {
+ env []string
+ cmd []string
+ mounts []volumeMounts
+ enableNetworking bool
+ expectedStdout string
+ expectedStderr string
+ }{
+
+ {
+ // trying environment variable replacement
+ env: []string{"FOO=BAR"},
+ cmd: []string{"echo", "hello", "$FOO"},
+ mounts: []volumeMounts{},
+ expectedStdout: "hello BAR\n",
+ expectedStderr: "",
+ },
+
+ {
+ env: []string{},
+ cmd: []string{"cat", "/testFile.txt"},
+ mounts: []volumeMounts{
+ {
+ hostPath: localKnownFile,
+ containerPath: "/testFile.txt",
+ readonly: true,
+ },
+ },
+
+ expectedStdout: "known content from a file in the host",
+ expectedStderr: "",
+ },
+ } {
+ containerCfg := &ContainerConfig{
+ Env: trial.env,
+ Cmd: trial.cmd,
+ Mounts: trial.mounts,
+ EnableNetworking: trial.enableNetworking,
+ }
+ f, err := os.Open("./fixtures/busybox_uclibc.tar")
+ c.Check(err, check.IsNil)
+ defer f.Close()
+
+ ce, err := NewSingularityClient()
+ c.Check(err, check.IsNil)
+
+ imageID, err := ce.LoadImage(f)
+ fmt.Printf("IMAGE %s", string(imageID))
+ defer ce.ImageRemove(imageID)
+
+ err = ce.CreateContainer(*containerCfg)
+ c.Check(err, check.IsNil)
+ stdout, err := ce.StdoutPipe()
+ c.Check(err, check.IsNil)
+ defer stdout.Close()
+
+ stderr, err := ce.StderrPipe()
+ c.Check(err, check.IsNil)
+ defer stderr.Close()
+
+ err = ce.StartContainer()
+ c.Check(err, check.IsNil)
+
+ buf := new(bytes.Buffer)
+ buf.ReadFrom(stdout)
+ c.Check(buf.String(), check.Equals, trial.expectedStdout)
+
+ buf2 := new(bytes.Buffer)
+ buf2.ReadFrom(stderr)
+ c.Check(buf2.String(), check.Equals, trial.expectedStderr)
+
+ err = ce.Wait()
+ c.Check(err, check.IsNil)
+ }
+}
diff --git a/lib/crunchrun/fixtures/hello.tar b/lib/crunchrun/fixtures/hello.tar
new file mode 100644
index 000000000..6cfaab5ff
Binary files /dev/null and b/lib/crunchrun/fixtures/hello.tar differ
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
new file mode 100644
index 000000000..7d7ef3c1a
--- /dev/null
+++ b/lib/crunchrun/singularity.go
@@ -0,0 +1,196 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+ "bytes"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+type SingularityClient struct {
+ exec.Cmd
+ containerConfig ContainerConfig
+ imageSIFLocation ImageID
+ scratchSIFDirectory string
+}
+
+func (c *SingularityClient) CheckImageIsLoaded(imageID ImageID) bool {
+ if c.imageSIFLocation == "" {
+ return false
+ }
+ if _, err := os.Stat(string(c.imageSIFLocation)); err != nil {
+ return false
+ }
+ return true
+
+}
+
+// LoadImage will satisfy ContainerExecuter interface transforming
+// containerImage into a sif file for later use.
+func (c *SingularityClient) LoadImage(containerImage io.Reader) (ImageID, error) {
+ randBytes := make([]byte, 16)
+ rand.Read(randBytes)
+ tempFilePrefix := filepath.Join(c.scratchSIFDirectory, hex.EncodeToString(randBytes)) //ioutil.TempFile maybe is better?
+
+ sifFile := tempFilePrefix + ".sif"
+ tarFile := tempFilePrefix + ".tar"
+
+ f, err := os.OpenFile(tarFile, os.O_WRONLY|os.O_CREATE, 0600)
+ if err != nil {
+ return "", err
+ }
+ io.Copy(f, containerImage)
+ err = f.Close()
+ if err != nil {
+ return "", err
+ }
+ archiveRef := fmt.Sprintf("docker-archive://%s", tarFile)
+
+ buildCommand := exec.Cmd{
+ Path: "/usr/bin/singularity",
+ Args: []string{"/usr/bin/singularity", "build", sifFile, archiveRef},
+ }
+
+ buildCommand.Stdin = containerImage
+ var out bytes.Buffer
+ buildCommand.Stdout = &out
+ var errBuf bytes.Buffer
+ buildCommand.Stderr = &errBuf
+
+ err = buildCommand.Run()
+ if err != nil {
+ fmt.Printf("%#v", buildCommand.Args)
+ fmt.Printf("OUTPUT:'%s'\n", out.String())
+ fmt.Printf("ERROR:'%s'\n", errBuf.String())
+ return ImageID(""), err
+ }
+ //review: should we checkout out "out" for extra errors?
+ // this is roughfly a successful output in singulariry
+ // INFO: Starting build...
+ // Getting image source signatures
+ // Copying blob ab15617702de done
+ // Copying config 651e02b8a2 done
+ // Writing manifest to image destination
+ // Storing signatures
+ // 2021/04/22 14:42:14 info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
+ // INFO: Creating SIF file...
+ // INFO: Build complete: arvados-jobs.latest.sif
+ if err := os.Remove(string(tarFile)); err != nil {
+ return ImageID(""), err
+ }
+ c.imageSIFLocation = ImageID(sifFile)
+
+ return c.imageSIFLocation, nil
+}
+
+func (c *SingularityClient) ImageRemove(imageID ImageID) error {
+ if c.imageSIFLocation == "" {
+ return errors.New("No image loaded")
+ }
+
+ if _, err := os.Stat(string(c.imageSIFLocation)); err != nil {
+ return fmt.Errorf("Image '%s' is invalid", string(imageID))
+ }
+
+ if err := os.Remove(string(c.imageSIFLocation)); err != nil {
+ return err
+ }
+ c.imageSIFLocation = ""
+ return nil
+}
+
+func (c *SingularityClient) ContainerState() (ContainerState, error) {
+
+ return ContainerState{}, nil
+}
+func (c *SingularityClient) CreateContainer(containerConfig ContainerConfig) error {
+ c.containerConfig = containerConfig
+ // maybe we should ne doing extra checks if mounts don't exist for example
+ return nil
+}
+
+func (c *SingularityClient) StartContainer() error {
+ c.Cmd.Path = "/usr/bin/singularity"
+ if c.containerConfig.EnableNetworking {
+ c.Cmd.Args = []string{"/usr/bin/singularity", "run", "--contain", string(c.imageSIFLocation)}
+ } else {
+ c.Cmd.Args = []string{"/usr/bin/singularity", "run", "--nonet", "--contain"}
+ }
+
+ for _, mounts := range c.containerConfig.Mounts {
+ var opt string
+ if mounts.readonly {
+ opt = "ro"
+ } else {
+ opt = "rw"
+ }
+ // this might need coma separated
+ c.Cmd.Args = append(c.Cmd.Args, "--bind", fmt.Sprintf("%s:%s:%s", mounts.hostPath, mounts.containerPath, opt))
+ }
+ c.Cmd.Args = append(c.Cmd.Args, string(c.imageSIFLocation)) // image should be the last parameter before the external command
+ c.Cmd.Args = append(c.Cmd.Args, c.containerConfig.Cmd...)
+
+ // keyEqualsValuePair is assumed to be strings in the form "foo=bar" to
+ // inject in the container environment
+ for _, keyEqualsValuePair := range c.containerConfig.Env {
+ // there are some behaviours that changed in singularity 3.6, please see:
+ // https://sylabs.io/guides/3.7/user-guide/environment_and_metadata.html
+ // https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
+ c.Cmd.Env = append(c.Cmd.Env, fmt.Sprintf("SINGULARITYENV_%s", keyEqualsValuePair))
+ }
+ return c.Cmd.Start()
+}
+
+func (c *SingularityClient) Kill() error {
+ return nil
+}
+
+/*
+func (c *SingularityClient) StdinPipe() (io.WriteCloser, error) {
+ var x io.WriteCloser = (*os.File)(nil)
+ return x, nil
+}
+
+func (c *SingularityClient) StdoutPipe() (io.ReadCloser, error) {
+ var x io.ReadCloser = (*os.File)(nil)
+ return x, nil
+}
+
+func (c *SingularityClient) StderrPipe() (io.ReadCloser, error) {
+ var x io.ReadCloser = (*os.File)(nil)
+ return x, nil
+}
+func (c *SingularityClient) Wait() {
+}
+*/
+
+func (c *SingularityClient) ExitCode() (int, error) {
+ return c.ProcessState.ExitCode(), nil
+}
+
+// NewSingularityClient creates client that satisfy the ContainerExecuter interface
+// to install singularity in ArvBox (debian buster) do the following:
+// echo 'deb http://httpredir.debian.org/debian unstable main' > /etc/apt/sources.list.d/debian-unstable.list
+// cat > /etc/apt/preferences.d/pin-stable << EOF
+// Package: *
+// Pin: release stable
+// Pin-Priority: 600
+// EOF
+// apt update
+// apt install -y singularity-container/unstable
+
+func NewSingularityClient() (*SingularityClient, error) {
+ var s = &SingularityClient{}
+ // TODO: define if this should be a parameter.
+ s.scratchSIFDirectory = os.TempDir()
+ return s, nil
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list