[ARVADOS] updated: 2.1.0-624-ge587495bf

Git user git at public.arvados.org
Thu Apr 29 20:02:21 UTC 2021


Summary of changes:
 lib/crunchrun/container_exec.go      |  49 ++++-----
 lib/crunchrun/container_exec_test.go | 170 ++++++++++++++++++++++++++++++
 lib/crunchrun/fixtures/hello.tar     | Bin 0 -> 11264 bytes
 lib/crunchrun/singularity.go         | 196 +++++++++++++++++++++++++++++++++++
 4 files changed, 391 insertions(+), 24 deletions(-)
 create mode 100644 lib/crunchrun/container_exec_test.go
 create mode 100644 lib/crunchrun/fixtures/hello.tar
 create mode 100644 lib/crunchrun/singularity.go

       via  e587495bf03b354e68ae64abe7969d1cb7c66c83 (commit)
      from  43b39915bae3f3c24ab31cfbc7aefdce88f84dcb (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit e587495bf03b354e68ae64abe7969d1cb7c66c83
Author: Nico Cesar <nico at nicocesar.com>
Date:   Thu Apr 29 16:01:24 2021 -0400

    container_exec.go and test with busybox_uclibc.tar download
    
    Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>

diff --git a/lib/crunchrun/container_exec.go b/lib/crunchrun/container_exec.go
index 809446d73..b65e068a4 100644
--- a/lib/crunchrun/container_exec.go
+++ b/lib/crunchrun/container_exec.go
@@ -55,48 +55,55 @@ type ContainerState struct {
 }
 
 type ContainerConfig struct {
-	Env    []string       // List of environment variable to set in the container "VALUE=KEY" format
-	Cmd    []string       // Command to run when starting the container
-	Mounts []volumeMounts // List of Mounts used for the container.
-
+	Env              []string       // List of environment variable to set in the container "VALUE=KEY" format
+	Cmd              []string       // Command to run when starting the container
+	Mounts           []volumeMounts // List of Mounts used for the container.
+	EnableNetworking bool           // This will allow the container to reach the outside world
 }
 
 // ContainerExecuter interface will implement all the methods needed for Docker,
 // Singularity, and others to load images, run containers, get thier outputs,
 // etc.
 type ContainerExecuter interface {
+	// CheckImageIsLoaded checks if imageID is already in the local environment,
+	// either something we can reference later in Docker API or similar, or a
+	// filepath containing the image to be used
 	CheckImageIsLoaded(imageID ImageID) bool
-	LoadImage(containerImage io.Reader) error
+
+	// LoadImage translates a io.Reader that has a tarball in docker format
+	// (Usually created with 'docker save'), and load it to the local
+	// ContainerExecuter.
+	//
+	// Returns an ImageID to be referenced later, could be an identifier or a
+	// filepath or something else
+	LoadImage(containerImage io.Reader) (ImageID, error)
+
+	// ImageRemove removes the image loaded using LoadImage()
 	ImageRemove(imageID ImageID) error
 
 	ContainerState() (ContainerState, error)
 
-	// CreateContainer will prepare anything in the creation on the container
+	// CreateContainer prepares anything in the creation on the container
 	// env is an array of string "KEY=VALUE" that represents the environment variables
 	// containerConfig has all the parameters needed to start the container
-	// in the past we also had
-	// - volumes: Now this is ExecOptions.mounts
-	// - hostConfig: Now this is ExecOptions.enableNetwork and others
-	// this are called when StartContainer() is executed.
-	// this will also do any prep work needed to Std*Pipe() to work
-	CreateContainer(env []string, execOptions ExecOptions, containerConfig ContainerConfig) error
-
-	// StartContainer will start the container little questions asked
+	CreateContainer(containerConfig ContainerConfig) error
+
+	// StartContainer starts the container
 	StartContainer() error
 
 	// Kill the container and optionally remove the underlying image returns an
-	// error if it didn't work (including timeout)
+	// error  including timeout errors
 	Kill() error
 
-	// this is similar how https://golang.org/pkg/os/exec/#Cmd does it.
+	// This is how https://golang.org/pkg/os/exec/#Cmd does it.
 	StdinPipe() (io.WriteCloser, error)
 	StdoutPipe() (io.ReadCloser, error)
 	StderrPipe() (io.ReadCloser, error)
 
 	// Wait for the container to finish
-	Wait()
+	Wait() error
 
-	// Returns the exit coui
+	// Returns the exit code of the last executed container
 	ExitCode() (int, error)
 }
 
@@ -113,12 +120,6 @@ type ContainerExec struct {
 	// minimize the boilerplate for creating new ones.
 }
 
-// ExecOptions are the options used in StartContainer()
-type ExecOptions struct {
-	enableNetworking bool
-	mounts           []volumeMounts
-}
-
 type volumeMounts struct {
 	hostPath      string
 	containerPath string
diff --git a/lib/crunchrun/container_exec_test.go b/lib/crunchrun/container_exec_test.go
new file mode 100644
index 000000000..904b1f8c2
--- /dev/null
+++ b/lib/crunchrun/container_exec_test.go
@@ -0,0 +1,170 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&containerExecSuite{})
+
+type containerExecSuite struct {
+}
+
+func downloadFile(filepath string, url string) (err error) {
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Get the data
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Check server response
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad status: %s", resp.Status)
+	}
+
+	// Writer the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *containerExecSuite) SetUpTest(c *check.C) {
+	// Download to ./fixtures/busybox_uclibc.tar from
+	// http://cache.arvados.org/busybox_uclibc.tar if it's not there.
+
+	_, err := os.Stat("./fixtures/busybox_uclibc.tar")
+	if err != nil {
+		err := downloadFile("./fixtures/busybox_uclibc.tar", "http://cache.arvados.org/busybox_uclibc.tar")
+		c.Check(err, check.IsNil)
+	}
+}
+
+func (s *containerExecSuite) TearDownTest(c *check.C) {
+}
+
+func (s *containerExecSuite) TestContainerExecLoadImage(c *check.C) {
+	ce, err := NewSingularityClient()
+	c.Check(err, check.IsNil)
+
+	c.Check(ce.CheckImageIsLoaded("InvalidImageID"), check.Equals, false)
+	f, err := os.Open("./fixtures/hello.tar")
+	c.Check(err, check.IsNil)
+	defer f.Close()
+	imageID, err := ce.LoadImage(f)
+	c.Check(err, check.IsNil)
+	loaded := ce.CheckImageIsLoaded(imageID)
+	c.Check(loaded, check.Equals, true)
+	err = ce.ImageRemove(imageID)
+	c.Check(err, check.IsNil)
+	loaded = ce.CheckImageIsLoaded(imageID)
+	c.Check(loaded, check.Equals, false)
+}
+
+func (s *containerExecSuite) TestContainerExecRunContainer(c *check.C) {
+	// tempfiles c.MkDir() NOTE...
+	// This file will be a random file with known content
+	file, err := ioutil.TempFile("/tmp", "arvados_test")
+	c.Check(err, check.IsNil)
+	defer os.Remove(file.Name())
+	io.Copy(file, bytes.NewBufferString("known content from a file in the host"))
+	err = file.Close()
+	c.Check(err, check.IsNil)
+	localKnownFile := file.Name()
+
+	c.Check(err, check.IsNil)
+	for _, trial := range []struct {
+		env              []string
+		cmd              []string
+		mounts           []volumeMounts
+		enableNetworking bool
+		expectedStdout   string
+		expectedStderr   string
+	}{
+
+		{
+			// trying environment variable replacement
+			env:            []string{"FOO=BAR"},
+			cmd:            []string{"echo", "hello", "$FOO"},
+			mounts:         []volumeMounts{},
+			expectedStdout: "hello BAR\n",
+			expectedStderr: "",
+		},
+
+		{
+			env: []string{},
+			cmd: []string{"cat", "/testFile.txt"},
+			mounts: []volumeMounts{
+				{
+					hostPath:      localKnownFile,
+					containerPath: "/testFile.txt",
+					readonly:      true,
+				},
+			},
+
+			expectedStdout: "known content from a file in the host",
+			expectedStderr: "",
+		},
+	} {
+		containerCfg := &ContainerConfig{
+			Env:              trial.env,
+			Cmd:              trial.cmd,
+			Mounts:           trial.mounts,
+			EnableNetworking: trial.enableNetworking,
+		}
+		f, err := os.Open("./fixtures/busybox_uclibc.tar")
+		c.Check(err, check.IsNil)
+		defer f.Close()
+
+		ce, err := NewSingularityClient()
+		c.Check(err, check.IsNil)
+
+		imageID, err := ce.LoadImage(f)
+		fmt.Printf("IMAGE %s", string(imageID))
+		defer ce.ImageRemove(imageID)
+
+		err = ce.CreateContainer(*containerCfg)
+		c.Check(err, check.IsNil)
+		stdout, err := ce.StdoutPipe()
+		c.Check(err, check.IsNil)
+		defer stdout.Close()
+
+		stderr, err := ce.StderrPipe()
+		c.Check(err, check.IsNil)
+		defer stderr.Close()
+
+		err = ce.StartContainer()
+		c.Check(err, check.IsNil)
+
+		buf := new(bytes.Buffer)
+		buf.ReadFrom(stdout)
+		c.Check(buf.String(), check.Equals, trial.expectedStdout)
+
+		buf2 := new(bytes.Buffer)
+		buf2.ReadFrom(stderr)
+		c.Check(buf2.String(), check.Equals, trial.expectedStderr)
+
+		err = ce.Wait()
+		c.Check(err, check.IsNil)
+	}
+}
diff --git a/lib/crunchrun/fixtures/hello.tar b/lib/crunchrun/fixtures/hello.tar
new file mode 100644
index 000000000..6cfaab5ff
Binary files /dev/null and b/lib/crunchrun/fixtures/hello.tar differ
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
new file mode 100644
index 000000000..7d7ef3c1a
--- /dev/null
+++ b/lib/crunchrun/singularity.go
@@ -0,0 +1,196 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+type SingularityClient struct {
+	exec.Cmd
+	containerConfig     ContainerConfig
+	imageSIFLocation    ImageID
+	scratchSIFDirectory string
+}
+
+func (c *SingularityClient) CheckImageIsLoaded(imageID ImageID) bool {
+	if c.imageSIFLocation == "" {
+		return false
+	}
+	if _, err := os.Stat(string(c.imageSIFLocation)); err != nil {
+		return false
+	}
+	return true
+
+}
+
+// LoadImage will satisfy ContainerExecuter interface transforming
+// containerImage into a sif file for later use.
+func (c *SingularityClient) LoadImage(containerImage io.Reader) (ImageID, error) {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	tempFilePrefix := filepath.Join(c.scratchSIFDirectory, hex.EncodeToString(randBytes)) //ioutil.TempFile maybe is better?
+
+	sifFile := tempFilePrefix + ".sif"
+	tarFile := tempFilePrefix + ".tar"
+
+	f, err := os.OpenFile(tarFile, os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return "", err
+	}
+	io.Copy(f, containerImage)
+	err = f.Close()
+	if err != nil {
+		return "", err
+	}
+	archiveRef := fmt.Sprintf("docker-archive://%s", tarFile)
+
+	buildCommand := exec.Cmd{
+		Path: "/usr/bin/singularity",
+		Args: []string{"/usr/bin/singularity", "build", sifFile, archiveRef},
+	}
+
+	buildCommand.Stdin = containerImage
+	var out bytes.Buffer
+	buildCommand.Stdout = &out
+	var errBuf bytes.Buffer
+	buildCommand.Stderr = &errBuf
+
+	err = buildCommand.Run()
+	if err != nil {
+		fmt.Printf("%#v", buildCommand.Args)
+		fmt.Printf("OUTPUT:'%s'\n", out.String())
+		fmt.Printf("ERROR:'%s'\n", errBuf.String())
+		return ImageID(""), err
+	}
+	//review: should we checkout out "out" for extra errors?
+	// this is roughfly a successful output in singulariry
+	// INFO:    Starting build...
+	// Getting image source signatures
+	// Copying blob ab15617702de done
+	// Copying config 651e02b8a2 done
+	// Writing manifest to image destination
+	// Storing signatures
+	// 2021/04/22 14:42:14  info unpack layer: sha256:21cbfd3a344c52b197b9fa36091e66d9cbe52232703ff78d44734f85abb7ccd3
+	// INFO:    Creating SIF file...
+	// INFO:    Build complete: arvados-jobs.latest.sif
+	if err := os.Remove(string(tarFile)); err != nil {
+		return ImageID(""), err
+	}
+	c.imageSIFLocation = ImageID(sifFile)
+
+	return c.imageSIFLocation, nil
+}
+
+func (c *SingularityClient) ImageRemove(imageID ImageID) error {
+	if c.imageSIFLocation == "" {
+		return errors.New("No image loaded")
+	}
+
+	if _, err := os.Stat(string(c.imageSIFLocation)); err != nil {
+		return fmt.Errorf("Image '%s' is invalid", string(imageID))
+	}
+
+	if err := os.Remove(string(c.imageSIFLocation)); err != nil {
+		return err
+	}
+	c.imageSIFLocation = ""
+	return nil
+}
+
+func (c *SingularityClient) ContainerState() (ContainerState, error) {
+
+	return ContainerState{}, nil
+}
+func (c *SingularityClient) CreateContainer(containerConfig ContainerConfig) error {
+	c.containerConfig = containerConfig
+	// maybe we should ne doing extra checks if mounts don't exist for example
+	return nil
+}
+
+func (c *SingularityClient) StartContainer() error {
+	c.Cmd.Path = "/usr/bin/singularity"
+	if c.containerConfig.EnableNetworking {
+		c.Cmd.Args = []string{"/usr/bin/singularity", "run", "--contain", string(c.imageSIFLocation)}
+	} else {
+		c.Cmd.Args = []string{"/usr/bin/singularity", "run", "--nonet", "--contain"}
+	}
+
+	for _, mounts := range c.containerConfig.Mounts {
+		var opt string
+		if mounts.readonly {
+			opt = "ro"
+		} else {
+			opt = "rw"
+		}
+		// this might need coma separated
+		c.Cmd.Args = append(c.Cmd.Args, "--bind", fmt.Sprintf("%s:%s:%s", mounts.hostPath, mounts.containerPath, opt))
+	}
+	c.Cmd.Args = append(c.Cmd.Args, string(c.imageSIFLocation)) // image should be the last parameter before the external command
+	c.Cmd.Args = append(c.Cmd.Args, c.containerConfig.Cmd...)
+
+	// keyEqualsValuePair is assumed to be strings in the form "foo=bar" to
+	// inject in the container environment
+	for _, keyEqualsValuePair := range c.containerConfig.Env {
+		// there are some behaviours that changed in singularity 3.6, please see:
+		// https://sylabs.io/guides/3.7/user-guide/environment_and_metadata.html
+		// https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html
+		c.Cmd.Env = append(c.Cmd.Env, fmt.Sprintf("SINGULARITYENV_%s", keyEqualsValuePair))
+	}
+	return c.Cmd.Start()
+}
+
+func (c *SingularityClient) Kill() error {
+	return nil
+}
+
+/*
+func (c *SingularityClient) StdinPipe() (io.WriteCloser, error) {
+	var x io.WriteCloser = (*os.File)(nil)
+	return x, nil
+}
+
+func (c *SingularityClient) StdoutPipe() (io.ReadCloser, error) {
+	var x io.ReadCloser = (*os.File)(nil)
+	return x, nil
+}
+
+func (c *SingularityClient) StderrPipe() (io.ReadCloser, error) {
+	var x io.ReadCloser = (*os.File)(nil)
+	return x, nil
+}
+func (c *SingularityClient) Wait() {
+}
+*/
+
+func (c *SingularityClient) ExitCode() (int, error) {
+	return c.ProcessState.ExitCode(), nil
+}
+
+// NewSingularityClient creates  client that satisfy the ContainerExecuter interface
+// to install singularity in ArvBox (debian buster) do the following:
+// echo 'deb http://httpredir.debian.org/debian unstable main' > /etc/apt/sources.list.d/debian-unstable.list
+// cat > /etc/apt/preferences.d/pin-stable << EOF
+// Package: *
+// Pin: release stable
+// Pin-Priority: 600
+// EOF
+// apt update
+// apt install -y singularity-container/unstable
+
+func NewSingularityClient() (*SingularityClient, error) {
+	var s = &SingularityClient{}
+	// TODO: define if this should be a parameter.
+	s.scratchSIFDirectory = os.TempDir()
+	return s, nil
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list