[ARVADOS] updated: faf5784b78844e5da0b4c63860323eb1ee06bd2f
git at public.curoverse.com
git at public.curoverse.com
Sat Feb 13 21:50:00 EST 2016
Summary of changes:
services/crunch-run/crunchrun.go | 66 ++++++++++++++++++++---------------
services/crunch-run/crunchrun_test.go | 22 ++++++------
2 files changed, 49 insertions(+), 39 deletions(-)
via faf5784b78844e5da0b4c63860323eb1ee06bd2f (commit)
from 9eafed4c378012e914000658b30af4f8a1c51432 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit faf5784b78844e5da0b4c63860323eb1ee06bd2f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Sat Feb 13 21:49:55 2016 -0500
8015: Fix existing tests, still need tests for new functionality
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 5f63f5d..d197fa9 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -101,7 +101,8 @@ type ContainerRunner struct {
ArvMount *exec.Cmd
ArvMountPoint string
HostOutputDir string
- OutputPDH string
+ Binds []string
+ OutputPDH *string
CancelLock sync.Mutex
Cancelled bool
SigChan chan os.Signal
@@ -179,7 +180,7 @@ func (runner *ContainerRunner) LoadImage() (err error) {
return nil
}
-func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig) (err error) {
+func (runner *ContainerRunner) SetupMounts() (err error) {
runner.ArvMountPoint, err = ioutil.TempDir("", "keep")
if err != nil {
return fmt.Errorf("While creating keep mount temp dir: %v", err)
@@ -217,9 +218,9 @@ func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig)
if bind == runner.ContainerRecord.OutputPath {
runner.HostOutputDir = src
}
- hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s", src, bind))
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
} else {
- hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
}
collectionPaths = append(collectionPaths, src)
} else if mnt.Kind == "tmp" {
@@ -229,9 +230,9 @@ func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig)
return fmt.Errorf("While creating mount temp dir: %v", err)
}
- hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind))
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind))
} else {
- hostConfig.Binds = append(hostConfig.Binds, bind)
+ runner.Binds = append(runner.Binds, bind)
}
} else {
return fmt.Errorf("Unknown mount kind '%s'", mnt.Kind)
@@ -284,16 +285,12 @@ func (runner *ContainerRunner) StartContainer() (err error) {
for k, v := range runner.ContainerRecord.Environment {
runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v)
}
+ runner.ContainerConfig.NetworkDisabled = true
runner.ContainerID, err = runner.Docker.CreateContainer(&runner.ContainerConfig, "", nil)
if err != nil {
return fmt.Errorf("While creating container: %v", err)
}
- hostConfig := &dockerclient.HostConfig{}
-
- err = runner.SetupMounts(hostConfig)
- if err != nil {
- return fmt.Errorf("While setting up mounts: %v", err)
- }
+ hostConfig := &dockerclient.HostConfig{Binds: runner.Binds}
runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
err = runner.Docker.StartContainer(runner.ContainerID, hostConfig)
@@ -351,7 +348,7 @@ func (runner *ContainerRunner) WaitFinish() error {
}
// HandleOutput sets the output and unmounts the FUSE mount.
-func (runner *ContainerRunner) HandleOutput() error {
+func (runner *ContainerRunner) CaptureOutput() error {
if runner.ArvMount != nil {
defer func() {
umount := exec.Command("fusermount", "-z", "-u", runner.ArvMountPoint)
@@ -363,6 +360,10 @@ func (runner *ContainerRunner) HandleOutput() error {
return nil
}
+ if runner.HostOutputDir == "" {
+ return nil
+ }
+
_, err := os.Stat(runner.HostOutputDir)
if err != nil {
return fmt.Errorf("While checking host output path: %v", err)
@@ -403,7 +404,8 @@ func (runner *ContainerRunner) HandleOutput() error {
return fmt.Errorf("While creating output collection: %v", err)
}
- runner.OutputPDH = response.PortableDataHash
+ runner.OutputPDH = new(string)
+ *runner.OutputPDH = response.PortableDataHash
return nil
}
@@ -425,17 +427,17 @@ func (runner *ContainerRunner) CommitLogs() error {
return fmt.Errorf("While creating log manifest: %v", err)
}
- response := make(map[string]string)
+ var response CollectionRecord
err = runner.ArvClient.Create("collections",
arvadosclient.Dict{"name": "logs for " + runner.ContainerRecord.UUID,
"manifest_text": mt},
- response)
+ &response)
if err != nil {
return fmt.Errorf("While creating log collection: %v", err)
}
runner.LogsPDH = new(string)
- *runner.LogsPDH = response["portable_data_hash"]
+ *runner.LogsPDH = response.PortableDataHash
return nil
}
@@ -456,9 +458,11 @@ func (runner *ContainerRunner) UpdateContainerRecordComplete() error {
if runner.ExitCode != nil {
update["exit_code"] = *runner.ExitCode
}
+ if runner.OutputPDH != nil {
+ update["output"] = runner.OutputPDH
+ }
update["state"] = runner.finalState
- update["output"] = runner.OutputPDH
return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID, update, nil)
}
@@ -485,19 +489,19 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) {
runner.finalState = "Complete"
}
- // (6) handle output
- outputerr := runner.HandleOutput()
+ // (7) capture output
+ outputerr := runner.CaptureOutput()
if outputerr != nil {
runner.CrunchLog.Print(outputerr)
}
- // (7) write logs
+ // (8) write logs
logerr := runner.CommitLogs()
if logerr != nil {
runner.CrunchLog.Print(logerr)
}
- // (8) update container record with results
+ // (9) update container record with results
updateerr := runner.UpdateContainerRecordComplete()
if updateerr != nil {
runner.CrunchLog.Print(updateerr)
@@ -523,19 +527,25 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) {
return fmt.Errorf("While getting container record: %v", err)
}
- // (0) setup signal handling
+ // (1) setup signal handling
err = runner.SetupSignals()
if err != nil {
return fmt.Errorf("While setting up signal handling: %v", err)
}
- // (1) check for and/or load image
+ // (2) check for and/or load image
err = runner.LoadImage()
if err != nil {
return fmt.Errorf("While loading container image: %v", err)
}
- // (2) start container
+ // (3) set up FUSE mount and binds
+ err = runner.SetupMounts()
+ if err != nil {
+ return fmt.Errorf("While setting up mounts: %v", err)
+ }
+
+ // (3) create and start container
err = runner.StartContainer()
if err != nil {
if err == ErrCancelled {
@@ -544,19 +554,19 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) {
return
}
- // (3) update container record state
+ // (4) update container record state
err = runner.UpdateContainerRecordRunning()
if err != nil {
runner.CrunchLog.Print(err)
}
- // (4) attach container logs
+ // (5) attach container logs
runerr = runner.AttachLogs()
if runerr != nil {
runner.CrunchLog.Print(runerr)
}
- // (5) wait for container to finish
+ // (6) wait for container to finish
waiterr = runner.WaitFinish()
return
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 3c4dbf0..eaa16d1 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -151,8 +151,8 @@ func (this *ArvTestClient) Create(resourceType string,
if resourceType == "collections" && output != nil {
mt := parameters["manifest_text"].(string)
- outmap := output.(map[string]string)
- outmap["portable_data_hash"] = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
+ outmap := output.(*CollectionRecord)
+ outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
}
return nil
@@ -301,7 +301,7 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
cr.ContainerRecord.ContainerImage = hwPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "ArvError")
+ c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
}
func (s *TestSuite) TestLoadImageKeepError(c *C) {
@@ -311,7 +311,7 @@ func (s *TestSuite) TestLoadImageKeepError(c *C) {
cr.ContainerRecord.ContainerImage = hwPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "KeepError")
+ c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
}
func (s *TestSuite) TestLoadImageCollectionError(c *C) {
@@ -320,7 +320,7 @@ func (s *TestSuite) TestLoadImageCollectionError(c *C) {
cr.ContainerRecord.ContainerImage = otherPDH
err := cr.LoadImage()
- c.Check(err.Error(), Equals, "First file in the collection does not end in .tar")
+ c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
}
func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
@@ -490,7 +490,7 @@ func (s *TestSuite) TestFullRunHello(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
@@ -514,7 +514,7 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
@@ -540,7 +540,7 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
@@ -563,7 +563,7 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
@@ -586,7 +586,7 @@ func (s *TestSuite) TestCancel(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
"environment": {},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
@@ -641,7 +641,7 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
"environment": {"FROBIZ": "bilbo"},
- "mounts": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list