[ARVADOS] created: 34f9129a3d7d2a625455fccbd01c94fc18f6685a
Git user
git at public.curoverse.com
Mon May 15 15:52:06 EDT 2017
at 34f9129a3d7d2a625455fccbd01c94fc18f6685a (commit)
commit 34f9129a3d7d2a625455fccbd01c94fc18f6685a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 15 15:52:00 2017 -0400
11693: Add tests.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index a761619..5a89f6a 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -953,9 +953,8 @@ func (runner *ContainerRunner) CaptureOutput() error {
}
// go through mounts and reverse map to collection reference
for _, bind := range binds {
- if tgt == bind || strings.HasPrefix(tgt, bind+"/") {
- mnt := runner.Container.Mounts[bind]
-
+ mnt := runner.Container.Mounts[bind]
+ if tgt == bind || strings.HasPrefix(tgt, bind+"/") && mnt.Kind == "collection" {
// get path relative to bind
sourceSuffix := tgt[len(bind):]
// get path relative to output dir
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index c842756..2b2c232 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"io/ioutil"
+ "log"
"net"
"os"
"os/exec"
@@ -84,6 +85,7 @@ type TestDockerClient struct {
cwd string
env []string
api *ArvTestClient
+ realTemp string
}
func NewTestDockerClient(exitCode int) *TestDockerClient {
@@ -610,6 +612,8 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f
c.Assert(err, IsNil)
defer os.RemoveAll(realTemp)
+ docker.realTemp = realTemp
+
tempcount := 0
cr.MkTempDir = func(_ string, prefix string) (string, error) {
tempcount++
@@ -1419,6 +1423,47 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
}
}
+func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
+ helperRecord := `{
+ "command": ["/bin/sh", "-c", "echo $FROBIZ"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "environment": {"FROBIZ": "bilbo"},
+ "mounts": {
+ "/tmp": {"kind": "tmp"},
+ "/keep/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
+ "/keep/foo2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ extraMounts := []string{
+ "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
+ }
+
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ log.Printf("realtemp is %v", t.realTemp)
+ os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz")
+ os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2")
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ for _, v := range api.Content {
+ if v["collection"] != nil {
+ collection := v["collection"].(arvadosclient.Dict)
+ if strings.Index(collection["name"].(string), "output") == 0 {
+ manifest := collection["manifest_text"].(string)
+ c.Check(manifest, Equals, `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:baz 9:18:baz2
+`)
+ }
+ }
+ }
+}
+
func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
commit 1903e0e26b3677d9686e1d19cea897690945e3ed
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 15 14:06:33 2017 -0400
11693: Dereference symlinks to input files and add incorporate them into output collection manifest.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 2cd3417..a761619 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -769,14 +769,14 @@ func (runner *ContainerRunner) getStdoutFile(mntPath string) (*os.File, error) {
if err != nil {
return nil, fmt.Errorf("While Stat on temp dir: %v", err)
}
- stdoutPath := path.Join(runner.HostOutputDir, subdirs)
+ stdoutPath := filepath.Join(runner.HostOutputDir, subdirs)
err = os.MkdirAll(stdoutPath, st.Mode()|os.ModeSetgid|0777)
if err != nil {
return nil, fmt.Errorf("While MkdirAll %q: %v", stdoutPath, err)
}
}
}
- stdoutFile, err := os.Create(path.Join(runner.HostOutputDir, stdoutPath))
+ stdoutFile, err := os.Create(filepath.Join(runner.HostOutputDir, stdoutPath))
if err != nil {
return nil, fmt.Errorf("While creating file %q: %v", stdoutPath, err)
}
@@ -919,14 +919,74 @@ func (runner *ContainerRunner) CaptureOutput() error {
return fmt.Errorf("While checking host output path: %v", err)
}
+ // Pre-populate output from the configured mount points
+ var binds []string
+ for bind, _ := range runner.Container.Mounts {
+ binds = append(binds, bind)
+ }
+ sort.Strings(binds)
+
var manifestText string
collectionMetafile := fmt.Sprintf("%s/.arvados#collection", runner.HostOutputDir)
_, err = os.Stat(collectionMetafile)
if err != nil {
// Regular directory
+
+ // Find symlinks to arv-mounted files & dirs.
+ err = filepath.Walk(runner.HostOutputDir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.Mode()&os.ModeSymlink == 0 {
+ return nil
+ }
+ // read link to get container internal path
+ var tgt string
+ tgt, err = os.Readlink(path)
+ if err != nil {
+ return err
+ }
+ if !strings.HasPrefix(tgt, "/") {
+ // Link is relative, don't handle it
+ return nil
+ }
+ // go through mounts and reverse map to collection reference
+ for _, bind := range binds {
+ if tgt == bind || strings.HasPrefix(tgt, bind+"/") {
+ mnt := runner.Container.Mounts[bind]
+
+ // get path relative to bind
+ sourceSuffix := tgt[len(bind):]
+ // get path relative to output dir
+ bindSuffix := path[len(runner.HostOutputDir):]
+
+ // Copy mount and adjust the path to add path relative to the bind
+ adjustedMount := mnt
+ adjustedMount.Path = filepath.Join(adjustedMount.Path, sourceSuffix)
+
+ // get manifest text
+ var m string
+ m, err = runner.getCollectionManifestForPath(adjustedMount, bindSuffix)
+ if err != nil {
+ return err
+ }
+ manifestText = manifestText + m
+ // delete symlink so WriteTree won't try to to dereference it.
+ os.Remove(path)
+ return nil
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("While checking output symlinks: %v", err)
+ }
+
cw := CollectionWriter{0, runner.Kc, nil, nil, sync.Mutex{}}
- manifestText, err = cw.WriteTree(runner.HostOutputDir, runner.CrunchLog.Logger)
+ var m string
+ m, err = cw.WriteTree(runner.HostOutputDir, runner.CrunchLog.Logger)
+ manifestText = manifestText + m
if err != nil {
return fmt.Errorf("While uploading output files: %v", err)
}
@@ -946,13 +1006,6 @@ func (runner *ContainerRunner) CaptureOutput() error {
manifestText = rec.ManifestText
}
- // Pre-populate output from the configured mount points
- var binds []string
- for bind, _ := range runner.Container.Mounts {
- binds = append(binds, bind)
- }
- sort.Strings(binds)
-
for _, bind := range binds {
mnt := runner.Container.Mounts[bind]
commit bbe86c4a80d53807b325b46dd51557a7a01670ae
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon May 15 10:19:20 2017 -0400
11693: Mark container as cancelled if there is an error during finalization.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 0793729..2cd3417 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1190,6 +1190,10 @@ func (runner *ContainerRunner) Run() (err error) {
if err == nil {
err = e
}
+ if runner.finalState == "Complete" {
+ // There was an error in the finalization.
+ runner.finalState = "Cancelled"
+ }
}
// Log the error encountered in Run(), if any
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list