[ARVADOS] updated: 3afc0c377d2859d0aba622d1883af771b1b62594

[Git user]

Summary of changes:
 services/crunch-run/crunchrun.go      | 11 ++++++-----
 services/crunch-run/crunchrun_test.go | 15 ++++++++++++++-
 services/crunch-run/upload.go         | 15 +++++++++++++++
 3 files changed, 35 insertions(+), 6 deletions(-)

       via  3afc0c377d2859d0aba622d1883af771b1b62594 (commit)
      from  560e7071aa040e7ffef1d596fc3f5ae3c6233975 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 3afc0c377d2859d0aba622d1883af771b1b62594
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Sep 29 16:06:59 2017 -0400

    12183: Support uploading symlinks to directories.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index db2a892..b467c4f 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -915,12 +915,12 @@ func (runner *ContainerRunner) WaitFinish() (err error) {
 // they must remain within the output directory.
 //
 // Assumes initial value of "path" is absolute, and located within runner.HostOutputDir.
-func (runner *ContainerRunner) EvalSymlinks(path string, binds []string, symlinksToRemove *[]string) (manifestText string, err error) {
+func (runner *ContainerRunner) EvalSymlinks(path string, binds []string) (manifestText string, symlinksToRemove []string, err error) {
 	var links []string
 
 	defer func() {
 		if err != nil {
-			*symlinksToRemove = append(*symlinksToRemove, links...)
+			symlinksToRemove = append(symlinksToRemove, links...)
 		}
 	}()
 
@@ -978,7 +978,7 @@ func (runner *ContainerRunner) EvalSymlinks(path string, binds []string, symlink
 						return
 					}
 					manifestText = manifestText + m
-					*symlinksToRemove = append(*symlinksToRemove, l)
+					symlinksToRemove = append(symlinksToRemove, l)
 				}
 				return
 			}
@@ -1053,18 +1053,19 @@ func (runner *ContainerRunner) CaptureOutput() error {
 
 		var symlinksToRemove []string
 		var m string
+		var srm []string
 		// Find symlinks to arv-mounted files & dirs.
 		err = filepath.Walk(runner.HostOutputDir, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
-			m, err = runner.EvalSymlinks(path, binds, &symlinksToRemove)
+			m, srm, err = runner.EvalSymlinks(path, binds)
+			symlinksToRemove = append(symlinksToRemove, srm...)
 			if err == nil {
 				manifestText = manifestText + m
 			}
 			return err
 		})
-		runner.CrunchLog.Printf("sm %q", symlinksToRemove)
 		for _, l := range symlinksToRemove {
 			os.Remove(l)
 		}
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 0cb8aa4..4a94648 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1534,10 +1534,18 @@ func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
 		rf, _ := os.Create(t.realTemp + "/2/realfile")
 		rf.Write([]byte("foo"))
 		rf.Close()
+
+		os.Mkdir(t.realTemp+"/2/realdir", 0700)
+		rf, _ = os.Create(t.realTemp + "/2/realdir/subfile")
+		rf.Write([]byte("bar"))
+		rf.Close()
+
 		os.Symlink("/tmp/realfile", t.realTemp+"/2/file1")
 		os.Symlink("realfile", t.realTemp+"/2/file2")
 		os.Symlink("/tmp/file1", t.realTemp+"/2/file3")
 		os.Symlink("file2", t.realTemp+"/2/file4")
+		os.Symlink("realdir", t.realTemp+"/2/dir1")
+		os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2")
 		t.logWriter.Close()
 	})
 
@@ -1548,7 +1556,12 @@ func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
 			collection := v["collection"].(arvadosclient.Dict)
 			if strings.Index(collection["name"].(string), "output") == 0 {
 				manifest := collection["manifest_text"].(string)
-				c.Check(manifest, Equals, ". 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile\n")
+				c.Check(manifest, Equals,
+					`. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile
+./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+`)
 			}
 		}
 	}
diff --git a/services/crunch-run/upload.go b/services/crunch-run/upload.go
index bb2776a..faf20b4 100644
--- a/services/crunch-run/upload.go
+++ b/services/crunch-run/upload.go
@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -282,6 +283,20 @@ func (m *WalkUpload) WalkFunc(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("stat symlink %q target %q: %s", path, targetPath, err)
 		}
+		if targetInfo.Mode()&os.ModeDir != 0 {
+			// Symlinks to directories don't get walked, so do it
+			// here.  We've previously checked that they stay in
+			// the output directory and don't result in an endless
+			// loop.
+			var rd []os.FileInfo
+			rd, err = ioutil.ReadDir(path)
+			if err != nil {
+				return err
+			}
+			for _, ent := range rd {
+				err = filepath.Walk(filepath.Join(path, ent.Name()), m.WalkFunc)
+			}
+		}
 	}
 
 	if targetInfo.Mode()&os.ModeType != 0 {

-----------------------------------------------------------------------


hooks/post-receive
--