[ARVADOS] created: 1.1.1-154-gbff2da3

[Git user]

        at  bff2da32c5bdbc437e4453830efee53be48287c7 (commit)


commit bff2da32c5bdbc437e4453830efee53be48287c7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Dec 7 09:51:25 2017 -0500

    12764: Writable file/directory support wip
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f3f754b..498a1f5 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -384,6 +384,37 @@ func (runner *ContainerRunner) SetupArvMountPoint(prefix string) (err error) {
 	return
 }
 
+func copyfile(src string, dst string) (err error) {
+	srcfile, err := os.Open(src)
+	if err != nil {
+		return
+	}
+
+	os.MkdirAll(path.Dir(dst), 0770)
+
+	dstfile, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	_, err = io.Copy(srcfile, dstfile)
+	if err != nil {
+		return
+	}
+
+	err = srcfile.Close()
+	err2 := dstfile.Close()
+
+	if err != nil {
+		return
+	}
+
+	if err2 != nil {
+		return err2
+	}
+
+	return nil
+}
+
 func (runner *ContainerRunner) SetupMounts() (err error) {
 	err = runner.SetupArvMountPoint("keep")
 	if err != nil {
@@ -406,6 +437,11 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 	runner.Binds = nil
 	runner.Volumes = make(map[string]struct{})
 	needCertMount := true
+	type copyFile struct {
+		src  string
+		bind string
+	}
+	var copyFiles []copyFile
 
 	var binds []string
 	for bind := range runner.Container.Mounts {
@@ -461,7 +497,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 				pdhOnly = false
 				src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
 			} else if mnt.PortableDataHash != "" {
-				if mnt.Writable {
+				if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
 					return fmt.Errorf("Can never write to a collection specified by portable data hash")
 				}
 				idx := strings.Index(mnt.PortableDataHash, "/")
@@ -488,10 +524,12 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 			if mnt.Writable {
 				if bind == runner.Container.OutputPath {
 					runner.HostOutputDir = src
+					runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
 				} else if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
-					return fmt.Errorf("Writable mount points are not permitted underneath the output_path: %v", bind)
+					copyFiles = append(copyFiles, copyFile{src, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
+				} else {
+					runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
 				}
-				runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
 			} else {
 				runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
 			}
@@ -579,6 +617,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 		}
 	}
 
+	for _, cp := range copyFiles {
+		dir, err := os.Stat(cp.src)
+		if err == nil {
+			if dir.IsDir() {
+				err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
+					return copyfile(walkpath, path.Join(cp.bind, walkpath))
+				})
+			} else {
+				err = copyfile(cp.src, cp.bind)
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("While staging writable files: %v", err)
+		}
+	}
+
 	return nil
 }
 
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index e1d9fed..0557e57 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1209,20 +1209,25 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 		checkEmpty()
 	}
 
-	// Writable mount points are not allowed underneath output_dir mount point
+	// Writable mount points copied to output_dir mount point
 	{
 		i = 0
 		cr.ArvMountPoint = ""
 		cr.Container.Mounts = make(map[string]arvados.Mount)
 		cr.Container.Mounts = map[string]arvados.Mount{
-			"/tmp":     {Kind: "tmp"},
-			"/tmp/foo": {Kind: "collection", Writable: true},
+			"/tmp": {Kind: "tmp"},
+			"/tmp/foo": {Kind: "collection",
+				PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
+				Writable:         true},
 		}
 		cr.OutputPath = "/tmp"
 
+		os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
+
 		err := cr.SetupMounts()
-		c.Check(err, NotNil)
-		c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
+		c.Check(err, IsNil)
+		_, err = os.Stat(cr.HostOutputDir + "/foo")
+		c.Check(err, IsNil)
 		os.RemoveAll(cr.ArvMountPoint)
 		cr.CleanupDirs()
 		checkEmpty()

-----------------------------------------------------------------------


hooks/post-receive
--