[ARVADOS] updated: 1.1.3-338-g7466d0c

[Git user]

Summary of changes:
 services/crunch-run/copier.go      | 20 +++++++++++---------
 services/crunch-run/copier_test.go | 15 ++++++++++++---
 2 files changed, 23 insertions(+), 12 deletions(-)

       via  7466d0c1af1f7d9a0c1b4ae54e9f1bbc951f2711 (commit)
       via  bcb6f55e2edfddd9edfa08c6463013b89ed0b78d (commit)
      from  826bc4316d3afb0013b9951c37137323d0e00b08 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 7466d0c1af1f7d9a0c1b4ae54e9f1bbc951f2711
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Apr 9 10:19:59 2018 -0400

    13100: Handle writable collections mounted below output dir.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/crunch-run/copier.go b/services/crunch-run/copier.go
index b6093ce..3ba6eed 100644
--- a/services/crunch-run/copier.go
+++ b/services/crunch-run/copier.go
@@ -185,7 +185,7 @@ func (cp *copier) walkMountsBelow(dest, src string) error {
 		if !strings.HasPrefix(mnt, src+"/") {
 			continue
 		}
-		if mntinfo.Kind == "text" || mntinfo.Kind == "json" {
+		if cp.copyRegularFiles(mntinfo) {
 			// These got copied into the nearest parent
 			// mount as regular files during setup, so
 			// they get copied as regular files when we
@@ -284,15 +284,15 @@ func (cp *copier) walkHostFS(dest, src string, maxSymlinks int, includeMounts bo
 			if _, isSecret := cp.secretMounts[src]; isSecret {
 				continue
 			}
-			if mntinfo, isMount := cp.mounts[src]; isMount && mntinfo.Kind != "text" && mntinfo.Kind != "json" {
+			if mntinfo, isMount := cp.mounts[src]; isMount && !cp.copyRegularFiles(mntinfo) {
 				// If a regular file/dir somehow
 				// exists at a path that's also a
 				// mount target, ignore the file --
 				// the mount has already been included
 				// with walkMountsBelow().
 				//
-				// (...except json/text mounts, which
-				// are handled as regular files.)
+				// (...except mount types that are
+				// handled as regular files.)
 				continue
 			}
 			err = cp.walkHostFS(dest, src, maxSymlinks, false)
@@ -316,6 +316,10 @@ func (cp *copier) walkHostFS(dest, src string, maxSymlinks int, includeMounts bo
 	return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
 }
 
+func (cp *copier) copyRegularFiles(m arvados.Mount) bool {
+	return m.Kind == "text" || m.Kind == "json" || (m.Kind == "collection" && m.Writable)
+}
+
 func (cp *copier) getManifest(pdh string) (*manifest.Manifest, error) {
 	if mft, ok := cp.manifestCache[pdh]; ok {
 		return mft, nil
diff --git a/services/crunch-run/copier_test.go b/services/crunch-run/copier_test.go
index 7b13ef9..41c2acd 100644
--- a/services/crunch-run/copier_test.go
+++ b/services/crunch-run/copier_test.go
@@ -172,14 +172,23 @@ func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
 	c.Check(err, check.NotNil)
 }
 
-func (s *copierSuite) TestUnsupportedWritableMountBelow(c *check.C) {
-	s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{
+func (s *copierSuite) TestWritableMountBelow(c *check.C) {
+	s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
 		Kind:             "collection",
 		PortableDataHash: arvadostest.FooPdh,
 		Writable:         true,
 	}
+	c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
+	s.writeFileInOutputDir(c, "file", "file")
+	s.writeFileInOutputDir(c, "mount/foo", "foo")
+
 	err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
-	c.Check(err, check.NotNil)
+	c.Check(err, check.IsNil)
+	c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
+	c.Check(s.cp.files, check.DeepEquals, []filetodo{
+		{src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
+		{src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
+	})
 }
 
 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {

commit bcb6f55e2edfddd9edfa08c6463013b89ed0b78d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Apr 9 09:56:03 2018 -0400

    13100: Update comment.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/crunch-run/copier.go b/services/crunch-run/copier.go
index 273ec79..b6093ce 100644
--- a/services/crunch-run/copier.go
+++ b/services/crunch-run/copier.go
@@ -45,11 +45,9 @@ type filetodo struct {
 // Symlinks to other parts of the container's filesystem result in
 // errors.
 //
-// To use a copier: first call walkMount() to inspect the output
-// directory, grab the necessary parts of already-stored collections,
-// and prepare a list of files that need to be copied from the local
-// filesystem; then call commit() to copy the file data and return a
-// complete output manifest.
+// Use:
+//
+//	manifest, err := (&copier{...}).Copy()
 type copier struct {
 	client        *arvados.Client
 	arvClient     IArvadosClient

-----------------------------------------------------------------------


hooks/post-receive
--