[ARVADOS] updated: 1.1.3-339-g163dac6

[Git user]

Summary of changes:
 services/crunch-run/copier.go      | 26 ++++++++++++++++++++++----
 services/crunch-run/copier_test.go | 23 +++++++++++++++++++++--
 services/crunch-run/crunchrun.go   |  1 +
 3 files changed, 44 insertions(+), 6 deletions(-)

       via  163dac6272b8b75e7c1b73076ab922df8b03728e (commit)
      from  7466d0c1af1f7d9a0c1b4ae54e9f1bbc951f2711 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 163dac6272b8b75e7c1b73076ab922df8b03728e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Mon Apr 9 11:13:30 2018 -0400

    13100: Handle output symlinks to writable collections.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/crunch-run/copier.go b/services/crunch-run/copier.go
index 3ba6eed..4c45f6a 100644
--- a/services/crunch-run/copier.go
+++ b/services/crunch-run/copier.go
@@ -54,6 +54,7 @@ type copier struct {
 	keepClient    IKeepClient
 	hostOutputDir string
 	ctrOutputDir  string
+	binds         []string
 	mounts        map[string]arvados.Mount
 	secretMounts  map[string]arvados.Mount
 	logger        printfer
@@ -157,8 +158,12 @@ func (cp *copier) walkMount(dest, src string, maxSymlinks int, walkMountsBelow b
 			return err
 		}
 		cp.manifest += mft.Extract(srcRelPath, dest).Text
-	case srcRoot == cp.ctrOutputDir:
-		f, err := os.Open(filepath.Join(cp.hostOutputDir, ".arvados#collection"))
+	default:
+		hostRoot, err := cp.hostRoot(srcRoot)
+		if err != nil {
+			return err
+		}
+		f, err := os.Open(filepath.Join(hostRoot, ".arvados#collection"))
 		if err != nil {
 			return err
 		}
@@ -170,8 +175,6 @@ func (cp *copier) walkMount(dest, src string, maxSymlinks int, walkMountsBelow b
 		}
 		mft := manifest.Manifest{Text: coll.ManifestText}
 		cp.manifest += mft.Extract(srcRelPath, dest).Text
-	default:
-		return fmt.Errorf("cannot output %q as %q: writable collection mounted at %q", src, dest, srcRoot)
 	}
 	if walkMountsBelow {
 		return cp.walkMountsBelow(dest, src)
@@ -316,6 +319,21 @@ func (cp *copier) walkHostFS(dest, src string, maxSymlinks int, includeMounts bo
 	return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
 }
 
+// Return the host path that was mounted at the given path in the
+// container.
+func (cp *copier) hostRoot(ctrRoot string) (string, error) {
+	if ctrRoot == cp.ctrOutputDir {
+		return cp.hostOutputDir, nil
+	}
+	for _, bind := range cp.binds {
+		tokens := strings.Split(bind, ":")
+		if len(tokens) >= 2 && tokens[1] == ctrRoot {
+			return tokens[0], nil
+		}
+	}
+	return "", fmt.Errorf("not bind-mounted: %q", ctrRoot)
+}
+
 func (cp *copier) copyRegularFiles(m arvados.Mount) bool {
 	return m.Kind == "text" || m.Kind == "json" || (m.Kind == "collection" && m.Writable)
 }
diff --git a/services/crunch-run/copier_test.go b/services/crunch-run/copier_test.go
index 41c2acd..d678cf6 100644
--- a/services/crunch-run/copier_test.go
+++ b/services/crunch-run/copier_test.go
@@ -108,16 +108,35 @@ func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 }
 
 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
+	// simulate mounted read-only collection
 	s.cp.mounts["/mnt"] = arvados.Mount{
 		Kind:             "collection",
 		PortableDataHash: arvadostest.FooPdh,
 	}
+
+	// simulate mounted writable collection
+	bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(bindtmp)
+	f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
+	c.Assert(err, check.IsNil)
+	_, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
+	c.Assert(err, check.IsNil)
+	c.Assert(f.Close(), check.IsNil)
+	s.cp.mounts["/mnt-w"] = arvados.Mount{
+		Kind:             "collection",
+		PortableDataHash: arvadostest.FooPdh,
+		Writable:         true,
+	}
+	s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w")
+
 	c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
 	c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
+	c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
 
-	err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
+	err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 	c.Check(err, check.IsNil)
-	c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n`)
+	c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
 }
 
 func (s *copierSuite) TestSymlink(c *check.C) {
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index aea5917..2f9ccf5 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1126,6 +1126,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
 		keepClient:    runner.Kc,
 		hostOutputDir: runner.HostOutputDir,
 		ctrOutputDir:  runner.Container.OutputPath,
+		binds:         runner.Binds,
 		mounts:        runner.Container.Mounts,
 		secretMounts:  runner.SecretMounts,
 		logger:        runner.CrunchLog,

-----------------------------------------------------------------------


hooks/post-receive
--