[ARVADOS] updated: fbc6c71ef67461cde5ba914e61f3f3b8740a4045

Git user git at public.curoverse.com
Fri Feb 10 13:51:49 EST 2017


Summary of changes:
 sdk/go/manifest/manifest.go           |  11 +--
 services/crunch-run/crunchrun.go      |  36 +++++----
 services/crunch-run/crunchrun_test.go | 144 ++++++++++++----------------------
 3 files changed, 80 insertions(+), 111 deletions(-)

       via  fbc6c71ef67461cde5ba914e61f3f3b8740a4045 (commit)
       via  397a747639fe360c18f3aa79846fef530b263184 (commit)
      from  ad464d416de6996a41d0c752124e0d201de0d3fc (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit fbc6c71ef67461cde5ba914e61f3f3b8740a4045
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Feb 10 13:51:42 2017 -0500

    9397: Fix bind behavior to include Path.  Update tests to check container binds
    as well as output manifest.

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f73ec73..5f4de14 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -317,7 +317,21 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 				if mnt.Writable {
 					return fmt.Errorf("Can never write to a collection specified by portable data hash")
 				}
+				idx := strings.Index(mnt.PortableDataHash, "/")
+				if idx > 0 {
+					mnt.Path = path.Clean(mnt.PortableDataHash[idx:])
+					mnt.PortableDataHash = mnt.PortableDataHash[0:idx]
+					runner.Container.Mounts[bind] = mnt
+				}
 				src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.PortableDataHash)
+				if mnt.Path != "" && mnt.Path != "." {
+					if strings.HasPrefix(mnt.Path, "./") {
+						mnt.Path = mnt.Path[2:]
+					} else if strings.HasPrefix(mnt.Path, "/") {
+						mnt.Path = mnt.Path[1:]
+					}
+					src += "/" + mnt.Path
+				}
 			} else {
 				src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount)
 				arvMountCmd = append(arvMountCmd, "--mount-tmp")
@@ -674,20 +688,10 @@ func (runner *ContainerRunner) CaptureOutput() error {
 			continue
 		}
 
-		if strings.HasPrefix(bindSuffix, "/") == false {
-			bindSuffix = "/" + bindSuffix
-		}
-
 		if mnt.ExcludeFromOutput == true {
 			continue
 		}
 
-		idx := strings.Index(mnt.PortableDataHash, "/")
-		if idx > 0 {
-			mnt.Path = mnt.PortableDataHash[idx:]
-			mnt.PortableDataHash = mnt.PortableDataHash[0:idx]
-		}
-
 		// append to manifest_text
 		m, err := runner.getCollectionManifestForPath(mnt, bindSuffix)
 		if err != nil {
@@ -699,7 +703,8 @@ func (runner *ContainerRunner) CaptureOutput() error {
 
 	// Save output
 	var response arvados.Collection
-	manifestText := manifest.Manifest{Text: manifestText}.NormalizedText()
+	manifest := manifest.Manifest{Text: manifestText}
+	manifestText = manifest.Extract(".", ".").Text
 	err = runner.ArvClient.Create("collections",
 		arvadosclient.Dict{
 			"collection": arvadosclient.Dict{
@@ -747,9 +752,12 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
 		return "", nil
 	}
 
-	manifest := manifest.Manifest{Text: collection.ManifestText}
-	manifestText := manifest.ManifestTextForPath(mnt.Path, bindSuffix)
-	return manifestText, nil
+	mft := manifest.Manifest{Text: collection.ManifestText}
+	extracted := mft.Extract(mnt.Path, bindSuffix)
+	if extracted.Err != nil {
+		return "", fmt.Errorf("Error parsing manifest for %v: %v", mnt.PortableDataHash, extracted.Err.Error())
+	}
+	return extracted.Text, nil
 }
 
 func (runner *ContainerRunner) loadDiscoveryVars() {
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index d4b7c1b..d1d2cee 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -57,10 +57,10 @@ var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea
 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d at 5693216f 0:46:md5sum.txt\n"
 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 
-var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
+var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 
-var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 5676:10:subdir1/file1_in_subdir1.txt 5686:18:subdir1/file2_in_subdir1.txt\n"
+var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 
 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
@@ -536,7 +536,7 @@ func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 
 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 // dress rehearsal of the Run() function, starting from a JSON container record.
-func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner) {
+func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 	rec := arvados.Container{}
 	err := json.Unmarshal([]byte(record), &rec)
 	c.Check(err, IsNil)
@@ -552,6 +552,22 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
 	am := &ArvMountCmdLine{}
 	cr.RunArvMount = am.ArvMountTest
 
+	realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(realTemp)
+
+	tempcount := 0
+	cr.MkTempDir = func(_ string, prefix string) (string, error) {
+		tempcount++
+		d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
+		err := os.Mkdir(d, os.ModePerm)
+		if err != nil && strings.Contains(err.Error(), ": file exists") {
+			// Test case must have pre-populated the tempdir
+			err = nil
+		}
+		return d, err
+	}
+
 	if extraMounts != nil && len(extraMounts) > 0 {
 		err := cr.SetupArvMountPoint("keep")
 		c.Check(err, IsNil)
@@ -578,7 +594,7 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
 }
 
 func (s *TestSuite) TestFullRunHello(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["echo", "hello world"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": ".",
@@ -600,7 +616,7 @@ func (s *TestSuite) TestFullRunHello(c *C) {
 }
 
 func (s *TestSuite) TestCrunchstat(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
 		"command": ["sleep", "1"],
 		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 		"cwd": ".",
@@ -633,7 +649,7 @@ func (s *TestSuite) TestCrunchstat(c *C) {
 }
 
 func (s *TestSuite) TestFullRunStderr(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": ".",
@@ -659,7 +675,7 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
 }
 
 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["pwd"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": ".",
@@ -681,7 +697,7 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 }
 
 func (s *TestSuite) TestFullRunSetCwd(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["pwd"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": "/bin",
@@ -755,7 +771,7 @@ func (s *TestSuite) TestCancel(c *C) {
 }
 
 func (s *TestSuite) TestFullRunSetEnv(c *C) {
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": "/bin",
@@ -1022,7 +1038,7 @@ func (s *TestSuite) TestStdout(c *C) {
 		"runtime_constraints": {}
 	}`
 
-	api, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
 		t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1085,7 +1101,7 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
 func (s *TestSuite) TestFullRunWithAPI(c *C) {
 	os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
 	defer os.Unsetenv("ARVADOS_API_HOST")
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": "/bin",
@@ -1109,7 +1125,7 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
 func (s *TestSuite) TestFullRunSetOutput(c *C) {
 	os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
 	defer os.Unsetenv("ARVADOS_API_HOST")
-	api, _ := FullRunHelper(c, `{
+	api, _, _ := FullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": "/bin",
@@ -1150,7 +1166,7 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
 
 	extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
 
-	api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
 		t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1169,64 +1185,10 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
 		"environment": {"FROBIZ": "bilbo"},
 		"mounts": {
         "/tmp": {"kind": "tmp"},
-        "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
-        "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
-        "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
-    },
-		"output_path": "/tmp",
-		"priority": 1,
-		"runtime_constraints": {}
-	}`
-
-	extraMounts := []string{"a0def87f80dd594d4675809e83bd4f15+367"}
-
-	api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
-		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
-		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
-	})
-
-	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
-	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-	for _, v := range api.Content {
-		if v["collection"] != nil {
-			collection := v["collection"].(arvadosclient.Dict)
-			if strings.Index(collection["name"].(string), "output") == 0 {
-				manifest := collection["manifest_text"].(string)
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
-				origManifestWithDotReplacedAsFoo := strings.Replace(normalizedManifestWithSubdirs, "./", "./foo/", -1)
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2"))
-			}
-		}
-	}
-}
-
-func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
-	helperRecord := `{
-		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
-		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
-		"cwd": "/bin",
-		"environment": {"FROBIZ": "bilbo"},
-		"mounts": {
-        "/tmp": {"kind": "tmp"},
-        "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"},
-        "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt"},
-        "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/"},
-        "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
-        "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt"},
+        "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
     },
 		"output_path": "/tmp",
@@ -1235,18 +1197,24 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
 	}`
 
 	extraMounts := []string{
-		"a0def87f80dd594d4675809e83bd4f15+367",
 		"a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
 		"a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
 		"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
 	}
 
-	api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
 		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
+	c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
+		realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
+		realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
+		realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
+		realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
+	})
+
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 	for _, v := range api.Content {
@@ -1255,18 +1223,12 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
 			if strings.Index(collection["name"].(string), "output") == 0 {
 				manifest := collection["manifest_text"].(string)
 
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
-				origManifestWithDotReplacedAsFoo := strings.Replace(normalizedManifestWithSubdirs, "./", "./foo/", -1)
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2"))
+				c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar 9:18:sub1file2
+./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2
+./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+`)
 			}
 		}
 	}
@@ -1280,8 +1242,7 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
 		"environment": {"FROBIZ": "bilbo"},
 		"mounts": {
         "/tmp": {"kind": "tmp"},
-        "/tmp/foo": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
-        "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"subdir1/file2_in_subdir1.txt"},
+        "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
     },
 		"output_path": "/tmp",
@@ -1289,9 +1250,11 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
 		"runtime_constraints": {}
 	}`
 
-	extraMounts := []string{"b0def87f80dd594d4675809e83bd4f15+367"}
+	extraMounts := []string{
+		"b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
+	}
 
-	api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
 		t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1305,12 +1268,9 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
 			if strings.Index(collection["name"].(string), "output") == 0 {
 				manifest := collection["manifest_text"].(string)
 
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
-				origManifestWithDotReplacedAsFoo := strings.Replace(denormalizedManifestWithSubdirs, "./", "./foo/", -1)
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
-				c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 5686:18:subdir1/file2_in_subdir1.txt"))
+				c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 10:17:bar
+`)
 			}
 		}
 	}

commit 397a747639fe360c18f3aa79846fef530b263184
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Feb 10 13:40:31 2017 -0500

    9397: Adjust stream normalization to use just digest (no hints) to de-duplicate
    block locators.

diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index cb60560..9038960 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -356,16 +356,16 @@ func (stream segmentedStream) normalizedText(name string) string {
 
 	stream_tokens := []string{EscapeName(name)}
 
-	blocks := make(map[string]int64)
+	blocks := make(map[blockdigest.BlockDigest]int64)
 	var streamoffset int64
 
 	// Go through each file and add each referenced block exactly once.
 	for _, streamfile := range sortedfiles {
 		for _, segment := range stream[streamfile] {
-			if _, ok := blocks[segment.Locator]; !ok {
+			b, _ := ParseBlockLocator(segment.Locator)
+			if _, ok := blocks[b.Digest]; !ok {
 				stream_tokens = append(stream_tokens, segment.Locator)
-				blocks[segment.Locator] = streamoffset
-				b, _ := ParseBlockLocator(segment.Locator)
+				blocks[b.Digest] = streamoffset
 				streamoffset += int64(b.Size)
 			}
 		}
@@ -382,7 +382,8 @@ func (stream segmentedStream) normalizedText(name string) string {
 		fout := EscapeName(streamfile)
 		for _, segment := range stream[streamfile] {
 			// Collapse adjacent segments
-			streamoffset = blocks[segment.Locator] + int64(segment.Offset)
+			b, _ := ParseBlockLocator(segment.Locator)
+			streamoffset = blocks[b.Digest] + int64(segment.Offset)
 			if span_start == -1 {
 				span_start = streamoffset
 				span_end = streamoffset + int64(segment.Len)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list